<commit_before>from time import time, sleep
from copy import deepcopy
import threading
import sqlite3
import json
import os
import decimal
from bitcoin.core import COIN, b2lx
import feemodel.config
from feemodel.config import config, historyFile
from feemodel.util import proxy, logWrite

pollPeriod = config['pollPeriod']
keepHistory = config['keepHistory']

class TxMempool(threading.Thread):
    # Have to handle RPC errors
    def __init__(self):
        # Writehistory means write to db the mempool state at each block.
        # We keep <keepHistory> number of past blocks.
        super(TxMempool, self).__init__()
        self._stop = threading.Event()

    def update(self):
        currHeight, mapTxNew = proxy.pollMempool()
        if currHeight > self.bestSeenBlock:
            threading.Thread(target=self.processBlocks,
                args=(range(self.bestSeenBlock+1,currHeight+1),
                    self.mapTx, deepcopy(mapTxNew)),
                name='mempool-processBlocks').start()
            self.mapTx = mapTxNew
            self.bestSeenBlock = currHeight
            return True
            # Be careful here: may have to pass deepcopy of self.mapTx to processBlocks,
            # if we are going to return self.mapTx for other functions to use.
        else:
            self.mapTx = mapTxNew
            return False

    def run(self):
        feemodel.config.apprun = True
        logWrite("Starting mempool")
        self.bestSeenBlock, self.mapTx = proxy.pollMempool()
        while not self._stop.is_set():
            self.update()
            self._stop.wait(timeout=pollPeriod)
        logWrite("Closing up mempool...")
        for thread in threading.enumerate():
            if thread.name.startswith('mempool'):
                thread.join()
        logWrite("Finished everything.")

    def stop(self):
        self._stop.set()

    @staticmethod
    def processBlocks(blockHeightRange, currPool, newPool, blockTime=None):
        if not blockTime:
            blockTime = int(time())
        blocks = []
        for blockHeight in blockHeightRange:
            blockData = proxy.getblock(proxy.getblockhash(blockHeight))
            blockSize = len(blockData.serialize())
            blockTxList = [b2lx(tx.GetHash()) for tx in blockData.vtx]
            entries = deepcopy(currPool)
            numMempoolTxsInBlock = 0

            for txid, entry in entries.iteritems():
                if txid in blockTxList:
                    entry['inBlock'] = True
                    numMempoolTxsInBlock += 1
                    del currPool[txid]
                else:
                    entry['inBlock'] = False
                entry['leadTime'] = blockTime - entry['time']
                entry['feeRate'] = int(entry['fee']*COIN) * 1000 // entry['size']
                entry['isConflict'] = False

            blocks.append(Block(entries,blockHeight,blockSize,blockTime))
            logWrite(str(numMempoolTxsInBlock) + ' of ' +
                str(len(blockTxList)-1) + ' in block ' + str(blockHeight))

        # To-do: insert warnings if block inclusion ratio is too low, or conflicts are too high
        conflicts = set(currPool) - set(newPool)

        numConflicts = 0
        for block in blocks:
            numConflicts += block.removeConflicts(conflicts)

        logWrite("%d conflicts removed." % numConflicts)

        if feemodel.config.apprun:
            for block in blocks:
                block.writeHistory()

        return blocks

class Block(object):
    def __init__(self, entries, blockHeight, blockSize, blockTime):
        # To-do: add a 'tx-coverage' field
        self.entries = entries
        self.height = blockHeight
        self.size = blockSize
        self.time = blockTime

    def removeConflicts(self, conflicts):
        numConflicts = 0
        for txid in self.entries.keys():
            if txid in conflicts:
                self.entries[txid]['isConflict'] = True
                numConflicts += 1

        return numConflicts

    def writeHistory(self, dbFile=historyFile):
        db = None
        dbExists = os.path.exists(dbFile)
        try:
            db = sqlite3.connect(dbFile)
            if not dbExists:
                with db:
                    db.execute('CREATE TABLE blocks (height INTEGER UNIQUE, size INTEGER, time REAL)')
                    db.execute('CREATE TABLE txs (blockheight INTEGER, txid TEXT, data TEXT)')
            with db:
                db.execute('INSERT INTO blocks VALUES (?,?,?)', (self.height, self.size, self.time))
                db.executemany('INSERT INTO txs VALUES (?,?,?)',
                    [(self.height, txid, json.dumps(entry,default=decimalDefault))
                    for txid,entry in self.entries.iteritems()])
                historyLimit = self.height - keepHistory
                if keepHistory > 0:
                    db.execute('DELETE FROM blocks WHERE height<=?', (historyLimit,))
                    db.execute('DELETE FROM txs WHERE blockheight<=?', (historyLimit,))
        except Exception as e:
            logWrite(repr(e))
            logWrite("Exception in writing/cleaning history.")
        finally:
            if db:
                db.close()

    @classmethod
    def blockFromHistory(cls, blockHeight, dbFile=historyFile):
        db = None
        try:
            db = sqlite3.connect(dbFile)
            block = db.execute('SELECT size,time FROM blocks WHERE height=?',
                (blockHeight,)).fetchall()
            if block:
                blockSize,blockTime = block[0]
            else:
                return None
            txlist = db.execute('SELECT txid,data FROM txs WHERE blockheight=?', (blockHeight,))
            entries = {txid: json.loads(str(data)) for txid,data in txlist}
            for entry in entries.itervalues():
                entry['fee'] = decimal.Decimal(entry['fee'])
                entry['startingpriority'] = decimal.Decimal(entry['startingpriority'])
                entry['currentpriority'] = decimal.Decimal(entry['currentpriority'])
            return cls(entries,blockHeight,blockSize,blockTime)
        except Exception as e:
            logWrite(repr(e))
            return None
        finally:
            if db:
                db.close()

    def __eq__(self,other):
        if not isinstance(other,Block):
            return False
        return all([
            self.entries == other.entries,
            self.height == other.height,
            self.size == other.size,
            self.time == other.time,
        ])

class LoadHistory(object):
    def __init__(self, dbFile=historyFile):
        self.fns = []
        self.dbFile = dbFile

    def registerFn(self, fn, blockHeightRange):
        # blockHeightRange tuple (start,end) includes start but not end, to adhere to range() convention
        self.fns.append((fn, blockHeightRange))

    def loadBlocks(self):
        startHeight = min([f[1][0] for f in self.fns])
        endHeight = max([f[1][1] for f in self.fns])

        for height in range(startHeight, endHeight):
            block = Block.blockFromHistory(height, self.dbFile)
            for fn, blockHeightRange in self.fns:
                if height >= blockHeightRange[0] and height < blockHeightRange[1]:
                    fn([block])

def decimalDefault(obj):
    if isinstance(obj, decimal.Decimal):
        return str(obj)
    raise TypeError

<commit_msg>added historyLock to txmempool<commit_after>from time import time, sleep
from copy import deepcopy
import threading
import sqlite3
import json
import os
import decimal
from bitcoin.core import COIN, b2lx
import feemodel.config
from feemodel.config import config, historyFile
from feemodel.util import proxy, logWrite

pollPeriod = config['pollPeriod']
keepHistory = config['keepHistory']
historyLock = threading.Lock()

class TxMempool(threading.Thread):
    # Have to handle RPC errors
    def __init__(self):
        # Writehistory means write to db the mempool state at each block.
        # We keep <keepHistory> number of past blocks.
        super(TxMempool, self).__init__()
        self._stop = threading.Event()

    def update(self):
        currHeight, mapTxNew = proxy.pollMempool()
        if currHeight > self.bestSeenBlock:
            threading.Thread(target=self.processBlocks,
                args=(range(self.bestSeenBlock+1,currHeight+1),
                    self.mapTx, deepcopy(mapTxNew)),
                name='mempool-processBlocks').start()
            self.mapTx = mapTxNew
            self.bestSeenBlock = currHeight
            return True
            # Be careful here: may have to pass deepcopy of self.mapTx to processBlocks,
            # if we are going to return self.mapTx for other functions to use.
        else:
            self.mapTx = mapTxNew
            return False

    def run(self):
        feemodel.config.apprun = True
        logWrite("Starting mempool")
        self.bestSeenBlock, self.mapTx = proxy.pollMempool()
        while not self._stop.is_set():
            self.update()
            self._stop.wait(timeout=pollPeriod)
        logWrite("Closing up mempool...")
        for thread in threading.enumerate():
            if thread.name.startswith('mempool'):
                thread.join()
        logWrite("Finished everything.")

    def stop(self):
        self._stop.set()

    @staticmethod
    def processBlocks(blockHeightRange, currPool, newPool, blockTime=None):
        with historyLock:
            if not blockTime:
                blockTime = int(time())
            blocks = []
            for blockHeight in blockHeightRange:
                blockData = proxy.getblock(proxy.getblockhash(blockHeight))
                blockSize = len(blockData.serialize())
                blockTxList = [b2lx(tx.GetHash()) for tx in blockData.vtx]
                entries = deepcopy(currPool)
                numMempoolTxsInBlock = 0

                for txid, entry in entries.iteritems():
                    if txid in blockTxList:
                        entry['inBlock'] = True
                        numMempoolTxsInBlock += 1
                        del currPool[txid]
                    else:
                        entry['inBlock'] = False
                    entry['leadTime'] = blockTime - entry['time']
                    entry['feeRate'] = int(entry['fee']*COIN) * 1000 // entry['size']
                    entry['isConflict'] = False

                blocks.append(Block(entries,blockHeight,blockSize,blockTime))
                logWrite(str(numMempoolTxsInBlock) + ' of ' +
                    str(len(blockTxList)-1) + ' in block ' + str(blockHeight))

            # To-do: insert warnings if block inclusion ratio is too low, or conflicts are too high
            conflicts = set(currPool) - set(newPool)

            numConflicts = 0
            for block in blocks:
                numConflicts += block.removeConflicts(conflicts)

            logWrite("%d conflicts removed." % numConflicts)

            if feemodel.config.apprun:
                for block in blocks:
                    block.writeHistory()

            return blocks

class Block(object):
    def __init__(self, entries, blockHeight, blockSize, blockTime):
        # To-do: add a 'tx-coverage' field
        self.entries = entries
        self.height = blockHeight
        self.size = blockSize
        self.time = blockTime

    def removeConflicts(self, conflicts):
        numConflicts = 0
        for txid in self.entries.keys():
            if txid in conflicts:
                self.entries[txid]['isConflict'] = True
                numConflicts += 1

        return numConflicts

    def writeHistory(self, dbFile=historyFile):
        db = None
        dbExists = os.path.exists(dbFile)
        try:
            db = sqlite3.connect(dbFile)
            if not dbExists:
                with db:
                    db.execute('CREATE TABLE blocks (height INTEGER UNIQUE, size INTEGER, time REAL)')
                    db.execute('CREATE TABLE txs (blockheight INTEGER, txid TEXT, data TEXT)')
            with db:
                db.execute('INSERT INTO blocks VALUES (?,?,?)', (self.height, self.size, self.time))
                db.executemany('INSERT INTO txs VALUES (?,?,?)',
                    [(self.height, txid, json.dumps(entry,default=decimalDefault))
                    for txid,entry in self.entries.iteritems()])
                historyLimit = self.height - keepHistory
                if keepHistory > 0:
                    db.execute('DELETE FROM blocks WHERE height<=?', (historyLimit,))
                    db.execute('DELETE FROM txs WHERE blockheight<=?', (historyLimit,))
        except Exception as e:
            logWrite(repr(e))
            logWrite("Exception in writing/cleaning history.")
        finally:
            if db:
                db.close()

    @classmethod
    def blockFromHistory(cls, blockHeight, dbFile=historyFile):
        db = None
        try:
            db = sqlite3.connect(dbFile)
            block = db.execute('SELECT size,time FROM blocks WHERE height=?',
                (blockHeight,)).fetchall()
            if block:
                blockSize,blockTime = block[0]
            else:
                return None
            txlist = db.execute('SELECT txid,data FROM txs WHERE blockheight=?', (blockHeight,))
            entries = {txid: json.loads(str(data)) for txid,data in txlist}
            for entry in entries.itervalues():
                entry['fee'] = decimal.Decimal(entry['fee'])
                entry['startingpriority'] = decimal.Decimal(entry['startingpriority'])
                entry['currentpriority'] = decimal.Decimal(entry['currentpriority'])
            return cls(entries,blockHeight,blockSize,blockTime)
        except Exception as e:
            logWrite(repr(e))
            return None
        finally:
            if db:
                db.close()

    def __eq__(self,other):
        if not isinstance(other,Block):
            return False
        return all([
            self.entries == other.entries,
            self.height == other.height,
            self.size == other.size,
            self.time == other.time,
        ])

class LoadHistory(object):
    def __init__(self, dbFile=historyFile):
        self.fns = []
        self.dbFile = dbFile

    def registerFn(self, fn, blockHeightRange):
        # blockHeightRange tuple (start,end) includes start but not end, to adhere to range() convention
        self.fns.append((fn, blockHeightRange))

    def loadBlocks(self):
        startHeight = min([f[1][0] for f in self.fns])
        endHeight = max([f[1][1] for f in self.fns])

        for height in range(startHeight, endHeight):
            block = Block.blockFromHistory(height, self.dbFile)
            for fn, blockHeightRange in self.fns:
                if height >= blockHeightRange[0] and height < blockHeightRange[1]:
                    fn([block])

def decimalDefault(obj):
    if isinstance(obj, decimal.Decimal):
        return str(obj)
    raise TypeError

<|endoftext|>


<commit_before># Copyright 2015 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os

import testtools
from mock import MagicMock
from trove.common.context import TroveContext
from trove.common.instance import ServiceStatuses
from trove.guestagent import volume
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.couchdb import (
    service as couchdb_service)
from trove.guestagent.datastore.experimental.couchdb import (
    manager as couchdb_manager)
from trove.guestagent import pkg as pkg

class GuestAgentCouchDBManagerTest(testtools.TestCase):

    def setUp(self):
        super(GuestAgentCouchDBManagerTest, self).setUp()
        self.real_status = couchdb_service.CouchDBAppStatus.set_status

        class FakeInstanceServiceStatus(object):
            status = ServiceStatuses.NEW

            def save(self):
                pass

        couchdb_service.CouchDBAppStatus.set_status = MagicMock(
            return_value=FakeInstanceServiceStatus())
        self.context = TroveContext()
        self.manager = couchdb_manager.Manager()
        self.pkg = couchdb_service.packager
        self.real_db_app_status = couchdb_service.CouchDBAppStatus
        self.origin_os_path_exists = os.path.exists
        self.origin_format = volume.VolumeDevice.format
        self.origin_migrate_data = volume.VolumeDevice.migrate_data
        self.origin_mount = volume.VolumeDevice.mount
        self.origin_mount_points = volume.VolumeDevice.mount_points
        self.origin_stop_db = couchdb_service.CouchDBApp.stop_db
        self.origin_start_db = couchdb_service.CouchDBApp.start_db
        self.original_get_ip = operating_system.get_ip_address
        self.orig_make_host_reachable = (
            couchdb_service.CouchDBApp.make_host_reachable)

    def tearDown(self):
        super(GuestAgentCouchDBManagerTest, self).tearDown()
        couchdb_service.packager = self.pkg
        couchdb_service.CouchDBAppStatus.set_status = self.real_db_app_status
        os.path.exists = self.origin_os_path_exists
        volume.VolumeDevice.format = self.origin_format
        volume.VolumeDevice.migrate_data = self.origin_migrate_data
        volume.VolumeDevice.mount = self.origin_mount
        volume.VolumeDevice.mount_points = self.origin_mount_points
        couchdb_service.CouchDBApp.stop_db = self.origin_stop_db
        couchdb_service.CouchDBApp.start_db = self.origin_start_db
        operating_system.get_ip_address = self.original_get_ip
        couchdb_service.CouchDBApp.make_host_reachable = (
            self.orig_make_host_reachable)

    def test_update_status(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        self.manager.update_status(self.context)
        mock_status.update.assert_any_call()

    def _prepare_dynamic(self, packages,
                         config_content=None, device_path='/dev/vdb',
                         is_db_installed=True, backup_id=None, overrides=None):
        mock_status = MagicMock()
        mock_app = MagicMock()
        self.manager.appStatus = mock_status
        self.manager.app = mock_app

        mock_status.begin_install = MagicMock(return_value=None)
        mock_app.install_if_needed = MagicMock(return_value=None)
        pkg.Package.pkg_is_installed = MagicMock(return_value=is_db_installed)
        mock_app.make_host_reachable = MagicMock(return_value=None)
        mock_app.restart = MagicMock(return_value=None)
        mock_app.start_db = MagicMock(return_value=None)
        mock_app.stop_db = MagicMock(return_value=None)
        os.path.exists = MagicMock(return_value=True)
        volume.VolumeDevice.format = MagicMock(return_value=None)
        volume.VolumeDevice.migrate_data = MagicMock(return_value=None)
        volume.VolumeDevice.mount = MagicMock(return_value=None)
        volume.VolumeDevice.mount_points = MagicMock(return_value=[])

        # invocation
        self.manager.prepare(context=self.context, packages=packages,
                             config_contents=config_content,
                             databases=None,
                             memory_mb='2048', users=None,
                             device_path=device_path,
                             mount_point="/var/lib/couchdb",
                             backup_info=None,
                             overrides=None,
                             cluster_config=None)

        # verification/assertion
        mock_status.begin_install.assert_any_call()
        mock_app.install_if_needed.assert_any_call(packages)
        mock_app.make_host_reachable.assert_any_call()
        mock_app.change_permissions.assert_any_call()

    def test_prepare_pkg(self):
        self._prepare_dynamic(['couchdb'])

    def test_prepare_no_pkg(self):
        self._prepare_dynamic([])

    def test_restart(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        couchdb_service.CouchDBApp.restart = MagicMock(return_value=None)
        #invocation
        self.manager.restart(self.context)
        #verification/assertion
        couchdb_service.CouchDBApp.restart.assert_any_call()

    def test_stop_db(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        couchdb_service.CouchDBApp.stop_db = MagicMock(return_value=None)
        #invocation
        self.manager.stop_db(self.context)
        #verification/assertion
        couchdb_service.CouchDBApp.stop_db.assert_any_call(
            do_not_start_on_reboot=False)
<commit_msg>Fix CouchDB unit test to properly mock restart<commit_after># Copyright 2015 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os

import testtools
from mock import MagicMock
from mock import patch
from trove.common.context import TroveContext
from trove.common.instance import ServiceStatuses
from trove.guestagent import volume
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.couchdb import (
    service as couchdb_service)
from trove.guestagent.datastore.experimental.couchdb import (
    manager as couchdb_manager)
from trove.guestagent import pkg as pkg

class GuestAgentCouchDBManagerTest(testtools.TestCase):

    def setUp(self):
        super(GuestAgentCouchDBManagerTest, self).setUp()
        self.real_status = couchdb_service.CouchDBAppStatus.set_status

        class FakeInstanceServiceStatus(object):
            status = ServiceStatuses.NEW

            def save(self):
                pass

        couchdb_service.CouchDBAppStatus.set_status = MagicMock(
            return_value=FakeInstanceServiceStatus())
        self.context = TroveContext()
        self.manager = couchdb_manager.Manager()
        self.pkg = couchdb_service.packager
        self.real_db_app_status = couchdb_service.CouchDBAppStatus
        self.origin_os_path_exists = os.path.exists
        self.origin_format = volume.VolumeDevice.format
        self.origin_migrate_data = volume.VolumeDevice.migrate_data
        self.origin_mount = volume.VolumeDevice.mount
        self.origin_mount_points = volume.VolumeDevice.mount_points
        self.origin_stop_db = couchdb_service.CouchDBApp.stop_db
        self.origin_start_db = couchdb_service.CouchDBApp.start_db
        self.original_get_ip = operating_system.get_ip_address
        self.orig_make_host_reachable = (
            couchdb_service.CouchDBApp.make_host_reachable)

    def tearDown(self):
        super(GuestAgentCouchDBManagerTest, self).tearDown()
        couchdb_service.packager = self.pkg
        couchdb_service.CouchDBAppStatus.set_status = self.real_db_app_status
        os.path.exists = self.origin_os_path_exists
        volume.VolumeDevice.format = self.origin_format
        volume.VolumeDevice.migrate_data = self.origin_migrate_data
        volume.VolumeDevice.mount = self.origin_mount
        volume.VolumeDevice.mount_points = self.origin_mount_points
        couchdb_service.CouchDBApp.stop_db = self.origin_stop_db
        couchdb_service.CouchDBApp.start_db = self.origin_start_db
        operating_system.get_ip_address = self.original_get_ip
        couchdb_service.CouchDBApp.make_host_reachable = (
            self.orig_make_host_reachable)

    def test_update_status(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        self.manager.update_status(self.context)
        mock_status.update.assert_any_call()

    def _prepare_dynamic(self, packages,
                         config_content=None, device_path='/dev/vdb',
                         is_db_installed=True, backup_id=None, overrides=None):
        mock_status = MagicMock()
        mock_app = MagicMock()
        self.manager.appStatus = mock_status
        self.manager.app = mock_app

        mock_status.begin_install = MagicMock(return_value=None)
        mock_app.install_if_needed = MagicMock(return_value=None)
        pkg.Package.pkg_is_installed = MagicMock(return_value=is_db_installed)
        mock_app.make_host_reachable = MagicMock(return_value=None)
        mock_app.restart = MagicMock(return_value=None)
        mock_app.start_db = MagicMock(return_value=None)
        mock_app.stop_db = MagicMock(return_value=None)
        os.path.exists = MagicMock(return_value=True)
        volume.VolumeDevice.format = MagicMock(return_value=None)
        volume.VolumeDevice.migrate_data = MagicMock(return_value=None)
        volume.VolumeDevice.mount = MagicMock(return_value=None)
        volume.VolumeDevice.mount_points = MagicMock(return_value=[])

        # invocation
        self.manager.prepare(context=self.context, packages=packages,
                             config_contents=config_content,
                             databases=None,
                             memory_mb='2048', users=None,
                             device_path=device_path,
                             mount_point="/var/lib/couchdb",
                             backup_info=None,
                             overrides=None,
                             cluster_config=None)

        # verification/assertion
        mock_status.begin_install.assert_any_call()
        mock_app.install_if_needed.assert_any_call(packages)
        mock_app.make_host_reachable.assert_any_call()
        mock_app.change_permissions.assert_any_call()

    def test_prepare_pkg(self):
        self._prepare_dynamic(['couchdb'])

    def test_prepare_no_pkg(self):
        self._prepare_dynamic([])

    def test_restart(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        with patch.object(couchdb_service.CouchDBApp, 'restart',
                          return_value=None):
            #invocation
            self.manager.restart(self.context)
            #verification/assertion
            couchdb_service.CouchDBApp.restart.assert_any_call()

    def test_stop_db(self):
        mock_status = MagicMock()
        self.manager.appStatus = mock_status
        couchdb_service.CouchDBApp.stop_db = MagicMock(return_value=None)
        #invocation
        self.manager.stop_db(self.context)
        #verification/assertion
        couchdb_service.CouchDBApp.stop_db.assert_any_call(
            do_not_start_on_reboot=False)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2015 Michael Scherer <mscherer@redhat.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# This script trigger a middleman rebuild if upstream git or any submodule
# did change

import yaml
import sys
import os
import subprocess
import datetime
import atexit
import syslog
import argparse

parser = argparse.ArgumentParser(description="Build middleman sites based "
                                             "on changes in git")
parser.add_argument("-f", "--force", help="force the rebuild",
                    action="store_true")
parser.add_argument("-n", "--dry-run", help="do not sync remotely",
                    action="store_true")
parser.add_argument("-d", "--debug", help="show debug output",
                    action="store_true")
parser.add_argument("-s", "--sync-only", help="do not build, only sync",
                    action="store_true")
parser.add_argument("config_file", help="yaml file for the builder config")

args = parser.parse_args()

builder_info = {
    'middleman': {
        'build_command': ['bundle', 'exec', 'middleman', 'build', '--verbose'],
        'build_subdir': 'build',
        'deploy_command': ['bundle', 'exec', 'middleman', 'deploy', '--no-build-before']
    },
    'jekyll': {
        'build_command': ['bundle', 'exec', 'jekyll', 'build', '--incremental', '--verbose', '--trace'],
        'build_subdir': '_site',
        'deploy_command': None
    }
}

def debug_print(message):
    if args.debug:
        print message

def refresh_checkout(checkout_dir):
    os.chdir(checkout_dir)
    result = subprocess.check_output(['git', 'fetch', '-q'], stderr=subprocess.STDOUT)
    debug_print(result)

def get_last_commit(checkout_dir):
    os.chdir(checkout_dir)
    r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                 'refs/remotes/origin/HEAD'])
    return r.split()[0]

def get_last_commit_submodule(checkout_dir, submodule):
    os.chdir("{}/{}/".format(checkout_dir, submodule))
    r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                 'refs/remotes/origin/HEAD'])
    return r.split()[0]

def get_submodules_checkout(checkout_dir):
    os.chdir(checkout_dir)
    result = []
    submodule_status = subprocess.check_output(['git', 'submodule', 'status'])
    for s in submodule_status.split('\n'):
        # there is a empty line at the end...
        if s:
            result.append(s.split()[1])
    return result

def load_config(config_file):
    if not os.path.exists(config_file):
        print "Error %s, do not exist" % config_file
        sys.exit(1)
    if not os.path.isfile(config_file):
        print "Error %s is not a file" % config_file
        sys.exit(1)

    with open(config_file) as f:
        config = yaml.safe_load(f)

    return config

def has_submodules(checkout_dir):
    os.chdir(checkout_dir)
    r = subprocess.check_output(['git', 'submodule', 'status'])
    return len(r) > 0

# TODO complete that
def notify_error(stage, error):
    print error
    sys.exit(3)

if not args.config_file:
    print "This script take only 1 single argument, the config file"
    sys.exit(1)

config = load_config(args.config_file)

if 'name' not in config:
    print "Incorrect config file: {}".format(args.config_file)
    sys.exit(1)

name = config['name']

lock_file = os.path.expanduser('{}/lock_{}'.format(
    os.environ.get('XDG_RUNTIME_DIR', '~'), name))
if os.path.exists(lock_file):
    # TODO verify if the PID in the file still exist
    debug_print("Builder already running, exiting")
    sys.exit(2)

# TODO try/except, show a better error message
fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.write(fd, str(os.getpid()))
os.close(fd)
atexit.register(os.unlink, lock_file)

status_file = os.path.expanduser('~/status_%s.yml' % name)
status = {}
if os.path.exists(status_file):
    with open(status_file) as f:
        status = yaml.safe_load(f)

checkout_dir = os.path.expanduser("~/%s" % name)
if not os.path.isdir(checkout_dir):
    os.unlink(lock_file)
    print "Checkout not existing, exiting"
    sys.exit(2)

refresh_checkout(checkout_dir)

start_build = False

last_build = datetime.datetime.fromtimestamp(
    int(status.get("last_build", "0")))

if ('regular_rebuild_interval' in config
        and datetime.datetime.now() - last_build
        > datetime.timedelta(hours=config['regular_rebuild_interval'])):
    start_build = True

current_commit = get_last_commit(checkout_dir)
if current_commit != status.get('last_build_commit', ''):
    start_build = True

submodule_commits = status.get('submodule_commits', {})
current_submodule_commits = {}
for submodule in get_submodules_checkout(checkout_dir):
    debug_print('Looking for %s' % submodule)
    current_submodule_commits[submodule] = \
        get_last_commit_submodule(checkout_dir, submodule)

    if current_submodule_commits[submodule] != \
            submodule_commits.get(submodule, ''):
        start_build = True

if args.force:
    start_build = True

if not start_build:
    debug_print("Nothing to build")
    sys.exit(0)

syslog.syslog("Start the build of {}".format(name))

os.chdir(checkout_dir)
result = subprocess.check_output(['git', 'stash'], stderr=subprocess.STDOUT)
debug_print(result)
result = subprocess.check_output(['git', 'stash', 'clear'], stderr=subprocess.STDOUT)
debug_print(result)
result = subprocess.check_output(['git', 'pull', '--rebase'], stderr=subprocess.STDOUT)
debug_print(result)

if has_submodules(checkout_dir):
    result = subprocess.check_output(['git', 'submodule', 'init'], stderr=subprocess.STDOUT)
    debug_print(result)
    result = subprocess.check_output(['git', 'submodule', 'sync'], stderr=subprocess.STDOUT)
    debug_print(result)

if config.get('update_submodule_head', False):
    result = subprocess.check_output(['git', 'submodule', 'foreach',
                     '"git pull -qf origin master"'], stderr=subprocess.STDOUT)
    debug_print(result)

if not args.sync_only:
    os.environ['PATH'] = "/usr/local/bin:/srv/builder/bin:" + \
                         os.environ['PATH']
    try:
        syslog.syslog("Build of {}: bundle install".format(name))
        result = subprocess.check_output(['bundle', 'install'], stderr=subprocess.STDOUT)
        debug_print(result)
    except subprocess.CalledProcessError, C:
        notify_error('install', C.output)

    try:
        command = builder_info[config['builder']]['build_command']
        syslog.syslog("Build of {}: {}".format(name, ' '.join(command)))
        result = subprocess.check_output(command, stderr=subprocess.STDOUT)
        debug_print(result)
    except subprocess.CalledProcessError, C:
        notify_error('build', C.output)

if not args.dry_run:
    syslog.syslog("Build of {}: start sync".format(name))
    try:
        if config['remote']:
            build_subdir = builder_info[config['builder']]['build_subdir']
            result = subprocess.check_output(
                            ['rsync',
                             '-e',
                             'ssh '
                             '-o '
                             'UserKnownHostsFile=/dev/null '
                             '-o '
                             'StrictHostKeyChecking=no '
                             '-i ' +
                             os.path.expanduser('~/.ssh/{}_id.rsa'.format(name)),
                             '--delete-after',
                             '-rltogvz',
                             '--omit-dir-times',
                             '%s/%s/' % (checkout_dir, build_subdir),
                             config['remote']],
                             stderr=subprocess.STDOUT)
        else:
            command = builder_info[config['builder']]['deploy_command']
            if command:
                result = subprocess.check_output(command)
            else:
                notify_error('deploy', "builder '%s' does not possess a deploy method" % config['builder'])
        debug_print(result)
    except subprocess.CalledProcessError, C:
        notify_error('deploy', C.output)
    syslog.syslog("Build of {}: finish sync".format(name))
else:
    syslog.syslog("Build of {}: not syncing, dry-run".format(name))

status = {}
status['last_build_commit'] = current_commit
status['last_build'] = datetime.datetime.now().strftime("%s")
status['last_build_human'] = datetime.datetime.now().strftime("%c")
status['submodule_commits'] = current_submodule_commits

with open(status_file, 'w+') as f:
    f.write(yaml.dump(status, default_flow_style=False))
<commit_msg>save debug messages in a log file and publish it<commit_after>#!/usr/bin/python
#
# Copyright (c) 2015 Michael Scherer <mscherer@redhat.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# This script trigger a middleman rebuild if upstream git or any submodule
# did change

import yaml
import sys
import os
import subprocess
import datetime
import atexit
import syslog
import argparse
import shutil

parser = argparse.ArgumentParser(description="Build middleman sites based "
                                             "on changes in git")
parser.add_argument("-f", "--force", help="force the rebuild",
                    action="store_true")
parser.add_argument("-n", "--dry-run", help="do not sync remotely",
                    action="store_true")
parser.add_argument("-d", "--debug", help="show debug output",
                    action="store_true")
parser.add_argument("-s", "--sync-only", help="do not build, only sync",
                    action="store_true")
parser.add_argument("config_file", help="yaml file for the builder config")

args = parser.parse_args()

builder_info = {
    'middleman': {
        'build_command': ['bundle', 'exec', 'middleman', 'build', '--verbose'],
        'build_subdir': 'build',
        'deploy_command': ['bundle', 'exec', 'middleman', 'deploy', '--no-build-before']
    },
    'jekyll': {
        'build_command': ['bundle', 'exec', 'jekyll', 'build', '--incremental', '--verbose', '--trace'],
        'build_subdir': '_site',
        'deploy_command': None
    }
}

def log_print(message):
    try:
        log_fd
    except NameError:
        pass
    else:
        log_fd.write(message + "\n")
        log_fd.flush()

def debug_print(message):
    if args.debug:
        print message
    log_print(message)

def refresh_checkout(checkout_dir):
    os.chdir(checkout_dir)
    result = subprocess.check_output(['git', 'fetch', '-q'], stderr=subprocess.STDOUT)
    debug_print(result)

def get_last_commit(checkout_dir):
    os.chdir(checkout_dir)
    r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                 'refs/remotes/origin/HEAD'])
    return r.split()[0]

def get_last_commit_submodule(checkout_dir, submodule):
    os.chdir("{}/{}/".format(checkout_dir, submodule))
    r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                 'refs/remotes/origin/HEAD'])
    return r.split()[0]

def get_submodules_checkout(checkout_dir):
    os.chdir(checkout_dir)
    result = []
    submodule_status = subprocess.check_output(['git', 'submodule', 'status'])
    for s in submodule_status.split('\n'):
        # there is a empty line at the end...
        if s:
            result.append(s.split()[1])
    return result

def load_config(config_file):
    if not os.path.exists(config_file):
        print "Error %s, do not exist" % config_file
        sys.exit(1)
    if not os.path.isfile(config_file):
        print "Error %s is not a file" % config_file
        sys.exit(1)

    with open(config_file) as f:
        config = yaml.safe_load(f)

    return config

def has_submodules(checkout_dir):
    os.chdir(checkout_dir)
    r = subprocess.check_output(['git', 'submodule', 'status'])
    return len(r) > 0

# TODO complete that
def notify_error(stage, error):
    print error
    sys.exit(3)

if not args.config_file:
    print "This script take only 1 single argument, the config file"
    sys.exit(1)

config = load_config(args.config_file)

if 'name' not in config:
    print "Incorrect config file: {}".format(args.config_file)
    sys.exit(1)

name = config['name']

lock_file = os.path.expanduser('{}/lock_{}'.format(
    os.environ.get('XDG_RUNTIME_DIR', '~'), name))
if os.path.exists(lock_file):
    # TODO verify if the PID in the file still exist
    debug_print("Builder already running, exiting")
    sys.exit(2)

# TODO try/except, show a better error message
fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.write(fd, str(os.getpid()))
os.close(fd)
atexit.register(os.unlink, lock_file)

status_file = os.path.expanduser('~/status_%s.yml' % name)
status = {}
if os.path.exists(status_file):
    with open(status_file) as f:
        status = yaml.safe_load(f)

checkout_dir = os.path.expanduser("~/%s" % name)
if not os.path.isdir(checkout_dir):
    os.unlink(lock_file)
    print "Checkout not existing, exiting"
    sys.exit(2)

refresh_checkout(checkout_dir)

start_build = False

last_build = datetime.datetime.fromtimestamp(
    int(status.get("last_build", "0")))

if ('regular_rebuild_interval' in config
        and datetime.datetime.now() - last_build
        > datetime.timedelta(hours=config['regular_rebuild_interval'])):
    start_build = True

current_commit = get_last_commit(checkout_dir)
if current_commit != status.get('last_build_commit', ''):
    start_build = True

submodule_commits = status.get('submodule_commits', {})
current_submodule_commits = {}
for submodule in get_submodules_checkout(checkout_dir):
    debug_print('Looking for %s' % submodule)
    current_submodule_commits[submodule] = \
        get_last_commit_submodule(checkout_dir, submodule)

    if current_submodule_commits[submodule] != \
            submodule_commits.get(submodule, ''):
        start_build = True

if args.force:
    start_build = True

if not start_build:
    debug_print("Nothing to build")
    sys.exit(0)

# Do not open earlier or we would end-up logging a lot of
# "Nothing to build" messages and lose the last build log.
log_file = os.path.expanduser('~/%s.log' % name)
log_fd = open(log_file, "w")
log_fd.write("last_build_date: {}\n".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S (UTC)")))
log_fd.write("last_build_commit: {}\n".format(current_commit))
log_fd.write("submodule_commits: {}\n".format(current_submodule_commits))
log_fd.write("\n")

syslog.syslog("Start the build of {}".format(name))

os.chdir(checkout_dir)
result = subprocess.check_output(['git', 'stash'], stderr=subprocess.STDOUT)
debug_print(result)
result = subprocess.check_output(['git', 'stash', 'clear'], stderr=subprocess.STDOUT)
debug_print(result)
result = subprocess.check_output(['git', 'pull', '--rebase'], stderr=subprocess.STDOUT)
debug_print(result)

if has_submodules(checkout_dir):
    result = subprocess.check_output(['git', 'submodule', 'init'], stderr=subprocess.STDOUT)
    debug_print(result)
    result = subprocess.check_output(['git', 'submodule', 'sync'], stderr=subprocess.STDOUT)
    debug_print(result)

if config.get('update_submodule_head', False):
    result = subprocess.check_output(['git', 'submodule', 'foreach',
                     '"git pull -qf origin master"'], stderr=subprocess.STDOUT)
    debug_print(result)

build_subdir = builder_info[config['builder']]['build_subdir']
build_dir = '%s/%s' % (checkout_dir, build_subdir)

if not args.sync_only:
    os.environ['PATH'] = "/usr/local/bin:/srv/builder/bin:" + \
                         os.environ['PATH']
    try:
        syslog.syslog("Build of {}: bundle install".format(name))
        result = subprocess.check_output(['bundle', 'install'], stderr=subprocess.STDOUT)
        debug_print(result)
    except subprocess.CalledProcessError, C:
        log_print(C.output)
        # TODO: find a way to publish the log on the website (as the sync step will never occur)
        notify_error('install', C.output)

    try:
        command = builder_info[config['builder']]['build_command']
        syslog.syslog("Build of {}: {}".format(name, ' '.join(command)))
        result = subprocess.check_output(command, stderr=subprocess.STDOUT)
        debug_print(result)
    except subprocess.CalledProcessError, C:
        log_print(C.output)
        # TODO: find a way to publish the log on the website (as the sync step will never occur)
        notify_error('build', C.output)

    # copy log in build dir to make it available to users
    # Using a .txt extension to ensure webservers will allow access to it
    shutil.copy2(log_file, '%s/build_log.txt' % build_dir)

if not args.dry_run:
    syslog.syslog("Build of {}: start sync".format(name))
    try:
        if config['remote']:
            result = subprocess.check_output(
                            ['rsync',
                             '-e',
                             'ssh '
                             '-o '
                             'UserKnownHostsFile=/dev/null '
                             '-o '
                             'StrictHostKeyChecking=no '
                             '-i ' +
                             os.path.expanduser('~/.ssh/{}_id.rsa'.format(name)),
                             '--delete-after',
                             '-rltogvz',
                             '--omit-dir-times',
                             '%s/' % build_dir,
                             config['remote']],
                             stderr=subprocess.STDOUT)
        else:
            command = builder_info[config['builder']]['deploy_command']
            if command:
                result = subprocess.check_output(command)
            else:
                notify_error('deploy', "builder '%s' does not possess a deploy method" % config['builder'])
        debug_print(result)
    except subprocess.CalledProcessError, C:
        notify_error('deploy', C.output)
    syslog.syslog("Build of {}: finish sync".format(name))
else:
    syslog.syslog("Build of {}: not syncing, dry-run".format(name))

status = {}
status['last_build_commit'] = current_commit
status['last_build'] = datetime.datetime.now().strftime("%s")
status['last_build_human'] = datetime.datetime.now().strftime("%c")
status['submodule_commits'] = current_submodule_commits

with open(status_file, 'w+') as f:
    f.write(yaml.dump(status, default_flow_style=False))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Tool to generate a NMF topic model on one or more corpora, using a fixed number of topics.
"""
import os, sys, random, operator
import logging as log
from optparse import OptionParser
import numpy as np
import text.util
import unsupervised.nmf, unsupervised.rankings, unsupervised.coherence

# --------------------------------------------------------------

def main():
	parser = OptionParser(usage="usage: %prog [options] window_matrix1 window_matrix2...")
	parser.add_option("--seed", action="store", type="int", dest="seed", help="initial random seed", default=1000)
	parser.add_option("-k", action="store", type="string", dest="krange", help="number of topics", default=None)
	parser.add_option("--maxiters", action="store", type="int", dest="maxiter", help="maximum number of iterations", default=200)
	parser.add_option("-o","--outdir", action="store", type="string", dest="dir_out", help="base output directory (default is current directory)", default=None)
	parser.add_option("-m", "--model", action="store", type="string", dest="model_path", help="path to Word2Vec model, if performing automatic selection of number of topics", default=None)
	parser.add_option("-t", "--top", action="store", type="int", dest="top", help="number of top terms to use, if performing automatic selection of number of topics", default=20)
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="display topic descriptors")
	(options, args) = parser.parse_args()
	if( len(args) < 1 ):
		parser.error( "Must specify at least one time window matrix file" )
	log.basicConfig(level=20, format='%(message)s')

	# Parse user-specified range for number of topics K
	if options.krange is None:
		parser.error("Must specific number of topics, or a range for the number of topics")
	kparts = options.krange.split(",")
	kmin = int(kparts[0])

	# Output directory for results
	if options.dir_out is None:
		dir_out = os.getcwd()
	else:
		dir_out = options.dir_out

	# Set random state
	random_seed = options.seed
	if random_seed < 0:
		random_seed = random.randint(1,100000)
	np.random.seed( random_seed )
	random.seed( random_seed )
	log.info("Using random seed %s" % random_seed )

	# Will we use automatic model selection?
	validation_measure = None
	if len(kparts) == 1:
		kmax = kmin
	else:
		kmax = int(kparts[1])
		# any word2vec model specified?
		if not options.model_path is None:
			log.info( "Loading Word2Vec model from %s ..." % options.model_path )
			import gensim
			model = gensim.models.Word2Vec.load(options.model_path)
			validation_measure = unsupervised.coherence.WithinTopicMeasure( unsupervised.coherence.ModelSimilarity(model) )

	# NMF implementation
	impl = unsupervised.nmf.SklNMF( max_iters = options.maxiter, init_strategy = "nndsvd" )

	# Process each specified time window document-term matrix
	for matrix_filepath in args:
		# Load the cached corpus
		window_name = os.path.splitext( os.path.split( matrix_filepath )[-1] )[0]
		log.info( "- Processing time window matrix for '%s' from %s ..." % (window_name,matrix_filepath) )
		(X,terms,doc_ids) = text.util.load_corpus( matrix_filepath )
		log.info( "Read %dx%d document-term matrix" % ( X.shape[0], X.shape[1] ) )

		# Generate window topic model for the specified range of numbers of topics
		coherence_scores = {}
		for k in range(kmin,kmax+1):
			log.info( "Applying window topic modeling to matrix for k=%d topics ..." % k )
			try:
				impl.apply( X, k )
			except IndexError as e:
				# a sklearn error can happen when applying NMF for high values of K and very small datasets
				log.warning( "Error applying NMF for k=%d: %s" % ( k, str(e) ) )
				break
			log.info( "Generated %dx%d factor W and %dx%d factor H" % ( impl.W.shape[0], impl.W.shape[1], impl.H.shape[0], impl.H.shape[1] ) )
			# Create a disjoint partition of documents
			partition = impl.generate_partition()
			# Create topic labels
			topic_labels = []
			for i in range( k ):
				topic_labels.append( "%s_%02d" % ( window_name, (i+1) ) )
			# Create term rankings for each topic
			term_rankings = []
			for topic_index in range(k):
				ranked_term_indices = impl.rank_terms( topic_index )
				term_ranking = [terms[i] for i in ranked_term_indices]
				term_rankings.append(term_ranking)
			# Print out the top terms?
			if options.verbose:
				log.info( unsupervised.rankings.format_term_rankings( term_rankings, top=10 ) )
			# Evaluate topic coherence of this topic model?
			if not validation_measure is None:
				truncated_term_rankings = unsupervised.rankings.truncate_term_rankings( term_rankings, options.top )
				coherence_scores[k] = validation_measure.evaluate_rankings( truncated_term_rankings )
				log.info("Model coherence (k=%d) = %.4f" % (k,coherence_scores[k]) )
			# Write results
			results_out_path = os.path.join( dir_out, "%s_windowtopics_k%02d.pkl"  % (window_name, k) )
			unsupervised.nmf.save_nmf_results( results_out_path, doc_ids, terms, term_rankings, partition, impl.W, impl.H, topic_labels )

		# Need to select value of k?
		if len(coherence_scores) > 0:
			sx = sorted(coherence_scores.items(), key=operator.itemgetter(1))
			sx.reverse()
			top_k = [ p[0] for p in sx ][0:min(3,len(sx))]
			log.info("- Top recommendations for number of topics for '%s': %s" % (window_name,",".join(map(str, top_k))) )

# --------------------------------------------------------------

if __name__ == "__main__":
	main()

<commit_msg>Ensure k values are not > than number of documents<commit_after>#!/usr/bin/env python
"""
Tool to generate a NMF topic model on one or more corpora, using a fixed number of topics.
"""
import os, sys, random, operator
import logging as log
from optparse import OptionParser
import numpy as np
import text.util
import unsupervised.nmf, unsupervised.rankings, unsupervised.coherence

# --------------------------------------------------------------

def main():
	parser = OptionParser(usage="usage: %prog [options] window_matrix1 window_matrix2...")
	parser.add_option("--seed", action="store", type="int", dest="seed", help="initial random seed", default=1000)
	parser.add_option("-k", action="store", type="string", dest="krange", help="number of topics", default=None)
	parser.add_option("--maxiters", action="store", type="int", dest="maxiter", help="maximum number of iterations", default=200)
	parser.add_option("-o","--outdir", action="store", type="string", dest="dir_out", help="base output directory (default is current directory)", default=None)
	parser.add_option("-m", "--model", action="store", type="string", dest="model_path", help="path to Word2Vec model, if performing automatic selection of number of topics", default=None)
	parser.add_option("-t", "--top", action="store", type="int", dest="top", help="number of top terms to use, if performing automatic selection of number of topics", default=20)
	parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="display topic descriptors")
	(options, args) = parser.parse_args()
	if( len(args) < 1 ):
		parser.error( "Must specify at least one time window matrix file" )
	log.basicConfig(level=20, format='%(message)s')

	# Parse user-specified range for number of topics K
	if options.krange is None:
		parser.error("Must specific number of topics, or a range for the number of topics")
	kparts = options.krange.split(",")
	kmin = int(kparts[0])

	# Output directory for results
	if options.dir_out is None:
		dir_out = os.getcwd()
	else:
		dir_out = options.dir_out

	# Set random state
	random_seed = options.seed
	if random_seed < 0:
		random_seed = random.randint(1,100000)
	np.random.seed( random_seed )
	random.seed( random_seed )
	log.info("Using random seed %s" % random_seed )

	# Will we use automatic model selection?
	validation_measure = None
	if len(kparts) == 1:
		kmax = kmin
	else:
		kmax = int(kparts[1])
		# any word2vec model specified?
		if not options.model_path is None:
			log.info( "Loading Word2Vec model from %s ..." % options.model_path )
			import gensim
			model = gensim.models.Word2Vec.load(options.model_path)
			validation_measure = unsupervised.coherence.WithinTopicMeasure( unsupervised.coherence.ModelSimilarity(model) )

	# NMF implementation
	impl = unsupervised.nmf.SklNMF( max_iters = options.maxiter, init_strategy = "nndsvd" )

	# Process each specified time window document-term matrix
	for matrix_filepath in args:
		# Load the cached corpus
		window_name = os.path.splitext( os.path.split( matrix_filepath )[-1] )[0]
		log.info( "- Processing time window matrix for '%s' from %s ..." % (window_name,matrix_filepath) )
		(X,terms,doc_ids) = text.util.load_corpus( matrix_filepath )
		log.info( "Read %dx%d document-term matrix" % ( X.shape[0], X.shape[1] ) )

		# Ensure that value of kmin and kmax are not greater than the number of documents
		num_docs = len(doc_ids)
		actual_kmin = min( num_docs, kmin )
		actual_kmax = min( num_docs, kmax )

		# Generate window topic model for the specified range of numbers of topics
		log.info( "Generating models in range [%d,%d] ..." % ( actual_kmin, actual_kmax ) )
		coherence_scores = {}
		for k in range(actual_kmin,actual_kmax+1):
			log.info( "Applying window topic modeling to matrix for k=%d topics ..." % k )
			try:
				impl.apply( X, k )
			except IndexError as e:
				# a sklearn error can happen when applying NMF for high values of K and very small datasets
				log.warning( "Error applying NMF for k=%d: %s" % ( k, str(e) ) )
				break
			log.info( "Generated %dx%d factor W and %dx%d factor H" % ( impl.W.shape[0], impl.W.shape[1], impl.H.shape[0], impl.H.shape[1] ) )
			# Create a disjoint partition of documents
			partition = impl.generate_partition()
			# Create topic labels
			topic_labels = []
			for i in range( k ):
				topic_labels.append( "%s_%02d" % ( window_name, (i+1) ) )
			# Create term rankings for each topic
			term_rankings = []
			for topic_index in range(k):
				ranked_term_indices = impl.rank_terms( topic_index )
				term_ranking = [terms[i] for i in ranked_term_indices]
				term_rankings.append(term_ranking)
			# Print out the top terms?
			if options.verbose:
				log.info( unsupervised.rankings.format_term_rankings( term_rankings, top=10 ) )
			# Evaluate topic coherence of this topic model?
			if not validation_measure is None:
				truncated_term_rankings = unsupervised.rankings.truncate_term_rankings( term_rankings, options.top )
				coherence_scores[k] = validation_measure.evaluate_rankings( truncated_term_rankings )
				log.info("Model coherence (k=%d) = %.4f" % (k,coherence_scores[k]) )
			# Write results
			results_out_path = os.path.join( dir_out, "%s_windowtopics_k%02d.pkl"  % (window_name, k) )
			unsupervised.nmf.save_nmf_results( results_out_path, doc_ids, terms, term_rankings, partition, impl.W, impl.H, topic_labels )

		# Need to select value of k?
		if len(coherence_scores) > 0:
			sx = sorted(coherence_scores.items(), key=operator.itemgetter(1))
			sx.reverse()
			top_k = [ p[0] for p in sx ][0:min(3,len(sx))]
			log.info("- Top recommendations for number of topics for '%s': %s" % (window_name,",".join(map(str, top_k))) )

# --------------------------------------------------------------

if __name__ == "__main__":
	main()

<|endoftext|>


<commit_before>from .enums import CardClass, CardType, GameTag, Race, Rarity

class Manager(object):
	def __init__(self, obj):
		self.obj = obj
		self.observers = []

	def __getitem__(self, tag):
		return getattr(self.obj, self.map[tag])

	def __setitem__(self, tag, value):
		setattr(self.obj, self.map[tag], value)

	def register(self, observer):
		self.observers.append(observer)

	def update(self, tags):
		for k, v in tags.items():
			if self.map[k]:
				self[k] = v

class GameManager(Manager):
	map = {
		GameTag.NUM_MINIONS_KILLED_THIS_TURN: "minionsKilledThisTurn",
		GameTag.PROPOSED_ATTACKER: "proposedAttacker",
		GameTag.PROPOSED_DEFENDER: "proposedDefender",
		GameTag.TURN: "turn",
	}

	def new_entity(self, entity):
		for observer in self.observers:
			observer.new_entity(entity)

class PlayerManager(Manager):
	map = {
		GameTag.CARDTYPE: "type",
		GameTag.COMBO_ACTIVE: "combo",
		GameTag.FATIGUE: "fatigueCounter",
		GameTag.FIRST_PLAYER: "firstPlayer",
		GameTag.HERO_ENTITY: "hero",
		GameTag.LAST_CARD_PLAYED: "lastCardPlayed",
		GameTag.MAXHANDSIZE: "maxHandSize",
		GameTag.MAXRESOURCES: "maxResources",
		GameTag.NUM_CARDS_DRAWN_THIS_TURN: "cardsDrawnThisTurn",
		GameTag.NUM_CARDS_PLAYED_THIS_TURN: "cardsPlayedThisTurn",
		GameTag.NUM_MINIONS_PLAYED_THIS_TURN: "minionsPlayedThisTurn",
		GameTag.NUM_MINIONS_PLAYER_KILLED_THIS_TURN: "minionsKilledThisTurn",
		GameTag.NUM_TIMES_HERO_POWER_USED_THIS_GAME: "timesHeroPowerUsedThisGame",
		GameTag.OUTGOING_HEALING_ADJUSTMENT: "outgoingHealingAdjustment",
		GameTag.CURRENT_SPELLPOWER: "spellpower",
		GameTag.RECALL_OWED: "overloaded",
		GameTag.RESOURCES_USED: "usedMana",
		GameTag.TEMP_RESOURCES: "tempMana",
		GameTag.TIMEOUT: "timeout",
		GameTag.TURN_START: "turnStart",
	}

class CardManager(Manager):
	map = {
		GameTag.AURA: "aura",
		GameTag.CARD_ID: "id",
		GameTag.CONTROLLER: "controller",
		GameTag.CARDNAME: "name",
		GameTag.CARDTYPE: "type",
		GameTag.CLASS: "cardClass",
		GameTag.COST: "cost",
		GameTag.DEATHRATTLE: "hasDeathrattle",
		GameTag.EXHAUSTED: "exhausted",
		GameTag.RARITY: "rarity",
		GameTag.SPELLPOWER: "spellpower",
		GameTag.ARTISTNAME: None,
		GameTag.AttackVisualType: None,
		GameTag.CARD_SET: None,
		GameTag.CARDTEXT_INHAND: None,
		GameTag.CardTextInPlay: None,
		GameTag.DevState: None,
		GameTag.ENCHANTMENT_IDLE_VISUAL: None,
		GameTag.ENCHANTMENT_BIRTH_VISUAL: None,
		GameTag.FACTION: None,
		GameTag.FLAVORTEXT: None,
		GameTag.HealTarget: None,
		GameTag.HOW_TO_EARN: None,
		GameTag.HOW_TO_EARN_GOLDEN: None,
		GameTag.SILENCE: None,
		GameTag.TRIGGER_VISUAL: None,
	}

class PlayableCardManager(Manager):
	map = CardManager.map.copy()
	map.update({
		GameTag.BATTLECRY: "hasBattlecry",
		GameTag.Collectible: "collectible",
		GameTag.CARD_TARGET: "target",
		GameTag.COMBO: "hasCombo",
		GameTag.DEFENDING: "defending",
		GameTag.FREEZE: "freeze",
		GameTag.RECALL: "overload",
		GameTag.WINDFURY: "windfury",
		GameTag.TARGETING_ARROW_TEXT: None,
		GameTag.TOPDECK: None,
	})

class CharacterManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.ATTACKING: "attacking",
		GameTag.CANT_ATTACK: "cantAttack",
		GameTag.CANT_BE_DAMAGED: "immune",
		GameTag.CANT_BE_TARGETED_BY_ABILITIES: "cantBeTargetedByAbilities",
		GameTag.CANT_BE_TARGETED_BY_HERO_POWERS: "cantBeTargetedByHeroPowers",
		GameTag.DAMAGE: "damage",
		GameTag.FROZEN: "frozen",
		GameTag.HEALTH: "maxHealth",
		GameTag.NUM_ATTACKS_THIS_TURN: "numAttacks",
		GameTag.SHOULDEXITCOMBAT: "shouldExitCombat",
	})

class HeroManager(Manager):
	map = CharacterManager.map.copy()
	map.update({
		GameTag.ARMOR: "armor",
	})

class MinionManager(Manager):
	map = CharacterManager.map.copy()
	map.update({
		GameTag.ADJACENT_BUFF: "adjacentBuff",
		GameTag.CARDRACE: "race",
		GameTag.CHARGE: "charge",
		GameTag.DIVINE_SHIELD: "divineShield",
		GameTag.ENRAGED: "enrage",
		GameTag.FORGETFUL: "forgetful",
		GameTag.POISONOUS: "poisonous",
		GameTag.SILENCED: "silenced",
		GameTag.STEALTH: "stealthed",
		GameTag.TAUNT: "taunt",
		GameTag.ELITE: None,
	})

class WeaponManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.DURABILITY: "durability",
	})

class SpellManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ImmuneToSpellpower: "immuneToSpellpower",
		GameTag.SECRET: "secret",
		GameTag.AFFECTED_BY_SPELL_POWER: None,
	})

class EnchantmentManager(Manager):
	map = CardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.ATTACHED: "owner",
		GameTag.CANT_BE_DAMAGED: "immune",
		GameTag.CHARGE: "charge",
		GameTag.CREATOR: "creator",
		GameTag.DURABILITY: "durability",
		GameTag.EXTRA_DEATHRATTLES: "extraDeathrattles",
		GameTag.HEALTH: "maxHealth",
		GameTag.HEALTH_MINIMUM: "minHealth",
		GameTag.OneTurnEffect: "oneTurnEffect",
		GameTag.OUTGOING_HEALING_ADJUSTMENT: "outgoingHealingAdjustment",
		GameTag.STEALTH: "stealthed",
		GameTag.TAUNT: "taunt",
	})
<commit_msg>Handle missing attributes in Manager.__getitem__()<commit_after>from .enums import CardClass, CardType, GameTag, Race, Rarity

class Manager(object):
	def __init__(self, obj):
		self.obj = obj
		self.observers = []

	def __getitem__(self, tag):
		if self.map.get(tag):
			return getattr(self.obj, self.map[tag], 0)
		raise KeyError

	def __setitem__(self, tag, value):
		setattr(self.obj, self.map[tag], value)

	def register(self, observer):
		self.observers.append(observer)

	def update(self, tags):
		for k, v in tags.items():
			if self.map[k]:
				self[k] = v

class GameManager(Manager):
	map = {
		GameTag.NUM_MINIONS_KILLED_THIS_TURN: "minionsKilledThisTurn",
		GameTag.PROPOSED_ATTACKER: "proposedAttacker",
		GameTag.PROPOSED_DEFENDER: "proposedDefender",
		GameTag.TURN: "turn",
	}

	def new_entity(self, entity):
		for observer in self.observers:
			observer.new_entity(entity)

class PlayerManager(Manager):
	map = {
		GameTag.CARDTYPE: "type",
		GameTag.COMBO_ACTIVE: "combo",
		GameTag.FATIGUE: "fatigueCounter",
		GameTag.FIRST_PLAYER: "firstPlayer",
		GameTag.HERO_ENTITY: "hero",
		GameTag.LAST_CARD_PLAYED: "lastCardPlayed",
		GameTag.MAXHANDSIZE: "maxHandSize",
		GameTag.MAXRESOURCES: "maxResources",
		GameTag.NUM_CARDS_DRAWN_THIS_TURN: "cardsDrawnThisTurn",
		GameTag.NUM_CARDS_PLAYED_THIS_TURN: "cardsPlayedThisTurn",
		GameTag.NUM_MINIONS_PLAYED_THIS_TURN: "minionsPlayedThisTurn",
		GameTag.NUM_MINIONS_PLAYER_KILLED_THIS_TURN: "minionsKilledThisTurn",
		GameTag.NUM_TIMES_HERO_POWER_USED_THIS_GAME: "timesHeroPowerUsedThisGame",
		GameTag.OUTGOING_HEALING_ADJUSTMENT: "outgoingHealingAdjustment",
		GameTag.CURRENT_SPELLPOWER: "spellpower",
		GameTag.RECALL_OWED: "overloaded",
		GameTag.RESOURCES_USED: "usedMana",
		GameTag.TEMP_RESOURCES: "tempMana",
		GameTag.TIMEOUT: "timeout",
		GameTag.TURN_START: "turnStart",
	}

class CardManager(Manager):
	map = {
		GameTag.AURA: "aura",
		GameTag.CARD_ID: "id",
		GameTag.CONTROLLER: "controller",
		GameTag.CARDNAME: "name",
		GameTag.CARDTYPE: "type",
		GameTag.CLASS: "cardClass",
		GameTag.COST: "cost",
		GameTag.DEATHRATTLE: "hasDeathrattle",
		GameTag.EXHAUSTED: "exhausted",
		GameTag.RARITY: "rarity",
		GameTag.SPELLPOWER: "spellpower",
		GameTag.ARTISTNAME: None,
		GameTag.AttackVisualType: None,
		GameTag.CARD_SET: None,
		GameTag.CARDTEXT_INHAND: None,
		GameTag.CardTextInPlay: None,
		GameTag.DevState: None,
		GameTag.ENCHANTMENT_IDLE_VISUAL: None,
		GameTag.ENCHANTMENT_BIRTH_VISUAL: None,
		GameTag.FACTION: None,
		GameTag.FLAVORTEXT: None,
		GameTag.HealTarget: None,
		GameTag.HOW_TO_EARN: None,
		GameTag.HOW_TO_EARN_GOLDEN: None,
		GameTag.SILENCE: None,
		GameTag.TRIGGER_VISUAL: None,
	}

class PlayableCardManager(Manager):
	map = CardManager.map.copy()
	map.update({
		GameTag.BATTLECRY: "hasBattlecry",
		GameTag.Collectible: "collectible",
		GameTag.CARD_TARGET: "target",
		GameTag.COMBO: "hasCombo",
		GameTag.DEFENDING: "defending",
		GameTag.FREEZE: "freeze",
		GameTag.RECALL: "overload",
		GameTag.WINDFURY: "windfury",
		GameTag.TARGETING_ARROW_TEXT: None,
		GameTag.TOPDECK: None,
	})

class CharacterManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.ATTACKING: "attacking",
		GameTag.CANT_ATTACK: "cantAttack",
		GameTag.CANT_BE_DAMAGED: "immune",
		GameTag.CANT_BE_TARGETED_BY_ABILITIES: "cantBeTargetedByAbilities",
		GameTag.CANT_BE_TARGETED_BY_HERO_POWERS: "cantBeTargetedByHeroPowers",
		GameTag.DAMAGE: "damage",
		GameTag.FROZEN: "frozen",
		GameTag.HEALTH: "maxHealth",
		GameTag.NUM_ATTACKS_THIS_TURN: "numAttacks",
		GameTag.SHOULDEXITCOMBAT: "shouldExitCombat",
	})

class HeroManager(Manager):
	map = CharacterManager.map.copy()
	map.update({
		GameTag.ARMOR: "armor",
	})

class MinionManager(Manager):
	map = CharacterManager.map.copy()
	map.update({
		GameTag.ADJACENT_BUFF: "adjacentBuff",
		GameTag.CARDRACE: "race",
		GameTag.CHARGE: "charge",
		GameTag.DIVINE_SHIELD: "divineShield",
		GameTag.ENRAGED: "enrage",
		GameTag.FORGETFUL: "forgetful",
		GameTag.POISONOUS: "poisonous",
		GameTag.SILENCED: "silenced",
		GameTag.STEALTH: "stealthed",
		GameTag.TAUNT: "taunt",
		GameTag.ELITE: None,
	})

class WeaponManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.DURABILITY: "durability",
	})

class SpellManager(Manager):
	map = PlayableCardManager.map.copy()
	map.update({
		GameTag.ImmuneToSpellpower: "immuneToSpellpower",
		GameTag.SECRET: "secret",
		GameTag.AFFECTED_BY_SPELL_POWER: None,
	})

class EnchantmentManager(Manager):
	map = CardManager.map.copy()
	map.update({
		GameTag.ATK: "atk",
		GameTag.ATTACHED: "owner",
		GameTag.CANT_BE_DAMAGED: "immune",
		GameTag.CHARGE: "charge",
		GameTag.CREATOR: "creator",
		GameTag.DURABILITY: "durability",
		GameTag.EXTRA_DEATHRATTLES: "extraDeathrattles",
		GameTag.HEALTH: "maxHealth",
		GameTag.HEALTH_MINIMUM: "minHealth",
		GameTag.OneTurnEffect: "oneTurnEffect",
		GameTag.OUTGOING_HEALING_ADJUSTMENT: "outgoingHealingAdjustment",
		GameTag.STEALTH: "stealthed",
		GameTag.TAUNT: "taunt",
	})
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyDecorator(PythonPackage):
    """The aim of the decorator module it to simplify the usage of decorators
       for the average programmer, and to popularize decorators by showing
       various non-trivial examples."""
    homepage = "https://github.com/micheles/decorator"
    url      = "https://pypi.io/packages/source/d/decorator/decorator-4.0.9.tar.gz"

    version('4.3.0', sha256='c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c')
    version('4.0.9', sha256='90022e83316363788a55352fe39cfbed357aa3a71d90e5f2803a35471de4bba8')

    depends_on('py-setuptools', type='build')
<commit_msg>Add latest version of py-decorator (#13318)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyDecorator(PythonPackage):
    """The aim of the decorator module it to simplify the usage of decorators
       for the average programmer, and to popularize decorators by showing
       various non-trivial examples."""

    homepage = "https://github.com/micheles/decorator"
    url      = "https://pypi.io/packages/source/d/decorator/decorator-4.4.0.tar.gz"

    version('4.4.0', sha256='86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de')
    version('4.3.0', sha256='c39efa13fbdeb4506c476c9b3babf6a718da943dab7811c206005a4a956c080c')
    version('4.0.9', sha256='90022e83316363788a55352fe39cfbed357aa3a71d90e5f2803a35471de4bba8')

    depends_on('python@2.6:2.8,3.2:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyQtconsole(PythonPackage):
    """Jupyter Qt console"""

    homepage = "http://ipython.org"
    pypi = "qtconsole/qtconsole-4.2.1.tar.gz"

    version('4.5.1', sha256='4af84facdd6f00a6b9b2927255f717bb23ae4b7a20ba1d9ef0a5a5a8dbe01ae2')
    version('4.2.1', sha256='25ec7d345528b3e8f3c91be349dd3c699755f206dc4b6ec668e2e5dd60ea18ef')

    variant('doc', default=False, description='Build documentation')

    depends_on('python@2.7:2.8,3.3:',    type=('build', 'run'))
    depends_on('py-ipykernel@4.1:',      type=('build', 'run'))
    depends_on('py-jupyter-client@4.1:', type=('build', 'run'))
    depends_on('py-jupyter-core',        type=('build', 'run'))
    depends_on('py-pygments',            type=('build', 'run'))
    depends_on('py-traitlets',           type=('build', 'run'))
    depends_on('py-ipython-genutils',    type=('build', 'run'), when='@4.5.1:')
    depends_on('py-sphinx@1.3:',         type=('build', 'run'), when='+docs')
    depends_on('py-pyqt5',               type='run')
<commit_msg>py-qtconsole: fix misnamed variant (#23746)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyQtconsole(PythonPackage):
    """Jupyter Qt console"""

    homepage = "http://ipython.org"
    pypi = "qtconsole/qtconsole-4.2.1.tar.gz"

    version('4.5.1', sha256='4af84facdd6f00a6b9b2927255f717bb23ae4b7a20ba1d9ef0a5a5a8dbe01ae2')
    version('4.2.1', sha256='25ec7d345528b3e8f3c91be349dd3c699755f206dc4b6ec668e2e5dd60ea18ef')

    variant('docs', default=False, description='Build documentation')

    depends_on('python@2.7:2.8,3.3:',    type=('build', 'run'))
    depends_on('py-ipykernel@4.1:',      type=('build', 'run'))
    depends_on('py-jupyter-client@4.1:', type=('build', 'run'))
    depends_on('py-jupyter-core',        type=('build', 'run'))
    depends_on('py-pygments',            type=('build', 'run'))
    depends_on('py-traitlets',           type=('build', 'run'))
    depends_on('py-ipython-genutils',    type=('build', 'run'), when='@4.5.1:')
    depends_on('py-sphinx@1.3:',         type=('build', 'run'), when='+docs')
    depends_on('py-pyqt5',               type='run')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012-2013, Cenobit Technologies, Inc. http://cenobit.es/
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
# * Neither the name of the Cenobit Technologies nor the names of
#    its contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import re
import decimal
import datetime
from uuid import uuid1
from functools import wraps

from flask import json, jsonify, current_app, got_request_exception

from flask_jsonrpc.helpers import extract_raw_data_request, log_exception
from flask_jsonrpc.types import Object, Array, Any
from flask_jsonrpc.exceptions import (Error, ParseError, InvalidRequestError,
                                      MethodNotFoundError, InvalidParamsError,
                                      ServerError, RequestPostError,
                                      InvalidCredentialsError, OtherError)

empty_dec = lambda f: f
try:
    # TODO: Add CSRF check
    csrf_exempt = empty_dec
except (NameError, ImportError):
    csrf_exempt = empty_dec

NoneType = type(None)
encode_kw = lambda p: dict([(str(k), v) for k, v in p.iteritems()])

def encode_kw11(p):
    if not type(p) is dict:
        return {}
    ret = p.copy()
    removes = []
    for k, v in ret.iteritems():
        try:
            int(k)
        except ValueError:
            pass
        else:
            removes.append(k)
    for k in removes:
        ret.pop(k)
    return ret

def encode_arg11(p):
    if type(p) is list:
        return p
    elif not type(p) is dict:
        return []
    else:
        pos = []
        d = encode_kw(p)
        for k, v in d.iteritems():
            try:
                pos.append(int(k))
            except ValueError:
                pass
        pos = list(set(pos))
        pos.sort()
        return [d[str(i)] for i in pos]

def validate_params(method, D):
    if type(D['params']) == Object:
        keys = method.json_arg_types.keys()
        if len(keys) != len(D['params']):
            raise InvalidParamsError('Not eough params provided for %s' % method.json_sig)
        for k in keys:
            if not k in D['params']:
                raise InvalidParamsError('%s is not a valid parameter for %s'
                                         % (k, method.json_sig))
            if not Any.kind(D['params'][k]) == method.json_arg_types[k]:
                raise InvalidParamsError('%s is not the correct type %s for %s'
                    % (type(D['params'][k]), method.json_arg_types[k], method.json_sig))
    elif type(D['params']) == Array:
        arg_types = method.json_arg_types.values()
        try:
            for i, arg in enumerate(D['params']):
                if not Any.kind(arg) == arg_types[i]:
                    raise InvalidParamsError('%s is not the correct type %s for %s'
                                             % (type(arg), arg_types[i], method.json_sig))
        except IndexError:
            raise InvalidParamsError('Too many params provided for %s' % method.json_sig)
        else:
            if len(D['params']) != len(arg_types):
                raise InvalidParamsError('Not enouh params provided for %s' % method.json_sig)

class JSONRPCSite(object):
    """A JSON-RPC Site
    """

    def __init__(self):
        self.urls = {}
        self.uuid = str(uuid1())
        self.version = '1.0'
        self.name = 'Flask-JSONRPC'
        self.register('system.describe', self.describe)

    def register(self, name, method):
        self.urls[unicode(name)] = method

    def extract_id_request(self, raw_data):
        if not raw_data is None and raw_data.find('id') != -1:
            find_id = re.findall(r'["|\']id["|\']:([0-9]+)|["|\']id["|\']:["|\'](.+?)["|\']',
                                 raw_data.replace(' ', ''), re.U)
            if find_id:
                g1, g2 = find_id[0]
                return g1 if g1 else g2
        return None

    def empty_response(self, version='1.0'):
        resp = {'id': None}
        if version == '1.1':
            resp['version'] = version
            return resp
        if version == '2.0':
            resp['jsonrpc'] = version
        resp.update({'error': None, 'result': None})
        return resp

    def validate_get(self, request, method):
        encode_get_params = lambda r: dict([(k, v[0] if len(v) == 1 else v) for k, v in r])
        if request.method == 'GET':
            method = unicode(method)
            if method in self.urls and getattr(self.urls[method], 'json_safe', False):
                D = {
                    'params': request.args.to_dict(),
                    'method': method,
                    'id': 'jsonrpc',
                    'version': '1.1'
                }
                return True, D
        return False, {}

    def response_dict(self, request, D, is_batch=False, version_hint='1.0'):
        version = version_hint
        response = self.empty_response(version=version)
        apply_version = {
            '2.0': lambda f, r, p: f(**encode_kw(p)) if type(p) is dict else f(*p),
            '1.1': lambda f, r, p: f(*encode_arg11(p), **encode_kw(encode_kw11(p))),
            '1.0': lambda f, r, p: f(*p)
        }

        try:
            # params: An Array or Object, that holds the actual parameter values
            # for the invocation of the procedure. Can be omitted if empty.
            if 'params' not in D or not D['params']:
                 D['params'] = []
            if 'method' not in D or 'params' not in D:
                raise InvalidParamsError('Request requires str:"method" and list:"params"')
            if D['method'] not in self.urls:
                raise MethodNotFoundError('Method not found. Available methods: %s' % ('\n'.join(self.urls.keys())))

            if 'jsonrpc' in D:
                if str(D['jsonrpc']) not in apply_version:
                    raise InvalidRequestError('JSON-RPC version %s not supported.' % D['jsonrpc'])
                version = request.jsonrpc_version = response['jsonrpc'] = str(D['jsonrpc'])
            elif 'version' in D:
                if str(D['version']) not in apply_version:
                    raise InvalidRequestError('JSON-RPC version %s not supported.' % D['version'])
                version = request.jsonrpc_version = response['version'] = str(D['version'])
            else:
                request.jsonrpc_version = '1.0'

            method = self.urls[str(D['method'])]
            if getattr(method, 'json_validate', False):
                validate_params(method, D)
            R = apply_version[version](method, request, D['params'])

            encoder = json.JSONEncoder()
            if not sum(map(lambda e: isinstance(R, e), # type of `R` should be one of these or...
                 (dict, str, unicode, int, long, list, set, NoneType, bool))):
                try:
                    rs = encoder.default(R) # ...or something this thing supports
                except TypeError, exc:
                    raise TypeError("Return type not supported, for %r" % R)

            if 'id' in D and D['id'] is not None: # regular request
                response['result'] = R
                response['id'] = D['id']
                if version in ('1.1', '2.0') and 'error' in response:
                    response.pop('error')
            elif is_batch: # notification, not ok in a batch format, but happened anyway
                raise InvalidRequestError
            else: # notification
                return None, 204

            status = 200

        except Error, e:
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            response['error'] = e.json_rpc_format
            if version in ('1.1', '2.0') and 'result' in response:
                response.pop('result')
            status = e.status
        except Exception, e:
            # exception missed by others
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            other_error = OtherError(e)
            response['error'] = other_error.json_rpc_format
            status = other_error.status
            if version in ('1.1', '2.0') and 'result' in response:
                response.pop('result')

        # Exactly one of result or error MUST be specified. It's not
        # allowed to specify both or none.
        if version in ('1.1', '2.0') and 'error' in response and not response['error']:
            response.pop('error')

        return response, status

    @csrf_exempt
    def dispatch(self, request, method=''):

        # in case we do something json doesn't like, we always get back valid
        # json-rpc response
        response = self.empty_response()
        raw_data = extract_raw_data_request(request)

        try:
            if request.method == 'GET':
                valid, D = self.validate_get(request, method)
                if not valid:
                    raise InvalidRequestError('The method you are trying to access is '
                                              'not availble by GET requests')
            elif not request.method == 'POST':
                raise RequestPostError
            else:
                try:
                    D = json.loads(raw_data)
                except Exception, e:
                    raise InvalidRequestError(e.message)

            if type(D) is list:
                response = [self.response_dict(request, d, is_batch=True)[0] for d in D]
                status = 200
            else:
                response, status = self.response_dict(request, D)
                if response is None and (not u'id' in D or D[u'id'] is None): # a notification
                    response = ''
                    return response, status
        except Error, e:
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            response['error'] = e.json_rpc_format
            status = e.status
        except Exception, e:
            # exception missed by others
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            other_error = OtherError(e)

            response['result'] = None
            response['error'] = other_error.json_rpc_format
            status = other_error.status

        # extract id the request
        json_request_id = self.extract_id_request(raw_data)
        response['id'] = json_request_id

        return response, status

    def procedure_desc(self, key):
        M = self.urls[key]
        return {
            'name': M.json_method,
            'summary': M.__doc__,
            'idempotent': M.json_safe,
            'params': [{'type': str(Any.kind(t)), 'name': k}
                for k, t in M.json_arg_types.iteritems()],
            'return': {'type': str(Any.kind(M.json_return_type))}}

    def service_desc(self, request):
        return {
            'sdversion': '1.0',
            'name': self.name,
            'id': 'urn:uuid:%s' % str(self.uuid),
            'summary': self.__doc__,
            'version': self.version,
            'procs': [self.procedure_desc(k)
                for k in self.urls.iterkeys()
                    if self.urls[k] != self.describe and (request is None or self.urls[k].json_method == request]}

    def describe(self, request=None):
        return self.service_desc(request)

jsonrpc_site = JSONRPCSite()
<commit_msg>Update site.py<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012-2013, Cenobit Technologies, Inc. http://cenobit.es/
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
# * Neither the name of the Cenobit Technologies nor the names of
#    its contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import re
import decimal
import datetime
from uuid import uuid1
from functools import wraps

from flask import json, jsonify, current_app, got_request_exception

from flask_jsonrpc.helpers import extract_raw_data_request, log_exception
from flask_jsonrpc.types import Object, Array, Any
from flask_jsonrpc.exceptions import (Error, ParseError, InvalidRequestError,
                                      MethodNotFoundError, InvalidParamsError,
                                      ServerError, RequestPostError,
                                      InvalidCredentialsError, OtherError)
from werkzeug.exceptions import HTTPException

empty_dec = lambda f: f
try:
    # TODO: Add CSRF check
    csrf_exempt = empty_dec
except (NameError, ImportError):
    csrf_exempt = empty_dec

NoneType = type(None)
encode_kw = lambda p: dict([(str(k), v) for k, v in p.iteritems()])

def encode_kw11(p):
    if not type(p) is dict:
        return {}
    ret = p.copy()
    removes = []
    for k, v in ret.iteritems():
        try:
            int(k)
        except ValueError:
            pass
        else:
            removes.append(k)
    for k in removes:
        ret.pop(k)
    return ret

def encode_arg11(p):
    if type(p) is list:
        return p
    elif not type(p) is dict:
        return []
    else:
        pos = []
        d = encode_kw(p)
        for k, v in d.iteritems():
            try:
                pos.append(int(k))
            except ValueError:
                pass
        pos = list(set(pos))
        pos.sort()
        return [d[str(i)] for i in pos]

def validate_params(method, D):
    if type(D['params']) == Object:
        keys = method.json_arg_types.keys()
        if len(keys) != len(D['params']):
            raise InvalidParamsError('Not eough params provided for %s' % method.json_sig)
        for k in keys:
            if not k in D['params']:
                raise InvalidParamsError('%s is not a valid parameter for %s'
                                         % (k, method.json_sig))
            if not Any.kind(D['params'][k]) == method.json_arg_types[k]:
                raise InvalidParamsError('%s is not the correct type %s for %s'
                    % (type(D['params'][k]), method.json_arg_types[k], method.json_sig))
    elif type(D['params']) == Array:
        arg_types = method.json_arg_types.values()
        try:
            for i, arg in enumerate(D['params']):
                if not Any.kind(arg) == arg_types[i]:
                    raise InvalidParamsError('%s is not the correct type %s for %s'
                                             % (type(arg), arg_types[i], method.json_sig))
        except IndexError:
            raise InvalidParamsError('Too many params provided for %s' % method.json_sig)
        else:
            if len(D['params']) != len(arg_types):
                raise InvalidParamsError('Not enouh params provided for %s' % method.json_sig)

class JSONRPCSite(object):
    """A JSON-RPC Site
    """

    def __init__(self):
        self.urls = {}
        self.uuid = str(uuid1())
        self.version = '1.0'
        self.name = 'Flask-JSONRPC'
        self.register('system.describe', self.describe)

    def register(self, name, method):
        self.urls[unicode(name)] = method

    def extract_id_request(self, raw_data):
        if not raw_data is None and raw_data.find('id') != -1:
            find_id = re.findall(r'["|\']id["|\']:([0-9]+)|["|\']id["|\']:["|\'](.+?)["|\']',
                                 raw_data.replace(' ', ''), re.U)
            if find_id:
                g1, g2 = find_id[0]
                return g1 if g1 else g2
        return None

    def empty_response(self, version='1.0'):
        resp = {'id': None}
        if version == '1.1':
            resp['version'] = version
            return resp
        if version == '2.0':
            resp['jsonrpc'] = version
        resp.update({'error': None, 'result': None})
        return resp

    def validate_get(self, request, method):
        encode_get_params = lambda r: dict([(k, v[0] if len(v) == 1 else v) for k, v in r])
        if request.method == 'GET':
            method = unicode(method)
            if method in self.urls and getattr(self.urls[method], 'json_safe', False):
                D = {
                    'params': request.args.to_dict(),
                    'method': method,
                    'id': 'jsonrpc',
                    'version': '1.1'
                }
                return True, D
        return False, {}

    def response_dict(self, request, D, is_batch=False, version_hint='1.0'):
        version = version_hint
        response = self.empty_response(version=version)
        apply_version = {
            '2.0': lambda f, r, p: f(**encode_kw(p)) if type(p) is dict else f(*p),
            '1.1': lambda f, r, p: f(*encode_arg11(p), **encode_kw(encode_kw11(p))),
            '1.0': lambda f, r, p: f(*p)
        }

        try:
            # params: An Array or Object, that holds the actual parameter values
            # for the invocation of the procedure. Can be omitted if empty.
            if 'params' not in D or not D['params']:
                 D['params'] = []
            if 'method' not in D or 'params' not in D:
                raise InvalidParamsError('Request requires str:"method" and list:"params"')
            if D['method'] not in self.urls:
                raise MethodNotFoundError('Method not found. Available methods: %s' % ('\n'.join(self.urls.keys())))

            if 'jsonrpc' in D:
                if str(D['jsonrpc']) not in apply_version:
                    raise InvalidRequestError('JSON-RPC version %s not supported.' % D['jsonrpc'])
                version = request.jsonrpc_version = response['jsonrpc'] = str(D['jsonrpc'])
            elif 'version' in D:
                if str(D['version']) not in apply_version:
                    raise InvalidRequestError('JSON-RPC version %s not supported.' % D['version'])
                version = request.jsonrpc_version = response['version'] = str(D['version'])
            else:
                request.jsonrpc_version = '1.0'

            method = self.urls[str(D['method'])]
            if getattr(method, 'json_validate', False):
                validate_params(method, D)
            R = apply_version[version](method, request, D['params'])

            encoder = json.JSONEncoder()
            if not sum(map(lambda e: isinstance(R, e), # type of `R` should be one of these or...
                 (dict, str, unicode, int, long, list, set, NoneType, bool))):
                try:
                    rs = encoder.default(R) # ...or something this thing supports
                except TypeError, exc:
                    raise TypeError("Return type not supported, for %r" % R)

            if 'id' in D and D['id'] is not None: # regular request
                response['result'] = R
                response['id'] = D['id']
                if version in ('1.1', '2.0') and 'error' in response:
                    response.pop('error')
            elif is_batch: # notification, not ok in a batch format, but happened anyway
                raise InvalidRequestError
            else: # notification
                return None, 204

            status = 200

        except Error, e:
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            response['error'] = e.json_rpc_format
            if version in ('1.1', '2.0') and 'result' in response:
                response.pop('result')
            status = e.status
        except HTTPException, e:
            other_error = OtherError(e)
            response['error'] = other_error.json_rpc_format
            status = e.code

            if version in ('1.1', '2.0') and 'result' in response:
                response.pop('result')
        except Exception, e:
            # exception missed by others
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            other_error = OtherError(e)
            response['error'] = other_error.json_rpc_format
            status = other_error.status
            if version in ('1.1', '2.0') and 'result' in response:
                response.pop('result')

        # Exactly one of result or error MUST be specified. It's not
        # allowed to specify both or none.
        if version in ('1.1', '2.0') and 'error' in response and not response['error']:
            response.pop('error')

        return response, status

    @csrf_exempt
    def dispatch(self, request, method=''):

        # in case we do something json doesn't like, we always get back valid
        # json-rpc response
        response = self.empty_response()
        raw_data = extract_raw_data_request(request)

        try:
            if request.method == 'GET':
                valid, D = self.validate_get(request, method)
                if not valid:
                    raise InvalidRequestError('The method you are trying to access is '
                                              'not availble by GET requests')
            elif not request.method == 'POST':
                raise RequestPostError
            else:
                try:
                    D = json.loads(raw_data)
                except Exception, e:
                    raise InvalidRequestError(e.message)

            if type(D) is list:
                response = [self.response_dict(request, d, is_batch=True)[0] for d in D]
                status = 200
            else:
                response, status = self.response_dict(request, D)
                if response is None and (not u'id' in D or D[u'id'] is None): # a notification
                    response = ''
                    return response, status
        except Error, e:
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            response['error'] = e.json_rpc_format
            status = e.status
        except Exception, e:
            # exception missed by others
            #got_request_exception.connect(log_exception, current_app._get_current_object())

            other_error = OtherError(e)

            response['result'] = None
            response['error'] = other_error.json_rpc_format
            status = other_error.status

        # extract id the request
        json_request_id = self.extract_id_request(raw_data)
        response['id'] = json_request_id

        return response, status

    def procedure_desc(self, key):
        M = self.urls[key]
        return {
            'name': M.json_method,
            'summary': M.__doc__,
            'idempotent': M.json_safe,
            'params': [{'type': str(Any.kind(t)), 'name': k}
                for k, t in M.json_arg_types.iteritems()],
            'return': {'type': str(Any.kind(M.json_return_type))}}

    def service_desc(self, request):
        return {
            'sdversion': '1.0',
            'name': self.name,
            'id': 'urn:uuid:%s' % str(self.uuid),
            'summary': self.__doc__,
            'version': self.version,
            'procs': [self.procedure_desc(k)
                for k in self.urls.iterkeys()
                    if self.urls[k] != self.describe and (request is None or self.urls[k].json_method == request]}

    def describe(self, request=None):
        return self.service_desc(request)

jsonrpc_site = JSONRPCSite()
<|endoftext|>