<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from floreal import models as m

def delivery_description(delivery, subgroups, **kwargs):
    """Generate a description of the purchases performed by users in `subgroups`
    for `delivery`. The resulting dictionary is used to render HTML as well as
    PDF and MS-Excel views. It's used both to display the complete order to network staff,
    and their subgroup's purchases to subgroup staff (in this case `subgroups` is expected
    to have only one element).

    The resulting dictionary is structured as follows, with many data organized by numeric
    indexes rather than hashtables in order to ease tabular rendering:

        { "delivery": delivery,
          "products": product_idx -> product,
          "table": subgroup_idx -> { "subgroup": subgroup,
                                     "totals": product_idx -> { "product": product,
                                                                "quantity": number,
                                                                "full_packages": number,
                                                                "out_of_packages": number,
                                                                "weight": number,
                                                                 "price": number},
                                     "users": user_idx -> { "user": user,
                                                            "orders": product_idx -> order,
                                                            "price": number,
                                                            "weight": number },
                                     "price": number,
                                     "weight": number},
          "product_totals": product_idx -> { "product": product,
                                             "quantity": number,
                                             "full_packages": number,
                                             "out_of_packages": number },
          "price": number }
    """
    # List of products, ordered by name
    products = delivery.product_set.all()
    # Iterable of all users in subgroups
    users = m.User.objects.filter(user_of_subgroup__in=subgroups)
    # Dictionary user -> list of ordered, indexed as products
    orders = m.Order.by_user_and_product(delivery, users, products)

    # Compute totals per product per subgroup 1: initialize
    # Dictionary subgroup -> product -> { product, granted_quantity }
    sg_pd_totals = {
        sg: {
            pd: {
                'product': pd,
                'quantity': 0,
            } for pd in products
        } for sg in subgroups
    }

    # Generate user->subgroup dict, helper to compute totals per subgroup
    user_to_subgroup = {}
    for sg in subgroups:
        for u in sg.users.iterator():
            user_to_subgroup[u] = sg

    # Sum quantities per subgroup and per product;
    for od in orders.itervalues():
        sg = user_to_subgroup[od.user]
        for pc in od.purchases:
            if pc:
                sg_pd_totals[sg][pc.product]['quantity'] += pc.granted

    # Break up quantities in packages + loose items, compute price
    for pd_totals in sg_pd_totals.itervalues():
        for pd, totals in pd_totals.iteritems():
            qty = totals['quantity']
            qpp = pd.quantity_per_package
            totals['price'] = qty * totals['product'].price
            totals['weight'] = qty * totals['product'].unit_weight
            if qpp:
                totals['full_packages'] = qty // qpp
                totals['out_of_packages'] = qty % qpp

    # Sum up grand total (all subgroups together) per product
    product_totals = []
    for i, pd in enumerate(products):
        qty = sum(sg_pd_totals[sg][pd]['quantity'] for sg in subgroups)
        qpp = pd.quantity_per_package
        total = {'product': pd, 'quantity': qty}
        if qpp:
            total['full_packages'] = qty // qpp
            total['out_of_packages'] = qty % qpp
        product_totals.append(total)

    # Convert dictionaries into ordered lists in `table`:
    # subgroup_idx -> { "subgroup": subgroup,
    #                   "totals": product_idx -> { "product": product,
    #                                              "quantity": number,
    #                                              "full_packages": number,
    #                                              "out_of_packages": number,
    #                                              "price": number},
    #                   "users": user_idx -> { "user": user,
    #                                          "orders": product_idx -> order,
    #                                          "price": number,
    #                                          "weight": number },
    #                   "price": number,
    #                   "weight": number }
    table = []
    for i, sg in enumerate(subgroups):
        pd_totals = sg_pd_totals[sg]
        pd_totals_list = []
        user_records = []
        # Get and order users alphabetically, except for the extra user which gets last
        sg_item = {'subgroup': sg, 'totals': pd_totals_list, 'users': user_records}
        table.append(sg_item)
        for j, pd in enumerate(products):
            pd_totals_list.append(pd_totals[pd])
        for k, u in enumerate(sg.sorted_users):
            order = orders[u]
            user_records.append({
                'user': u,
                'orders': order,
                'price': sum(pc.price for pc in order.purchases if pc),
                'weight': sum(pc.weight for pc in order.purchases if pc)})
        sg_item['price'] = sum(uo['price'] for uo in user_records)
        sg_item['weight'] = sum(uo['weight'] for uo in user_records)

    price = sum(x['price'] for x in table)

    result = {
        'delivery': delivery,
        'products': products,
        'product_totals': product_totals,
        'table': table,
        'price': price
    }
    result.update(kwargs)
    return result
<commit_msg>check for useractiveness everywhere.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from floreal import models as m

def delivery_description(delivery, subgroups, **kwargs):
    """Generate a description of the purchases performed by users in `subgroups`
    for `delivery`. The resulting dictionary is used to render HTML as well as
    PDF and MS-Excel views. It's used both to display the complete order to network staff,
    and their subgroup's purchases to subgroup staff (in this case `subgroups` is expected
    to have only one element).

    The resulting dictionary is structured as follows, with many data organized by numeric
    indexes rather than hashtables in order to ease tabular rendering:

        { "delivery": delivery,
          "products": product_idx -> product,
          "table": subgroup_idx -> { "subgroup": subgroup,
                                     "totals": product_idx -> { "product": product,
                                                                "quantity": number,
                                                                "full_packages": number,
                                                                "out_of_packages": number,
                                                                "weight": number,
                                                                 "price": number},
                                     "users": user_idx -> { "user": user,
                                                            "orders": product_idx -> order,
                                                            "price": number,
                                                            "weight": number },
                                     "price": number,
                                     "weight": number},
          "product_totals": product_idx -> { "product": product,
                                             "quantity": number,
                                             "full_packages": number,
                                             "out_of_packages": number },
          "price": number }
    """
    # List of products, ordered by name
    products = delivery.product_set.all()
    # Iterable of all users in subgroups
    users = m.User.objects.filter(user_of_subgroup__in=subgroups, is_active=True)
    # Dictionary user -> list of ordered, indexed as products
    orders = m.Order.by_user_and_product(delivery, users, products)

    # Compute totals per product per subgroup 1: initialize
    # Dictionary subgroup -> product -> { product, granted_quantity }
    sg_pd_totals = {
        sg: {
            pd: {
                'product': pd,
                'quantity': 0,
            } for pd in products
        } for sg in subgroups
    }

    # Generate user->subgroup dict, helper to compute totals per subgroup
    user_to_subgroup = {}
    for sg in subgroups:
        for u in sg.users.iterator():
            user_to_subgroup[u] = sg

    # Sum quantities per subgroup and per product;
    for od in orders.itervalues():
        sg = user_to_subgroup[od.user]
        for pc in od.purchases:
            if pc:
                sg_pd_totals[sg][pc.product]['quantity'] += pc.granted

    # Break up quantities in packages + loose items, compute price
    for pd_totals in sg_pd_totals.itervalues():
        for pd, totals in pd_totals.iteritems():
            qty = totals['quantity']
            qpp = pd.quantity_per_package
            totals['price'] = qty * totals['product'].price
            totals['weight'] = qty * totals['product'].unit_weight
            if qpp:
                totals['full_packages'] = qty // qpp
                totals['out_of_packages'] = qty % qpp

    # Sum up grand total (all subgroups together) per product
    product_totals = []
    for i, pd in enumerate(products):
        qty = sum(sg_pd_totals[sg][pd]['quantity'] for sg in subgroups)
        qpp = pd.quantity_per_package
        total = {'product': pd, 'quantity': qty}
        if qpp:
            total['full_packages'] = qty // qpp
            total['out_of_packages'] = qty % qpp
        product_totals.append(total)

    # Convert dictionaries into ordered lists in `table`:
    # subgroup_idx -> { "subgroup": subgroup,
    #                   "totals": product_idx -> { "product": product,
    #                                              "quantity": number,
    #                                              "full_packages": number,
    #                                              "out_of_packages": number,
    #                                              "price": number},
    #                   "users": user_idx -> { "user": user,
    #                                          "orders": product_idx -> order,
    #                                          "price": number,
    #                                          "weight": number },
    #                   "price": number,
    #                   "weight": number }
    table = []
    for i, sg in enumerate(subgroups):
        pd_totals = sg_pd_totals[sg]
        pd_totals_list = []
        user_records = []
        # Get and order users alphabetically, except for the extra user which gets last
        sg_item = {'subgroup': sg, 'totals': pd_totals_list, 'users': user_records}
        table.append(sg_item)
        for j, pd in enumerate(products):
            pd_totals_list.append(pd_totals[pd])
        for k, u in enumerate(sg.sorted_users):
            order = orders[u]
            user_records.append({
                'user': u,
                'orders': order,
                'price': sum(pc.price for pc in order.purchases if pc),
                'weight': sum(pc.weight for pc in order.purchases if pc)})
        sg_item['price'] = sum(uo['price'] for uo in user_records)
        sg_item['weight'] = sum(uo['weight'] for uo in user_records)

    price = sum(x['price'] for x in table)

    result = {
        'delivery': delivery,
        'products': products,
        'product_totals': product_totals,
        'table': table,
        'price': price
    }
    result.update(kwargs)
    return result
<|endoftext|>


<commit_before>import unittest

import fact
import lists
import matrix

class TestPython(unittest.TestCase):
  "Testing basic concepts in Python"

  def setUp(self):
  	pass

  def tearDown(self):
  	pass

  def test_basic_addition(self):
  	self.assertEqual(2 + 2, 4)

  def test_factorial(self):
  	self.assertEqual(fact.factorial(5), 120)

  def test_matrix(self):
  	m = matrix.get_matrix()
  	self.assertEqual(m[1][1], 5)

  def test_list(self):
    ls = lists.get_default_list()
    self.assertEqual(ls[-1], 13)

  def test_list_replacement(self):
    ls = lists.get_default_list()
    ls[1] -= ls[0]
    self.assertEqual(ls[1], 1)

  def test_step(self):
    ls = lists.get_default_list()
    sl = ls[::2]
    self.assertEqual(sl[-1], ls[-2])

  def test_diffeomorphism(self):
    ls = lists.get_default_list()
    self.assertEqual(ls, ls[::-1][::-1])

  def test_append(self):
    ls = lists.get_default_list()
    old_len = len(ls)
    ls.append(17)
    self.assertEqual(old_len + 1, len(ls))

  def test_combine(self):
    ls1 = lists.get_default_list()
    ls2 = lists.get_next_list()
    ls1 += ls2
    self.assertEqual(10, len(ls1))

  def test_append_instead_of_combine(self):
    ls1 = lists.get_default_list()
    ls2 = lists.get_next_list()
    ls1.append(ls2)
    self.assertEqual(7, len(ls1))
    self.assertEqual(ls1[6][0], 17)

  def test_delete(self):
    ls1 = lists.get_default_list()
    del ls1[0]
    self.assertEqual(ls1[0], 3)

if __name__ == '__main__':
    unittest.main()
<commit_msg>test push-pop<commit_after>import unittest

import fact
import lists
import matrix

class TestPython(unittest.TestCase):
  "Testing basic concepts in Python"

  def setUp(self):
  	pass

  def tearDown(self):
  	pass

  def test_basic_addition(self):
  	self.assertEqual(2 + 2, 4)

  def test_factorial(self):
  	self.assertEqual(fact.factorial(5), 120)

  def test_matrix(self):
  	m = matrix.get_matrix()
  	self.assertEqual(m[1][1], 5)

  def test_list(self):
    ls = lists.get_default_list()
    self.assertEqual(ls[-1], 13)

  def test_list_replacement(self):
    ls = lists.get_default_list()
    ls[1] -= ls[0]
    self.assertEqual(ls[1], 1)

  def test_step(self):
    ls = lists.get_default_list()
    sl = ls[::2]
    self.assertEqual(sl[-1], ls[-2])

  def test_diffeomorphism(self):
    ls = lists.get_default_list()
    self.assertEqual(ls, ls[::-1][::-1])

  def test_append(self):
    ls = lists.get_default_list()
    old_len = len(ls)
    ls.append(17)
    self.assertEqual(old_len + 1, len(ls))

  def test_combine(self):
    ls1 = lists.get_default_list()
    ls2 = lists.get_next_list()
    ls1 += ls2
    self.assertEqual(10, len(ls1))

  def test_append_instead_of_combine(self):
    ls1 = lists.get_default_list()
    ls2 = lists.get_next_list()
    ls1.append(ls2)
    self.assertEqual(7, len(ls1))
    self.assertEqual(ls1[6][0], 17)

  def test_delete(self):
    ls1 = lists.get_default_list()
    del ls1[0]
    self.assertEqual(ls1[0], 3)

  def test_append_and_pop(self):
    ls = lists.get_default_list()
    ls.append(17)
    lifo = ls.pop()
    self.assertEqual(lifo, 17)
    self.assertEqual(ls, lists.get_default_list())

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import re
from pathlib import Path
from typing import Mapping
import logging

from msc_pyparser import MSCParser
from lib.context import Context
from lib.operators.assembler import Assembler

class Parser(object):
    rule_id_regex = re.compile(r"^(\d{6})(?:-chain(\d+))?")
    parsers: Mapping[str, MSCParser] = {}
    prefix_to_file_map: Mapping[str, str] = {}
    logger = logging.getLogger()

    def __init__(self, context: Context):
        self.context = context

    def perform_compare_or_update(self, process_all: bool, func=None):
        files = [file for file in self.context.data_files_directory.iterdir()]
        files.sort()
        for file in files:
            if process_all:
                match = self.rule_id_regex.match(file.name)
                rule_id = match.group(1)
                chain_offset = int(match.group(2)) if match.group(2) else 0
            else:
                rule_id = self.context.single_rule_id
                chain_offset = self.context.single_chain_offset

            if rule_id in file and ('chain' not in file.name or f'chain{chain_offset}' in file.name):
                self.process_regex(
                    rule_id,
                    chain_offset,
                    file,
                    Assembler(self.context),
                    func,
                )
                if not process_all:
                    break

    def process_regex(self, rule_id: str, chain_offset: int, file_path: Path, assembler: Assembler, func):
        self.logger.info("Processing %s, chain offset %s", rule_id, chain_offset)

        with file_path.open("rt") as file:
            regex = assembler.run(file)

        rule_prefix = rule_id[:3]
        if rule_prefix in self.parsers:
            parser = self.parsers[rule_prefix]
        else:
            for rule_file in self.context.rules_directory.iterdir():
                if rule_prefix in rule_file:
                    self.prefix_to_file_map[rule_prefix] = rule_file
                    with (self.context.rules_directory / rule_file).open("rt") as handle:
                        parser = MSCParser()
                        self.parsers[rule_prefix] = parser
                        parser.parser.parse(handle.read())
                        break
            if not parser:
                raise Warning(f"No rule file found for data file {file}")

        self.process_configlines(parser.configlines, file_path, rule_id, chain_offset, regex, func)

    def process_configlines(self, configlines, file_path, rule_id, chain_offset, regex, func):
        current_chain_offset = 0
        check_chain_rules = False

        for config in configlines:
            if config["type"] == "SecRule":
                if check_chain_rules and "id" not in config["actions"]:
                    current_chain_offset += 1
                    if current_chain_offset == chain_offset:
                        if config["operator"] != "@rx":
                            raise Warning(f"Rule {rule_id} does not use the @rx operator, but {config['operator']}")

                        func(
                            rule_id,
                            regex,
                            config["operator_argument"],
                            config,
                            "operator_argument",
                        )
                        return
                else:
                    if check_chain_rules:
                        raise Warning(f"Chain rule with ID {rule_id} and offset {chain_offset} not found in file {file_path}")

                    for action in config["actions"]:
                        if action["act_name"] == "id" and action["act_arg"] == rule_id:
                            if chain_offset == 0:
                                if config["operator"] != "@rx":
                                    raise Warning(f"Rule {rule_id} does not use the @rx operator, but {config['operator']}")

                                func(
                                    rule_id,
                                    regex,
                                    config["operator_argument"],
                                    config,
                                    "operator_argument",
                                )
                                return
                            else:
                                check_chain_rules = True
                                break
<commit_msg>fix(regexp-assemble): Use correct object in conditional<commit_after>import re
from pathlib import Path
from typing import Mapping
import logging

from msc_pyparser import MSCParser
from lib.context import Context
from lib.operators.assembler import Assembler

class Parser(object):
    rule_id_regex = re.compile(r"^(\d{6})(?:-chain(\d+))?")
    parsers: Mapping[str, MSCParser] = {}
    prefix_to_file_map: Mapping[str, str] = {}
    logger = logging.getLogger()

    def __init__(self, context: Context):
        self.context = context

    def perform_compare_or_update(self, process_all: bool, func=None):
        files = [file for file in self.context.data_files_directory.iterdir()]
        files.sort()
        for file in files:
            if process_all:
                match = self.rule_id_regex.match(file.name)
                rule_id = match.group(1)
                chain_offset = int(match.group(2)) if match.group(2) else 0
            else:
                rule_id = self.context.single_rule_id
                chain_offset = self.context.single_chain_offset

            if rule_id in file.name and ('chain' not in file.name or f'chain{chain_offset}' in file.name):
                self.process_regex(
                    rule_id,
                    chain_offset,
                    file,
                    Assembler(self.context),
                    func,
                )
                if not process_all:
                    break

    def process_regex(self, rule_id: str, chain_offset: int, file_path: Path, assembler: Assembler, func):
        self.logger.info("Processing %s, chain offset %s", rule_id, chain_offset)

        with file_path.open("rt") as file:
            regex = assembler.run(file)

        rule_prefix = rule_id[:3]
        if rule_prefix in self.parsers:
            parser = self.parsers[rule_prefix]
        else:
            for rule_file in self.context.rules_directory.iterdir():
                if rule_prefix in rule_file:
                    self.prefix_to_file_map[rule_prefix] = rule_file
                    with (self.context.rules_directory / rule_file).open("rt") as handle:
                        parser = MSCParser()
                        self.parsers[rule_prefix] = parser
                        parser.parser.parse(handle.read())
                        break
            if not parser:
                raise Warning(f"No rule file found for data file {file}")

        self.process_configlines(parser.configlines, file_path, rule_id, chain_offset, regex, func)

    def process_configlines(self, configlines, file_path, rule_id, chain_offset, regex, func):
        current_chain_offset = 0
        check_chain_rules = False

        for config in configlines:
            if config["type"] == "SecRule":
                if check_chain_rules and "id" not in config["actions"]:
                    current_chain_offset += 1
                    if current_chain_offset == chain_offset:
                        if config["operator"] != "@rx":
                            raise Warning(f"Rule {rule_id} does not use the @rx operator, but {config['operator']}")

                        func(
                            rule_id,
                            regex,
                            config["operator_argument"],
                            config,
                            "operator_argument",
                        )
                        return
                else:
                    if check_chain_rules:
                        raise Warning(f"Chain rule with ID {rule_id} and offset {chain_offset} not found in file {file_path}")

                    for action in config["actions"]:
                        if action["act_name"] == "id" and action["act_arg"] == rule_id:
                            if chain_offset == 0:
                                if config["operator"] != "@rx":
                                    raise Warning(f"Rule {rule_id} does not use the @rx operator, but {config['operator']}")

                                func(
                                    rule_id,
                                    regex,
                                    config["operator_argument"],
                                    config,
                                    "operator_argument",
                                )
                                return
                            else:
                                check_chain_rules = True
                                break
<|endoftext|>


<commit_before>#!/usr/bin/env python
# pylint: disable=invalid-name

"""
CherryPy-based webservice daemon with background threads
"""
from cStringIO import StringIO
import codecs
import json
import os, os.path
import random
import string
import threading
import subprocess
import cherrypy
from cherrypy.process import plugins
import docx
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from cStringIO import StringIO
from pptx import Presentation
import pyteaser
import textrazor
import os
import re
from unidecode import unidecode

# import pyteaser
def worker():
    """Background Timer that runs the hello() function every 5 seconds
    TODO: this needs to be /optimized. I don't like creating the thread
    repeatedly.fixed
    """

    while True:
        t = threading.Timer(10.0, hello)
        t.start()
        t.join()

def hello():
    """Output 'hello' on the console"""

    print ('Server running...')
    # Summarize("hi my name is nipoon")
    # x = pyteaser.Summarize("Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document.")
    # print(x)

class MyBackgroundThread(plugins.SimplePlugin):
    """CherryPy plugin to create a background worker thread"""

    def __init__(self, bus):
        plugins.SimplePlugin.__init__(self, bus)

        self.t = None

    def start(self):
        """Plugin entrypoint"""

        self.t = threading.Thread(target=worker)
        self.t.daemon = True
        self.t.start()

    # Start at a higher priority that "Daemonize" (which we're not using
    # yet but may in the future)
    start.priority = 85

class APIController(object): \
        # pylint: disable=too-few-public-methods

    """Controller for fictional "nodes" webservice APIs"""

    # #     @cherrypy.tools.json_out()
    def upload(self):
        # Regular request for '/nodes' URI
        return file('./Public/html/index.html')

    @cherrypy.expose
    def test(self):
        return file("./Public/html/summaries.html")

    @cherrypy.expose
    def extractPage(self):
        return file("./Public/html/ExtractText.html")

    @cherrypy.expose
    def newcv(self):
        return file("./Public/html/select-file.html")

    @cherrypy.expose
    def keywordsearch(self):
        return file("./Public/html/KeyWordSearch.html")

    def result(self, myFile):
        out = """

Summary

%s

        """
        print(myFile.filename)
        upload_file = './temp_files/' + myFile.filename

        if os.path.splitext(myFile.filename)[1] == '.pdf':
            size = 0
            with open(upload_file, 'wb') as out:
                while True:
                    data = myFile.file.read(8192)
                    if not data:
                        break
                    out.write(data)

            data = convertPdf('./temp_files/' + myFile.filename)
        elif os.path.splitext(myFile.filename)[1] == '.pptx':
            target = './temp_files'

            size = 0
            if not os.path.isdir(target):
                os.mkdir(target)

            with open(target + myFile.filename, 'wb') as out:
                while True:
                    data = myFile.file.read(8192)
                    if not data:
                        break
                    out.write(data)

            data = convertPptx(target + myFile.filename)
        elif os.path.splitext(myFile.filename)[1] == '.docx':
            target = './temp_files'
            size = 0
            if not os.path.isdir(target):
                os.mkdir(target)

            with open(target + myFile.filename, 'wb') as out:
                while True:
                    data = myFile.file.read(8192)
                    if not data:
                        break
                    out.write(data)

            data = convertDocx(target + myFile.filename)
        elif os.path.splitext(myFile.filename)[1] == '.doc':
            target = './temp_files'
            size = 0
            if not os.path.isdir(target):
                os.mkdir(target)

            with open(myFile.filename, 'wb') as out:
                while True:
                    data = myFile.file.read(8192)
                    if not data:
                        break
                    out.write(data)

            data = convertDocxx(myFile.filename)
        elif os.path.splitext(myFile.filename)[1] == '.ppt':
            target = './temp_files'
            size = 0
            if not os.path.isdir(target):
                os.mkdir(target)

            with open(myFile.filename, 'wb') as out:
                while True:
                    data = myFile.file.read(8192)
                    if not data:
                        break
                    out.write(data)

            data = convertPptxx(myFile.filename)
        else:
            data = "Invalid file type!"
        return data

    def fetchFilteredSummaries(self, data, keywords):
        return pyteaser.Summarize(data, keywords)

    def extractKeywords(self, data):

        return pyteaser.extract_keywords(data)

def convertDocx(path):
    document = docx.Document(path)
    docText = ''.join([
        (unidecode(unicode(paragraph.text))) for paragraph in document.paragraphs
    ])
    #print(docText)
    return docText

def convertDocxx(path):

    for filename in  os.listdir(os.getcwd()):
        if filename.endswith('.doc'):
            subprocess.call(['soffice', '--headless', '--convert-to', 'docx', filename])
    document = docx.Document(path[:-4] + ".docx")
    docText = ''.join([
        (unidecode(unicode(paragraph.text))) for paragraph in document.paragraphs
    ])
    # print(docText)
    return docText
def convertPptx(path):
    prs = Presentation(path)
    # text_runs will be populated with a list of strings,
    # one for each text run in presentation
    text_runs = []
    full_string = ""

    for slide in prs.slides:
        for shape in slide.shapes:
            if not shape.has_text_frame:
                continue
            for paragraph in shape.text_frame.paragraphs:
                for run in paragraph.runs:
                    text_runs.append(unidecode(unicode(run.text)))

    full_string = ''.join(text_runs)
    print(full_string)
    return full_string
def convertPptxx(path):
    for filename in  os.listdir(os.getcwd()):
        if filename.endswith('.ppt'):
            subprocess.call(['soffice', '--headless', '--convert-to', 'pptx', filename])
    prs = Presentation(path[:-4] + ".pptx")

    # text_runs will be populated with a list of strings,
    # one for each text run in presentation
    text_runs = []
    full_string = ""

    for slide in prs.slides:
        for shape in slide.shapes:
            if not shape.has_text_frame:
                continue
            for paragraph in shape.text_frame.paragraphs:
                for run in paragraph.runs:
                    text_runs.append(unidecode(unicode(run.text)))

    full_string = ''.join(text_runs)
    # print(full_string)
    return full_string

def convertPdf(path):
    rsrcmgr = PDFResourceManager()
    retstr = StringIO()
    codec = 'utf-8'
    laparams = LAParams()
    device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)

    fp = file(path, 'rb')
    for page in PDFPage.get_pages(fp):
        interpreter.process_page(page)

    text = retstr.getvalue()
    # Cleanup
    fp.close()
    device.close()
    retstr.close()
    text = unidecode(unicode(text, encoding = "utf-8"))
    print(text)
    return text

def jsonify_error(status, message, traceback, version): \
        # pylint: disable=unused-argument

    """JSONify all CherryPy error responses (created by raising the
    cherrypy.HTTPError exception)
    """

    cherrypy.response.headers['Content-Type'] = 'application/json'
    response_body = json.dumps(
        {
            'error': {
                'http_status': status,
                'message': message,
            }
        })

    cherrypy.response.status = status

    return response_body

if __name__ == '__main__':
    MyBackgroundThread(cherrypy.engine).subscribe()

    dispatcher = cherrypy.dispatch.RoutesDispatcher()

    # /nodes (GET)
    dispatcher.connect(name='upload',
                       route='/',
                       action='upload',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='result',
                       route='/result',
                       action='result',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    # /nodes (GET)
    dispatcher.connect(name='fetchFilteredSummaries',
                       route='/fetchFilteredSummaries',
                       action='fetchFilteredSummaries',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    dispatcher.connect(name='extractKeywords',
                       route='/extractKeywords',
                       action='extractKeywords',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    # /nodes (GET)
    dispatcher.connect(name='extractPage',
                       route='/extractPage',
                       action='extractPage',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='newcv',
                       route='/newcv',
                       action='newcv',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='keywordsearch',
                       route='/keywordsearch',
                       action='keywordsearch',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='test',
                       route='/test',
                       action='test',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    config = {
        '/': {
            'request.dispatch': dispatcher,
            'error_page.default': jsonify_error,
            'tools.sessions.on': True,
            'tools.staticdir.root': os.path.abspath(os.getcwd())
        },

        '/generator': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },

        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'Public/'
        }
    }

    cherrypy.tree.mount(root=None, config=config)
    cherrypy.server.socket_host = '0.0.0.0'
    cherrypy.server.socket_port = 80

    cherrypy.engine.start()
    cherrypy.engine.block()
<commit_msg>File gets deleted once text is extracted from uploaded file<commit_after>#!/usr/bin/env python
# pylint: disable=invalid-name

"""
CherryPy-based webservice daemon with background threads
"""
from cStringIO import StringIO
import codecs
import json
import os, os.path
import random
import string
import threading
import subprocess
import cherrypy
from cherrypy.process import plugins
import docx
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from cStringIO import StringIO
from pptx import Presentation
import pyteaser
import textrazor
import os
import re
from unidecode import unidecode

# import pyteaser
def worker():
    """Background Timer that runs the hello() function every 5 seconds
    TODO: this needs to be /optimized. I don't like creating the thread
    repeatedly.fixed
    """

    while True:
        t = threading.Timer(10.0, hello)
        t.start()
        t.join()

def hello():
    """Output 'hello' on the console"""

    print ('Server running...')
    # Summarize("hi my name is nipoon")
    # x = pyteaser.Summarize("Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document.")
    # print(x)

class MyBackgroundThread(plugins.SimplePlugin):
    """CherryPy plugin to create a background worker thread"""

    def __init__(self, bus):
        plugins.SimplePlugin.__init__(self, bus)

        self.t = None

    def start(self):
        """Plugin entrypoint"""

        self.t = threading.Thread(target=worker)
        self.t.daemon = True
        self.t.start()

    # Start at a higher priority that "Daemonize" (which we're not using
    # yet but may in the future)
    start.priority = 85

class APIController(object): \
        # pylint: disable=too-few-public-methods

    """Controller for fictional "nodes" webservice APIs"""

    # #     @cherrypy.tools.json_out()
    def upload(self):
        # Regular request for '/nodes' URI
        return file('./Public/html/index.html')

    @cherrypy.expose
    def test(self):
        return file("./Public/html/summaries.html")

    @cherrypy.expose
    def extractPage(self):
        return file("./Public/html/ExtractText.html")

    @cherrypy.expose
    def newcv(self):
        return file("./Public/html/select-file.html")

    @cherrypy.expose
    def keywordsearch(self):
        return file("./Public/html/KeyWordSearch.html")

    def result(self, myFile):
        out = """

Summary

%s

        """
        print(myFile.filename)
        upload_file = './temp_files/' + myFile.filename

        if not os.path.isdir('./temp_files'):
            os.mkdir('./temp_files')

        with open(upload_file, 'wb') as out:
            while True:
                data = myFile.file.read(8192)
                if not data:
                    break
                out.write(data)

        if os.path.splitext(myFile.filename)[1] == '.pdf':
            data = convertPdf(upload_file)
        elif os.path.splitext(myFile.filename)[1] == '.pptx':
            data = convertPptx(upload_file)
        elif os.path.splitext(myFile.filename)[1] == '.docx':
            data = convertDocx(upload_file)
        elif os.path.splitext(myFile.filename)[1] == '.doc':
            data = convertDocxx(upload_file)
        elif os.path.splitext(myFile.filename)[1] == '.ppt':
            data = convertPptxx(upload_file)
        else:
            data = "Invalid file type!"

        # Remove this line if you do not want to remove files from temp_files
        os.remove('./temp_files/' + myFile.filename)

        return data

    def fetchFilteredSummaries(self, data, keywords):
        return pyteaser.Summarize(data, keywords)

    def extractKeywords(self, data):

        return pyteaser.extract_keywords(data)

def convertDocx(path):
    document = docx.Document(path)
    docText = ''.join([
        (unidecode(unicode(paragraph.text))) for paragraph in document.paragraphs
    ])
    #print(docText)
    return docText

def convertDocxx(path):

    for filename in  os.listdir(os.getcwd()):
        if filename.endswith('.doc'):
            subprocess.call(['soffice', '--headless', '--convert-to', 'docx', filename])
    document = docx.Document(path[:-4] + ".docx")
    docText = ''.join([
        (unidecode(unicode(paragraph.text))) for paragraph in document.paragraphs
    ])
    # print(docText)
    return docText
def convertPptx(path):
    prs = Presentation(path)
    # text_runs will be populated with a list of strings,
    # one for each text run in presentation
    text_runs = []
    full_string = ""

    for slide in prs.slides:
        for shape in slide.shapes:
            if not shape.has_text_frame:
                continue
            for paragraph in shape.text_frame.paragraphs:
                for run in paragraph.runs:
                    text_runs.append(unidecode(unicode(run.text)))

    full_string = ''.join(text_runs)
    print(full_string)
    return full_string
def convertPptxx(path):
    for filename in  os.listdir(os.getcwd()):
        if filename.endswith('.ppt'):
            subprocess.call(['soffice', '--headless', '--convert-to', 'pptx', filename])
    prs = Presentation(path[:-4] + ".pptx")

    # text_runs will be populated with a list of strings,
    # one for each text run in presentation
    text_runs = []
    full_string = ""

    for slide in prs.slides:
        for shape in slide.shapes:
            if not shape.has_text_frame:
                continue
            for paragraph in shape.text_frame.paragraphs:
                for run in paragraph.runs:
                    text_runs.append(unidecode(unicode(run.text)))

    full_string = ''.join(text_runs)
    # print(full_string)
    return full_string

def convertPdf(path):
    rsrcmgr = PDFResourceManager()
    retstr = StringIO()
    codec = 'utf-8'
    laparams = LAParams()
    device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)

    fp = file(path, 'rb')
    for page in PDFPage.get_pages(fp):
        interpreter.process_page(page)

    text = retstr.getvalue()
    # Cleanup
    fp.close()
    device.close()
    retstr.close()
    text = unidecode(unicode(text, encoding = "utf-8"))
    # print(text)
    return text

def jsonify_error(status, message, traceback, version): \
        # pylint: disable=unused-argument

    """JSONify all CherryPy error responses (created by raising the
    cherrypy.HTTPError exception)
    """

    cherrypy.response.headers['Content-Type'] = 'application/json'
    response_body = json.dumps(
        {
            'error': {
                'http_status': status,
                'message': message,
            }
        })

    cherrypy.response.status = status

    return response_body

if __name__ == '__main__':
    MyBackgroundThread(cherrypy.engine).subscribe()

    dispatcher = cherrypy.dispatch.RoutesDispatcher()

    # /nodes (GET)
    dispatcher.connect(name='upload',
                       route='/',
                       action='upload',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='result',
                       route='/result',
                       action='result',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    # /nodes (GET)
    dispatcher.connect(name='fetchFilteredSummaries',
                       route='/fetchFilteredSummaries',
                       action='fetchFilteredSummaries',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    dispatcher.connect(name='extractKeywords',
                       route='/extractKeywords',
                       action='extractKeywords',
                       controller=APIController(),
                       conditions={'method': ['POST']})

    # /nodes (GET)
    dispatcher.connect(name='extractPage',
                       route='/extractPage',
                       action='extractPage',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='newcv',
                       route='/newcv',
                       action='newcv',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='keywordsearch',
                       route='/keywordsearch',
                       action='keywordsearch',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    # /nodes (GET)
    dispatcher.connect(name='test',
                       route='/test',
                       action='test',
                       controller=APIController(),
                       conditions={'method': ['GET']})

    config = {
        '/': {
            'request.dispatch': dispatcher,
            'error_page.default': jsonify_error,
            'tools.sessions.on': True,
            'tools.staticdir.root': os.path.abspath(os.getcwd())
        },

        '/generator': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },

        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'Public/'
        }
    }

    cherrypy.tree.mount(root=None, config=config)
    cherrypy.server.socket_host = '0.0.0.0'
    cherrypy.server.socket_port = 80

    cherrypy.engine.start()
    cherrypy.engine.block()
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import numpy
import os
import pandas
import random

from collections import Counter
from utils.plotting import PlotNLP

try:
  import simplejson as json
except ImportError:
  import json

class ClassificationModel(object):
  """
  Base class for NLP models of classification tasks. When inheriting from this
  class please take note of which methods MUST be overridden, as documented
  below.

  The Model superclass implements:
    - classifyRandomly() calculates accuracy of a random classifier
    - encodeRandomly() creates a random SDR encoding
    - evaluateTrialResults() calculates result stats
    - evaluateResults() calculates result stats for a list of trial results
    - printTrialReport() prints classifications of an evaluation trial
    - printFinalReport() prints evaluation metrics and confusion matrix
    - densifyPattern() returns a binary SDR vector for a given bitmap

  Methods/properties that must be implemented by subclasses:
    - encodePattern(); note the specified format in the docstring below.
    - trainModel()
    - testModel()
  """

  def __init__(self, n=16384, w=328, verbosity=1, plot=True):
    """The SDR dimensions are standard for Cortical.io fingerprints."""
    self.n = n
    self.w = w
    self.verbosity = verbosity
    self.plot = plot

  def classifyRandomly(self, labels):
    """Return accuracy of random classifications for the labels."""
    randomLabels = numpy.random.randint(0, labels.max(), labels.shape)
    return (randomLabels == labels).sum() / float(labels.shape[0])

  def encodeRandomly(self, sample):
    """Return a random bitmap representation of the sample."""
    random.seed(sample)
    return numpy.sort(random.sample(xrange(self.n), self.w))

  def logEncodings(self, patterns, path):
    """Log the encoding dictionaries to a txt file."""
    if not os.path.isdir(path):
      raise ValueError("Invalid path to write file.")

    # Cast numpy arrays to list objects for serialization.
    jsonPatterns = copy.deepcopy(patterns)
    for jp in jsonPatterns:
      jp["bitmap"] = jp.get("bitmap", None).tolist()

    with open(os.path.join(path, "encoding_log.txt"), "w") as f:
      f.write(json.dumps(jsonPatterns, indent=1))

  def evaluateTrialResults(self, classifications, references, idx): ## TODO: evaluation metrics for multiple classifcations
    """
    Calculate statistics for the predicted classifications against the actual.

    @param classifications  (list)            Two lists: (0) predictions and (1)
                                              actual classifications. Items in
                                              the predictions list are lists of
                                              ints or None, and items in actual
                                              classifications list are ints.
    @param references       (list)            Classification label strings.
    @return                 (tuple)           Returns a 2-item tuple w/ the
                                              accuracy (float) and confusion
                                              matrix (numpy array).
    """
    if len(classifications[0]) != len(classifications[1]):
      raise ValueError("Classification lists must have same length.")
    if self.verbosity > 0:
      self._printTrialReport(classifications, references, idx)

    actual = numpy.array(classifications[1])
    predicted = numpy.array([c[0] for c in classifications[0]])  ## TODO: see above; this forces evaluation metrics to consider only the first predicted classification
    accuracy = (actual == predicted).sum() / float(len(actual))

    # Calculate confusion matrix.
    total = len(references)
    cm = numpy.zeros((total, total+1))
    for i, p in enumerate(predicted):
      if p is not None:
        cm[actual[i]][p] += 1
      else:
        # No predicted label, so increment the "(none)" column.
        cm[actual[i]][total] += 1
    cm = numpy.vstack((cm, numpy.sum(cm, axis=0)))
    cm = numpy.hstack((cm, numpy.sum(cm, axis=1).reshape(total+1,1)))

    cm = pandas.DataFrame(
      data=cm,
      columns=references+["(none)"]+["Actual Totals"],
      index=references+["Prediction Totals"])

    return (accuracy, cm)

  def evaluateFinalResults(self, intermResults):
    """
    Cumulative statistics for the outputs of evaluateTrialResults().

    @param intermResults      (list)          List of returned results from
                                              evaluateTrialResults().
    @return                   (list)          Returns a dictionary with entries
                                              for max, mean, and min accuracies,
                                              and the mean confusion matrix.
    """
    accuracy = []
    cm = numpy.zeros((intermResults[0][1].shape))

    # Find mean, max, and min values for the metrics.
    for result in intermResults:
      accuracy.append(result[0])
      cm = numpy.add(cm, result[1])

    results = {"max_accuracy":max(accuracy),
               "mean_accuracy":sum(accuracy)/float(len(accuracy)),
               "min_accuracy":min(accuracy),
               "total_cm":cm}

    if self.verbosity > 0:
      self._printFinalReport(results)

    if self.plot:
      self.plotConfusionMatrix(cm)

    return results

  @staticmethod
  def _printTrialReport(labels, refs, idx):
    """Print columns for sample #, actual label, and predicted label."""
    template = "{0:10}|{1:30}|{2:30}"
    print "Evaluation results for this fold:"
    print template.format("#", "Actual", "Predicted")
    for i in xrange(len(labels[0])):
      if labels[0][i][0] == None:
        print template.format(idx[i], refs[labels[1][i]], "(none)")
      else:
        print template.format(
          idx[i], refs[labels[1][i]], [refs[label] for label in labels[0][i]])

  @staticmethod
  def _printFinalReport(results):  ## TODO: pprint
    """Prints results as returned by evaluateResults()."""
    print "---------- RESULTS ----------"
    print "max, mean, min accuracies = "
    print "{0:.3f}, {1:.3f}, {2:.3f}".format(
    results["max_accuracy"], results["mean_accuracy"], results["min_accuracy"])
    print "total confusion matrix =\n", results["total_cm"]

  def _densifyPattern(self, bitmap):
    """Return a numpy array of 0s and 1s to represent the input bitmap."""
    densePattern = numpy.zeros(self.n)
    for i in bitmap:
      densePattern[i] = 1.0
    return densePattern

  def _winningLabels(self, labels, n=3):
    """
    Returns the most frequent item in the input list of labels. If there are
    ties for the most frequent item, the x most frequent are returned,
    where x<=n.
    """
    labelCount = Counter(labels).most_common()
    maxCount = 0
    for c in labelCount:  ## TODO: better way to do this?
      if c[1] > maxCount:
        maxCount = c[1]
    winners = [c[0] for c in labelCount if c[1]==maxCount]

    return winners if len(winners) <= n else winners[:n]

  def plotConfusionMatrix(self, cm):
    """Output plotly confusion matrix."""
    PlotNLP().confusionMatrix(cm)

  def encodePattern(self, pattern):
    """
    The subclass implementations must return the encoding in the following
    format:
      {
        ["text"]:sample,
        ["sparsity"]:sparsity,
        ["bitmap"]:bitmapSDR
      }
    Note: sample is a string, sparsity is float, and bitmapSDR is a numpy array.
    """
    raise NotImplementedError

  def resetModel(self):
    raise NotImplementedError

  def trainModel(self, sample, label):
    raise NotImplementedError

  def testModel(self, sample):
    raise NotImplementedError
<commit_msg>plotly staticmethod<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import numpy
import os
import pandas
import random

from collections import Counter
from utils.plotting import PlotNLP

try:
  import simplejson as json
except ImportError:
  import json

class ClassificationModel(object):
  """
  Base class for NLP models of classification tasks. When inheriting from this
  class please take note of which methods MUST be overridden, as documented
  below.

  The Model superclass implements:
    - classifyRandomly() calculates accuracy of a random classifier
    - encodeRandomly() creates a random SDR encoding
    - evaluateTrialResults() calculates result stats
    - evaluateResults() calculates result stats for a list of trial results
    - printTrialReport() prints classifications of an evaluation trial
    - printFinalReport() prints evaluation metrics and confusion matrix
    - densifyPattern() returns a binary SDR vector for a given bitmap

  Methods/properties that must be implemented by subclasses:
    - encodePattern(); note the specified format in the docstring below.
    - trainModel()
    - testModel()
  """

  def __init__(self, n=16384, w=328, verbosity=1, plot=True):
    """The SDR dimensions are standard for Cortical.io fingerprints."""
    self.n = n
    self.w = w
    self.verbosity = verbosity
    self.plot = plot

  def classifyRandomly(self, labels):
    """Return accuracy of random classifications for the labels."""
    randomLabels = numpy.random.randint(0, labels.max(), labels.shape)
    return (randomLabels == labels).sum() / float(labels.shape[0])

  def encodeRandomly(self, sample):
    """Return a random bitmap representation of the sample."""
    random.seed(sample)
    return numpy.sort(random.sample(xrange(self.n), self.w))

  def logEncodings(self, patterns, path):
    """Log the encoding dictionaries to a txt file."""
    if not os.path.isdir(path):
      raise ValueError("Invalid path to write file.")

    # Cast numpy arrays to list objects for serialization.
    jsonPatterns = copy.deepcopy(patterns)
    for jp in jsonPatterns:
      jp["bitmap"] = jp.get("bitmap", None).tolist()

    with open(os.path.join(path, "encoding_log.txt"), "w") as f:
      f.write(json.dumps(jsonPatterns, indent=1))

  def evaluateTrialResults(self, classifications, references, idx): ## TODO: evaluation metrics for multiple classifcations
    """
    Calculate statistics for the predicted classifications against the actual.

    @param classifications  (list)            Two lists: (0) predictions and (1)
                                              actual classifications. Items in
                                              the predictions list are lists of
                                              ints or None, and items in actual
                                              classifications list are ints.
    @param references       (list)            Classification label strings.
    @return                 (tuple)           Returns a 2-item tuple w/ the
                                              accuracy (float) and confusion
                                              matrix (numpy array).
    """
    if len(classifications[0]) != len(classifications[1]):
      raise ValueError("Classification lists must have same length.")
    if self.verbosity > 0:
      self._printTrialReport(classifications, references, idx)

    actual = numpy.array(classifications[1])
    predicted = numpy.array([c[0] for c in classifications[0]])  ## TODO: see above; this forces evaluation metrics to consider only the first predicted classification
    accuracy = (actual == predicted).sum() / float(len(actual))

    # Calculate confusion matrix.
    total = len(references)
    cm = numpy.zeros((total, total+1))
    for i, p in enumerate(predicted):
      if p is not None:
        cm[actual[i]][p] += 1
      else:
        # No predicted label, so increment the "(none)" column.
        cm[actual[i]][total] += 1
    cm = numpy.vstack((cm, numpy.sum(cm, axis=0)))
    cm = numpy.hstack((cm, numpy.sum(cm, axis=1).reshape(total+1,1)))

    cm = pandas.DataFrame(
      data=cm,
      columns=references+["(none)"]+["Actual Totals"],
      index=references+["Prediction Totals"])

    return (accuracy, cm)

  def evaluateFinalResults(self, intermResults):
    """
    Cumulative statistics for the outputs of evaluateTrialResults().

    @param intermResults      (list)          List of returned results from
                                              evaluateTrialResults().
    @return                   (list)          Returns a dictionary with entries
                                              for max, mean, and min accuracies,
                                              and the mean confusion matrix.
    """
    accuracy = []
    cm = numpy.zeros((intermResults[0][1].shape))

    # Find mean, max, and min values for the metrics.
    for result in intermResults:
      accuracy.append(result[0])
      cm = numpy.add(cm, result[1])

    results = {"max_accuracy":max(accuracy),
               "mean_accuracy":sum(accuracy)/float(len(accuracy)),
               "min_accuracy":min(accuracy),
               "total_cm":cm}

    if self.verbosity > 0:
      self._printFinalReport(results)

    if self.plot:
      self.plotConfusionMatrix(cm)

    return results

  @staticmethod
  def _printTrialReport(labels, refs, idx):
    """Print columns for sample #, actual label, and predicted label."""
    template = "{0:10}|{1:30}|{2:30}"
    print "Evaluation results for this fold:"
    print template.format("#", "Actual", "Predicted")
    for i in xrange(len(labels[0])):
      if labels[0][i][0] == None:
        print template.format(idx[i], refs[labels[1][i]], "(none)")
      else:
        print template.format(
          idx[i], refs[labels[1][i]], [refs[label] for label in labels[0][i]])

  @staticmethod
  def _printFinalReport(results):  ## TODO: pprint
    """Prints results as returned by evaluateResults()."""
    print "---------- RESULTS ----------"
    print "max, mean, min accuracies = "
    print "{0:.3f}, {1:.3f}, {2:.3f}".format(
    results["max_accuracy"], results["mean_accuracy"], results["min_accuracy"])
    print "total confusion matrix =\n", results["total_cm"]

  def _densifyPattern(self, bitmap):
    """Return a numpy array of 0s and 1s to represent the input bitmap."""
    densePattern = numpy.zeros(self.n)
    for i in bitmap:
      densePattern[i] = 1.0
    return densePattern

  def _winningLabels(self, labels, n=3):
    """
    Returns the most frequent item in the input list of labels. If there are
    ties for the most frequent item, the x most frequent are returned,
    where x<=n.
    """
    labelCount = Counter(labels).most_common()
    maxCount = 0
    for c in labelCount:  ## TODO: better way to do this?
      if c[1] > maxCount:
        maxCount = c[1]
    winners = [c[0] for c in labelCount if c[1]==maxCount]

    return winners if len(winners) <= n else winners[:n]

  @staticmethod
  def plotConfusionMatrix(cm):
    """Output plotly confusion matrix."""
    PlotNLP().confusionMatrix(cm)

  def encodePattern(self, pattern):
    """
    The subclass implementations must return the encoding in the following
    format:
      {
        ["text"]:sample,
        ["sparsity"]:sparsity,
        ["bitmap"]:bitmapSDR
      }
    Note: sample is a string, sparsity is float, and bitmapSDR is a numpy array.
    """
    raise NotImplementedError

  def resetModel(self):
    raise NotImplementedError

  def trainModel(self, sample, label):
    raise NotImplementedError

  def testModel(self, sample):
    raise NotImplementedError
<|endoftext|>


<commit_before>import serial
import schedule
import time
import json
import forecastio
import sys
from flask import Flask
from threading import Thread
from influxdb import InfluxDBClient
from twilio.rest import TwilioRestClient

try:
  from flask.ext.cors import CORS  # The typical way to import flask-cors
except ImportError:
  # Path hack allows examples to be run without installation.
  import os
  parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  os.sys.path.insert(0, parentdir)
  from flask.ext.cors import CORS

FILE_NAME = "config.json"
config = {}
current_forecast = {}
client = None
influxdb = None
ser = None

app = Flask(__name__, static_url_path='/static')
app.debug = True
cors = CORS(app)

def load_file():
  with open(FILE_NAME, "r") as data_file:
    config = json.load(data_file)
    client = TwilioRestClient(
        config['config']['alert']['sid'], config['config']['alert']['token'])
    influxdb = InfluxDBClient(config['config']['db']['host'], config['config']['db']['port'], config[
                              'config']['db']['username'], config['config']['db']['password'], config['config']['db']['name'])
    ser = serial.Serial(
        config['config']['arduino']['com_port'], config['config']['arduino']['baudrate'])
    data_file.close()

def send_sms(content):
  client.messages.create(
      to=config['config']['alert']['to'],
      from_=config['config']['alert']['from'],
      body=content,
  )

def get_sensors():
  slow = get_sensors.counter == (
      (config['config']['arduino']['read_sensors_timer'] / config['config']['arduino']['read_sensors_fast_timer']) - 1)
  if slow:
    ser.write('r')
    get_sensors.counter = 0
  else:
    ser.write('x')
  get_sensors.counter += 1
  json_info = ser.readline()
  json_info = json_info.replace('\n', '')
  json_info = json_info.replace('\r', '')
  json_info = json_info.replace('\'', '\"')
  m = json.loads(json_info)
  if slow:
    m.append(current_forecast)
  try:
    influxdb.write_points(m)
  except:
    print "Unexpected error InfluxDB:", sys.exc_info()[0]

get_sensors.counter = 0

def get_meteo():
  try:
    forecast = forecastio.load_forecast(
        config['config']['forecast']['api'], config['config']['forecast']['lat'], config['config']['forecast']['long'])
    temp = forecast.currently().temperature
    humi = forecast.currently().humidity * 100
  except:
    print "Unexpected error Forecast.io:", sys.exc_info()[0]
  else:
    global current_forecast
    current_forecast = {
        "points": [[temp, humi]],
        "name": "forecastio",
        "columns": ["temperature", "humidity"]
    }

def run_schedule():
  while 1:
    schedule.run_pending()
    time.sleep(1)

@app.route('/', methods=['GET'])
def index():
  return app.send_static_file('index.html')

@app.route('/config', methods=['POST'])
def index():
  data = request.form["config"]
  with open(FILE_NAME, "w") as data_file:
    data_file.write(json.dumps(data))
  load_file()
  return app.send_static_file('index.html')

@app.route('/<path:path>')
def static_proxy(path):
  # send_static_file will guess the correct MIME type
  return app.send_static_file(path)

if __name__ == '__main__':
  load_file()
  get_meteo()  # init current_forecast
  schedule.every(
      config['config']['arduino']['read_sensors_fast_timer']).seconds.do(get_sensors)
  schedule.every(
      config['config']['arduino']['get_meteo_timer']).seconds.do(get_meteo)
  t = Thread(target=run_schedule)
  t.daemon = True
  t.start()
  app.run(debug=True, use_reloader=False, host='0.0.0.0', port=8080)
  ser.close()
<commit_msg>routes<commit_after>import serial
import schedule
import time
import json
import forecastio
import sys
from flask import Flask
from threading import Thread
from influxdb import InfluxDBClient
from twilio.rest import TwilioRestClient

try:
  from flask.ext.cors import CORS  # The typical way to import flask-cors
except ImportError:
  # Path hack allows examples to be run without installation.
  import os
  parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  os.sys.path.insert(0, parentdir)
  from flask.ext.cors import CORS

FILE_NAME = "config.json"
config = {}
current_forecast = {}
client = None
influxdb = None
ser = None

app = Flask(__name__, static_url_path='/static')
app.debug = True
cors = CORS(app)

def load_file():
  with open(FILE_NAME, "r") as data_file:
    config = json.load(data_file)
    client = TwilioRestClient(
        config['config']['alert']['sid'], config['config']['alert']['token'])
    influxdb = InfluxDBClient(config['config']['db']['host'], config['config']['db']['port'], config[
                              'config']['db']['username'], config['config']['db']['password'], config['config']['db']['name'])
    ser = serial.Serial(
        config['config']['arduino']['com_port'], config['config']['arduino']['baudrate'])
    data_file.close()

def send_sms(content):
  client.messages.create(
      to=config['config']['alert']['to'],
      from_=config['config']['alert']['from'],
      body=content,
  )

def get_sensors():
  slow = get_sensors.counter == (
      (config['config']['arduino']['read_sensors_timer'] / config['config']['arduino']['read_sensors_fast_timer']) - 1)
  if slow:
    ser.write('r')
    get_sensors.counter = 0
  else:
    ser.write('x')
  get_sensors.counter += 1
  json_info = ser.readline()
  json_info = json_info.replace('\n', '')
  json_info = json_info.replace('\r', '')
  json_info = json_info.replace('\'', '\"')
  m = json.loads(json_info)
  if slow:
    m.append(current_forecast)
  try:
    influxdb.write_points(m)
  except:
    print "Unexpected error InfluxDB:", sys.exc_info()[0]

get_sensors.counter = 0

def get_meteo():
  try:
    forecast = forecastio.load_forecast(
        config['config']['forecast']['api'], config['config']['forecast']['lat'], config['config']['forecast']['long'])
    temp = forecast.currently().temperature
    humi = forecast.currently().humidity * 100
  except:
    print "Unexpected error Forecast.io:", sys.exc_info()[0]
  else:
    global current_forecast
    current_forecast = {
        "points": [[temp, humi]],
        "name": "forecastio",
        "columns": ["temperature", "humidity"]
    }

def run_schedule():
  while 1:
    schedule.run_pending()
    time.sleep(1)

@app.route('/', methods=['GET'])
def index():
  return app.send_static_file('index.html')

@app.route('/', methods=['GET'])
def index():
  return app.send_static_file('index.html')

@app.route('/api/get_current_config')
def get_current_user():
  return config

@app.route('/config', methods=['GET'])
def get_current_user():
  return app.send_static_file('configuration/index.html')

@app.route('/config', methods=['POST'])
def index():
  data = request.form["config"]
  with open(FILE_NAME, "w") as data_file:
    data_file.write(json.dumps(data))
  load_file()
  return app.send_static_file('index.html')

@app.route('/<path:path>')
def static_proxy(path):
  # send_static_file will guess the correct MIME type
  return app.send_static_file(path)

if __name__ == '__main__':
  load_file()
  get_meteo()  # init current_forecast
  schedule.every(
      config['config']['arduino']['read_sensors_fast_timer']).seconds.do(get_sensors)
  schedule.every(
      config['config']['arduino']['get_meteo_timer']).seconds.do(get_meteo)
  t = Thread(target=run_schedule)
  t.daemon = True
  t.start()
  app.run(debug=True, use_reloader=False, host='0.0.0.0', port=8080)
  ser.close()
<|endoftext|>


<commit_before>__author__ = 'abdul'

import server

from mongoctl.utils import resolve_path
from mongoctl.mongoctl_logging import log_verbose, log_debug, log_exception, \
    log_warning, log_info

from bson.son import SON
from mongoctl.errors import MongoctlException
from replicaset_cluster import ReplicaSetCluster, get_member_repl_lag
from sharded_cluster import ShardedCluster
from mongoctl.mongodb_version import make_version_info
###############################################################################
# CONSTANTS
###############################################################################

# This is mongodb's default dbpath
DEFAULT_DBPATH='/data/db'

LOCK_FILE_NAME = "mongod.lock"

###############################################################################
# MongodServer Class
###############################################################################

class MongodServer(server.Server):

    ###########################################################################
    # Constructor
    ###########################################################################
    def __init__(self, server_doc):
        super(MongodServer, self).__init__(server_doc)

    ###########################################################################
    # Properties
    ###########################################################################

    def get_db_path(self):
        dbpath = self.get_cmd_option("dbpath")
        if not dbpath:
            dbpath = super(MongodServer, self).get_server_home()
        if not dbpath:
            dbpath = DEFAULT_DBPATH

        return resolve_path(dbpath)

    ###########################################################################
    def get_server_home(self):
        """
            Override!
        :return:
        """
        home_dir = super(MongodServer, self).get_server_home()
        if not home_dir:
            home_dir = self.get_db_path()

        return home_dir

    ###########################################################################
    def export_cmd_options(self, options_override=None, standalone=False):
        """
            Override!
        :return:
        """
        cmd_options = super(MongodServer, self).export_cmd_options(
            options_override=options_override)

        # reset some props to exporting vals
        cmd_options['dbpath'] = self.get_db_path()

        if 'repairpath' in cmd_options:
            cmd_options['repairpath'] = resolve_path(cmd_options['repairpath'])

        # Add ReplicaSet args if a cluster is configured

        repl_cluster = self.get_replicaset_cluster()
        if repl_cluster is not None:
            if "replSet" not in cmd_options:
                cmd_options["replSet"] = repl_cluster.id

        # apply standalone if specified
        if standalone:
            if "replSet" in cmd_options:
                del cmd_options["replSet"]
            if "keyFile" in cmd_options:
                del cmd_options["keyFile"]

        # add configsvr as needed
        if self.is_config_server():
            cmd_options["configsvr"] = True

        # add shardsvr as needed
        if self.is_shard_server():
            cmd_options["shardsvr"] = True

        # remove wiredTigerCacheSizeGB if specified since we set it in runtime parameter wiredTigerEngineRuntimeConfig
        # if it was specified
        if "wiredTigerCacheSizeGB" in cmd_options:
            del cmd_options["wiredTigerCacheSizeGB"]

        return cmd_options

    ###########################################################################
    def get_seed_users(self):
        """
            Override!
        :return:
        """
        seed_users = super(MongodServer, self).get_seed_users()
        # exempt database users for config servers
        if seed_users and self.is_config_server():
            for dbname in seed_users.keys():
                if dbname not in ["admin", "local", "config"]:
                    del seed_users[dbname]

        return seed_users

    ###########################################################################
    def get_lock_file_path(self):
        return self.get_default_file_path(LOCK_FILE_NAME)

    ###########################################################################
    def is_master(self):
        return self.get_cmd_option("master")

    ###########################################################################
    def is_slave(self):
        return self.get_cmd_option("slave")

    ###########################################################################
    def set_auth(self,auth):
        self.set_cmd_option("auth", auth)

    ###########################################################################
    def is_administrable(self):
        return not self.is_arbiter_server() and self.can_function()

    ###########################################################################
    def get_status(self, admin=False):

        # get status for super
        status = super(MongodServer, self).get_status(admin=admin)

        if "error" not in status and admin:
            rs_summary = self.get_rs_status_summary()
            if rs_summary:
                status["selfReplicaSetStatusSummary"] = rs_summary

        return status

    ###########################################################################
    def get_rs_status_summary(self):
        if self.is_replicaset_member():
            member_rs_status = self.get_member_rs_status()
            if member_rs_status:
                return {
                    "name": member_rs_status['name'],
                    "stateStr": member_rs_status['stateStr']
                }

    ###########################################################################
    def is_cluster_connection_member(self):
        return not self.is_arbiter_server()

    ###########################################################################
    def is_arbiter_server(self):
        cluster = self.get_cluster()
        return (isinstance(cluster, ReplicaSetCluster) and
                cluster.get_member_for(self).is_arbiter())

    ###########################################################################
    def is_replicaset_member(self):
        cluster = self.get_cluster()
        return isinstance(cluster, ReplicaSetCluster)

    ###########################################################################
    def get_replicaset_cluster(self):
        cluster = self.get_cluster()
        if isinstance(cluster, ReplicaSetCluster):
            return cluster

    ###########################################################################
    def is_config_server(self):
        cluster = self.get_cluster()

        return ((isinstance(cluster, ShardedCluster) and
                 cluster.has_config_server(self)) or
                self.get_cmd_option("configsvr"))

    ###########################################################################
    def is_shard_server(self):
        cluster = self.get_cluster()
        if isinstance(cluster, ShardedCluster):
            return cluster.has_shard(self)
        elif isinstance(cluster, ReplicaSetCluster):
            return cluster.is_shard_member()

    ###########################################################################
    def command_needs_auth(self, dbname, cmd):
        # isMaster command does not need auth
        if "isMaster" in cmd or "ismaster" in cmd:
            return False
        if 'shutdown' in cmd and self.is_arbiter_server():
            return False

        # otherwise use default behavior
        return super(MongodServer, self).command_needs_auth(dbname, cmd)

    ###########################################################################
    def get_mongo_uri_template(self, db=None):
        if not db:
            if self.is_auth():
                db = "/[dbname]"
            else:
                db = ""
        else:
            db = "/" + db

        creds = "[dbuser]:[dbpass]@" if self.is_auth() else ""
        return "mongodb://%s%s%s" % (creds, self.get_address_display(), db)

    ###########################################################################
    def get_rs_status(self):
        try:
            rs_status_cmd = SON([('replSetGetStatus', 1)])
            rs_status =  self.db_command(rs_status_cmd, 'admin')
            return rs_status
        except (Exception,RuntimeError), e:
            log_debug("Cannot get rs status from server '%s'. cause: %s" %
                        (self.id, e))
            log_exception(e)
            return None

    ###########################################################################
    def get_member_rs_status(self):
        rs_status =  self.get_rs_status()
        if rs_status:
            try:
                for member in rs_status['members']:
                    if 'self' in member and member['self']:
                        return member
            except (Exception,RuntimeError), e:
                log_debug("Cannot get member rs status from server '%s'."
                          " cause: %s" % (self.id, e))
                log_exception(e)

                return None

    ###########################################################################
    def is_primary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("ismaster")

    ###########################################################################
    def is_secondary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("secondary")

    ###########################################################################
    def is_master_command(self):
        try:
            if self.is_online():
                result = self.db_command({"isMaster" : 1}, "admin")
                return result

        except(Exception, RuntimeError),e:
            log_verbose("isMaster command failed on server '%s'. Cause %s" %
                        (self.id, e))

    ###########################################################################
    def has_joined_replica(self):
        master_result = self.is_master_command()
        if master_result:
            return (master_result.get("setName") is not None or
                    master_result.get("ismaster") or
                    master_result.get("arbiterOnly") or
                    master_result.get("secondary"))

    ###########################################################################
    def get_repl_lag(self, master_status):
        """
            Given two 'members' elements from rs.status(),
            return lag between their optimes (in secs).
        """
        member_status = self.get_member_rs_status()

        if not member_status:
            raise MongoctlException("Unable to determine replicaset status for"
                                    " member '%s'" %
                                    self.id)

        return get_member_repl_lag(member_status, master_status)

    ###########################################################################
    def get_server_executable_env_vars(self):
        env_vars = {}
        # Workaround for mongod SERVER-24303 issue
        if self.is_wired_tiger():
            env_vars["TCMALLOC_AGGRESSIVE_DECOMMIT"] = "y"

        return env_vars

    ###########################################################################
    def is_wired_tiger(self):
        version = self.get_mongo_version_info()
        storage_engine = self.get_cmd_option("storageEngine")

        if version >= make_version_info("3.4.0"):
            #TODO XXX when we figure out 3.4.x is out. See SERVER-24303
            raise Exception("is_wired_tiger() not supported >= 3.4.x yet.")
        elif version >= make_version_info("3.2.0"):
            return not storage_engine or storage_engine == "wiredTiger"
        elif version >= make_version_info("3.0.0"):
            return storage_engine == "wiredTiger"

    ###########################################################################
    def set_runtime_parameters(self):
        params = self.generate_runtime_parameters()

        if params:
            for name, val in params.items():
                self.set_runtime_parameter_cmd(name, val)

            # verify that params were applied properly
            self.verify_runtime_parameters()

    ###########################################################################
    def verify_runtime_parameters(self):
        log_info("Verifying runtime params...")
        wtcs_gb = self.get_cmd_option("wiredTigerCacheSizeGB")
        if wtcs_gb is not None:
            wtcs_bytes = int(wtcs_gb * 1024 * 1024 * 1024)
            server_status = self.server_status()
            if not(server_status and "wiredTiger" in server_status and "cache" in server_status["wiredTiger"] and
                           "maximum bytes configured" in server_status["wiredTiger"]["cache"] and
                           server_status["wiredTiger"]["cache"]["maximum bytes configured"] == wtcs_bytes):
                raise MongoctlException("CONFIG ERROR: wiredTigerCacheSizeGB was not applied")

        log_info("Runtime params verification passed!")

    ###########################################################################
    def generate_runtime_parameters(self):
        parameters = {}
        wtcs_gb = self.get_cmd_option("wiredTigerCacheSizeGB")
        if wtcs_gb is not None:
            wtcs_mb = int(wtcs_gb * 1024)
            parameters["wiredTigerEngineRuntimeConfig"] = "cache_size=%sM" % wtcs_mb

        return parameters

    ###########################################################################
    def set_runtime_parameter_cmd(self, name, value):
        log_info("Setting runtime parameter '%s' to '%s', for server '%s'..." %
                 (name, value, self.id))
        cmd = {
            "setParameter": 1,
            name: value
        }
        self.db_command(cmd, "admin")

<commit_msg>no longer apply wiredTigerCacheSizeGB in runtime params<commit_after>__author__ = 'abdul'

import server

from mongoctl.utils import resolve_path
from mongoctl.mongoctl_logging import log_verbose, log_debug, log_exception, \
    log_warning, log_info

from bson.son import SON
from mongoctl.errors import MongoctlException
from replicaset_cluster import ReplicaSetCluster, get_member_repl_lag
from sharded_cluster import ShardedCluster
from mongoctl.mongodb_version import make_version_info
###############################################################################
# CONSTANTS
###############################################################################

# This is mongodb's default dbpath
DEFAULT_DBPATH='/data/db'

LOCK_FILE_NAME = "mongod.lock"

###############################################################################
# MongodServer Class
###############################################################################

class MongodServer(server.Server):

    ###########################################################################
    # Constructor
    ###########################################################################
    def __init__(self, server_doc):
        super(MongodServer, self).__init__(server_doc)

    ###########################################################################
    # Properties
    ###########################################################################

    def get_db_path(self):
        dbpath = self.get_cmd_option("dbpath")
        if not dbpath:
            dbpath = super(MongodServer, self).get_server_home()
        if not dbpath:
            dbpath = DEFAULT_DBPATH

        return resolve_path(dbpath)

    ###########################################################################
    def get_server_home(self):
        """
            Override!
        :return:
        """
        home_dir = super(MongodServer, self).get_server_home()
        if not home_dir:
            home_dir = self.get_db_path()

        return home_dir

    ###########################################################################
    def export_cmd_options(self, options_override=None, standalone=False):
        """
            Override!
        :return:
        """
        cmd_options = super(MongodServer, self).export_cmd_options(
            options_override=options_override)

        # reset some props to exporting vals
        cmd_options['dbpath'] = self.get_db_path()

        if 'repairpath' in cmd_options:
            cmd_options['repairpath'] = resolve_path(cmd_options['repairpath'])

        # Add ReplicaSet args if a cluster is configured

        repl_cluster = self.get_replicaset_cluster()
        if repl_cluster is not None:
            if "replSet" not in cmd_options:
                cmd_options["replSet"] = repl_cluster.id

        # apply standalone if specified
        if standalone:
            if "replSet" in cmd_options:
                del cmd_options["replSet"]
            if "keyFile" in cmd_options:
                del cmd_options["keyFile"]

        # add configsvr as needed
        if self.is_config_server():
            cmd_options["configsvr"] = True

        # add shardsvr as needed
        if self.is_shard_server():
            cmd_options["shardsvr"] = True

        # remove wiredTigerCacheSizeGB if specified since we set it in runtime parameter wiredTigerEngineRuntimeConfig
        # if it was specified
        if "wiredTigerCacheSizeGB" in cmd_options and self.get_cmd_option("wiredTigerCacheSizeGB") < 1:
            del cmd_options["wiredTigerCacheSizeGB"]

        return cmd_options

    ###########################################################################
    def get_seed_users(self):
        """
            Override!
        :return:
        """
        seed_users = super(MongodServer, self).get_seed_users()
        # exempt database users for config servers
        if seed_users and self.is_config_server():
            for dbname in seed_users.keys():
                if dbname not in ["admin", "local", "config"]:
                    del seed_users[dbname]

        return seed_users

    ###########################################################################
    def get_lock_file_path(self):
        return self.get_default_file_path(LOCK_FILE_NAME)

    ###########################################################################
    def is_master(self):
        return self.get_cmd_option("master")

    ###########################################################################
    def is_slave(self):
        return self.get_cmd_option("slave")

    ###########################################################################
    def set_auth(self,auth):
        self.set_cmd_option("auth", auth)

    ###########################################################################
    def is_administrable(self):
        return not self.is_arbiter_server() and self.can_function()

    ###########################################################################
    def get_status(self, admin=False):

        # get status for super
        status = super(MongodServer, self).get_status(admin=admin)

        if "error" not in status and admin:
            rs_summary = self.get_rs_status_summary()
            if rs_summary:
                status["selfReplicaSetStatusSummary"] = rs_summary

        return status

    ###########################################################################
    def get_rs_status_summary(self):
        if self.is_replicaset_member():
            member_rs_status = self.get_member_rs_status()
            if member_rs_status:
                return {
                    "name": member_rs_status['name'],
                    "stateStr": member_rs_status['stateStr']
                }

    ###########################################################################
    def is_cluster_connection_member(self):
        return not self.is_arbiter_server()

    ###########################################################################
    def is_arbiter_server(self):
        cluster = self.get_cluster()
        return (isinstance(cluster, ReplicaSetCluster) and
                cluster.get_member_for(self).is_arbiter())

    ###########################################################################
    def is_replicaset_member(self):
        cluster = self.get_cluster()
        return isinstance(cluster, ReplicaSetCluster)

    ###########################################################################
    def get_replicaset_cluster(self):
        cluster = self.get_cluster()
        if isinstance(cluster, ReplicaSetCluster):
            return cluster

    ###########################################################################
    def is_config_server(self):
        cluster = self.get_cluster()

        return ((isinstance(cluster, ShardedCluster) and
                 cluster.has_config_server(self)) or
                self.get_cmd_option("configsvr"))

    ###########################################################################
    def is_shard_server(self):
        cluster = self.get_cluster()
        if isinstance(cluster, ShardedCluster):
            return cluster.has_shard(self)
        elif isinstance(cluster, ReplicaSetCluster):
            return cluster.is_shard_member()

    ###########################################################################
    def command_needs_auth(self, dbname, cmd):
        # isMaster command does not need auth
        if "isMaster" in cmd or "ismaster" in cmd:
            return False
        if 'shutdown' in cmd and self.is_arbiter_server():
            return False

        # otherwise use default behavior
        return super(MongodServer, self).command_needs_auth(dbname, cmd)

    ###########################################################################
    def get_mongo_uri_template(self, db=None):
        if not db:
            if self.is_auth():
                db = "/[dbname]"
            else:
                db = ""
        else:
            db = "/" + db

        creds = "[dbuser]:[dbpass]@" if self.is_auth() else ""
        return "mongodb://%s%s%s" % (creds, self.get_address_display(), db)

    ###########################################################################
    def get_rs_status(self):
        try:
            rs_status_cmd = SON([('replSetGetStatus', 1)])
            rs_status =  self.db_command(rs_status_cmd, 'admin')
            return rs_status
        except (Exception,RuntimeError), e:
            log_debug("Cannot get rs status from server '%s'. cause: %s" %
                        (self.id, e))
            log_exception(e)
            return None

    ###########################################################################
    def get_member_rs_status(self):
        rs_status =  self.get_rs_status()
        if rs_status:
            try:
                for member in rs_status['members']:
                    if 'self' in member and member['self']:
                        return member
            except (Exception,RuntimeError), e:
                log_debug("Cannot get member rs status from server '%s'."
                          " cause: %s" % (self.id, e))
                log_exception(e)

                return None

    ###########################################################################
    def is_primary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("ismaster")

    ###########################################################################
    def is_secondary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("secondary")

    ###########################################################################
    def is_master_command(self):
        try:
            if self.is_online():
                result = self.db_command({"isMaster" : 1}, "admin")
                return result

        except(Exception, RuntimeError),e:
            log_verbose("isMaster command failed on server '%s'. Cause %s" %
                        (self.id, e))

    ###########################################################################
    def has_joined_replica(self):
        master_result = self.is_master_command()
        if master_result:
            return (master_result.get("setName") is not None or
                    master_result.get("ismaster") or
                    master_result.get("arbiterOnly") or
                    master_result.get("secondary"))

    ###########################################################################
    def get_repl_lag(self, master_status):
        """
            Given two 'members' elements from rs.status(),
            return lag between their optimes (in secs).
        """
        member_status = self.get_member_rs_status()

        if not member_status:
            raise MongoctlException("Unable to determine replicaset status for"
                                    " member '%s'" %
                                    self.id)

        return get_member_repl_lag(member_status, master_status)

    ###########################################################################
    def get_server_executable_env_vars(self):
        env_vars = {}
        # Workaround for mongod SERVER-24303 issue
        if self.is_wired_tiger():
            env_vars["TCMALLOC_AGGRESSIVE_DECOMMIT"] = "y"

        return env_vars

    ###########################################################################
    def is_wired_tiger(self):
        version = self.get_mongo_version_info()
        storage_engine = self.get_cmd_option("storageEngine")

        if version >= make_version_info("3.4.0"):
            #TODO XXX when we figure out 3.4.x is out. See SERVER-24303
            raise Exception("is_wired_tiger() not supported >= 3.4.x yet.")
        elif version >= make_version_info("3.2.0"):
            return not storage_engine or storage_engine == "wiredTiger"
        elif version >= make_version_info("3.0.0"):
            return storage_engine == "wiredTiger"

    ###########################################################################
    def set_runtime_parameters(self):
        params = self.generate_runtime_parameters()

        if params:
            for name, val in params.items():
                self.set_runtime_parameter_cmd(name, val)

            # verify that params were applied properly
            self.verify_runtime_parameters()

    ###########################################################################
    def verify_runtime_parameters(self):
        log_info("Verifying runtime params...")
        wtcs_gb = self.get_cmd_option("wiredTigerCacheSizeGB")
        if wtcs_gb is not None and wtcs_gb < 1:
            wtcs_bytes = int(wtcs_gb * 1024 * 1024 * 1024)
            server_status = self.server_status()
            if not(server_status and "wiredTiger" in server_status and "cache" in server_status["wiredTiger"] and
                           "maximum bytes configured" in server_status["wiredTiger"]["cache"] and
                           server_status["wiredTiger"]["cache"]["maximum bytes configured"] == wtcs_bytes):
                raise MongoctlException("CONFIG ERROR: wiredTigerCacheSizeGB was not applied")

        log_info("Runtime params verification passed!")

    ###########################################################################
    def generate_runtime_parameters(self):
        parameters = {}
        return parameters
        """ TODO Enable later
        wtcs_gb = self.get_cmd_option("wiredTigerCacheSizeGB")
        if wtcs_gb is not None and wtcs_gb < 1:
            wtcs_mb = int(wtcs_gb * 1024)
            parameters["wiredTigerEngineRuntimeConfig"] = "cache_size=%sM" % wtcs_mb

        return parameters
        """

    ###########################################################################
    def set_runtime_parameter_cmd(self, name, value):
        log_info("Setting runtime parameter '%s' to '%s', for server '%s'..." %
                 (name, value, self.id))
        cmd = {
            "setParameter": 1,
            name: value
        }
        self.db_command(cmd, "admin")

<|endoftext|>