<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2014, Andreas Pakulat <apaku@gmx.de>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from PyQt4 import QtCore, QtGui

from settings import SettingsWidget
from appdirs import user_config_dir
import os
import json
import threading
import traceback
from ..jenkinsmonitor import JenkinsMonitor
from ..jenkinsjob import JenkinsJob, JenkinsState

CONFIG_FILENAME = "jenkinstray.json"

def refreshMonitors(trayObject):
    for monitor in trayObject.monitors:
        try:
            monitor.refreshFromServer()
        except Exception, e:
            print "Error refreshing jenkins server %s: %s" % (monitor.serverurl, traceback.format_exc())
    trayObject.serverInfoUpdated.emit()

class JenkinsTray(QtCore.QObject):

    serverInfoUpdated = QtCore.pyqtSignal()

    def __init__(self, parent):
        QtCore.QObject.__init__(self, parent)
        self.monitors = []
        self.trayicon = QtGui.QSystemTrayIcon(self)
        self.menu = QtGui.QMenu()
        self.settingsAct = QtGui.QAction("Settings...", self.menu)
        self.settingsAct.activated.connect(self.openSettings)
        self.aboutQtAct = QtGui.QAction("About Qt", self.menu)
        self.aboutQtAct.activated.connect(self.aboutQt)
        self.aboutAct = QtGui.QAction("About %s" % QtGui.qApp.applicationName(), self.menu)
        self.aboutAct.activated.connect(self.aboutApp)
        self.quitAct = QtGui.QAction("Quit", self.menu)
        self.quitAct.activated.connect(QtGui.qApp.quit)
        self.menu.addAction(self.settingsAct)
        self.menu.addSeparator()
        self.menu.addAction(self.aboutAct)
        self.menu.addAction(self.aboutQtAct)
        self.menu.addSeparator()
        self.menu.addAction(self.quitAct)
        self.trayicon.setContextMenu(self.menu)
        self.image = QtGui.QImage(":///images/jenkinstray.png")
        assert(not self.image.isNull())
        self.trayicon.setIcon(QtGui.QIcon(QtGui.QPixmap.fromImage(self.image)))
        self.trayicon.setVisible(True)
        self.cfgDir = user_config_dir("jenkinstray", appauthor="jenkinstray", version="0.1")
        self.timer = QtCore.QTimer(self)
        self.timer.timeout.connect(lambda: threading.Thread(target=refreshMonitors(self)).start())
        self.serverInfoUpdated.connect(self.updateUiFromMonitors)
        self.updateFromSettings(self.readSettings())
        self.timer.start()

    def aboutApp(self):
        QtGui.QMessageBox.about(None,
                                "About %s" % QtGui.qApp.applicationName(),
                                "About %s

Version: %s

Jenkins Tray is a system tray application that monitors one or more jobs running on one or more jenkins instances and shows the overal status of the jobs as well as allowing access to the job web pages." % (QtGui.qApp.applicationName(),
                                                    QtGui.qApp.applicationVersion()))

    def aboutQt(self):
        QtGui.QMessageBox.aboutQt(None, "About Qt")

    def updateFromSettings(self, settings):
        self.timer.setInterval(settings["refreshInterval"] * 1000)
        self.notificationTimeout = settings["notificationTimeout"] * 1000
        for server in settings["servers"]:
            match = filter(lambda monitor: monitor.serverurl == server["url"], self.monitors)
            monitor = None
            if match:
                monitor = match[0]
            else:
                monitor = JenkinsMonitor(server["url"])
                try:
                    monitor.refreshFromServer()
                except Exception, e:
                    print "Error loading jobs from server %s: %s" % (server["url"], repr(e))
                self.monitors.append(monitor)
            for job in server["jobs"]:
                monitorjobmatches = filter(lambda monitorjob: monitorjob.name == job["name"], monitor.allJobs())
                if monitorjobmatches:
                    monitorjob = monitorjobmatches[0]
                else:
                    monitorjob = JenkinsJob(job["name"], job["monitored"], "Unknown", JenkinsState.Unknown)
                if job["monitored"]:
                    monitorjob.enableMonitoring()
                else:
                    monitorjob.disableMonitoring()
        for monitor in list(self.monitors):
            if len(filter(lambda server: monitor.serverurl == server["url"], settings["servers"])) == 0:
                self.monitors.remove(monitor)

    def addCountToImage(self, number):
        painter = QtGui.QPainter(self.image)
        font = painter.font()
        maxwidth = self.image.width() - round(self.image.width() * .3)
        maxheight = self.image.height() - round(self.image.height() * .3)
        metrics = QtGui.QFontMetrics(font)
        text = str(number)
        while metrics.width(text) < maxwidth and metrics.height() < maxheight:
            font.setPointSize(font.pointSize() + 1)
            metrics = QtGui.QFontMetrics(font)

        painter.setFont(font)
        painter.drawText(self.image.rect(), QtCore.Qt.AlignVCenter | QtCore.Qt.AlignCenter, text)
        painter.end()

    def updateUiFromMonitors(self):
        failCnt = 0
        unstableCnt = 0
        successfulCnt = 0
        failedjobs = []
        unstablejobs = []
        fixedjobs = []
        for monitor in self.monitors:
            failCnt += monitor.numFailedMonitoredJobs()
            unstableCnt += monitor.numUnstableMonitoredJobs()
            successfulCnt += monitor.numSuccessfulMonitoredJobs()
            for job in monitor.monitoredJobs():
                if job.lastState != job.state:
                    if job.state == JenkinsState.Failed and job.lastState in [JenkinsState.Unstable, JenkinsState.Successful]:
                        failedjobs.append(job.name)
                    elif job.state == JenkinsState.Unstable and job.lastState in [JenkinsState.Failed, JenkinsState.Successful]:
                        unstablejobs.append(job.name)
                    elif job.state == JenkinsState.Successful and job.lastState in [JenkinsState.Failed, JenkinsState.Unstable]:
                        fixedjobs.append(job.name)
        if failCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_failed.png")
        elif unstableCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_unstable.png")
        elif successfulCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_success.png")
        else:
            self.image = QtGui.QImage(":///images/jenkinstray.png")
        if len(failedjobs) > 0:
            self.trayicon.showMessage("Failed Jobs", "\n".join(failedjobs), QtGui.QSystemTrayIcon.Critical, self.notificationTimeout)
        elif len(fixedjobs) > 0:
            self.trayicon.showMessage("Fixed Jobs", "\n".join(fixedjobs), QtGui.QSystemTrayIcon.Information, self.notificationTimeout)
        elif len(unstablejobs) > 0:
            self.trayicon.showMessage("Unstable Jobs", "\n".join(unstablejobs), QtGui.QSystemTrayIcon.Warning, self.notificationTimeout)
        if failCnt > 0 or unstableCnt > 0:
            self.addCountToImage(failCnt + unstableCnt)
        self.trayicon.setIcon(QtGui.QIcon(QtGui.QPixmap.fromImage(self.image)))
        self.trayicon.setToolTip("%s failed jobs\n%s unstable jobs\n%s successful jobs" % (failCnt, unstableCnt, successfulCnt))

    def readSettings(self):
        try:
            return json.load(open(os.path.join(self.cfgDir, CONFIG_FILENAME), "r"))
        except:
            return {"refreshInterval": 60, "servers": [], "notificationTimeout": 10}

    def openSettings(self):
        dialog = QtGui.QDialog()
        dialog.setWindowTitle("Jenkins Tray Settings")
        settingsdata = self.createSettingsFromMonitors()
        settingswidget = SettingsWidget(dialog, settingsdata)
        layout = QtGui.QVBoxLayout(dialog)
        layout.addWidget(settingswidget)
        buttonbox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.StandardButtons(QtGui.QDialogButtonBox.Ok | QtGui.QDialogButtonBox.Cancel), QtCore.Qt.Horizontal, dialog)
        layout.addWidget(buttonbox)
        buttonbox.accepted.connect(dialog.accept)
        buttonbox.rejected.connect(dialog.reject)
        dialog.setModal(True)
        if dialog.exec_() == QtGui.QDialog.Accepted:
            self.writeSettings(settingsdata)
            self.updateFromSettings(settingsdata)
            self.updateUiFromMonitors()

    def writeSettings(self, settings):
        if not os.path.exists(self.cfgDir):
            os.makedirs(self.cfgDir)
        json.dump(settings, open(os.path.join(self.cfgDir, CONFIG_FILENAME), "w"), indent=4, separators=(",", ": "))

    def createSettingsFromMonitors(self):
        return {
                "refreshInterval": int(self.timer.interval() / 1000),
                "notificationTimeout": int(self.notificationTimeout / 1000),
                "servers": map(lambda monitor: {
                                                "url": monitor.serverurl,
                                                "jobs": map(lambda job: {
                                                                         "name": job.name,
                                                                         "monitored": job.monitored
                                                                        },
                                                            monitor.allJobs())
                                               },
                               self.monitors)
               }
<commit_msg>Report issues while connecting to a server via popup<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2014, Andreas Pakulat <apaku@gmx.de>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from PyQt4 import QtCore, QtGui

from settings import SettingsWidget
from appdirs import user_config_dir
import os
import json
import threading
import traceback
from ..jenkinsmonitor import JenkinsMonitor
from ..jenkinsjob import JenkinsJob, JenkinsState

CONFIG_FILENAME = "jenkinstray.json"

def refreshMonitors(trayObject):
    errors = []
    for monitor in trayObject.monitors:
        try:
            monitor.refreshFromServer()
            if monitor in trayObject.monitorsWithConnectivityProblems:
                trayObject.monitorsWithConnectivityProblems.remove(monitor)
        except Exception, e:
            if monitor not in trayObject.monitorsWithConnectivityProblems:
                errors.append(str(e))
            trayObject.monitorsWithConnectivityProblems.append(monitor)
            print "Error refreshing jenkins server %s: %s" % (monitor.serverurl, traceback.format_exc())
    trayObject.serverInfoUpdated.emit(errors)

class JenkinsTray(QtCore.QObject):

    serverInfoUpdated = QtCore.pyqtSignal(list)

    def __init__(self, parent):
        QtCore.QObject.__init__(self, parent)
        self.monitors = []
        self.monitorsWithConnectivityProblems = []
        self.trayicon = QtGui.QSystemTrayIcon(self)
        self.menu = QtGui.QMenu()
        self.settingsAct = QtGui.QAction("Settings...", self.menu)
        self.settingsAct.activated.connect(self.openSettings)
        self.aboutQtAct = QtGui.QAction("About Qt", self.menu)
        self.aboutQtAct.activated.connect(self.aboutQt)
        self.aboutAct = QtGui.QAction("About %s" % QtGui.qApp.applicationName(), self.menu)
        self.aboutAct.activated.connect(self.aboutApp)
        self.quitAct = QtGui.QAction("Quit", self.menu)
        self.quitAct.activated.connect(QtGui.qApp.quit)
        self.menu.addAction(self.settingsAct)
        self.menu.addSeparator()
        self.menu.addAction(self.aboutAct)
        self.menu.addAction(self.aboutQtAct)
        self.menu.addSeparator()
        self.menu.addAction(self.quitAct)
        self.trayicon.setContextMenu(self.menu)
        self.image = QtGui.QImage(":///images/jenkinstray.png")
        assert(not self.image.isNull())
        self.trayicon.setIcon(QtGui.QIcon(QtGui.QPixmap.fromImage(self.image)))
        self.trayicon.setVisible(True)
        self.cfgDir = user_config_dir("jenkinstray", appauthor="jenkinstray", version="0.1")
        self.timer = QtCore.QTimer(self)
        self.timer.timeout.connect(lambda: threading.Thread(target=refreshMonitors(self)).start())
        self.serverInfoUpdated.connect(self.updateUiFromMonitors)
        self.updateFromSettings(self.readSettings())
        self.timer.start()

    def aboutApp(self):
        QtGui.QMessageBox.about(None,
                                "About %s" % QtGui.qApp.applicationName(),
                                "About %s

Version: %s

Jenkins Tray is a system tray application that monitors one or more jobs running on one or more jenkins instances and shows the overal status of the jobs as well as allowing access to the job web pages." % (QtGui.qApp.applicationName(),
                                                    QtGui.qApp.applicationVersion()))

    def aboutQt(self):
        QtGui.QMessageBox.aboutQt(None, "About Qt")

    def updateFromSettings(self, settings):
        self.timer.setInterval(settings["refreshInterval"] * 1000)
        self.notificationTimeout = settings["notificationTimeout"] * 1000
        for server in settings["servers"]:
            match = filter(lambda monitor: monitor.serverurl == server["url"], self.monitors)
            monitor = None
            if match:
                monitor = match[0]
            else:
                monitor = JenkinsMonitor(server["url"])
                try:
                    monitor.refreshFromServer()
                except Exception, e:
                    print "Error loading jobs from server %s: %s" % (server["url"], repr(e))
                self.monitors.append(monitor)
            for job in server["jobs"]:
                monitorjobmatches = filter(lambda monitorjob: monitorjob.name == job["name"], monitor.allJobs())
                if monitorjobmatches:
                    monitorjob = monitorjobmatches[0]
                else:
                    monitorjob = JenkinsJob(job["name"], job["monitored"], "Unknown", JenkinsState.Unknown)
                if job["monitored"]:
                    monitorjob.enableMonitoring()
                else:
                    monitorjob.disableMonitoring()
        for monitor in list(self.monitors):
            if len(filter(lambda server: monitor.serverurl == server["url"], settings["servers"])) == 0:
                self.monitors.remove(monitor)

    def addCountToImage(self, number):
        painter = QtGui.QPainter(self.image)
        font = painter.font()
        maxwidth = self.image.width() - round(self.image.width() * .3)
        maxheight = self.image.height() - round(self.image.height() * .3)
        metrics = QtGui.QFontMetrics(font)
        text = str(number)
        while metrics.width(text) < maxwidth and metrics.height() < maxheight:
            font.setPointSize(font.pointSize() + 1)
            metrics = QtGui.QFontMetrics(font)

        painter.setFont(font)
        painter.drawText(self.image.rect(), QtCore.Qt.AlignVCenter | QtCore.Qt.AlignCenter, text)
        painter.end()

    def updateUiFromMonitors(self, errors):
        failCnt = 0
        unstableCnt = 0
        successfulCnt = 0
        failedjobs = []
        unstablejobs = []
        fixedjobs = []
        for monitor in self.monitors:
            failCnt += monitor.numFailedMonitoredJobs()
            unstableCnt += monitor.numUnstableMonitoredJobs()
            successfulCnt += monitor.numSuccessfulMonitoredJobs()
            for job in monitor.monitoredJobs():
                if job.lastState != job.state:
                    if job.state == JenkinsState.Failed and job.lastState in [JenkinsState.Unstable, JenkinsState.Successful]:
                        failedjobs.append(job.name)
                    elif job.state == JenkinsState.Unstable and job.lastState in [JenkinsState.Failed, JenkinsState.Successful]:
                        unstablejobs.append(job.name)
                    elif job.state == JenkinsState.Successful and job.lastState in [JenkinsState.Failed, JenkinsState.Unstable]:
                        fixedjobs.append(job.name)
        if failCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_failed.png")
        elif unstableCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_unstable.png")
        elif successfulCnt > 0:
            self.image = QtGui.QImage(":///images/jenkinstray_success.png")
        else:
            self.image = QtGui.QImage(":///images/jenkinstray.png")
        if len(errors) > 0:
            self.trayicon.showMessage("Connectivity Problems", "\n".join(errors), QtGui.QSystemTrayIcon.Critical, self.notificationTimeout)
        elif len(failedjobs) > 0:
            self.trayicon.showMessage("Failed Jobs", "\n".join(failedjobs), QtGui.QSystemTrayIcon.Critical, self.notificationTimeout)
        elif len(fixedjobs) > 0:
            self.trayicon.showMessage("Fixed Jobs", "\n".join(fixedjobs), QtGui.QSystemTrayIcon.Information, self.notificationTimeout)
        elif len(unstablejobs) > 0:
            self.trayicon.showMessage("Unstable Jobs", "\n".join(unstablejobs), QtGui.QSystemTrayIcon.Warning, self.notificationTimeout)
        if failCnt > 0 or unstableCnt > 0:
            self.addCountToImage(failCnt + unstableCnt)
        self.trayicon.setIcon(QtGui.QIcon(QtGui.QPixmap.fromImage(self.image)))
        self.trayicon.setToolTip("%s failed jobs\n%s unstable jobs\n%s successful jobs" % (failCnt, unstableCnt, successfulCnt))

    def readSettings(self):
        try:
            return json.load(open(os.path.join(self.cfgDir, CONFIG_FILENAME), "r"))
        except:
            return {"refreshInterval": 60, "servers": [], "notificationTimeout": 10}

    def openSettings(self):
        dialog = QtGui.QDialog()
        dialog.setWindowTitle("Jenkins Tray Settings")
        settingsdata = self.createSettingsFromMonitors()
        settingswidget = SettingsWidget(dialog, settingsdata)
        layout = QtGui.QVBoxLayout(dialog)
        layout.addWidget(settingswidget)
        buttonbox = QtGui.QDialogButtonBox(QtGui.QDialogButtonBox.StandardButtons(QtGui.QDialogButtonBox.Ok | QtGui.QDialogButtonBox.Cancel), QtCore.Qt.Horizontal, dialog)
        layout.addWidget(buttonbox)
        buttonbox.accepted.connect(dialog.accept)
        buttonbox.rejected.connect(dialog.reject)
        dialog.setModal(True)
        if dialog.exec_() == QtGui.QDialog.Accepted:
            self.writeSettings(settingsdata)
            self.updateFromSettings(settingsdata)
            self.updateUiFromMonitors()

    def writeSettings(self, settings):
        if not os.path.exists(self.cfgDir):
            os.makedirs(self.cfgDir)
        json.dump(settings, open(os.path.join(self.cfgDir, CONFIG_FILENAME), "w"), indent=4, separators=(",", ": "))

    def createSettingsFromMonitors(self):
        return {
                "refreshInterval": int(self.timer.interval() / 1000),
                "notificationTimeout": int(self.notificationTimeout / 1000),
                "servers": map(lambda monitor: {
                                                "url": monitor.serverurl,
                                                "jobs": map(lambda job: {
                                                                         "name": job.name,
                                                                         "monitored": job.monitored
                                                                        },
                                                            monitor.allJobs())
                                               },
                               self.monitors)
               }
<|endoftext|>


<commit_before>#    Author: Leonardo Pistone
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
from openerp.tests.common import TransactionCase

class TestSourcingPercentage(TransactionCase):
    def test_one_sourced_out_of_two_is_50_percent(self):
        self.lr = self.LR.new({
            'line_ids': (
                self.LRL.new({'state': 'draft'}) |
                self.LRL.new({'state': 'sourced'})
            )
        })
        # "or 0.0" because odoo returns False instead of 0.0.
        # I need float for assertAlmostEqual to give a good failure.
        self.assertAlmostEqual(0.5, self.lr._get_sourced() or 0.0)

    def setUp(self):
        super(TestSourcingPercentage, self).setUp()
        self.LR = self.env['logistic.requisition']
        self.LRL = self.env['logistic.requisition.line']
<commit_msg>fix test on sourcing percentage<commit_after>#    Author: Leonardo Pistone
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
from openerp.tests.common import TransactionCase

class TestSourcingPercentage(TransactionCase):
    def test_one_sourced_out_of_two_is_50_percent(self):
        self.lr = self.LR.new({
            'line_ids': (
                self.LRL.new({'state': 'draft'}) |
                self.LRL.new({'state': 'sourced'})
            )
        })
        self.assertAlmostEqual(50.0, self.lr.sourced)

    def setUp(self):
        super(TestSourcingPercentage, self).setUp()
        self.LR = self.env['logistic.requisition']
        self.LRL = self.env['logistic.requisition.line']
<|endoftext|>


<commit_before># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

# from package_lib import SoftwarePackage
from time import time
from datetime import datetime
from csmpe.plugins import CSMPlugin
from install import expand_subpkgs
from install import install_activate_reload
from install import install_activate_write_memory
from install import install_activate_issu
from csmpe.core_plugins.csm_get_software_packages.ios_xe.plugin import get_package
from csmpe.core_plugins.csm_install_operations.utils import update_device_info_udi
from utils import remove_exist_image
from utils import xe_show_platform
from utils import install_add_remove

class Plugin(CSMPlugin):
    """This plugin Activates packages on the device."""
    name = "Install Activate Plugin"
    platforms = {'ASR900'}
    phases = {'Activate'}
    os = {'XE'}

    def run(self):
        """
        Performs install activate operation
        """

        rsp_count = int(self.ctx.load_data('xe_rsp_count')[0])
        pkg = self.ctx.load_data('xe_activate_pkg')[0]
        mode = self.ctx.load_data('xe_boot_mode')[0]
        folder = self.ctx.load_data('xe_install_folder')[0]

        self.ctx.info("Activate number of RSP = {}".format(rsp_count))
        self.ctx.info("Activate package = {}".format(pkg))
        self.ctx.info("Activate package mode = {}".format(mode))
        self.ctx.info("Install folder = {}".format(folder))

        self.ctx.info("Activate package(s) pending")
        self.ctx.post_status("Activate Package(s) Pending")

        prompt = self.ctx._connection.hostname

        # issu: need to copy the consolidated image to the installed folder
        if mode == 'issu':
            cmd = 'copy bootflash:' + pkg + ' ' + folder + '/' + pkg
            install_add_remove(self.ctx, cmd)
            package = 'bootflash:' + pkg
            remove_exist_image(self.ctx, package)

        # subpackage: need to expand the consolidated image to the installed folder
        if mode == 'subpackage':
            result = expand_subpkgs(self.ctx, rsp_count, folder, pkg)
            if not result:
                self.ctx.error("Error in extracting sub-images from the consolidated "
                               "image {}".format(pkg))
                return

        # configurations
        cmd = "configure terminal"
        self.ctx.send(cmd, wait_for_string=prompt)
        cmd = "config-register 0x2102"
        self.ctx.send(cmd, wait_for_string=prompt)

        if mode == 'issu':
            cmd = 'redundancy'
            self.ctx.send(cmd, wait_for_string=prompt)
            cmd = 'mode sso'
            self.ctx.send(cmd, wait_for_string=prompt)
        else:
            cmd = "no boot system"
            self.ctx.send(cmd, wait_for_string=prompt)
            if mode == 'consolidated':
                cmd = "boot system bootflash:" + pkg
            else:
                cmd = 'boot system ' + folder + '/packages.conf'
            self.ctx.send(cmd, wait_for_string=prompt)

        self.ctx.send('end', wait_for_string=prompt)

        cmd = "write memory"
        install_activate_write_memory(self.ctx, cmd, self.ctx._connection.hostname)
        # self.ctx.send(cmd, timeout=300, wait_for_string=prompt)

        # Start activation
        if mode == 'issu':
            cmd = 'request platform software package install node file ' + \
                  folder + '/' + pkg + ' interface-module-delay 160'
            install_activate_issu(self.ctx, cmd)

            # Remove all-in-one image from the installed folder
            package = folder + '/' + pkg
            remove_exist_image(self.ctx, package)
            package = 'stby-' + package
            remove_exist_image(self.ctx, package)

            # Remove the all-in-one image from stby-bootflash:
            package = 'stby-bootflash:' + pkg
            remove_exist_image(self.ctx, package)
        else:
            install_activate_reload(self.ctx)

        self.ctx.info("Activate package done")

        # Refresh package information
        get_package(self.ctx)

        update_device_info_udi(self.ctx)

        # Verify the version after activation
        if self.ctx._connection.os_version not in pkg:
            self.ctx.error('The post-activate OS Version: {}'.self.ctx._connection.os_version)

        # Verify the status of RP and SIP
        previous_data, timestamp = self.ctx.load_data('xe_show_platform')
        self.ctx.info("Pre-Activate data collected on {}".format(
            datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')))
        if timestamp < time() - (60 * 60 * 2):  # two hours
            self.ctx.warning("Pre-Activate phase 'show platform' "
                             "data older than 2 hours")

        current_data = xe_show_platform(self.ctx)
        if not current_data:
            self.ctx.error("The CLI 'show platform' is not able to determine the status of RP and SIP ")
            return

        for Slot, Status in previous_data.items():
            Type = Status[0]
            previous_state = Status[1]
            current_state = current_data[Slot][1]
            if previous_state != current_state:
                if previous_state == 'ok, active' and current_state == 'ok, standby' \
                        or previous_state == 'ok, standby' and current_state == 'ok, active':
                    continue
                self.ctx.warning("Slot {} Type {} state changes after Activation".format(Slot, Type))
                self.ctx.warning("\t Pre-Activate State = {} vs. Post-Activate State = "
                                 "{}".format(previous_state, current_state))
            if 'ok' not in current_state:
                self.ctx.error("Slot {} Type {} is not in 'ok' state after "
                               "activation".format(Slot, Type))

        self.ctx.info("The status of RP and SIP has been verified. Please check any warnings in plugins.log")
<commit_msg>move show platform erro to warning<commit_after># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

# from package_lib import SoftwarePackage
from time import time
from datetime import datetime
from csmpe.plugins import CSMPlugin
from install import expand_subpkgs
from install import install_activate_reload
from install import install_activate_write_memory
from install import install_activate_issu
from csmpe.core_plugins.csm_get_software_packages.ios_xe.plugin import get_package
from csmpe.core_plugins.csm_install_operations.utils import update_device_info_udi
from utils import remove_exist_image
from utils import xe_show_platform
from utils import install_add_remove

class Plugin(CSMPlugin):
    """This plugin Activates packages on the device."""
    name = "Install Activate Plugin"
    platforms = {'ASR900'}
    phases = {'Activate'}
    os = {'XE'}

    def run(self):
        """
        Performs install activate operation
        """

        rsp_count = int(self.ctx.load_data('xe_rsp_count')[0])
        pkg = self.ctx.load_data('xe_activate_pkg')[0]
        mode = self.ctx.load_data('xe_boot_mode')[0]
        folder = self.ctx.load_data('xe_install_folder')[0]

        self.ctx.info("Activate number of RSP = {}".format(rsp_count))
        self.ctx.info("Activate package = {}".format(pkg))
        self.ctx.info("Activate package mode = {}".format(mode))
        self.ctx.info("Install folder = {}".format(folder))

        self.ctx.info("Activate package(s) pending")
        self.ctx.post_status("Activate Package(s) Pending")

        prompt = self.ctx._connection.hostname

        # issu: need to copy the consolidated image to the installed folder
        if mode == 'issu':
            cmd = 'copy bootflash:' + pkg + ' ' + folder + '/' + pkg
            install_add_remove(self.ctx, cmd)
            package = 'bootflash:' + pkg
            remove_exist_image(self.ctx, package)

        # subpackage: need to expand the consolidated image to the installed folder
        if mode == 'subpackage':
            result = expand_subpkgs(self.ctx, rsp_count, folder, pkg)
            if not result:
                self.ctx.error("Error in extracting sub-images from the consolidated "
                               "image {}".format(pkg))
                return

        # configurations
        cmd = "configure terminal"
        self.ctx.send(cmd, wait_for_string=prompt)
        cmd = "config-register 0x2102"
        self.ctx.send(cmd, wait_for_string=prompt)

        if mode == 'issu':
            cmd = 'redundancy'
            self.ctx.send(cmd, wait_for_string=prompt)
            cmd = 'mode sso'
            self.ctx.send(cmd, wait_for_string=prompt)
        else:
            cmd = "no boot system"
            self.ctx.send(cmd, wait_for_string=prompt)
            if mode == 'consolidated':
                cmd = "boot system bootflash:" + pkg
            else:
                cmd = 'boot system ' + folder + '/packages.conf'
            self.ctx.send(cmd, wait_for_string=prompt)

        self.ctx.send('end', wait_for_string=prompt)

        cmd = "write memory"
        install_activate_write_memory(self.ctx, cmd, self.ctx._connection.hostname)
        # self.ctx.send(cmd, timeout=300, wait_for_string=prompt)

        # Start activation
        if mode == 'issu':
            cmd = 'request platform software package install node file ' + \
                  folder + '/' + pkg + ' interface-module-delay 160'
            install_activate_issu(self.ctx, cmd)

            # Remove all-in-one image from the installed folder
            package = folder + '/' + pkg
            remove_exist_image(self.ctx, package)
            package = 'stby-' + package
            remove_exist_image(self.ctx, package)

            # Remove the all-in-one image from stby-bootflash:
            package = 'stby-bootflash:' + pkg
            remove_exist_image(self.ctx, package)
        else:
            install_activate_reload(self.ctx)

        self.ctx.info("Activate package done")

        # Refresh package information
        get_package(self.ctx)

        update_device_info_udi(self.ctx)

        # Verify the version after activation
        if self.ctx._connection.os_version not in pkg:
            self.ctx.error('The post-activate OS Version: {}'.self.ctx._connection.os_version)

        # Verify the status of RP and SIP
        previous_data, timestamp = self.ctx.load_data('xe_show_platform')
        self.ctx.info("Pre-Activate data collected on {}".format(
            datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')))
        if timestamp < time() - (60 * 60 * 2):  # two hours
            self.ctx.warning("Pre-Activate phase 'show platform' "
                             "data older than 2 hours")

        current_data = xe_show_platform(self.ctx)
        if not current_data:
            self.ctx.error("The CLI 'show platform' is not able to determine the status of RP and SIP ")
            return

        for Slot, Status in previous_data.items():
            Type = Status[0]
            previous_state = Status[1]
            current_state = current_data[Slot][1]
            if previous_state != current_state:
                if previous_state == 'ok, active' and current_state == 'ok, standby' \
                        or previous_state == 'ok, standby' and current_state == 'ok, active':
                    continue
                self.ctx.warning("Slot {} Type {} state changes after Activation".format(Slot, Type))
                self.ctx.warning("\t Pre-Activate State = {} vs. Post-Activate State = "
                                 "{}".format(previous_state, current_state))
            if 'ok' not in current_state:
                self.ctx.warning("Slot {} Type {} is not in 'ok' state after "
                                 "activation".format(Slot, Type))

        self.ctx.info("The status of RP and SIP has been verified. Please check any warnings in plugins.log")
<|endoftext|>


<commit_before>__guide__ = 'aaronweaver'
__author__ = 'Rajarshi333'

from defusedxml import ElementTree
from dateutil import parser
import ntpath
import re
from dojo.utils import add_language
from dojo.models import Finding

class FortifyXMLParser(object):
    language_list = []
    def __init__(self, filename, test):
        Fortifyscan = ElementTree.parse(filename)
        root = Fortifyscan.getroot()

        ##Get Date##
        date_string = root.getchildren()[5].getchildren()[1].getchildren()[2].text
        date_list = date_string.split()[1:4]
        date_act = "".join(date_list)
        find_date = parser.parse(date_act)

        ##Get Language##
        lang_string = root[8][4][2].text
        lang_need_string = re.findall("^.*com.fortify.sca.Phase0HigherOrder.Languages.*$",lang_string,re.MULTILINE)
        lang_my_string = lang_need_string[0]
        language = lang_my_string.split('=')[1]
        if language not in self.language_list:
            self.language_list.append(language)

        ##Get Finding Details##
        dupes = dict()

        for ReportSection in root.findall('ReportSection'):
            if ReportSection.findtext('Title')=="Results Outline":
                kingdom = ''
                category = ''
                mitigation = 'N/A'
                impact = 'N/A'
                references = ''
                findingdetail = ''
                title = ''
                filename = ''
                cwe = ''
                filepath = ''
                linestart = ''
                dupe_key = ''
                filename = ''
                snipet = ''
                linestart = ''

            for GroupingSection in ReportSection.iter('GroupingSection'):
                for groupTitle in GroupingSection.iter('groupTitle'):
                    grouptitle = groupTitle.text
                cwe_id = grouptitle.split(' ')
                if len(cwe_id) > 2:
                    cwe_id = cwe_id[2]
                    if "," in cwe_id:
                        cwe_id = cwe_id[:1]
                else:
                    cwe_id = 0

                for Friority in GroupingSection.iter('Friority'):
                    sev = Friority.text

                for Category in GroupingSection.iter('Category'):
				    category = Category.text

                for Kingdom in GroupingSection.iter('Kingdom'):
                    kingdom = Kingdom.text

                for LineStart in GroupingSection.iter('LineStart'):
                    linestart=LineStart.text
                    if linestart is not None:
                        findingdetail += "**Line Start:**" + linestart +'\n'

                for Snippet in GroupingSection.iter('Snippet'):
                    snippet=Snippet.text
                    if snippet is not None:
				        findingdetail += "\n**Code:**\n'''\n" + snippet +"\n'''\n"

                    for FileName in GroupingSection.iter('FileName'):
    				    filename = FileName.text
    				    if filename is not None:
    					    findingdetail += "**FileName:**" + filename +'\n'
                    for FilePath in GroupingSection.iter('FilePath'):
                        filepath=FilePath.text
                        if filepath is not None:
                            findingdetail += "**Filepath:**" + filepath +'\n'

                    title = category + " " + kingdom
                    dupe_key = (title + sev + cwe_id)

                    if dupe_key in dupes:
                        find = dupes[dupe_key]
                    else:
                        dupes[dupe_key]=True
                        find = Finding(title=title,
                                        cwe=cwe_id,
                                        test=test,
                                        active=False,
                                        verified=False,
                						description=findingdetail,
                						severity=sev,
                                        numerical_severity=Finding.get_numerical_severity(sev),
                                        mitigation=mitigation,
                                        impact=impact,
                                        references=references,
                                        file_path=filepath,
                                        line=linestart,
                                        url='N/A',
                                        date=find_date,
                                        static_finding=True)
                    dupes[dupe_key] = find
                    findingdetail = ''

        self.items = dupes.values()
<commit_msg>flake8<commit_after>__guide__ = 'aaronweaver'
__author__ = 'Rajarshi333'

from defusedxml import ElementTree
from dateutil import parser
import re
from dojo.models import Finding

class FortifyXMLParser(object):
    language_list = []

    def __init__(self, filename, test):
        Fortifyscan = ElementTree.parse(filename)
        root = Fortifyscan.getroot()

        # Get Date
        date_string = root.getchildren()[5].getchildren()[1].getchildren()[2].text
        date_list = date_string.split()[1:4]
        date_act = "".join(date_list)
        find_date = parser.parse(date_act)

        # Get Language
        lang_string = root[8][4][2].text
        lang_need_string = re.findall("^.*com.fortify.sca.Phase0HigherOrder.Languages.*$", lang_string, re.MULTILINE)
        lang_my_string = lang_need_string[0]
        language = lang_my_string.split('=')[1]
        if language not in self.language_list:
            self.language_list.append(language)

        # Get Finding Details
        dupes = dict()

        for ReportSection in root.findall('ReportSection'):
            if ReportSection.findtext('Title') == "Results Outline":
                kingdom = ''
                category = ''
                mitigation = 'N/A'
                impact = 'N/A'
                references = ''
                findingdetail = ''
                title = ''
                filename = ''
                filepath = ''
                linestart = ''
                dupe_key = ''
                filename = ''
                linestart = ''

            for GroupingSection in ReportSection.iter('GroupingSection'):
                for groupTitle in GroupingSection.iter('groupTitle'):
                    grouptitle = groupTitle.text
                cwe_id = grouptitle.split(' ')
                if len(cwe_id) > 2:
                    cwe_id = cwe_id[2]
                    if "," in cwe_id:
                        cwe_id = cwe_id[:1]
                else:
                    cwe_id = 0

                for Friority in GroupingSection.iter('Friority'):
                    sev = Friority.text

                for Category in GroupingSection.iter('Category'):
                    category = Category.text

                for Kingdom in GroupingSection.iter('Kingdom'):
                    kingdom = Kingdom.text

                for LineStart in GroupingSection.iter('LineStart'):
                    linestart = LineStart.text
                    if linestart is not None:
                        findingdetail += "**Line Start:**" + linestart + '\n'

                for Snippet in GroupingSection.iter('Snippet'):
                    snippet = Snippet.text
                    if snippet is not None:
                        findingdetail += "\n**Code:**\n'''\n" + snippet + "\n\n"

                    for FileName in GroupingSection.iter('FileName'):
                        filename = FileName.text
                        if filename is not None:
                            findingdetail += "**FileName:**" + filename + '\n'
                    for FilePath in GroupingSection.iter('FilePath'):
                        filepath = FilePath.text
                        if filepath is not None:
                            findingdetail += "**Filepath:**" + filepath + '\n'

                    title = category + " " + kingdom
                    dupe_key = (title + sev + cwe_id)

                    if dupe_key in dupes:
                        find = dupes[dupe_key]
                    else:
                        dupes[dupe_key] = True
                        find = Finding(title=title,
                                        cwe=cwe_id,
                                        test=test,
                                        active=False,
                                        verified=False,
                                        description=findingdetail,
                                        severity=sev,
                                        numerical_severity=Finding.get_numerical_severity(sev),
                                        mitigation=mitigation,
                                        impact=impact,
                                        references=references,
                                        file_path=filepath,
                                        line=linestart,
                                        url='N/A',
                                        date=find_date,
                                        static_finding=True)
                    dupes[dupe_key] = find
                    findingdetail = ''

        self.items = dupes.values()
<|endoftext|>


<commit_before>import inspect
from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db import connections

import attr
import dateutil

from dimagi.utils.chunked import chunked

from corehq.apps.hqadmin.management.commands.stale_data_in_es import (
    get_sql_case_data_for_db,
)
from corehq.apps.userreports.util import get_table_name
from corehq.form_processor.models import XFormInstanceSQL
from corehq.form_processor.utils import should_use_sql_backend
from corehq.sql_db.connections import get_icds_ucr_citus_db_alias
from corehq.sql_db.util import get_db_aliases_for_partitioned_query

@attr.s
class RunConfig(object):
    domain = attr.ib()
    table_id = attr.ib()
    start_date = attr.ib()
    end_date = attr.ib()
    case_type = attr.ib()
    xmlns = attr.ib()

    def run_with_forms(self):
        return bool(self.xmlns)

class Command(BaseCommand):
    """
    Returns list of (doc_id, doc_type, doc_subtype, ucr_insert_on, modified_on)
    tuples for all househould cases that are not found static-household_cases UCR.

    Can be used in conjunction with republish_doc_changes

        1. Generate tuples not updated in ES with extra debug columns
        $ ./manage.py stale_data_in_househould_ucr <DOMAIN> --start 2019-09-19 --end 2019-09-28 > stale_ids.txt

        2. Republish case changes
        $ ./manage.py republish_doc_changes <DOMAIN> stale_ids.txt
    """
    help = inspect.cleandoc(__doc__).split('\n')[0]

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument('table_id')
        parser.add_argument(
            '--case-type',
            action='store',
            help='Mutually exclusive with XMLNS',
        )
        parser.add_argument(
            '--xmlns',
            action='store',
            help='Mutually exclusive with case_type',
        )
        parser.add_argument(
            '--start',
            action='store',
            help='Only include data modified after this date',
        )
        parser.add_argument(
            '--end',
            action='store',
            help='Only include data modified before this date',
        )

    def handle(self, domain, table_id, **options):
        if options['case_type'] and options['xmlns']:
            raise CommandError('You may only specify one of case_type or XMLNS')

        start = dateutil.parser.parse(options['start']) if options['start'] else datetime(2010, 1, 1)
        end = dateutil.parser.parse(options['end']) if options['end'] else datetime.utcnow()

        run_config = RunConfig(domain, table_id, start, end, options['case_type'], options['xmlns'])
        if not should_use_sql_backend(run_config.domain):
            raise CommandError('This command only supports SQL domains.')

        for doc_id, doc_type, ucr_date, primary_date in _get_stale_data(run_config):
            print(f"{doc_id},{doc_type},{ucr_date},{primary_date}")

def _get_stale_data(run_config):
    for db in get_db_aliases_for_partitioned_query():
        matching_records_for_db = get_sql_case_data_for_db(db, run_config)
        chunk_size = 1000
        for chunk in chunked(matching_records_for_db, chunk_size):
            doc_ids = [val[0] for val in chunk]
            ucr_insertion_dates = _get_ucr_insertion_dates(run_config.domain, run_config.table_id, doc_ids)
            for doc_id, doc_type, sql_modified_on in chunk:
                ucr_insert_date = ucr_insertion_dates.get(doc_id)
                if not ucr_insert_date or (ucr_insert_date < sql_modified_on):
                    ucr_date_string = ucr_insert_date.isoformat() if ucr_insert_date else ''
                    yield (doc_id, doc_type, ucr_date_string, sql_modified_on.isoformat())

def _get_ucr_insertion_dates(domain, table_id, doc_ids):
    table_name = get_table_name(domain, table_id)
    with connections[get_icds_ucr_citus_db_alias()].cursor() as cursor:
        query = f'''
            SELECT
                doc_id,
                inserted_at
            FROM "{table_name}"
            WHERE doc_id = ANY(%(doc_ids)s);
        '''
        cursor.execute(query, {'doc_ids': doc_ids})
        return dict(cursor.fetchall())

def _get_primary_data_for_db(run_config, db):
    if run_config.run_with_forms:
        return XFormInstanceSQL.objects.using(db).filter(
            domain=run_config.domain,
            received_on__gte=run_config.start_date,
            received_on__lte=run_config.end_date,
        ).values_list('form_id', 'xmlns', 'received_on')
    else:
        return get_sql_case_data_for_db(db, run_config)
<commit_msg>Filter by XMLNS<commit_after>import inspect
from datetime import datetime

from django.core.management.base import BaseCommand, CommandError
from django.db import connections

import attr
import dateutil

from dimagi.utils.chunked import chunked

from corehq.apps.hqadmin.management.commands.stale_data_in_es import (
    get_sql_case_data_for_db,
)
from corehq.apps.userreports.util import get_table_name
from corehq.form_processor.models import XFormInstanceSQL
from corehq.form_processor.utils import should_use_sql_backend
from corehq.sql_db.connections import get_icds_ucr_citus_db_alias
from corehq.sql_db.util import get_db_aliases_for_partitioned_query

@attr.s
class RunConfig(object):
    domain = attr.ib()
    table_id = attr.ib()
    start_date = attr.ib()
    end_date = attr.ib()
    case_type = attr.ib()
    xmlns = attr.ib()

    def run_with_forms(self):
        return bool(self.xmlns)

class Command(BaseCommand):
    """
    Returns list of (doc_id, doc_type, doc_subtype, ucr_insert_on, modified_on)
    tuples for all househould cases that are not found static-household_cases UCR.

    Can be used in conjunction with republish_doc_changes

        1. Generate tuples not updated in ES with extra debug columns
        $ ./manage.py stale_data_in_househould_ucr <DOMAIN> --start 2019-09-19 --end 2019-09-28 > stale_ids.txt

        2. Republish case changes
        $ ./manage.py republish_doc_changes <DOMAIN> stale_ids.txt
    """
    help = inspect.cleandoc(__doc__).split('\n')[0]

    def add_arguments(self, parser):
        parser.add_argument('domain')
        parser.add_argument('table_id')
        parser.add_argument(
            '--case-type',
            action='store',
            help='Mutually exclusive with XMLNS',
        )
        parser.add_argument(
            '--xmlns',
            action='store',
            help='Mutually exclusive with case_type',
        )
        parser.add_argument(
            '--start',
            action='store',
            help='Only include data modified after this date',
        )
        parser.add_argument(
            '--end',
            action='store',
            help='Only include data modified before this date',
        )

    def handle(self, domain, table_id, **options):
        if options['case_type'] and options['xmlns']:
            raise CommandError('You may only specify one of case_type or XMLNS')

        start = dateutil.parser.parse(options['start']) if options['start'] else datetime(2010, 1, 1)
        end = dateutil.parser.parse(options['end']) if options['end'] else datetime.utcnow()

        run_config = RunConfig(domain, table_id, start, end, options['case_type'], options['xmlns'])
        if not should_use_sql_backend(run_config.domain):
            raise CommandError('This command only supports SQL domains.')

        for doc_id, doc_type, ucr_date, primary_date in _get_stale_data(run_config):
            print(f"{doc_id},{doc_type},{ucr_date},{primary_date}")

def _get_stale_data(run_config):
    for db in get_db_aliases_for_partitioned_query():
        matching_records_for_db = get_sql_case_data_for_db(db, run_config)
        chunk_size = 1000
        for chunk in chunked(matching_records_for_db, chunk_size):
            doc_ids = [val[0] for val in chunk]
            ucr_insertion_dates = _get_ucr_insertion_dates(run_config.domain, run_config.table_id, doc_ids)
            for doc_id, doc_type, sql_modified_on in chunk:
                ucr_insert_date = ucr_insertion_dates.get(doc_id)
                if not ucr_insert_date or (ucr_insert_date < sql_modified_on):
                    ucr_date_string = ucr_insert_date.isoformat() if ucr_insert_date else ''
                    yield (doc_id, doc_type, ucr_date_string, sql_modified_on.isoformat())

def _get_ucr_insertion_dates(domain, table_id, doc_ids):
    table_name = get_table_name(domain, table_id)
    with connections[get_icds_ucr_citus_db_alias()].cursor() as cursor:
        query = f'''
            SELECT
                doc_id,
                inserted_at
            FROM "{table_name}"
            WHERE doc_id = ANY(%(doc_ids)s);
        '''
        cursor.execute(query, {'doc_ids': doc_ids})
        return dict(cursor.fetchall())

def _get_primary_data_for_db(run_config, db):
    if run_config.run_with_forms:
        matching_xforms = XFormInstanceSQL.objects.using(db).filter(
            domain=run_config.domain,
            received_on__gte=run_config.start_date,
            received_on__lte=run_config.end_date,
        )
        if run_config.xmlns:
            matching_xforms = matching_xforms.filter(xmlns=run_config.xmlns)
        return matching_xforms.values_list('form_id', 'xmlns', 'received_on')
    else:
        return get_sql_case_data_for_db(db, run_config)
<|endoftext|>


<commit_before>##   Copyright 2016 Sayak Brahmachari
##
##   Licensed under the Apache License, Version 2.0 (the "License");
##   you may not use this file except in compliance with the License.
##   You may obtain a copy of the License at
##
##       http://www.apache.org/licenses/LICENSE-2.0
##
##   Unless required by applicable law or agreed to in writing, software
##   distributed under the License is distributed on an "AS IS" BASIS,
##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
##   See the License for the specific language governing permissions and
##   limitations under the License.

class AddWords:
    ''' Provides Built-in Words for the struixLang Interpreter. '''
    def __init__(self, terp):
        wordSets = ['output', 'control', 'math', 'stack', 'variables']
        for wordSet in wordSets:
            terp.addWords(eval('self.words4{}()'.format(wordSet)))

    @staticmethod
    def words4output():
        ''' Provides Words for output operations. '''
        def PRINT(terp):
            ''' Pops & Displays the Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            print(terp.stack.pop())
        def PSTACK(terp):
            ''' Displays the complete stack. '''
            print("\n".join(str(val) for val in terp.stack))
        return {
            "PRINT":  PRINT,
            "PSTACK": PSTACK
            }

    @staticmethod
    def words4control():
        ''' Provides Words for controlling execution. '''
        def EXIT(terp):
            ''' Terminates the execution. '''
            exit()
        return {
            "EXIT": EXIT
            }

    @staticmethod
    def words4math():
        ''' Provides Words for several mathematical and logical operations. '''
        def CALCGEN(op):
            ''' Generates Words for a specefic operation. '''
            def CALC(terp):
                if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
                terp.stack.append(eval('terp.stack.pop(){}terp.stack.pop()'.format(op)))
            return CALC
        ops = ['+', '-', '*', '/', '%', '//', '**', '|', '^', '&', '<<', '>>']
        return dict(zip(ops, [CALCGEN(op) for op in ops]))

    @staticmethod
    def words4stack():
        ''' Provides Words for Stack Operations. '''
        def DUP(terp):
            ''' Duplicate Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            terp.stack.append(terp.stack[-1])
        def DROP(terp):
            ''' Remove Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            terp.stack.pop()
        def SWAP(terp):
            ''' Exchange positions of ToS and second item on stack (2oS). '''
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            terp.stack.append(tos)
            terp.stack.append(_2os)
        def OVER(terp):
            ''' Copy 2oS on top of stack. '''
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            terp.stack.append(_2os)
            terp.stack.append(tos)
            terp.stack.append(_2os)
        def ROT(terp):
            ''' Copy 3oS on top of stack. '''
            if terp.stack.__len__() < 3: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            _3os = terp.stack.pop()
            terp.stack.append(_2os)
            terp.stack.append(tos)
            terp.stack.append(_3os)
        return {
            "DUP":  DUP,
            "DROP": DROP,
            "SWAP": SWAP,
            "OVER": OVER,
            "ROT":  ROT
            }
    @staticmethod
    def words4variables():
        class Variable:
            def __init__(self, val=None):
                self.value = val
            def access(self, terp):
                terp.stack.append(self)
        class Constant:
            def __init__(self, val):
                import random
                self.__vHjp3nfr6N3JJs9y = random.uniform(-1000, 0)
                self.__vHjp3nG4qN3JJs9y = random.randint(-100, 100)
                self.__vH5WKNf5y96k3a9y = random.randint(0, 1000)
                self.__vHjp3nG4qN3JJs9y = val
                self.__vHAKUW42nQaCpm9y = random.uniform(-1000, 0)
                self.__vH5WKNG4y96k3a9y = random.uniform(-100, 100)
                self.__vHjp3nfr6N3JJs9y = random.uniform(0, 1000)
                self.__dict__['_Constant__vHjp3n4GqN3JJs9y'] = self.__dict__.pop('_Constant__vHjp3nG4qN3JJs9y')
            def access(self, terp):
                self.__dict__['_Constant__vHjp3nfr6N3JJs9y'] = self.__dict__.pop('_Constant__vHjp3n4GqN3JJs9y')
                terp.stack.append(self.__vHjp3nfr6N3JJs9y)
        def VAR(terp):
            name = terp.lexer.nextWord()
            if name is None: raise SyntaxError('Invalid Syntax')
            var = Variable()
            terp.define(name, var.access)
        def CONST(terp):
            name = terp.lexer.nextWord()
            val = terp.lexer.nextWord()
            if name is None or val is None: raise SyntaxError('Invalid Syntax')
            const = Constant(val)
            terp.define(name, const.access)
        def STORE(terp):
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            val = terp.stack.pop()
            ref = terp.stack.pop()
            ref.value = val
        def FETCH(terp):
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            ref = terp.stack.pop()
            terp.stack.append(ref.value)
        return {
            "VAR":   VAR,
            "CONST": CONST,
            "STORE": STORE,
            "FETCH": FETCH
            }
<commit_msg>Made Constants difficult to change<commit_after>##   Copyright 2016 Sayak Brahmachari
##
##   Licensed under the Apache License, Version 2.0 (the "License");
##   you may not use this file except in compliance with the License.
##   You may obtain a copy of the License at
##
##       http://www.apache.org/licenses/LICENSE-2.0
##
##   Unless required by applicable law or agreed to in writing, software
##   distributed under the License is distributed on an "AS IS" BASIS,
##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
##   See the License for the specific language governing permissions and
##   limitations under the License.

class AddWords:
    ''' Provides Built-in Words for the struixLang Interpreter. '''
    def __init__(self, terp):
        wordSets = ['output', 'control', 'math', 'stack', 'variables']
        for wordSet in wordSets:
            terp.addWords(eval('self.words4{}()'.format(wordSet)))

    @staticmethod
    def words4output():
        ''' Provides Words for output operations. '''
        def PRINT(terp):
            ''' Pops & Displays the Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            print(terp.stack.pop())
        def PSTACK(terp):
            ''' Displays the complete stack. '''
            print("\n".join(str(val) for val in terp.stack))
        return {
            "PRINT":  PRINT,
            "PSTACK": PSTACK
            }

    @staticmethod
    def words4control():
        ''' Provides Words for controlling execution. '''
        def EXIT(terp):
            ''' Terminates the execution. '''
            exit()
        return {
            "EXIT": EXIT
            }

    @staticmethod
    def words4math():
        ''' Provides Words for several mathematical and logical operations. '''
        def CALCGEN(op):
            ''' Generates Words for a specefic operation. '''
            def CALC(terp):
                if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
                terp.stack.append(eval('terp.stack.pop(){}terp.stack.pop()'.format(op)))
            return CALC
        ops = ['+', '-', '*', '/', '%', '//', '**', '|', '^', '&', '<<', '>>']
        return dict(zip(ops, [CALCGEN(op) for op in ops]))

    @staticmethod
    def words4stack():
        ''' Provides Words for Stack Operations. '''
        def DUP(terp):
            ''' Duplicate Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            terp.stack.append(terp.stack[-1])
        def DROP(terp):
            ''' Remove Top of Stack (ToS). '''
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            terp.stack.pop()
        def SWAP(terp):
            ''' Exchange positions of ToS and second item on stack (2oS). '''
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            terp.stack.append(tos)
            terp.stack.append(_2os)
        def OVER(terp):
            ''' Copy 2oS on top of stack. '''
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            terp.stack.append(_2os)
            terp.stack.append(tos)
            terp.stack.append(_2os)
        def ROT(terp):
            ''' Copy 3oS on top of stack. '''
            if terp.stack.__len__() < 3: raise IndexError('Not enough items on stack')
            tos = terp.stack.pop()
            _2os = terp.stack.pop()
            _3os = terp.stack.pop()
            terp.stack.append(_2os)
            terp.stack.append(tos)
            terp.stack.append(_3os)
        return {
            "DUP":  DUP,
            "DROP": DROP,
            "SWAP": SWAP,
            "OVER": OVER,
            "ROT":  ROT
            }
    @staticmethod
    def words4variables():
        class Variable:
            def __init__(self, val=None):
                self.value = val
            def access(self, terp):
                terp.stack.append(self)
        class Constant:
            def __init__(self, val):
                import random
                self.__vHjp3nfr6N3JJs9y = random.randint(-1000, 0)
                self.__vHjp3nG4qN3JJs9y = random.uniform(-100, 100)
                self.__vH5WKNf5y96k3a9y = random.randint(0, 1000)
                self.__vHjp3nG4qN3JJs9y = val
                self.__vHAKUW42nQaCpm9y = random.uniform(-1000, 0)
                self.__vH5WKNG4y96k3a9y = random.randint(-100, 100)
                self.__vHjp3nfr6NEJJs9y = random.uniform(0, 1000)
                self.__dict__['_Constant__vHjp3n4GqN3JJs9y'] = self.__dict__.pop('_Constant__vHjp3nG4qN3JJs9y')
            def access(self, terp):
                import random, string
                var = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))
                self.__dict__['_Constant__{}'.format(var)] = self.__dict__.pop('_Constant__vHjp3n4GqN3JJs9y')
                terp.stack.append(eval('self._Constant__{}'.format(var)))
                self.__dict__['_Constant__vHjp3n4GqN3JJs9y'] = self.__dict__.pop('_Constant__{}'.format(var))
        def VAR(terp):
            name = terp.lexer.nextWord()
            if name is None: raise SyntaxError('Invalid Syntax')
            var = Variable()
            terp.define(name, var.access)
        def CONST(terp):
            name = terp.lexer.nextWord()
            val = terp.lexer.nextWord()
            if name is None or val is None: raise SyntaxError('Invalid Syntax')
            const = Constant(val)
            terp.define(name, const.access)
        def STORE(terp):
            if terp.stack.__len__() < 2: raise IndexError('Not enough items on stack')
            val = terp.stack.pop()
            ref = terp.stack.pop()
            ref.value = val
        def FETCH(terp):
            if terp.stack.__len__() < 1: raise IndexError('Not enough items on stack')
            ref = terp.stack.pop()
            terp.stack.append(ref.value)
        return {
            "VAR":   VAR,
            "CONST": CONST,
            "STORE": STORE,
            "FETCH": FETCH
            }
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

from dateutil.relativedelta import relativedelta
from custom.icds_reports.const import AGG_LS_VHND_TABLE, AGG_LS_AWC_VISIT_TABLE, AGG_LS_BENEFICIARY_TABLE
from corehq.apps.userreports.models import StaticDataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.util import get_table_name
from custom.icds_reports.utils.aggregation_helpers import BaseICDSAggregationHelper, transform_day_to_month, \
month_formatter

class AggLsHelper(BaseICDSAggregationHelper):

    base_tablename = 'agg_ls'
    awc_location_ucr = 'static-awc_location'

    def __init__(self, month):
        self.month_start = transform_day_to_month(month)
        self.next_month_start = self.month_start + relativedelta(months=1)

    def _tablename_func(self, agg_level):
        return "{}_{}_{}".format(self.base_tablename, self.month_start.strftime("%Y-%m-%d"), agg_level)

    def drop_table_if_exists(self, agg_level):
        return """
        DROP TABLE IF EXISTS "{table_name}"
        """.format(table_name=self._tablename_func(agg_level))

    def create_child_table(self, agg_level):
        return """
        CREATE TABLE "{table_name}" (
        CHECK (month=DATE %(start_date)s AND aggregation_level={agg_level})
        ) INHERITS ({base_tablename})
        """.format(
            table_name=self._tablename_func(agg_level),
            base_tablename=self.base_tablename,
            agg_level=agg_level
        ), {
            "start_date": self.month_start
        }

    @property
    def tablename(self):
        return self._tablename_func(4)

    def _ucr_tablename(self, ucr_id):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, ucr_id)
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    def aggregate_query(self):
        """
        Returns the base aggregate query which is used to insert all the locations
        into the LS data table.
        """

        columns = (
            ('state_id', 'location.state_id'),
            ('district_id', 'location.district_id'),
            ('block_id', 'location.block_id'),
            ('supervisor_id', 'location.supervisor_id'),
            ('month', month_formatter(self.month_start)),
            ('unique_awc_vists', 'sum(awc_table.unique_awc_vists)'),
            ('vhnd_observed', 'sum(vhnd_table.vhnd_observed)'),
            ('beneficiary_vists', 'sum(beneficiary_table.beneficiary_vists)'),
            ('aggregation_level', '5')
        )
        return """
        INSERT INTO "{tablename}" (
            {columns}
        )
        (
        SELECT
        {calculations}
        from "{awc_location_ucr}" location
        LEFT OUTER JOIN "{awc_table}" awc_table on location.supervisor_id=awc_table.supervisor_id
        LEFT OUTER JOIN "{vhnd_table}" vhnd_table on (
            awc_table.supervisor_id = vhnd_table.supervisor_id AND
            awc_table.month = vhnd_table.month
        )
        LEFT OUTER JOIN "{beneficiary_table}" beneficiary_table on (
        vhnd_table.supervisor_id = beneficiary_table.supervisor_id AND
        vhnd_table.month = beneficiary_table.month
        )
        WHERE awc_table.month = %(start_date)s
        GROUP BY location.state_id, location.district_id, location.block_id, location.supervisor_id
        )
        """.format(
            tablename=self.tablename,
            columns=", ".join([col[0] for col in columns]),
            calculations=", ".join([col[1] for col in columns]),
            awc_location_ucr=self._ucr_tablename(ucr_id=self.awc_location_ucr),
            awc_table=AGG_LS_AWC_VISIT_TABLE,
            vhnd_table=AGG_LS_VHND_TABLE,
            beneficiary_table=AGG_LS_BENEFICIARY_TABLE
        ), {
            'start_date': self.month_start
        }

    def indexes(self, aggregation_level):
        """
        Returns queries to create indices  for columns
        district_id, block_id, supervisor_id and state_id based on
        aggregation level
        """
        indexes = []
        agg_locations = ['state_id']
        if aggregation_level > 1:
            indexes.append('CREATE INDEX ON "{}" (district_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('district_id')
        if aggregation_level > 2:
            indexes.append('CREATE INDEX ON "{}" (block_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('block_id')
        if aggregation_level > 3:
            indexes.append('CREATE INDEX ON "{}" (supervisor_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('supervisor_id')

        indexes.append('CREATE INDEX ON "{}" ({})'.format(self._tablename_func(aggregation_level),
                                                          ', '.join(agg_locations)))
        return indexes

    def rollup_query(self, agg_level):
        """
        Returns the roll up query to the agg_level passed as argument.
        Roll up query is used to roll up the data from supervisor level
        to block level to district level to state level
        """
        locations = ['state_id', 'district_id', 'block_id', 'supervisor_id']

        for i in range(3, agg_level - 1, -1):
            locations[i] = "'All'"

        return """
            INSERT INTO "{to_table}" (
            vhnd_observed,
            beneficiary_vists,
            unique_awc_vists,
            aggregation_level,
            state_id,
            district_id,
            block_id,
            supervisor_id,
            month)
            (
                SELECT
                sum(vhnd_observed) as vhnd_observed,
                sum(beneficiary_vists) as beneficiary_vists,
                sum(unique_awc_vists) as unique_awc_vists,
                {agg_level},
                {locations},
                month
                FROM "{from_table}"
                GROUP BY {group_by}, month
            )
        """.format(
            agg_level=agg_level,
            to_table=self._tablename_func(agg_level),
            locations=','.join(locations),
            from_table=self._tablename_func(agg_level + 1),
            group_by=','.join(locations[:agg_level])
        )
<commit_msg>Use aggregation level 4 as base<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

from dateutil.relativedelta import relativedelta
from custom.icds_reports.const import AGG_LS_VHND_TABLE, AGG_LS_AWC_VISIT_TABLE, AGG_LS_BENEFICIARY_TABLE
from corehq.apps.userreports.models import StaticDataSourceConfiguration, get_datasource_config
from corehq.apps.userreports.util import get_table_name
from custom.icds_reports.utils.aggregation_helpers import BaseICDSAggregationHelper, transform_day_to_month, \
month_formatter

class AggLsHelper(BaseICDSAggregationHelper):

    base_tablename = 'agg_ls'
    awc_location_ucr = 'static-awc_location'

    def __init__(self, month):
        self.month_start = transform_day_to_month(month)
        self.next_month_start = self.month_start + relativedelta(months=1)

    def _tablename_func(self, agg_level):
        return "{}_{}_{}".format(self.base_tablename, self.month_start.strftime("%Y-%m-%d"), agg_level)

    def drop_table_if_exists(self, agg_level):
        return """
        DROP TABLE IF EXISTS "{table_name}"
        """.format(table_name=self._tablename_func(agg_level))

    def create_child_table(self, agg_level):
        return """
        CREATE TABLE "{table_name}" (
        CHECK (month=DATE %(start_date)s AND aggregation_level={agg_level})
        ) INHERITS ({base_tablename})
        """.format(
            table_name=self._tablename_func(agg_level),
            base_tablename=self.base_tablename,
            agg_level=agg_level
        ), {
            "start_date": self.month_start
        }

    @property
    def tablename(self):
        return self._tablename_func(4)

    def _ucr_tablename(self, ucr_id):
        doc_id = StaticDataSourceConfiguration.get_doc_id(self.domain, ucr_id)
        config, _ = get_datasource_config(doc_id, self.domain)
        return get_table_name(self.domain, config.table_id)

    def aggregate_query(self):
        """
        Returns the base aggregate query which is used to insert all the locations
        into the LS data table.
        """

        columns = (
            ('state_id', 'location.state_id'),
            ('district_id', 'location.district_id'),
            ('block_id', 'location.block_id'),
            ('supervisor_id', 'location.supervisor_id'),
            ('month', "'{}'".format(month_formatter(self.month_start))),
            ('unique_awc_vists', 'sum(awc_table.unique_awc_vists)'),
            ('vhnd_observed', 'sum(vhnd_table.vhnd_observed)'),
            ('beneficiary_vists', 'sum(beneficiary_table.beneficiary_vists)'),
            ('aggregation_level', '4')
        )
        return """
        INSERT INTO "{tablename}" (
            {columns}
        )
        (
        SELECT
        {calculations}
        from "{awc_location_ucr}" location
        LEFT OUTER JOIN "{awc_table}" awc_table on location.supervisor_id=awc_table.supervisor_id
        LEFT OUTER JOIN "{vhnd_table}" vhnd_table on (
            awc_table.supervisor_id = vhnd_table.supervisor_id AND
            awc_table.month = vhnd_table.month
        )
        LEFT OUTER JOIN "{beneficiary_table}" beneficiary_table on (
        vhnd_table.supervisor_id = beneficiary_table.supervisor_id AND
        vhnd_table.month = beneficiary_table.month
        )
        WHERE awc_table.month = %(start_date)s
        GROUP BY location.state_id, location.district_id, location.block_id, location.supervisor_id
        )
        """.format(
            tablename=self.tablename,
            columns=", ".join([col[0] for col in columns]),
            calculations=", ".join([col[1] for col in columns]),
            awc_location_ucr=self._ucr_tablename(ucr_id=self.awc_location_ucr),
            awc_table=AGG_LS_AWC_VISIT_TABLE,
            vhnd_table=AGG_LS_VHND_TABLE,
            beneficiary_table=AGG_LS_BENEFICIARY_TABLE
        ), {
            'start_date': self.month_start
        }

    def indexes(self, aggregation_level):
        """
        Returns queries to create indices  for columns
        district_id, block_id, supervisor_id and state_id based on
        aggregation level
        """
        indexes = []
        agg_locations = ['state_id']
        if aggregation_level > 1:
            indexes.append('CREATE INDEX ON "{}" (district_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('district_id')
        if aggregation_level > 2:
            indexes.append('CREATE INDEX ON "{}" (block_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('block_id')
        if aggregation_level > 3:
            indexes.append('CREATE INDEX ON "{}" (supervisor_id)'.format(self._tablename_func(aggregation_level)))
            agg_locations.append('supervisor_id')

        indexes.append('CREATE INDEX ON "{}" ({})'.format(self._tablename_func(aggregation_level),
                                                          ', '.join(agg_locations)))
        return indexes

    def rollup_query(self, agg_level):
        """
        Returns the roll up query to the agg_level passed as argument.
        Roll up query is used to roll up the data from supervisor level
        to block level to district level to state level
        """
        locations = ['state_id', 'district_id', 'block_id', 'supervisor_id']

        for i in range(3, agg_level - 1, -1):
            locations[i] = "'All'"

        return """
            INSERT INTO "{to_table}" (
            vhnd_observed,
            beneficiary_vists,
            unique_awc_vists,
            aggregation_level,
            state_id,
            district_id,
            block_id,
            supervisor_id,
            month)
            (
                SELECT
                sum(vhnd_observed) as vhnd_observed,
                sum(beneficiary_vists) as beneficiary_vists,
                sum(unique_awc_vists) as unique_awc_vists,
                {agg_level},
                {locations},
                month
                FROM "{from_table}"
                GROUP BY {group_by}, month
            )
        """.format(
            agg_level=agg_level,
            to_table=self._tablename_func(agg_level),
            locations=','.join(locations),
            from_table=self._tablename_func(agg_level + 1),
            group_by=','.join(locations[:agg_level])
        )
<|endoftext|>