<commit_before>#!/usr/bin/python
from azuremodules import *
import sys
import time
import re
import os
import linecache
import imp
import os.path
import zipfile

current_distro        = "unknown"
distro_version        = "unknown"
sudo_password        = "rdPa$$w0rd"
startup_file = ""

rpm_links = {}
tar_link = {}
current_distro = "unknown"
packages_list_xml = "./packages.xml"
python_cmd="python"
waagent_cmd="waagent"
waagent_bin_path="/usr/sbin"

def set_variables_OS_dependent():
        global current_distro
        global distro_version
        global startup_file

        RunLog.info ("\nset_variables_OS_dependent ..")
        [current_distro, distro_version] = DetectDistro()
        if(current_distro == 'unknown'):
                RunLog.error ("unknown distribution found, exiting")
                ResultLog.info('ABORTED')
                exit()
        if(current_distro == "ubuntu" or current_distro == "Debian"):
                startup_file = '/etc/rc.local'
        elif(current_distro == "centos" or current_distro == "rhel" or current_distro == "fedora" or current_distro == "Oracle"):
                startup_file = '/etc/rc.d/rc.local'
        elif(current_distro == "SUSE" or current_distro == "sles" or current_distro == "opensuse"):
                startup_file = '/etc/rc.d/after.local'

        if(current_distro == "coreos"):
                python_cmd="/usr/share/oem/python/bin/python"
                waagent_bin_path="/usr/share/oem/bin/python"
                waagent_cmd= "{0} {1}".format(python_cmd, waagent_bin_path)

        RunLog.info ("\nset_variables_OS_dependent ..[done]")

def download_and_install_rpm(package):
        RunLog.info("Installing Package: " + package+" from rpmlink")
        if package in rpm_links:
                if DownloadUrl(rpm_links.get(package), "/tmp/"):
                        if InstallRpm("/tmp/"+re.split("/",rpm_links.get(package))[-1], package):
                                RunLog.info("Installing Package: " + package+" from rpmlink done!")
                                return True

        RunLog.error("Installing Package: " + package+" from rpmlink failed!!")
        return False

def yum_package_install(package):
        if(YumPackageInstall(package) == True):
                return True
        elif(download_and_install_rpm(package) == True):
                return True
        else:
                return False

def zypper_package_install(package):
        if(ZypperPackageInstall(package) == True):
                return True
        elif(download_and_install_rpm(package) == True):
                return True
        else:
                return False

def coreos_package_install():
        binpath="/usr/share/oem/bin"
        pythonlibrary="/usr/share/oem/python/lib64/python2.7"

        # create /etc/hosts
        ExecMultiCmdsLocalSudo(["touch /etc/hosts",\
                "echo '127.0.0.1 localhost' > /etc/hosts",\
                "echo '** modify /etc/hosts successfully **' >> PackageStatus.txt"])
        # copy tools to bin folder
        Run("unzip -d CoreosPreparationTools ./CoreosPreparationTools.zip")
        ExecMultiCmdsLocalSudo(["cp ./CoreosPreparationTools/killall " + binpath, \
                "cp ./CoreosPreparationTools/iperf " + binpath,\
                "cp ./CoreosPreparationTools/iozone " + binpath,\
                "cp ./CoreosPreparationTools/dos2unix " + binpath,\
                "cp ./CoreosPreparationTools/at " + binpath,\
                "chmod 755 "+ binpath + "/*",\
                "echo '** copy tools successfully **' >> PackageStatus.txt"])
        # copy python library to python library folder
        Run("tar zxvf ./CoreosPreparationTools/pycrypto.tar.gz -C "+ pythonlibrary)
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/ecdsa-0.13.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/ecdsa-0.13",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/paramiko-1.15.1.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/paramiko-1.15.1",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../..",\
                "tar zxvf ./CoreosPreparationTools/pexpect-3.3.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/pexpect-3.3",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/dnspython-1.12.0.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/dnspython-1.12.0",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        if not os.path.exists (pythonlibrary + "/site-packages/pexpect"):
                RunLog.error ("pexpect package installation failed!")
                Run("echo '** pexpect package installation failed **' >> PackageStatus.txt")
                return False
        if not os.path.exists (pythonlibrary + "/site-packages/paramiko"):
                RunLog.error ("paramiko packages installation failed!")
                Run("echo '** paramiko packages installed failed **' >> PackageStatus.txt")
                return False
        if not os.path.exists (pythonlibrary + "/site-packages/dns"):
                RunLog.error ("dnspython packages installation failed!")
                Run("echo '** dnspython packages installed failed **' >> PackageStatus.txt")
                return False
        RunLog.info ("pexpect, paramiko and dnspython packages installed successfully!")
        Run("echo '** pexpect, paramiko and dnspython packages installed successfully **' >> PackageStatus.txt")
        return True

def install_waagent_from_github():
        RunLog.info ("Installing waagent from github...")

        DownloadUrl(tar_link.get("waagent"), "/tmp/")
        filename = tar_link.get("waagent").split('/')[-1]
        RunLog.info ("Waagent tar file name is: "+ filename+"|")

        pkgPath = os.path.join("/tmp", filename)
        unzipPath = os.path.join("/tmp", "agent")
        if not os.path.isfile(pkgPath):
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("File not found: {0}".format(pkgPath))
                return False

        try:
                zipfile.ZipFile(pkgPath).extractall(unzipPath)
        except IOError as e:
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("{0}".format(e))
                return False

        binPath20 = os.path.join(unzipPath, "waagent")
        binPath21 = os.path.join(unzipPath, "bin/waagent")
        if os.path.isfile(binPath20):
                #For 2.0, only one file(waagent) needs to be replaced.
                waagent_bin=os.path.join(unzipPath, 'waagent')
                ExecMultiCmdsLocalSudo([
                        "chmod +x {0}".format(waagent_bin),
                        "cp {0} {1}".format(waagent_bin, waagent_bin_path),
                        "rm -rf {0}".format(unzipPath)])
                return True
        elif os.path.isfile(binPath21):
                #For 2.1, use setup.py to install/uninstall package
                setup_py = os.path.join(unzipPath, 'setup.py')
                ExecMultiCmdsLocalSudo([
                        "{0} {1} -uninstall".format(python_cmd, setup_py),
                        "{0} {1} -install".format(python_cmd, setup_py),
                        "rm -rf {0}".format(unzipPath)])
                return True
        else:
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("Unknown waagent verions")
                return False

def install_package(package):
        RunLog.info ("\nInstall_package: "+package)
        if (package == "waagent"):
                return install_waagent_from_github()
        else:
                if ((current_distro == "ubuntu") or (current_distro == "Debian")):
                        return AptgetPackageInstall(package)
                elif ((current_distro == "rhel") or (current_distro == "Oracle") or (current_distro == 'centos') or (current_distro == 'fedora')):
                        return yum_package_install(package)
                elif (current_distro == "SUSE") or (current_distro == "openSUSE") or (current_distro == "sles") or (current_distro == "opensuse"):
                        return zypper_package_install(package)
                else:
                        RunLog.error (package + ": package installation failed!")
                        RunLog.info (current_distro + ": Unrecognised Distribution OS Linux found!")
                        return False

def ConfigFilesUpdate():
        firewall_disabled = False
        update_configuration = False
        IsStartUp = False
        RunLog.info("Updating configuration files..")

        #Provisioning.MonitorHostName=n -> y, in Ubuntu it is n by default
        Run("sed -i s/Provisioning.MonitorHostName=n/Provisioning.MonitorHostName=y/g  /etc/waagent.conf > /tmp/waagent.conf")

        # Disable 'requiretty' in sudoers
        Run("sed -r 's/^.*Defaults\s*requiretty.*$/#Defaults requiretty/g' /etc/sudoers -i")

        #Configuration of /etc/security/pam_env.conf
        Run("sed -i 's/^#REMOTEHOST/REMOTEHOST/g' /etc/security/pam_env.conf")
        Run("sed -i 's/^#DISPLAY/DISPLAY/g' /etc/security/pam_env.conf")
        pamconf = Run("cat /etc/security/pam_env.conf")

        if ((pamconf.find('#REMOTEHOST') == -1) and (pamconf.find('#DISPLAY') == -1)):
                RunLog.info("/etc/security/pam_env.conf updated successfully\n")
                update_configuration = True
                Run("echo '** Config files are updated successfully **' >> PackageStatus.txt")
        else:
                RunLog.error('Config file not updated\n')
                Run("echo '** updating of config file is failed **' >> PackageStatus.txt")
                update_configuration = False

        if (update_configuration == True):
                RunLog.info('Config file updation succesfully!\n')

        else:
                RunLog.error('[Error] Config file updation failed!')

        #Configuration of Firewall(Disable)
        if (current_distro == "ubuntu"):
                FirewallInfo = Run("ufw disable")
                if (FirewallInfo.find('Firewall stopped and disabled on system startup')):
                        RunLog.info('**Firewall Stopped Successfully** \n')
                        firewall_disabled = True
                else:
                        RunLog.error('**Failed to disable Firewall**')
                        firewall_disabled = False

        if ((current_distro == "SUSE") or (current_distro == "openSUSE")):
                FWBootInfo = Run("/sbin/yast2 firewall startup manual")
                if(FWBootInfo.find('Removing firewall from the boot process')):
                        RunLog.info('Firewall Removed successfully from boot process \n')

                FirewallInfo = Run("/sbin/rcSuSEfirewall2 status")

                if (FirewallInfo.find('SuSEfirewall2') and FirewallInfo.find('unused')):
                        RunLog.info('**Firewall Stopped Successfully** \n')
                        firewall_disabled = True
                else:
                        RunLog.error('**Failed to disable Firewall**')
                        firewall_disabled = False
        else:
                FirewallCmds = ("iptables -F","iptables -X","iptables -t nat -F","iptables -t nat -X","iptables -t mangle -F","iptables -t mangle -X","iptables -P INPUT ACCEPT","iptables -P OUTPUT ACCEPT","iptables -P FORWARD ACCEPT","systemctl stop iptables.service","systemctl disable iptables.service","systemctl stop firewalld.service","systemctl disable firewalld.service")
                output = ExecMultiCmdsLocalSudo(FirewallCmds)
                RunLog.info("Firewall disabled successfully\n")
                Run("echo '** Firewall disabled successfully **' >> PackageStatus.txt")
        #Verify startup file
        StartupStatus= Run('ls '+startup_file+' 2>&1')
        if("No such file or directory" in StartupStatus):
                RunLog.info( "'"+startup_file+"' not available.. ")
                RunLog.info( "creating '+startup_file+'.. ")
                cmds=["touch "+startup_file,"chmod a+x "+startup_file,"echo '#!/bin/sh' > "+startup_file,"echo ' ' >> "+startup_file,"echo 'exit 0' >> "+startup_file]
                ExecMultiCmdsLocalSudo(cmds)
                if(current_distro == "fedora"):
                        cmds = ["service rc-local start","service rc-local status"]
                        ExecMultiCmdsLocalSudo(cmds)
                RunLog.info("'+startup_file+' created successfully\n")
                Run("echo '** Is '"+startup_file+"' created successfully **' >> PackageStatus.txt")
                IsStartUp = True

        else:
                RunLog.info("'"+startup_file+"' available.. ")
                Run("echo '** Is '"+startup_file+"' verified successfully **' >> PackageStatus.txt")
                IsStartUp = True

        if(firewall_disabled and update_configuration and IsStartUp):
                success = True
        else:
                success = False

        return success

def RunTest():
        UpdateState("TestRunning")
        success = True
        try:
                import xml.etree.cElementTree as ET
        except ImportError:
                import xml.etree.ElementTree as ET

        #Parse the packages.xml file into memory
        packages_xml_file = ET.parse(packages_list_xml)
        xml_root = packages_xml_file.getroot()

        parse_success = False
        Run("echo '** Installing Packages for '"+current_distro+"' Started.. **' > PackageStatus.txt")
        for branch in xml_root:
                for node in branch:
                        if (node.tag == "packages"):
                                if(current_distro == node.attrib["distro"]):
                                        packages_list = node.text.split(",")
                        elif node.tag == "waLinuxAgent_link":
                                pass
                        elif node.tag == "rpm_link":
                                rpm_links[node.attrib["name"]] = node.text
                        elif node.tag == "tar_link":
                                tar_link[node.attrib["name"]] = node.text

        if not (current_distro=="coreos"):
                for package in packages_list:
                        if(not install_package(package)):
                                success = False
                                Run("echo '"+package+"' failed to install >> PackageStatus.txt")
                                #break
                        else:
                                Run("echo '"+package+"' installed successfully >> PackageStatus.txt")
        else:
                if (not coreos_package_install()):
                        success = False
                        Run("echo 'coreos packages failed to install' >> PackageStatus.txt")
                else:
                        Run("echo 'coreos support tools installed successfully' >> PackageStatus.txt")

        if (current_distro == "SUSE") or (current_distro == "openSUSE") or (current_distro == "sles") or (current_distro == "opensuse"):
                iperf_cmd = Run("command -v iperf")
                iperf3_cmd = Run("command -v iperf3")
                if not iperf_cmd and iperf3_cmd:
                        RunLog.info('iperf3 has been installed instead of iperf from default repository')
                        if (ZypperPackageRemove('iperf') and download_and_install_rpm('iperf')):
                                Run("echo 'iperf' installed successfully >> PackageStatus.txt")
                        else:
                                Run("echo 'iperf' failed to install >> PackageStatus.txt")

        Run("echo '** Packages Installation Completed **' >> PackageStatus.txt")
        if success == True:
                if not (current_distro=="coreos"):
                        ConfigFilesUpdate()
                if success == True:
                        RunLog.info('PACKAGE-INSTALL-CONFIG-PASS')
                        Run("echo 'PACKAGE-INSTALL-CONFIG-PASS' >> SetupStatus.txt")
                else:
                        RunLog.info('PACKAGE-INSTALL-CONFIG-FAIL')
                        Run("echo 'PACKAGE-INSTALL-CONFIG-FAIL' >> SetupStatus.txt")
        else:
                RunLog.info('PACKAGE-INSTALL-CONFIG-FAIL')
                Run("echo 'PACKAGE-INSTALL-CONFIG-FAIL' >> SetupStatus.txt")

#Code execution starts from here
if not os.path.isfile("packages.xml"):
        RunLog.info("'packages.xml' file is missing\n")
        exit ()

set_variables_OS_dependent()
UpdateRepos(current_distro)

RunTest()

<commit_msg>Update setup command for 2.1<commit_after>#!/usr/bin/python
from azuremodules import *
import sys
import time
import re
import os
import linecache
import imp
import os.path
import zipfile

current_distro        = "unknown"
distro_version        = "unknown"
sudo_password        = "rdPa$$w0rd"
startup_file = ""

rpm_links = {}
tar_link = {}
current_distro = "unknown"
packages_list_xml = "./packages.xml"
python_cmd="python"
waagent_cmd="waagent"
waagent_bin_path="/usr/sbin"

def set_variables_OS_dependent():
        global current_distro
        global distro_version
        global startup_file

        RunLog.info ("\nset_variables_OS_dependent ..")
        [current_distro, distro_version] = DetectDistro()
        if(current_distro == 'unknown'):
                RunLog.error ("unknown distribution found, exiting")
                ResultLog.info('ABORTED')
                exit()
        if(current_distro == "ubuntu" or current_distro == "Debian"):
                startup_file = '/etc/rc.local'
        elif(current_distro == "centos" or current_distro == "rhel" or current_distro == "fedora" or current_distro == "Oracle"):
                startup_file = '/etc/rc.d/rc.local'
        elif(current_distro == "SUSE" or current_distro == "sles" or current_distro == "opensuse"):
                startup_file = '/etc/rc.d/after.local'

        if(current_distro == "coreos"):
                python_cmd="/usr/share/oem/python/bin/python"
                waagent_bin_path="/usr/share/oem/bin/python"
                waagent_cmd= "{0} {1}".format(python_cmd, waagent_bin_path)

        RunLog.info ("\nset_variables_OS_dependent ..[done]")

def download_and_install_rpm(package):
        RunLog.info("Installing Package: " + package+" from rpmlink")
        if package in rpm_links:
                if DownloadUrl(rpm_links.get(package), "/tmp/"):
                        if InstallRpm("/tmp/"+re.split("/",rpm_links.get(package))[-1], package):
                                RunLog.info("Installing Package: " + package+" from rpmlink done!")
                                return True

        RunLog.error("Installing Package: " + package+" from rpmlink failed!!")
        return False

def yum_package_install(package):
        if(YumPackageInstall(package) == True):
                return True
        elif(download_and_install_rpm(package) == True):
                return True
        else:
                return False

def zypper_package_install(package):
        if(ZypperPackageInstall(package) == True):
                return True
        elif(download_and_install_rpm(package) == True):
                return True
        else:
                return False

def coreos_package_install():
        binpath="/usr/share/oem/bin"
        pythonlibrary="/usr/share/oem/python/lib64/python2.7"

        # create /etc/hosts
        ExecMultiCmdsLocalSudo(["touch /etc/hosts",\
                "echo '127.0.0.1 localhost' > /etc/hosts",\
                "echo '** modify /etc/hosts successfully **' >> PackageStatus.txt"])
        # copy tools to bin folder
        Run("unzip -d CoreosPreparationTools ./CoreosPreparationTools.zip")
        ExecMultiCmdsLocalSudo(["cp ./CoreosPreparationTools/killall " + binpath, \
                "cp ./CoreosPreparationTools/iperf " + binpath,\
                "cp ./CoreosPreparationTools/iozone " + binpath,\
                "cp ./CoreosPreparationTools/dos2unix " + binpath,\
                "cp ./CoreosPreparationTools/at " + binpath,\
                "chmod 755 "+ binpath + "/*",\
                "echo '** copy tools successfully **' >> PackageStatus.txt"])
        # copy python library to python library folder
        Run("tar zxvf ./CoreosPreparationTools/pycrypto.tar.gz -C "+ pythonlibrary)
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/ecdsa-0.13.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/ecdsa-0.13",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/paramiko-1.15.1.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/paramiko-1.15.1",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../..",\
                "tar zxvf ./CoreosPreparationTools/pexpect-3.3.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/pexpect-3.3",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        ExecMultiCmdsLocalSudo(["tar zxvf ./CoreosPreparationTools/dnspython-1.12.0.tar.gz -C ./CoreosPreparationTools",\
                "cd ./CoreosPreparationTools/dnspython-1.12.0",\
                "/usr/share/oem/python/bin/python setup.py install",\
                "cd ../.."])
        if not os.path.exists (pythonlibrary + "/site-packages/pexpect"):
                RunLog.error ("pexpect package installation failed!")
                Run("echo '** pexpect package installation failed **' >> PackageStatus.txt")
                return False
        if not os.path.exists (pythonlibrary + "/site-packages/paramiko"):
                RunLog.error ("paramiko packages installation failed!")
                Run("echo '** paramiko packages installed failed **' >> PackageStatus.txt")
                return False
        if not os.path.exists (pythonlibrary + "/site-packages/dns"):
                RunLog.error ("dnspython packages installation failed!")
                Run("echo '** dnspython packages installed failed **' >> PackageStatus.txt")
                return False
        RunLog.info ("pexpect, paramiko and dnspython packages installed successfully!")
        Run("echo '** pexpect, paramiko and dnspython packages installed successfully **' >> PackageStatus.txt")
        return True

def install_waagent_from_github():
        RunLog.info ("Installing waagent from github...")

        DownloadUrl(tar_link.get("waagent"), "/tmp/")
        filename = tar_link.get("waagent").split('/')[-1]
        RunLog.info ("Waagent tar file name is: "+ filename+"|")

        pkgPath = os.path.join("/tmp", filename)
        unzipPath = os.path.join("/tmp", "agent")
        if not os.path.isfile(pkgPath):
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("File not found: {0}".format(pkgPath))
                return False

        try:
                zipfile.ZipFile(pkgPath).extractall(unzipPath)
        except IOError as e:
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("{0}".format(e))
                return False

        binPath20 = os.path.join(unzipPath, "waagent")
        binPath21 = os.path.join(unzipPath, "bin/waagent")
        if os.path.isfile(binPath20):
                #For 2.0, only one file(waagent) needs to be replaced.
                waagent_bin=os.path.join(unzipPath, 'waagent')
                ExecMultiCmdsLocalSudo([
                        "chmod +x {0}".format(waagent_bin),
                        "cp {0} {1}".format(waagent_bin, waagent_bin_path),
                        "rm -rf {0}".format(unzipPath)])
                return True
        elif os.path.isfile(binPath21):
                #For 2.1, use setup.py to install/uninstall package
                setup_py = os.path.join(unzipPath, 'setup.py')
                ExecMultiCmdsLocalSudo([
                        "{0} {1} install --register-service --force".format(python_cmd, setup_py),
                        "rm -rf {0}".format(unzipPath)])
                return True
        else:
                RunLog.error("Installing waagent from github...[failed]")
                RunLog.error("Unknown waagent verions")
                return False

def install_package(package):
        RunLog.info ("\nInstall_package: "+package)
        if (package == "waagent"):
                return install_waagent_from_github()
        else:
                if ((current_distro == "ubuntu") or (current_distro == "Debian")):
                        return AptgetPackageInstall(package)
                elif ((current_distro == "rhel") or (current_distro == "Oracle") or (current_distro == 'centos') or (current_distro == 'fedora')):
                        return yum_package_install(package)
                elif (current_distro == "SUSE") or (current_distro == "openSUSE") or (current_distro == "sles") or (current_distro == "opensuse"):
                        return zypper_package_install(package)
                else:
                        RunLog.error (package + ": package installation failed!")
                        RunLog.info (current_distro + ": Unrecognised Distribution OS Linux found!")
                        return False

def ConfigFilesUpdate():
        firewall_disabled = False
        update_configuration = False
        IsStartUp = False
        RunLog.info("Updating configuration files..")

        #Provisioning.MonitorHostName=n -> y, in Ubuntu it is n by default
        Run("sed -i s/Provisioning.MonitorHostName=n/Provisioning.MonitorHostName=y/g  /etc/waagent.conf > /tmp/waagent.conf")

        # Disable 'requiretty' in sudoers
        Run("sed -r 's/^.*Defaults\s*requiretty.*$/#Defaults requiretty/g' /etc/sudoers -i")

        #Configuration of /etc/security/pam_env.conf
        Run("sed -i 's/^#REMOTEHOST/REMOTEHOST/g' /etc/security/pam_env.conf")
        Run("sed -i 's/^#DISPLAY/DISPLAY/g' /etc/security/pam_env.conf")
        pamconf = Run("cat /etc/security/pam_env.conf")

        if ((pamconf.find('#REMOTEHOST') == -1) and (pamconf.find('#DISPLAY') == -1)):
                RunLog.info("/etc/security/pam_env.conf updated successfully\n")
                update_configuration = True
                Run("echo '** Config files are updated successfully **' >> PackageStatus.txt")
        else:
                RunLog.error('Config file not updated\n')
                Run("echo '** updating of config file is failed **' >> PackageStatus.txt")
                update_configuration = False

        if (update_configuration == True):
                RunLog.info('Config file updation succesfully!\n')

        else:
                RunLog.error('[Error] Config file updation failed!')

        #Configuration of Firewall(Disable)
        if (current_distro == "ubuntu"):
                FirewallInfo = Run("ufw disable")
                if (FirewallInfo.find('Firewall stopped and disabled on system startup')):
                        RunLog.info('**Firewall Stopped Successfully** \n')
                        firewall_disabled = True
                else:
                        RunLog.error('**Failed to disable Firewall**')
                        firewall_disabled = False

        if ((current_distro == "SUSE") or (current_distro == "openSUSE")):
                FWBootInfo = Run("/sbin/yast2 firewall startup manual")
                if(FWBootInfo.find('Removing firewall from the boot process')):
                        RunLog.info('Firewall Removed successfully from boot process \n')

                FirewallInfo = Run("/sbin/rcSuSEfirewall2 status")

                if (FirewallInfo.find('SuSEfirewall2') and FirewallInfo.find('unused')):
                        RunLog.info('**Firewall Stopped Successfully** \n')
                        firewall_disabled = True
                else:
                        RunLog.error('**Failed to disable Firewall**')
                        firewall_disabled = False
        else:
                FirewallCmds = ("iptables -F","iptables -X","iptables -t nat -F","iptables -t nat -X","iptables -t mangle -F","iptables -t mangle -X","iptables -P INPUT ACCEPT","iptables -P OUTPUT ACCEPT","iptables -P FORWARD ACCEPT","systemctl stop iptables.service","systemctl disable iptables.service","systemctl stop firewalld.service","systemctl disable firewalld.service")
                output = ExecMultiCmdsLocalSudo(FirewallCmds)
                RunLog.info("Firewall disabled successfully\n")
                Run("echo '** Firewall disabled successfully **' >> PackageStatus.txt")
        #Verify startup file
        StartupStatus= Run('ls '+startup_file+' 2>&1')
        if("No such file or directory" in StartupStatus):
                RunLog.info( "'"+startup_file+"' not available.. ")
                RunLog.info( "creating '+startup_file+'.. ")
                cmds=["touch "+startup_file,"chmod a+x "+startup_file,"echo '#!/bin/sh' > "+startup_file,"echo ' ' >> "+startup_file,"echo 'exit 0' >> "+startup_file]
                ExecMultiCmdsLocalSudo(cmds)
                if(current_distro == "fedora"):
                        cmds = ["service rc-local start","service rc-local status"]
                        ExecMultiCmdsLocalSudo(cmds)
                RunLog.info("'+startup_file+' created successfully\n")
                Run("echo '** Is '"+startup_file+"' created successfully **' >> PackageStatus.txt")
                IsStartUp = True

        else:
                RunLog.info("'"+startup_file+"' available.. ")
                Run("echo '** Is '"+startup_file+"' verified successfully **' >> PackageStatus.txt")
                IsStartUp = True

        if(firewall_disabled and update_configuration and IsStartUp):
                success = True
        else:
                success = False

        return success

def RunTest():
        UpdateState("TestRunning")
        success = True
        try:
                import xml.etree.cElementTree as ET
        except ImportError:
                import xml.etree.ElementTree as ET

        #Parse the packages.xml file into memory
        packages_xml_file = ET.parse(packages_list_xml)
        xml_root = packages_xml_file.getroot()

        parse_success = False
        Run("echo '** Installing Packages for '"+current_distro+"' Started.. **' > PackageStatus.txt")
        for branch in xml_root:
                for node in branch:
                        if (node.tag == "packages"):
                                if(current_distro == node.attrib["distro"]):
                                        packages_list = node.text.split(",")
                        elif node.tag == "waLinuxAgent_link":
                                pass
                        elif node.tag == "rpm_link":
                                rpm_links[node.attrib["name"]] = node.text
                        elif node.tag == "tar_link":
                                tar_link[node.attrib["name"]] = node.text

        if not (current_distro=="coreos"):
                for package in packages_list:
                        if(not install_package(package)):
                                success = False
                                Run("echo '"+package+"' failed to install >> PackageStatus.txt")
                                #break
                        else:
                                Run("echo '"+package+"' installed successfully >> PackageStatus.txt")
        else:
                if (not coreos_package_install()):
                        success = False
                        Run("echo 'coreos packages failed to install' >> PackageStatus.txt")
                else:
                        Run("echo 'coreos support tools installed successfully' >> PackageStatus.txt")

        if (current_distro == "SUSE") or (current_distro == "openSUSE") or (current_distro == "sles") or (current_distro == "opensuse"):
                iperf_cmd = Run("command -v iperf")
                iperf3_cmd = Run("command -v iperf3")
                if not iperf_cmd and iperf3_cmd:
                        RunLog.info('iperf3 has been installed instead of iperf from default repository')
                        if (ZypperPackageRemove('iperf') and download_and_install_rpm('iperf')):
                                Run("echo 'iperf' installed successfully >> PackageStatus.txt")
                        else:
                                Run("echo 'iperf' failed to install >> PackageStatus.txt")

        Run("echo '** Packages Installation Completed **' >> PackageStatus.txt")
        if success == True:
                if not (current_distro=="coreos"):
                        ConfigFilesUpdate()
                if success == True:
                        RunLog.info('PACKAGE-INSTALL-CONFIG-PASS')
                        Run("echo 'PACKAGE-INSTALL-CONFIG-PASS' >> SetupStatus.txt")
                else:
                        RunLog.info('PACKAGE-INSTALL-CONFIG-FAIL')
                        Run("echo 'PACKAGE-INSTALL-CONFIG-FAIL' >> SetupStatus.txt")
        else:
                RunLog.info('PACKAGE-INSTALL-CONFIG-FAIL')
                Run("echo 'PACKAGE-INSTALL-CONFIG-FAIL' >> SetupStatus.txt")

#Code execution starts from here
if not os.path.isfile("packages.xml"):
        RunLog.info("'packages.xml' file is missing\n")
        exit ()

set_variables_OS_dependent()
UpdateRepos(current_distro)

RunTest()

<|endoftext|>


<commit_before>import json
import urllib2

class Ophion:
    def __init__(self, host):
        self.host = host
        self.url = host + "/gaia/vertex/query"

    def query(self):
        return OphionQuery(self)

    def vertex(self, gid):
        url = self.host + "/gaia/vertex/find/" + gid
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        request = urllib2.Request(url, headers=headers)
        response = urllib2.urlopen(request)
        result = response.read()
        return json.loads(result)

    def execute(self, query):
        payload = query.render()
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        request = urllib2.Request(self.url, payload, headers=headers)
        response = urllib2.urlopen(request)
        result = response.read()
        return json.loads(result)

class OphionQuery:
    def __init__(self, parent=None):
        self.query = []
        self.parent = parent

    def label(self, label):
        self.query.append({'label': label})
        return self

    def has(self, prop, within):
        if not isinstance(within, list):
            within = [within]
        self.query.append({'has': prop, 'within': within})
        return self

    def values(self, v):
        self.query.append({'values': v})
        return self

    def cap(self, c):
        if not isinstance(c, list):
            c = [c]
        self.query.append({'cap': c})
        return self

    def incoming(self, label):
        self.query.append({'in': label})
        return self

    def outgoing(self, label):
        self.query.append({'out': label})
        return self

    def inEdge(self, label):
        self.query.append({'inEdge': label})
        return self

    def outEdge(self, label):
        self.query.append({'outEdge': label})
        return self

    def inVertex(self, label):
        self.query.append({'inVertex': label})
        return self

    def outVertex(self, label):
        self.query.append({'outVertex': label})
        return self

    def mark(self, label):
        self.query.append({'as': label})
        return self

    def limit(self, l):
        self.query.append({'limit': l})
        return self

    def range(self, begin, end):
        self.query.append({'begin': begin, 'end': end})
        return self

    def count(self):
        self.query.append({'count': ''})
        return self

    def groupCount(self, label):
        self.query.append({'groupCount': label})
        return self

    def by(self, label):
        self.query.append({'by': label})
        return self

    def render(self):
        output = {'query': self.query}
        return json.dumps(output)

    def execute(self):
        return self.parent.execute(self)
<commit_msg>Adding select method to client<commit_after>import json
import urllib2

class Ophion:
    def __init__(self, host):
        self.host = host
        self.url = host + "/gaia/vertex/query"

    def query(self):
        return OphionQuery(self)

    def vertex(self, gid):
        url = self.host + "/gaia/vertex/find/" + gid
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        request = urllib2.Request(url, headers=headers)
        response = urllib2.urlopen(request)
        result = response.read()
        return json.loads(result)

    def execute(self, query):
        payload = query.render()
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        request = urllib2.Request(self.url, payload, headers=headers)
        response = urllib2.urlopen(request)
        result = response.read()
        return json.loads(result)

class OphionQuery:
    def __init__(self, parent=None):
        self.query = []
        self.parent = parent

    def label(self, label):
        self.query.append({'label': label})
        return self

    def has(self, prop, within):
        if not isinstance(within, list):
            within = [within]
        self.query.append({'has': prop, 'within': within})
        return self

    def values(self, v):
        self.query.append({'values': v})
        return self

    def cap(self, c):
        if not isinstance(c, list):
            c = [c]
        self.query.append({'cap': c})
        return self

    def incoming(self, label):
        self.query.append({'in': label})
        return self

    def outgoing(self, label):
        self.query.append({'out': label})
        return self

    def inEdge(self, label):
        self.query.append({'inEdge': label})
        return self

    def outEdge(self, label):
        self.query.append({'outEdge': label})
        return self

    def inVertex(self, label):
        self.query.append({'inVertex': label})
        return self

    def outVertex(self, label):
        self.query.append({'outVertex': label})
        return self

    def mark(self, label):
        self.query.append({'as': label})
        return self

    def select(self, labels):
        if not isinstance(labels, list):
            labels = [labels]
        self.query.append({'select' : labels})
        return self

    def limit(self, l):
        self.query.append({'limit': l})
        return self

    def range(self, begin, end):
        self.query.append({'begin': begin, 'end': end})
        return self

    def count(self):
        self.query.append({'count': ''})
        return self

    def groupCount(self, label):
        self.query.append({'groupCount': label})
        return self

    def by(self, label):
        self.query.append({'by': label})
        return self

    def render(self):
        output = {'query': self.query}
        return json.dumps(output)

    def execute(self):
        return self.parent.execute(self)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""RAPPOR client library.

Privacy is ensured without a third party by only sending RAPPOR'd data over the
network (as opposed to raw client data).

Note that we use MD5 for the Bloom filter hash function. (security not required)
"""
import csv
import hashlib
import hmac
import json
import os
import struct
import sys

class Error(Exception):
  pass

def log(msg, *args):
  if args:
    msg = msg % args
  print >>sys.stderr, msg

class Params(object):
  """RAPPOR encoding parameters.

  These affect privacy/anonymity.  See the paper for details.
  """
  def __init__(self):
    self.num_bloombits = 16      # Number of bloom filter bits (k)
    self.num_hashes = 2          # Number of bloom filter hashes (h)
    self.num_cohorts = 64        # Number of cohorts (m)
    self.prob_p = 0.50           # Probability p
    self.prob_q = 0.75           # Probability q
    self.prob_f = 0.50           # Probability f

  # For testing
  def __eq__(self, other):
    return self.__dict__ == other.__dict__

  def __repr__(self):
    return repr(self.__dict__)

  def to_json(self):
    """Convert this instance to JSON.

    The names are be compatible with the apps/api server.
    """
    return json.dumps({
        'numBits': self.num_bloombits,
        'numHashes': self.num_hashes,
        'numCohorts': self.num_cohorts,
        'probPrr': self.prob_f,
        'probIrr0': self.prob_p,
        'probIrr1': self.prob_q,
    })

  # NOTE:
  # - from_csv is currently used in sum_bits.py
  # - to_csv is in rappor_sim.print_params
  @staticmethod
  def from_csv(f):
    """Read the RAPPOR parameters from a CSV file.

    Args:
      f: file handle

    Returns:
      Params instance.

    Raises:
      rappor.Error: when the file is malformed.
    """
    c = csv.reader(f)
    ok = False
    p = Params()
    for i, row in enumerate(c):

      if i == 0:
        if row != ['k', 'h', 'm', 'p', 'q', 'f']:
          raise Error('Header %s is malformed; expected k,h,m,p,q,f' % row)

      elif i == 1:
        try:
          # NOTE: May raise exceptions
          p.num_bloombits = int(row[0])
          p.num_hashes = int(row[1])
          p.num_cohorts = int(row[2])
          p.prob_p = float(row[3])
          p.prob_p = float(row[4])
          p.prob_q = float(row[5])
        except (ValueError, IndexError) as e:
          raise Error('Row is malformed: %s' % e)
        ok = True

      else:
        raise Error('Params file should only have two rows')

    if not ok:
      raise Error("Expected second row with params")

    return p

class _SecureRandom(object):
  """Returns an integer where each bit has probability p of being 1."""

  def __init__(self, prob_one, num_bits):
    self.prob_one = prob_one
    self.num_bits = num_bits

  def __call__(self):
    p = self.prob_one
    bytes = os.urandom(self.num_bits)

    r = 0
    for i, b in enumerate(xrange(self.num_bits)):
      bit = b < p * 256.0
      r |= (bit << i)  # using bool as int
    return r

class SecureIrrRand(object):
  """Python's os.random()"""

  def __init__(self, params):
    """
    Args:
      params: rappor.Params
    """
    num_bits = params.num_bloombits
    # IRR probabilities

    self.p_gen = _SecureRandom(params.prob_p, num_bits)
    self.q_gen = _SecureRandom(params.prob_q, num_bits)

def to_big_endian(i):
  """Convert an integer to a 4 byte big endian string.  Used for hashing."""
  # https://docs.python.org/2/library/struct.html
  # - Big Endian (>) for consistent network byte order.
  # - L means 4 bytes when using >
  return struct.pack('>L', i)

def get_bloom_bits(word, cohort, num_hashes, num_bloombits):
  """Return an array of bits to set in the bloom filter.

  In the real report, we bitwise-OR them together.  In hash candidates, we put
  them in separate entries in the "map" matrix.
  """
  value = to_big_endian(cohort) + word  # Cohort is 4 byte prefix.
  md5 = hashlib.md5(value)

  digest = md5.digest()

  # Each has is a byte, which means we could have up to 256 bit Bloom filters.
  # There are 16 bytes in an MD5, in which case we can have up to 16 hash
  # functions per Bloom filter.
  if num_hashes > len(digest):
    raise RuntimeError("Can't have more than %d hashes" % md5)

  #log('hash_input %r', value)
  #log('Cohort %d', cohort)
  #log('MD5 %s', md5.hexdigest())

  return [ord(digest[i]) % num_bloombits for i in xrange(num_hashes)]

def get_prr_masks(secret, word, prob_f, num_bits):
  h = hmac.new(secret, word, digestmod=hashlib.sha256)
  #log('word %s, secret %s, HMAC-SHA256 %s', word, secret, h.hexdigest())

  # Now go through each byte
  digest_bytes = h.digest()
  assert len(digest_bytes) == 32

  # Use 32 bits.  If we want 64 bits, it may be fine to generate another 32
  # bytes by repeated HMAC.  For arbitrary numbers of bytes it's probably
  # better to use the HMAC-DRBG algorithm.
  if num_bits > len(digest_bytes):
    raise RuntimeError('%d bits is more than the max of %d', num_bits, len(d))

  threshold128 = prob_f * 128

  uniform = 0
  f_mask = 0

  for i in xrange(num_bits):
    ch = digest_bytes[i]
    byte = ord(ch)

    u_bit = byte & 0x01  # 1 bit of entropy
    uniform |= (u_bit << i)  # maybe set bit in mask

    rand128 = byte >> 1  # 7 bits of entropy
    noise_bit = (rand128 < threshold128)
    f_mask |= (noise_bit << i)  # maybe set bit in mask

  return uniform, f_mask

def bit_string(irr, num_bloombits):
  """Like bin(), but uses leading zeroes, and no '0b'."""
  s = ''
  bits = []
  for bit_num in xrange(num_bloombits):
    if irr & (1 << bit_num):
      bits.append('1')
    else:
      bits.append('0')
  return ''.join(reversed(bits))

class Encoder(object):
  """Obfuscates values for a given user using the RAPPOR privacy algorithm."""

  def __init__(self, params, cohort, secret, irr_rand):
    """
    Args:
      params: RAPPOR Params() controlling privacy
      cohort: integer cohort, for Bloom hashing.
      secret: secret string, for the PRR to be a deterministic function of the
        reported value.
      irr_rand: IRR randomness interface.
    """
    # RAPPOR params.  NOTE: num_cohorts isn't used.  p and q are used by
    # irr_rand.
    self.params = params
    self.cohort = cohort  # associated: MD5
    self.secret = secret  # associated: HMAC-SHA256
    self.irr_rand = irr_rand  # p and q used

  def _internal_encode_bits(self, bits):
    """Helper function for simulation / testing.

    Returns:
      The PRR and IRR.  The PRR should never be sent over the network.
    """
    # Compute Permanent Randomized Response (PRR).
    uniform, f_mask = get_prr_masks(
        self.secret, to_big_endian(bits), self.params.prob_f,
        self.params.num_bloombits)

    # Suppose bit i of the Bloom filter is B_i.  Then bit i of the PRR is
    # defined as:
    #
    # 1   with prob f/2
    # 0   with prob f/2
    # B_i with prob 1-f

    # Uniform bits are 1 with probability 1/2, and f_mask bits are 1 with
    # probability f.  So in the expression below:
    #
    # - Bits in (uniform & f_mask) are 1 with probability f/2.
    # - (bloom_bits & ~f_mask) clears a bloom filter bit with probability
    # f, so we get B_i with probability 1-f.
    # - The remaining bits are 0, with remaining probability f/2.

    prr = (bits & ~f_mask) | (uniform & f_mask)

    #log('U %s / F %s', bit_string(uniform, num_bits),
    #    bit_string(f_mask, num_bits))

    #log('B %s / PRR %s', bit_string(bloom_bits, num_bits),
    #    bit_string(prr, num_bits))

    # Compute Instantaneous Randomized Response (IRR).
    # If PRR bit is 0, IRR bit is 1 with probability p.
    # If PRR bit is 1, IRR bit is 1 with probability q.
    p_bits = self.irr_rand.p_gen()
    q_bits = self.irr_rand.q_gen()

    irr = (p_bits & ~prr) | (q_bits & prr)

    return prr, irr  # IRR is the rappor

  def _internal_encode(self, word):
    """Helper function for simulation / testing.

    Returns:
      The Bloom filter bits, PRR, and IRR.  The first two values should never
      be sent over the network.
    """
    bloom_bits = get_bloom_bits(word, self.cohort, self.params.num_hashes,
                                self.params.num_bloombits)

    bloom = 0
    for bit_to_set in bloom_bits:
      bloom |= (1 << bit_to_set)

    prr, irr = self._internal_encode_bits(bloom)
    return bloom, prr, irr

  def encode_bits(self, bits):
    """Encode a string with RAPPOR.

    Args:
      bits: An integer representing bits to encode.

    Returns:
      An integer that is the IRR (Instantaneous Randomized Response).
    """
    _, irr = self._internal_encode_bits(bits)
    return irr

  def encode(self, word):
    """Encode a string with RAPPOR.

    Args:
      word: the string that should be privately transmitted.

    Returns:
      An integer that is the IRR (Instantaneous Randomized Response).
    """
    _, _, irr = self._internal_encode(word)
    return irr
<commit_msg>Switch to system random to get full precision<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""RAPPOR client library.

Privacy is ensured without a third party by only sending RAPPOR'd data over the
network (as opposed to raw client data).

Note that we use MD5 for the Bloom filter hash function. (security not required)
"""
import csv
import hashlib
import hmac
import json
import struct
import sys

from random import SystemRandom

class Error(Exception):
  pass

def log(msg, *args):
  if args:
    msg = msg % args
  print >>sys.stderr, msg

class Params(object):
  """RAPPOR encoding parameters.

  These affect privacy/anonymity.  See the paper for details.
  """
  def __init__(self):
    self.num_bloombits = 16      # Number of bloom filter bits (k)
    self.num_hashes = 2          # Number of bloom filter hashes (h)
    self.num_cohorts = 64        # Number of cohorts (m)
    self.prob_p = 0.50           # Probability p
    self.prob_q = 0.75           # Probability q
    self.prob_f = 0.50           # Probability f

  # For testing
  def __eq__(self, other):
    return self.__dict__ == other.__dict__

  def __repr__(self):
    return repr(self.__dict__)

  def to_json(self):
    """Convert this instance to JSON.

    The names are be compatible with the apps/api server.
    """
    return json.dumps({
        'numBits': self.num_bloombits,
        'numHashes': self.num_hashes,
        'numCohorts': self.num_cohorts,
        'probPrr': self.prob_f,
        'probIrr0': self.prob_p,
        'probIrr1': self.prob_q,
    })

  # NOTE:
  # - from_csv is currently used in sum_bits.py
  # - to_csv is in rappor_sim.print_params
  @staticmethod
  def from_csv(f):
    """Read the RAPPOR parameters from a CSV file.

    Args:
      f: file handle

    Returns:
      Params instance.

    Raises:
      rappor.Error: when the file is malformed.
    """
    c = csv.reader(f)
    ok = False
    p = Params()
    for i, row in enumerate(c):

      if i == 0:
        if row != ['k', 'h', 'm', 'p', 'q', 'f']:
          raise Error('Header %s is malformed; expected k,h,m,p,q,f' % row)

      elif i == 1:
        try:
          # NOTE: May raise exceptions
          p.num_bloombits = int(row[0])
          p.num_hashes = int(row[1])
          p.num_cohorts = int(row[2])
          p.prob_p = float(row[3])
          p.prob_p = float(row[4])
          p.prob_q = float(row[5])
        except (ValueError, IndexError) as e:
          raise Error('Row is malformed: %s' % e)
        ok = True

      else:
        raise Error('Params file should only have two rows')

    if not ok:
      raise Error("Expected second row with params")

    return p

class _SecureRandom(object):
  """Returns an integer where each bit has probability p of being 1."""

  def __init__(self, prob_one, num_bits):
    self.prob_one = prob_one
    self.num_bits = num_bits

  def __call__(self):
    p = self.prob_one
    rand = SystemRandom()
    r = 0

    for i in xrange(self.num_bits):
      bit = rand.random() < p
      r |= (bit << i)  # using bool as int
    return r

class SecureIrrRand(object):
  """Python's os.random()"""

  def __init__(self, params):
    """
    Args:
      params: rappor.Params
    """
    num_bits = params.num_bloombits
    # IRR probabilities

    self.p_gen = _SecureRandom(params.prob_p, num_bits)
    self.q_gen = _SecureRandom(params.prob_q, num_bits)

def to_big_endian(i):
  """Convert an integer to a 4 byte big endian string.  Used for hashing."""
  # https://docs.python.org/2/library/struct.html
  # - Big Endian (>) for consistent network byte order.
  # - L means 4 bytes when using >
  return struct.pack('>L', i)

def get_bloom_bits(word, cohort, num_hashes, num_bloombits):
  """Return an array of bits to set in the bloom filter.

  In the real report, we bitwise-OR them together.  In hash candidates, we put
  them in separate entries in the "map" matrix.
  """
  value = to_big_endian(cohort) + word  # Cohort is 4 byte prefix.
  md5 = hashlib.md5(value)

  digest = md5.digest()

  # Each has is a byte, which means we could have up to 256 bit Bloom filters.
  # There are 16 bytes in an MD5, in which case we can have up to 16 hash
  # functions per Bloom filter.
  if num_hashes > len(digest):
    raise RuntimeError("Can't have more than %d hashes" % md5)

  #log('hash_input %r', value)
  #log('Cohort %d', cohort)
  #log('MD5 %s', md5.hexdigest())

  return [ord(digest[i]) % num_bloombits for i in xrange(num_hashes)]

def get_prr_masks(secret, word, prob_f, num_bits):
  h = hmac.new(secret, word, digestmod=hashlib.sha256)
  #log('word %s, secret %s, HMAC-SHA256 %s', word, secret, h.hexdigest())

  # Now go through each byte
  digest_bytes = h.digest()
  assert len(digest_bytes) == 32

  # Use 32 bits.  If we want 64 bits, it may be fine to generate another 32
  # bytes by repeated HMAC.  For arbitrary numbers of bytes it's probably
  # better to use the HMAC-DRBG algorithm.
  if num_bits > len(digest_bytes):
    raise RuntimeError('%d bits is more than the max of %d', num_bits, len(d))

  threshold128 = prob_f * 128

  uniform = 0
  f_mask = 0

  for i in xrange(num_bits):
    ch = digest_bytes[i]
    byte = ord(ch)

    u_bit = byte & 0x01  # 1 bit of entropy
    uniform |= (u_bit << i)  # maybe set bit in mask

    rand128 = byte >> 1  # 7 bits of entropy
    noise_bit = (rand128 < threshold128)
    f_mask |= (noise_bit << i)  # maybe set bit in mask

  return uniform, f_mask

def bit_string(irr, num_bloombits):
  """Like bin(), but uses leading zeroes, and no '0b'."""
  s = ''
  bits = []
  for bit_num in xrange(num_bloombits):
    if irr & (1 << bit_num):
      bits.append('1')
    else:
      bits.append('0')
  return ''.join(reversed(bits))

class Encoder(object):
  """Obfuscates values for a given user using the RAPPOR privacy algorithm."""

  def __init__(self, params, cohort, secret, irr_rand):
    """
    Args:
      params: RAPPOR Params() controlling privacy
      cohort: integer cohort, for Bloom hashing.
      secret: secret string, for the PRR to be a deterministic function of the
        reported value.
      irr_rand: IRR randomness interface.
    """
    # RAPPOR params.  NOTE: num_cohorts isn't used.  p and q are used by
    # irr_rand.
    self.params = params
    self.cohort = cohort  # associated: MD5
    self.secret = secret  # associated: HMAC-SHA256
    self.irr_rand = irr_rand  # p and q used

  def _internal_encode_bits(self, bits):
    """Helper function for simulation / testing.

    Returns:
      The PRR and IRR.  The PRR should never be sent over the network.
    """
    # Compute Permanent Randomized Response (PRR).
    uniform, f_mask = get_prr_masks(
        self.secret, to_big_endian(bits), self.params.prob_f,
        self.params.num_bloombits)

    # Suppose bit i of the Bloom filter is B_i.  Then bit i of the PRR is
    # defined as:
    #
    # 1   with prob f/2
    # 0   with prob f/2
    # B_i with prob 1-f

    # Uniform bits are 1 with probability 1/2, and f_mask bits are 1 with
    # probability f.  So in the expression below:
    #
    # - Bits in (uniform & f_mask) are 1 with probability f/2.
    # - (bloom_bits & ~f_mask) clears a bloom filter bit with probability
    # f, so we get B_i with probability 1-f.
    # - The remaining bits are 0, with remaining probability f/2.

    prr = (bits & ~f_mask) | (uniform & f_mask)

    #log('U %s / F %s', bit_string(uniform, num_bits),
    #    bit_string(f_mask, num_bits))

    #log('B %s / PRR %s', bit_string(bloom_bits, num_bits),
    #    bit_string(prr, num_bits))

    # Compute Instantaneous Randomized Response (IRR).
    # If PRR bit is 0, IRR bit is 1 with probability p.
    # If PRR bit is 1, IRR bit is 1 with probability q.
    p_bits = self.irr_rand.p_gen()
    q_bits = self.irr_rand.q_gen()

    irr = (p_bits & ~prr) | (q_bits & prr)

    return prr, irr  # IRR is the rappor

  def _internal_encode(self, word):
    """Helper function for simulation / testing.

    Returns:
      The Bloom filter bits, PRR, and IRR.  The first two values should never
      be sent over the network.
    """
    bloom_bits = get_bloom_bits(word, self.cohort, self.params.num_hashes,
                                self.params.num_bloombits)

    bloom = 0
    for bit_to_set in bloom_bits:
      bloom |= (1 << bit_to_set)

    prr, irr = self._internal_encode_bits(bloom)
    return bloom, prr, irr

  def encode_bits(self, bits):
    """Encode a string with RAPPOR.

    Args:
      bits: An integer representing bits to encode.

    Returns:
      An integer that is the IRR (Instantaneous Randomized Response).
    """
    _, irr = self._internal_encode_bits(bits)
    return irr

  def encode(self, word):
    """Encode a string with RAPPOR.

    Args:
      word: the string that should be privately transmitted.

    Returns:
      An integer that is the IRR (Instantaneous Randomized Response).
    """
    _, _, irr = self._internal_encode(word)
    return irr
<|endoftext|>


<commit_before>import threading
import time
import sys
import logging
import traceback
import pytest
from multiprocessing import Process, Queue

logger = logging.getLogger(__name__)

def run_test(webview, main_func, thread_func=None, param=None, no_destroy=False):
    __tracebackhide__ = True
    queue = Queue()

    try:
        _create_window(webview, main_func, thread_func, queue, param, no_destroy)
    except Exception as e:
        pytest.fail(e)

    #p = Process(target=_create_window, args=(main_func, thread_func, queue, param, no_destroy))
    #p.start()
    #p.join()
    #assert p.exitcode == 0

    #if not queue.empty():
    #    exception = queue.get()
    #

def assert_js(webview, func_name, expected_result, uid='master'):
    execute_func = 'window.pywebview.api.{0}()'.format(func_name)
    check_func = 'window.pywebview._returnValues["{0}"].value'.format(func_name)

    webview.evaluate_js(execute_func, uid)

    result = webview.evaluate_js(check_func, uid)
    counter, MAX = 0, 50

    while result is None:
        if counter == MAX:
            raise AssertionError('assert_js timeout')
        else:
            counter += 1
            time.sleep(0.1)
            result = webview.evaluate_js(check_func, uid)

    assert expected_result == result

def _create_window(webview, main_func, thread_func, queue, param, no_destroy):

    def thread(destroy_event, param):
        try:
            if thread_func:
                thread_func()
            destroy_event.set()
        except Exception as e:
            logger.exception(e, exc_info=True)
            queue.put(traceback.format_exc())
            destroy_event.set()

    if not no_destroy:
        args = (param,) if param else ()
        destroy_event = _destroy_window(webview)

        t = threading.Thread(target=thread, args=(destroy_event, args))
        t.start()

    main_func()

def _destroy_window(webview, delay=0):
    def stop():
        event.wait()
        time.sleep(delay)
        webview.destroy_window()

        if sys.platform == 'darwin':
            from .util_cocoa import mouseMoveRelative
            time.sleep(1)
            mouseMoveRelative(1, 1)

    event = threading.Event()
    event.clear()
    t = threading.Thread(target=stop)
    t.start()

    return event

<commit_msg>Remove commented code<commit_after>import threading
import time
import sys
import logging
import traceback
import pytest
from multiprocessing import Process, Queue

logger = logging.getLogger(__name__)

def run_test(webview, main_func, thread_func=None, param=None, no_destroy=False):
    __tracebackhide__ = True
    queue = Queue()

    try:
        _create_window(webview, main_func, thread_func, queue, param, no_destroy)
    except Exception as e:
        pytest.fail(e)

def assert_js(webview, func_name, expected_result, uid='master'):
    execute_func = 'window.pywebview.api.{0}()'.format(func_name)
    check_func = 'window.pywebview._returnValues["{0}"].value'.format(func_name)

    webview.evaluate_js(execute_func, uid)

    result = webview.evaluate_js(check_func, uid)
    counter, MAX = 0, 50

    while result is None:
        if counter == MAX:
            raise AssertionError('assert_js timeout')
        else:
            counter += 1
            time.sleep(0.1)
            result = webview.evaluate_js(check_func, uid)

    assert expected_result == result

def _create_window(webview, main_func, thread_func, queue, param, no_destroy):

    def thread(destroy_event, param):
        try:
            if thread_func:
                thread_func()
            destroy_event.set()
        except Exception as e:
            logger.exception(e, exc_info=True)
            queue.put(traceback.format_exc())
            destroy_event.set()

    if not no_destroy:
        args = (param,) if param else ()
        destroy_event = _destroy_window(webview)

        t = threading.Thread(target=thread, args=(destroy_event, args))
        t.start()

    main_func()

def _destroy_window(webview, delay=0):
    def stop():
        event.wait()
        time.sleep(delay)
        webview.destroy_window()

        if sys.platform == 'darwin':
            from .util_cocoa import mouseMoveRelative
            time.sleep(1)
            mouseMoveRelative(1, 1)

    event = threading.Event()
    event.clear()
    t = threading.Thread(target=stop)
    t.start()

    return event

<|endoftext|>


<commit_before>import sys
if sys.version_info <  (3,0):
	from BaseHTTPServer import HTTPServer
	from SimpleHTTPServer import SimpleHTTPRequestHandler
else:
	from http.server import HTTPServer, SimpleHTTPRequestHandler

httpd = HTTPServer(('0.0.0.0', 80), SimpleHTTPRequestHandler)
httpd.serve_forever()
<commit_msg>testserver: fix build and show message<commit_after>import sys
if sys.version_info <  (3,0):
	from BaseHTTPServer import HTTPServer
	from SimpleHTTPServer import SimpleHTTPRequestHandler
else:
	from http.server import HTTPServer, SimpleHTTPRequestHandler

httpd = HTTPServer(('0.0.0.0', 8080), SimpleHTTPRequestHandler)
print 'Medea server is launched.'
print 'Please go to http://localhost:8080 to find demos'
httpd.serve_forever()
<|endoftext|>


<commit_before># Module:   server
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Web Server

This module implements the Web Server Component.
"""

import os
from socket import gethostname as _gethostname

from circuits import Component

from circuits.net.sockets import TCPServer

from http import HTTP
from wrappers import Request, Host
from constants import SERVER_VERSION
from dispatchers import Dispatcher

class BaseServer(Component):

    channel = "web"

    def __init__(self, bind, **kwargs):
        super(BaseServer, self).__init__(**kwargs)

        self.server = (TCPServer(bind, **kwargs) + HTTP())
        self += self.server

        Request.server = self
        Request.local = Host(self.server.bind[0], self.server.bind[1])
        Request.host = self.host

        print "%s listening on %s" % (self.version, self.base)

    @property
    def version(self):
        return SERVER_VERSION

    @property
    def address(self):
        return self.server.bind[0] if hasattr(self, "server") else None

    @property
    def port(self):
        return self.server.bind[1] if hasattr(self, "server") else None

    @property
    def ssl(self):
        return self.server.ssl if hasattr(self, "server") else None

    @property
    def scheme(self):
        return "https" if self.ssl else "http"

    @property
    def host(self):
        host = self.address

        if host in ("0.0.0.0", "::", ""):
            # 0.0.0.0 is INADDR_ANY and :: is IN6ADDR_ANY.
            # Look up the host name, which should be the
            # safest thing to spit out in a URL.
            host = _gethostname()

        ssl = self.ssl
        port = self.port

        if not ((ssl and port == 433) or (not ssl and port == 80)):
            host = "%s:%s" % (host, port)

        return host

    @property
    def base(self):
        host = self.host
        port = self.port
        scheme = self.scheme

        return "%s://%s" % (scheme, host)

class Server(BaseServer):

    def __init__(self, bind, **kwargs):
        super(Server, self).__init__(bind, **kwargs)

        docroot = kwargs.get("docroot", None)
        self.dispatcher = Dispatcher(docroot=docroot)
        self += self.dispatcher

    def __get_docroot(self):
        return self.dispatcher.docroot if hasattr(self, "dispatcher") else None

    def __set_docroot(self, docroot):
        if os.path.exists(docroot):
            self.dispatcher.docroot = docroot
        else:
            raise IOError(2, "Invalid docroot path", docroot)

    docroot = property(__get_docroot, __set_docroot, None, """\
            Document Root of this Server's Dispatcher.""")
<commit_msg>web.servers: Fixed a typo with ssl port no. (Thanks riskable).<commit_after># Module:   server
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Web Server

This module implements the Web Server Component.
"""

import os
from socket import gethostname as _gethostname

from circuits import Component

from circuits.net.sockets import TCPServer

from http import HTTP
from wrappers import Request, Host
from constants import SERVER_VERSION
from dispatchers import Dispatcher

class BaseServer(Component):

    channel = "web"

    def __init__(self, bind, **kwargs):
        super(BaseServer, self).__init__(**kwargs)

        self.server = (TCPServer(bind, **kwargs) + HTTP())
        self += self.server

        Request.server = self
        Request.local = Host(self.server.bind[0], self.server.bind[1])
        Request.host = self.host

        print "%s listening on %s" % (self.version, self.base)

    @property
    def version(self):
        return SERVER_VERSION

    @property
    def address(self):
        return self.server.bind[0] if hasattr(self, "server") else None

    @property
    def port(self):
        return self.server.bind[1] if hasattr(self, "server") else None

    @property
    def ssl(self):
        return self.server.ssl if hasattr(self, "server") else None

    @property
    def scheme(self):
        return "https" if self.ssl else "http"

    @property
    def host(self):
        host = self.address

        if host in ("0.0.0.0", "::", ""):
            # 0.0.0.0 is INADDR_ANY and :: is IN6ADDR_ANY.
            # Look up the host name, which should be the
            # safest thing to spit out in a URL.
            host = _gethostname()

        ssl = self.ssl
        port = self.port

        if not ((ssl and port == 443) or (not ssl and port == 80)):
            host = "%s:%s" % (host, port)

        return host

    @property
    def base(self):
        host = self.host
        port = self.port
        scheme = self.scheme

        return "%s://%s" % (scheme, host)

class Server(BaseServer):

    def __init__(self, bind, **kwargs):
        super(Server, self).__init__(bind, **kwargs)

        docroot = kwargs.get("docroot", None)
        self.dispatcher = Dispatcher(docroot=docroot)
        self += self.dispatcher

    def __get_docroot(self):
        return self.dispatcher.docroot if hasattr(self, "dispatcher") else None

    def __set_docroot(self, docroot):
        if os.path.exists(docroot):
            self.dispatcher.docroot = docroot
        else:
            raise IOError(2, "Invalid docroot path", docroot)

    docroot = property(__get_docroot, __set_docroot, None, """\
            Document Root of this Server's Dispatcher.""")
<|endoftext|>


<commit_before>#
# Copyright (C) 2007-2013 by Johan De Taeye, frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os.path
import sys
from datetime import datetime
from subprocess import Popen

from django.conf import settings
from django.views.decorators.cache import never_cache
from django.shortcuts import render
from django.db.models import get_apps
from django.utils.translation import ugettext_lazy as _
from django.db import DEFAULT_DB_ALIAS
from django.contrib.admin.views.decorators import staff_member_required
from django.views.decorators.csrf import csrf_protect
from django.http import Http404, HttpResponseRedirect, HttpResponseServerError, HttpResponse
from django.contrib import messages
from django.utils.encoding import force_unicode

from freppledb.execute.models import Task, Scenario
from freppledb.common.report import exportWorkbook, importWorkbook
from freppledb.common.report import GridReport, GridFieldDateTime, GridFieldText, GridFieldInteger
from freppledb.execute.management.commands.frepple_runworker import checkActive

import logging
logger = logging.getLogger(__name__)

class TaskReport(GridReport):
  '''
  A list report to review the history of actions.
  '''
  template = 'execute/execute.html'
  title = _('Task status')
  basequeryset = Task.objects.all()
  model = Task
  frozenColumns = 0
  multiselect = False
  editable = False
  height = 150
  default_sort = (0, 'desc')

  rows = (
    GridFieldInteger('id', title=_('identifier'), key=True),
    GridFieldText('name', title=_('name'), editable=False, align='center'),
    GridFieldDateTime('submitted', title=_('submitted'), editable=False, align='center'),
    GridFieldDateTime('started', title=_('started'), editable=False, align='center'),
    GridFieldDateTime('finished', title=_('finished'), editable=False, align='center'),
    GridFieldText('status', title=_('status'), editable=False, align='center', extra="formatter:status"),
    GridFieldText('message', title=_('message'), editable=False, width=500),
    GridFieldText('arguments', title=_('arguments'), editable=False),
    GridFieldText('user', title=_('user'), field_name='user__username', editable=False, align='center'),
    )

  @classmethod
  def extra_context(reportclass, request, *args, **kwargs):
    try:
      constraint = int(request.session['constraint'])
    except:
      constraint = 15

    # Synchronize the scenario table with the settings
    Scenario.syncWithSettings()

    # Loop over all fixtures of all apps and directories
    fixtures = set()
    folders = list(settings.FIXTURE_DIRS)
    for app in get_apps():
      if app.__name__.startswith('django'):
        continue
      folders.append(os.path.join(os.path.dirname(app.__file__), 'fixtures'))
    for f in folders:
      try:
        for root, dirs, files in os.walk(f):
          for i in files:
            if i.endswith('.json'):
              fixtures.add(i.split('.')[0])
      except:
        pass  # Silently ignore failures
    fixtures = sorted(fixtures)

    # Send to template
    return {'capacityconstrained': constraint & 4,
            'materialconstrained': constraint & 2,
            'leadtimeconstrained': constraint & 1,
            'fenceconstrained': constraint & 8,
            'scenarios': Scenario.objects.all(),
            'fixtures': fixtures,
            'openbravo': 'freppledb.openbravo' in settings.INSTALLED_APPS,
            'odoo': 'freppledb.odoo' in settings.INSTALLED_APPS,
            }

@staff_member_required
@never_cache
@csrf_protect
def LaunchTask(request, action):
  try:
    if action == 'exportworkbook':
      return exportWorkbook(request)
    elif action == 'importworkbook':
      return importWorkbook(request)
    else:
      wrapTask(request, action)
      return HttpResponseRedirect('%s/execute/' % request.prefix)
  except Exception as e:
    messages.add_message(
      request, messages.ERROR,
      force_unicode(_('Failure launching action: %(msg)s') % {'msg': e})
      )
    return HttpResponseRedirect('%s/execute/' % request.prefix)

def wrapTask(request, action):
  # Allow only post
  if request.method != 'POST':
    raise Exception('Only post requests allowed')

  # Check user permissions
  if not request.user.has_perm('execute'):
    raise Exception('Missing execution privileges')

  # Parse the posted parameters as arguments for an asynchronous task to add to the queue.    TODO MAKE MODULAR WITH SEPERATE TASK CLASS
  worker_database = request.database

  now = datetime.now()
  task = None
  # A
  if action == 'frepple_run':
    if not request.user.has_perm('execute.generate_plan'):
      raise Exception('Missing execution privileges')
    constraint = 0
    for value in request.POST.getlist('constraint'):
      try:
        constraint += int(value)
      except:
        pass
    task = Task(name='generate plan', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--constraint=%s --plantype=%s" % (constraint, request.POST.get('plantype'))
    env = []
    if request.POST.get('odoo_read', None) == u'1':
      env.append("odoo_read")
    if request.POST.get('odoo_write', None) == u'1':
      env.append("odoo_write")
    if env:
      task.arguments = "%s --env=%s" % (task.arguments, ','.join(env))
    task.save(using=request.database)
    # Update the session object   TODO REPLACE WITH PREFERENCE INFO
    request.session['plantype'] = request.POST.get('plantype')
    request.session['constraint'] = constraint
  # B
  elif action == 'frepple_createmodel':
    task = Task(name='generate model', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--cluster=%s --demand=%s --forecast_per_item=%s --level=%s --resource=%s " \
      "--resource_size=%s --components=%s --components_per=%s --deliver_lt=%s --procure_lt=%s" % (
        request.POST['clusters'], request.POST['demands'], request.POST['fcst'], request.POST['levels'],
        request.POST['rsrc_number'], request.POST['rsrc_size'], request.POST['components'],
        request.POST['components_per'], request.POST['deliver_lt'], request.POST['procure_lt']
        )
    task.save(using=request.database)
  # C
  elif action == 'frepple_flush':
    task = Task(name='empty database', submitted=now, status='Waiting', user=request.user)
    if not request.POST.get('all'):
      task.arguments = "--models=%s" % ','.join(request.POST.getlist('entities'))
    task.save(using=request.database)
  # D
  elif action == 'loaddata':
    task = Task(name='load dataset', submitted=now, status='Waiting', user=request.user, arguments=request.POST['datafile'])
    task.save(using=request.database)
  # E
  elif action == 'frepple_copy':
    worker_database = DEFAULT_DB_ALIAS
    if 'copy' in request.POST:
      if not request.user.has_perm('execute.copy_scenario'):
        raise Exception('Missing execution privileges')
      source = request.POST.get('source', DEFAULT_DB_ALIAS)
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == 'on' and sc.status == u'Free':
          task = Task(name='copy scenario', submitted=now, status='Waiting', user=request.user, arguments="%s %s" % (source, sc.name))
          task.save()
    elif 'release' in request.POST:
      # Note: release is immediate and synchronous.
      if not request.user.has_perm('execute.release_scenario'):
        raise Exception('Missing execution privileges')
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == u'on' and sc.status != u'Free':
          sc.status = u'Free'
          sc.lastrefresh = now
          sc.save()
          if request.database == sc.name:
            # Erasing the database that is currently selected.
            request.prefix = ''
    elif 'update' in request.POST:
      # Note: update is immediate and synchronous.
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == 'on':
          sc.description = request.POST.get('description', None)
          sc.save()
    else:
      raise Exception('Invalid scenario task')
  # F
  elif action == 'frepple_backup':
    task = Task(name='backup database', submitted=now, status='Waiting', user=request.user)
    task.save(using=request.database)
  # G
  elif action == 'frepple_createbuckets':
    task = Task(name='generate buckets', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--start=%s --end=%s --weekstart=%s" % (
      request.POST['start'], request.POST['end'], request.POST['weekstart']
      )
    task.save(using=request.database)
  # H
  elif action == 'openbravo_import' and 'freppledb.openbravo' in settings.INSTALLED_APPS:
    task = Task(name='Openbravo import', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--delta=%s" % request.POST['delta']
    task.save(using=request.database)
  # I
  elif action == 'openbravo_export' and 'freppledb.openbravo' in settings.INSTALLED_APPS:
    task = Task(name='Openbravo export', submitted=now, status='Waiting', user=request.user)
    task.save(using=request.database)
  else:
    # Task not recognized
    raise Exception('Invalid launching task')

  # Launch a worker process
  if task and not checkActive(worker_database):
    if os.path.isfile(os.path.join(settings.FREPPLE_APP, "frepplectl.py")):
      if "python" in sys.executable:
        # Development layout
        Popen([
          sys.executable,  # Python executable
          os.path.join(settings.FREPPLE_APP, "frepplectl.py"),
          "frepple_runworker",
          "--database=%s" % worker_database
          ])
      else:
        # Deployment on Apache web server
        Popen([
          "python",
          os.path.join(settings.FREPPLE_APP, "frepplectl.py"),
          "frepple_runworker",
          "--database=%s" % worker_database
          ], creationflags=0x08000000)
    elif sys.executable.find('freppleserver.exe') >= 0:
      # Py2exe executable
      Popen([
        sys.executable.replace('freppleserver.exe', 'frepplectl.exe'),  # frepplectl executable
        "frepple_runworker",
        "--database=%s" % worker_database
        ], creationflags=0x08000000)  # Do not create a console window
    else:
      # Linux standard installation
      Popen([
        "frepplectl",
        "frepple_runworker",
        "--database=%s" % worker_database
        ])
  return task

@staff_member_required
@never_cache
@csrf_protect
def CancelTask(request, taskid):
  # Allow only post
  if request.method != 'POST'or not request.is_ajax():
    raise Http404('Only ajax post requests allowed')
  try:
    task = Task.objects.all().using(request.database).get(pk=taskid)
    if task.status != 'Waiting':
      raise Exception('Task is not in waiting status')
    task.status = 'Canceled'
    task.save(using=request.database)
    return HttpResponse(content="OK")
  except Exception as e:
    logger.error("Error saving report settings: %s" % e)
    return HttpResponseServerError('Error canceling task')

@staff_member_required
@never_cache
def logfile(request):
  '''
  This view shows the frePPLe log file of the last planning run in this database.
  '''
  try:
    if request.database == DEFAULT_DB_ALIAS:
      f = open(os.path.join(settings.FREPPLE_LOGDIR, 'frepple.log'), 'rb')
    else:
      f = open(os.path.join(settings.FREPPLE_LOGDIR, 'frepple_%s.log' % request.database), 'rb')
  except:
    logdata = "File not found"
  else:
    try:
      f.seek(-1, os.SEEK_END)
      if f.tell() >= 50000:
        # Too big to display completely
        f.seek(-50000, os.SEEK_END)
        logdata = force_unicode(_("Displaying only the last 50K from the log file")) + '...\n\n...' + force_unicode(f.read(50000))
      else:
        # Displayed completely
        f.seek(0, os.SEEK_SET)
        logdata = f.read(50000)
    finally:
      f.close()

  return render(request, 'execute/logfrepple.html', {
    'title': _('Log file'),
    'logdata': logdata,
    } )
<commit_msg>View log file: strip first incomplete line<commit_after>#
# Copyright (C) 2007-2013 by Johan De Taeye, frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os.path
import sys
from datetime import datetime
from subprocess import Popen

from django.conf import settings
from django.views.decorators.cache import never_cache
from django.shortcuts import render
from django.db.models import get_apps
from django.utils.translation import ugettext_lazy as _
from django.db import DEFAULT_DB_ALIAS
from django.contrib.admin.views.decorators import staff_member_required
from django.views.decorators.csrf import csrf_protect
from django.http import Http404, HttpResponseRedirect, HttpResponseServerError, HttpResponse
from django.contrib import messages
from django.utils.encoding import force_unicode

from freppledb.execute.models import Task, Scenario
from freppledb.common.report import exportWorkbook, importWorkbook
from freppledb.common.report import GridReport, GridFieldDateTime, GridFieldText, GridFieldInteger
from freppledb.execute.management.commands.frepple_runworker import checkActive

import logging
logger = logging.getLogger(__name__)

class TaskReport(GridReport):
  '''
  A list report to review the history of actions.
  '''
  template = 'execute/execute.html'
  title = _('Task status')
  basequeryset = Task.objects.all()
  model = Task
  frozenColumns = 0
  multiselect = False
  editable = False
  height = 150
  default_sort = (0, 'desc')

  rows = (
    GridFieldInteger('id', title=_('identifier'), key=True),
    GridFieldText('name', title=_('name'), editable=False, align='center'),
    GridFieldDateTime('submitted', title=_('submitted'), editable=False, align='center'),
    GridFieldDateTime('started', title=_('started'), editable=False, align='center'),
    GridFieldDateTime('finished', title=_('finished'), editable=False, align='center'),
    GridFieldText('status', title=_('status'), editable=False, align='center', extra="formatter:status"),
    GridFieldText('message', title=_('message'), editable=False, width=500),
    GridFieldText('arguments', title=_('arguments'), editable=False),
    GridFieldText('user', title=_('user'), field_name='user__username', editable=False, align='center'),
    )

  @classmethod
  def extra_context(reportclass, request, *args, **kwargs):
    try:
      constraint = int(request.session['constraint'])
    except:
      constraint = 15

    # Synchronize the scenario table with the settings
    Scenario.syncWithSettings()

    # Loop over all fixtures of all apps and directories
    fixtures = set()
    folders = list(settings.FIXTURE_DIRS)
    for app in get_apps():
      if app.__name__.startswith('django'):
        continue
      folders.append(os.path.join(os.path.dirname(app.__file__), 'fixtures'))
    for f in folders:
      try:
        for root, dirs, files in os.walk(f):
          for i in files:
            if i.endswith('.json'):
              fixtures.add(i.split('.')[0])
      except:
        pass  # Silently ignore failures
    fixtures = sorted(fixtures)

    # Send to template
    return {'capacityconstrained': constraint & 4,
            'materialconstrained': constraint & 2,
            'leadtimeconstrained': constraint & 1,
            'fenceconstrained': constraint & 8,
            'scenarios': Scenario.objects.all(),
            'fixtures': fixtures,
            'openbravo': 'freppledb.openbravo' in settings.INSTALLED_APPS,
            'odoo': 'freppledb.odoo' in settings.INSTALLED_APPS,
            }

@staff_member_required
@never_cache
@csrf_protect
def LaunchTask(request, action):
  try:
    if action == 'exportworkbook':
      return exportWorkbook(request)
    elif action == 'importworkbook':
      return importWorkbook(request)
    else:
      wrapTask(request, action)
      return HttpResponseRedirect('%s/execute/' % request.prefix)
  except Exception as e:
    messages.add_message(
      request, messages.ERROR,
      force_unicode(_('Failure launching action: %(msg)s') % {'msg': e})
      )
    return HttpResponseRedirect('%s/execute/' % request.prefix)

def wrapTask(request, action):
  # Allow only post
  if request.method != 'POST':
    raise Exception('Only post requests allowed')

  # Check user permissions
  if not request.user.has_perm('execute'):
    raise Exception('Missing execution privileges')

  # Parse the posted parameters as arguments for an asynchronous task to add to the queue.    TODO MAKE MODULAR WITH SEPERATE TASK CLASS
  worker_database = request.database

  now = datetime.now()
  task = None
  # A
  if action == 'frepple_run':
    if not request.user.has_perm('execute.generate_plan'):
      raise Exception('Missing execution privileges')
    constraint = 0
    for value in request.POST.getlist('constraint'):
      try:
        constraint += int(value)
      except:
        pass
    task = Task(name='generate plan', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--constraint=%s --plantype=%s" % (constraint, request.POST.get('plantype'))
    env = []
    if request.POST.get('odoo_read', None) == u'1':
      env.append("odoo_read")
    if request.POST.get('odoo_write', None) == u'1':
      env.append("odoo_write")
    if env:
      task.arguments = "%s --env=%s" % (task.arguments, ','.join(env))
    task.save(using=request.database)
    # Update the session object   TODO REPLACE WITH PREFERENCE INFO
    request.session['plantype'] = request.POST.get('plantype')
    request.session['constraint'] = constraint
  # B
  elif action == 'frepple_createmodel':
    task = Task(name='generate model', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--cluster=%s --demand=%s --forecast_per_item=%s --level=%s --resource=%s " \
      "--resource_size=%s --components=%s --components_per=%s --deliver_lt=%s --procure_lt=%s" % (
        request.POST['clusters'], request.POST['demands'], request.POST['fcst'], request.POST['levels'],
        request.POST['rsrc_number'], request.POST['rsrc_size'], request.POST['components'],
        request.POST['components_per'], request.POST['deliver_lt'], request.POST['procure_lt']
        )
    task.save(using=request.database)
  # C
  elif action == 'frepple_flush':
    task = Task(name='empty database', submitted=now, status='Waiting', user=request.user)
    if not request.POST.get('all'):
      task.arguments = "--models=%s" % ','.join(request.POST.getlist('entities'))
    task.save(using=request.database)
  # D
  elif action == 'loaddata':
    task = Task(name='load dataset', submitted=now, status='Waiting', user=request.user, arguments=request.POST['datafile'])
    task.save(using=request.database)
  # E
  elif action == 'frepple_copy':
    worker_database = DEFAULT_DB_ALIAS
    if 'copy' in request.POST:
      if not request.user.has_perm('execute.copy_scenario'):
        raise Exception('Missing execution privileges')
      source = request.POST.get('source', DEFAULT_DB_ALIAS)
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == 'on' and sc.status == u'Free':
          task = Task(name='copy scenario', submitted=now, status='Waiting', user=request.user, arguments="%s %s" % (source, sc.name))
          task.save()
    elif 'release' in request.POST:
      # Note: release is immediate and synchronous.
      if not request.user.has_perm('execute.release_scenario'):
        raise Exception('Missing execution privileges')
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == u'on' and sc.status != u'Free':
          sc.status = u'Free'
          sc.lastrefresh = now
          sc.save()
          if request.database == sc.name:
            # Erasing the database that is currently selected.
            request.prefix = ''
    elif 'update' in request.POST:
      # Note: update is immediate and synchronous.
      for sc in Scenario.objects.all():
        if request.POST.get(sc.name, 'off') == 'on':
          sc.description = request.POST.get('description', None)
          sc.save()
    else:
      raise Exception('Invalid scenario task')
  # F
  elif action == 'frepple_backup':
    task = Task(name='backup database', submitted=now, status='Waiting', user=request.user)
    task.save(using=request.database)
  # G
  elif action == 'frepple_createbuckets':
    task = Task(name='generate buckets', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--start=%s --end=%s --weekstart=%s" % (
      request.POST['start'], request.POST['end'], request.POST['weekstart']
      )
    task.save(using=request.database)
  # H
  elif action == 'openbravo_import' and 'freppledb.openbravo' in settings.INSTALLED_APPS:
    task = Task(name='Openbravo import', submitted=now, status='Waiting', user=request.user)
    task.arguments = "--delta=%s" % request.POST['delta']
    task.save(using=request.database)
  # I
  elif action == 'openbravo_export' and 'freppledb.openbravo' in settings.INSTALLED_APPS:
    task = Task(name='Openbravo export', submitted=now, status='Waiting', user=request.user)
    task.save(using=request.database)
  else:
    # Task not recognized
    raise Exception('Invalid launching task')

  # Launch a worker process
  if task and not checkActive(worker_database):
    if os.path.isfile(os.path.join(settings.FREPPLE_APP, "frepplectl.py")):
      if "python" in sys.executable:
        # Development layout
        Popen([
          sys.executable,  # Python executable
          os.path.join(settings.FREPPLE_APP, "frepplectl.py"),
          "frepple_runworker",
          "--database=%s" % worker_database
          ])
      else:
        # Deployment on Apache web server
        Popen([
          "python",
          os.path.join(settings.FREPPLE_APP, "frepplectl.py"),
          "frepple_runworker",
          "--database=%s" % worker_database
          ], creationflags=0x08000000)
    elif sys.executable.find('freppleserver.exe') >= 0:
      # Py2exe executable
      Popen([
        sys.executable.replace('freppleserver.exe', 'frepplectl.exe'),  # frepplectl executable
        "frepple_runworker",
        "--database=%s" % worker_database
        ], creationflags=0x08000000)  # Do not create a console window
    else:
      # Linux standard installation
      Popen([
        "frepplectl",
        "frepple_runworker",
        "--database=%s" % worker_database
        ])
  return task

@staff_member_required
@never_cache
@csrf_protect
def CancelTask(request, taskid):
  # Allow only post
  if request.method != 'POST'or not request.is_ajax():
    raise Http404('Only ajax post requests allowed')
  try:
    task = Task.objects.all().using(request.database).get(pk=taskid)
    if task.status != 'Waiting':
      raise Exception('Task is not in waiting status')
    task.status = 'Canceled'
    task.save(using=request.database)
    return HttpResponse(content="OK")
  except Exception as e:
    logger.error("Error saving report settings: %s" % e)
    return HttpResponseServerError('Error canceling task')

@staff_member_required
@never_cache
def logfile(request):
  '''
  This view shows the frePPLe log file of the last planning run in this database.
  '''
  try:
    if request.database == DEFAULT_DB_ALIAS:
      f = open(os.path.join(settings.FREPPLE_LOGDIR, 'frepple.log'), 'rb')
    else:
      f = open(os.path.join(settings.FREPPLE_LOGDIR, 'frepple_%s.log' % request.database), 'rb')
  except:
    logdata = "File not found"
  else:
    try:
      f.seek(-1, os.SEEK_END)
      if f.tell() >= 50000:
        # Too big to display completely
        f.seek(-50000, os.SEEK_END)
        d = f.read(50000)
        d = d[d.index('\n'):] # Strip the first, incomplete line
        logdata = force_unicode(_("Displaying only the last 50K from the log file")) + '...\n\n...' + force_unicode(d)
      else:
        # Displayed completely
        f.seek(0, os.SEEK_SET)
        logdata = f.read(50000)
    finally:
      f.close()

  return render(request, 'execute/logfrepple.html', {
    'title': _('Log file'),
    'logdata': logdata,
    } )
<|endoftext|>