<commit_before># xpyBuild - eXtensible Python-based Build System
#
# This module holds definitions that are used throughout the build system, and
# typically all names from this module will be imported.
#
# Copyright (c) 2013 - 2017, 2019 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: buildcommon.py 301527 2017-02-06 15:31:43Z matj $
#

import traceback, os, sys, locale, inspect, io
import re
import platform

import logging
# do NOT define a 'log' variable here or targets will use it by mistake

from xpybuild.utils.flatten import flatten
import xpybuild.utils.fileutils
from xpybuild.utils.fileutils import parsePropertiesFile

def __getXpybuildVersion():

	try:
		with open(os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "XPYBUILD_VERSION")) as f:
			return f.read().strip()
	except Exception:
		raise
		return "<unknown>"
XPYBUILD_VERSION: str = __getXpybuildVersion()
"""The current xpybuild version."""

def include(file):
	""" Parse and register the targets and properties in in the specified
	xpybuild.py file.

	Targets should only be defined in files included using this method,
	not using python import statements.

	@param file: a path relative to the directory containing this file.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	from xpybuild.utils.buildfilelocation import BuildFileLocation

	file = getBuildInitializationContext().expandPropertyValues(file)

	assert file.endswith('.xpybuild.py') # enforce recommended naming convention

	filepath = getBuildInitializationContext().getFullPath(file, os.path.dirname(BuildFileLocation._currentBuildFile[-1]))

	BuildFileLocation._currentBuildFile.append(filepath) # add to stack of files being parsed

	namespace = {}
	exec(compile(open(filepath, "rb").read(), filepath, 'exec'), namespace, namespace)

	del BuildFileLocation._currentBuildFile[-1]

	return namespace

def requireXpyBuildVersion(version: str):
	""" Checks that this xpybuild is at least a certain version number. """
	from xpybuild.utils.stringutils import compareVersions
	if compareVersions(XPYBUILD_VERSION, version) < 0: raise Exception("This build file requires xpyBuild at least version "+version+" but this is xpyBuild "+XPYBUILD_VERSION)

isDirPath = xpybuild.utils.fileutils.isDirPath
""" Returns true if the path is a directory (ends with / or \\). """

def normpath(path):
	""" Normalizes the specified file or dir path to remove ".." sequences and
	differences in the capitalization of Windows drive letters.

	Does not add Windows long-path safety or absolutization.

	Leaves in place any  trailing platform-appropriate character to indicate
	directory if appropriate.

	See also L{xpybuild.utils.fileutils.normLongPath} and L{xpybuild.utils.fileutils.toLongPathSafe}.

	"""
	path = os.path.normpath(path)+(os.path.sep if isDirPath(path) else '')

	# normpath does nothing to normalize case, and windows seems to be quite random about upper/lower case
	# for drive letters (more so than directory names), with different cmd prompts frequently using different
	# capitalization, so normalize at least that bit, to prevent spurious rebuilding from different prompts
	if len(path)>2 and IS_WINDOWS and path[1] == ':':
		path = path[0].lower()+path[1:]

	return path

""" A boolean that specifies whether this is Windows or some other operating system. """
IS_WINDOWS = platform.system()=='Windows'
# (we won't want constants for every possible OS here, but since there is so much conditionalization between
# windows and unix-based systems, much of it on the critical path, it is worthwhile having a constant for this).

if IS_WINDOWS:
	def isWindows():
		""" Returns True if this is a windows platform.
		@deprecated: Use the IS_WINDOWS constant instead.
		"""
		return True
else:
	def isWindows():
		""" Returns True if this is a windows platform.
		@deprecated: Use the IS_WINDOWS constant instead.
		"""
		return False

_stdoutEncoding = None
try:
	_stdoutEncoding = sys.stdout.encoding or locale.getpreferredencoding() # stdout encoding will be None unless in a terminal
except:
	pass # probably in epydoc

def getStdoutEncoding():
	""" Returns the most likely encoding used by subprocesses, based on
	whether the build is running in a console, etc
	which is typically what should be used for converting byte strings from
	subprocesses to python unicode.
	"""
	return _stdoutEncoding

def formatFileLocation(path, lineNumber):
	""" A functor to format a file and a line number for output. """
	# use format vim uses for opening file at line number; easy enough to convert to ultra edit path/linenumber syntax
	return '"%s" +%d' % (os.path.normpath(path), lineNumber)

def defineAtomicTargetGroup(*targets):
	""" The given targets must all be built before anything which depends on any of those targets.

	Returns the flattened list of targets.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	targets = flatten(targets)
	getBuildInitializationContext().defineAtomicTargetGroup(targets)
	return targets

def registerPreBuildCheck(fn):
	""" Defines a check which will be called after any clean but before any build actions take place.
	    fn should be a functor that takes a context and raises a BuildException if the check fails. """
	from buildcontext import getBuildInitializationContext
	getBuildInitializationContext().registerPreBuildCheck(fn)

class StringFormatter(object):
	""" A simple named functor for applying a %s-style string format, useful
	in situations where a function is needed to add a suffix/prefix for the
	value of an option.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'StringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return self.fmt % args[0]

class FilenameStringFormatter(object):
	""" A simple named functor for applying a %s-style string format.
		 Formatter is just applied to the basename part of the filename,
		 the dirname part is preserved as-is.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'FilenameStringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return os.path.join(os.path.dirname(args[0]), self.fmt % os.path.basename(args[0]))
	<commit_msg>minor formatting improvement<commit_after># xpyBuild - eXtensible Python-based Build System
#
# This module holds definitions that are used throughout the build system, and
# typically all names from this module will be imported.
#
# Copyright (c) 2013 - 2017, 2019 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: buildcommon.py 301527 2017-02-06 15:31:43Z matj $
#

import traceback, os, sys, locale, inspect, io
import re
import platform

import logging
# do NOT define a 'log' variable here or targets will use it by mistake

from xpybuild.utils.flatten import flatten
import xpybuild.utils.fileutils
from xpybuild.utils.fileutils import parsePropertiesFile

def __getXpybuildVersion():

	try:
		with open(os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "XPYBUILD_VERSION")) as f:
			return f.read().strip()
	except Exception:
		raise
		return "<unknown>"
XPYBUILD_VERSION: str = __getXpybuildVersion()
"""The current xpybuild version."""

def include(file):
	""" Parse and register the targets and properties in in the specified
	xpybuild.py file.

	Targets should only be defined in files included using this method,
	not using python import statements.

	@param file: a path relative to the directory containing this file.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	from xpybuild.utils.buildfilelocation import BuildFileLocation

	file = getBuildInitializationContext().expandPropertyValues(file)

	assert file.endswith('.xpybuild.py') # enforce recommended naming convention

	filepath = getBuildInitializationContext().getFullPath(file, os.path.dirname(BuildFileLocation._currentBuildFile[-1]))

	BuildFileLocation._currentBuildFile.append(filepath) # add to stack of files being parsed

	namespace = {}
	exec(compile(open(filepath, "rb").read(), filepath, 'exec'), namespace, namespace)

	del BuildFileLocation._currentBuildFile[-1]

	return namespace

def requireXpyBuildVersion(version: str):
	""" Checks that this xpybuild is at least a certain version number. """
	from xpybuild.utils.stringutils import compareVersions
	if compareVersions(XPYBUILD_VERSION, version) < 0: raise Exception("This build file requires xpyBuild at least version "+version+" but this is xpyBuild "+XPYBUILD_VERSION)

isDirPath = xpybuild.utils.fileutils.isDirPath
"""Returns true if the path is a directory (ends with a slash, / or \\\\). """

def normpath(path):
	""" Normalizes the specified file or dir path to remove ".." sequences and
	differences in the capitalization of Windows drive letters.

	Does not add Windows long-path safety or absolutization.

	Leaves in place any  trailing platform-appropriate character to indicate
	directory if appropriate.

	See also L{xpybuild.utils.fileutils.normLongPath} and L{xpybuild.utils.fileutils.toLongPathSafe}.

	"""
	path = os.path.normpath(path)+(os.path.sep if isDirPath(path) else '')

	# normpath does nothing to normalize case, and windows seems to be quite random about upper/lower case
	# for drive letters (more so than directory names), with different cmd prompts frequently using different
	# capitalization, so normalize at least that bit, to prevent spurious rebuilding from different prompts
	if len(path)>2 and IS_WINDOWS and path[1] == ':':
		path = path[0].lower()+path[1:]

	return path

IS_WINDOWS: bool = platform.system()=='Windows'
""" A boolean that specifies whether this is Windows or some other operating system. """
# (we won't want constants for every possible OS here, but since there is so much conditionalization between
# windows and unix-based systems, much of it on the critical path, it is worthwhile having a constant for this).

if IS_WINDOWS:
	def isWindows():
		""" Returns True if this is a windows platform.
		@deprecated: Use the IS_WINDOWS constant instead.
		"""
		return True
else:
	def isWindows():
		""" Returns True if this is a windows platform.
		@deprecated: Use the IS_WINDOWS constant instead.
		"""
		return False

_stdoutEncoding = None
try:
	_stdoutEncoding = sys.stdout.encoding or locale.getpreferredencoding() # stdout encoding will be None unless in a terminal
except:
	pass # probably in epydoc

def getStdoutEncoding():
	""" Returns the most likely encoding used by subprocesses, based on
	whether the build is running in a console, etc
	which is typically what should be used for converting byte strings from
	subprocesses to python unicode.
	"""
	return _stdoutEncoding

def formatFileLocation(path, lineNumber):
	""" A functor to format a file and a line number for output. """
	# use format vim uses for opening file at line number; easy enough to convert to ultra edit path/linenumber syntax
	return '"%s" +%d' % (os.path.normpath(path), lineNumber)

def defineAtomicTargetGroup(*targets):
	""" The given targets must all be built before anything which depends on any of those targets.

	Returns the flattened list of targets.
	"""

	from xpybuild.buildcontext import getBuildInitializationContext
	targets = flatten(targets)
	getBuildInitializationContext().defineAtomicTargetGroup(targets)
	return targets

def registerPreBuildCheck(fn):
	""" Defines a check which will be called after any clean but before any build actions take place.
	    fn should be a functor that takes a context and raises a BuildException if the check fails. """
	from buildcontext import getBuildInitializationContext
	getBuildInitializationContext().registerPreBuildCheck(fn)

class StringFormatter(object):
	""" A simple named functor for applying a %s-style string format, useful
	in situations where a function is needed to add a suffix/prefix for the
	value of an option.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'StringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return self.fmt % args[0]

class FilenameStringFormatter(object):
	""" A simple named functor for applying a %s-style string format.
		 Formatter is just applied to the basename part of the filename,
		 the dirname part is preserved as-is.
	"""
	def __init__(self, formatstring):
		self.fmt = formatstring
	def __repr__(self):
		return 'FilenameStringFormatter<"%s">'%self.fmt
	def __call__(self, *args, **kwargs):
		assert not kwargs
		assert len(args)==1
		return os.path.join(os.path.dirname(args[0]), self.fmt % os.path.basename(args[0]))
	<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

# Copyright (c) 2012 theo crevon
#
# See the file LICENSE for copying permission.

import time

from collections import namedtuple
from functools import wraps
from inspect import getmembers, ismethod

ExecTimeCollection = namedtuple('ExecTimeCollection', ['times', 'scale'])

def extra_setup(setup_code):
    """Allows to setup some extra context to it's decorated function.

    As a convention, the bench decorated function should always handle
    *args and **kwargs. Kwargs will be updated with the extra context
    set by the decorator.

    Example:
        @extra_setup("l = [x for x in xrange(100)]")
        def bench_len(self, *args, **kwargs):
            print len(kwargs['l'])
    """
    def decorator(func):
        @wraps(func)
        def decorated_function(*args, **kwargs):
            exec setup_code in {}, kwargs
            return func(*args, **kwargs)
        return decorated_function
    return decorator

class BenchCase(object):
    def __init__(self):
        self._benchmarks = []
        self.results = {
            'exec_times': {},
            'averages': {},
        }

    def setUp(self):
        """Hook method for setting up the benchmark
        fixture before exercising it."""
        pass

    def tearDown(self):
        """Hook method for deconstructing the benchmark
        fixture after testing it."""
        pass

    @property
    def benchmarks(self):
        if not self._benchmarks:
            bench_case_methods = getmembers(self.__class__, predicate=ismethod)
            for (method_name, method_value) in bench_case_methods:
                if method_name.startswith('bench_'):
                    self._benchmarks.append((method_name, method_value))
        return self._benchmarks

    def tick(self, func, *args, **kwargs):
        """Times a function execution in miliseconds"""
        start = time.time()
        func(*args, **kwargs)
        end = time.time()
        exec_time = round(((end - start) * 1000), 2)

        return exec_time

    def run(self, repeat=10):
        for method_name, method_value in self.benchmarks:
            self.setUp()

            exec_times = ExecTimeCollection(times=[self.tick(method_value, self) for x in [0.0] * repeat],
                                                              scale='ms')
            average = sum(exec_times.times) / repeat

            self.results['exec_times'].update({method_name: exec_times})
            self.results['averages'].update({method_name: average})

            self.tearDown()
<commit_msg>Update : write benchmarks results on stdout<commit_after># -*- coding:utf-8 -*-

# Copyright (c) 2012 theo crevon
#
# See the file LICENSE for copying permission.

import sys
import time

from collections import namedtuple
from functools import wraps
from inspect import getmembers, ismethod

ExecTimeCollection = namedtuple('ExecTimeCollection', ['times', 'scale'])

def extra_setup(setup_code):
    """Allows to setup some extra context to it's decorated function.

    As a convention, the bench decorated function should always handle
    *args and **kwargs. Kwargs will be updated with the extra context
    set by the decorator.

    Example:
        @extra_setup("l = [x for x in xrange(100)]")
        def bench_len(self, *args, **kwargs):
            print len(kwargs['l'])
    """
    def decorator(func):
        @wraps(func)
        def decorated_function(*args, **kwargs):
            exec setup_code in {}, kwargs
            return func(*args, **kwargs)
        return decorated_function
    return decorator

class BenchCase(object):
    def __init__(self):
        self._benchmarks = []
        self.results = {
            'exec_times': {},
            'averages': {},
        }

    def setUp(self):
        """Hook method for setting up the benchmark
        fixture before exercising it."""
        pass

    def tearDown(self):
        """Hook method for deconstructing the benchmark
        fixture after testing it."""
        pass

    @property
    def benchmarks(self):
        if not self._benchmarks:
            bench_case_methods = getmembers(self.__class__, predicate=ismethod)
            for (method_name, method_value) in bench_case_methods:
                if method_name.startswith('bench_'):
                    self._benchmarks.append((method_name, method_value))
        return self._benchmarks

    def tick(self, func, *args, **kwargs):
        """Times a function execution in miliseconds"""
        start = time.time()
        func(*args, **kwargs)
        end = time.time()
        exec_time = round(((end - start) * 1000), 2)

        return exec_time

    def run(self, repeat=10):
        for method_name, method_value in self.benchmarks:
            self.setUp()

            exec_times = ExecTimeCollection(times=[self.tick(method_value, self) for x in [0.0] * repeat],
                                                              scale='ms')
            average = sum(exec_times.times) / repeat
            ref_class = self.__class__.__name__

            sys.stdout.write("{0}.{1} ... {2} {3}\n".format(ref_class,
                                                                            method_name,
                                                                            average,
                                                                            exec_times.scale))

            self.results['exec_times'].update({method_name: exec_times})
            self.results['averages'].update({method_name: average})

            self.tearDown()
<|endoftext|>


<commit_before>######################################################################
#
# Copyright (C) 2013
# Associated Universities, Inc. Washington DC, USA,
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Library General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
# License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
#
# Correspondence concerning VLA Pipelines should be addressed as follows:
#    Please register and submit helpdesk tickets via: https://help.nrao.edu
#    Postal address:
#              National Radio Astronomy Observatory
#              VLA Pipeline Support Office
#              PO Box O
#              Socorro, NM,  USA
#
######################################################################
# EVLA pipeline
# For continuum modes (contiguous spws within a baseband)
# May work for other modes as well
#
# 06/13/12 C. Chandler
# 07/20/12 B. Kent
# 02/05/13 C. Chandler initial release for CASA 4.1
# 09/23/14 C. Chandler modified to work on CASA 4.2.2, and updated to
#          use Perley-Butler 2013 flux density scale
# 06/08/15 C. Chandler modified for CASA 4.3.1
# 06/08/15 C. Chandler modified for CASA 4.4.0
# 07/13/15 E. Momjian for CASA 4.4.0 split target rflag and final uv
#          plots into two scripts
#          Separate calibrators and targets in final rflag
#          Separate calibrators and targets in statwt
# 07/29/15 E. Momjian force SETJY to create the model column, otherwise
#          calibration will not be correct
# 08/25/15 C. Chandler moved plots to after statwt
# 10/13/15 E. Momjian modified for CASA 4.5.0
#          The use of the real vs. virtual model in setjy is a choice (y/n)
#          at the start of the pipeline
# 02/20/16 E. Momjian modified for CASA 4.5.2
#          Using mstransform based split2 instead of split
# 04/12/16 E. Momjian modified for CASA 4.5.3
# 04/12/16 E. Momjian modified for CASA 4.6.0
#          Mstransform based split2 has been renamed as split.
#          Also using Mstransform based hanningsmooth.
######################################################################

# Change version and date below with each svn commit.  Note changes in the
# .../trunk/doc/CHANGELOG.txt and .../trunk/doc/bugs_features.txt files

import sys
import os
import time

# Import path to pipeline
from paths import sixteenB_pipe_path
# Define location of pipeline
pipepath = sixteenB_pipe_path

version = "1.3.8"
svnrevision = '11nnn'
date = "2016Apr12"

print "Pipeline version " + version + " for use with CASA 4.6.0"

[major, minor, revision] = casadef.casa_version.split('.')
casa_version = 100 * int(major) + 10 * int(minor) + int(revision)
if casa_version < 460:
    sys.exit("Your CASA version is " + casadef.casa_version +
             ", please re-start using CASA 4.6.0")
if casa_version > 460:
    sys.exit("Your CASA version is " + casadef.casa_version +
             ", please re-start using CASA 4.6.0")

# This is the default time-stamped casa log file, in case we
#    need to return to it at any point in the script
log_dir = 'logs'
if not os.path.exists(log_dir):
    os.makedirs(log_dir)

maincasalog = casalogger.func_globals['thelogfile']

def logprint(msg, logfileout=maincasalog):
    print(msg)
    casalog.setlogfile(logfileout)
    casalog.post(msg)
    casalog.setlogfile(maincasalog)
    casalog.post(msg)
    return

# Create timing profile list and file if they don't already exist
if 'time_list' not in globals():
    time_list = []

timing_file = 'logs/timing.log'

if not os.path.exists(timing_file):
    timelog = open(timing_file, 'w')
else:
    timelog = open(timing_file, 'a')

def runtiming(pipestate, status):
    '''Determine profile for a given state/stage of the pipeline
    '''
    time_list.append(
        {'pipestate': pipestate, 'time': time.time(), 'status': status})
#
    if (status == "end"):
        timelog = open(timing_file, 'a')
        timelog.write(
            pipestate + ': ' + str(time_list[-1]['time'] -
                                   time_list[-2]['time']) + ' sec \n')
        timelog.flush()
        timelog.close()
        # with open(maincasalog, 'a') as casalogfile:
        #    tempfile = open('logs/'+pipestate+'.log','r')
        #    casalogfile.write(tempfile.read())
        #    tempfile.close()
        # casalogfile.close()
#
    return time_list

######################################################################

# The following script includes all the definitions and functions and
# prior inputs needed by a run of the pipeline.

time_list = runtiming('startup', 'start')
execfile(pipepath + 'EVLA_pipe_startup.py')
time_list = runtiming('startup', 'end')
pipeline_save()

######################################################################

try:

    ######################################################################

    # IMPORT THE DATA TO CASA

    execfile(pipepath + 'EVLA_pipe_import.py')

    # GET SOME INFORMATION FROM THE MS THAT WILL BE NEEDED LATER, LIST
    # THE DATA, AND MAKE SOME PLOTS

    execfile(pipepath + 'EVLA_pipe_msinfo.py')

    # DETERMINISTIC FLAGGING:
    # TIME-BASED: online flags, shadowed data, zeroes, pointing scans, quacking
    # CHANNEL-BASED: end 5% of channels of each spw, 10 end channels at
    # edges of basebands

    execfile(pipepath + 'EVLA_pipe_flagall.py')

    # Split into continuum and lines

    execfile(pipepath + 'EVLA_pipe_mixed_setup_split.py')

    current_path = os.getcwd()

    if line_run:
        line_path = line_settings["folder_name"]
        if not os.path.exists(line_path):
            raise Warning("speclines path was not created.")
        os.chdir(line_path)

        # Change the SDM name and other needed settings
        SDM_name = line_settings['SDM_Name']
        myHanning = line_settings['myHanning']
        scratch = line_settings['scratch']

        execfile(pipepath + 'EVLA_pipeline_lines.py')

    os.chdir(current_path)

    if cont_run:
        cont_path = cont_settings["folder_name"]
        if not os.path.exists(cont_path):
            raise Warning("continuum path was not created.")
        os.chdir(cont_path)

        # Change the SDM name and other needed settings
        SDM_name = cont_settings['SDM_Name']
        myHanning = cont_settings['myHanning']
        scratch = cont_settings['scratch']

        execfile(pipepath + 'EVLA_pipeline_continuum.py')

# Quit if there have been any exceptions caught:

except KeyboardInterrupt, keyboardException:
    logprint("Keyboard Interrupt: " + str(keyboardException))
except Exception, generalException:
    logprint("Exiting script: " + str(generalException))
<commit_msg>Be sure to copy the online flags plot into each split directory<commit_after>######################################################################
#
# Copyright (C) 2013
# Associated Universities, Inc. Washington DC, USA,
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Library General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
# License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with this library; if not, write to the Free Software Foundation,
# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
#
# Correspondence concerning VLA Pipelines should be addressed as follows:
#    Please register and submit helpdesk tickets via: https://help.nrao.edu
#    Postal address:
#              National Radio Astronomy Observatory
#              VLA Pipeline Support Office
#              PO Box O
#              Socorro, NM,  USA
#
######################################################################
# EVLA pipeline
# For continuum modes (contiguous spws within a baseband)
# May work for other modes as well
#
# 06/13/12 C. Chandler
# 07/20/12 B. Kent
# 02/05/13 C. Chandler initial release for CASA 4.1
# 09/23/14 C. Chandler modified to work on CASA 4.2.2, and updated to
#          use Perley-Butler 2013 flux density scale
# 06/08/15 C. Chandler modified for CASA 4.3.1
# 06/08/15 C. Chandler modified for CASA 4.4.0
# 07/13/15 E. Momjian for CASA 4.4.0 split target rflag and final uv
#          plots into two scripts
#          Separate calibrators and targets in final rflag
#          Separate calibrators and targets in statwt
# 07/29/15 E. Momjian force SETJY to create the model column, otherwise
#          calibration will not be correct
# 08/25/15 C. Chandler moved plots to after statwt
# 10/13/15 E. Momjian modified for CASA 4.5.0
#          The use of the real vs. virtual model in setjy is a choice (y/n)
#          at the start of the pipeline
# 02/20/16 E. Momjian modified for CASA 4.5.2
#          Using mstransform based split2 instead of split
# 04/12/16 E. Momjian modified for CASA 4.5.3
# 04/12/16 E. Momjian modified for CASA 4.6.0
#          Mstransform based split2 has been renamed as split.
#          Also using Mstransform based hanningsmooth.
######################################################################

# Change version and date below with each svn commit.  Note changes in the
# .../trunk/doc/CHANGELOG.txt and .../trunk/doc/bugs_features.txt files

import sys
import os
import time

# Import path to pipeline
from paths import sixteenB_pipe_path
# Define location of pipeline
pipepath = sixteenB_pipe_path

version = "1.3.8"
svnrevision = '11nnn'
date = "2016Apr12"

print "Pipeline version " + version + " for use with CASA 4.6.0"

[major, minor, revision] = casadef.casa_version.split('.')
casa_version = 100 * int(major) + 10 * int(minor) + int(revision)
if casa_version < 460:
    sys.exit("Your CASA version is " + casadef.casa_version +
             ", please re-start using CASA 4.6.0")
if casa_version > 460:
    sys.exit("Your CASA version is " + casadef.casa_version +
             ", please re-start using CASA 4.6.0")

# This is the default time-stamped casa log file, in case we
#    need to return to it at any point in the script
log_dir = 'logs'
if not os.path.exists(log_dir):
    os.makedirs(log_dir)

maincasalog = casalogger.func_globals['thelogfile']

def logprint(msg, logfileout=maincasalog):
    print(msg)
    casalog.setlogfile(logfileout)
    casalog.post(msg)
    casalog.setlogfile(maincasalog)
    casalog.post(msg)
    return

# Create timing profile list and file if they don't already exist
if 'time_list' not in globals():
    time_list = []

timing_file = 'logs/timing.log'

if not os.path.exists(timing_file):
    timelog = open(timing_file, 'w')
else:
    timelog = open(timing_file, 'a')

def runtiming(pipestate, status):
    '''Determine profile for a given state/stage of the pipeline
    '''
    time_list.append(
        {'pipestate': pipestate, 'time': time.time(), 'status': status})
#
    if (status == "end"):
        timelog = open(timing_file, 'a')
        timelog.write(
            pipestate + ': ' + str(time_list[-1]['time'] -
                                   time_list[-2]['time']) + ' sec \n')
        timelog.flush()
        timelog.close()
        # with open(maincasalog, 'a') as casalogfile:
        #    tempfile = open('logs/'+pipestate+'.log','r')
        #    casalogfile.write(tempfile.read())
        #    tempfile.close()
        # casalogfile.close()
#
    return time_list

######################################################################

# The following script includes all the definitions and functions and
# prior inputs needed by a run of the pipeline.

time_list = runtiming('startup', 'start')
execfile(pipepath + 'EVLA_pipe_startup.py')
time_list = runtiming('startup', 'end')
pipeline_save()

######################################################################

try:

    ######################################################################

    # IMPORT THE DATA TO CASA

    execfile(pipepath + 'EVLA_pipe_import.py')

    # GET SOME INFORMATION FROM THE MS THAT WILL BE NEEDED LATER, LIST
    # THE DATA, AND MAKE SOME PLOTS

    execfile(pipepath + 'EVLA_pipe_msinfo.py')

    # DETERMINISTIC FLAGGING:
    # TIME-BASED: online flags, shadowed data, zeroes, pointing scans, quacking
    # CHANNEL-BASED: end 5% of channels of each spw, 10 end channels at
    # edges of basebands

    execfile(pipepath + 'EVLA_pipe_flagall.py')

    # Split into continuum and lines

    execfile(pipepath + 'EVLA_pipe_mixed_setup_split.py')

    current_path = os.getcwd()

    if line_run:
        line_path = line_settings["folder_name"]
        if not os.path.exists(line_path):
            raise Warning("speclines path was not created.")

        # Copy the online flags plot into each split directory
        os.system("cp ../onlineFlags.png .")

        os.chdir(line_path)

        # Change the SDM name and other needed settings
        SDM_name = line_settings['SDM_Name']
        myHanning = line_settings['myHanning']
        scratch = line_settings['scratch']

        execfile(pipepath + 'EVLA_pipeline_lines.py')

    os.chdir(current_path)

    if cont_run:
        cont_path = cont_settings["folder_name"]
        if not os.path.exists(cont_path):
            raise Warning("continuum path was not created.")

        # Copy the online flags plot into each split directory
        os.system("cp ../onlineFlags.png .")

        os.chdir(cont_path)

        # Change the SDM name and other needed settings
        SDM_name = cont_settings['SDM_Name']
        myHanning = cont_settings['myHanning']
        scratch = cont_settings['scratch']

        execfile(pipepath + 'EVLA_pipeline_continuum.py')

# Quit if there have been any exceptions caught:

except KeyboardInterrupt, keyboardException:
    logprint("Keyboard Interrupt: " + str(keyboardException))
except Exception, generalException:
    logprint("Exiting script: " + str(generalException))
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging
import time

import pykka
from spotify import Link, SpotifyError

from mopidy import settings
from mopidy.backends import base
from mopidy.models import Track

from . import translator

logger = logging.getLogger('mopidy.backends.spotify')

class SpotifyTrack(Track):
    """Proxy object for unloaded Spotify tracks."""
    def __init__(self, uri=None, track=None):
        super(SpotifyTrack, self).__init__()
        if uri:
            self._spotify_track = Link.from_string(uri).as_track()
        elif track:
            self._spotify_track = track
        else:
            raise AttributeError('uri or track must be provided')
        self._unloaded_track = Track(uri=uri, name='[loading...]')
        self._track = None

    @property
    def _proxy(self):
        if self._track:
            return self._track
        elif self._spotify_track.is_loaded():
            self._track = translator.to_mopidy_track(self._spotify_track)
            return self._track
        else:
            return self._unloaded_track

    def __getattribute__(self, name):
        if name.startswith('_'):
            return super(SpotifyTrack, self).__getattribute__(name)
        return self._proxy.__getattribute__(name)

    def __repr__(self):
        return self._proxy.__repr__()

    def __hash__(self):
        return hash(self._proxy.uri)

    def __eq__(self, other):
        if not isinstance(other, Track):
            return False
        return self._proxy.uri == other.uri

    def copy(self, **values):
        return self._proxy.copy(**values)

class SpotifyLibraryProvider(base.BaseLibraryProvider):
    def find_exact(self, **query):
        return self.search(**query)

    def lookup(self, uri):
        try:
            link = Link.from_string(uri)
            if link.type() == Link.LINK_TRACK:
                return self._lookup_track(uri)
            if link.type() == Link.LINK_ALBUM:
                return self._lookup_album(uri)
            elif link.type() == Link.LINK_ARTIST:
                return self._lookup_artist(uri)
            elif link.type() == Link.LINK_PLAYLIST:
                return self._lookup_playlist(uri)
            else:
                return []
        except SpotifyError as error:
            logger.debug(u'Failed to lookup "%s": %s', uri, error)
            return []

    def _lookup_track(self, uri):
        return [SpotifyTrack(uri)]

    def _lookup_album(self, uri):
        album = Link.from_string(uri).as_album()
        album_browser = self.backend.spotify.session.browse_album(album)
        self._wait_for_object_to_load(album_browser)
        return [SpotifyTrack(track=t) for t in album_browser]

    def _lookup_artist(self, uri):
        artist = Link.from_string(uri).as_artist()
        artist_browser = self.backend.spotify.session.browse_artist(artist)
        self._wait_for_object_to_load(artist_browser)
        return [SpotifyTrack(track=t) for t in artist_browser]

    def _lookup_playlist(self, uri):
        playlist = Link.from_string(uri).as_playlist()
        self._wait_for_object_to_load(playlist)
        return [SpotifyTrack(track=t) for t in playlist]

    def _wait_for_object_to_load(
            self, spotify_obj, timeout=settings.SPOTIFY_TIMEOUT):
        # XXX Sleeping to wait for the Spotify object to load is an ugly hack,
        # but it works. We should look into other solutions for this.
        start = time.time()
        while not spotify_obj.is_loaded():
            time.sleep(0.1)
            if time.time() > (start + timeout):
                logger.debug(
                    'Timeout: Spotify object did not load in %ds', timeout)
                return

    def refresh(self, uri=None):
        pass  # TODO

    def search(self, **query):
        if not query:
            return self._get_all_tracks()

        spotify_query = self._translate_search_query(query)
        logger.debug('Spotify search query: %s' % spotify_query)

        future = pykka.ThreadingFuture()

        def callback(results, userdata=None):
            # TODO Include results from results.albums(), etc. too
            # TODO Consider launching a second search if results.total_tracks()
            # is larger than len(results.tracks())
            tracks = [
                translator.to_mopidy_track(t) for t in results.tracks()]
            future.set(tracks)

        if not self.backend.spotify.connected.wait(settings.SPOTIFY_TIMEOUT):
            logger.debug('Not connected: Spotify search cancelled')
            return []

        self.backend.spotify.session.search(
            spotify_query, callback,
            track_count=100, album_count=0, artist_count=0)

        try:
            return future.get(timeout=settings.SPOTIFY_TIMEOUT)
        except pykka.Timeout:
            logger.debug(
                'Timeout: Spotify search did not return in %ds',
                settings.SPOTIFY_TIMEOUT)
            return []

    def _get_all_tracks(self):
        # Since we can't search for the entire Spotify library, we return
        # all tracks in the playlists when the query is empty.
        tracks = []
        for playlist in self.backend.playlists.playlists:
            tracks += playlist.tracks
        return tracks

    def _translate_search_query(self, mopidy_query):
        spotify_query = []
        for (field, values) in mopidy_query.iteritems():
            if field == 'uri':
                tracks = []
                for value in values:
                    track = self.lookup(value)
                    if track:
                        tracks.append(track)
                return tracks
            elif field == 'track':
                field = 'title'
            elif field == 'date':
                field = 'year'
            if not hasattr(values, '__iter__'):
                values = [values]
            for value in values:
                if field == 'any':
                    spotify_query.append(value)
                elif field == 'year':
                    value = int(value.split('-')[0])  # Extract year
                    spotify_query.append('%s:%d' % (field, value))
                else:
                    spotify_query.append('%s:"%s"' % (field, value))
        spotify_query = ' '.join(spotify_query)
        return spotify_query
<commit_msg>spotify: SpotifyTrack fails when both uri and track is provided<commit_after>from __future__ import unicode_literals

import logging
import time

import pykka
from spotify import Link, SpotifyError

from mopidy import settings
from mopidy.backends import base
from mopidy.models import Track

from . import translator

logger = logging.getLogger('mopidy.backends.spotify')

class SpotifyTrack(Track):
    """Proxy object for unloaded Spotify tracks."""
    def __init__(self, uri=None, track=None):
        super(SpotifyTrack, self).__init__()
        if (uri and track) or (not uri and not track):
            raise AttributeError('uri or track must be provided')
        elif uri:
            self._spotify_track = Link.from_string(uri).as_track()
        elif track:
            self._spotify_track = track
        self._unloaded_track = Track(uri=uri, name='[loading...]')
        self._track = None

    @property
    def _proxy(self):
        if self._track:
            return self._track
        elif self._spotify_track.is_loaded():
            self._track = translator.to_mopidy_track(self._spotify_track)
            return self._track
        else:
            return self._unloaded_track

    def __getattribute__(self, name):
        if name.startswith('_'):
            return super(SpotifyTrack, self).__getattribute__(name)
        return self._proxy.__getattribute__(name)

    def __repr__(self):
        return self._proxy.__repr__()

    def __hash__(self):
        return hash(self._proxy.uri)

    def __eq__(self, other):
        if not isinstance(other, Track):
            return False
        return self._proxy.uri == other.uri

    def copy(self, **values):
        return self._proxy.copy(**values)

class SpotifyLibraryProvider(base.BaseLibraryProvider):
    def find_exact(self, **query):
        return self.search(**query)

    def lookup(self, uri):
        try:
            link = Link.from_string(uri)
            if link.type() == Link.LINK_TRACK:
                return self._lookup_track(uri)
            if link.type() == Link.LINK_ALBUM:
                return self._lookup_album(uri)
            elif link.type() == Link.LINK_ARTIST:
                return self._lookup_artist(uri)
            elif link.type() == Link.LINK_PLAYLIST:
                return self._lookup_playlist(uri)
            else:
                return []
        except SpotifyError as error:
            logger.debug(u'Failed to lookup "%s": %s', uri, error)
            return []

    def _lookup_track(self, uri):
        return [SpotifyTrack(uri)]

    def _lookup_album(self, uri):
        album = Link.from_string(uri).as_album()
        album_browser = self.backend.spotify.session.browse_album(album)
        self._wait_for_object_to_load(album_browser)
        return [SpotifyTrack(track=t) for t in album_browser]

    def _lookup_artist(self, uri):
        artist = Link.from_string(uri).as_artist()
        artist_browser = self.backend.spotify.session.browse_artist(artist)
        self._wait_for_object_to_load(artist_browser)
        return [SpotifyTrack(track=t) for t in artist_browser]

    def _lookup_playlist(self, uri):
        playlist = Link.from_string(uri).as_playlist()
        self._wait_for_object_to_load(playlist)
        return [SpotifyTrack(track=t) for t in playlist]

    def _wait_for_object_to_load(
            self, spotify_obj, timeout=settings.SPOTIFY_TIMEOUT):
        # XXX Sleeping to wait for the Spotify object to load is an ugly hack,
        # but it works. We should look into other solutions for this.
        start = time.time()
        while not spotify_obj.is_loaded():
            time.sleep(0.1)
            if time.time() > (start + timeout):
                logger.debug(
                    'Timeout: Spotify object did not load in %ds', timeout)
                return

    def refresh(self, uri=None):
        pass  # TODO

    def search(self, **query):
        if not query:
            return self._get_all_tracks()

        spotify_query = self._translate_search_query(query)
        logger.debug('Spotify search query: %s' % spotify_query)

        future = pykka.ThreadingFuture()

        def callback(results, userdata=None):
            # TODO Include results from results.albums(), etc. too
            # TODO Consider launching a second search if results.total_tracks()
            # is larger than len(results.tracks())
            tracks = [
                translator.to_mopidy_track(t) for t in results.tracks()]
            future.set(tracks)

        if not self.backend.spotify.connected.wait(settings.SPOTIFY_TIMEOUT):
            logger.debug('Not connected: Spotify search cancelled')
            return []

        self.backend.spotify.session.search(
            spotify_query, callback,
            track_count=100, album_count=0, artist_count=0)

        try:
            return future.get(timeout=settings.SPOTIFY_TIMEOUT)
        except pykka.Timeout:
            logger.debug(
                'Timeout: Spotify search did not return in %ds',
                settings.SPOTIFY_TIMEOUT)
            return []

    def _get_all_tracks(self):
        # Since we can't search for the entire Spotify library, we return
        # all tracks in the playlists when the query is empty.
        tracks = []
        for playlist in self.backend.playlists.playlists:
            tracks += playlist.tracks
        return tracks

    def _translate_search_query(self, mopidy_query):
        spotify_query = []
        for (field, values) in mopidy_query.iteritems():
            if field == 'uri':
                tracks = []
                for value in values:
                    track = self.lookup(value)
                    if track:
                        tracks.append(track)
                return tracks
            elif field == 'track':
                field = 'title'
            elif field == 'date':
                field = 'year'
            if not hasattr(values, '__iter__'):
                values = [values]
            for value in values:
                if field == 'any':
                    spotify_query.append(value)
                elif field == 'year':
                    value = int(value.split('-')[0])  # Extract year
                    spotify_query.append('%s:%d' % (field, value))
                else:
                    spotify_query.append('%s:"%s"' % (field, value))
        spotify_query = ' '.join(spotify_query)
        return spotify_query
<|endoftext|>


<commit_before># Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from rply import LexerGenerator

lg = LexerGenerator()

# A regexp for something that should end a quoting/unquoting operator
# i.e. a space or a closing brace/paren/curly
end_quote = r'(?![\s\)\]\}])'

lg.add('LPAREN', r'\(')
lg.add('RPAREN', r'\)')
lg.add('LBRACKET', r'\[')
lg.add('RBRACKET', r'\]')
lg.add('LCURLY', r'\{')
lg.add('RCURLY', r'\}')
lg.add('HLCURLY', r'#\{')
lg.add('QUOTE', r'\'%s' % end_quote)
lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('HASHREADER', r'#[^{]')

# A regexp which matches incomplete strings, used to support
# multi-line strings in the interpreter
partial_string = r'''(?x)
    (?:u|r|ur|ru)? # prefix
    "  # start string
    (?:
       | [^"\\]             # non-quote or backslash
       | \\.                # or escaped single character
       | \\x[0-9a-fA-F]{2}  # or escaped raw character
       | \\u[0-9a-fA-F]{4}  # or unicode escape
       | \\U[0-9a-fA-F]{8}  # or long unicode escape
    )* # one or more times
'''

lg.add('STRING', r'%s"' % partial_string)
lg.add('PARTIAL_STRING', partial_string)

lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')

lg.ignore(r';.*(?=\r|\n|$)')
lg.ignore(r'\s+')

lexer = lg.build()
<commit_msg>Fix #831<commit_after># Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from rply import LexerGenerator

lg = LexerGenerator()

# A regexp for something that should end a quoting/unquoting operator
# i.e. a space or a closing brace/paren/curly
end_quote = r'(?![\s\)\]\}])'

lg.add('LPAREN', r'\(')
lg.add('RPAREN', r'\)')
lg.add('LBRACKET', r'\[')
lg.add('RBRACKET', r'\]')
lg.add('LCURLY', r'\{')
lg.add('RCURLY', r'\}')
lg.add('HLCURLY', r'#\{')
lg.add('QUOTE', r'\'%s' % end_quote)
lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('HASHREADER', r'#[^{]')

# A regexp which matches incomplete strings, used to support
# multi-line strings in the interpreter
partial_string = r'''(?x)
    (?:u|r|ur|ru)? # prefix
    "  # start string
    (?:
       | [^"\\]             # non-quote or backslash
       | \\(.|\n)           # or escaped single character or newline
       | \\x[0-9a-fA-F]{2}  # or escaped raw character
       | \\u[0-9a-fA-F]{4}  # or unicode escape
       | \\U[0-9a-fA-F]{8}  # or long unicode escape
    )* # one or more times
'''

lg.add('STRING', r'%s"' % partial_string)
lg.add('PARTIAL_STRING', partial_string)

lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')

lg.ignore(r';.*(?=\r|\n|$)')
lg.ignore(r'\s+')

lexer = lg.build()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from fabric.api import task, local
import sys
import platform
import os

def is_ubuntu():
    """test sif the platform is ubuntu"""
    return platform.dist()[0] == 'Ubuntu'

def is_centos():
    """test if the platform is centos"""
    (centos, version, release) = platform.dist()
    if centos == "centos" and version != "6.4":
        print "Warning: centos %s is not tested" % version
    return centos == "centos"

def is_osx():
    return platform.system().lower() == 'darwin'

@task
def deploy():
    """deploys the system on supported distributions"""
    # download()
    (major, minor, micro, releaselevel, serial) = sys.version_info
    if major != 2 or (major == 2 and minor < 7):
        print "Your version of python is not supported.  Please install python 2.7 for cloudmesh"
        sys.exit()
    if not hasattr(sys, 'real_prefix'):
        print "You do not appear to be in a vitualenv.  Please create and/or activate a virtualenv for cloudmesh installation"
        sys.exit()
    if is_ubuntu():
        ubuntu()
    elif is_centos():
        centos()
    elif is_osx():
        osx()
    else:
        print "OS distribution not supported; please see documatation for manual installation instructions."
        sys.exit()

    # install()

@task
def download():
    '''downloads cloudmesh'''
    local("git clone git@github.com:cloudmesh/cloudmesh.git")

@task
def install():
    sphinx_updates()
    local("pip install -r requirements.txt")
    local("python setup.py install")

@task
def install_mongodb():
    local("fab mongo.install")

def install_package(package):
    if is_ubuntu():
        local ("sudo apt-get -y install {0}".format(package))
    if is_centos():
        local("sudo yum -y install {0}".format(package))
    elif sys.platform == "darwin":
        print "Not yet supported"
        sys.exit()
    elif sys.platform == "win32":
        print "Windows is not supported"
        print "Use Linux instead"
        sys.exit()

@task
def install_packages(packages):
    for package in packages:
        install_package (package)

@task
def ubuntu():
    '''prepares an system and installs all
    needed packages before we install cloudmesch'''

    local ("sudo apt-get update")
    install_packages(["git",
                      "mercurial",
                      "curl",
                      "python-dev",
                      "libldap2-dev",
                      "libsasl2-dev",
                      "libpng-dev",
                      "mongodb-server"])
    install_packages(["rabbitmq-server"])
    install()
    install_mongodb()#important that mongo_db installation be done only after all we install all needed python packages(as per requiremnts.txt)

def centos():
    install_packages (["git",
                       "mercurial",
                       "wget",
                       "gcc",
                       "make",
                       "readline-devel",
                       "zlib-devel",
                       "openssl-devel",
                       "openldap-devel",
                       "bzip2-devel",
                       "python-matplotlib",
                       "libpng-devel"])

    install_packages(["rabbitmq-server"])
    local('sudo sh -c "chkconfig rabbitmq-server on && service rabbitmq-server start"')
    install()
    install_mongodb()

def osx():
    local('brew install wget')
    local('brew install mercurial')
    local('brew install freetype')
    local('brew install libpng')
    try:
        import numpy
        print "numpy already installed"
    except:
        local('pip install numpy')
    try:
        import matplotlib
        print "matplotlib already installed"
    except:
        local ('LDFLAGS="-L/usr/local/opt/freetype/lib -L/usr/local/opt/libpng/lib" CPPFLAGS="-I/usr/local/opt/freetype/include -I/usr/local/opt/libpng/include -I/usr/local/opt/freetype/include/freetype2" pip install matplotlib')

        # local('pip install matplotlib')

    install()
    install_mongodb()

def sphinx_updates():
    local('rm -rf /tmp/install-cloudmesh')
    local('mkdir -p /tmp/install-cloudmesh')
    local('cd /tmp/install-cloudmesh; hg clone http://bitbucket.org/birkenfeld/sphinx-contrib/')
    local('~/ENV/bicd /tmp/install-cloudmesh/sphinx-contrib/autorun; python setup.py install')

<commit_msg>adding the export to ignore the unused options<commit_after>#!/usr/bin/env python
from fabric.api import task, local
import sys
import platform
import os

def is_ubuntu():
    """test sif the platform is ubuntu"""
    return platform.dist()[0] == 'Ubuntu'

def is_centos():
    """test if the platform is centos"""
    (centos, version, release) = platform.dist()
    if centos == "centos" and version != "6.4":
        print "Warning: centos %s is not tested" % version
    return centos == "centos"

def is_osx():
    return platform.system().lower() == 'darwin'

@task
def deploy():
    """deploys the system on supported distributions"""
    # download()
    (major, minor, micro, releaselevel, serial) = sys.version_info
    if major != 2 or (major == 2 and minor < 7):
        print "Your version of python is not supported.  Please install python 2.7 for cloudmesh"
        sys.exit()
    if not hasattr(sys, 'real_prefix'):
        print "You do not appear to be in a vitualenv.  Please create and/or activate a virtualenv for cloudmesh installation"
        sys.exit()
    if is_ubuntu():
        ubuntu()
    elif is_centos():
        centos()
    elif is_osx():
        osx()
    else:
        print "OS distribution not supported; please see documatation for manual installation instructions."
        sys.exit()

    # install()

@task
def download():
    '''downloads cloudmesh'''
    local("git clone git@github.com:cloudmesh/cloudmesh.git")

@task
def install():
    sphinx_updates()
    local("pip install -r requirements.txt")
    local("python setup.py install")

@task
def install_mongodb():
    local("fab mongo.install")

def install_package(package):
    if is_ubuntu():
        local ("sudo apt-get -y install {0}".format(package))
    if is_centos():
        local("sudo yum -y install {0}".format(package))
    elif sys.platform == "darwin":
        print "Not yet supported"
        sys.exit()
    elif sys.platform == "win32":
        print "Windows is not supported"
        print "Use Linux instead"
        sys.exit()

@task
def install_packages(packages):
    for package in packages:
        install_package (package)

@task
def ubuntu():
    '''prepares an system and installs all
    needed packages before we install cloudmesch'''

    local ("sudo apt-get update")
    install_packages(["git",
                      "mercurial",
                      "curl",
                      "python-dev",
                      "libldap2-dev",
                      "libsasl2-dev",
                      "libpng-dev",
                      "mongodb-server"])
    install_packages(["rabbitmq-server"])
    install()
    install_mongodb()#important that mongo_db installation be done only after all we install all needed python packages(as per requiremnts.txt)

def centos():
    install_packages (["git",
                       "mercurial",
                       "wget",
                       "gcc",
                       "make",
                       "readline-devel",
                       "zlib-devel",
                       "openssl-devel",
                       "openldap-devel",
                       "bzip2-devel",
                       "python-matplotlib",
                       "libpng-devel"])

    install_packages(["rabbitmq-server"])
    local('sudo sh -c "chkconfig rabbitmq-server on && service rabbitmq-server start"')
    install()
    install_mongodb()

def osx():

    local("export CFLAGS=-Qunused-arguments")
    local("export CPPFLAGS=-Qunused-arguments")
    local('brew install wget')
    local('brew install mercurial')
    local('brew install freetype')
    local('brew install libpng')
    try:
        import numpy
        print "numpy already installed"
    except:
        local('pip install numpy')
    try:
        import matplotlib
        print "matplotlib already installed"
    except:
        local ('LDFLAGS="-L/usr/local/opt/freetype/lib -L/usr/local/opt/libpng/lib" CPPFLAGS="-I/usr/local/opt/freetype/include -I/usr/local/opt/libpng/include -I/usr/local/opt/freetype/include/freetype2" pip install matplotlib')

        # local('pip install matplotlib')

    install()
    install_mongodb()

def sphinx_updates():
    local('rm -rf /tmp/install-cloudmesh')
    local('mkdir -p /tmp/install-cloudmesh')
    local('cd /tmp/install-cloudmesh; hg clone http://bitbucket.org/birkenfeld/sphinx-contrib/')
    local('~/ENV/bicd /tmp/install-cloudmesh/sphinx-contrib/autorun; python setup.py install')

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import subprocess
import json
import platform
import argparse
import mmap
import traceback
import ConfigParser
from distutils import dir_util

from space_checker_utils import wget_wrapper

def create_directory(directory):
    """Create parent directories as necessary.
    :param directory: (~str) Path of directory to be made.
    :return: True - if directory is created, and False - if not.
    """
    try:
        os.makedirs(directory)
        return True
    except OSError:
        # Checks if the folder is empty
        if not os.listdir(directory):
            return True
        return False

def run_command(command):
    """Execute the provided shell command.
    :param command: (~str) Linux shell command.
    :return: True - if command executed, and False if not.
    """
    Colorizer.normal("[*] Running following command")
    Colorizer.info("%s" % command)
    # If command is `wget`, then before execution, `wget_wrapper` checks whether there is enough disk space available
    if not wget_wrapper(command):
        return False
    return os.system(command)

def owtf_last_commit():
    """Prints the local git repo's last commit hash."""
    if os.path.exists(os.path.join(root_dir, '.git')):
        command = 'git log -n 1 --pretty=format:"%H"'
        commit_hash = os.popen(command).read()
        return commit_hash
    else:
        return "*Not a git repository.*"

def check_sudo():
    """Checks if the user has sudo access."""
    sudo = os.system("sudo -v")
    if not sudo:
        return
    else:
        Colorizer.warning("[!] Your user does not have sudo privileges. Some OWTF components require"
                          "sudo permissions to install")
        sys.exit()

def install_in_directory(directory, command):
    """Execute a certain command while staying inside one directory.
    :param directory: (~str) Path of directory in which installation command has to be executed.
    :param command: (~str) Linux shell command (most likely `wget` here)
    :return: True - if installation successful or directory already exists, and False if not.
    """
    if create_directory(directory):
        Colorizer.info("[*] Switching to %s" % directory)
        os.chdir(directory)
        return run_command(command)
    else:
        Colorizer.warning("[!] Directory %s already exists, so skipping installation for this" % directory)
        return True

def install_using_pip(requirements_file):
    """Install pip libraries as mentioned in a requirements file.
    :param requirements_file: (~str) Path to requirements file - in which libraries are listed.
    :return: True - if installation successful, and False if not.
    """
    # Instead of using file directly with pip which can crash because of single library
    return run_command("pip2 install --upgrade -r %s" % requirements_file)

def install_restricted_from_cfg(config_file):
    """Install restricted tools and dependencies which are distro independent.
    :param config_file: (~str) Path to configuration file having information about restricted content.
    """
    cp = ConfigParser.ConfigParser({"RootDir": root_dir, "Pid": pid})
    cp.read(config_file)
    for section in cp.sections():
        Colorizer.info("[*] Installing %s" % section)
        install_in_directory(os.path.expanduser(cp.get(section, "directory")), cp.get(section, "command"))

def is_compatible():
        compatible_value = os.system("which apt-get >> /dev/null 2>&1")
        if (compatible_value >> 8) == 1:
            return False
        else:
            return True

def finish():
    Colorizer.success("[*] Finished!")
    Colorizer.info("[*] Run following command to start virtualenv: source ~/.%src; workon owtf"
                   % os.environ["SHELL"].split(os.sep)[-1])
    Colorizer.info("[*] Start OWTF by running 'cd path/to/pentest/directory; ./path/to/owtf.py'")

def setup_virtualenv():
    Colorizer.info("[*] Seting up virtual environment named owtf...")
    # sources files and commands
    source = 'source /usr/local/bin/virtualenvwrapper.sh'
    setup_env = 'cd $WORKON_HOME; virtualenv -q --always-copy -p %s owtf >/dev/null 2>&1;'\
        ' source owtf/bin/activate' % sys.executable
    dump = '%s -c "import os, json;print json.dumps(dict(os.environ))"' % sys.executable

    pipe = subprocess.Popen(['/bin/bash', '-c', '%s >/dev/null 2>&1; %s; %s' % (source, setup_env, dump)],
                            stdout=subprocess.PIPE)
    env = json.loads(pipe.stdout.read())

    # Update the os environment variable
    os.environ.update(env)
    try:
        if os.path.join(os.environ["WORKON_HOME"], "owtf") == os.environ["VIRTUAL_ENV"]:
            # Add source to shell config file only if not present
            Colorizer.info("[*] Adding virtualenvwrapper source to shell config file")
            shell_rc_path = os.path.join(os.environ["HOME"], ".%src" % os.environ["SHELL"].split(os.sep)[-1])
            with open(shell_rc_path, "r") as f:
                if mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ).find(source) == -1:
                    run_command("echo '%s' >> %s" % (source, shell_rc_path))
                else:
                    Colorizer.info("[+] Source line already added to the $SHELL config ")
            return True
    except KeyError:
        traceback.print_exc()
    return False

def setup_pip():
    # Installing pip
    Colorizer.info("[*] Installing pip")
    directory = "/tmp/owtf-install/pip/%s" % os.getpid()
    command = 'command -v pip2 >/dev/null || { wget --user-agent="Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101'\
        ' Firefox/15.0" --tries=3 https://bootstrap.pypa.io/get-pip.py; sudo python get-pip.py;}'
    install_in_directory(os.path.expanduser(directory), command)
    Colorizer.info("[*] Installing required packages for pipsecure")
    run_command("sudo pip2 install --upgrade pyopenssl ndg-httpsclient pyasn1")

    # Installing virtualenv
    Colorizer.info("[*] Installing virtualenv and virtualenvwrapper")
    install_in_directory(os.path.expanduser(str(os.getpid())), "sudo pip2 install --upgrade virtualenv virtualenvwrapper")

def install(cmd_arguments):
    """Perform installation of OWTF Framework. Wraps around all helper methods made in this module.

    :param cmd_arguments:
    """
    args = parser.parse_args(cmd_arguments)
    # User asked to select distro (in case it can't be automatically detected) and distro related stuff is installed
    cp = ConfigParser.ConfigParser({"RootDir": root_dir, "Pid": pid})
    cp.read(distros_cfg)

    # Try get the distro automatically
    distro, version, arch = platform.linux_distribution()
    distro_num = 0
    if "kali" in distro.lower():
        distro_num = 1
    elif "samurai" in distro.lower():
        distro_num = 2
    elif is_compatible():
        distro_num = 3

    if distro_num != 0:
        Colorizer.info("[*] %s has been automatically detected... " % distro)
        Colorizer.normal("[*] Continuing in auto-mode")
    elif (distro_num == 0) and args.no_user_input:
        Colorizer.info("[*] Cannot auto-detect a supported distro...")
        Colorizer.normal("[*] Continuing in auto-mode with the core installation...")
    else:
        # Loop until proper input is received
        while True:
            print("")
            for i, item in enumerate(cp.sections()):
                Colorizer.warning("(%d) %s" % (i + 1, item))
            Colorizer.warning("(0) My distro is not listed :( %s" % distro)

            num_input = raw_input("Select a number based on your distribution : ")
            try:
                if int(num_input) <= len(cp.sections()):
                    distro_num = int(num_input)
                    break
                else:
                    print("")
                    Colorizer.warning("[!] Invalid number - not a supported distro")
                    continue
            except ValueError:
                print("")
                Colorizer.warning("[!] Invalid Number specified")
                continue

    # Install distro specific dependencies and packages needed for OWTF to work
    if distro_num != 0:
        run_command(cp.get(cp.sections()[int(distro_num) - 1], "install"))
    else:
        Colorizer.normal("[*] Skipping distro related installation :(")

    # Installing pip and setting up virtualenv.
    # This requires distro specific dependencies to be installed properly.
    setup_pip()
    if setup_virtualenv():
        install_using_pip(owtf_pip)
    else:
        Colorizer.danger("Unable to setup virtualenv...")
        Colorizer.danger("Skipping installation of OWTF python dependencies ...")

    # Now install distro independent stuff - optional
    # This is due to db config setup included in this. Should run only after PostgreSQL is installed.
    # See https://github.com/owtf/owtf/issues/797.
    install_restricted_from_cfg(restricted_cfg)

    Colorizer.normal("[*] Upgrading pip to the latest version ...")
    # Upgrade pip before install required libraries
    run_command("pip2 install --upgrade pip")
    Colorizer.normal("Upgrading setuptools to the latest version ...")
    # Upgrade setuptools
    run_command("pip2 install --upgrade setuptools")
    Colorizer.normal("Upgrading cffi to the latest version ...")
    # Mitigate cffi errors by upgrading it first
    run_command("pip2 install --upgrade cffi")

class Colorizer:

    """Helper class for colorized strings.

    Different statements will have different colors:

        - `normal`, denoting ongoing procedure (WHITE)
        - `info`, any file path, commit hash or any other info (BLUE)
        - `warning`, any potential hindrance in installation (YELLOW)
        - `danger`, abrupt failure, desired file/dir not found etc. (RED)
    """

    BOLD = '\033[1m'
    RED = BOLD + '\033[91m'
    GREEN = BOLD + '\033[92m'
    YELLOW = BOLD + '\033[93m'
    BLUE = BOLD + '\033[34m'
    PURPLE = BOLD + '\033[95m'
    CYAN = BOLD + '\033[36m'
    WHITE = BOLD + '\033[37m'
    END = '\033[0m\033[0m'

    def __init__(self):
        pass

    @classmethod
    def normal(cls, string):
        print(cls.WHITE + string + cls.END)

    @classmethod
    def info(cls, string):
        print(cls.CYAN + string + cls.END)

    @classmethod
    def warning(cls, string):
        print(cls.YELLOW + string + cls.END)

    @classmethod
    def success(cls, string):
        print(cls.GREEN + string + cls.END)

    @classmethod
    def danger(cls, string):
        print(cls.RED + string + cls.END)

if __name__ == "__main__":
    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    pid = os.getpid()

    # Path to custom scripts for tasks such as setting up/ running PostgreSQL db, run arachni, nikto, wapiti etc.
    scripts_path = os.path.join(root_dir, "scripts")

    # OWTF python libraries
    owtf_pip = os.path.join(root_dir, "install", "owtf.pip")

    # Restricted tools and dictionaries which are distro independent
    restricted_cfg = os.path.join(root_dir, "install", "distro-independent.cfg")

    # Various distros and install scripts
    distros_cfg = os.path.join(root_dir, "install", "linux-distributions.cfg")

    parser = argparse.ArgumentParser()
    parser.add_argument('--no-user-input', help='run script with default options for user input', action="store_true")

    Colorizer.normal("[*] Great that you are installing OWTF :D")
    Colorizer.warning("[!] There will be lot of output, please be patient")

    Colorizer.info("[*] Last commit hash: %s" % owtf_last_commit())
    check_sudo()
    install(sys.argv[1:])

    # Copying config files
    dest_config_path = os.path.join(os.path.expanduser('~'), '.owtf', 'configuration')
    create_directory(dest_config_path)
    src_config_path = os.path.join(root_dir, 'configuration')
    dir_util.copy_tree(src_config_path, dest_config_path)

    finish()
<commit_msg>[virtualenv] Select python 2.7 as the default version for venv<commit_after>#!/usr/bin/env python

import os
import sys
import subprocess
import json
import platform
import argparse
import mmap
import traceback
import ConfigParser
from distutils import dir_util

from space_checker_utils import wget_wrapper

def create_directory(directory):
    """Create parent directories as necessary.
    :param directory: (~str) Path of directory to be made.
    :return: True - if directory is created, and False - if not.
    """
    try:
        os.makedirs(directory)
        return True
    except OSError:
        # Checks if the folder is empty
        if not os.listdir(directory):
            return True
        return False

def run_command(command):
    """Execute the provided shell command.
    :param command: (~str) Linux shell command.
    :return: True - if command executed, and False if not.
    """
    Colorizer.normal("[*] Running following command")
    Colorizer.info("%s" % command)
    # If command is `wget`, then before execution, `wget_wrapper` checks whether there is enough disk space available
    if not wget_wrapper(command):
        return False
    return os.system(command)

def owtf_last_commit():
    """Prints the local git repo's last commit hash."""
    if os.path.exists(os.path.join(root_dir, '.git')):
        command = 'git log -n 1 --pretty=format:"%H"'
        commit_hash = os.popen(command).read()
        return commit_hash
    else:
        return "*Not a git repository.*"

def check_sudo():
    """Checks if the user has sudo access."""
    sudo = os.system("sudo -v")
    if not sudo:
        return
    else:
        Colorizer.warning("[!] Your user does not have sudo privileges. Some OWTF components require"
                          "sudo permissions to install")
        sys.exit()

def install_in_directory(directory, command):
    """Execute a certain command while staying inside one directory.
    :param directory: (~str) Path of directory in which installation command has to be executed.
    :param command: (~str) Linux shell command (most likely `wget` here)
    :return: True - if installation successful or directory already exists, and False if not.
    """
    if create_directory(directory):
        Colorizer.info("[*] Switching to %s" % directory)
        os.chdir(directory)
        return run_command(command)
    else:
        Colorizer.warning("[!] Directory %s already exists, so skipping installation for this" % directory)
        return True

def install_using_pip(requirements_file):
    """Install pip libraries as mentioned in a requirements file.
    :param requirements_file: (~str) Path to requirements file - in which libraries are listed.
    :return: True - if installation successful, and False if not.
    """
    # Instead of using file directly with pip which can crash because of single library
    return run_command("pip2 install --upgrade -r %s" % requirements_file)

def install_restricted_from_cfg(config_file):
    """Install restricted tools and dependencies which are distro independent.
    :param config_file: (~str) Path to configuration file having information about restricted content.
    """
    cp = ConfigParser.ConfigParser({"RootDir": root_dir, "Pid": pid})
    cp.read(config_file)
    for section in cp.sections():
        Colorizer.info("[*] Installing %s" % section)
        install_in_directory(os.path.expanduser(cp.get(section, "directory")), cp.get(section, "command"))

def is_compatible():
        compatible_value = os.system("which apt-get >> /dev/null 2>&1")
        if (compatible_value >> 8) == 1:
            return False
        else:
            return True

def finish():
    Colorizer.success("[*] Finished!")
    Colorizer.info("[*] Run following command to start virtualenv: source ~/.%src; workon owtf"
                   % os.environ["SHELL"].split(os.sep)[-1])
    Colorizer.info("[*] Start OWTF by running 'cd path/to/pentest/directory; ./path/to/owtf.py'")

def setup_virtualenv():
    Colorizer.info("[*] Setting up virtual environment named owtf...")
    # If /usr/local/bin/virtualenvwrapper.sh doesn't exist, create a symlink from /usr/bin/
    if not os.path.isfile('/usr/local/bin/virtualenvwrapper.sh'):
        run_command('sudo ln -s /usr/bin/virtualenvwrapper.sh /usr/local/bin/virtualenvwrapper.sh >/dev/null 2>&1;')

    # sources files and commands
    source = 'source /usr/local/bin/virtualenvwrapper.sh'
    setup_env = 'cd $WORKON_HOME; virtualenv -q --always-copy --python=python2.7 owtf >/dev/null 2>&1;'\
        ' source owtf/bin/activate'
    dump = '%s -c "import os, json;print json.dumps(dict(os.environ))"' % sys.executable

    pipe = subprocess.Popen(['/bin/bash', '-c', '%s >/dev/null 2>&1; %s; %s' % (source, setup_env, dump)],
                            stdout=subprocess.PIPE)
    env = json.loads(pipe.stdout.read())

    # Update the os environment variable
    os.environ.update(env)
    try:
        if os.path.join(os.environ["WORKON_HOME"], "owtf") == os.environ["VIRTUAL_ENV"]:
            # Add source to shell config file only if not present
            Colorizer.info("[*] Adding virtualenvwrapper source to shell config file")
            shell_rc_path = os.path.join(os.environ["HOME"], ".%src" % os.environ["SHELL"].split(os.sep)[-1])
            with open(shell_rc_path, "r") as f:
                if mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ).find(source) == -1:
                    run_command("echo '%s' >> %s" % (source, shell_rc_path))
                else:
                    Colorizer.info("[+] Source line already added to the $SHELL config ")
            return True
    except KeyError:
        traceback.print_exc()
    return False

def setup_pip():
    # Installing pip
    Colorizer.info("[*] Installing pip")
    directory = "/tmp/owtf-install/pip/%s" % os.getpid()
    command = 'command -v pip2 >/dev/null || { wget --user-agent="Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101'\
        ' Firefox/15.0" --tries=3 https://bootstrap.pypa.io/get-pip.py; sudo python get-pip.py;}'
    install_in_directory(os.path.expanduser(directory), command)
    Colorizer.info("[*] Installing required packages for pipsecure")
    run_command("sudo pip2 install --upgrade pyopenssl ndg-httpsclient pyasn1")

    # Installing virtualenv
    Colorizer.info("[*] Installing virtualenv and virtualenvwrapper")
    install_in_directory(os.path.expanduser(str(os.getpid())), "sudo pip2 install --upgrade virtualenv virtualenvwrapper")

def install(cmd_arguments):
    """Perform installation of OWTF Framework. Wraps around all helper methods made in this module.

    :param cmd_arguments:
    """
    args = parser.parse_args(cmd_arguments)
    # User asked to select distro (in case it can't be automatically detected) and distro related stuff is installed
    cp = ConfigParser.ConfigParser({"RootDir": root_dir, "Pid": pid})
    cp.read(distros_cfg)

    # Try get the distro automatically
    distro, version, arch = platform.linux_distribution()
    distro_num = 0
    if "kali" in distro.lower():
        distro_num = 1
    elif "samurai" in distro.lower():
        distro_num = 2
    elif is_compatible():
        distro_num = 3

    if distro_num != 0:
        Colorizer.info("[*] %s has been automatically detected... " % distro)
        Colorizer.normal("[*] Continuing in auto-mode")
    elif (distro_num == 0) and args.no_user_input:
        Colorizer.info("[*] Cannot auto-detect a supported distro...")
        Colorizer.normal("[*] Continuing in auto-mode with the core installation...")
    else:
        # Loop until proper input is received
        while True:
            print("")
            for i, item in enumerate(cp.sections()):
                Colorizer.warning("(%d) %s" % (i + 1, item))
            Colorizer.warning("(0) My distro is not listed :( %s" % distro)

            num_input = raw_input("Select a number based on your distribution : ")
            try:
                if int(num_input) <= len(cp.sections()):
                    distro_num = int(num_input)
                    break
                else:
                    print("")
                    Colorizer.warning("[!] Invalid number - not a supported distro")
                    continue
            except ValueError:
                print("")
                Colorizer.warning("[!] Invalid Number specified")
                continue

    # Install distro specific dependencies and packages needed for OWTF to work
    if distro_num != 0:
        run_command(cp.get(cp.sections()[int(distro_num) - 1], "install"))
    else:
        Colorizer.normal("[*] Skipping distro related installation :(")

    # Installing pip and setting up virtualenv.
    # This requires distro specific dependencies to be installed properly.
    setup_pip()
    if setup_virtualenv():
        install_using_pip(owtf_pip)
    else:
        Colorizer.danger("Unable to setup virtualenv...")
        Colorizer.danger("Skipping installation of OWTF python dependencies ...")

    # Now install distro independent stuff - optional
    # This is due to db config setup included in this. Should run only after PostgreSQL is installed.
    # See https://github.com/owtf/owtf/issues/797.
    install_restricted_from_cfg(restricted_cfg)

    Colorizer.normal("[*] Upgrading pip to the latest version ...")
    # Upgrade pip before install required libraries
    run_command("pip2 install --upgrade pip")
    Colorizer.normal("Upgrading setuptools to the latest version ...")
    # Upgrade setuptools
    run_command("pip2 install --upgrade setuptools")
    Colorizer.normal("Upgrading cffi to the latest version ...")
    # Mitigate cffi errors by upgrading it first
    run_command("pip2 install --upgrade cffi")

class Colorizer:

    """Helper class for colorized strings.

    Different statements will have different colors:

        - `normal`, denoting ongoing procedure (WHITE)
        - `info`, any file path, commit hash or any other info (BLUE)
        - `warning`, any potential hindrance in installation (YELLOW)
        - `danger`, abrupt failure, desired file/dir not found etc. (RED)
    """

    BOLD = '\033[1m'
    RED = BOLD + '\033[91m'
    GREEN = BOLD + '\033[92m'
    YELLOW = BOLD + '\033[93m'
    BLUE = BOLD + '\033[34m'
    PURPLE = BOLD + '\033[95m'
    CYAN = BOLD + '\033[36m'
    WHITE = BOLD + '\033[37m'
    END = '\033[0m\033[0m'

    def __init__(self):
        pass

    @classmethod
    def normal(cls, string):
        print(cls.WHITE + string + cls.END)

    @classmethod
    def info(cls, string):
        print(cls.CYAN + string + cls.END)

    @classmethod
    def warning(cls, string):
        print(cls.YELLOW + string + cls.END)

    @classmethod
    def success(cls, string):
        print(cls.GREEN + string + cls.END)

    @classmethod
    def danger(cls, string):
        print(cls.RED + string + cls.END)

if __name__ == "__main__":
    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    pid = os.getpid()

    # Path to custom scripts for tasks such as setting up/ running PostgreSQL db, run arachni, nikto, wapiti etc.
    scripts_path = os.path.join(root_dir, "scripts")

    # OWTF python libraries
    owtf_pip = os.path.join(root_dir, "install", "owtf.pip")

    # Restricted tools and dictionaries which are distro independent
    restricted_cfg = os.path.join(root_dir, "install", "distro-independent.cfg")

    # Various distros and install scripts
    distros_cfg = os.path.join(root_dir, "install", "linux-distributions.cfg")

    parser = argparse.ArgumentParser()
    parser.add_argument('--no-user-input', help='run script with default options for user input', action="store_true")

    Colorizer.normal("[*] Great that you are installing OWTF :D")
    Colorizer.warning("[!] There will be lot of output, please be patient")

    Colorizer.info("[*] Last commit hash: %s" % owtf_last_commit())
    check_sudo()
    install(sys.argv[1:])

    # Copying config files
    dest_config_path = os.path.join(os.path.expanduser('~'), '.owtf', 'configuration')
    create_directory(dest_config_path)
    src_config_path = os.path.join(root_dir, 'configuration')
    dir_util.copy_tree(src_config_path, dest_config_path)

    finish()
<|endoftext|>


<commit_before>import datetime
import logging
import urllib
import uuid

from consts.client_type import ClientType
from consts.notification_type import NotificationType

from google.appengine.api import urlfetch

from helpers.push_helper import PushHelper

from models.event import Event
from models.sitevar import Sitevar

from notifications.alliance_selections import AllianceSelectionNotification
from notifications.level_starting import CompLevelStartingNotification
from notifications.broadcast import BroadcastNotification
from notifications.match_score import MatchScoreNotification
from notifications.awards_updated import AwardsUpdatedNotification
from notifications.schedule_updated import ScheduleUpdatedNotification
from notifications.upcoming_match import UpcomingMatchNotification
from notifications.update_favorites import UpdateFavoritesNotification
from notifications.update_subscriptions import UpdateSubscriptionsNotification
from notifications.verification import VerificationNotification

class NotificationHelper(object):

    """
    Helper class for sending push notifications.
    Methods here should build a Notification object and use their send method
    """

    @classmethod
    def send_match_score_update(cls, match):
        users = PushHelper.get_users_subscribed_to_match(match, NotificationType.MATCH_SCORE)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = MatchScoreNotification(match)
        notification.send(keys)

    @classmethod
    def send_favorite_update(cls, user_id, sending_device_key=""):
        clients = PushHelper.get_client_ids_for_users([user_id])

        notification = UpdateFavoritesNotification(user_id, sending_device_key)
        notification.send(clients)

    @classmethod
    def send_subscription_update(cls, user_id, sending_device_key=""):
        clients = PushHelper.get_client_ids_for_users([user_id])

        notification = UpdateSubscriptionsNotification(user_id, sending_device_key)
        notification.send(clients)

    @classmethod
    def send_upcoming_match_notification(cls, match, event):
        users = PushHelper.get_users_subscribed_to_match(match, NotificationType.UPCOMING_MATCH)
        keys = PushHelper.get_client_ids_for_users(users)

        if match.set_number == 1 and match.match_number == 1:
            # First match of a new type, send level starting notifications
            start_users = PushHelper.get_users_subscribed_to_match(match, NotificationType.LEVEL_STARTING)
            start_keys = PushHelper.get_client_ids_for_users(start_users)
            level_start = CompLevelStartingNotification(match, event)
            level_start.send(start_keys)

        # Send upcoming match notification
        notification = UpcomingMatchNotification(match, event)
        notification.send(keys)
        match.push_sent = True  # Make sure we don't send updates for this match again
        match.put()

    @classmethod
    def send_upcoming_matches(cls, live_events):
        from helpers.match_helper import MatchHelper  # PJL: Hacky :P
        # Causes circular import, otherwise
        # https://github.com/the-blue-alliance/the-blue-alliance/pull/1098#discussion_r25128966

        now = datetime.datetime.utcnow()
        for event in live_events:
            matches = event.matches
            if not matches:
                continue
            last_matches = MatchHelper.recentMatches(matches, num=1)
            next_matches = MatchHelper.upcomingMatches(matches, num=2)

            # First, compare the difference between scheduled times of next/last match
            # Send an upcoming notification if it's <10 minutes, to account for events ahead of schedule
            if last_matches != []:
                last_match = last_matches[0]
                for i, next_match in enumerate(next_matches):
                    if not next_match.push_sent and last_match.time and next_match.time:
                        diff = next_match.time - last_match.time
                        if diff < datetime.timedelta(minutes=10*(i+1)):
                            cls.send_upcoming_match_notification(next_match, event)
                            # We're done here, no need to send anywhere else
                            continue

            for match in next_matches:
                if match and not match.push_sent:
                    # Only continue sending for the next match if a push hasn't already been sent for it
                    if match.time is None or match.time + datetime.timedelta(minutes=-7) <= now:
                        # Only send notifications for matches no more than 7 minutes (average-ish match cycle time) before it's scheduled to start
                        # Unless, the match has no time info. Then #yolo and send it
                        cls.send_upcoming_match_notification(match, event)

                        # Don't send update for any further matches
                        continue

    @classmethod
    def send_schedule_update(cls, event):
        users = PushHelper.get_users_subscribed_to_event(event, NotificationType.SCHEDULE_UPDATED)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = ScheduleUpdatedNotification(event)
        notification.send(keys)

    @classmethod
    def send_alliance_update(cls, event):
        users = PushHelper.get_users_subscribed_for_alliances(event, NotificationType.ALLIANCE_SELECTION)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = AllianceSelectionNotification(event)
        notification.send(keys)

    @classmethod
    def send_award_update(cls, event):
        users = PushHelper.get_users_subscribed_to_event(event, NotificationType.AWARDS)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = AwardsUpdatedNotification(event)
        notification.send(keys)

    @classmethod
    def send_broadcast(cls, client_types, title, message, url, app_version=''):
        users = PushHelper.get_all_mobile_clients(client_types)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = BroadcastNotification(title, message, url, app_version)
        notification.send(keys)

    @classmethod
    def verify_webhook(cls, url, secret):
        key = {ClientType.WEBHOOK: [(url, secret)]}
        notification = VerificationNotification(url, secret)
        notification.send(key)
        return notification.verification_key
<commit_msg>Upcoming match notification fixes<commit_after>import datetime
import logging
import urllib
import uuid

from consts.client_type import ClientType
from consts.notification_type import NotificationType

from google.appengine.api import urlfetch

from helpers.push_helper import PushHelper

from models.event import Event
from models.sitevar import Sitevar

from notifications.alliance_selections import AllianceSelectionNotification
from notifications.level_starting import CompLevelStartingNotification
from notifications.broadcast import BroadcastNotification
from notifications.match_score import MatchScoreNotification
from notifications.awards_updated import AwardsUpdatedNotification
from notifications.schedule_updated import ScheduleUpdatedNotification
from notifications.upcoming_match import UpcomingMatchNotification
from notifications.update_favorites import UpdateFavoritesNotification
from notifications.update_subscriptions import UpdateSubscriptionsNotification
from notifications.verification import VerificationNotification

class NotificationHelper(object):

    """
    Helper class for sending push notifications.
    Methods here should build a Notification object and use their send method
    """

    @classmethod
    def send_match_score_update(cls, match):
        users = PushHelper.get_users_subscribed_to_match(match, NotificationType.MATCH_SCORE)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = MatchScoreNotification(match)
        notification.send(keys)

    @classmethod
    def send_favorite_update(cls, user_id, sending_device_key=""):
        clients = PushHelper.get_client_ids_for_users([user_id])

        notification = UpdateFavoritesNotification(user_id, sending_device_key)
        notification.send(clients)

    @classmethod
    def send_subscription_update(cls, user_id, sending_device_key=""):
        clients = PushHelper.get_client_ids_for_users([user_id])

        notification = UpdateSubscriptionsNotification(user_id, sending_device_key)
        notification.send(clients)

    @classmethod
    def send_upcoming_match_notification(cls, match, event):
        users = PushHelper.get_users_subscribed_to_match(match, NotificationType.UPCOMING_MATCH)
        keys = PushHelper.get_client_ids_for_users(users)

        if match.set_number == 1 and match.match_number == 1:
            # First match of a new type, send level starting notifications
            start_users = PushHelper.get_users_subscribed_to_match(match, NotificationType.LEVEL_STARTING)
            start_keys = PushHelper.get_client_ids_for_users(start_users)
            level_start = CompLevelStartingNotification(match, event)
            level_start.send(start_keys)

        # Send upcoming match notification
        notification = UpcomingMatchNotification(match, event)
        notification.send(keys)
        match.push_sent = True  # Make sure we don't send updates for this match again
        match.put()

    @classmethod
    def send_upcoming_matches(cls, live_events):
        from helpers.match_helper import MatchHelper  # PJL: Hacky :P
        # Causes circular import, otherwise
        # https://github.com/the-blue-alliance/the-blue-alliance/pull/1098#discussion_r25128966

        now = datetime.datetime.utcnow()
        for event in live_events:
            matches = event.matches
            if not matches:
                continue
            last_matches = MatchHelper.recentMatches(matches, num=1)
            next_matches = MatchHelper.upcomingMatches(matches, num=2)

            # First, compare the difference between scheduled times of next/last match
            # Send an upcoming notification if it's <10 minutes, to account for events ahead of schedule
            event_done = False
            if last_matches != []:
                last_match = last_matches[0]
                for i, next_match in enumerate(next_matches):
                    if not next_match.push_sent and last_match.time and next_match.time:
                        diff = next_match.time - last_match.time
                        if diff < datetime.timedelta(minutes=10*(i+1)):
                            cls.send_upcoming_match_notification(next_match, event)
                            # We're done here, no need to send anywhere else
                            event_done = True
                            break
            if event_done:
                continue

            for match in next_matches:
                if match and not match.push_sent:
                    # Only continue sending for the next match if a push hasn't already been sent for it
                    if match.time is None or match.time + datetime.timedelta(minutes=-7) <= now:
                        # Only send notifications for matches no more than 7 minutes (average-ish match cycle time) before it's scheduled to start
                        # Unless, the match has no time info. Then #yolo and send it
                        cls.send_upcoming_match_notification(match, event)

                        # Don't send update for any further matches
                        break

    @classmethod
    def send_schedule_update(cls, event):
        users = PushHelper.get_users_subscribed_to_event(event, NotificationType.SCHEDULE_UPDATED)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = ScheduleUpdatedNotification(event)
        notification.send(keys)

    @classmethod
    def send_alliance_update(cls, event):
        users = PushHelper.get_users_subscribed_for_alliances(event, NotificationType.ALLIANCE_SELECTION)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = AllianceSelectionNotification(event)
        notification.send(keys)

    @classmethod
    def send_award_update(cls, event):
        users = PushHelper.get_users_subscribed_to_event(event, NotificationType.AWARDS)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = AwardsUpdatedNotification(event)
        notification.send(keys)

    @classmethod
    def send_broadcast(cls, client_types, title, message, url, app_version=''):
        users = PushHelper.get_all_mobile_clients(client_types)
        keys = PushHelper.get_client_ids_for_users(users)

        notification = BroadcastNotification(title, message, url, app_version)
        notification.send(keys)

    @classmethod
    def verify_webhook(cls, url, secret):
        key = {ClientType.WEBHOOK: [(url, secret)]}
        notification = VerificationNotification(url, secret)
        notification.send(key)
        return notification.verification_key
<|endoftext|>