<commit_before>from neurogenesis.util import Logger
import pickle
import imp

class SimulationConfig():
    def __init__(self):
        self.x_limits = (0,100,10)
        self.y_limits = (0, 100, 10)

        self.x_axis_parameter = ""
        self.y_axis_parameter = ""

def check_filter(sim, filter_list):
    for filter in filter_list:
        try:
            if not sim.parameters[filter[0]] == filter[1]:
               return False
        except KeyError:
            Logger.warning("The dataset does not contain filter value: %s" % (filter[0]))
            pass
    return True

def load_plot_config(path):
    for line in config_file:
        if line.lstrip()[0] is "#": #comment
            continue

        tokens = line.strip().split(" ")

def get_datapoints_in_buckets(simulations, x_axis_attr, y_axis_attr, filter=None):
    datapoints = []
    data_buckets = {}
    for sim in simulations.values():
        if type(filter) is tuple:
            if not filter or sim.results[filter[0]] == filter[1]:
                datapoints.append((sim.results[x_axis_attr], sim.results[y_axis_attr]))
        else:
            result = check_filter(sim, filter)
            if result:
                datapoints.append((sim.results[x_axis_attr], sim.results[y_axis_attr]))

    for point in datapoints:
        if point[0] not in data_buckets:
            data_buckets[point[0]] = []
        data_buckets[point[0]].append(point[1])
    tuples = []
    for (key, l) in data_buckets.items():
        tpl = (key, l)
        tuples.append(tpl)

    return sorted(tuples, key=lambda x: x[0])

def plot_simulations(simulations, plot_script_path):
    mod = imp.load_source("plot", plot_script_path)
    mod.plot(simulations)

<commit_msg>created data driven plotting methods<commit_after>import matplotlib as mpl
mpl.use('pdf')
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from neurogenesis.util import Logger
import pickle
import imp
import itertools
import datetime
from scipy import stats

class SimulationConfig():
    def __init__(self):
        self.x_limits = (0,100,10)
        self.y_limits = (0, 100, 10)

        self.x_axis_parameter = ""
        self.y_axis_parameter = ""

def check_filter(sim, filter_list):
    for filter in filter_list:
        try:
            if not sim.parameters[filter[0]] == filter[1]:
               return False
        except KeyError:
            Logger.warning("The dataset does not contain filter value: %s" % (filter[0]))
            Logger.warning("Available are: %s" % (sim.parameters))
            pass
    return True

def load_plot_config(path):
    for line in config_file:
        if line.lstrip()[0] is "#": #comment
            continue

        tokens = line.strip().split(" ")

def get_datapoints_in_buckets(simulations, x_axis_attr, y_axis_attr, filter=None):
    datapoints = []
    data_buckets = {}
    for sim in simulations.values():
        if type(filter) is tuple:
            if not filter or sim.results[filter[0]] == filter[1]:
                datapoints.append((sim.results[x_axis_attr], sim.results[y_axis_attr]))
        else:
            result = check_filter(sim, filter)
            if result:
                try:
                    datapoints.append((sim.results[x_axis_attr], sim.results[y_axis_attr]))
                except KeyError, e:
                    Logger.error("Could not find desired axis %s in data set!" % (e))
                    Logger.error("Available are: %s" % (sim.results.keys()))
                    exit(-1)

    for point in datapoints:
        if point[0] not in data_buckets:
            data_buckets[point[0]] = []
        data_buckets[point[0]].append(point[1])
    tuples = []
    for (key, l) in data_buckets.items():
        tpl = (key, l)
        tuples.append(tpl)

    return sorted(tuples, key=lambda x: x[0])

def generate_plot(simulation, plot_description, pdf):
    dimensions = plot_description['dimensions']
    config_map = {}
    all_filter_values = []

    for filter_values in dimensions:
        all_filter_values.append(filter_values[1])
        config_map[filter_values[0]] = filter_values[1]
    plot_configs = []
    for perm in itertools.product(*all_filter_values):
        filter = []
        for idx, dim in enumerate(perm):
            filter.append((dimensions[idx][0], dim))
        plot_configs.append(filter)

    plot_groups = plot_description['group-by'][:]
    plot_groups.insert(0, plot_description['line'])
    plot_groups.insert(0, plot_description['color'])

    for group in plot_groups:
        def getKey(plotConfig):
            for entry in plotConfig:
                if entry[0] == group:
                    return entry[1]
            Logger.warning("couldn't find group-key %s" % (group))
        plot_configs = sorted(plot_configs, key= getKey)

    line_parameter = plot_description['line']
    col_parameter = plot_description['color']

    nr_line_parameters = len(config_map[line_parameter])
    nr_col_parameters = len(config_map[col_parameter])

    line_mode = False
    ax = None
    box = None

    for idx, filter in enumerate(plot_configs):
        current_config = dict(filter)

        datapoints = get_datapoints_in_buckets(simulation.simulation_runs, plot_description['x-axis'], plot_description['y-axis'], filter)
        if len(datapoints) == 0:
            Logger.error("Error in plot config! Simdata does not contain filter: %s" % (filter))
            exit(-1)

        if idx > 0 and idx % (nr_line_parameters * nr_col_parameters) == 0:
            # save old stuff
            generate_legend(ax, len(datapoints[0][1]), plot_description)
            plt.savefig(pdf, format='pdf')

        if idx % (nr_line_parameters * nr_col_parameters) == 0:
            fig = plt.figure()
            title = generate_title(plot_description, current_config)
            fig.suptitle(title , fontsize=14, fontweight='bold')
            ax = plt.subplot(111)
            box = ax.get_position()
            ax.set_position([box.x0, box.y0 + box.height * 0.3 , box.width, box.height * 0.7])

        x_s = [dat[0] for dat in datapoints]
        y_s = [np.mean(dat[1]) for dat in datapoints]
        y_err = [stats.sem(dat[1]) for dat in datapoints]
        if idx % nr_col_parameters == 0:
            line_mode = not line_mode
            plt.gca().set_color_cycle(None)

        label = "%s | %s" % (current_config[line_parameter], current_config[col_parameter])
        if line_mode:
            ax.errorbar(x_s, y_s, yerr=y_err, label=label , marker="o", lw=3)
            #print("plotting dot: %s" %(idx))
        else:
            ax.errorbar(x_s, y_s, yerr=y_err, label=label , fmt="-.", lw=3)
            #print("plotting dash: %s" %(idx))
    plt.savefig(pdf, format='pdf')

def generate_legend(ax, n, plot_description):
    ax.set_xlabel(plot_description['x-axis-label'])
    ax.set_ylabel(plot_description['y-axis-label'])
    plt.figtext(0.02, 0.02, "n=%s" % (n))
    plt.legend(bbox_to_anchor=(0.0, -0.6, 1.0,  0), loc=3, ncol=2, mode="expand", borderaxespad=0.)

def generate_title(plot_description, current_config):
    title = plot_description['title' ]

    for entry in plot_description['group-by']:
        value = current_config[entry]
        title += " | %s" % (value)
    return title

def plot_simulations(simulations, plot_script_path):
    mod = imp.load_source("plot", plot_script_path)
    mod.plot(simulations)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for OSS-Fuzz infrastructure."""

import logging
import os
import re
import stat
import subprocess

import helper

ALLOWED_FUZZ_TARGET_EXTENSIONS = ['', '.exe']
FUZZ_TARGET_SEARCH_STRING = 'LLVMFuzzerTestOneInput'
VALID_TARGET_NAME = re.compile(r'^[a-zA-Z0-9_-]+$')

def chdir_to_root():
  """Changes cwd to OSS-Fuzz root directory."""
  # Change to oss-fuzz main directory so helper.py runs correctly.
  if os.getcwd() != helper.OSS_FUZZ_DIR:
    os.chdir(helper.OSS_FUZZ_DIR)

def execute(command, location=None, check_result=False):
  """ Runs a shell command in the specified directory location.

  Args:
    command: The command as a list to be run.
    location: The directory the command is run in.
    check_result: Should an exception be thrown on failed command.

  Returns:
    stdout, stderr, error code.

  Raises:
    RuntimeError: running a command resulted in an error.
  """

  if not location:
    location = os.getcwd()
  process = subprocess.Popen(command,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             cwd=location)
  out, err = process.communicate()
  out = out.decode('utf-8', errors='ignore')
  err = err.decode('utf-8', errors='ignore')
  if err:
    logging.debug('Stderr of command \'%s\' is %s.', ' '.join(command), err)
  if check_result and process.returncode:
    raise RuntimeError(
        'Executing command \'{0}\' failed with error: {1}.'.format(
            ' '.join(command), err))
  return out, err, process.returncode

def get_fuzz_targets(path):
  """Get list of fuzz targets in a directory.

  Args:
    path: A path to search for fuzz targets in.

  Returns:
    A list of paths to fuzzers or an empty list if None.
  """
  if not os.path.exists(path):
    return []
  fuzz_target_paths = []
  for root, _, fuzzers in os.walk(path):
    for fuzzer in fuzzers:
      file_path = os.path.join(root, fuzzer)
      if is_fuzz_target_local(file_path):
        fuzz_target_paths.append(file_path)

  return fuzz_target_paths

def get_container_name():
  """Gets the name of the current docker container you are in.
  /proc/self/cgroup can be used to check control groups e.g. Docker.
  See: https://docs.docker.com/config/containers/runmetrics/ for more info.

  Returns:
    Container name or None if not in a container.
  """
  with open('/proc/self/cgroup') as file_handle:
    if 'docker' not in file_handle.read():
      return None
  with open('/etc/hostname') as file_handle:
    return file_handle.read().strip()

def is_fuzz_target_local(file_path):
  """Returns whether |file_path| is a fuzz target binary (local path).
  Copied from clusterfuzz src/python/bot/fuzzers/utils.py
  with slight modifications.
  """
  filename, file_extension = os.path.splitext(os.path.basename(file_path))
  if not VALID_TARGET_NAME.match(filename):
    # Check fuzz target has a valid name (without any special chars).
    return False

  if file_extension not in ALLOWED_FUZZ_TARGET_EXTENSIONS:
    # Ignore files with disallowed extensions (to prevent opening e.g. .zips).
    return False

  if not os.path.exists(file_path) or not os.access(file_path, os.X_OK):
    return False

  if filename.endswith('_fuzzer'):
    return True

  if os.path.exists(file_path) and not stat.S_ISREG(os.stat(file_path).st_mode):
    return False

  with open(file_path, 'rb') as file_handle:
    return file_handle.read().find(FUZZ_TARGET_SEARCH_STRING.encode()) != -1
<commit_msg>CIFuzz: switch to systemd-detect-virt to detect docker (#4101)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for OSS-Fuzz infrastructure."""

import logging
import os
import re
import stat
import subprocess

import helper

ALLOWED_FUZZ_TARGET_EXTENSIONS = ['', '.exe']
FUZZ_TARGET_SEARCH_STRING = 'LLVMFuzzerTestOneInput'
VALID_TARGET_NAME = re.compile(r'^[a-zA-Z0-9_-]+$')

def chdir_to_root():
  """Changes cwd to OSS-Fuzz root directory."""
  # Change to oss-fuzz main directory so helper.py runs correctly.
  if os.getcwd() != helper.OSS_FUZZ_DIR:
    os.chdir(helper.OSS_FUZZ_DIR)

def execute(command, location=None, check_result=False):
  """ Runs a shell command in the specified directory location.

  Args:
    command: The command as a list to be run.
    location: The directory the command is run in.
    check_result: Should an exception be thrown on failed command.

  Returns:
    stdout, stderr, error code.

  Raises:
    RuntimeError: running a command resulted in an error.
  """

  if not location:
    location = os.getcwd()
  process = subprocess.Popen(command,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             cwd=location)
  out, err = process.communicate()
  out = out.decode('utf-8', errors='ignore')
  err = err.decode('utf-8', errors='ignore')
  if err:
    logging.debug('Stderr of command \'%s\' is %s.', ' '.join(command), err)
  if check_result and process.returncode:
    raise RuntimeError(
        'Executing command \'{0}\' failed with error: {1}.'.format(
            ' '.join(command), err))
  return out, err, process.returncode

def get_fuzz_targets(path):
  """Get list of fuzz targets in a directory.

  Args:
    path: A path to search for fuzz targets in.

  Returns:
    A list of paths to fuzzers or an empty list if None.
  """
  if not os.path.exists(path):
    return []
  fuzz_target_paths = []
  for root, _, fuzzers in os.walk(path):
    for fuzzer in fuzzers:
      file_path = os.path.join(root, fuzzer)
      if is_fuzz_target_local(file_path):
        fuzz_target_paths.append(file_path)

  return fuzz_target_paths

def get_container_name():
  """Gets the name of the current docker container you are in.

  Returns:
    Container name or None if not in a container.
  """
  result = subprocess.run(['systemd-detect-virt', '-c'],
                          stdout=subprocess.PIPE).stdout
  if b'docker' not in result:
    return None
  with open('/etc/hostname') as file_handle:
    return file_handle.read().strip()

def is_fuzz_target_local(file_path):
  """Returns whether |file_path| is a fuzz target binary (local path).
  Copied from clusterfuzz src/python/bot/fuzzers/utils.py
  with slight modifications.
  """
  filename, file_extension = os.path.splitext(os.path.basename(file_path))
  if not VALID_TARGET_NAME.match(filename):
    # Check fuzz target has a valid name (without any special chars).
    return False

  if file_extension not in ALLOWED_FUZZ_TARGET_EXTENSIONS:
    # Ignore files with disallowed extensions (to prevent opening e.g. .zips).
    return False

  if not os.path.exists(file_path) or not os.access(file_path, os.X_OK):
    return False

  if filename.endswith('_fuzzer'):
    return True

  if os.path.exists(file_path) and not stat.S_ISREG(os.stat(file_path).st_mode):
    return False

  with open(file_path, 'rb') as file_handle:
    return file_handle.read().find(FUZZ_TARGET_SEARCH_STRING.encode()) != -1
<|endoftext|>


<commit_before>from flask import Flask, render_template
from flask_socketio import SocketIO

print("Flask-SocketIO running...")
app = Flask(__name__)
app.debug = False
socketio = SocketIO(app)

@app.before_first_request
def initialize():
    print "Called only once, when the first request comes in"

@app.route('/')
def index():
    print('Rendering index.html')
    return render_template('index.html')

@socketio.on('ledCtrl', namespace='/test')
def ledCtrl(message):
    print(message['led'])

if __name__ == '__main__':
    socketio.run(app, host='0.0.0.0', port=5001)<commit_msg>serial write<commit_after>from flask import Flask, render_template
from flask_socketio import SocketIO
import serial

ser = serial.Serial()

print("Flask-SocketIO running...")
app = Flask(__name__)
app.debug = True
socketio = SocketIO(app)

@app.before_first_request
def initialize():
    print "Called only once, when the first request comes in"
    try:
        ser.port = "/dev/ttyACM0"
        ser.baudrate = 115200
        ser.open()
    except serial.SerialException as e:
        print(e)

@app.route('/')
def index():
    print('Rendering index.html')
    return render_template('index.html')

@socketio.on('ledCtrl', namespace='/test')
def ledCtrl(message):
    print(message['led'])
    if message['led']:
        ser.write('l1\r')
    else:
        ser.write('l0\r')

if __name__ == '__main__':
    socketio.run(app, host='0.0.0.0', port=5001)<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Anything related to deploying your project modules, media, and data
"""
import os, shutil, sys

from django.template.loader import render_to_string

from fabric.state import env
from fabric.operations import local, run, put, sudo
from fabric.decorators import runs_once
from fabric.contrib.files import exists
from fabric.contrib.console import confirm
#Required for a bug in 0.9
from fabric.version import get_version

from woven.decorators import run_once_per_version
from woven.deployment import deploy_files
from woven.environment import deployment_root, _root_domain

@runs_once
def _make_local_sitesettings(overwrite=False):
    local_settings_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings')
    if not os.path.exists(local_settings_dir) or overwrite:
        if overwrite:
            shutil.rmtree(local_settings_dir,ignore_errors=True)
        os.mkdir(local_settings_dir)
        f = open(os.path.join(local_settings_dir,'__init__.py'),"w")
        f.close()

    settings_file_path = os.path.join(local_settings_dir,'settings.py')
    if not os.path.exists(settings_file_path):
        root_domain = _root_domain()
        u_domain = root_domain.replace('.','_')
        output = render_to_string('woven/sitesettings.txt',
                {"deployment_root":deployment_root(),
                "site_id":"1",
                "project_name": env.project_name,
                "project_fullname": env.project_fullname,
                "project_package_name": env.project_package_name,
                "u_domain":u_domain,
                "domain":root_domain,
                "user":env,
                "MEDIA_URL":env.MEDIA_URL,
                "STATIC_URL":env.STATIC_URL}
            )

        f = open(settings_file_path,"w+")
        f.writelines(output)
        f.close()
        if env.verbosity:
            print "Created local sitesettings folder and default settings file"
        #copy manage.py into that directory
        manage_path = os.path.join(os.getcwd(),env.project_package_name,'manage.py')
        dest_manage_path = os.path.join(os.getcwd(),env.project_package_name,'sitesettings','manage.py')
        shutil.copy(manage_path, dest_manage_path)

    return

@run_once_per_version
def deploy_project():
    """
    Deploy to the project directory in the virtualenv
    """

    project_root = '/'.join([deployment_root(),'env',env.project_fullname,'project'])
    local_dir = os.getcwd()

    if env.verbosity:
        print env.host,"DEPLOYING project", env.project_fullname
    #Exclude a few things that we don't want deployed as part of the project folder
    rsync_exclude = ['local_settings*','*.pyc','*.log','.*','/build','/dist','/media*','/static*','/www','/public','/template*']

    #make site local settings if they don't already exist
    _make_local_sitesettings()
    created = deploy_files(local_dir, project_root, rsync_exclude=rsync_exclude)
    if not env.patch:
        #hook the project into sys.path - #TODO make the python version not fixed
        link_name = '/'.join([deployment_root(),'env',env.project_fullname,'lib/python2.6/site-packages',env.project_package_name])
        target = '/'.join([project_root,env.project_package_name])
        run(' '.join(['ln -s',target,link_name]))

    return created

def deploy_sitesettings():
    """
    Deploy to the project directory in the virtualenv
    """

    sitesettings = '/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])
    local_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings')

    created = deploy_files(local_dir, sitesettings)
    if env.verbosity and created:
        print env.host,"DEPLOYING sitesettings"
        for path in created:
            tail = path.split('/')[-1]
            print ' * uploaded',tail

@run_once_per_version
def deploy_templates():
    """
    Deploy any templates from your shortest TEMPLATE_DIRS setting
    """

    deployed = None
    if not hasattr(env, 'project_template_dir'):
        #the normal pattern would mean the shortest path is the main one.
        #its probably the last listed
        length = 1000
        for dir in env.TEMPLATE_DIRS:
            if dir:
                len_dir = len(dir)
                if len_dir < length:
                    length = len_dir
                    env.project_template_dir = dir

    if hasattr(env,'project_template_dir'):
        remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'templates'])
        if env.verbosity:
            print env.host,"DEPLOYING templates", remote_dir
        deployed = deploy_files(env.project_template_dir,remote_dir)
    return deployed

@run_once_per_version
def deploy_static():
    """
    Deploy static (application) versioned media
    """

    if not env.STATIC_URL or 'http://' in env.STATIC_URL: return
    from django.core.servers.basehttp import AdminMediaHandler
    remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'static'])
    m_prefix = len(env.MEDIA_URL)
    #if app media is not handled by django-staticfiles we can install admin media by default
    if 'django.contrib.admin' in env.INSTALLED_APPS and not 'django.contrib.staticfiles' in env.INSTALLED_APPS:

        if env.MEDIA_URL and env.MEDIA_URL == env.ADMIN_MEDIA_PREFIX[:m_prefix]:
            print "ERROR: Your ADMIN_MEDIA_PREFIX (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        admin = AdminMediaHandler('DummyApp')
        local_dir = admin.media_dir
        remote_dir =  ''.join([remote_dir,env.ADMIN_MEDIA_PREFIX])
    else:
        if env.MEDIA_URL and env.MEDIA_URL == env.STATIC_URL[:m_prefix]:
            print "ERROR: Your STATIC_URL (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        elif env.STATIC_ROOT:
            local_dir = env.STATIC_ROOT
            static_url = env.STATIC_URL[1:]
            if static_url:
                remote_dir = '/'.join([remote_dir,static_url])
        else: return
    if env.verbosity:
        print env.host,"DEPLOYING static",remote_dir
    return deploy_files(local_dir,remote_dir)

@run_once_per_version
def deploy_media():
    """
    Deploy MEDIA_ROOT unversioned on host
    """
    if not env.MEDIA_URL or not env.MEDIA_ROOT or 'http://' in env.MEDIA_URL: return
    local_dir = env.MEDIA_ROOT

    remote_dir = '/'.join([deployment_root(),'public'])
    media_url = env.MEDIA_URL[1:]
    if media_url:
        remote_dir = '/'.join([remote_dir,media_url])
    if env.verbosity:
        print env.host,"DEPLOYING media",remote_dir
    deployed = deploy_files(local_dir,remote_dir)

    #make writable for www-data for file uploads
    sudo("chown -R www-data:sudo %s" % remote_dir)
    sudo("chmod -R ug+w %s"% remote_dir)
    return deployed

@runs_once
def deploy_db(rollback=False):
    """
    Deploy a sqlite database from development
    """
    if not rollback:

        if env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
            db_dir = '/'.join([deployment_root(),'database'])
            db_name = ''.join([env.project_name,'_','site_1','.db'])
            dest_db_path = '/'.join([db_dir,db_name])
            if exists(dest_db_path): return
            if env.verbosity:
                print env.host,"DEPLOYING DEFAULT SQLITE DATABASE"
            if not env.DEFAULT_DATABASE_NAME:
                print "ERROR: A database name has not been defined in your Django settings file"
                sys.exit(1)

            if env.DEFAULT_DATABASE_NAME[0] not in [os.path.sep,'.']: #relative path
                db_path = os.path.join(os.getcwd(),env.project_package_name,env.DEFAULT_DATABASE_NAME)

            elif env.DEFAULT_DATABASE_NAME[:2] == '..':
                print "ERROR: Use a full expanded path to the database in your Django settings"
                sys.exit(1)
            else:
                db_path = env.DEFAULT_DATABASE_NAME

            if not db_path or not os.path.exists(db_path):
                print "ERROR: the database %s does not exist. \nRun python manage.py syncdb to create your database locally first, or check your settings."% db_path
                sys.exit(1)

            db_name = os.path.split(db_path)[1]
            run('mkdir -p '+db_dir)
            put(db_path,dest_db_path)
            #directory and file must be writable by webserver
            sudo("chown -R %s:www-data %s"% (env.user,db_dir))
            sudo("chmod -R ug+w %s"% db_dir)

        elif env.DEFAULT_DATABASE_ENGINE=='django.db.backends.':
            print "ERROR: The default database engine has not been defined in your Django settings file"
            print "At a minimum you must define an sqlite3 database for woven to deploy, or define a database that is managed outside of woven."
            sys.exit(1)
    elif rollback and env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
        if env.INTERACTIVE:
            delete = confirm('DELETE the database on the host?',default=False)
            if delete:
                run('rm -f '+db_name)
    return
<commit_msg>make sure sitesettings manage.py has chmod +x <commit_after>#!/usr/bin/env python
"""
Anything related to deploying your project modules, media, and data
"""
import os, shutil, sys

from django.template.loader import render_to_string

from fabric.state import env
from fabric.operations import local, run, put, sudo
from fabric.decorators import runs_once
from fabric.contrib.files import exists
from fabric.contrib.console import confirm
#Required for a bug in 0.9
from fabric.version import get_version

from woven.decorators import run_once_per_version
from woven.deployment import deploy_files
from woven.environment import deployment_root, _root_domain

@runs_once
def _make_local_sitesettings(overwrite=False):
    local_settings_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings')
    if not os.path.exists(local_settings_dir) or overwrite:
        if overwrite:
            shutil.rmtree(local_settings_dir,ignore_errors=True)
        os.mkdir(local_settings_dir)
        f = open(os.path.join(local_settings_dir,'__init__.py'),"w")
        f.close()

    settings_file_path = os.path.join(local_settings_dir,'settings.py')
    if not os.path.exists(settings_file_path):
        root_domain = _root_domain()
        u_domain = root_domain.replace('.','_')
        output = render_to_string('woven/sitesettings.txt',
                {"deployment_root":deployment_root(),
                "site_id":"1",
                "project_name": env.project_name,
                "project_fullname": env.project_fullname,
                "project_package_name": env.project_package_name,
                "u_domain":u_domain,
                "domain":root_domain,
                "user":env,
                "MEDIA_URL":env.MEDIA_URL,
                "STATIC_URL":env.STATIC_URL}
            )

        f = open(settings_file_path,"w+")
        f.writelines(output)
        f.close()
        if env.verbosity:
            print "Created local sitesettings folder and default settings file"
        #copy manage.py into that directory
        manage_path = os.path.join(os.getcwd(),env.project_package_name,'manage.py')
        dest_manage_path = os.path.join(os.getcwd(),env.project_package_name,'sitesettings','manage.py')
        shutil.copy(manage_path, dest_manage_path)

    return

@run_once_per_version
def deploy_project():
    """
    Deploy to the project directory in the virtualenv
    """

    project_root = '/'.join([deployment_root(),'env',env.project_fullname,'project'])
    local_dir = os.getcwd()

    if env.verbosity:
        print env.host,"DEPLOYING project", env.project_fullname
    #Exclude a few things that we don't want deployed as part of the project folder
    rsync_exclude = ['local_settings*','*.pyc','*.log','.*','/build','/dist','/media*','/static*','/www','/public','/template*']

    #make site local settings if they don't already exist
    _make_local_sitesettings()
    created = deploy_files(local_dir, project_root, rsync_exclude=rsync_exclude)
    if not env.patch:
        #hook the project into sys.path - #TODO make the python version not fixed
        link_name = '/'.join([deployment_root(),'env',env.project_fullname,'lib/python2.6/site-packages',env.project_package_name])
        target = '/'.join([project_root,env.project_package_name])
        run(' '.join(['ln -s',target,link_name]))

        #make sure manage.py has exec permissions
        managepy = '/'.join([target,'sitesettings','manage.py'])
        if exists(managepy):
            sudo('chmod ugo+x %s'% managepy)

    return created

def deploy_sitesettings():
    """
    Deploy to the project directory in the virtualenv
    """

    sitesettings = '/'.join([deployment_root(),'env',env.project_fullname,'project',env.project_package_name,'sitesettings'])
    local_dir = os.path.join(os.getcwd(),env.project_package_name,'sitesettings')

    created = deploy_files(local_dir, sitesettings)
    if env.verbosity and created:
        print env.host,"DEPLOYING sitesettings"
        for path in created:
            tail = path.split('/')[-1]
            print ' * uploaded',tail

@run_once_per_version
def deploy_templates():
    """
    Deploy any templates from your shortest TEMPLATE_DIRS setting
    """

    deployed = None
    if not hasattr(env, 'project_template_dir'):
        #the normal pattern would mean the shortest path is the main one.
        #its probably the last listed
        length = 1000
        for dir in env.TEMPLATE_DIRS:
            if dir:
                len_dir = len(dir)
                if len_dir < length:
                    length = len_dir
                    env.project_template_dir = dir

    if hasattr(env,'project_template_dir'):
        remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'templates'])
        if env.verbosity:
            print env.host,"DEPLOYING templates", remote_dir
        deployed = deploy_files(env.project_template_dir,remote_dir)
    return deployed

@run_once_per_version
def deploy_static():
    """
    Deploy static (application) versioned media
    """

    if not env.STATIC_URL or 'http://' in env.STATIC_URL: return
    from django.core.servers.basehttp import AdminMediaHandler
    remote_dir = '/'.join([deployment_root(),'env',env.project_fullname,'static'])
    m_prefix = len(env.MEDIA_URL)
    #if app media is not handled by django-staticfiles we can install admin media by default
    if 'django.contrib.admin' in env.INSTALLED_APPS and not 'django.contrib.staticfiles' in env.INSTALLED_APPS:

        if env.MEDIA_URL and env.MEDIA_URL == env.ADMIN_MEDIA_PREFIX[:m_prefix]:
            print "ERROR: Your ADMIN_MEDIA_PREFIX (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        admin = AdminMediaHandler('DummyApp')
        local_dir = admin.media_dir
        remote_dir =  ''.join([remote_dir,env.ADMIN_MEDIA_PREFIX])
    else:
        if env.MEDIA_URL and env.MEDIA_URL == env.STATIC_URL[:m_prefix]:
            print "ERROR: Your STATIC_URL (Application media) must not be on the same path as your MEDIA_URL (User media)"
            sys.exit(1)
        elif env.STATIC_ROOT:
            local_dir = env.STATIC_ROOT
            static_url = env.STATIC_URL[1:]
            if static_url:
                remote_dir = '/'.join([remote_dir,static_url])
        else: return
    if env.verbosity:
        print env.host,"DEPLOYING static",remote_dir
    return deploy_files(local_dir,remote_dir)

@run_once_per_version
def deploy_media():
    """
    Deploy MEDIA_ROOT unversioned on host
    """
    if not env.MEDIA_URL or not env.MEDIA_ROOT or 'http://' in env.MEDIA_URL: return
    local_dir = env.MEDIA_ROOT

    remote_dir = '/'.join([deployment_root(),'public'])
    media_url = env.MEDIA_URL[1:]
    if media_url:
        remote_dir = '/'.join([remote_dir,media_url])
    if env.verbosity:
        print env.host,"DEPLOYING media",remote_dir
    deployed = deploy_files(local_dir,remote_dir)

    #make writable for www-data for file uploads
    sudo("chown -R www-data:sudo %s" % remote_dir)
    sudo("chmod -R ug+w %s"% remote_dir)
    return deployed

@runs_once
def deploy_db(rollback=False):
    """
    Deploy a sqlite database from development
    """
    if not rollback:

        if env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
            db_dir = '/'.join([deployment_root(),'database'])
            db_name = ''.join([env.project_name,'_','site_1','.db'])
            dest_db_path = '/'.join([db_dir,db_name])
            if exists(dest_db_path): return
            if env.verbosity:
                print env.host,"DEPLOYING DEFAULT SQLITE DATABASE"
            if not env.DEFAULT_DATABASE_NAME:
                print "ERROR: A database name has not been defined in your Django settings file"
                sys.exit(1)

            if env.DEFAULT_DATABASE_NAME[0] not in [os.path.sep,'.']: #relative path
                db_path = os.path.join(os.getcwd(),env.project_package_name,env.DEFAULT_DATABASE_NAME)

            elif env.DEFAULT_DATABASE_NAME[:2] == '..':
                print "ERROR: Use a full expanded path to the database in your Django settings"
                sys.exit(1)
            else:
                db_path = env.DEFAULT_DATABASE_NAME

            if not db_path or not os.path.exists(db_path):
                print "ERROR: the database %s does not exist. \nRun python manage.py syncdb to create your database locally first, or check your settings."% db_path
                sys.exit(1)

            db_name = os.path.split(db_path)[1]
            run('mkdir -p '+db_dir)
            put(db_path,dest_db_path)
            #directory and file must be writable by webserver
            sudo("chown -R %s:www-data %s"% (env.user,db_dir))
            sudo("chmod -R ug+w %s"% db_dir)

        elif env.DEFAULT_DATABASE_ENGINE=='django.db.backends.':
            print "ERROR: The default database engine has not been defined in your Django settings file"
            print "At a minimum you must define an sqlite3 database for woven to deploy, or define a database that is managed outside of woven."
            sys.exit(1)
    elif rollback and env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
        if env.INTERACTIVE:
            delete = confirm('DELETE the database on the host?',default=False)
            if delete:
                run('rm -f '+db_name)
    return
<|endoftext|>


<commit_before># encoding: utf-8

from __future__ import unicode_literals

import logging
import time

from threading import local

logger = logging.getLogger(__name__)

_thread_locals = local()

def get_current_request():
    """ returns the request object for this thead """
    return getattr(_thread_locals, "request", None)

class RequestLoggerMiddleware(object):

    def process_request(self, request):
        _thread_locals.request = request
        request.start_request_time = time.time()
        logger.info("{method} {path}".format(
            method=request.method,
            path=request.get_full_path()),
            extra={
                'request_method': request.method,
                'http_host': request.META.get('HTTP_HOST'),
                'http_path': request.get_full_path(),
                'request_id': request.META.get('HTTP_REQUEST_ID')
        })

    def process_response(self, request, response):
        if hasattr(request, 'start_request_time'):
            elapsed_time = time.time() - request.start_request_time
            logger.info("{method} {path} : {status} {secs:.6f}s".format(
                method=request.method,
                path=request.get_full_path(),
                status=response.status_code,
                secs=elapsed_time),
                extra={
                    'request_method': request.method,
                    'http_host': request.META.get('HTTP_HOST'),
                    'http_path': request.get_full_path(),
                    'status': response.status_code,
                    'request_time': elapsed_time,
            })
        return response

class AdditionalFieldsFilter(logging.Filter):

    def filter(self, record):
        request = get_current_request()
        if request:
            record.__dict__['request_method'] = request.method,
            record.__dict__['http_host'] = request.META.get('HTTP_HOST'),
            record.__dict__['http_path'] = request.get_full_path(),
            record.__dict__['request_id'] = request.META.get('HTTP_REQUEST_ID')
        return 1
<commit_msg>Stop using the LogRecord dict for extra fields<commit_after># encoding: utf-8

from __future__ import unicode_literals

import logging
import time

from threading import local

logger = logging.getLogger(__name__)

_thread_locals = local()

def get_current_request():
    """ returns the request object for this thead """
    return getattr(_thread_locals, "request", None)

class RequestLoggerMiddleware(object):

    def process_request(self, request):
        _thread_locals.request = request
        request.start_request_time = time.time()
        logger.info("{method} {path}".format(
            method=request.method,
            path=request.get_full_path()),
            extra={
                'request_method': request.method,
                'http_host': request.META.get('HTTP_HOST'),
                'http_path': request.get_full_path(),
                'request_id': request.META.get('HTTP_REQUEST_ID')
        })

    def process_response(self, request, response):
        if hasattr(request, 'start_request_time'):
            elapsed_time = time.time() - request.start_request_time
            logger.info("{method} {path} : {status} {secs:.6f}s".format(
                method=request.method,
                path=request.get_full_path(),
                status=response.status_code,
                secs=elapsed_time),
                extra={
                    'request_method': request.method,
                    'http_host': request.META.get('HTTP_HOST'),
                    'http_path': request.get_full_path(),
                    'status': response.status_code,
                    'request_time': elapsed_time,
            })
        return response

class AdditionalFieldsFilter(logging.Filter):

    def filter(self, record):
        request = get_current_request()
        if request:
            record.request_method = request.method,
            record.http_host = request.META.get('HTTP_HOST'),
            record.http_path = request.get_full_path(),
            record.request_id = request.META.get('HTTP_REQUEST_ID')
        return 1
<|endoftext|>


<commit_before>"""Render source models as simulated survey observations.
"""

import math
import inspect

import numpy as np

import galsim

class Engine(object):
    """Rendering engine to simulate survey observations.

    Args:
        survey(descwl.survey.Survey): Survey that rendered images will simulate.
        min_snr(float): Simulate signals from individual sources down to this S/N threshold,
            where the signal N is calculated for the full exposure time and the noise N is
            set by the expected fluctuations in the sky background during a full exposure.
        truncate_radius(float): All extended sources are truncated at this radius in arcseconds.
        no_margin(bool): Do not simulate the tails of objects just outside the field.
        verbose_render(bool): Provide verbose output on rendering process.
    """
    def __init__(self,survey,min_snr,truncate_radius,no_margin,verbose_render):
        self.survey = survey
        self.min_snr = min_snr
        self.truncate_radius = truncate_radius
        self.no_margin = no_margin
        self.verbose_render = verbose_render
        # Calculate pixel flux threshold in electrons per pixel that determines how big a
        # bounding box we simulate for each source.
        sky_noise = math.sqrt(survey.get_sky_level())
        self.pixel_cut = self.min_snr*sky_noise
        # Initialize our GalSim parameters.
        self.galsim_params = galsim.GSParams(maximum_fft_size=32768)
        # Evaluate the PSF dilution factor as the maximum fraction of a source's total flux
        # that can end up in a single pixel after convolution with the PSF.
        psf_stamp = galsim.ImageD(1,1,scale=self.survey.pixel_scale)
        self.survey.psf_model.drawImage(image = psf_stamp)
        self.psf_dilution = psf_stamp.array[0]
        # We will render each source into a square stamp with width = height = 2*padding + 1.
        self.padding = int(math.ceil(self.truncate_radius/self.survey.pixel_scale - 0.5))

    def description(self):
        """Describe our rendering configuration.

        Returns:
            str: Description of the rendering configuration that we be used to simulate
                the survey.
        """
        return '\n'.join([
            ('Will render all pixels with at least %.1f detected electrons.' % self.pixel_cut),
            ('PSF dilution factor is %.6f.' % self.psf_dilution)
            ])

    def render_galaxy(self,galaxy):
        """Render a galaxy model for a simulated survey.

        Args:
            galaxy(descwl.model.Galaxy): Model of the galaxy to render.

        Returns:
            :class:`numpy.ndarray`: Array of shape (nstamp,width,height) pixel values that represent
                nstamp postage-stamp images with the same dimensions (width,height) calculated
                based on the rendering options provided. Returns None if this galaxy has no
                pixels above threshold that are visible in the simulated survey image.
        """
        # Skip sources that are too faint to possibly be above our cut after PSF convolution.
        if galaxy.model.getFlux()*self.psf_dilution < self.pixel_cut:
            return None

        # Calculate the offset of the source center from the bottom-left corner of the
        # simulated image in floating-point pixel units.
        centroid = galaxy.model.centroid()
        x_center_pixels,y_center_pixels = self.survey.get_image_coordinates(centroid.x,centroid.y)

        # Calculate the corresponding central pixel indices in the full image, where (0,0) is the
        # bottom-left corner.
        x_center_index = int(math.floor(x_center_pixels))
        y_center_index = int(math.floor(y_center_pixels))

        # Calculate the bounding box to use for simulating this galaxy.
        x_min = x_center_index - self.padding
        x_max = x_center_index + self.padding
        y_min = y_center_index - self.padding
        y_max = y_center_index + self.padding
        bounds = galsim.BoundsI(x_min,x_max,y_min,y_max)

        # Is there any overlap with the simulated image?
        survey_overlap = bounds & self.survey.image.bounds
        if survey_overlap.area() == 0:
            print 'no overlap!'
            return None

        # Calculate the offset of the bounding box center from the image center in arcsecs.
        dx_stamp_arcsec = 0.5*(x_min + x_max+1 - self.survey.image_width)*self.survey.pixel_scale
        dy_stamp_arcsec = 0.5*(y_min + y_max+1 - self.survey.image_height)*self.survey.pixel_scale

        # Shift the model to the bounding box center and convolve with the survey PSF.
        # We do not convolve by the pixel response since drawImage takes care of this.
        model = galsim.Convolve([
            galaxy.model.shift(dx=-dx_stamp_arcsec,dy=-dy_stamp_arcsec),
            self.survey.psf_model
            ],gsparams=self.galsim_params)

        # Render the model in its own postage stamp.
        stamp = galsim.Image(bounds = bounds,scale = self.survey.pixel_scale, dtype = np.float64)
        model.drawImage(image = stamp, use_true_center = True)
        self.survey.image[survey_overlap] += stamp[survey_overlap]

        # Draw directly into the survey image, with no truncation.
        #model.drawImage(image = self.survey.image,add_to_image = True,use_true_center = True)

        if self.verbose_render:
            print 'Rendering galaxy model for id = %d with z = %.3f' % (
                galaxy.identifier,galaxy.redshift)
            print 'bounds: [%d:%d,%d:%d] w,h = %d,%d' % (
                x_min,x_max,y_min,y_max,x_max-x_min+1,y_max-y_min+1)
            print ' shift: (%.6f,%.6f) arcsec relative to stamp center' % (
                model.centroid().x,model.centroid().y)

        return stamp.array[np.newaxis,:,:]

    @staticmethod
    def add_args(parser):
        """Add command-line arguments for constructing a new :class:`Engine`.

        The added arguments are our constructor parameters with '_' replaced by '-' in the names.
        Note that constructor parameter defaults are specified here rather than in the constructor,
        so that they are included in command-line help.

        Args:
            parser(argparse.ArgumentParser): Arguments will be added to this parser object using its
                add_argument method.
        """
        parser.add_argument('--min-snr', type = float, default = 0.5, metavar = 'SNR',
            help = 'Simulate signals from individual sources down to this S/N threshold.')
        parser.add_argument('--truncate-radius', type = float, default = 30., metavar = 'SIZE',
            help = 'All extended sources are truncated at this radius in arcseconds.')
        parser.add_argument('--no-margin', action = 'store_true',
            help = 'Do not simulate the tails of objects just outside the field.')
        parser.add_argument('--verbose-render', action = 'store_true',
            help = 'Provide verbose output on rendering process.')

    @classmethod
    def from_args(cls,survey,args):
        """Create a new :class:`Engine` object from a set of arguments.

        Args:
            survey(descwl.survey.Survey): Survey that rendered images will simulate.
            args(object): A set of arguments accessed as a :py:class:`dict` using the
                built-in :py:func:`vars` function. Any extra arguments beyond those defined
                in :func:`add_args` will be silently ignored.

        Returns:
            :class:`Engine`: A newly constructed Engine object.
        """
        # Look up the named constructor parameters.
        pnames = (inspect.getargspec(cls.__init__)).args[1:]
        # Get a dictionary of the arguments provided.
        args_dict = vars(args)
        # Filter the dictionary to only include constructor parameters.
        filtered_dict = { key:args_dict[key] for key in (set(pnames) & set(args_dict)) }
        return cls(survey,**filtered_dict)
<commit_msg>Crop rendered images to SNR cut and truncation radius<commit_after>"""Render source models as simulated survey observations.
"""

import math
import inspect

import numpy as np

import galsim

class Engine(object):
    """Rendering engine to simulate survey observations.

    Any pixels outside of the truncation radius or below the minimum S/N cut will have their
    flux set to zero in the rendered image. As a result the total rendered flux may be below
    the total model flux.

    Args:
        survey(descwl.survey.Survey): Survey that rendered images will simulate.
        min_snr(float): Simulate signals from individual sources down to this S/N threshold,
            where the signal N is calculated for the full exposure time and the noise N is
            set by the expected fluctuations in the sky background during a full exposure.
        truncate_radius(float): All extended sources are truncated at this radius in arcseconds.
        no_margin(bool): Do not simulate the tails of objects just outside the field.
        verbose_render(bool): Provide verbose output on rendering process.
    """
    def __init__(self,survey,min_snr,truncate_radius,no_margin,verbose_render):
        self.survey = survey
        self.min_snr = min_snr
        self.truncate_radius = truncate_radius
        self.no_margin = no_margin
        self.verbose_render = verbose_render
        # Calculate pixel flux threshold in electrons per pixel that determines how big a
        # bounding box we simulate for each source.
        sky_noise = math.sqrt(survey.get_sky_level())
        self.pixel_cut = self.min_snr*sky_noise
        # Initialize our GalSim parameters.
        self.galsim_params = galsim.GSParams(maximum_fft_size=32768)
        # Evaluate the PSF dilution factor as the maximum fraction of a source's total flux
        # that can end up in a single pixel after convolution with the PSF.
        psf_stamp = galsim.ImageD(1,1,scale=self.survey.pixel_scale)
        self.survey.psf_model.drawImage(image = psf_stamp)
        self.psf_dilution = psf_stamp.array[0]
        # We will render each source into a square stamp with width = height = 2*padding + 1.
        self.padding = int(math.ceil(self.truncate_radius/self.survey.pixel_scale - 0.5))
        # Prepare a truncation mask.
        pixel_grid = np.arange(-self.padding,self.padding+1)*self.survey.pixel_scale
        pixel_x,pixel_y = np.meshgrid(pixel_grid,pixel_grid)
        pixel_radius = np.sqrt(pixel_x**2 + pixel_y**2)
        self.truncation_mask = (pixel_radius <= self.truncate_radius)

    def description(self):
        """Describe our rendering configuration.

        Returns:
            str: Description of the rendering configuration that we be used to simulate
                the survey.
        """
        return '\n'.join([
            ('Will render all pixels with at least %.1f detected electrons.' % self.pixel_cut),
            ('PSF dilution factor is %.6f.' % self.psf_dilution)
            ])

    def render_galaxy(self,galaxy):
        """Render a galaxy model for a simulated survey.

        Args:
            galaxy(descwl.model.Galaxy): Model of the galaxy to render.

        Returns:
            :class:`numpy.ndarray`: Array of shape (nstamp,width,height) pixel values that represent
                nstamp postage-stamp images with the same dimensions (width,height) calculated
                based on the rendering options provided. Returns None if this galaxy has no
                pixels above threshold that are visible in the simulated survey image.
        """
        # Skip sources that are too faint to possibly be above our cut after PSF convolution.
        if galaxy.model.getFlux()*self.psf_dilution < self.pixel_cut:
            return None

        # Calculate the offset of the source center from the bottom-left corner of the
        # simulated image in floating-point pixel units.
        centroid = galaxy.model.centroid()
        x_center_pixels,y_center_pixels = self.survey.get_image_coordinates(centroid.x,centroid.y)

        # Calculate the corresponding central pixel indices in the full image, where (0,0) is the
        # bottom-left corner.
        x_center_index = int(math.floor(x_center_pixels))
        y_center_index = int(math.floor(y_center_pixels))

        # Calculate the bounding box to use for simulating this galaxy.
        x_min = x_center_index - self.padding
        x_max = x_center_index + self.padding
        y_min = y_center_index - self.padding
        y_max = y_center_index + self.padding
        bounds = galsim.BoundsI(x_min,x_max,y_min,y_max)

        # Calculate the offset of the bounding box center from the image center in arcsecs.
        dx_stamp_arcsec = 0.5*(x_min + x_max+1 - self.survey.image_width)*self.survey.pixel_scale
        dy_stamp_arcsec = 0.5*(y_min + y_max+1 - self.survey.image_height)*self.survey.pixel_scale

        # Shift the model to the bounding box center and convolve with the survey PSF.
        # We do not convolve by the pixel response since drawImage takes care of this.
        model = galsim.Convolve([
            galaxy.model.shift(dx=-dx_stamp_arcsec,dy=-dy_stamp_arcsec),
            self.survey.psf_model
            ],gsparams=self.galsim_params)

        # Render the model in its own postage stamp.
        stamp = galsim.Image(bounds = bounds,scale = self.survey.pixel_scale, dtype = np.float64)
        model.drawImage(image = stamp, use_true_center = True)

        # Identify pixels with flux above our cut and within our truncation radius
        # and zero all other pixel fluxes.
        keep_mask = (stamp.array*self.truncation_mask > self.pixel_cut)
        if np.sum(keep_mask) == 0:
            return None
        stamp.array[np.logical_not(keep_mask)] = 0.

        # Crop the bounding box.
        x_projection = (np.sum(keep_mask,axis=0) > 0)
        y_projection = (np.sum(keep_mask,axis=1) > 0)
        x_min_inset = np.argmax(x_projection)
        x_max_inset = np.argmax(x_projection[::-1])
        y_min_inset = np.argmax(y_projection)
        y_max_inset = np.argmax(y_projection[::-1])
        cropped_bounds = galsim.BoundsI(
            x_min+x_min_inset,x_max-x_max_inset,
            y_min+y_min_inset,y_max-y_max_inset)
        cropped_stamp = stamp[cropped_bounds]

        # Add the rendered model to the survey image.
        survey_overlap = cropped_bounds & self.survey.image.bounds
        if survey_overlap.area() == 0:
            return None
        self.survey.image[survey_overlap] += cropped_stamp[survey_overlap]

        # Draw directly into the survey image, with no truncation.
        #model.drawImage(image = self.survey.image,add_to_image = True,use_true_center = True)

        if self.verbose_render:
            print 'Rendered galaxy model for id = %d with z = %.3f' % (
                galaxy.identifier,galaxy.redshift)
            print 'bounds: [%d:%d,%d:%d] w,h = %d,%d' % (
                x_min,x_max,y_min,y_max,x_max-x_min+1,y_max-y_min+1)
            print ' shift: (%.6f,%.6f) arcsec relative to stamp center' % (
                model.centroid().x,model.centroid().y)

        return cropped_stamp.array[np.newaxis,:,:]

    @staticmethod
    def add_args(parser):
        """Add command-line arguments for constructing a new :class:`Engine`.

        The added arguments are our constructor parameters with '_' replaced by '-' in the names.
        Note that constructor parameter defaults are specified here rather than in the constructor,
        so that they are included in command-line help.

        Args:
            parser(argparse.ArgumentParser): Arguments will be added to this parser object using its
                add_argument method.
        """
        parser.add_argument('--min-snr', type = float, default = 0.5, metavar = 'SNR',
            help = 'Simulate signals from individual sources down to this S/N threshold.')
        parser.add_argument('--truncate-radius', type = float, default = 30., metavar = 'SIZE',
            help = 'All extended sources are truncated at this radius in arcseconds.')
        parser.add_argument('--no-margin', action = 'store_true',
            help = 'Do not simulate the tails of objects just outside the field.')
        parser.add_argument('--verbose-render', action = 'store_true',
            help = 'Provide verbose output on rendering process.')

    @classmethod
    def from_args(cls,survey,args):
        """Create a new :class:`Engine` object from a set of arguments.

        Args:
            survey(descwl.survey.Survey): Survey that rendered images will simulate.
            args(object): A set of arguments accessed as a :py:class:`dict` using the
                built-in :py:func:`vars` function. Any extra arguments beyond those defined
                in :func:`add_args` will be silently ignored.

        Returns:
            :class:`Engine`: A newly constructed Engine object.
        """
        # Look up the named constructor parameters.
        pnames = (inspect.getargspec(cls.__init__)).args[1:]
        # Get a dictionary of the arguments provided.
        args_dict = vars(args)
        # Filter the dictionary to only include constructor parameters.
        filtered_dict = { key:args_dict[key] for key in (set(pnames) & set(args_dict)) }
        return cls(survey,**filtered_dict)
<|endoftext|>


<commit_before>import os
import sys
import optparse
import lib.vcrparser as vcrparser
import lib.spider_stats as spider_stats
import lib.html_lib as html

#########################################################################
# PARSER
#########################################################################
desc = "aRNApipe: SPIDER module"
parser = optparse.OptionParser(description = desc)
parser.add_option("-p", "--path", dest = "path", default = "", help = "Required: Path to the project folder")
(opt, args) = parser.parse_args()

#########################################################################
# INITIAL ARRANGEMENTS
#########################################################################
pathscript = os.path.dirname(sys.argv[0]) + "/R/" # PATH TO R SCRIPTS
project, path = html.check_project(opt.path) # CHECKS IF PROJECT FOLDER, SAMPLES FILE AND CONFIGURATION FILE EXIST
html.skeleton(path, os.path.dirname(sys.argv[0])) # CREATES THE SKELETON FOR THE HTML AND OUTPUT RESULTS
config = html.check_config(path) # PARSES THE CONFIGURATION FILE
try:
    samples = vcrparser.get_samples(path.replace(project, ""), project, path + "/samples.list") # PARSES THE SAMPLE FILE AND ASSOCIATED FASTQ FILES
except:
    samples = vcrparser.get_samples(path.replace(project, ""), project, path + "/samples.list", no_check=True)
f = open(path + "/samples.list", 'r')
h = f.readline()
samples_ordered = list()
for i in f:
    if len(i.split('\t')) > 0:
        samples_ordered.append(i.split("\t")[0])
f.close()
lmenu = html.get_menu(config, len(samples))

#########################################################################
# PROCESSES AND ARRANGES LOGS AND OUTPUT DATA FROM STAR, KALLISTO AND HTSEQ
#########################################################################
try:
    spider_stats.stats_varcall(path, "varscan")
except:
    print "Unexpected error generating stats of VARSCAN."
try:
    spider_stats.stats_varcall(path, "gatk")
except:
    print "Unexpected error generating stats of GATK."
try:
    spider_stats.stats_trimgalore(path)
except:
    print "Unexpected error generating stats of TRIMGALORE."
try:
    spider_stats.stats_fusion(path)
except:
    print "Unexpected error generating stats of Star-Fusion."
try:
    spider_stats.stats_log(path + "/logs/") # PARSES THE ASSOCIATED LSF LOG FILES AND WRTIES THE RELEVANT PARAMETERS IN OUTPUTS
except:
    print "Unexpected error generating stats of HPC."
try:
    spider_stats.stats_kallisto(path + "/results_kallisto/", samples_ordered) # GENERATES STATISTICS, ESTIMATED COUNTS, TPM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of KALLISTO not yet ready or unexpected error."
try:
    spider_stats.stats_star(path + "/results_star/", samples_ordered)  # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of STAR not yet ready or unexpected error."
try:
    spider_stats.stats_htseq(path + "/results_htseq-gene/", samples_ordered, "gene") # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of HTSEQ-GENE not yet ready or unexpected error."
try:
    spider_stats.stats_htseq(path + "/results_htseq-exon/", samples_ordered, "exon") # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of HTSEQ-EXON not yet ready or unexpected error."
#########################################################################
# HTML SUMMARY WEBPAGE
#########################################################################
try:
    print "> Generating webpage with samples list and configuration..."
    print "  - " + path + "/HTML/summary.html"
    html_table = html.print_samples(path,config) # PROVIDES HTML TABLE WITH SAMPLES STATS
    html_table2 = html.config_file(path, "config.txt") # PROVIDES HTML TABLE WITH CONFIGURATION SETTINGS
    html.build_from_template("SUMMARY", project, "", html_table, html_table2, path+"/HTML/summary.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_SUMMARY.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTML DOWNLOADS
#########################################################################
try:
    print "> Generating webpage with download links..."
    print "  - " + path + "/HTML/downloads.html"
    html.build_from_template("DOWNLOADS", project, "", "", "", path+"/HTML/downloads.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_DOWNLOAD.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HPC STATS
#########################################################################
try:
    print "> Generating webpage with HPC and LOG statistics..."
    print "  - " + path + "/HTML/hpc.html"
    os.system("Rscript "+pathscript+"/log_stats.R " + path + "/outputs/log_stats.txt") # PLOT OF HPC USAGE
    html_table = html.print_table_default(path + "/outputs/log_stats.txt", 2, []) # PROVIDES HTML TABLE WITH HPC STATS
    fils = os.listdir(path + "/outputs/")
    gg = ""
    for i in fils:
        if i.endswith(".png") and i.startswith("log_stats_"):
            gg = gg + '
Analysis run number '+i.split("_")[2].replace(".png","")+'

'
    html_table2 = "
" + gg + "
"
    html.build_from_template("HPC", project, "", html_table, html_table2, path+"/HTML/hpc.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_HPC.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# TRIM_GALORE
#########################################################################
try:
    if config.has_key("trimgalore"):
        print "> Generating webpage with TrimGalore/Cutadapt statistics..."
        print "  - " + path + "/HTML/trim.html"
        html_table = html.print_table_default(path + "/outputs/stats_trim.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_trim_plot.txt",0,[],[])
        html.build_from_template("TrimGalore", project, data, html_table, "", path+"/HTML/trim.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_TRIMG.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# FASTQ
#########################################################################
try:
    if config.has_key("fastqc"):
        print "> Generating webpage with fastqc statistics..."
        print "  - " + path + "/HTML/fastqc.html"
        html_table = spider_stats.stats_fastq(path,samples,config) # PROVIDES HTML TABLE WITH FASTQ STATS
        html.build_from_template("FASTQC", project, "", html_table, "", path+"/HTML/fastqc.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_FASTQC.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# PICARD
#########################################################################
try:
    if config.has_key("picard"):
        print "> Generating webpage with picard statistics..."
        print "  - " + path + "/HTML/picard.html"
        html_table = html.stats_picard(path,samples,config) # PROVIDES HTML TABLE WITH PICARD STATS
        data = html.bar_getdata (path + "/outputs/stats_picard.txt",0,range(1,7), range(7,11))
        html.build_from_template("PICARD", project, data, html_table, "", path+"/HTML/picard.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PICARD.html", lmenu)
    if config.has_key("picard_IS"):
        print "> Generating webpage with picard insert size statistics..."
        print "  - " + path + "/HTML/picard-is.html"
        x = html.stats_picard_2(path,samples,config)
        html_table = html.print_table_default(path + "/outputs/stats_picard2.txt", -1, [])
        print html_table
        data = html.bar_getdata (path + "/outputs/stats_picard2.txt",0,range(1,2),[])
        print data
        html.build_from_template("PICARD-InsertSize", project, data, html_table, "", path+"/HTML/picard-is.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PICARDIS.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTML STAR-FUSION WEBPAGE
#########################################################################
try:
    if config.has_key("star-fusion"):
        print "> Generating webpage with Star-Fusion results..."
        print "  - " + path + "/HTML/star-fusion.html"
        html_table = html.print_table_default(path + "/outputs/starfusion_aggregate.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        html.build_from_template("STAR-FUSION", project, "", html_table, "", path+"/HTML/star-fusion.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_STARFUSION.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# KALLISTO_QC
#########################################################################
try:
    if config.has_key("kallisto"):
        print "> Generating webpage with Kallisto statistics..."
        print "  - " + path + "/HTML/kallisto.html"
        html_table = html.print_table_default(path + "/outputs/kallisto_stats_est_counts.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/kallisto_stats_est_counts.txt",0,range(1,2),[])
        html.build_from_template("KALLISTO", project, data, html_table, "", path+"/HTML/kallisto.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_KALLISTO.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# STAR_QC
#########################################################################
try:
    if config.has_key("star"):
        print "> Generating webpage with STAR statistics..."
        print "  - " + path + "/HTML/star.html"
        html_table = html.print_table_default(path + "/outputs/star_unstranded_stats.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/star_unstranded_stats.txt",0,range(1,6),[])
        html.build_from_template("STAR", project, data, html_table, "", path+"/HTML/star.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_STAR.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTSEQ_QC
#########################################################################
try:
    for ij in ["htseq-gene", "htseq-exon"]:
        if config.has_key(ij):
            print "> Generating webpage with "+ij+" statistics..."
            print "  - " + path + "/HTML/"+ij+".html"
            html_table = html.print_table_default(path + "/outputs/"+ij+"_stats.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
            data = html.bar_getdata (path + "/outputs/"+ij+"_stats.txt",0,range(1,7),[])
            html.build_from_template(ij.upper(), project, data, html_table, "", path+"/HTML/" + ij + ".html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_HTSEQ.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# STAR & HTSEQ STATS ON COUNTS/RPKMS
#########################################################################
try:
    if len(samples) > 1:
        if config["programs"]["strandedness"] == "yes":
            n = {"star":["STAR","star_stranded"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        elif config["programs"]["strandedness"] == "no":
            n = {"star":["STAR","star_unstranded"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        elif config["programs"]["strandedness"] == "reverse":
            n = {"star":["STAR","star_stranded-reverse"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        for prog, pname in n.iteritems():
            if config.has_key(prog):
                os.system("Rscript "+pathscript+"/stats_algs.R " + path + "/outputs/ " + pname[1]) # PLOT OF HPC USAGE
                html_table = html.print_table_default(path + "/outputs/" + pname[1] + "_pca.txt", -1, [0, 1, 2, 3, 4, 6, 7, 8, 12, 13, 14, 15, 17, 18, 19])
                if prog != 'kallisto':
                    html.build_amcharts(os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PCA.html", path + "/HTML/" + prog + "2.html", prog, pname, path, html_table, project, lmenu)
                else:
                    html.build_amcharts(os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PCA2.html", path + "/HTML/" + prog + "2.html", prog, pname, path, html_table, project, lmenu)
except:
    print "  - Not ready"

#########################################################################
# VARSCAN
#########################################################################
try:
    if config.has_key("varscan"):
        print "> Generating webpage with VARSCAN statistics..."
        print "  - " + path + "/HTML/varscan.html"
        html_table = html.print_table_default(path + "/outputs/stats_varscan.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_varscan.txt",0,[],[])
        html.build_from_template("VARSCAN", project, data, html_table, "", path+"/HTML/varscan.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_GATK.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# GATK
#########################################################################
try:
    if config.has_key("gatk"):
        print "> Generating webpage with GATK statistics..."
        print "  - " + path + "/HTML/gatk.html"
        html_table = html.print_table_default(path + "/outputs/stats_gatk.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_gatk.txt",0,[],[])
        html.build_from_template("GATK", project, data, html_table, "", path+"/HTML/gatk.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_GATK.html", lmenu)
except:
    print "  - Not ready"
<commit_msg>test spider kallisto<commit_after>import os
import sys
import optparse
import lib.vcrparser as vcrparser
import lib.spider_stats as spider_stats
import lib.html_lib as html

#########################################################################
# PARSER
#########################################################################
desc = "aRNApipe: SPIDER module"
parser = optparse.OptionParser(description = desc)
parser.add_option("-p", "--path", dest = "path", default = "", help = "Required: Path to the project folder")
(opt, args) = parser.parse_args()

#########################################################################
# INITIAL ARRANGEMENTS
#########################################################################
pathscript = os.path.dirname(sys.argv[0]) + "/R/" # PATH TO R SCRIPTS
project, path = html.check_project(opt.path) # CHECKS IF PROJECT FOLDER, SAMPLES FILE AND CONFIGURATION FILE EXIST
html.skeleton(path, os.path.dirname(sys.argv[0])) # CREATES THE SKELETON FOR THE HTML AND OUTPUT RESULTS
config = html.check_config(path) # PARSES THE CONFIGURATION FILE
try:
    samples = vcrparser.get_samples(path.replace(project, ""), project, path + "/samples.list") # PARSES THE SAMPLE FILE AND ASSOCIATED FASTQ FILES
except:
    samples = vcrparser.get_samples(path.replace(project, ""), project, path + "/samples.list", no_check=True)
f = open(path + "/samples.list", 'r')
h = f.readline()
samples_ordered = list()
for i in f:
    if len(i.split('\t')) > 0:
        samples_ordered.append(i.split("\t")[0])
f.close()
lmenu = html.get_menu(config, len(samples))

#########################################################################
# PROCESSES AND ARRANGES LOGS AND OUTPUT DATA FROM STAR, KALLISTO AND HTSEQ
#########################################################################
try:
    spider_stats.stats_varcall(path, "varscan")
except:
    print "Unexpected error generating stats of VARSCAN."
try:
    spider_stats.stats_varcall(path, "gatk")
except:
    print "Unexpected error generating stats of GATK."
try:
    spider_stats.stats_trimgalore(path)
except:
    print "Unexpected error generating stats of TRIMGALORE."
try:
    spider_stats.stats_fusion(path)
except:
    print "Unexpected error generating stats of Star-Fusion."
try:
    spider_stats.stats_log(path + "/logs/") # PARSES THE ASSOCIATED LSF LOG FILES AND WRTIES THE RELEVANT PARAMETERS IN OUTPUTS
except:
    print "Unexpected error generating stats of HPC."
try:
    spider_stats.stats_kallisto(path + "/results_kallisto/", samples_ordered) # GENERATES STATISTICS, ESTIMATED COUNTS, TPM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of KALLISTO not yet ready or unexpected error."
try:
    spider_stats.stats_star(path + "/results_star/", samples_ordered)  # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of STAR not yet ready or unexpected error."
try:
    spider_stats.stats_htseq(path + "/results_htseq-gene/", samples_ordered, "gene") # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of HTSEQ-GENE not yet ready or unexpected error."
try:
    spider_stats.stats_htseq(path + "/results_htseq-exon/", samples_ordered, "exon") # GENERATES STATISTICS, COUNTS, RPKM AND ANNOTATION MATRICES
except:
    print "Error: Statistics of HTSEQ-EXON not yet ready or unexpected error."
#########################################################################
# HTML SUMMARY WEBPAGE
#########################################################################
try:
    print "> Generating webpage with samples list and configuration..."
    print "  - " + path + "/HTML/summary.html"
    html_table = html.print_samples(path,config) # PROVIDES HTML TABLE WITH SAMPLES STATS
    html_table2 = html.config_file(path, "config.txt") # PROVIDES HTML TABLE WITH CONFIGURATION SETTINGS
    html.build_from_template("SUMMARY", project, "", html_table, html_table2, path+"/HTML/summary.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_SUMMARY.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTML DOWNLOADS
#########################################################################
try:
    print "> Generating webpage with download links..."
    print "  - " + path + "/HTML/downloads.html"
    html.build_from_template("DOWNLOADS", project, "", "", "", path+"/HTML/downloads.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_DOWNLOAD.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HPC STATS
#########################################################################
try:
    print "> Generating webpage with HPC and LOG statistics..."
    print "  - " + path + "/HTML/hpc.html"
    os.system("Rscript "+pathscript+"/log_stats.R " + path + "/outputs/log_stats.txt") # PLOT OF HPC USAGE
    html_table = html.print_table_default(path + "/outputs/log_stats.txt", 2, []) # PROVIDES HTML TABLE WITH HPC STATS
    fils = os.listdir(path + "/outputs/")
    gg = ""
    for i in fils:
        if i.endswith(".png") and i.startswith("log_stats_"):
            gg = gg + '
Analysis run number '+i.split("_")[2].replace(".png","")+'

'
    html_table2 = "
" + gg + "
"
    html.build_from_template("HPC", project, "", html_table, html_table2, path+"/HTML/hpc.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_HPC.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# TRIM_GALORE
#########################################################################
try:
    if config.has_key("trimgalore"):
        print "> Generating webpage with TrimGalore/Cutadapt statistics..."
        print "  - " + path + "/HTML/trim.html"
        html_table = html.print_table_default(path + "/outputs/stats_trim.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_trim_plot.txt",0,[],[])
        html.build_from_template("TrimGalore", project, data, html_table, "", path+"/HTML/trim.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_TRIMG.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# FASTQ
#########################################################################
try:
    if config.has_key("fastqc"):
        print "> Generating webpage with fastqc statistics..."
        print "  - " + path + "/HTML/fastqc.html"
        html_table = spider_stats.stats_fastq(path,samples,config) # PROVIDES HTML TABLE WITH FASTQ STATS
        html.build_from_template("FASTQC", project, "", html_table, "", path+"/HTML/fastqc.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_FASTQC.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# PICARD
#########################################################################
try:
    if config.has_key("picard"):
        print "> Generating webpage with picard statistics..."
        print "  - " + path + "/HTML/picard.html"
        html_table = html.stats_picard(path,samples,config) # PROVIDES HTML TABLE WITH PICARD STATS
        data = html.bar_getdata (path + "/outputs/stats_picard.txt",0,range(1,7), range(7,11))
        html.build_from_template("PICARD", project, data, html_table, "", path+"/HTML/picard.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PICARD.html", lmenu)
    if config.has_key("picard_IS"):
        print "> Generating webpage with picard insert size statistics..."
        print "  - " + path + "/HTML/picard-is.html"
        x = html.stats_picard_2(path,samples,config)
        html_table = html.print_table_default(path + "/outputs/stats_picard2.txt", -1, [])
        print html_table
        data = html.bar_getdata (path + "/outputs/stats_picard2.txt",0,range(1,2),[])
        print data
        html.build_from_template("PICARD-InsertSize", project, data, html_table, "", path+"/HTML/picard-is.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PICARDIS.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTML STAR-FUSION WEBPAGE
#########################################################################
try:
    if config.has_key("star-fusion"):
        print "> Generating webpage with Star-Fusion results..."
        print "  - " + path + "/HTML/star-fusion.html"
        html_table = html.print_table_default(path + "/outputs/starfusion_aggregate.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        html.build_from_template("STAR-FUSION", project, "", html_table, "", path+"/HTML/star-fusion.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_STARFUSION.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# KALLISTO_QC
#########################################################################
try:
    if config.has_key("kallisto"):
        print "> Generating webpage with Kallisto statistics..."
        print "  - " + path + "/HTML/kallisto.html"
        html_table = html.print_table_default(path + "/outputs/kallisto_stats_est_counts.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        print 1
        data = html.bar_getdata (path + "/outputs/kallisto_stats_est_counts.txt",0,range(1,2),[])
        print 2
        html.build_from_template("KALLISTO", project, data, html_table, "", path+"/HTML/kallisto.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_KALLISTO.html", lmenu)
        print 3
except:
    print "  - Not ready"

#########################################################################
# STAR_QC
#########################################################################
try:
    if config.has_key("star"):
        print "> Generating webpage with STAR statistics..."
        print "  - " + path + "/HTML/star.html"
        html_table = html.print_table_default(path + "/outputs/star_unstranded_stats.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/star_unstranded_stats.txt",0,range(1,6),[])
        html.build_from_template("STAR", project, data, html_table, "", path+"/HTML/star.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_STAR.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# HTSEQ_QC
#########################################################################
try:
    for ij in ["htseq-gene", "htseq-exon"]:
        if config.has_key(ij):
            print "> Generating webpage with "+ij+" statistics..."
            print "  - " + path + "/HTML/"+ij+".html"
            html_table = html.print_table_default(path + "/outputs/"+ij+"_stats.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
            data = html.bar_getdata (path + "/outputs/"+ij+"_stats.txt",0,range(1,7),[])
            html.build_from_template(ij.upper(), project, data, html_table, "", path+"/HTML/" + ij + ".html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_HTSEQ.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# STAR & HTSEQ STATS ON COUNTS/RPKMS
#########################################################################
try:
    if len(samples) > 1:
        if config["programs"]["strandedness"] == "yes":
            n = {"star":["STAR","star_stranded"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        elif config["programs"]["strandedness"] == "no":
            n = {"star":["STAR","star_unstranded"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        elif config["programs"]["strandedness"] == "reverse":
            n = {"star":["STAR","star_stranded-reverse"],"htseq-gene":["HTseq-count Gene", "htseq-gene"],"htseq-exon":["HTseq-count Exon", "htseq-exon"], 'kallisto': ['Kallisto', 'kallisto']}
        for prog, pname in n.iteritems():
            if config.has_key(prog):
                os.system("Rscript "+pathscript+"/stats_algs.R " + path + "/outputs/ " + pname[1]) # PLOT OF HPC USAGE
                html_table = html.print_table_default(path + "/outputs/" + pname[1] + "_pca.txt", -1, [0, 1, 2, 3, 4, 6, 7, 8, 12, 13, 14, 15, 17, 18, 19])
                if prog != 'kallisto':
                    html.build_amcharts(os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PCA.html", path + "/HTML/" + prog + "2.html", prog, pname, path, html_table, project, lmenu)
                else:
                    html.build_amcharts(os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_PCA2.html", path + "/HTML/" + prog + "2.html", prog, pname, path, html_table, project, lmenu)
except:
    print "  - Not ready"

#########################################################################
# VARSCAN
#########################################################################
try:
    if config.has_key("varscan"):
        print "> Generating webpage with VARSCAN statistics..."
        print "  - " + path + "/HTML/varscan.html"
        html_table = html.print_table_default(path + "/outputs/stats_varscan.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_varscan.txt",0,[],[])
        html.build_from_template("VARSCAN", project, data, html_table, "", path+"/HTML/varscan.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_GATK.html", lmenu)
except:
    print "  - Not ready"

#########################################################################
# GATK
#########################################################################
try:
    if config.has_key("gatk"):
        print "> Generating webpage with GATK statistics..."
        print "  - " + path + "/HTML/gatk.html"
        html_table = html.print_table_default(path + "/outputs/stats_gatk.txt", -1, []) # PROVIDES HTML TABLE WITH HPC STATS
        data = html.bar_getdata (path + "/outputs/stats_gatk.txt",0,[],[])
        html.build_from_template("GATK", project, data, html_table, "", path+"/HTML/gatk.html", os.path.dirname(sys.argv[0]) + "/template/TEMPLATE_GATK.html", lmenu)
except:
    print "  - Not ready"
<|endoftext|>


<commit_before>#!/usr/bin/env python
## category General
## desc Find the nearest annotated BED region for each read
## experimental
'''
For each read, find the nearest annotated region from a BED file.

For example, if you have a BED file with the transcription starting site (TSS)
for all genes in the genome, this will find the nearest TSS for each read, and
report the distance to the site.

'''

import sys
import os
import ngsutils.bam
from ngsutils.bed import BedFile

def find_nearest(bam, bed, maxdist=100000, out=sys.stdout):
    for read in ngsutils.bam.bam_iter(bam):
        if read.is_unmapped:
            continue

        dists = []  # will be an list tuples: (abs_val of the distance, 'up/down') (respective to + strand)

        chrom = bam.getrname(read.tid)
        strand = '-' if read.is_reverse else '+'
        start = max(0, read.pos - maxdist)
        end = min(read.aend + maxdist, bam.lengths[read.tid])

        for region in bed.fetch(chrom, start, end, strand):
            sys.stderr.write("  - checking region: %s = " % region)
            if region.start <= read.pos <= region.end:
                # start is w/in region
                dists.append((0, ''))
                continue
            elif region.start <= read.aend <= region.end:
                # end is w/in region
                dists.append((0, ''))
                continue
            elif read.pos <= region.start <= region.end <= read.aend:
                # read spans the entire region
                dists.append((0,''))
                continue
            elif region.end < read.pos:
                dists.append((read.pos - region.end, 'up'))
            else:
                dists.append((region.start - read.aend, 'down'))

        if dists:
            dists.sort()

            distance = dists[0][0]
            orient = dists[0][1]

            if distance > 0:
                if not read.is_reverse and orient == 'down':
                    distance = -distance
                elif read.is_reverse and orient == 'up':
                    distance = -distance

            out.write('%s\t%s\n' % (read.qname, distance))
        else:
            out.write('%s\t*\n' % (read.qname,))

def usage(msg=None):
    if msg:
        sys.stdout.write('%s\n\n' % msg)
    sys.stdout.write(__doc__)
    sys.stdout.write('''\
Usage: bamutils nearest {-max val} filename.bam regions.bed

Options:
  -max    The maximal distance to look for a nearest region
          (default: 100K)

''')

    sys.exit(1)

if __name__ == '__main__':
    bam_fname = None
    bed_fname = None
    maxdist = 100000

    last = None

    for arg in sys.argv[1:]:
        if last == '-max':
            maxdist = int(arg)
            last = None
        elif arg in ['-max']:
            last = arg
        elif not bam_fname and os.path.exists(arg):
            bam_fname = arg
        elif not bed_fname and os.path.exists(arg):
            bed_fname = arg

    if not bam_fname or not bed_fname:
        usage()

    bed = BedFile(bed_fname)
    bam = ngsutils.bam.bam_open(bam_fname)

    find_nearest(bam, bed, maxdist)
<commit_msg>logging<commit_after>#!/usr/bin/env python
## category General
## desc Find the nearest annotated BED region for each read
## experimental
'''
For each read, find the nearest annotated region from a BED file.

For example, if you have a BED file with the transcription starting site (TSS)
for all genes in the genome, this will find the nearest TSS for each read, and
report the distance to the site.

'''

import sys
import os
import ngsutils.bam
from ngsutils.bed import BedFile

def find_nearest(bam, bed, maxdist=100000, out=sys.stdout):
    for read in ngsutils.bam.bam_iter(bam):
        if read.is_unmapped:
            continue

        dists = []  # will be an list tuples: (abs_val of the distance, 'up/down') (respective to + strand)

        chrom = bam.getrname(read.tid)
        strand = '-' if read.is_reverse else '+'
        start = max(0, read.pos - maxdist)
        end = min(read.aend + maxdist, bam.lengths[read.tid])

        for region in bed.fetch(chrom, start, end, strand):
            if region.start <= read.pos <= region.end:
                # start is w/in region
                dists.append((0, ''))
                continue
            elif region.start <= read.aend <= region.end:
                # end is w/in region
                dists.append((0, ''))
                continue
            elif read.pos <= region.start <= region.end <= read.aend:
                # read spans the entire region
                dists.append((0,''))
                continue
            elif region.end < read.pos:
                dists.append((read.pos - region.end, 'up'))
            else:
                dists.append((region.start - read.aend, 'down'))

        if dists:
            dists.sort()

            distance = dists[0][0]
            orient = dists[0][1]

            if distance > 0:
                if not read.is_reverse and orient == 'down':
                    distance = -distance
                elif read.is_reverse and orient == 'up':
                    distance = -distance

            out.write('%s\t%s\n' % (read.qname, distance))
        else:
            out.write('%s\t*\n' % (read.qname,))

def usage(msg=None):
    if msg:
        sys.stdout.write('%s\n\n' % msg)
    sys.stdout.write(__doc__)
    sys.stdout.write('''\
Usage: bamutils nearest {-max val} filename.bam regions.bed

Options:
  -max    The maximal distance to look for a nearest region
          (default: 100K)

''')

    sys.exit(1)

if __name__ == '__main__':
    bam_fname = None
    bed_fname = None
    maxdist = 100000

    last = None

    for arg in sys.argv[1:]:
        if last == '-max':
            maxdist = int(arg)
            last = None
        elif arg in ['-max']:
            last = arg
        elif not bam_fname and os.path.exists(arg):
            bam_fname = arg
        elif not bed_fname and os.path.exists(arg):
            bed_fname = arg

    if not bam_fname or not bed_fname:
        usage()

    bed = BedFile(bed_fname)
    bam = ngsutils.bam.bam_open(bam_fname)

    find_nearest(bam, bed, maxdist)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from netmiko.ssh_connection import SSHConnection
from netmiko.netmiko_globals import MAX_BUFFER
import time

class CiscoAsaSSH(SSHConnection):

    def session_preparation(self):
        '''
        Prepare the session after the connection has been established

        ASA must go into enable mode to disable_paging
        '''

        self.enable()
        self.disable_paging(command="terminal pager 0\n")
        self.set_base_prompt()

    def enable(self):
        '''
        Enter enable mode

        Must manually control the channel at this point for ASA
        '''

        delay_factor = .5

        self.clear_buffer()
        self.remote_conn.send("\nenable\n")
        time.sleep(1*delay_factor)

        output = self.remote_conn.recv(MAX_BUFFER)
        if 'password' in output.lower():
            self.remote_conn.send(self.secret+'\n')
            self.remote_conn.send('\n')
            time.sleep(1*delay_factor)
            output += self.remote_conn.recv(MAX_BUFFER)

        self.set_base_prompt()
        self.clear_buffer()
<commit_msg>Fixing merge conflicts<commit_after>from __future__ import unicode_literals
from netmiko.ssh_connection import SSHConnection
from netmiko.netmiko_globals import MAX_BUFFER
import time
import re

class CiscoAsaSSH(SSHConnection):

    def session_preparation(self):
        '''
        Prepare the session after the connection has been established

        ASA must go into enable mode to disable_paging
        '''

        self.enable()
        self.disable_paging(command="terminal pager 0\n")
        self.set_base_prompt()

    def send_command(self, command_string, delay_factor=.5, max_loops=30,
                 strip_prompt=True, strip_command=True):
        '''
        If ASA is in multi-context mode, then the base_prompt needs to be
        updated after each change of context to make strip_prompt behave
        properly.
        '''

        output=super(CiscoAsaSSH,self).send_command(command_string,delay_factor,
                max_loops,strip_prompt,strip_command)
        if "changeto" in command_string:
            self.set_base_prompt()
        return output

    def set_base_prompt(self):
        '''
        Cisco ASA in multi-context mode needs to have the base prompt updated
        (if you switch contexts i.e. 'changeto')

        This switch of ASA contexts can occur in configuration mode. If this
        happens the trailing '(config*' needs stripped off.
        '''
        cur_base_prompt = super(CiscoAsaSSH, self).set_base_prompt()
        match = re.search(r'(.*)\(conf.*', a)
        if match:
            # strip off (conf.* from base_prompt
            self.base_prompt = match.group(1)
            return self.base_prompt

    def enable(self):
        '''
        Enter enable mode

        Must manually control the channel at this point for ASA
        '''

        delay_factor = .5

        self.clear_buffer()
        self.remote_conn.send("\nenable\n")
        time.sleep(1*delay_factor)

        output = self.remote_conn.recv(MAX_BUFFER)
        if 'password' in output.lower():
            self.remote_conn.send(self.secret+'\n')
            self.remote_conn.send('\n')
            time.sleep(1*delay_factor)
            output += self.remote_conn.recv(MAX_BUFFER)

        self.set_base_prompt()
        self.clear_buffer()
<|endoftext|>