<commit_before>#!/usr/bin/env python
import datetime
import optparse
import os
import re
import socket
import struct
import subprocess
import sys
import threading
import time
import urllib
import wsgiref.simple_server

BG_RESCAN_INTERVAL_SECS = 60 * 60 * 30  # Rescan network every 30 mins.

CONTENT = """
<!doctype html>

  <title>Wake My Subnet</title>

  <form method="POST">

      Wake My Subnet

    MAC or cached hostname:
    <input type="text" name="t" list="hosts">
    <datalist id="hosts">
      <select>$1</select>
    </datalist>
    <input type="submit" value="WOL">
    </form>

Network scan status: $2

How to setup your machine for WOL
 1. Enable wake on lan in the BIOS.
 2. Add "ethtool -s eth0 wol g" to your rc.local

"""

class WMS(object):
  def __init__(self):
    parser = optparse.OptionParser()
    parser.add_option('-d', '--daemon', help='Daemonize', action='store_true')
    parser.add_option('-p', '--port', help='HTTP Port', default=8965,
                      type='int')
    parser.add_option('-i', '--interface', help='Interface IP addr',
                      default=socket.gethostbyname(socket.gethostname()))
    (options, _) = parser.parse_args()

    print 'Using interface', options.interface
    self._subnet = options.interface.split('.')[0:3]
    self._bacast_addr = '.'.join(self._subnet + ['255'])
    self._known_hosts = {}
    self._scan_status = 'Not started'

    print 'Serving on http://localhost:%d/' % options.port

    if options.daemon:
      print 'Forking into daemon land.'
      Daemonize()

    self._rescan_thread = threading.Thread(target=self._RescanThread)
    self._rescan_thread.daemon = True
    self._rescan_thread.start()

    httpd = wsgiref.simple_server.make_server(
        '', options.port, self._HttpHandler)
    httpd.serve_forever()

  def _HttpHandler(self, environ, start_response):
    # path = environ['PATH_INFO']
    method = environ['REQUEST_METHOD']
    start_response('200 OK', [('Content-Type', 'text/html'),
                              ('Cache-Control', 'no-cache'),
                              ('Expires', 'Fri, 19 Sep 1986 05:00:00 GMT')])
    if method == 'POST':
      req_body_size = int(environ.get('CONTENT_LENGTH', 0))
      req_body = environ['wsgi.input'].read(req_body_size)
      if req_body.startswith('t='):
        target = urllib.unquote(req_body[2:])
        mac = self._known_hosts.get(target, target)
        try:
          SendWOLPacket(mac, self._bacast_addr)
          return ['Sending WOL packet to %s' % mac]
        except:
          return ['Error while waking "%s": %s' % (mac, sys.exc_info())]
          raise

    else:
      hosts_datalist = ''
      for host, mac in self._known_hosts.iteritems():
        hosts_datalist += '<option value="%s">%s</option>' % (host, mac)
      html = str(CONTENT.replace('$1', hosts_datalist))
      html = str(html.replace('$2', str(self._scan_status)))
      return [html]

  def _RescanNetwork(self):
    print 'Scanning subnet'
    subnet_addrs = ['.'.join(self._subnet + [str(i)]) for i in xrange(1, 255)]
    pinged_addrs_count = 0
    for target_ip in subnet_addrs:
      host_or_ip, mac_addr = Lookup(target_ip)
      pinged_addrs_count += 1
      ping_stat = 'Ping: %d/%d' % (pinged_addrs_count, len(subnet_addrs))
      self._scan_status = ping_stat
      print ping_stat, '  \r',
      if mac_addr:
        self._known_hosts[host_or_ip] = mac_addr
    print '\rDiscovered %d hosts' % len(self._known_hosts)
    self._scan_status = datetime.datetime.now()

  def _RescanThread(self):
    while(True):
      self._RescanNetwork()
      time.sleep(BG_RESCAN_INTERVAL_SECS)

def Lookup(target_ip):
  p = subprocess.Popen(['arping', '-f', '-w1', target_ip],
                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  time.sleep(1)
  if p.poll() is None:
    p.kill()
    os.wait()
    return (target_ip, None)
  output, _ = p.communicate()
  match = re.search('reply.*\[([\w:]+)\]', output)
  if not match:
    return (target_ip, None)
  mac = match.group(1)
  try:
    socket.setdefaulttimeout(1)
    host_info = socket.gethostbyaddr(target_ip)
  except socket.herror:
    host_info = None
  host_or_ip = host_info[0] if host_info else target_ip
  return (host_or_ip, mac)

def SendWOLPacket(mac, bcast_addr='<broadcast>'):
  mac = re.sub('[^\w]', '', mac)
  assert(len(mac) == 12), 'Expecting 6-octects hex MAC addr'
  magic = 'FFFFFFFFFFFF' + (mac * 20)
  payload = ''
  for i in range(0, len(magic), 2):
    payload = ''.join(
        [payload, struct.pack('B', int(magic[i: i + 2], 16))])
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
  sock.sendto(payload, (bcast_addr, 7))

def Daemonize():
  pid = os.fork()
  if pid > 0:
    sys.exit(0)  # exit first parent

  os.setsid()
  os.umask(0)

  pid = os.fork()
  if pid > 0:
    sys.exit(0)  # exit second parent

  sys.stdout.flush()
  sys.stderr.flush()
  null_in = file('/dev/null', 'r')
  null_out = file('/dev/null', 'a+')
  os.dup2(null_in.fileno(), sys.stdin.fileno())
  os.dup2(null_out.fileno(), sys.stdout.fileno())
  os.dup2(null_out.fileno(), sys.stderr.fileno())

if __name__ == '__main__':
  WMS()
<commit_msg>Fix rescan interval<commit_after>#!/usr/bin/env python
import datetime
import optparse
import os
import re
import socket
import struct
import subprocess
import sys
import threading
import time
import urllib
import wsgiref.simple_server

BG_RESCAN_INTERVAL_SECS = 60 * 30  # Rescan network every 30 mins.

CONTENT = """
<!doctype html>

  <title>Wake My Subnet</title>

  <form method="POST">

      Wake My Subnet

    MAC or cached hostname:
    <input type="text" name="t" list="hosts">
    <datalist id="hosts">
      <select>$1</select>
    </datalist>
    <input type="submit" value="WOL">
    </form>

Network scan status: $2

How to setup your machine for WOL
 1. Enable wake on lan in the BIOS.
 2. Add "ethtool -s eth0 wol g" to your rc.local

"""

class WMS(object):
  def __init__(self):
    parser = optparse.OptionParser()
    parser.add_option('-d', '--daemon', help='Daemonize', action='store_true')
    parser.add_option('-p', '--port', help='HTTP Port', default=8965,
                      type='int')
    parser.add_option('-i', '--interface', help='Interface IP addr',
                      default=socket.gethostbyname(socket.gethostname()))
    (options, _) = parser.parse_args()

    print 'Using interface', options.interface
    self._subnet = options.interface.split('.')[0:3]
    self._bacast_addr = '.'.join(self._subnet + ['255'])
    self._known_hosts = {}
    self._scan_status = 'Not started'

    print 'Serving on http://localhost:%d/' % options.port

    if options.daemon:
      print 'Forking into daemon land.'
      Daemonize()

    self._rescan_thread = threading.Thread(target=self._RescanThread)
    self._rescan_thread.daemon = True
    self._rescan_thread.start()

    httpd = wsgiref.simple_server.make_server(
        '', options.port, self._HttpHandler)
    httpd.serve_forever()

  def _HttpHandler(self, environ, start_response):
    # path = environ['PATH_INFO']
    method = environ['REQUEST_METHOD']
    start_response('200 OK', [('Content-Type', 'text/html'),
                              ('Cache-Control', 'no-cache'),
                              ('Expires', 'Fri, 19 Sep 1986 05:00:00 GMT')])
    if method == 'POST':
      req_body_size = int(environ.get('CONTENT_LENGTH', 0))
      req_body = environ['wsgi.input'].read(req_body_size)
      if req_body.startswith('t='):
        target = urllib.unquote(req_body[2:])
        mac = self._known_hosts.get(target, target)
        try:
          SendWOLPacket(mac, self._bacast_addr)
          return ['Sending WOL packet to %s' % mac]
        except:
          return ['Error while waking "%s": %s' % (mac, sys.exc_info())]
          raise

    else:
      hosts_datalist = ''
      for host, mac in self._known_hosts.iteritems():
        hosts_datalist += '<option value="%s">%s</option>' % (host, mac)
      html = str(CONTENT.replace('$1', hosts_datalist))
      html = str(html.replace('$2', str(self._scan_status)))
      return [html]

  def _RescanNetwork(self):
    print 'Scanning subnet'
    subnet_addrs = ['.'.join(self._subnet + [str(i)]) for i in xrange(1, 255)]
    pinged_addrs_count = 0
    for target_ip in subnet_addrs:
      host_or_ip, mac_addr = Lookup(target_ip)
      pinged_addrs_count += 1
      ping_stat = 'Ping: %d/%d' % (pinged_addrs_count, len(subnet_addrs))
      self._scan_status = ping_stat
      print ping_stat, '  \r',
      if mac_addr:
        self._known_hosts[host_or_ip] = mac_addr
    print '\rDiscovered %d hosts' % len(self._known_hosts)
    self._scan_status = datetime.datetime.now()

  def _RescanThread(self):
    while(True):
      self._RescanNetwork()
      time.sleep(BG_RESCAN_INTERVAL_SECS)

def Lookup(target_ip):
  p = subprocess.Popen(['arping', '-f', '-w1', target_ip],
                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  time.sleep(1)
  if p.poll() is None:
    p.kill()
    os.wait()
    return (target_ip, None)
  output, _ = p.communicate()
  match = re.search('reply.*\[([\w:]+)\]', output)
  if not match:
    return (target_ip, None)
  mac = match.group(1)
  try:
    socket.setdefaulttimeout(1)
    host_info = socket.gethostbyaddr(target_ip)
  except socket.herror:
    host_info = None
  host_or_ip = host_info[0] if host_info else target_ip
  return (host_or_ip, mac)

def SendWOLPacket(mac, bcast_addr='<broadcast>'):
  mac = re.sub('[^\w]', '', mac)
  assert(len(mac) == 12), 'Expecting 6-octects hex MAC addr'
  magic = 'FFFFFFFFFFFF' + (mac * 20)
  payload = ''
  for i in range(0, len(magic), 2):
    payload = ''.join(
        [payload, struct.pack('B', int(magic[i: i + 2], 16))])
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
  sock.sendto(payload, (bcast_addr, 7))

def Daemonize():
  pid = os.fork()
  if pid > 0:
    sys.exit(0)  # exit first parent

  os.setsid()
  os.umask(0)

  pid = os.fork()
  if pid > 0:
    sys.exit(0)  # exit second parent

  sys.stdout.flush()
  sys.stderr.flush()
  null_in = file('/dev/null', 'r')
  null_out = file('/dev/null', 'a+')
  os.dup2(null_in.fileno(), sys.stdin.fileno())
  os.dup2(null_out.fileno(), sys.stdout.fileno())
  os.dup2(null_out.fileno(), sys.stderr.fileno())

if __name__ == '__main__':
  WMS()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import os
import stat

import getpass

import sha512_wrapper

def create_pass_hash(filename):

    if os.path.exists(hash_fn_full):
        print("[%s] already exists. Aborting." % hash_fn_full)
        return False

    # ask passphrase
    passphrase = getpass.getpass("Type in...\n")

    # generate hash
    v, r = sha512_wrapper.hash_sha_512_app_content(passphrase)
    if not v:
        return False

    # write out to file
    with open(filename, "w") as f:
        f.write(r)

    # set user-read-only permissions
    os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

    return True

if __name__ == "__main__":

    filename_base = os.path.expanduser("~/")
    hash_filename = ".private_passphrase_hash"

    if len(sys.argv) == 2:
        filename_base = os.getcwd()
        hash_filename = sys.argv[1]

    hash_fn_full = os.path.join(filename_base, hash_filename)
    if not create_pass_hash(hash_fn_full):
        print("Failed creating pass hash file")
        sys.exit(1)
    else:
        print("Successfuly created hash file [%s]." % hash_fn_full)
<commit_msg>replacing os.path.join with concat_path, create_pass_hash<commit_after>#!/usr/bin/env python3

import sys
import os
import stat

import getpass

import sha512_wrapper
import path_utils

def create_pass_hash(filename):

    if os.path.exists(hash_fn_full):
        print("[%s] already exists. Aborting." % hash_fn_full)
        return False

    # ask passphrase
    passphrase = getpass.getpass("Type in...\n")

    # generate hash
    v, r = sha512_wrapper.hash_sha_512_app_content(passphrase)
    if not v:
        return False

    # write out to file
    with open(filename, "w") as f:
        f.write(r)

    # set user-read-only permissions
    os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

    return True

if __name__ == "__main__":

    filename_base = os.path.expanduser("~/")
    hash_filename = ".private_passphrase_hash"

    if len(sys.argv) == 2:
        filename_base = os.getcwd()
        hash_filename = sys.argv[1]

    hash_fn_full = path_utils.concat_path(filename_base, hash_filename)
    if not create_pass_hash(hash_fn_full):
        print("Failed creating pass hash file")
        sys.exit(1)
    else:
        print("Successfuly created hash file [%s]." % hash_fn_full)
<|endoftext|>


<commit_before># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import yaml

from oslo.serialization import jsonutils
from oslo.utils import importutils

from senlinclient.openstack.common import cliutils

# Using common methods from oslo cliutils
# Will change when the module graduates
arg = cliutils.arg
env = cliutils.env
print_list = cliutils.print_list
exit = cliutils.exit

supported_formats = {
    "json": lambda x: jsonutils.dumps(x, indent=2),
    "yaml": yaml.safe_dump
}

def import_versioned_module(version, submodule=None):
    module = 'senlinclient.v%s' % version
    if submodule:
        module = '.'.join((module, submodule))
    return importutils.import_module(module)
<commit_msg>Fix oslo namespace errors<commit_after># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import yaml

from oslo_serialization import jsonutils
from oslo_utils import importutils

from senlinclient.openstack.common import cliutils

# Using common methods from oslo cliutils
# Will change when the module graduates
arg = cliutils.arg
env = cliutils.env
print_list = cliutils.print_list
exit = cliutils.exit

supported_formats = {
    "json": lambda x: jsonutils.dumps(x, indent=2),
    "yaml": yaml.safe_dump
}

def import_versioned_module(version, submodule=None):
    module = 'senlinclient.v%s' % version
    if submodule:
        module = '.'.join((module, submodule))
    return importutils.import_module(module)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""waltz: make web apps in 3/4 time (http://github.com/mekarpeles/waltz)"""

__version__ = "0.1.62"
__author__ = [
    "Mek <michael.karpeles@gmail.com>"
]
__license__ = "public domain"
__contributors__ = "see AUTHORS"

import web

session = lambda: getattr(web.ctx, 'session', None)
render = lambda: getattr(web.ctx, 'render', None)

from security import User
from decorations import *
from utils import *
from treasury import *
from setup import *
<commit_msg>Adding db handler reference (lambda) to web.ctx['waltz']<commit_after>#!/usr/bin/env python

"""waltz: make web apps in 3/4 time (http://github.com/mekarpeles/waltz)"""

__version__ = "0.1.62"
__author__ = [
    "Mek <michael.karpeles@gmail.com>"
]
__license__ = "public domain"
__contributors__ = "see AUTHORS"

import web

session = lambda: getattr(web.ctx, 'session', None)
render = lambda: getattr(web.ctx, 'render', None)
db = lambda: Db(web.ctx['waltz']['db'])

from security import User
from decorations import *
from utils import *
from treasury import *
from setup import *
<|endoftext|>


<commit_before>import logging
import os
import os.path as path
import json

from django.conf import settings
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.http import HttpResponseRedirect, HttpResponse,\
    HttpResponseForbidden, HttpResponseNotFound
from django.utils.text import slugify

from tardis.tardis_portal.auth import decorators as authz
from tardis.tardis_portal.models import Experiment, ExperimentParameter, \
    DatafileParameter, DatasetParameter, ObjectACL, DataFile, \
    DatafileParameterSet, ParameterName, GroupAdmin, Schema, \
    Dataset, ExperimentParameterSet, DatasetParameterSet, \
    License, UserProfile, UserAuthentication, Token

from tardis.tardis_portal.shortcuts import render_response_index, \
    return_response_error, return_response_not_found, \
    render_response_search, get_experiment_referer

from tardis.tardis_portal.views import _add_protocols_and_organizations

logger = logging.getLogger(__name__)

def _get_paramset_by_subtype(model, schema_subtype):
    for param_set in model.getParameterSets():
        if param_set.schema.subtype == schema_subtype:
            return param_set

    return None

def _get_param_value(model, param_name, schema_subtype):
    param_set = _get_paramset_by_subtype(model, schema_subtype)
    if param_set:
        return param_set.get_param(param_name)
    else:
        return None

def _format_bootstrap_table_json(fastqc_summary, fastqc_dataset_id):
    """
    Unpack the datastructure representing a summary of
    FastQC results for all samples in the project and repackage it
    as JSON suitable for presentation by bootstrap-table, including a
    separate list used for table column headers.

    :type fastqc_dataset_id: int
    :type fastqc_summary: dict
    :rtype: str, list
    """

    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None, None

    qc_descriptions = []
    qc_fields = []
    fastqc_data = []
    for sample in fastqc_summary['samples']:
        sample_name = sample['sample_name']
        sample_id_text = sample_name + " (R%s)" % (sample['read'])
        qc_checks = sample['qc_checks']
        fastqc_filename = sample.get('fastqc_report_filename', None)
        if fastqc_filename:
            _alnk = '%s'
            sample_link = _alnk % (fastqc_dataset_id,
                                   fastqc_filename,
                                   sample_id_text)
        else:
            sample_link = '%s' % sample_name
        results_for_sample = {'sample_name': sample_link}
        qc_anchor_index = 0
        for desc, result in qc_checks:
            qc_field_name = slugify(desc)
            if qc_field_name not in qc_fields:
                qc_fields.append(qc_field_name)
            if desc not in qc_descriptions:
                qc_descriptions.append(desc)
            if fastqc_filename:
                _alnk = '%s'
                result_link = _alnk % (fastqc_dataset_id,
                                       fastqc_filename,
                                       qc_anchor_index,
                                       result)
            else:
                result_link = '%s' % result
            results_for_sample[qc_field_name] = result_link
            results_for_sample['filename'] = fastqc_filename
            results_for_sample['fastqc_report_filename'] = fastqc_filename
            qc_anchor_index += 1
        fastqc_data.append(results_for_sample)

    fastqc_summary_headers = zip(qc_fields, qc_descriptions)
    if fastqc_data:
        fastqc_data_json = json.dumps(fastqc_data)
    else:
        fastqc_data_json = None

    return fastqc_data_json, fastqc_summary_headers

def _get_fastqc_json_parameter(dataset):
    """
    Find the Parameter containing the FastQC results (JSON) in the Dataset,
    return the deserialized Python datastructure, or None if not found.

    :type dataset: tardis.tardis_portal.models.Dataset
    :rtype: dict
    """

    subtype = 'hidden-fastqc-project-summary'
    fastqc_summary = None
    param_set = _get_paramset_by_subtype(dataset, subtype)
    if param_set:
        try:
            fastqc_summary_json = param_set.get_param(
                'hidden_fastqc_summary_json',
                True)
            fastqc_summary = json.loads(fastqc_summary_json)
        except DatasetParameter.DoesNotExist:
            pass

    return fastqc_summary

def _get_project_stats_from_fastqc(fastqc_summary):
    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None

    import numpy
    read_numbers = []
    gc_percents = []
    read_lengths = []
    for sample in fastqc_summary['samples']:
        params = sample['basic_stats']
        read_numbers.append(params.get('number_of_reads', None))
        read_lengths.append(params.get('read_length', None))
        gc_percents.append(params.get('percent_gc', None))

    # if one of the values isn't set, don't return misleading
    # overall stats, just return None
    if None in read_lengths or \
       None in read_numbers or \
       None in gc_percents:
        return None

    read_length_mean = numpy.mean(read_lengths)
    read_length_stddev = numpy.std(read_lengths)
    gc_mean = numpy.mean(gc_percents)
    gc_stddev = numpy.std(gc_percents)

    return {'total_reads': sum(read_numbers),
            'read_length_mean': read_length_mean,
            'read_length_stddev': read_length_stddev,
            'gc_mean': gc_mean,
            'gc_stddev': gc_stddev}

def _get_project_stats_from_datafiles(dataset):
    import numpy
    read_numbers = []
    gc_percents = []
    read_lengths = []
    subtype = 'fastq-raw-reads'
    for datafile in dataset.datafile_set.all():
        param_set = _get_paramset_by_subtype(datafile, subtype)
        read_numbers.append(int(param_set.get_param('number_of_reads', True)))
        read_lengths.append(int(param_set.get_param('read_length', True)))
        # no GC content in we can't get this from FASTQ DataFile metadata,
        # gc_percents.append(param_set.get_param('percent_gc', True))

    read_length_mean = numpy.mean(read_lengths)
    read_length_stddev = numpy.std(read_lengths)

    return {'total_reads': sum(read_numbers),
            'read_length_mean': read_length_mean,
            'read_length_stddev': read_length_stddev}

def _format_read_number_summary(fastqc_summary):

    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None

    sample_stats_table = {'thead': [], 'tbody': []}
    for sample in fastqc_summary['samples']:
        # sample_name = sample['sample_name']
        sample_name = sample['sample_name']

        basic_stats = sample['basic_stats']
        # TODO: use fcs-table format for this ?
        #       or also a bootstrap-table JSON blob ?
        sample_stats_table['thead'] = sorted(basic_stats.keys())
        extra_fields = ['sample_name', 'index', 'lane', 'read']
        for k in extra_fields:
            basic_stats[k] = sample[k]
        sample_stats_table['thead'] = extra_fields + sample_stats_table['thead']
        row = []
        for col in sample_stats_table['thead']:
            row.append(basic_stats[col])
        sample_stats_table['tbody'].append(row)

    return sample_stats_table

@authz.dataset_access_required
def view_fastqc_report(request, fastqc_dataset_id, filename):

    dataset = Dataset.objects.get(id=fastqc_dataset_id)
    datafile = DataFile.objects.filter(filename__exact=filename,
                                       dataset__id=fastqc_dataset_id).get()

    from tardis.tardis_portal.download import view_datafile
    return view_datafile(request, datafile.id)

@authz.dataset_access_required
def view_fastq_dataset(request, dataset_id):
    """

    This view should be used for Datasets using the FASTQ raw reads schema, by
    defining a mapping in settings.DATASET_VIEWS.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The Django HTTP response object
    :rtype: HttpResponse
    """
    c, dataset = prepare_project_dataset_view(request, dataset_id)

    # inspect parameters to retrieve the associated fastqc_dataset id,
    # if present
    fastqc_dataset_id = None
    fq_dataset_subtype = 'nucleotide-raw-reads-dataset'
    param_set = _get_paramset_by_subtype(dataset, fq_dataset_subtype)
    if param_set:
        fastqc_link = param_set.get_param('fastqc_dataset')
        if fastqc_link.link_id:
            fastqc_dataset_id = fastqc_link.link_id
        else:
            # this is a fallback, just in case link_id isn't set
            # but string_value is ...
            from urlparse import urlparse
            fastqc_dataset_id = urlparse(fastqc_link.link_url).path[1]

    # TODO: rather than use the fastqc_summary JSON blob associated with the
    #       FASTQ dataset, always use the one associated with a FastQC dataset,
    #       if one exists (then we can make the ingestor only add that
    #       ParameterSet set to the FastQC dataset, not the FASTQ one,
    #       to reduce duplication)
    #       Then we remove can also fastqc_summary and fastqc_version from
    #       prepare_project_dataset_view
    fastqc_summary = c.get('fastqc_summary', None)
    if fastqc_summary and fastqc_dataset_id:
        fastqc_table_json, fastqc_table_headers = \
            _format_bootstrap_table_json(fastqc_summary, fastqc_dataset_id)
        c['fastqc_table_json'] = fastqc_table_json
        c['fastqc_table_headers'] = fastqc_table_headers

    overall_stats = None
    if fastqc_summary:
        overall_stats = _get_project_stats_from_fastqc(fastqc_summary)
    else:
        overall_stats = _get_project_stats_from_datafiles(dataset)

    sample_stats_table = _format_read_number_summary(fastqc_summary)

    c['overall_stats'] = overall_stats
    c['sample_stats_table'] = sample_stats_table

    return HttpResponse(render_response_index(
        request,
        'view_project_dataset.html',
        c)
    )

@authz.dataset_access_required
def view_fastqc_reports_dataset(request, dataset_id):
    """

    This view should be used for Datasets using the FastQC reports schema, by
    defining a mapping in settings.DATASET_VIEWS.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The Django HTTP response object
    :rtype: HttpResponse
    """
    c, dataset = prepare_project_dataset_view(request, dataset_id)

    fastqc_summary = c.get('fastqc_summary', None)
    if fastqc_summary:
        fastqc_table_json, fastqc_table_headers = \
            _format_bootstrap_table_json(fastqc_summary, dataset_id)
        c['fastqc_table_json'] = fastqc_table_json
        c['fastqc_table_headers'] = fastqc_table_headers

    return HttpResponse(render_response_index(
        request,
        'view_project_dataset.html',
        c)
    )

def prepare_project_dataset_view(request, dataset_id):
    """
    Prepares the basic dictionary template for a 'Project' Dataset
    and associated information, and the Dataset instance.
    Caller of this method will typically add or override various
    values before returning their own HttpResponse (and template).

    Views using this method show the dataset, its metadata and a list of
    associated files with the option to show metadata of each file and ways
    to download those files.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The template dictionary and Dataset model instance
    :rtype: (dict, tardis.tardis_portal.Dataset)
    """
    dataset = Dataset.objects.get(id=dataset_id)

    def get_datafiles_page():
        # pagination was removed by someone in the interface but not here.
        # need to fix.
        pgresults = 100

        paginator = Paginator(dataset.datafile_set.all(), pgresults)

        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1

        # If page request (9999) is out of range, deliver last page of results.
        try:
            return paginator.page(page)
        except (EmptyPage, InvalidPage):
            return paginator.page(paginator.num_pages)

    upload_method = getattr(settings, "UPLOAD_METHOD", False)

    fastqc_summary = _get_fastqc_json_parameter(dataset)

    fastqc_version = ''
    if fastqc_summary:
        fastqc_version = fastqc_summary.get('fastqc_version', '')

    c = {
        'dataset': dataset,
        'datafiles': get_datafiles_page(),
        'parametersets': dataset.getParameterSets()
                                .exclude(schema__hidden=True),
        'has_download_permissions':
        authz.has_dataset_download_access(request, dataset_id),
        'has_write_permissions':
        authz.has_dataset_write(request, dataset_id),
        'from_experiment':
        get_experiment_referer(request, dataset_id),
        'other_experiments':
        authz.get_accessible_experiments_for_dataset(request, dataset_id),
        'upload_method': upload_method,
        'fastqc_version': fastqc_version,
        'fastqc_summary': fastqc_summary,
    }
    _add_protocols_and_organizations(request, dataset, c)

    return c, dataset
<commit_msg>Temporary workaround so that FastQC dataset links work with current mytardis develop.<commit_after>import logging
import os
import os.path as path
import json

from django.conf import settings
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.http import HttpResponseRedirect, HttpResponse,\
    HttpResponseForbidden, HttpResponseNotFound
from django.utils.text import slugify

from tardis.tardis_portal.auth import decorators as authz
from tardis.tardis_portal.models import Experiment, ExperimentParameter, \
    DatafileParameter, DatasetParameter, ObjectACL, DataFile, \
    DatafileParameterSet, ParameterName, GroupAdmin, Schema, \
    Dataset, ExperimentParameterSet, DatasetParameterSet, \
    License, UserProfile, UserAuthentication, Token

from tardis.tardis_portal.shortcuts import render_response_index, \
    return_response_error, return_response_not_found, \
    render_response_search, get_experiment_referer

from tardis.tardis_portal.views import _add_protocols_and_organizations

logger = logging.getLogger(__name__)

def _get_paramset_by_subtype(model, schema_subtype):
    for param_set in model.getParameterSets():
        if param_set.schema.subtype == schema_subtype:
            return param_set

    return None

def _get_param_value(model, param_name, schema_subtype):
    param_set = _get_paramset_by_subtype(model, schema_subtype)
    if param_set:
        return param_set.get_param(param_name)
    else:
        return None

def _format_bootstrap_table_json(fastqc_summary, fastqc_dataset_id):
    """
    Unpack the datastructure representing a summary of
    FastQC results for all samples in the project and repackage it
    as JSON suitable for presentation by bootstrap-table, including a
    separate list used for table column headers.

    :type fastqc_dataset_id: int
    :type fastqc_summary: dict
    :rtype: str, list
    """

    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None, None

    qc_descriptions = []
    qc_fields = []
    fastqc_data = []
    for sample in fastqc_summary['samples']:
        sample_name = sample['sample_name']
        sample_id_text = sample_name + " (R%s)" % (sample['read'])
        qc_checks = sample['qc_checks']
        fastqc_filename = sample.get('fastqc_report_filename', None)
        if fastqc_filename:
            _alnk = '%s'
            sample_link = _alnk % (fastqc_dataset_id,
                                   fastqc_filename,
                                   sample_id_text)
        else:
            sample_link = '%s' % sample_name
        results_for_sample = {'sample_name': sample_link}
        qc_anchor_index = 0
        for desc, result in qc_checks:
            qc_field_name = slugify(desc)
            if qc_field_name not in qc_fields:
                qc_fields.append(qc_field_name)
            if desc not in qc_descriptions:
                qc_descriptions.append(desc)
            if fastqc_filename:
                _alnk = '%s'
                result_link = _alnk % (fastqc_dataset_id,
                                       fastqc_filename,
                                       qc_anchor_index,
                                       result)
            else:
                result_link = '%s' % result
            results_for_sample[qc_field_name] = result_link
            results_for_sample['filename'] = fastqc_filename
            results_for_sample['fastqc_report_filename'] = fastqc_filename
            qc_anchor_index += 1
        fastqc_data.append(results_for_sample)

    fastqc_summary_headers = zip(qc_fields, qc_descriptions)
    if fastqc_data:
        fastqc_data_json = json.dumps(fastqc_data)
    else:
        fastqc_data_json = None

    return fastqc_data_json, fastqc_summary_headers

def _get_fastqc_json_parameter(dataset):
    """
    Find the Parameter containing the FastQC results (JSON) in the Dataset,
    return the deserialized Python datastructure, or None if not found.

    :type dataset: tardis.tardis_portal.models.Dataset
    :rtype: dict
    """

    subtype = 'hidden-fastqc-project-summary'
    fastqc_summary = None
    param_set = _get_paramset_by_subtype(dataset, subtype)
    if param_set:
        try:
            fastqc_summary_json = param_set.get_param(
                'hidden_fastqc_summary_json',
                True)
            fastqc_summary = json.loads(fastqc_summary_json)
        except DatasetParameter.DoesNotExist:
            pass

    return fastqc_summary

def _get_project_stats_from_fastqc(fastqc_summary):
    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None

    import numpy
    read_numbers = []
    gc_percents = []
    read_lengths = []
    for sample in fastqc_summary['samples']:
        params = sample['basic_stats']
        read_numbers.append(params.get('number_of_reads', None))
        read_lengths.append(params.get('read_length', None))
        gc_percents.append(params.get('percent_gc', None))

    # if one of the values isn't set, don't return misleading
    # overall stats, just return None
    if None in read_lengths or \
       None in read_numbers or \
       None in gc_percents:
        return None

    read_length_mean = numpy.mean(read_lengths)
    read_length_stddev = numpy.std(read_lengths)
    gc_mean = numpy.mean(gc_percents)
    gc_stddev = numpy.std(gc_percents)

    return {'total_reads': sum(read_numbers),
            'read_length_mean': read_length_mean,
            'read_length_stddev': read_length_stddev,
            'gc_mean': gc_mean,
            'gc_stddev': gc_stddev}

def _get_project_stats_from_datafiles(dataset):
    import numpy
    read_numbers = []
    gc_percents = []
    read_lengths = []
    subtype = 'fastq-raw-reads'
    for datafile in dataset.datafile_set.all():
        param_set = _get_paramset_by_subtype(datafile, subtype)
        read_numbers.append(int(param_set.get_param('number_of_reads', True)))
        read_lengths.append(int(param_set.get_param('read_length', True)))
        # no GC content in we can't get this from FASTQ DataFile metadata,
        # gc_percents.append(param_set.get_param('percent_gc', True))

    read_length_mean = numpy.mean(read_lengths)
    read_length_stddev = numpy.std(read_lengths)

    return {'total_reads': sum(read_numbers),
            'read_length_mean': read_length_mean,
            'read_length_stddev': read_length_stddev}

def _format_read_number_summary(fastqc_summary):

    if not fastqc_summary or ('samples' not in fastqc_summary):
        return None

    sample_stats_table = {'thead': [], 'tbody': []}
    for sample in fastqc_summary['samples']:
        # sample_name = sample['sample_name']
        sample_name = sample['sample_name']

        basic_stats = sample['basic_stats']
        # TODO: use fcs-table format for this ?
        #       or also a bootstrap-table JSON blob ?
        sample_stats_table['thead'] = sorted(basic_stats.keys())
        extra_fields = ['sample_name', 'index', 'lane', 'read']
        for k in extra_fields:
            basic_stats[k] = sample[k]
        sample_stats_table['thead'] = extra_fields + sample_stats_table['thead']
        row = []
        for col in sample_stats_table['thead']:
            row.append(basic_stats[col])
        sample_stats_table['tbody'].append(row)

    return sample_stats_table

@authz.dataset_access_required
def view_fastqc_report(request, fastqc_dataset_id, filename):

    dataset = Dataset.objects.get(id=fastqc_dataset_id)
    datafile = DataFile.objects.filter(filename__exact=filename,
                                       dataset__id=fastqc_dataset_id).get()

    from tardis.tardis_portal.download import view_datafile
    return view_datafile(request, datafile.id)

@authz.dataset_access_required
def view_fastq_dataset(request, dataset_id):
    """

    This view should be used for Datasets using the FASTQ raw reads schema, by
    defining a mapping in settings.DATASET_VIEWS.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The Django HTTP response object
    :rtype: HttpResponse
    """
    c, dataset = prepare_project_dataset_view(request, dataset_id)

    # inspect parameters to retrieve the associated fastqc_dataset id,
    # if present
    fastqc_dataset_id = None
    fq_dataset_subtype = 'nucleotide-raw-reads-dataset'
    param_set = _get_paramset_by_subtype(dataset, fq_dataset_subtype)
    if param_set:
        fastqc_link = param_set.get_param('fastqc_dataset')
        if fastqc_link.link_id:
            fastqc_dataset_id = fastqc_link.link_id
        else:
            # this is a fallback, just in case link_id isn't set
            # but string_value is ...
            from urlparse import urlparse
            # TODO: We would prefer to use link_url rather than string_value
            #       but need to wait for appropriate patches in mytardis
            #       develop branch that return string_value as a fallback
            #       rather than NotImplementedError
            # fastqc_dataset_id = urlparse(fastqc_link.link_url).path[1]
            fastqc_dataset_id = urlparse(fastqc_link.string_value).path[1]

    # TODO: rather than use the fastqc_summary JSON blob associated with the
    #       FASTQ dataset, always use the one associated with a FastQC dataset,
    #       if one exists (then we can make the ingestor only add that
    #       ParameterSet set to the FastQC dataset, not the FASTQ one,
    #       to reduce duplication)
    #       Then we remove can also fastqc_summary and fastqc_version from
    #       prepare_project_dataset_view
    fastqc_summary = c.get('fastqc_summary', None)
    if fastqc_summary and fastqc_dataset_id:
        fastqc_table_json, fastqc_table_headers = \
            _format_bootstrap_table_json(fastqc_summary, fastqc_dataset_id)
        c['fastqc_table_json'] = fastqc_table_json
        c['fastqc_table_headers'] = fastqc_table_headers

    overall_stats = None
    if fastqc_summary:
        overall_stats = _get_project_stats_from_fastqc(fastqc_summary)
    else:
        overall_stats = _get_project_stats_from_datafiles(dataset)

    sample_stats_table = _format_read_number_summary(fastqc_summary)

    c['overall_stats'] = overall_stats
    c['sample_stats_table'] = sample_stats_table

    return HttpResponse(render_response_index(
        request,
        'view_project_dataset.html',
        c)
    )

@authz.dataset_access_required
def view_fastqc_reports_dataset(request, dataset_id):
    """

    This view should be used for Datasets using the FastQC reports schema, by
    defining a mapping in settings.DATASET_VIEWS.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The Django HTTP response object
    :rtype: HttpResponse
    """
    c, dataset = prepare_project_dataset_view(request, dataset_id)

    fastqc_summary = c.get('fastqc_summary', None)
    if fastqc_summary:
        fastqc_table_json, fastqc_table_headers = \
            _format_bootstrap_table_json(fastqc_summary, dataset_id)
        c['fastqc_table_json'] = fastqc_table_json
        c['fastqc_table_headers'] = fastqc_table_headers

    return HttpResponse(render_response_index(
        request,
        'view_project_dataset.html',
        c)
    )

def prepare_project_dataset_view(request, dataset_id):
    """
    Prepares the basic dictionary template for a 'Project' Dataset
    and associated information, and the Dataset instance.
    Caller of this method will typically add or override various
    values before returning their own HttpResponse (and template).

    Views using this method show the dataset, its metadata and a list of
    associated files with the option to show metadata of each file and ways
    to download those files.

    :param request: The Django HTTP request object
    :type request: HTTPRequest
    :param dataset_id: The ID of the Dataset to be viewed (primary key)
    :type dataset_id: int
    :return: The template dictionary and Dataset model instance
    :rtype: (dict, tardis.tardis_portal.Dataset)
    """
    dataset = Dataset.objects.get(id=dataset_id)

    def get_datafiles_page():
        # pagination was removed by someone in the interface but not here.
        # need to fix.
        pgresults = 100

        paginator = Paginator(dataset.datafile_set.all(), pgresults)

        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1

        # If page request (9999) is out of range, deliver last page of results.
        try:
            return paginator.page(page)
        except (EmptyPage, InvalidPage):
            return paginator.page(paginator.num_pages)

    upload_method = getattr(settings, "UPLOAD_METHOD", False)

    fastqc_summary = _get_fastqc_json_parameter(dataset)

    fastqc_version = ''
    if fastqc_summary:
        fastqc_version = fastqc_summary.get('fastqc_version', '')

    c = {
        'dataset': dataset,
        'datafiles': get_datafiles_page(),
        'parametersets': dataset.getParameterSets()
                                .exclude(schema__hidden=True),
        'has_download_permissions':
        authz.has_dataset_download_access(request, dataset_id),
        'has_write_permissions':
        authz.has_dataset_write(request, dataset_id),
        'from_experiment':
        get_experiment_referer(request, dataset_id),
        'other_experiments':
        authz.get_accessible_experiments_for_dataset(request, dataset_id),
        'upload_method': upload_method,
        'fastqc_version': fastqc_version,
        'fastqc_summary': fastqc_summary,
    }
    _add_protocols_and_organizations(request, dataset, c)

    return c, dataset
<|endoftext|>


<commit_before># This file is part of the Juju GUI, which lets users view and manage Juju
# environments within a graphical interface (https://launchpad.net/juju-gui).
# Copyright (C) 2013 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License version 3, as published by
# the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Juju GUI server.

The GUI server is a custom-made application based on the
[Tornado](http://www.tornadoweb.org/) framework.

It directly serves static files to the browser, including images, HTML, CSS and
JavaScript files via an HTTPS connection to port 443. HTTP connections to port
80 are redirected to the former one. All other URLs serve the common
`index.html` file.

It also acts as a proxy between the browser and the Juju API server that
performs the actual orchestration work. Both browser-server and server-Juju
connections are bidirectional, using the WebSocket protocol on the same port as
the HTTPS connection, allowing changes in the Juju environment to be propagated
and shown immediately by the browser. """

VERSION = (0, 2, 0)

def get_version():
    """Return the Juju GUI server version as a string."""
    return '.'.join(map(str, VERSION))
<commit_msg>Bumped GUI server version up.<commit_after># This file is part of the Juju GUI, which lets users view and manage Juju
# environments within a graphical interface (https://launchpad.net/juju-gui).
# Copyright (C) 2013 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License version 3, as published by
# the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Juju GUI server.

The GUI server is a custom-made application based on the
[Tornado](http://www.tornadoweb.org/) framework.

It directly serves static files to the browser, including images, HTML, CSS and
JavaScript files via an HTTPS connection to port 443. HTTP connections to port
80 are redirected to the former one. All other URLs serve the common
`index.html` file.

It also acts as a proxy between the browser and the Juju API server that
performs the actual orchestration work. Both browser-server and server-Juju
connections are bidirectional, using the WebSocket protocol on the same port as
the HTTPS connection, allowing changes in the Juju environment to be propagated
and shown immediately by the browser. """

VERSION = (0, 2, 1)

def get_version():
    """Return the Juju GUI server version as a string."""
    return '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ts=2 sw=2 et ai
###############################################################################
# Copyright (c) 2012,2013 Andreas Vogel andreas@wellenvogel.net
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included
#  in all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.
#
#  parts from this software (AIS decoding) are taken from the gpsd project
#  so refer to this BSD licencse also (see ais.py) or omit ais.py
###############################################################################

import time
import subprocess
import threading
import os
import datetime
import glob
import sys
import traceback
import json
import datetime
import threading

from avnav_util import *
from avnav_worker import *
from wpa_control import WpaControl
import avnav_handlerList

#a handler to set up a wpa claient
class AVNWpaHandler(AVNWorker):
  def __init__(self,param):
    AVNWorker.__init__(self, param)
    self.wpaHandler=None
    self.lastScan=datetime.datetime.utcnow()
    self.scanLock=threading.Lock()
    self.getRequestParam=AVNUtil.getHttpRequestParam
  @classmethod
  def getConfigName(cls):
    return "AVNWpaHandler"
  @classmethod
  def getConfigParam(cls, child=None):
    if child is not None:
      return None
    return {
            'ownSocket':'/tmp/avnav-wpa-ctrl', #my own socket endpoint
            'wpaSocket':"/var/run/wpa_supplicant/wlan-av1", #the wpa control socket
            'ownSsid':'avnav'
    }
  @classmethod
  def preventMultiInstance(cls):
    return True
  def getName(self):
    return "WpaControl"

  def run(self):
    self.setName("[%s]%s"%(AVNLog.getThreadId(),self.getConfigName()))
    wpaSocket=self.getStringParam('wpaSocket')
    ownSocket=self.getStringParam('ownSocket')
    while True:
      try:
        newWpaSocket=self.getStringParam('wpaSocket')
        if newWpaSocket != wpaSocket:
          if self.wpaHandler is not None:
            self.wpaHandler.close(False)
            self.wpaHandler=None
          wpaSocket=newWpaSocket
        if os.path.exists(wpaSocket):
          if self.wpaHandler is None:
            AVNLog.info("connecting to wpa_supplicant %s",wpaSocket)
            self.wpaHandler=WpaControl(wpaSocket,ownSocket)
            self.wpaHandler.open()
            self.setInfo('main','connected to %s'%(wpaSocket),AVNWorker.Status.STARTED)
          else:
            try:
              self.wpaHandler.checkOpen()
            except:
              AVNLog.error("wpa handler closed...")
              self.wpaHandler=None
        else:
          if self.wpaHandler is not None:
            self.wpaHandler.close(False)
            self.wpaHandler=None
            AVNLog.info("disconnecting from wpa_supplicant %s",wpaSocket)
            self.setInfo('main','disconnected from %s'%(wpaSocket),AVNWorker.Status.INACTIVE)
          time.sleep(5)
          continue
        #we should have an active wpa handler here...
        #todo: cache scan results here
      except Exception as e:
        AVNLog.error("exception in WPAHandler: %s",traceback.format_exc())
        pass
      time.sleep(5)

  def startScan(self):
    if self.wpaHandler is None:
      return
    self.scanLock.acquire()
    now=datetime.datetime.utcnow()
    if now > (self.lastScan + datetime.timedelta(seconds=30)):
      self.lastScan=now
      self.scanLock.release()
      self.wpaHandler.startScan()
      return
    self.scanLock.release()

  def getList(self):
    rt=[]
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      self.startScan()
    except:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
    try:
      list=wpaHandler.scanResultWithInfo()
      ownSSid=self.getStringParam('ownSsid')
      if ownSSid is not None and ownSSid != "":
        #remove own ssid
        for net in list:
          netSsid=net.get('ssid')
          if netSsid is not None and netSsid==ownSSid:
            continue
          rt.append(net)
      else:
        rt=list
      AVNLog.debug("wpa list",rt)
      return rt
    except Exception:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return rt
  def removeNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa remove network",id)
      wpaHandler.removeNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def enableNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa enable network",id)
      wpaHandler.enableNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}
  def disableNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa disable network",id)
      wpaHandler.disableNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:disableNetwork: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def connect(self,param):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa connect",param)
      wpaHandler.connect(param)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:connect: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def getStatus(self):
    rt={'wpa_state':'OFFLINE'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      rt=wpaHandler.status()
      AVNLog.debug("wpa status",rt)
      return rt
    except Exception:
      AVNLog.error("exception in WPAHandler:getStatus: %s",traceback.format_exc())
      return {'wpa_state','COMMANDERROR'}

  def getHandledCommands(self):
    return "wpa"

  def handleApiRequest(self,type,requestparam):
    start=datetime.datetime.utcnow()
    command=self.getRequestParam(requestparam, 'command')
    rt=None
    if command is None:
      raise Exception('missing command for wpa request')
    if command == 'list':
      rt=json.dumps(self.getList())
    if command == 'status':
      rt=json.dumps(self.getStatus())
    if command == 'all':
      rt=json.dumps({'status':self.getStatus(),'list':self.getList()})
    if command == 'enable':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.enableNetwork(id))
    if command == 'disable':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.disableNetwork(id))
    if command == 'remove':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.removeNetwork(id))
    if command == 'connect':
      param={}
      for k in ['ssid','psk']:
        v=self.getRequestParam(requestparam,k)
        if v is not None:
          param[k]=v
      rt=json.dumps(self.connect(param))
    if rt is None:
      raise Exception("unknown command %s"%(command))
    end=datetime.datetime.utcnow()
    AVNLog.debug("wpa request %s lasted %d millis",command,(end-start).total_seconds()*1000)
    return rt
avnav_handlerList.registerHandler(AVNWpaHandler)

<commit_msg>correctly handle api request in wpa handler allow for multiple own ssids<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ts=2 sw=2 et ai
###############################################################################
# Copyright (c) 2012,2013 Andreas Vogel andreas@wellenvogel.net
#
#  Permission is hereby granted, free of charge, to any person obtaining a
#  copy of this software and associated documentation files (the "Software"),
#  to deal in the Software without restriction, including without limitation
#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
#  and/or sell copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included
#  in all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
#  OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
#  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
#  DEALINGS IN THE SOFTWARE.
#
#  parts from this software (AIS decoding) are taken from the gpsd project
#  so refer to this BSD licencse also (see ais.py) or omit ais.py
###############################################################################

import time
import subprocess
import threading
import os
import datetime
import glob
import sys
import traceback
import json
import datetime
import threading

from avnav_util import *
from avnav_worker import *
from wpa_control import WpaControl
import avnav_handlerList

#a handler to set up a wpa claient
class AVNWpaHandler(AVNWorker):
  def __init__(self,param):
    AVNWorker.__init__(self, param)
    self.wpaHandler=None
    self.lastScan=datetime.datetime.utcnow()
    self.scanLock=threading.Lock()
    self.getRequestParam=AVNUtil.getHttpRequestParam
  @classmethod
  def getConfigName(cls):
    return "AVNWpaHandler"
  @classmethod
  def getConfigParam(cls, child=None):
    if child is not None:
      return None
    return {
            'ownSocket':'/tmp/avnav-wpa-ctrl', #my own socket endpoint
            'wpaSocket':"/var/run/wpa_supplicant/wlan-av1", #the wpa control socket
            'ownSsid':'avnav,avnav1,avnav2'
    }
  @classmethod
  def preventMultiInstance(cls):
    return True
  def getName(self):
    return "WpaControl"

  def run(self):
    self.setName("[%s]%s"%(AVNLog.getThreadId(),self.getConfigName()))
    wpaSocket=self.getStringParam('wpaSocket')
    ownSocket=self.getStringParam('ownSocket')
    while True:
      try:
        newWpaSocket=self.getStringParam('wpaSocket')
        if newWpaSocket != wpaSocket:
          if self.wpaHandler is not None:
            self.wpaHandler.close(False)
            self.wpaHandler=None
          wpaSocket=newWpaSocket
        if os.path.exists(wpaSocket):
          if self.wpaHandler is None:
            AVNLog.info("connecting to wpa_supplicant %s",wpaSocket)
            self.wpaHandler=WpaControl(wpaSocket,ownSocket)
            self.wpaHandler.open()
            self.setInfo('main','connected to %s'%(wpaSocket),AVNWorker.Status.STARTED)
          else:
            try:
              self.wpaHandler.checkOpen()
            except:
              AVNLog.error("wpa handler closed...")
              self.wpaHandler=None
        else:
          if self.wpaHandler is not None:
            self.wpaHandler.close(False)
            self.wpaHandler=None
            AVNLog.info("disconnecting from wpa_supplicant %s",wpaSocket)
            self.setInfo('main','disconnected from %s'%(wpaSocket),AVNWorker.Status.INACTIVE)
          time.sleep(5)
          continue
        #we should have an active wpa handler here...
        #todo: cache scan results here
      except Exception as e:
        AVNLog.error("exception in WPAHandler: %s",traceback.format_exc())
        pass
      time.sleep(5)

  def startScan(self):
    if self.wpaHandler is None:
      return
    self.scanLock.acquire()
    now=datetime.datetime.utcnow()
    if now > (self.lastScan + datetime.timedelta(seconds=30)):
      self.lastScan=now
      self.scanLock.release()
      self.wpaHandler.startScan()
      return
    self.scanLock.release()

  def getList(self):
    rt=[]
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      self.startScan()
    except:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
    try:
      list=wpaHandler.scanResultWithInfo()
      ownSSid=self.getStringParam('ownSsid')
      if ownSSid is not None and ownSSid != "":
        #remove own ssid
        for net in list:
          netSsid=net.get('ssid')
          if netSsid is not None and netSsid in ownSSid.split(","):
            continue
          rt.append(net)
      else:
        rt=list
      AVNLog.debug("wpa list",rt)
      return rt
    except Exception:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return rt
  def removeNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa remove network",id)
      wpaHandler.removeNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def enableNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa enable network",id)
      wpaHandler.enableNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:getList: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}
  def disableNetwork(self,id):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa disable network",id)
      wpaHandler.disableNetwork(id)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:disableNetwork: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def connect(self,param):
    rt={'status':'no WLAN'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      AVNLog.debug("wpa connect",param)
      wpaHandler.connect(param)
      wpaHandler.saveConfig()
      return {'status':'OK'}
    except Exception as e:
      AVNLog.error("exception in WPAHandler:connect: %s",traceback.format_exc())
      return {'status':'commandError','info':str(e)}

  def getStatus(self):
    rt={'wpa_state':'OFFLINE'}
    wpaHandler=self.wpaHandler
    if wpaHandler is None:
      return rt
    try:
      rt=wpaHandler.status()
      AVNLog.debug("wpa status",rt)
      return rt
    except Exception:
      AVNLog.error("exception in WPAHandler:getStatus: %s",traceback.format_exc())
      return {'wpa_state','COMMANDERROR'}

  def getHandledCommands(self):
    return "wpa"

  def handleApiRequest(self,type,subtype,requestparam):
    start=datetime.datetime.utcnow()
    command=self.getRequestParam(requestparam, 'command')
    rt=None
    if command is None:
      raise Exception('missing command for wpa request')
    if command == 'list':
      rt=json.dumps(self.getList())
    if command == 'status':
      rt=json.dumps(self.getStatus())
    if command == 'all':
      rt=json.dumps({'status':self.getStatus(),'list':self.getList()})
    if command == 'enable':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.enableNetwork(id))
    if command == 'disable':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.disableNetwork(id))
    if command == 'remove':
      id=self.getRequestParam(requestparam,'id')
      rt=json.dumps(self.removeNetwork(id))
    if command == 'connect':
      param={}
      for k in ['ssid','psk']:
        v=self.getRequestParam(requestparam,k)
        if v is not None:
          param[k]=v
      rt=json.dumps(self.connect(param))
    if rt is None:
      raise Exception("unknown command %s"%(command))
    end=datetime.datetime.utcnow()
    AVNLog.debug("wpa request %s lasted %d millis",command,(end-start).total_seconds()*1000)
    return rt
avnav_handlerList.registerHandler(AVNWpaHandler)

<|endoftext|>


<commit_before>from flask import jsonify, request
from penn.base import APIError
from sqlalchemy import or_

from server import app
from server.models import Account

@app.route('/studyspaces/user/search', methods=['GET'])
def get_nam():
    """
    Gets users that match search query
    """

    query = request.args.get('query')
    if not query:
        return jsonify({'error': 'Query argument not found.'}), 400

    if len(query) <= 1:
    	return jsonify({'error': 'Query is too short. Minimum length is two characters.'}), 400

    # query = query.lower()
    starting_query = '{}%'.format(query)
    general_query = '%{}%'.format(query)

    starting_first_name_matches = Account.query.filter(Account.first.like(starting_query)).all()
    starting_last_name_matches = Account.query.filter(Account.last.like(starting_query)).all()
    general_first_name_matches = Account.query.filter(Account.first.like(general_query)).all()
    general_last_name_matches = Account.query.filter(Account.last.like(general_query)).all()

    users = starting_first_name_matches
    users.extend(starting_last_name_matches)
    users.extend(general_first_name_matches)
    users.extend(general_last_name_matches)

    seen_pennkeys = set()
    filtered_users = []
    for user in users:
    	if user.pennkey not in seen_pennkeys:
    		filtered_users.append(user)
    		seen_pennkeys.add(user.pennkey)

    filtered_users = [
    	{
    		'first': x.first,
    		'last': x.last,
    		'pennkey': x.pennkey,
    		'email': x.email,
    	} for x in filtered_users
    ]

    return jsonify({'results': filtered_users})
<commit_msg>first and last name search<commit_after>from flask import jsonify, request
from penn.base import APIError
from sqlalchemy import and_

from server import app
from server.models import Account

@app.route('/studyspaces/user/search', methods=['GET'])
def get_nam():
    """
    Gets users that match search query
    """

    query = request.args.get('query')
    if not query:
        return jsonify({'error': 'Query argument not found.'}), 400

    if len(query) <= 1:
    	return jsonify({'error': 'Query is too short. Minimum length is two characters.'}), 400

    # query = query.lower()
    if ' ' in query:
    	split = query.split(' ')
    	if len(split) >= 2:
    		first = split[0]
    		last = split[1]

    users = []
    if first and last:
	    matches = Account.query.filter(and_(Account.first.like('{}%'.format(first)), Account.last.like('{}%'.format(last)))).all()
	    users.extend(matches)
    else:
	    starting_query = '{}%'.format(query)
	    general_query = '%{}%'.format(query)

	    starting_first_name_matches = Account.query.filter(Account.first.like(starting_query)).all()
	    starting_last_name_matches = Account.query.filter(Account.last.like(starting_query)).all()
	    general_first_name_matches = Account.query.filter(Account.first.like(general_query)).all()
	    general_last_name_matches = Account.query.filter(Account.last.like(general_query)).all()

	    users.extend(starting_first_name_matches)
	    users.extend(starting_last_name_matches)
	    users.extend(general_first_name_matches)
	    users.extend(general_last_name_matches)

    seen_pennkeys = set()
    filtered_users = []
    for user in users:
    	if user.pennkey not in seen_pennkeys:
    		filtered_users.append(user)
    		seen_pennkeys.add(user.pennkey)

    filtered_users = [
    	{
    		'first': x.first,
    		'last': x.last,
    		'pennkey': x.pennkey,
    		'email': x.email,
    	} for x in filtered_users
    ]

    return jsonify({'results': filtered_users})
<|endoftext|>


<commit_before># Copyright (C) 2013-2016 Martin Vejmelka, UC Denver
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR
# A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from ssh_shuttle import SSHShuttle
import json
import os
import os.path as osp
import sys
import logging
import shutil
import glob
import signal
import subprocess
from utils import kill_process, Dict, process_create_time, load_sys_cfg

cfg = load_sys_cfg()

def remote_rmdir(s,dirname):
    logging.info('SHUTTLE removing remote directory %s' % dirname)
    try:
        s.rmdir(dirname)
        logging.info('SHUTTLE delete successful.')
        return 0
    except:
        logging.error('SHUTTLE cannot delete directory %s' % dirname)
        return 'Error'

def local_rmdir(dirname):
    work_dir = osp.abspath(osp.join(cfg['workspace_path'], dirname))
    logging.info('Deleting directory %s' % work_dir)
    try:
        shutil.rmtree(work_dir)
        logging.info('Deleted directory %s' % work_dir)
        return 0
    except:
        logging.error('Deleting directory %s failed' % work_dir)
        return 'Error'

def cancel_job(job_num,qsys):
        if job_num is not None:
            logging.info('Deleting parallel job %s on %s.' % (job_num, qsys))
            cluster = load_cluster_file(qsys)
            try:
                ret = subprocess.check_output([cluster['qdel_cmd'], job_num])
                logging.info(ret)
            except:
                logging.error('Deleting parallel job %s failed.' % job_num)

def load_cluster_file(qsys):
    clusters = json.load(open('etc/clusters.json','r'))
    return clusters[qsys]

def load_job_file(job_id):
    jobfile = osp.join(cfg['workspace_path'], job_id, 'job.json')
    logging.info('Loading job state from %s' % jobfile)
    try:
        js = Dict(json.load(open(jobfile,'r')))
    except IOError:
        logging.error('Cannot open %s' % jobfile)
        js = None
    except:
        logging.error('Cannot load %s' % jobfile)
        js = None
    # print(json.dumps(js, indent=4, separators=(',', ': ')))
    return js, jobfile

def forecast_process_running(js):
    if 'pid' in js and 'process_create_time' in js:
        fs =  js.process_create_time == process_create_time(js.pid)
        logging.info('Forecast process is running: %s pid=%s' % (fs, js.pid))
        return fs
    else:
        logging.warning('Cannot determine if forecast process is running - old job file?')
        return None

def parallel_job_running(js):
    cluster = load_cluster_file(js.qsys)
    ret = subprocess.check_output(cluster['qstat_cmd'])
    ws = False
    for line in ret.split('\n'):
        ls=line.split()
        if len(ls) >0 and ls[0] == js.job_num:
             ws = True
             break
    if ws:
        logging.debug('WRF job %s is running.' % js.job_num)
    else:
        logging.info('WRF job %s is not running.' % js.job_num)
    return ws

# the cleanup functions called as argv[2]:
# connecting to remote host if needed

def list(s):
    s.connect()
    cat = s.retrieve_catalog()
    s.disconnect()
    print('%-60s desc' % 'id')
    print('-' * 70)
    for k in sorted(cat):
        print('%-60s %s' % (k, cat[k]['description']))

def workspace(s):
    logging.info('Deleting all directories in local workspace that are not in the remote catalog.')
    s.connect()
    cat = s.retrieve_catalog(s)
    s.disconnect()
    for f in glob.glob(osp.join(cfg['workspace_path'],'*')):
        if osp.isdir(f):
            ff = osp.basename(f)
            if ff not in cat:
                logging.error('%s not in the catalog' % ff)
                local_rmdir(cfg, f)

def output(s,name):
    logging.info('Trying to delete WRF output and visualization of job %s' % name)
    s.connect()
    remote_rmdir(s, name)
    s.disconnect()
    local_rmdir(cfg,osp.join(name,'products'))
    wrf_dir = osp.join(cfg['workspace_path'], name,'wrf')
    files = glob.glob(osp.join(wrf_dir,'rsl.*'))+glob.glob(osp.join(wrf_dir,'wrfout_*'))
    logging.info('Deleting %s WRF output files in  %s' % (len(files),wrf_dir ))
    for f in files:
        os.remove(f)

def cancel(name):
    logging.info('Trying to cancel job %s' % name)
    js, jobfile = load_job_file(name)
    if js is not None:
        if 'state' in js and js.state == 'Cancelled':
            logging.warning('Job %s was already cancelled, proceeding anyway' % name)
        if 'job_num' in js and js.job_num is not None:
            try:
                logging.info('Trying to delete job %s from the queue scheduler' % js.job_num)
                cancel_job(js.job_num,js.qsys)
            except:
                logging.error('Delete failed, please check your queue scheduler for job %s' % js.job_num)
            js.old_job_num = js.job_num
        js.job_num = None
        if 'pid' in js and js.pid is not None:
            logging.info('Trying to kill forecast process %s' % js.pid)
            kill_process(js.pid)
        js.pid = None
        js.state = "Cancelled"
        json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))

def delete(s,name):
    s.connect()
    logging.info('Trying to delete all files of job %s' % name)
    remote_rmdir(s,name)
    local_rmdir(name)
    s.simple_command('wrfxweb/join_catalog.sh')
    s.disconnect()

def update(name):
    logging.info('Updating state of job %s' % name)
    js, jobfile = load_job_file(name)
    if js is not None:
        logging.info('Old state is: %s' % js.state)
        if not forecast_process_running(js):
            #logging.info('Forecast process is not running')
            js.pid = None
        if not parallel_job_running(js):
            #logging.info('WRF is not running')
            js.old_job_num = js.job_num
            js.job_num = None
        if js.state == 'Completed' or js.state == 'Cancelled':
            if js.pid is not None:
                 js.state = 'Forecast runaway'
            if js.job_num is not None:
                 js.state = 'WRF runaway'
        elif js.state == 'Preparing':
                 if js.pid is None:
                      js.state = 'Stopped'
                 if js.job_num is not None:
                      js.state = 'Cannot happen'
        elif js.state == 'Processing':
                 if js.pid is None and js.job_num is None:
                      js.state = 'Stopped'
                 if js.pid is not None and js.job_num is None:
                      js.state = 'WRF stopped'
                 if js.pid is None and js.job_num is not None:
                      js.state = 'Forecast stopped'

        logging.info('New state is: %s' % js.state)

        json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))

if __name__ == '__main__':

    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

    commands = [ 'list', 'cancel', 'output', 'delete', 'workspace', 'update']

    if len(sys.argv) < 2 or sys.argv[1] not in commands:
        print('usage: ./cleanup.sh ' + '|'.join(commands) +' [job_id]')
        print('list            : show list of current simulations with their job_id and description')
        print('cancel <job_id> : kill all processes and the WRF parallel job, do not delete any files')
        print('output <job id> : cancel, and delete all WRF output and visualization files only')
        print('delete <job_id> : cancel, and delete all files')
        print('workspace       : delete jobs that are not on the visulalization server')
        print('update <job_id> : check if the job is running and update its job state file')
        sys.exit(1)

    cmd = sys.argv[1]
    if cmd in ['delete' , 'cancel', 'output', 'update']:
        if len(sys.argv) == 3 and not sys.argv[2] == "" :
            job_id = sys.argv[2]
        else:
            print('%s function requires one job id' % cmd)
            sys.exit(1)
    else:
        job_id = None

    logging.info('cleanup: command=%s job_id=%s' % (cmd,job_id))
    s = SSHShuttle(cfg)

    if cmd == 'list':
        list(s)

    if cmd == 'cancel':
        cancel(job_id)

    if cmd == 'output':
        cancel(job_id)
        output(s,job_id)

    if cmd == 'delete':
        cancel(job_id)
        delete(s,job_id)

    if cmd == 'workspace':
        workspace(s)

    if cmd == 'update':
        update(job_id)

<commit_msg>changed a few output messages for clarity<commit_after># Copyright (C) 2013-2016 Martin Vejmelka, UC Denver
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR
# A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from ssh_shuttle import SSHShuttle
import json
import os
import os.path as osp
import sys
import logging
import shutil
import glob
import signal
import subprocess
from utils import kill_process, Dict, process_create_time, load_sys_cfg

cfg = load_sys_cfg()

def remote_rmdir(s,dirname):
    logging.info('SHUTTLE removing remote directory %s' % dirname)
    try:
        s.rmdir(dirname)
        logging.info('SHUTTLE delete successful.')
        return 0
    except:
        logging.error('SHUTTLE cannot delete directory %s' % dirname)
        return 'Error'

def local_rmdir(dirname):
    work_dir = osp.abspath(osp.join(cfg['workspace_path'], dirname))
    logging.info('Deleting directory %s' % work_dir)
    try:
        shutil.rmtree(work_dir)
        logging.info('Deleted directory %s' % work_dir)
        return 0
    except:
        logging.error('Deleting directory %s failed' % work_dir)
        return 'Error'

def cancel_job(job_num,qsys):
        if job_num is not None:
            logging.info('Deleting parallel job %s on %s.' % (job_num, qsys))
            cluster = load_cluster_file(qsys)
            try:
                ret = subprocess.check_output([cluster['qdel_cmd'], job_num])
                logging.info(ret)
            except:
                logging.error('Deleting parallel job %s failed.' % job_num)

def load_cluster_file(qsys):
    clusters = json.load(open('etc/clusters.json','r'))
    return clusters[qsys]

def load_job_file(job_id):
    jobfile = osp.join(cfg['workspace_path'], job_id, 'job.json')
    logging.info('Loading job state from %s' % jobfile)
    try:
        js = Dict(json.load(open(jobfile,'r')))
    except IOError:
        logging.error('Cannot open %s' % jobfile)
        js = None
    except:
        logging.error('Cannot load %s' % jobfile)
        js = None
    # print(json.dumps(js, indent=4, separators=(',', ': ')))
    return js, jobfile

def forecast_process_running(js):
    if 'pid' in js and 'process_create_time' in js:
        fs =  js.process_create_time == process_create_time(js.pid)
        if fs:
           logging.info('Forecast process is running. pid=%s' % (js.pid))
        else:
            logging.info('Forecast process is not running')
        return fs
    else:
        logging.warning('Cannot determine if forecast process is running - old job file?')
        return None

def parallel_job_running(js):
    cluster = load_cluster_file(js.qsys)
    logging.debug('Checking for WRF job %s' % js.job_num)
    ret = subprocess.check_output(cluster['qstat_cmd'])
    ws = False
    for line in ret.split('\n'):
        ls=line.split()
        if len(ls) >0 and ls[0] == js.job_num:
             ws = True
             break
    if ws:
        logging.info('WRF job %s is running.' % js.job_num)
    else:
        logging.info('WRF job is not running.')
    return ws

# the cleanup functions called as argv[2]:
# connecting to remote host if needed

def list(s):
    s.connect()
    cat = s.retrieve_catalog()
    s.disconnect()
    print('%-60s desc' % 'id')
    print('-' * 70)
    for k in sorted(cat):
        print('%-60s %s' % (k, cat[k]['description']))

def workspace(s):
    logging.info('Deleting all directories in local workspace that are not in the remote catalog.')
    s.connect()
    cat = s.retrieve_catalog(s)
    s.disconnect()
    for f in glob.glob(osp.join(cfg['workspace_path'],'*')):
        if osp.isdir(f):
            ff = osp.basename(f)
            if ff not in cat:
                logging.error('%s not in the catalog' % ff)
                local_rmdir(cfg, f)

def output(s,name):
    logging.info('Trying to delete WRF output and visualization of job %s' % name)
    s.connect()
    remote_rmdir(s, name)
    s.disconnect()
    local_rmdir(cfg,osp.join(name,'products'))
    wrf_dir = osp.join(cfg['workspace_path'], name,'wrf')
    files = glob.glob(osp.join(wrf_dir,'rsl.*'))+glob.glob(osp.join(wrf_dir,'wrfout_*'))
    logging.info('Deleting %s WRF output files in  %s' % (len(files),wrf_dir ))
    for f in files:
        os.remove(f)

def cancel(name):
    logging.info('Trying to cancel job %s' % name)
    js, jobfile = load_job_file(name)
    if js is not None:
        if 'state' in js and js.state == 'Cancelled':
            logging.warning('Job %s was already cancelled, proceeding anyway' % name)
        if 'job_num' in js and js.job_num is not None:
            try:
                logging.info('Trying to delete job %s from the queue scheduler' % js.job_num)
                cancel_job(js.job_num,js.qsys)
            except:
                logging.error('Delete failed, please check your queue scheduler for job %s' % js.job_num)
            js.old_job_num = js.job_num
        js.job_num = None
        if 'pid' in js and js.pid is not None:
            logging.info('Trying to kill forecast process %s' % js.pid)
            kill_process(js.pid)
        js.pid = None
        js.state = "Cancelled"
        json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))

def delete(s,name):
    s.connect()
    logging.info('Trying to delete all files of job %s' % name)
    remote_rmdir(s,name)
    local_rmdir(name)
    s.simple_command('wrfxweb/join_catalog.sh')
    s.disconnect()

def update(name):
    logging.info('Updating state of job %s' % name)
    js, jobfile = load_job_file(name)
    if js is not None:
        logging.debug('Old state is: %s' % js.state)
        if not forecast_process_running(js):
            logging.debug('Forecast process is not running')
            js.pid = None
        if not parallel_job_running(js):
            logging.debug('WRF is not running')
            js.old_job_num = js.job_num
            js.job_num = None
        if js.state == 'Completed' or js.state == 'Cancelled':
            if js.pid is not None:
                 js.state = 'Forecast runaway'
            if js.job_num is not None:
                 js.state = 'WRF runaway'
        elif js.state == 'Preparing':
                 if js.pid is None:
                      js.state = 'Stopped'
                 if js.job_num is not None:
                      js.state = 'Cannot happen'
        elif js.state == 'Processing':
                 if js.pid is None and js.job_num is None:
                      js.state = 'Stopped'
                 if js.pid is not None and js.job_num is None:
                      js.state = 'WRF stopped'
                 if js.pid is None and js.job_num is not None:
                      js.state = 'Forecast stopped'

        logging.info('State is: %s' % js.state)

        json.dump(js, open(jobfile,'w'), indent=4, separators=(',', ': '))

if __name__ == '__main__':

    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

    commands = [ 'list', 'cancel', 'output', 'delete', 'workspace', 'update']

    if len(sys.argv) < 2 or sys.argv[1] not in commands:
        print('usage: ./cleanup.sh ' + '|'.join(commands) +' [job_id]')
        print('list            : show list of current simulations with their job_id and description')
        print('cancel <job_id> : kill all processes and the WRF parallel job, do not delete any files')
        print('output <job id> : cancel, and delete all WRF output and visualization files only')
        print('delete <job_id> : cancel, and delete all files')
        print('workspace       : delete jobs that are not on the visulalization server')
        print('update <job_id> : check if the job is running and update its job state file')
        sys.exit(1)

    cmd = sys.argv[1]
    if cmd in ['delete' , 'cancel', 'output', 'update']:
        if len(sys.argv) == 3 and not sys.argv[2] == "" :
            job_id = sys.argv[2]
        else:
            print('%s function requires one job id' % cmd)
            sys.exit(1)
    else:
        job_id = None

    logging.info('cleanup: command=%s job_id=%s' % (cmd,job_id))
    s = SSHShuttle(cfg)

    if cmd == 'list':
        list(s)

    if cmd == 'cancel':
        cancel(job_id)

    if cmd == 'output':
        cancel(job_id)
        output(s,job_id)

    if cmd == 'delete':
        cancel(job_id)
        delete(s,job_id)

    if cmd == 'workspace':
        workspace(s)

    if cmd == 'update':
        update(job_id)

<|endoftext|>