<commit_before># -*- coding: utf-8 -*-
import calendar
import hashlib
import json
from multiprocessing import Process
import os
import shlex
import subprocess
from subprocess import PIPE
import sys
from datetime import datetime
import time
import collections
import traceback
from com.dtmilano.android.viewclient import ViewClient
import re

class ApkChecker(object):
    def __init__(self, conf_file):
        # initialize check_result dict
        self.result = collections.defaultdict(lambda: collections.defaultdict(dict))
        self.result['running_log'] = []
        self._check_not_finished()
        self._check_not_passed()
        self.conf_data = self.read_conf(conf_file)

        try:
            self.apk_file = self.conf_data['apk_file']
            self.serialno = self.conf_data['serialno']
            self.screenshot_path = self.conf_data['screenshot_path']
            self.log_verbose = self.conf_data['log_verbose']
        except KeyError:
            self._error_log(traceback.format_exc())

        # get apk file basic info
        self.get_apk_info()
        self.package = self.result['apk_result']['package_name']
        self.activity = self.result['apk_result']['launch_activity']
        # begin connect to phone
        self.adb = self.connect()

    def read_conf(self, conf_file):
        if not os.path.exists(conf_file):
            self._error_log('apk conf_file: {0} not found'.format(conf_file))
        return json.load(open(conf_file))

    def get_apk_info(self):
        if not os.path.exists(self.apk_file):
            self._error_log('apk file: {0} not found'.format(self.apk_file))
        # get apk file info
        self.result['apk_result']['file_name'] = os.path.basename(self.apk_file)
        self.result['apk_result']['file_size'] = os.path.getsize(self.apk_file)
        self.result['apk_result']['file_md5'] = hashlib.md5(open(self.apk_file, 'rb').read()).hexdigest()
        # dump apk package info
        ret = self._run_wrapper('aapt dump badging {0}'.format(self.apk_file))
        # get apk package info
        self.result['apk_result']['package_name'] = re.search("(?<=package: name=')[^']+", ret).group()
        self.result['apk_result']['version_code'] = re.search("(?<=versionCode=')[^']+", ret).group()
        self.result['apk_result']['version_name'] = re.search("(?<=versionName=')[^']+", ret).group()
        self.result['apk_result']['launch_activity'] = re.search("(?<=launchable-activity: name=')[^']+", ret).group()

    def connect(self):
        # check devices is connected
        ret = self._run_wrapper('adb devices')
        if self.serialno not in ret:
            self._error_log('device {0} not connected'.format(self.serialno))
        # connecting to device
        try:
            _adb, _serialno = ViewClient.connectToDeviceOrExit(verbose=False, serialno=self.serialno)
            return _adb
        except Exception as e:
            self._error_log('connecting to Android Device {0} Failed: {1}'.format(self.serialno, e))

    def run_check(self):
        self.unlock_device()
        # self.install_apk()
        self.gather_info()
        self.start_logcat_daemon()
        self.start_app()
        while not self._is_check_finished():
            self.gather_info()
            print "Parent: {0}".format(self._is_check_finished())
        self.lock_device()
        self._save_result()

    def unlock_device(self):
        try:
            # wake screen on
            self.adb.wake()
            # try unlock screen
            if self.adb.isLocked():
                (w, h) = (self.adb.getProperty("display.width"), self.adb.getProperty("display.height"))
                self.adb.drag((w * 0.5, h * 0.7), (w, h * 0.7))
        except Exception as e:
            self._error_log('device not support screen unlock: {0}'.format(e))

    def lock_device(self):
        if self.adb.isScreenOn():
            self.adb.shell('input keyevent POWER')

    def install_apk(self):
        ret = self._run_wrapper('adb -s {0} install {1}'.format(self.serialno, self.apk_file))
        if "Failure " in ret:
            self._error_log('install {0} to device {1} failed: {2}'.format(self.apk_file, self.serialno, ret))

    def start_app(self):
        self._run_wrapper('adb shell am start -n {0}/{1}'.format(self.package, self.activity))

    def is_app_alive(self):
        return True if self.package in self.adb.shell('ps') else False

    def gather_info(self):
        timestamp = self.get_timestamp()
        self._data_log(timestamp=timestamp, cpu_data=self.get_cpu_data(), mem_data=self.get_mem_data(),
                       screenshot=self.take_screenshot(timestamp))

    def get_mem_data(self):
        ret = self.adb.shell('dumpsys meminfo {0}'.format(self.package))
        mem_regex = re.search("(?<=TOTAL)\s+\d+", ret)
        mem = mem_regex.group().lstrip() if mem_regex else 0
        return round(float(mem) / 1024, 2)

    def get_cpu_data(self):
        ret = self.adb.shell('dumpsys cpuinfo')
        cpu_line = filter(lambda line: self.package in line, ret.splitlines())
        if cpu_line:
            cpu_regex = re.search("\d+(?=% )", cpu_line[0])
            cpu = cpu_regex.group()
        else:
            cpu = 0
        return round(float(cpu), 2)

    def take_screenshot(self, timestamp):
        full_file_path = '{0}/{1}.png'.format(self.screenshot_path, timestamp)
        self.adb.takeSnapshot(reconnect=True).save(full_file_path, 'PNG')
        return '{0}.png'.format(timestamp)

    def start_logcat_daemon(self):
        logcat_watcher = Process(target=self.watch_logcat)
        logcat_watcher.daemon = True
        logcat_watcher.start()

    def watch_logcat(self):
        logcat = self.start_logcat()
        for i in range(1, 10):
            print "watching logcat: {0}".format(i)
            time.sleep(1)
        print "changing varible..."
        self._check_finished()
        print "changing varible done..:{0}".format(self._is_check_finished())

    def start_logcat(self):
        # clear log before starting logcat
        adb_clear_cmd = shlex.split('adb -s {0} logcat -c'.format(self.serialno))
        adb_clear = subprocess.Popen(adb_clear_cmd)
        while adb_clear.poll() is None:
            pass
        adb_logcat_cmd = shlex.split('adb -s {0} logcat'.format(self.serialno))
        return subprocess.Popen(adb_logcat_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)

    def _run_wrapper(self, cmd):
        ret = self.run_cmd(cmd)
        if ret['retcode'] != 0:
            self._error_log('command: {0} execution failed, return: {0}'.format(cmd, ret['retval']))
        self._cmd_log(cmd, ret['retval'])
        return ret['retval']

    @staticmethod
    def run_cmd(cmd, cwd=None, daemon=False):
        args = map(lambda s: s.decode('utf-8'), shlex.split(cmd.encode('utf-8')))
        child = subprocess.Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
        if not daemon:
            stdout, stderr = child.communicate()
            retcode = child.returncode
            if retcode is 0:
                result = {'retcode': retcode, 'retval': stdout}
            else:
                result = {'retcode': retcode, 'retval': stderr}
            return result
        else:
            return child.pid

    @staticmethod
    def get_timestamp():
        return calendar.timegm(datetime.now().utctimetuple())

    def _error_log(self, content):
        log_content = {
            'timestamp': self.get_timestamp(),
            'content': content
        }
        self._check_not_finished()
        self.result['error_log'] = log_content
        self._save_result()
        print >> sys.stderr, log_content
        sys.exit(1)

    def _cmd_log(self, cmd, ret, level='v'):
        log_content = {
            'timestamp': self.get_timestamp(),
            'type': 'cmd',
            'level': level,
            'cmd': cmd,
            'ret': ret
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _logcat_data(self, tag, text, level):
        log_content = {
            'timestamp': self.get_timestamp(),
            'type': 'data',
            'tag': tag,
            'text': text,
            'level': level
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _data_log(self, timestamp, cpu_data, mem_data, screenshot):
        log_content = {
            'timestamp': timestamp,
            'type': 'data',
            'cpu_data': cpu_data,
            'mem_data': mem_data,
            'screenshot': screenshot
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _is_check_finished(self):
        return self.result['apk_result']['finished']

    def _check_finished(self):
        self.result['apk_result']['finished'] = 1

    def _check_not_finished(self):
        self.result['apk_result']['finished'] = 0

    def _check_passed(self):
        self.result['apk_result']['passed'] = 1

    def _check_not_passed(self):
        self.result['apk_result']['passed'] = 0

    def _save_result(self, filename='check_result.json'):
        with open(filename, 'w') as outfile:
            json.dump(self.result, outfile, sort_keys=True, indent=4, separators=(',', ': '))

if __name__ == '__main__':
    apk_checker = ApkChecker('test_conf.json')
    apk_checker.run_check()
<commit_msg>add logcat process<commit_after># -*- coding: utf-8 -*-
import calendar
import hashlib
import json
from multiprocessing import Process, Manager
import os
import shlex
import subprocess
from subprocess import PIPE
import sys
from datetime import datetime
import time
import collections
import traceback
from com.dtmilano.android.viewclient import ViewClient
import re

class ApkChecker(object):
    def __init__(self, conf_file):
        # initialize check_result dict
        self.result = collections.defaultdict(lambda: collections.defaultdict(dict))
        self.result['running_log'] = []
        self._check_not_finished()
        self._check_not_passed()
        self.conf_data = self.read_conf(conf_file)
        # manipulate locat data between process
        manager = Manager()
        self.logcat_data = manager.dict()

        try:
            self.apk_file = self.conf_data['apk_file']
            self.serialno = self.conf_data['serialno']
            self.screenshot_path = self.conf_data['screenshot_path']
            self.log_verbose = self.conf_data['log_verbose']
        except KeyError:
            self._error_log(traceback.format_exc())

        # get apk file basic info
        self.get_apk_info()
        self.package = self.result['apk_result']['package_name']
        self.activity = self.result['apk_result']['launch_activity']
        # begin connect to phone
        self.adb = self.connect()

    def read_conf(self, conf_file):
        if not os.path.exists(conf_file):
            self._error_log('apk conf_file: {0} not found'.format(conf_file))
        return json.load(open(conf_file))

    def get_apk_info(self):
        if not os.path.exists(self.apk_file):
            self._error_log('apk file: {0} not found'.format(self.apk_file))
        # get apk file info
        self.result['apk_result']['file_name'] = os.path.basename(self.apk_file)
        self.result['apk_result']['file_size'] = os.path.getsize(self.apk_file)
        self.result['apk_result']['file_md5'] = hashlib.md5(open(self.apk_file, 'rb').read()).hexdigest()
        # dump apk package info
        ret = self._run_wrapper('aapt dump badging {0}'.format(self.apk_file))
        # get apk package info
        self.result['apk_result']['package_name'] = re.search("(?<=package: name=')[^']+", ret).group()
        self.result['apk_result']['version_code'] = re.search("(?<=versionCode=')[^']+", ret).group()
        self.result['apk_result']['version_name'] = re.search("(?<=versionName=')[^']+", ret).group()
        self.result['apk_result']['launch_activity'] = re.search("(?<=launchable-activity: name=')[^']+", ret).group()

    def connect(self):
        # check devices is connected
        ret = self._run_wrapper('adb devices')
        if self.serialno not in ret:
            self._error_log('device {0} not connected'.format(self.serialno))
        # connecting to device
        try:
            _adb, _serialno = ViewClient.connectToDeviceOrExit(verbose=False, serialno=self.serialno)
            return _adb
        except Exception as e:
            self._error_log('connecting to Android Device {0} Failed: {1}'.format(self.serialno, e))

    def run_check(self):
        self.unlock_device()
        # self.install_apk()
        self.gather_info()
        logcat_watcher = self.start_logcat_daemon()
        self.start_app()
        while logcat_watcher.is_alive():
            self.gather_info()
        print self.logcat_data
        self.lock_device()
        self._save_result()

    def unlock_device(self):
        try:
            # wake screen on
            self.adb.wake()
            # try unlock screen
            if self.adb.isLocked():
                (w, h) = (self.adb.getProperty("display.width"), self.adb.getProperty("display.height"))
                self.adb.drag((w * 0.5, h * 0.7), (w, h * 0.7))
        except Exception as e:
            self._error_log('device not support screen unlock: {0}'.format(e))

    def lock_device(self):
        if self.adb.isScreenOn():
            self.adb.shell('input keyevent POWER')

    def install_apk(self):
        ret = self._run_wrapper('adb -s {0} install {1}'.format(self.serialno, self.apk_file))
        if "Failure " in ret:
            self._error_log('install {0} to device {1} failed: {2}'.format(self.apk_file, self.serialno, ret))

    def start_app(self):
        self._run_wrapper('adb shell am start -n {0}/{1}'.format(self.package, self.activity))

    def is_app_alive(self):
        return True if self.package in self.adb.shell('ps') else False

    def gather_info(self):
        timestamp = self.get_timestamp()
        self._data_log(timestamp=timestamp, cpu_data=self.get_cpu_data(), mem_data=self.get_mem_data(),
                       screenshot=self.take_screenshot(timestamp))

    def get_mem_data(self):
        ret = self.adb.shell('dumpsys meminfo {0}'.format(self.package))
        mem_regex = re.search("(?<=TOTAL)\s+\d+", ret)
        mem = mem_regex.group().lstrip() if mem_regex else 0
        return round(float(mem) / 1024, 2)

    def get_cpu_data(self):
        ret = self.adb.shell('dumpsys cpuinfo')
        cpu_line = filter(lambda line: self.package in line, ret.splitlines())
        if cpu_line:
            cpu_regex = re.search("\d+(?=% )", cpu_line[0])
            cpu = cpu_regex.group()
        else:
            cpu = 0
        return round(float(cpu), 2)

    def take_screenshot(self, timestamp):
        full_file_path = '{0}/{1}.png'.format(self.screenshot_path, timestamp)
        self.adb.takeSnapshot(reconnect=True).save(full_file_path, 'PNG')
        return '{0}.png'.format(timestamp)

    def start_logcat_daemon(self):
        logcat_watcher = Process(target=self.watch_logcat, args=(self.logcat_data,))
        logcat_watcher.daemon = True
        logcat_watcher.start()
        return logcat_watcher

    def watch_logcat(self, logcat_data):
        logcat_subp = self.start_logcat()
        for i in range(1, 10):
            logcat_data[i] = i
            time.sleep(1)

    def start_logcat(self):
        # clear log before starting logcat
        adb_clear_cmd = shlex.split('adb -s {0} logcat -c'.format(self.serialno))
        adb_clear = subprocess.Popen(adb_clear_cmd)
        while adb_clear.poll() is None:
            pass
        adb_logcat_cmd = shlex.split('adb -s {0} logcat'.format(self.serialno))
        return subprocess.Popen(adb_logcat_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)

    def _run_wrapper(self, cmd):
        ret = self.run_cmd(cmd)
        if ret['retcode'] != 0:
            self._error_log('command: {0} execution failed, return: {0}'.format(cmd, ret['retval']))
        self._cmd_log(cmd, ret['retval'])
        return ret['retval']

    @staticmethod
    def run_cmd(cmd, cwd=None, daemon=False):
        args = map(lambda s: s.decode('utf-8'), shlex.split(cmd.encode('utf-8')))
        child = subprocess.Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
        if not daemon:
            stdout, stderr = child.communicate()
            retcode = child.returncode
            if retcode is 0:
                result = {'retcode': retcode, 'retval': stdout}
            else:
                result = {'retcode': retcode, 'retval': stderr}
            return result
        else:
            return child.pid

    @staticmethod
    def get_timestamp():
        return calendar.timegm(datetime.now().utctimetuple())

    def _error_log(self, content):
        log_content = {
            'timestamp': self.get_timestamp(),
            'content': content
        }
        self._check_not_finished()
        self.result['error_log'] = log_content
        self._save_result()
        print >> sys.stderr, log_content
        sys.exit(1)

    def _cmd_log(self, cmd, ret, level='v'):
        log_content = {
            'timestamp': self.get_timestamp(),
            'type': 'cmd',
            'level': level,
            'cmd': cmd,
            'ret': ret
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _logcat_data(self, tag, text, level):
        log_content = {
            'timestamp': self.get_timestamp(),
            'type': 'data',
            'tag': tag,
            'text': text,
            'level': level
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _data_log(self, timestamp, cpu_data, mem_data, screenshot):
        log_content = {
            'timestamp': timestamp,
            'type': 'data',
            'cpu_data': cpu_data,
            'mem_data': mem_data,
            'screenshot': screenshot
        }
        self.result['running_log'].append(log_content)
        if self.log_verbose:
            print >> sys.stdout, log_content

    def _check_finished(self):
        self.result['apk_result']['finished'] = 1

    def _check_not_finished(self):
        self.result['apk_result']['finished'] = 0

    def _check_passed(self):
        self.result['apk_result']['passed'] = 1

    def _check_not_passed(self):
        self.result['apk_result']['passed'] = 0

    def _save_result(self, filename='check_result.json'):
        with open(filename, 'w') as outfile:
            json.dump(self.result, outfile, sort_keys=True, indent=4, separators=(',', ': '))

if __name__ == '__main__':
    apk_checker = ApkChecker('test_conf.json')
    apk_checker.run_check()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Created 19-Mar-2013 by Daniel Margala (University of California, Irvine) <dmargala@uci.edu>
#
# usage:

# Import libraries
import argparse
import math
from datetime import datetime

import numpy as np

import os
import yanny
from astropy.io import fits

from astropy import units as u
from astropy.time import Time
from astropy.coordinates import EarthLocation
from astropy.coordinates import Angle

# APO Geographical Location
apolat = Angle('32d46m49s')
apolon = Angle('-105d49m13s')
apo = EarthLocation.from_geodetic(apolon, apolat)

def equatorial_to_horizontal(ra, dec, lat, ha):
    sin_alt = np.sin(dec)*np.sin(lat) + np.cos(dec)*np.cos(lat)*np.cos(ha)
    alt = np.arcsin(sin_alt)
    cos_az = (np.sin(dec) - sin_alt*np.sin(lat))/(np.cos(alt)*np.cos(lat))
    if np.sin(ha) < 0:
        az = np.arccos(cos_az)
    else:
        az = 2*np.pi*u.radian - np.arccos(cos_az)
    return alt, az

def examine_exposure(info, cframe, cframe_keys):
    # Process CFrame header keywords
    for keyword in cframe_keys:
        info[keyword] = cframe.header[keyword]

    obs_ra = cframe.header['RADEG']
    obs_dec = cframe.header['DECDEG']
    taibeg = cframe.header['TAI-BEG']
    taiend = cframe.header['TAI-END']

    taimid = 0.5*(taibeg+taiend)
    dec = Angle(obs_dec, u.degree)
    ra = Angle(obs_ra, u.degree)

    time = Time(taimid/86400.0, format='mjd', scale='tai', location=apo)
    lst = time.sidereal_time('apparent')
    ha = (lst - ra)

    alt, az = equatorial_to_horizontal(ra, dec, apolat, ha)

    info['mean_alt'] = alt.to(u.degree).value
    if ha > np.pi*u.radian:
        ha -= 2*np.pi*u.radian
    info['mean_ha'] = ha.to(u.degree).value

class Fluxcalib(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.data = hdulist[0].data
        hdulist.close()

class CFrame(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.flux = hdulist[0].data
        self.ivar = hdulist[1].data
        self.mask = hdulist[2].data
        self.loglam = hdulist[3].data
        self.wdisp = hdulist[4].data
        self.sky = hdulist[6].data
        hdulist.close()

class Plate(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.flux = hdulist[0].data
        self.ivar = hdulist[1].data
        self.andmask = hdulist[2].data
        self.ormask = hdulist[3].data
        self.wdisp = hdulist[4].data
        self.sky = hdulist[6].data
        hdulist.close()

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-v','--verbose', action='store_true', help='provide more verbose output')
    parser.add_argument('-p', '--plate', type=str, help='Plate')
    parser.add_argument('-m', '--mjd', type=str, help='MJD')
    parser.add_argument('--bossdir', type=str, help='Input default boss reduction $BOSS_SPECTRO_REDUX/$RUN2D')
    parser.add_argument('-i', '--input', type=str, help='Input plate list file to read')
    parser.add_argument('-d', '--delim', type=str, default=' ', help='Table deliminator')
    parser.add_argument('--speclog', type=str, help='Speclog base dir $SPECLOG_DIR')
    parser.add_argument('-o', '--output', type=str, default='', help='Output prefix')

    args = parser.parse_args()

    if not (args.input or (args.plate and args.mjd)):
        print 'Must specify a plate list file to read or a specific plate and mjd pair!'
        return -1

    plate_keys = ['SEEING50', 'RMSOFF50', 'AIRMASS', 'ALT', 'BESTEXP', 'NSTD']
    plugmap_keys = ['haMin']#,'cartridgeId']#,'raCen','decCen']
    cframe_keys = ['MJD', 'SEEING50', 'RMSOFF50', 'AIRMASS', 'ALT']
    new_exp_keys = ['id', 'mean_alt', 'mean_ha']
    new_plate_keys = ['plate', 'mjd', 'design_alt']

    plate_mjd_list = []
    if args.input:
        with open(args.input) as platelist:
            for line in platelist:
                plate_mjd_list.append(line.strip().split())
    else:
        plate_mjd_list.append((args.plate, args.mjd))

    mean_psf_fwhm = []
    mean_ha = []
    mean_alt = []

    for plate, mjd in plate_mjd_list:
        plate_name = os.path.join(args.bossdir, str(plate), 'spPlate-%s-%s.fits' % (plate, mjd))
        spPlate = Plate(plate_name)

        plan_name = os.path.join(args.bossdir, str(plate), 'spPlancomb-%s-%s.par' % (plate, mjd))
        plan = yanny.yanny(plan_name)
        unique_mapnames = set(zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
        (mapmjd, mapname) = unique_mapnames.pop()
        plugmap_name = os.path.join(args.speclog, str(mapmjd), 'plPlugMapM-%s.par' % mapname)
        plugmap = yanny.yanny(plugmap_name)

        # Construct list of exposure IDs
        exposures = list()
        nexp = spPlate.header['NEXP']
        if (nexp % 4) != 0:
            print 'Warning, unexpected number of exposures...'
        for i in range(nexp/4):
            exposures.append('-'.join(spPlate.header['EXPID%02d'%(i+1)].split('-')[0:2]))

        plate_info = {}
        plate_info['plate'] = plate
        plate_info['mjd'] = mjd
        plate_info['exposures'] = exposures
        # Process Plate header keywords
        for keyword in plate_keys:
            plate_info[keyword] = spPlate.header[keyword]
        # Process plugmap header keywords
        for keyword in plugmap_keys:
            plate_info[keyword] = plugmap[keyword]

        design_ra = Angle(float(plugmap['raCen']), unit=u.degree)
        design_dec = Angle(float(plugmap['decCen']), unit=u.degree)
        design_ha = Angle(float(plugmap['haMin'])%360, unit=u.degree)
        design_alt, design_az = equatorial_to_horizontal(design_ra, design_dec, apolat, design_ha)
        plate_info['design_alt'] = design_alt.to(u.degree).value

        # Iterate over exposures and gather information of interest
        psf_fwhm_list = []
        ha_list = []
        alt_list = []

        print args.delim.join([str(plate_info[key]) for key in (new_plate_keys + plate_keys + plugmap_keys)])

        for exposure in exposures:
            exposure_info = dict()
            exposure_info['id'] = exposure

            cframe = CFrame(os.path.join(args.bossdir, str(plate), 'spCFrame-%s.fits' % exposure))

            examine_exposure(exposure_info, cframe, cframe_keys)

            print '\t', args.delim.join([str(exposure_info[key]) for key in (new_exp_keys + cframe_keys)])

            psf_fwhm_list.append(exposure_info['SEEING50'])
            ha_list.append(exposure_info['mean_ha'])
            alt_list.append(exposure_info['mean_alt'])

        print np.mean(psf_fwhm_list), np.mean(ha_list), np.mean(alt_list)

        mean_psf_fwhm.append(np.mean(psf_fwhm_list))
        mean_ha.append(np.mean(ha_list))
        mean_alt.append(np.mean(alt_list))

    if args.output:
        with open(args.output, 'w') as output:
            for i, (plate, mjd) in enumerate(plate_mjd_list):
                output.write('%s %s %.4f %.4f %.4f\n' % (plate, mjd, mean_ha[i], mean_psf_fwhm[i], mean_alt[i]))

if __name__ == '__main__':
    main()

<commit_msg>clean up argument help strings and default values<commit_after>#!/usr/bin/env python
#
# Created 19-Mar-2013 by Daniel Margala (University of California, Irvine) <dmargala@uci.edu>
#
# usage:

# ./python/exposure_query.py --verbose --input work/blue-plate-mjd-list.txt --output work/plate-mjd-ha-psf.txt

# Import libraries
import argparse
import math
from datetime import datetime

import numpy as np

import os
import yanny
from astropy.io import fits

from astropy import units as u
from astropy.time import Time
from astropy.coordinates import EarthLocation
from astropy.coordinates import Angle

# APO Geographical Location
apolat = Angle('32d46m49s')
apolon = Angle('-105d49m13s')
apo = EarthLocation.from_geodetic(apolon, apolat)

def equatorial_to_horizontal(ra, dec, lat, ha):
    sin_alt = np.sin(dec)*np.sin(lat) + np.cos(dec)*np.cos(lat)*np.cos(ha)
    alt = np.arcsin(sin_alt)
    cos_az = (np.sin(dec) - sin_alt*np.sin(lat))/(np.cos(alt)*np.cos(lat))
    if np.sin(ha) < 0:
        az = np.arccos(cos_az)
    else:
        az = 2*np.pi*u.radian - np.arccos(cos_az)
    return alt, az

def examine_exposure(info, cframe, cframe_keys):
    # Process CFrame header keywords
    for keyword in cframe_keys:
        info[keyword] = cframe.header[keyword]

    obs_ra = cframe.header['RADEG']
    obs_dec = cframe.header['DECDEG']
    taibeg = cframe.header['TAI-BEG']
    taiend = cframe.header['TAI-END']

    taimid = 0.5*(taibeg+taiend)
    dec = Angle(obs_dec, u.degree)
    ra = Angle(obs_ra, u.degree)

    time = Time(taimid/86400.0, format='mjd', scale='tai', location=apo)
    lst = time.sidereal_time('apparent')
    ha = (lst - ra)

    alt, az = equatorial_to_horizontal(ra, dec, apolat, ha)

    info['mean_alt'] = alt.to(u.degree).value
    if ha > np.pi*u.radian:
        ha -= 2*np.pi*u.radian
    info['mean_ha'] = ha.to(u.degree).value

class Fluxcalib(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.data = hdulist[0].data
        hdulist.close()

class CFrame(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.flux = hdulist[0].data
        self.ivar = hdulist[1].data
        self.mask = hdulist[2].data
        self.loglam = hdulist[3].data
        self.wdisp = hdulist[4].data
        self.sky = hdulist[6].data
        hdulist.close()

class Plate(object):
    def __init__(self, name):
        hdulist = fits.open(name)
        self.header = hdulist[0].header
        self.flux = hdulist[0].data
        self.ivar = hdulist[1].data
        self.andmask = hdulist[2].data
        self.ormask = hdulist[3].data
        self.wdisp = hdulist[4].data
        self.sky = hdulist[6].data
        hdulist.close()

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-v','--verbose', action='store_true',
        help='provide more verbose output')
    parser.add_argument('-p', '--plate', type=str, default=None,
        help='specify plate number for single plate-mjd query')
    parser.add_argument('-m', '--mjd', type=str, default=None,
        help='specify mjd for single plate-mjd query')
    parser.add_argument('-i', '--input', type=str, default=None
        help='specify file with plate,mjd entries to query')
    parser.add_argument('--bossdir', type=str, default='/clusterfs/riemann/raid008/bosswork/boss/spectro/redux/v5_7_0',
        help='path to 2D reduction dir ($BOSS_SPECTRO_REDUX/$RUN2D)')
    parser.add_argument('--speclog', type=str, default='/home/boss/products/NULL/speclog/trunk',
        help='path to speclog base dir ($SPECLOG_DIR)')
    parser.add_argument('-d', '--delim', type=str, default=' ',
        help='output token deliminator (useful for making tables)')
    parser.add_argument('-o', '--output', type=str, default=None,
        help='output filename')
    args = parser.parse_args()

    if not (args.input or (args.plate and args.mjd)):
        print 'Must specify a plate list file to read or a specific plate and mjd pair!'
        return -1

    plate_keys = ['SEEING50', 'RMSOFF50', 'AIRMASS', 'ALT', 'BESTEXP', 'NSTD']
    plugmap_keys = ['haMin']
    cframe_keys = ['MJD', 'SEEING50', 'RMSOFF50', 'AIRMASS', 'ALT']
    new_exp_keys = ['id', 'mean_alt', 'mean_ha']
    new_plate_keys = ['plate', 'mjd', 'design_alt']

    plate_mjd_list = []
    if args.input:
        with open(args.input) as platelist:
            for line in platelist:
                plate_mjd_list.append(line.strip().split())
    else:
        plate_mjd_list.append((args.plate, args.mjd))

    mean_psf_fwhm = []
    mean_ha = []
    mean_alt = []

    for plate, mjd in plate_mjd_list:
        plate_name = os.path.join(args.bossdir, str(plate), 'spPlate-%s-%s.fits' % (plate, mjd))
        spPlate = Plate(plate_name)

        plan_name = os.path.join(args.bossdir, str(plate), 'spPlancomb-%s-%s.par' % (plate, mjd))
        plan = yanny.yanny(plan_name)
        unique_mapnames = set(zip(plan['SPEXP']['mjd'], plan['SPEXP']['mapname']))
        (mapmjd, mapname) = unique_mapnames.pop()
        plugmap_name = os.path.join(args.speclog, str(mapmjd), 'plPlugMapM-%s.par' % mapname)
        plugmap = yanny.yanny(plugmap_name)

        # Construct list of exposure IDs
        exposures = list()
        nexp = spPlate.header['NEXP']
        if (nexp % 4) != 0:
            print 'Warning, unexpected number of exposures...'
        for i in range(nexp/4):
            exposures.append('-'.join(spPlate.header['EXPID%02d'%(i+1)].split('-')[0:2]))

        plate_info = {}
        plate_info['plate'] = plate
        plate_info['mjd'] = mjd
        plate_info['exposures'] = exposures
        # Process Plate header keywords
        for keyword in plate_keys:
            plate_info[keyword] = spPlate.header[keyword]
        # Process plugmap header keywords
        for keyword in plugmap_keys:
            plate_info[keyword] = plugmap[keyword]

        design_ra = Angle(float(plugmap['raCen']), unit=u.degree)
        design_dec = Angle(float(plugmap['decCen']), unit=u.degree)
        design_ha = Angle(float(plugmap['haMin'])%360, unit=u.degree)
        design_alt, design_az = equatorial_to_horizontal(design_ra, design_dec, apolat, design_ha)
        plate_info['design_alt'] = design_alt.to(u.degree).value

        # Iterate over exposures and gather information of interest
        psf_fwhm_list = []
        ha_list = []
        alt_list = []

        if args.verbose:
            print args.delim.join([str(plate_info[key]) for key in (new_plate_keys + plate_keys + plugmap_keys)])

        for exposure in exposures:
            exposure_info = dict()
            exposure_info['id'] = exposure

            cframe = CFrame(os.path.join(args.bossdir, str(plate), 'spCFrame-%s.fits' % exposure))

            examine_exposure(exposure_info, cframe, cframe_keys)

            if args.verbose:
                print '\t', args.delim.join([str(exposure_info[key]) for key in (new_exp_keys + cframe_keys)])

            psf_fwhm_list.append(exposure_info['SEEING50'])
            ha_list.append(exposure_info['mean_ha'])
            alt_list.append(exposure_info['mean_alt'])

        if args.verbose:
            print np.mean(psf_fwhm_list), np.mean(ha_list), np.mean(alt_list)

        mean_psf_fwhm.append(np.mean(psf_fwhm_list))
        mean_ha.append(np.mean(ha_list))
        mean_alt.append(np.mean(alt_list))

    if args.output:
        with open(args.output, 'w') as output:
            for i, (plate, mjd) in enumerate(plate_mjd_list):
                output.write('%s %s %.4f %.4f %.4f\n' % (plate, mjd, mean_ha[i], mean_psf_fwhm[i], mean_alt[i]))

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import serial
import time

# this is the port you're connecting to the irobot with
# on windows, this is something like COM2
N = 2

def ints2str(lst):
    '''
    Taking a list of notes/lengths, convert it to a string
    '''
    s = ""
    for i in lst:
        if i < 0 or i > 255:
            raise Exception
        s = s + str(chr(i))
    return s

# do some initialization magic
s = serial.Serial(N, 57600, timeout=4)
print("sending start...")
s.write(ints2str([128]))
print("switch to full mode...")
s.write(ints2str([132]))

# define silence
r = 30

# map note names in the lilypad notation to irobot commands
c4 = 60
cis4 = des4 = 61
d4 = 62
dis4 = ees4 = 63
e4 = 64
f4 = 65
fis4 = ges4 = 66
g4 = 67
gis4 = aes4 = 68
a4 = 69
ais4 = bes4 = 70
b4 = 71
c5 = 72
cis5 = des5 = 73
d5 = 74
dis5 = ees5 = 75
e5 = 76
f5 = 77
fis5 = ges5 = 78
g5 = 79
gis5 = aes5 = 80
a5 = 81
ais5 = bes5 = 82
b5 = 83
c6 = 84
cis6 = des6 = 85
d6 = 86
dis6 = ees6 = 87
e6 = 88
f6 = 89
fis6 = ges6 = 90

# define some note lengths
# change the top MEASURE (4/4 time) to get faster/slower speeds
MEASURE = 160
HALF = MEASURE/2
Q = MEASURE/4
E = MEASURE/8
Ed = MEASURE*3/16
S = MEASURE/16

MEASURE_TIME = MEASURE/64.

# send song
# [140, num, len, (note, dur)_1, ...]
# durations are multiples of 1/64
# a4 a a f8. c'16 | a4 f8. c'16 a2
# e2 e e f8. c'16 | aes4 f8. c'16 a2
# a'4 a,8. a16 a'4 aes8 g | ges16 f g8 r8 bes, ees4 d8 des
# c16 b c8 r8 f,8 aes4 f8. aes16 | c4 a8. c16 e2
# a4 a,8. a16 a'4 aes8 g | ges16 f g8 r8 bes, ees4 d8 des
# c16 b c8 r8 f,8 aes4 f8. c'16 | a4 f8. c,16 a2
# 40/64 bps
print("send songs...")
# first upload the songs to the irobot...
s.write(ints2str([140, 0, 9,
                  a4,Q, a4,Q, a4,Q, f4,Ed, c5,S,
                  a4,Q, f4,Ed, c5,S, a4,HALF]))
s.write(ints2str([140, 1, 9,
                  e5,Q, e5,Q, e5,Q, f5,Ed, c5,S,
                  aes4,Q, f4,Ed, c5,S, a4,HALF]))
s.write(ints2str([140, 2, 9,
                  a5,Q, a4,Ed, a4,S, a5,Q, aes5,E, g5,E,
                  ges5,S, f5,S, ges5,S]))
s.write(ints2str([140, 3, 8,
                  r,E, bes4,E, ees5,Q, d5,E, des5,E,
                  c5,S, b4,S, c5,E]))
s.write(ints2str([140, 4, 9,
                  r,E, f4,E, aes4,Q, f4,Ed, aes4,S,
                  c5,Q, a4,Ed, c5,S, e5,HALF]))
# play 2 again
# play 3 again
s.write(ints2str([140, 5, 9,
                  r,E, f4,E, aes4,Q, f4,Ed, c5,S,
                  a4,Q, f4,Ed, c5,S, a4,HALF]))

# once all the songs are uploaded, play them at the right times
# add a little extra time, b/c otherwise cuts off the end
print("play songs...")
s.write(ints2str([141, 0]))
time.sleep(MEASURE_TIME*2.01)

s.write(ints2str([141, 1]))
time.sleep(MEASURE_TIME*2.01)

s.write(ints2str([141, 2]))
time.sleep(MEASURE_TIME*1.26)

s.write(ints2str([141, 3]))
time.sleep(MEASURE_TIME*1.01)

s.write(ints2str([141, 4]))
time.sleep(MEASURE_TIME*1.76)

s.write(ints2str([141, 2]))
time.sleep(MEASURE_TIME*1.26)

s.write(ints2str([141, 3]))
time.sleep(MEASURE_TIME*1.01)

s.write(ints2str([141, 5]))
time.sleep(MEASURE_TIME*1.76)<commit_msg>Update imperial_march.py<commit_after>#! /usr/bin/env python
import serial
import time

# this is the port you're connecting to the irobot with
# on windows, this is something like COM2
N = "/dev/ttyUSB0"

def ints2str(lst):
    '''
    Taking a list of notes/lengths, convert it to a string
    '''
    s = ""
    for i in lst:
        if i < 0 or i > 255:
            raise Exception
        s = s + str(chr(i))
    return s

# do some initialization magic
s = serial.Serial(N, 57600, timeout=4)
print("sending start...")
s.write(ints2str([128]))
print("switch to full mode...")
s.write(ints2str([132]))

# define silence
r = 30

# map note names in the lilypad notation to irobot commands
c4 = 60
cis4 = des4 = 61
d4 = 62
dis4 = ees4 = 63
e4 = 64
f4 = 65
fis4 = ges4 = 66
g4 = 67
gis4 = aes4 = 68
a4 = 69
ais4 = bes4 = 70
b4 = 71
c5 = 72
cis5 = des5 = 73
d5 = 74
dis5 = ees5 = 75
e5 = 76
f5 = 77
fis5 = ges5 = 78
g5 = 79
gis5 = aes5 = 80
a5 = 81
ais5 = bes5 = 82
b5 = 83
c6 = 84
cis6 = des6 = 85
d6 = 86
dis6 = ees6 = 87
e6 = 88
f6 = 89
fis6 = ges6 = 90

# define some note lengths
# change the top MEASURE (4/4 time) to get faster/slower speeds
MEASURE = 160
HALF = MEASURE/2
Q = MEASURE/4
E = MEASURE/8
Ed = MEASURE*3/16
S = MEASURE/16

MEASURE_TIME = MEASURE/64.

# send song
# [140, num, len, (note, dur)_1, ...]
# durations are multiples of 1/64
# a4 a a f8. c'16 | a4 f8. c'16 a2
# e2 e e f8. c'16 | aes4 f8. c'16 a2
# a'4 a,8. a16 a'4 aes8 g | ges16 f g8 r8 bes, ees4 d8 des
# c16 b c8 r8 f,8 aes4 f8. aes16 | c4 a8. c16 e2
# a4 a,8. a16 a'4 aes8 g | ges16 f g8 r8 bes, ees4 d8 des
# c16 b c8 r8 f,8 aes4 f8. c'16 | a4 f8. c,16 a2
# 40/64 bps
print("send songs...")
# first upload the songs to the irobot...
s.write(ints2str([140, 0, 9,
                  a4,Q, a4,Q, a4,Q, f4,Ed, c5,S,
                  a4,Q, f4,Ed, c5,S, a4,HALF]))
s.write(ints2str([140, 1, 9,
                  e5,Q, e5,Q, e5,Q, f5,Ed, c5,S,
                  aes4,Q, f4,Ed, c5,S, a4,HALF]))
s.write(ints2str([140, 2, 9,
                  a5,Q, a4,Ed, a4,S, a5,Q, aes5,E, g5,E,
                  ges5,S, f5,S, ges5,S]))
s.write(ints2str([140, 3, 8,
                  r,E, bes4,E, ees5,Q, d5,E, des5,E,
                  c5,S, b4,S, c5,E]))
s.write(ints2str([140, 4, 9,
                  r,E, f4,E, aes4,Q, f4,Ed, aes4,S,
                  c5,Q, a4,Ed, c5,S, e5,HALF]))
# play 2 again
# play 3 again
s.write(ints2str([140, 5, 9,
                  r,E, f4,E, aes4,Q, f4,Ed, c5,S,
                  a4,Q, f4,Ed, c5,S, a4,HALF]))

# once all the songs are uploaded, play them at the right times
# add a little extra time, b/c otherwise cuts off the end
print("play songs...")
s.write(ints2str([141, 0]))
time.sleep(MEASURE_TIME*2.01)

s.write(ints2str([141, 1]))
time.sleep(MEASURE_TIME*2.01)

s.write(ints2str([141, 2]))
time.sleep(MEASURE_TIME*1.26)

s.write(ints2str([141, 3]))
time.sleep(MEASURE_TIME*1.01)

s.write(ints2str([141, 4]))
time.sleep(MEASURE_TIME*1.76)

s.write(ints2str([141, 2]))
time.sleep(MEASURE_TIME*1.26)

s.write(ints2str([141, 3]))
time.sleep(MEASURE_TIME*1.01)

s.write(ints2str([141, 5]))
time.sleep(MEASURE_TIME*1.76)
<|endoftext|>


<commit_before>"""Functions to call lolo Merit classes, which describe the performance of a machine learning model

These are very similar to the "metrics" from scikit-learn, which is the reason for the name of this module.
"""

from lolopy.loloserver import get_java_gateway
from lolopy.utils import send_1D_array
import numpy as np

def _call_lolo_merit(metric_name, y_true, y_pred, y_std=None, *args):
    """Call a metric from lolopy

    Args:
        metric_name (str): Name of a Merit class (e.g., UncertaintyCorrelation)
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_uncert ([double]): Prediction uncertainties
        *args (list): Any parameters to the constructor of the Metric
    Returns:
        (double): Metric score
    """

    # If needed, set y_std to 1 for all entries
    if y_std is None:
        y_std = np.ones(len(y_true))

    # Get the metric object
    gateway = get_java_gateway()
    metric = getattr(gateway.jvm.io.citrine.lolo.validation, metric_name)(*args)

    # Convert the data arrays to Java
    y_true_java = send_1D_array(gateway, y_true, True)
    y_pred_java = send_1D_array(gateway, y_pred, True)
    y_std_java = send_1D_array(gateway, y_std, True)

    # Make the prediction result
    pred_result = gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.makeRegressionPredictionResult(y_pred_java, y_std_java)

    # Run the prediction result through the metric
    return metric.evaluate(pred_result, y_true_java)

def root_mean_squared_error(y_true, y_pred):
    """Compute the root mean squared error

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
    Returns:
        (double): RMSE
    """

    return _call_lolo_merit('RootMeanSquareError', y_true, y_pred)

def standard_confidence(y_true, y_pred, y_std):
    """Fraction of entries that have errors within the predicted confidence interval.

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
    Returns:
        (double): standard confidence
    """

    return _call_lolo_merit('StandardConfidence', y_true, y_pred, y_std)

def standard_error(y_true, y_pred, y_std, rescale=1.0):
    """Root mean square of the error divided by the predicted uncertainty

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
        rescale (double): Multiplicative factor with which to rescale error
    Returns:
        (double): standard error
    """

    return _call_lolo_merit('StandardError', y_true, y_pred, y_std, float(rescale))

def uncertainty_correlation(y_true, y_pred, y_std):
    """Measure of the correlation between the predicted uncertainty and error magnitude

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
    Returns:
        (double):
    """

    return _call_lolo_merit('UncertaintyCorrelation', y_true, y_pred, y_std)

<commit_msg>Fix lolopy uncertainty correlation test.<commit_after>"""Functions to call lolo Merit classes, which describe the performance of a machine learning model

These are very similar to the "metrics" from scikit-learn, which is the reason for the name of this module.
"""

from lolopy.loloserver import get_java_gateway
from lolopy.utils import send_1D_array
import numpy as np

def _call_lolo_merit(metric_name, y_true, y_pred, y_std=None, *args):
    """Call a metric from lolopy

    Args:
        metric_name (str): Name of a Merit class (e.g., UncertaintyCorrelation)
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_uncert ([double]): Prediction uncertainties
        *args (list): Any parameters to the constructor of the Metric
    Returns:
        (double): Metric score
    """

    # If needed, set y_std to 1 for all entries
    if y_std is None:
        y_std = np.ones(len(y_true))

    # Get the metric object
    gateway = get_java_gateway()
    metric = getattr(gateway.jvm.io.citrine.lolo.validation, metric_name)
    if len(args) > 0:
        metric = metric(*args)
    else:
        metric = metric()

    # Convert the data arrays to Java
    y_true_java = send_1D_array(gateway, y_true, True)
    y_pred_java = send_1D_array(gateway, y_pred, True)
    y_std_java = send_1D_array(gateway, y_std, True)

    # Make the prediction result
    pred_result = gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.makeRegressionPredictionResult(y_pred_java, y_std_java)

    # Run the prediction result through the metric
    return metric.evaluate(pred_result, y_true_java)

def root_mean_squared_error(y_true, y_pred):
    """Compute the root mean squared error

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
    Returns:
        (double): RMSE
    """

    return _call_lolo_merit('RootMeanSquareError', y_true, y_pred)

def standard_confidence(y_true, y_pred, y_std):
    """Fraction of entries that have errors within the predicted confidence interval.

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
    Returns:
        (double): standard confidence
    """

    return _call_lolo_merit('StandardConfidence', y_true, y_pred, y_std)

def standard_error(y_true, y_pred, y_std, rescale=1.0):
    """Root mean square of the error divided by the predicted uncertainty

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
        rescale (double): Multiplicative factor with which to rescale error
    Returns:
        (double): standard error
    """

    return _call_lolo_merit('StandardError', y_true, y_pred, y_std, float(rescale))

def uncertainty_correlation(y_true, y_pred, y_std, random_seed=None):
    """Measure of the correlation between the predicted uncertainty and error magnitude

    Args:
        y_true ([double]): True value
        y_pred ([double]): Predicted values
        y_std ([double]): Predicted uncertainty
    Returns:
        (double):
    """

    gateway = get_java_gateway()
    rng = gateway.jvm.scala.util.Random() if random_seed is None else self.gateway.jvm.scala.util.Random(random_seed)
    return _call_lolo_merit('UncertaintyCorrelation', y_true, y_pred, y_std, rng)

<|endoftext|>


<commit_before># Copyright 2008-2009 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data filters used by Test objects"""

import re
import time

# Gracefully disable xml/xpath support if not found
try:
    from lxml import etree
except ImportError:
    etree = None

from twisted.python import failure

from nagcat import errors, log, util

# Accept filter specs in the format "name[default]:arguments"
# where [default] and :arguments are optional
_SPEC = re.compile(r'^([a-z][a-z0-9]*)(\[([^\]]*)\])?(:)?')

def Filter(test, spec):
    """Generator that create a Filter class from the spec"""

    assert isinstance(spec, str)

    match = _SPEC.match(spec)
    if not match:
        raise errors.InitError("Invalid filter spec: '%s'" % spec)

    name = match.group(1)
    default = match.group(3)

    if match.group(4):
        arguments = spec[match.end():]
    else:
        arguments = ""

    filter_class = globals().get("Filter_%s" % name, None)
    if filter_class:
        assert issubclass(filter_class, _Filter)
        return filter_class(test, default, arguments)
    else:
        raise errors.InitError("Invalid filter type '%s'" % name)

class _Filter(object):
    """Filter class template"""

    # Set weather this filter allows default values
    handle_default = True
    # Set weather this filter should be on the errorback chain
    # in addition to the normal callback chain.
    handle_errors = False

    def __init__(self, test, default, arguments):
        self.test = test
        self.default = default
        self.arguments = arguments

        if not self.handle_default and self.default is not None:
            raise errors.InitError("'%s' filters cannot take default values"
                    % self.__class__.__name__.replace("Filter_",""))

    @errors.callback
    def filter(self, result):
        """Run the filter on the given input.

        All filters must expect and return a str.
        """
        raise Exception("Unimplemented!")

class Filter_regex(_Filter):
    """Filter data based on a regular expression"""

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)
        try:
            self.regex = re.compile(self.arguments, re.MULTILINE | re.DOTALL)
        except re.error, ex:
            raise errors.InitError(
                    "Invalid regex '%s': %s" % (self.arguments, ex))

    @errors.callback
    def filter(self, result):
        log.debug("Matching regex '%s'", self.arguments)

        match = self.regex.search(result)
        if match:
            if match.groups():
                return match.group(1)
            else:
                return match.group(0)
        else:
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                        "Failed to match regex '%s'" % self.arguments)

class Filter_date2epoch(_Filter):
    """Transform a string into the time in seconds since epoch"""
    # This filter could use a better name

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        # Test for bogus patterns
        testing = time.strftime(self.arguments)
        if testing == self.arguments:
            raise errors.InitError("Invalid date format: %s" % self.arguments)

    @errors.callback
    def filter(self, result):
        log.debug("Converting date using format '%s'", self.arguments)

        try:
            return str(time.mktime(time.strptime(result, self.arguments)))
        except ValueError:
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                    "Failed to parse date with format '%s'" % self.arguments)

class Filter_xpath(_Filter):
    """Fetch something out of an XML document using XPath."""

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        if not etree:
            raise errors.InitError("lxml is required for XPath support.")

        try:
            self.xpath = etree.XPath(self.arguments)
        except etree.XPathSyntaxError, ex:
            raise errors.InitError(
                    "Invalid XPath query '%s': %s"  % (self.arguments, ex))

    @errors.callback
    def filter(self, result):
        def format(data):
            if etree.iselement(data):
                ret = etree.tostring(data)
            else:
                ret = str(data)
            return ret.strip()

        log.debug("Fetching XML element %s", self.arguments)

        try:
            root = etree.fromstring(result)
        except etree.XMLSyntaxError, ex:
            raise errors.TestCritical("Invalid XML: %s" % ex)

        data = self.xpath(root)

        if isinstance(data, list) and data:
            return "\n".join([format(x) for x in data])
        elif isinstance(data, list):
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                        "Failed to find xml element %s" % self.arguments)
        else:
            return format(data)

class Filter_save(_Filter):
    """Save the current result for use in the test report"""

    handle_default = False
    handle_errors = True

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        if not arguments:
            raise errors.InitError("save filters must provide an identifier")

    @errors.callback
    def filter(self, result):
        # Pull the value out of the error if needed
        if isinstance(result, failure.Failure):
            if isinstance(result, errors.Failure):
                value = result.result
            else:
                value = None
        else:
            value = result

        self.test.saved[self.arguments] = value

        return result

class Filter_critical(_Filter):
    """Mark the test as CRITICAL if the given test fails."""

    handle_default = False
    handle_errors = True

    # Supported operators
    ops = ('>','<','=','==','>=','<=','<>','!=','=~','!~')
    # Expression format
    format = re.compile("\s*([<>=!~]{1,2})\s*(\S+.*)")

    # Error to raise, overridden in Filter_warning
    error = errors.TestCritical

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        match = self.format.match(arguments)
        if not match:
            raise errors.InitError("Invalid %s test: %s"
                    % (self.error.state.lower(), arguments))

        self.test_op = match.group(1)
        self.test_val = match.group(2)

        if self.test_op not in self.ops:
            raise errors.InitError("Invalid %s test operator: %s"
                    % (self.error.state.lower(), self.test_op))

        if '~' in self.test_op:
            # Check for a valid regular expression
            try:
                self.test_regex = re.compile(self.test_val, re.MULTILINE)
            except re.error, ex:
                raise errors.InitError("Invalid %s test regex '%s': %s"
                        % (self.error.state.lower(), self.test_val, ex))
        else:
            # not a regular expression, let MathString do its magic.
            self.test_val = util.MathString(self.test_val)
            self.test_regex = None

        # Convert non-python operator
        if self.test_op == '=':
            self.test_op = '=='

    def filter(self, result):
        # Allow critical to override warning
        if (isinstance(result, errors.Failure) and
                isinstance(result.value, errors.TestWarning)):
            true_result = result.result
        elif isinstance(result, failure.Failure):
            return result
        else:
            true_result = result

        # mimic the error.method decorator since we need to
        # report true_result, not result.
        try:
            if self.test_op == '=~':
                if self.test_regex.search(true_result):
                    raise self.error(
                            "Failed to match regex '%s'" % self.test_val)
            elif self.test_op == '!~':
                if not self.test_regex.search(true_result):
                    raise self.error("Matched regex '%s'" % self.test_val)
            else:
                eval_dict = {'a':util.MathString(true_result),
                             'b':self.test_val}

                if eval("a %s b" % self.test_op, eval_dict):
                    raise self.error("Test failed: %s %s"
                            % (self.test_op, self.test_val))
        except Exception, ex:
            result = errors.Failure(result=true_result)

        return result

class Filter_warning(Filter_critical):
    """Mark the test as WARNING if the given test fails."""

    handle_errors = False
    error = errors.TestWarning

<commit_msg>Finally fix the backwards warning/critical messages<commit_after># Copyright 2008-2009 ITA Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data filters used by Test objects"""

import re
import time

# Gracefully disable xml/xpath support if not found
try:
    from lxml import etree
except ImportError:
    etree = None

from twisted.python import failure

from nagcat import errors, log, util

# Accept filter specs in the format "name[default]:arguments"
# where [default] and :arguments are optional
_SPEC = re.compile(r'^([a-z][a-z0-9]*)(\[([^\]]*)\])?(:)?')

def Filter(test, spec):
    """Generator that create a Filter class from the spec"""

    assert isinstance(spec, str)

    match = _SPEC.match(spec)
    if not match:
        raise errors.InitError("Invalid filter spec: '%s'" % spec)

    name = match.group(1)
    default = match.group(3)

    if match.group(4):
        arguments = spec[match.end():]
    else:
        arguments = ""

    filter_class = globals().get("Filter_%s" % name, None)
    if filter_class:
        assert issubclass(filter_class, _Filter)
        return filter_class(test, default, arguments)
    else:
        raise errors.InitError("Invalid filter type '%s'" % name)

class _Filter(object):
    """Filter class template"""

    # Set weather this filter allows default values
    handle_default = True
    # Set weather this filter should be on the errorback chain
    # in addition to the normal callback chain.
    handle_errors = False

    def __init__(self, test, default, arguments):
        self.test = test
        self.default = default
        self.arguments = arguments

        if not self.handle_default and self.default is not None:
            raise errors.InitError("'%s' filters cannot take default values"
                    % self.__class__.__name__.replace("Filter_",""))

    @errors.callback
    def filter(self, result):
        """Run the filter on the given input.

        All filters must expect and return a str.
        """
        raise Exception("Unimplemented!")

class Filter_regex(_Filter):
    """Filter data based on a regular expression"""

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)
        try:
            self.regex = re.compile(self.arguments, re.MULTILINE | re.DOTALL)
        except re.error, ex:
            raise errors.InitError(
                    "Invalid regex '%s': %s" % (self.arguments, ex))

    @errors.callback
    def filter(self, result):
        log.debug("Matching regex '%s'", self.arguments)

        match = self.regex.search(result)
        if match:
            if match.groups():
                return match.group(1)
            else:
                return match.group(0)
        else:
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                        "Failed to match regex '%s'" % self.arguments)

class Filter_date2epoch(_Filter):
    """Transform a string into the time in seconds since epoch"""
    # This filter could use a better name

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        # Test for bogus patterns
        testing = time.strftime(self.arguments)
        if testing == self.arguments:
            raise errors.InitError("Invalid date format: %s" % self.arguments)

    @errors.callback
    def filter(self, result):
        log.debug("Converting date using format '%s'", self.arguments)

        try:
            return str(time.mktime(time.strptime(result, self.arguments)))
        except ValueError:
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                    "Failed to parse date with format '%s'" % self.arguments)

class Filter_xpath(_Filter):
    """Fetch something out of an XML document using XPath."""

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        if not etree:
            raise errors.InitError("lxml is required for XPath support.")

        try:
            self.xpath = etree.XPath(self.arguments)
        except etree.XPathSyntaxError, ex:
            raise errors.InitError(
                    "Invalid XPath query '%s': %s"  % (self.arguments, ex))

    @errors.callback
    def filter(self, result):
        def format(data):
            if etree.iselement(data):
                ret = etree.tostring(data)
            else:
                ret = str(data)
            return ret.strip()

        log.debug("Fetching XML element %s", self.arguments)

        try:
            root = etree.fromstring(result)
        except etree.XMLSyntaxError, ex:
            raise errors.TestCritical("Invalid XML: %s" % ex)

        data = self.xpath(root)

        if isinstance(data, list) and data:
            return "\n".join([format(x) for x in data])
        elif isinstance(data, list):
            if self.default is not None:
                return self.default
            else:
                raise errors.TestCritical(
                        "Failed to find xml element %s" % self.arguments)
        else:
            return format(data)

class Filter_save(_Filter):
    """Save the current result for use in the test report"""

    handle_default = False
    handle_errors = True

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        if not arguments:
            raise errors.InitError("save filters must provide an identifier")

    @errors.callback
    def filter(self, result):
        # Pull the value out of the error if needed
        if isinstance(result, failure.Failure):
            if isinstance(result, errors.Failure):
                value = result.result
            else:
                value = None
        else:
            value = result

        self.test.saved[self.arguments] = value

        return result

class Filter_critical(_Filter):
    """Mark the test as CRITICAL if the given test fails."""

    handle_default = False
    handle_errors = True

    # Supported operators
    ops = ('>','<','=','==','>=','<=','<>','!=','=~','!~')
    # Expression format
    format = re.compile("\s*([<>=!~]{1,2})\s*(\S+.*)")

    # Error to raise, overridden in Filter_warning
    error = errors.TestCritical

    def __init__(self, test, default, arguments):
        _Filter.__init__(self, test, default, arguments)

        match = self.format.match(arguments)
        if not match:
            raise errors.InitError("Invalid %s test: %s"
                    % (self.error.state.lower(), arguments))

        self.test_op = match.group(1)
        self.test_val = match.group(2)

        if self.test_op not in self.ops:
            raise errors.InitError("Invalid %s test operator: %s"
                    % (self.error.state.lower(), self.test_op))

        if '~' in self.test_op:
            # Check for a valid regular expression
            try:
                self.test_regex = re.compile(self.test_val, re.MULTILINE)
            except re.error, ex:
                raise errors.InitError("Invalid %s test regex '%s': %s"
                        % (self.error.state.lower(), self.test_val, ex))
        else:
            # not a regular expression, let MathString do its magic.
            self.test_val = util.MathString(self.test_val)
            self.test_regex = None

        # Convert non-python operator
        if self.test_op == '=':
            self.test_op = '=='

    def filter(self, result):
        # Allow critical to override warning
        if (isinstance(result, errors.Failure) and
                isinstance(result.value, errors.TestWarning)):
            true_result = result.result
        elif isinstance(result, failure.Failure):
            return result
        else:
            true_result = result

        # mimic the error.method decorator since we need to
        # report true_result, not result.
        try:
            if self.test_op == '=~':
                if self.test_regex.search(true_result):
                    raise self.error("Matched regex '%s'" % self.test_val)
            elif self.test_op == '!~':
                if not self.test_regex.search(true_result):
                    raise self.error(
                            "Failed to match regex '%s'" % self.test_val)
            else:
                eval_dict = {'a':util.MathString(true_result),
                             'b':self.test_val}

                if eval("a %s b" % self.test_op, eval_dict):
                    raise self.error("Test failed: %s %s"
                            % (self.test_op, self.test_val))
        except Exception, ex:
            result = errors.Failure(result=true_result)

        return result

class Filter_warning(Filter_critical):
    """Mark the test as WARNING if the given test fails."""

    handle_errors = False
    error = errors.TestWarning

<|endoftext|>


<commit_before># Copyright 2010-2011 Google
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Min Cost Flow and Max Flow examples."""

from google.apputils import app
import gflags
from graph import pywrapflow

FLAGS = gflags.FLAGS

def MinCostFlow():
  """Min Cost Flow Example."""
  print 'Min Cost Flow on 4x4 Matrix'
  num_sources = 4
  num_targets = 4
  costs = [[90, 75, 75, 80],
           [35, 85, 55, 65],
           [125, 95, 90, 105],
           [45, 110, 95, 115]]
  expected_cost = 275
  graph = pywrapflow.StarGraph(num_sources + num_targets,
                               num_sources * num_targets)
  min_cost_flow = pywrapflow.MinCostFlow(graph)
  for source in range(0, num_sources):
    for target in range(0, num_targets):
      arc = graph.AddArc(source, num_sources + target)
      min_cost_flow.SetArcUnitCost(arc, costs[source][target])
      min_cost_flow.SetArcCapacity(arc, 1)

  for source in range(0, num_sources):
    min_cost_flow.SetNodeSupply(source, 1)
  for target in range(0, num_targets):
    min_cost_flow.SetNodeSupply(num_sources + target, -1)

  total_flow_cost = min_cost_flow.ComputeMinCostFlow()
  print 'total flow', total_flow_cost, '/', expected_cost

def MaxFeasibleFlow():
  """Max Flow Example."""
  print 'MaxFeasibleFlow'
  num_nodes = 6
  num_arcs = 9
  tails = [0, 0, 0, 0, 1, 2, 3, 3, 4]
  heads = [1, 2, 3, 4, 3, 4, 4, 5, 5]
  capacities = [5, 8, 5, 3, 4, 5, 6, 6, 4]
  expected_flows = [4, 4, 2, 0, 4, 4, 0, 6, 4]
  expected_total_flow = 10
  graph = pywrapflow.StarGraph(num_nodes, num_arcs)
  max_flow = pywrapflow.MaxFlow(graph, 0, num_nodes - 1)
  for i in range(0, num_arcs):
    arc = graph.AddArc(tails[i], heads[i])
    max_flow.SetArcCapacity(arc, capacities[i])

  total_flow = max_flow.ComputeMaxFlow()
  print 'total flow', total_flow, '/', expected_total_flow
  for i in range(num_arcs):
    print 'Arc', i, ':', max_flow.Flow(i), '/', expected_flows[i]

def main(unused_argv):
  MinCostFlow()
  MaxFeasibleFlow()

if __name__ == '__main__':
  app.run()
<commit_msg>fix pyflow_example<commit_after># Copyright 2010-2011 Google
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Min Cost Flow and Max Flow examples."""

from google.apputils import app
import gflags
from graph import pywrapflow

FLAGS = gflags.FLAGS

def MinCostFlow():
  """Min Cost Flow Example."""
  print 'Min Cost Flow on 4x4 Matrix'
  num_sources = 4
  num_targets = 4
  costs = [[90, 75, 75, 80],
           [35, 85, 55, 65],
           [125, 95, 90, 105],
           [45, 110, 95, 115]]
  expected_cost = 275
  graph = pywrapflow.StarGraph(num_sources + num_targets,
                               num_sources * num_targets)
  min_cost_flow = pywrapflow.MinCostFlow(graph)
  for source in range(0, num_sources):
    for target in range(0, num_targets):
      arc = graph.AddArc(source, num_sources + target)
      min_cost_flow.SetArcUnitCost(arc, costs[source][target])
      min_cost_flow.SetArcCapacity(arc, 1)

  for source in range(0, num_sources):
    min_cost_flow.SetNodeSupply(source, 1)
  for target in range(0, num_targets):
    min_cost_flow.SetNodeSupply(num_sources + target, -1)
  min_cost_flow.Solve()
  total_flow_cost = min_cost_flow.GetOptimalCost()
  print 'total flow', total_flow_cost, '/', expected_cost

def MaxFeasibleFlow():
  """Max Flow Example."""
  print 'MaxFeasibleFlow'
  num_nodes = 6
  num_arcs = 9
  tails = [0, 0, 0, 0, 1, 2, 3, 3, 4]
  heads = [1, 2, 3, 4, 3, 4, 4, 5, 5]
  capacities = [5, 8, 5, 3, 4, 5, 6, 6, 4]
  expected_flows = [4, 4, 2, 0, 4, 4, 0, 6, 4]
  expected_total_flow = 10
  graph = pywrapflow.StarGraph(num_nodes, num_arcs)
  max_flow = pywrapflow.MaxFlow(graph, 0, num_nodes - 1)
  for i in range(0, num_arcs):
    arc = graph.AddArc(tails[i], heads[i])
    max_flow.SetArcCapacity(arc, capacities[i])
  max_flow.Solve()
  total_flow = max_flow.GetOptimalFlow()
  print 'total flow', total_flow, '/', expected_total_flow
  for i in range(num_arcs):
    print 'Arc', i, ':', max_flow.Flow(i), '/', expected_flows[i]

def main(unused_argv):
  MinCostFlow()
  MaxFeasibleFlow()

if __name__ == '__main__':
  app.run()
<|endoftext|>


<commit_before>from rigor.database import transactional, reader, RowMapper, uuid_transform, polygon_transform, polygon_tuple_adapter

import uuid
from datetime import datetime, timedelta
import psycopg2

def resolution_transform(value, column_name, row):
	if value is None:
		return None
	return (row['x_resolution'], row['y_resolution'])

image_mapper = RowMapper(field_mappings={'x_resolution': 'resolution', 'y_resolution': None}, field_transforms={'locator':uuid_transform, 'resolution':resolution_transform})
annotation_mapper = RowMapper(field_transforms={'boundary':polygon_transform})

class DatabaseMapper(object):
	""" Reads and write Images to database """

	def __init__(self, database):
		self._db = database

	def _get_next_id(self, cursor, name):
		sql = "SELECT NEXTVAL('{0}_id_seq');".format(name)
		cursor.execute(sql)
		row = cursor.fetch_one()
		return row.values()[0]

	@reader
	def get_only_image_by_id(self, image_id):
		"""
		Retrieves the image, but none of its tags or annotations, from the database
		"""
		pass

	def _get_only_image_by_id(self, cursor, image_id):
		sql = "SELECT id, locator, hash, stamp, sensor, x_resolution, y_resolution, format, depth, location, source FROM image WHERE id = %s;"
		cursor.execute(sql, (image_id, ))
		image = cursor.fetch_only_one(image_mapper)
		return image

	@reader
	def get_image_by_id(self, image_id):
		""" Retrieves the image from the database """
		pass

	def _get_image_by_id(self, cursor, image_id):
		image = self._get_only_image_by_id(cursor, image_id)
		image['tags'] = self._get_tags_by_image_id(cursor, image_id)
		image['annotations'] = self._get_annotations_by_image_id(cursor, image_id)
		return image

	@reader
	def get_images_for_analysis(self, domain, limit=None, random=False):
		"""
		Retrieves all images for the domain from the database, used for
		algorithm analysis.  The optional limit limits the number of
		images returned.  If random, the images will be randomly selected.
		Otherwise, images will be returned sorted by ID.
		"""
		pass

	def _get_images_for_analysis(self, cursor, domain, limit, random):
		sql = "SELECT * FROM (SELECT distinct(image.id), image.locator, image.format FROM annotation LEFT JOIN image ON annotation.image_id = image.id WHERE annotation.domain = %s) image ORDER BY "
		if random and limit:
			sql += "random()"
		else:
			sql += "image.id"

		if limit:
			sql += " LIMIT %s;"
			cursor.execute(sql, (domain, limit))
		else:
			sql += ";"
			cursor.execute(sql, (domain, ))
		rows = cursor.fetch_all()
		images = list()
		for row in rows:
			image = image_mapper.map_row(row)
			sql = "SELECT model, boundary FROM annotation WHERE image_id = %s AND domain = %s;";
			cursor.execute(sql, (row[0], domain, ))
			image['annotations'] = cursor.fetch_all(annotation_mapper)
			images.append(image)
		return images

	def _get_tags_by_image_id(self, cursor, image_id):
		sql = "SELECT name FROM tag where image_id = %s;"
		cursor.execute(sql, (image_id, ))
		rows = cursor.fetch_all()
		return [row[0] for row in rows]

	@transactional
	def create_image(self, image):
		""" Stores the image metadata in the database, setting its id """
		pass

	def _create_image(self, cursor, image):
		id = self._get_next_id(cursor, 'image')
		sql = "INSERT INTO image (id, locator, hash, stamp, sensor, x_resolution, y_resolution, format, depth, location, source) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
		cursor.execute(sql, (id, image['locator'], image['hash'], image['stamp'], image['sensor'], image['resolution'][0], image['resolution'][1], image['format'], image['depth'], image['location'], image['source']))
		image['id'] = id
		if image['tags']:
			self._create_tags(cursor, image['tags'], id)
		if image['annotations']:
			for annotation in image['annotations']:
				self._create_annotation(cursor, annotation, id)

	def _create_tags(self, cursor, tags, image_id):
		sql = "INSERT INTO tag (image_id, name) VALUES (%s, %s);"
		cursor.executemany(sql, [(image_id, tag) for tag in tags])

	@reader
	def get_annotation_by_id(self, annotation_id):
		""" Retrieves the annotation from the database """
		pass

	def _get_annotation_by_id(self, cursor, annotation_id):
		sql = "SELECT id, stamp, boundary, domain, rank, model FROM annotation WHERE id = %s;"
		cursor.execute(sql, (annotation_id, ))
		return cursor.fetch_only_one(annotation_mapper)

	def _get_annotations_by_image_id(self, cursor, image_id):
		sql = "SELECT id, stamp, boundary, domain, rank, model FROM annotation WHERE image_id = %s;"
		cursor.execute(sql, (image_id, ))
		return cursor.fetch_all(annotation_mapper)

	@transactional
	def update_annotation_model(self, annotation_id, model):
		"""
		Replaces model in the database with new model.  The timestamp will be
		updated to the current time.  Nothing will be changed if the model does not
		change.  Returns True if a change was made, False otherwise.
		"""
		pass

	def _update_annotation_model(self, cursor, annotation_id, model):
		now = datetime.utcnow()
		sql = "SELECT model FROM annotation WHERE id = %s;"
		cursor.execute(sql, (annotation_id, ))
		original = cursor.fetch_one()[0]
		if model == original:
			return False
		sql = "UPDATE annotation SET stamp = %s, model = %s WHERE id = %s;"
		cursor.execute(sql, (now, model, annotation_id))
		return True

	@transactional
	def delete_annotations(self, image_id, domain):
		""" Removes all annotations for an image in a particular domain """
		pass

	def _delete_annotations(self, cursor, image_id, domain):
		sql = "DELETE FROM annotation WHERE domain = %s AND image_id = %s;"
		cursor.execute(sql, (domain, image_id))

	def _delete_annotation(self, cursor, annotation_id):
		sql = "DELETE FROM annotation WHERE annotation_id = %s;"
		cursor.execute(sql, (annotation_id))

	@transactional
	def create_annotation(self, annotation, image_id):
		""" Stores the annotation in the database, setting its id """
		pass

	def _create_annotation(self, cursor, annotation, image_id):
		id = self._get_next_id(cursor, 'annotation')
		sql = "INSERT INTO annotation (id, image_id, stamp, boundary, domain, rank, model) VALUES (%s, %s, %s, %s, %s, %s, %s);"
		cursor.execute(sql, (id, image_id, annotation['stamp'], polygon_tuple_adapter(annotation['boundary']), annotation['domain'], annotation['rank'], annotation['model']))
		annotation['id'] = id

	def acquire_lock(self, image_id, domain, duration=300):
		"""
		Locks a particular image so that it is not served to other users while a
		user is creating annotations in a particular domain.  The duration is the
		amount of time in seconds the lock will be valid before expiring.

		Returns either a key that is needed to release the lock, or None if the
		image is already locked.  FIXME: Currently, a new lock can't be acquired
		for an image/domain until the old lock has been removed completely, even if
		it has expired.
		"""
		cursor = self._db.get_cursor()
		retval = None
		try:
			retval = _acquire_lock(cursor, image_id, domain, duration)
			self._db.commit(cursor)
		except psycopg2.IntegrityError:
			self._db.rollback(cursor)
			# lock already exists; don't raise, just return None
		except:
			self._db.rollback(cursor)
			raise
		return retval

	def _acquire_lock(self, cursor, image_id, domain, duration):
		expiry = datetime.utcnow() + timedelta(seconds=int(duration))
		key = uuid.uuid4().hex
		sql = "INSERT INTO image_lock (image_id, domain, key, expiry) VALUES (%s, %s, %s, %s);"
		cursor.execute(sql, (image_id, domain, key, expiry))
		return key

	@transactional
	def release_lock(self, key, checked):
		"""
		Releases a lock that was previously acquired, using the key.

		If checked is True, and the lock doesn't exist or it has already expired,
		then an IntegrityError will be raised.  This allows it to be used to
		invalidate inserts on expired locks.
		"""
		pass

	def _release_lock(self, cursor, key, checked):
		now = datetime.utcnow()
		sql = "DELETE FROM image_lock WHERE key = %s AND expiry > %s;"
		cursor.execute(sql, (key, now))
		if not cursor.rowcount > 0:
			raise psycopg2.IntegrityError("Lock was not found")

	@transactional
	def expire_locks(self):
		"""
		Removes all expired locks from the database.  This is meant to be run by a
		periodic process, not by end-users.
		"""
		pass

	def _expire_locks(self, cursor):
		now = datetime.utcnow()
		sql = "DELETE FROM image_lock WHERE expiry <= %s;"
		cursor.execute(sql, (now, ))
<commit_msg>Lock cleanups part one<commit_after>from rigor.database import transactional, reader, RowMapper, uuid_transform, polygon_transform, polygon_tuple_adapter

import uuid
from datetime import datetime, timedelta
import psycopg2

def resolution_transform(value, column_name, row):
	if value is None:
		return None
	return (row['x_resolution'], row['y_resolution'])

image_mapper = RowMapper(field_mappings={'x_resolution': 'resolution', 'y_resolution': None}, field_transforms={'locator':uuid_transform, 'resolution':resolution_transform})
annotation_mapper = RowMapper(field_transforms={'boundary':polygon_transform})

class DatabaseMapper(object):
	""" Reads and write Images to database """

	def __init__(self, database):
		self._db = database

	def _get_next_id(self, cursor, name):
		sql = "SELECT NEXTVAL('{0}_id_seq');".format(name)
		cursor.execute(sql)
		row = cursor.fetch_one()
		return row.values()[0]

	@reader
	def get_only_image_by_id(self, image_id):
		"""
		Retrieves the image, but none of its tags or annotations, from the database
		"""
		pass

	def _get_only_image_by_id(self, cursor, image_id):
		sql = "SELECT id, locator, hash, stamp, sensor, x_resolution, y_resolution, format, depth, location, source FROM image WHERE id = %s;"
		cursor.execute(sql, (image_id, ))
		image = cursor.fetch_only_one(image_mapper)
		return image

	@reader
	def get_image_by_id(self, image_id):
		""" Retrieves the image from the database """
		pass

	def _get_image_by_id(self, cursor, image_id):
		image = self._get_only_image_by_id(cursor, image_id)
		image['tags'] = self._get_tags_by_image_id(cursor, image_id)
		image['annotations'] = self._get_annotations_by_image_id(cursor, image_id)
		return image

	@reader
	def get_images_for_analysis(self, domain, limit=None, random=False):
		"""
		Retrieves all images for the domain from the database, used for
		algorithm analysis.  The optional limit limits the number of
		images returned.  If random, the images will be randomly selected.
		Otherwise, images will be returned sorted by ID.
		"""
		pass

	def _get_images_for_analysis(self, cursor, domain, limit, random):
		sql = "SELECT * FROM (SELECT distinct(image.id), image.locator, image.format FROM annotation LEFT JOIN image ON annotation.image_id = image.id WHERE annotation.domain = %s) image ORDER BY "
		if random and limit:
			sql += "random()"
		else:
			sql += "image.id"

		if limit:
			sql += " LIMIT %s;"
			cursor.execute(sql, (domain, limit))
		else:
			sql += ";"
			cursor.execute(sql, (domain, ))
		rows = cursor.fetch_all()
		images = list()
		for row in rows:
			image = image_mapper.map_row(row)
			sql = "SELECT model, boundary FROM annotation WHERE image_id = %s AND domain = %s;";
			cursor.execute(sql, (row[0], domain, ))
			image['annotations'] = cursor.fetch_all(annotation_mapper)
			images.append(image)
		return images

	def _get_tags_by_image_id(self, cursor, image_id):
		sql = "SELECT name FROM tag where image_id = %s;"
		cursor.execute(sql, (image_id, ))
		rows = cursor.fetch_all()
		return [row[0] for row in rows]

	@transactional
	def create_image(self, image):
		""" Stores the image metadata in the database, setting its id """
		pass

	def _create_image(self, cursor, image):
		id = self._get_next_id(cursor, 'image')
		sql = "INSERT INTO image (id, locator, hash, stamp, sensor, x_resolution, y_resolution, format, depth, location, source) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
		cursor.execute(sql, (id, image['locator'], image['hash'], image['stamp'], image['sensor'], image['resolution'][0], image['resolution'][1], image['format'], image['depth'], image['location'], image['source']))
		image['id'] = id
		if image['tags']:
			self._create_tags(cursor, image['tags'], id)
		if image['annotations']:
			for annotation in image['annotations']:
				self._create_annotation(cursor, annotation, id)

	def _create_tags(self, cursor, tags, image_id):
		sql = "INSERT INTO tag (image_id, name) VALUES (%s, %s);"
		cursor.executemany(sql, [(image_id, tag) for tag in tags])

	@reader
	def get_annotation_by_id(self, annotation_id):
		""" Retrieves the annotation from the database """
		pass

	def _get_annotation_by_id(self, cursor, annotation_id):
		sql = "SELECT id, stamp, boundary, domain, rank, model FROM annotation WHERE id = %s;"
		cursor.execute(sql, (annotation_id, ))
		return cursor.fetch_only_one(annotation_mapper)

	def _get_annotations_by_image_id(self, cursor, image_id):
		sql = "SELECT id, stamp, boundary, domain, rank, model FROM annotation WHERE image_id = %s;"
		cursor.execute(sql, (image_id, ))
		return cursor.fetch_all(annotation_mapper)

	@transactional
	def update_annotation_model(self, annotation_id, model):
		"""
		Replaces model in the database with new model.  The timestamp will be
		updated to the current time.
		"""
		pass

	def _update_annotation_model(self, cursor, annotation_id, model):
		now = datetime.utcnow()
		sql = "UPDATE annotation SET stamp = %s, model = %s WHERE id = %s;"
		cursor.execute(sql, (now, model, annotation_id))

	@transactional
	def delete_annotations(self, image_id, domain):
		""" Removes all annotations for an image in a particular domain """
		pass

	def _delete_annotations(self, cursor, image_id, domain):
		sql = "DELETE FROM annotation WHERE domain = %s AND image_id = %s;"
		cursor.execute(sql, (domain, image_id))

	def _delete_annotation(self, cursor, annotation_id):
		sql = "DELETE FROM annotation WHERE annotation_id = %s;"
		cursor.execute(sql, (annotation_id))

	@transactional
	def create_annotation(self, annotation, image_id):
		""" Stores the annotation in the database, setting its id """
		pass

	def _create_annotation(self, cursor, annotation, image_id):
		id = self._get_next_id(cursor, 'annotation')
		sql = "INSERT INTO annotation (id, image_id, stamp, boundary, domain, rank, model) VALUES (%s, %s, %s, %s, %s, %s, %s);"
		cursor.execute(sql, (id, image_id, annotation['stamp'], polygon_tuple_adapter(annotation['boundary']), annotation['domain'], annotation['rank'], annotation['model']))
		annotation['id'] = id

	@transactional
	def acquire_lock(self, image_id, domain, key, duration):
		"""
		Locks a particular image so that it is not served to other users while a
		user is creating annotations in a particular domain.  The duration is the
		amount of time in seconds the lock will be valid before expiring.  If a key
		is supplied, it will be used to create the lock.  Otherwise, a new key will
		be generated.

		If the lock already exists, its expiry will be updated using the new
		duration.

		A single key should be used for all locks acquired by a user.  This makes
		it easier to navigate.

		Throws an exception if the lock can't be acquired, otherwise it returns the
		key.  FIXME: Currently, a new lock can't be acquired for an image/domain
		until the old lock has been removed completely, even if it has expired.
		"""
		pass

	def _acquire_lock(self, cursor, image_id, domain, key, duration):
		expiry = datetime.utcnow() + timedelta(seconds=int(duration))
		if key:
			sql = "UPDATE image_lock SET expiry = %s WHERE image_id = %s AND domain = %s AND key = %s;"
			cursor.execute(sql, (expiry, image_id, domain, key))
			if cursor.rowcount > 0:
				return key
		else:
			key = uuid.uuid4().hex

		sql = "INSERT INTO image_lock (image_id, domain, key, expiry) VALUES (%s, %s, %s, %s);"
		cursor.execute(sql, (image_id, domain, key, expiry))
		return key

	@transactional
	def release_lock(self, image_id, domain, key, checked):
		"""
		Releases a lock that was previously acquired, using the key.

		If checked is True, and the lock doesn't exist or it has already expired,
		then an IntegrityError will be raised.  This allows it to be used to
		invalidate inserts on expired locks.
		"""
		pass

	def _release_lock(self, cursor, image_id, domain, key, checked):
		now = datetime.utcnow()
		sql = "DELETE FROM image_lock WHERE image_id = %s AND domain = %s AND key = %s AND expiry > %s;"
		cursor.execute(sql, (image_id, domain, key, now))
		if not cursor.rowcount > 0:
			raise psycopg2.IntegrityError("Lock was not found")

	@transactional
	def expire_locks(self):
		"""
		Removes all expired locks from the database.  This is meant to be run by a
		periodic process, not by end-users.
		"""
		pass

	def _expire_locks(self, cursor):
		now = datetime.utcnow()
		sql = "DELETE FROM image_lock WHERE expiry <= %s;"
		cursor.execute(sql, (now, ))
<|endoftext|>