<commit_before>import os
import subprocess

import pynodegl as ngl
from PyQt5 import QtGui, QtCore

from com import query_inplace

class Exporter(QtCore.QThread):

    progressed = QtCore.pyqtSignal(int)
    failed = QtCore.pyqtSignal()
    finished = QtCore.pyqtSignal()

    def __init__(self, get_scene_func, filename, w, h, extra_enc_args=None):
        super(Exporter, self).__init__()
        self._get_scene_func = get_scene_func
        self._filename = filename
        self._width = w
        self._height = h
        self._extra_enc_args = extra_enc_args
        self._cancelled = False

    def run(self):
        filename, width, height = self._filename, self._width, self._height

        if filename.endswith('gif'):
            palette_filename = '/tmp/palette.png'
            pass1_args = ['-vf', 'palettegen']
            pass2_args = self._extra_enc_args + ['-i', palette_filename, '-lavfi', 'paletteuse']
            ok = self._export(palette_filename, width, height, pass1_args)
            if not ok:
                return
            ok = self._export(filename, width, height, pass2_args)
        else:
            ok = self._export(filename, width, height, self._extra_enc_args)
        if ok:
            self.finished.emit()

    def _export(self, filename, width, height, extra_enc_args=None):
        fd_r, fd_w = os.pipe()

        cfg = self._get_scene_func(pipe=(fd_w, width, height))
        if not cfg:
            self.failed.emit()
            return False

        fps = cfg['framerate']
        duration = cfg['duration']
        samples = cfg['samples']

        cmd = ['ffmpeg', '-r', '%d/%d' % fps,
               '-nostats', '-nostdin',
               '-f', 'rawvideo',
               '-video_size', '%dx%d' % (width, height),
               '-pixel_format', 'rgba',
               '-i', 'pipe:%d' % fd_r]
        if extra_enc_args:
            cmd += extra_enc_args
        cmd += ['-y', filename]

        def close_unused_child_fd():
            os.close(fd_w)

        def close_unused_parent_fd():
            os.close(fd_r)

        reader = subprocess.Popen(cmd, preexec_fn=close_unused_child_fd, close_fds=False)
        close_unused_parent_fd()

        # node.gl context
        ngl_viewer = ngl.Viewer()
        ngl_viewer.configure(
            platform=ngl.GLPLATFORM_AUTO,
            api=ngl.GLAPI_AUTO,
            wrapped=0,
            offscreen=1,
            width=width,
            height=height,
            samples=samples,
        )
        ngl_viewer.set_scene_from_string(cfg['scene'])
        ngl_viewer.set_viewport(0, 0, width, height)
        ngl_viewer.set_clearcolor(*cfg['clear_color'])

        # Draw every frame
        nb_frame = int(duration * fps[0] / fps[1])
        for i in range(nb_frame):
            if self._cancelled:
                break
            time = i * fps[1] / float(fps[0])
            ngl_viewer.draw(time)
            self.progressed.emit(i*100 / nb_frame)
        self.progressed.emit(100)

        os.close(fd_w)
        reader.wait()
        return True

    def cancel(self):
        self._cancelled = True

def test_export():
    import sys

    def _get_scene(**cfg_overrides):
        cfg = {
            'pkg': 'pynodegl_utils.examples',
            'scene': ('misc', 'triangle'),
            'duration': 5,
        }
        cfg.update(cfg_overrides)

        ret = query_inplace(query='scene', **cfg)
        if 'error' in ret:
            print ret['error']
            return None
        return ret

    def print_progress(progress):
        sys.stdout.write('\r%d%%' % progress)
        sys.stdout.flush()
        if progress == 100:
            sys.stdout.write('\n')

    if len(sys.argv) != 2:
        print 'Usage: %s <outfile>' % sys.argv[0]
        sys.exit(0)

    filename = sys.argv[1]
    app = QtGui.QGuiApplication(sys.argv)

    exporter = Exporter(_get_scene, filename, 320, 240)
    exporter.progressed.connect(print_progress)
    exporter.start()
    exporter.wait()

if __name__ == '__main__':
    test_export()
<commit_msg>utils/export: allow exporter to export a single specific frame<commit_after>import os
import subprocess

import pynodegl as ngl
from PyQt5 import QtGui, QtCore

from com import query_inplace

class Exporter(QtCore.QThread):

    progressed = QtCore.pyqtSignal(int)
    failed = QtCore.pyqtSignal()
    finished = QtCore.pyqtSignal()

    def __init__(self, get_scene_func, filename, w, h, extra_enc_args=None, time=None):
        super(Exporter, self).__init__()
        self._get_scene_func = get_scene_func
        self._filename = filename
        self._width = w
        self._height = h
        self._extra_enc_args = extra_enc_args
        self._time = time
        self._cancelled = False

    def run(self):
        filename, width, height = self._filename, self._width, self._height

        if filename.endswith('gif'):
            palette_filename = '/tmp/palette.png'
            pass1_args = ['-vf', 'palettegen']
            pass2_args = self._extra_enc_args + ['-i', palette_filename, '-lavfi', 'paletteuse']
            ok = self._export(palette_filename, width, height, pass1_args)
            if not ok:
                return
            ok = self._export(filename, width, height, pass2_args)
        else:
            ok = self._export(filename, width, height, self._extra_enc_args)
        if ok:
            self.finished.emit()

    def _export(self, filename, width, height, extra_enc_args=None):
        fd_r, fd_w = os.pipe()

        cfg = self._get_scene_func(pipe=(fd_w, width, height))
        if not cfg:
            self.failed.emit()
            return False

        fps = cfg['framerate']
        duration = cfg['duration']
        samples = cfg['samples']

        cmd = ['ffmpeg', '-r', '%d/%d' % fps,
               '-nostats', '-nostdin',
               '-f', 'rawvideo',
               '-video_size', '%dx%d' % (width, height),
               '-pixel_format', 'rgba',
               '-i', 'pipe:%d' % fd_r]
        if extra_enc_args:
            cmd += extra_enc_args
        cmd += ['-y', filename]

        def close_unused_child_fd():
            os.close(fd_w)

        def close_unused_parent_fd():
            os.close(fd_r)

        reader = subprocess.Popen(cmd, preexec_fn=close_unused_child_fd, close_fds=False)
        close_unused_parent_fd()

        # node.gl context
        ngl_viewer = ngl.Viewer()
        ngl_viewer.configure(
            platform=ngl.GLPLATFORM_AUTO,
            api=ngl.GLAPI_AUTO,
            wrapped=0,
            offscreen=1,
            width=width,
            height=height,
            samples=samples,
        )
        ngl_viewer.set_scene_from_string(cfg['scene'])
        ngl_viewer.set_viewport(0, 0, width, height)
        ngl_viewer.set_clearcolor(*cfg['clear_color'])

        if self._time is not None:
            ngl_viewer.draw(self._time)
            self.progressed.emit(100)
        else:
            # Draw every frame
            nb_frame = int(duration * fps[0] / fps[1])
            for i in range(nb_frame):
                if self._cancelled:
                    break
                time = i * fps[1] / float(fps[0])
                ngl_viewer.draw(time)
                self.progressed.emit(i*100 / nb_frame)
            self.progressed.emit(100)

        os.close(fd_w)
        reader.wait()
        return True

    def cancel(self):
        self._cancelled = True

def test_export():
    import sys

    def _get_scene(**cfg_overrides):
        cfg = {
            'pkg': 'pynodegl_utils.examples',
            'scene': ('misc', 'triangle'),
            'duration': 5,
        }
        cfg.update(cfg_overrides)

        ret = query_inplace(query='scene', **cfg)
        if 'error' in ret:
            print ret['error']
            return None
        return ret

    def print_progress(progress):
        sys.stdout.write('\r%d%%' % progress)
        sys.stdout.flush()
        if progress == 100:
            sys.stdout.write('\n')

    if len(sys.argv) != 2:
        print 'Usage: %s <outfile>' % sys.argv[0]
        sys.exit(0)

    filename = sys.argv[1]
    app = QtGui.QGuiApplication(sys.argv)

    exporter = Exporter(_get_scene, filename, 320, 240)
    exporter.progressed.connect(print_progress)
    exporter.start()
    exporter.wait()

if __name__ == '__main__':
    test_export()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import core.models.status_type

def go_back(apps, schema_editor):
    pass

def apply_data_migration(apps, schema_editor):
    MatchType = apps.get_model("core", "MatchType")
    add_match_types(MatchType)

def add_match_types(MatchType):
    MatchType.objects.get_or_create(name="BasicEmail")
    MatchType.objects.get_or_create(name="Username")

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0016_change_table_name_machine_export_to_export_request'),
    ]

    operations = [
        migrations.CreateModel(
            name='MatchType', fields=[
                ('id', models.AutoField(
                    verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(
                        max_length=128)), ], ), migrations.RunPython(
            apply_data_migration, go_back), migrations.CreateModel(
                            name='PatternMatch', fields=[
                                ('id', models.AutoField(
                                    verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('pattern', models.CharField(
                                        max_length=256)), ('type', models.ForeignKey(
                                            to='core.MatchType')), ], options={
                                                'db_table': 'pattern_match', }, ), migrations.AddField(
                                                    model_name='patternmatch', name='created_by', field=models.ForeignKey(
                                                        to=settings.AUTH_USER_MODEL), preserve_default=True, ), migrations.AddField(
                                                            model_name='license', name='access_list', field=models.ManyToManyField(
                                                                to='core.PatternMatch', blank=True), ), ]
<commit_msg>Spacing was jacked up<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import core.models.status_type

def go_back(apps, schema_editor):
    pass

def apply_data_migration(apps, schema_editor):
    MatchType = apps.get_model("core", "MatchType")
    add_match_types(MatchType)

def add_match_types(MatchType):
    MatchType.objects.get_or_create(name="BasicEmail")
    MatchType.objects.get_or_create(name="Username")

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0016_change_table_name_machine_export_to_export_request'),
    ]

    operations = [
        migrations.CreateModel(
            name='MatchType', fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=128)),
            ],
        ),
        migrations.RunPython(
            apply_data_migration, go_back),
        migrations.CreateModel(
            name='PatternMatch', fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('pattern', models.CharField(max_length=256)),
                ('type', models.ForeignKey(to='core.MatchType')),
            ], options={
                'db_table': 'pattern_match',
            },
        ),
        migrations.AddField(
            model_name='patternmatch', name='created_by', field=models.ForeignKey(to=settings.AUTH_USER_MODEL), preserve_default=True,
        ),
        migrations.AddField(
            model_name='license', name='access_list', field=models.ManyToManyField(to='core.PatternMatch', blank=True),
        ),
    ]
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Prepare data for diffuse all-sky analysis
"""
from __future__ import absolute_import, division, print_function

import os
import math

import yaml

from fermipy.jobs.utils import is_null
from fermipy.jobs.chain import Link
from fermipy.jobs.chain import Chain
from fermipy.jobs.scatter_gather import ScatterGather
from fermipy.jobs.slac_impl import make_nfs_path

from fermipy.diffuse.utils import create_inputlist
from fermipy.diffuse.name_policy import NameFactory
from fermipy.diffuse import defaults as diffuse_defaults
from fermipy.diffuse.binning import EVT_TYPE_DICT
from fermipy.diffuse.timefilter import MktimeFilterDict

from fermipy.diffuse.job_library import Gtlink_select, Gtlink_mktime,\
    Gtlink_ltcube, Gtlink_bin, Gtltsum_SG, Gtexpcube2_SG

from fermipy.diffuse.gt_coadd_split import CoaddSplit_SG

NAME_FACTORY = NameFactory()
try:
    MKTIME_DICT = MktimeFilterDict.build_from_yamlfile('config/mktime_filters.yaml')
except IOError:
    MKTIME_DICT = MktimeFilterDict(aliases=dict(quality='lat_config==1&&data_qual>0'),
                                   selections=dict(standard='{quality}'))

def make_full_path(basedir, outkey, origname):
    """Make a full file path by combining tokens

    Parameters
    -----------

    basedir : str
        The top level output area

    outkey : str
        The key for the particular instance of the analysis

    origname : str
        Template for the output file name

    Returns
    -------

    outpath : str
        This will be <basedir>:<outkey>:<newname>.fits
        Where newname = origname.replace('.fits', '_<outkey>.fits')

    """
    return os.path.join(basedir, outkey,
                        os.path.basename(origname).replace('.fits',
                                                           '_%s.fits' % outkey))

class SplitAndMktime(Chain):
    """Small class to split, apply mktime and bin data according to some user-provided specification

    This chain consists multiple `Link` objects:

    select-energy-EBIN-ZCUT : `Gtlink_select`
        Initial splitting by energy bin and zenith angle cut

    mktime-EBIN-ZCUT-FILTER : `Gtlink_mktime`
        Application of gtmktime filter for zenith angle cut

    ltcube-EBIN-ZCUT-FILTER : `Gtlink_ltcube`
        Computation of livetime cube for zenith angle cut

    select-type-EBIN-ZCUT-FILTER-TYPE : `Gtlink_select`
        Refinement of selection from event types

    bin-EBIN-ZCUT-FILTER-TYPE : `Gtlink_bin`
        Final binning of the data for each event type

    """
    appname = 'fermipy-split-and-mktime'
    linkname_default = 'split-and-mktime'
    usage = '%s [options]' % (appname)
    description = 'Run gtselect and gtbin together'

    default_options = dict(comp=diffuse_defaults.diffuse['comp'],
                           data=diffuse_defaults.diffuse['data'],
                           hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           evclass=(128, 'Event class bit mask', int),
                           outdir=('counts_cubes', 'Output directory', str),
                           outkey=(None, 'Key for this particular output file', str),
                           pfiles=(None, 'Directory for .par files', str),
                           do_ltsum=(False, 'Sum livetime cube files', bool),
                           scratch=(None, 'Scratch area', str),
                           dry_run=(False, 'Print commands but do not run them', bool))

    __doc__ += Link.construct_docstring(default_options)

    def __init__(self, **kwargs):
        """C'tor
        """
        super(SplitAndMktime, self).__init__(**kwargs)
        self.comp_dict = None

    def _map_arguments(self, args):
        """Map from the top-level arguments to the arguments provided to
        the indiviudal links """
        comp_file = args.get('comp', None)
        datafile = args.get('data', None)
        if is_null(comp_file):
            return
        if is_null(datafile):
            return

        NAME_FACTORY.update_base_dict(args['data'])

        outdir = args.get('outdir')
        outkey = args.get('outkey')
        ft1file = args['ft1file']
        ft2file = args['ft2file']
        if is_null(outdir) or is_null(outkey):
            return
        pfiles = os.path.join(outdir, outkey)

        self.comp_dict = yaml.safe_load(open(comp_file))
        coordsys = self.comp_dict.pop('coordsys')

        full_out_dir = make_nfs_path(os.path.join(outdir, outkey))

        for key_e, comp_e in sorted(self.comp_dict.items()):
            emin = math.pow(10., comp_e['log_emin'])
            emax = math.pow(10., comp_e['log_emax'])
            enumbins = comp_e['enumbins']
            zmax = comp_e['zmax']
            zcut = "zmax%i" % comp_e['zmax']
            evclassstr = NAME_FACTORY.base_dict['evclass']

            kwargs_select = dict(zcut=zcut,
                                 ebin=key_e,
                                 psftype='ALL',
                                 coordsys=coordsys)
            linkname = 'select-energy-%s-%s' % (key_e, zcut)
            selectfile_energy = make_full_path(outdir, outkey, NAME_FACTORY.select(**kwargs_select))
            self._set_link(linkname, Gtlink_select,
                           infile=ft1file,
                           outfile=selectfile_energy,
                           zmax=zmax,
                           emin=emin,
                           emax=emax,
                           evclass=NAME_FACTORY.evclassmask(evclassstr),
                           logfile=os.path.join(full_out_dir, "%s.log" % linkname))

            if 'mktimefilters' in comp_e:
                mktimefilters = comp_e['mktimefilters']
            else:
                mktimefilters = ['none']

            for mktimekey in mktimefilters:
                kwargs_mktime = kwargs_select.copy()
                kwargs_mktime['mktime'] = mktimekey
                filterstring = MKTIME_DICT[mktimekey]
                mktime_file = make_full_path(outdir, outkey, NAME_FACTORY.mktime(**kwargs_mktime))
                ltcube_file = make_full_path(outdir, outkey, NAME_FACTORY.ltcube(**kwargs_mktime))
                linkname_mktime = 'mktime-%s-%s-%s' % (key_e, zcut, mktimekey)
                linkname_ltcube = 'ltcube-%s-%s-%s' % (key_e, zcut, mktimekey)

                self._set_link(linkname_mktime, Gtlink_mktime,
                               evfile=selectfile_energy,
                               outfile=mktime_file,
                               scfile=ft2file,
                               filter=filterstring,
                               pfiles=pfiles,
                               logfile=os.path.join(full_out_dir, "%s.log" % linkname_mktime))
                self._set_link(linkname_ltcube, Gtlink_ltcube,
                               evfile=mktime_file,
                               outfile=ltcube_file,
                               scfile=ft2file,
                               zmax=zmax,
                               pfiles=pfiles,
                               logfile=os.path.join(full_out_dir, "%s.log" % linkname_ltcube))

                if 'evtclasses' in comp_e:
                    evtclasslist_vals = comp_e['evtclasses']
                else:
                    evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]

                for evtclassval in evtclasslist_vals:
                    for psf_type, psf_dict in sorted(comp_e['psf_types'].items()):
                        linkname_select = 'select-type-%s-%s-%s-%s-%s' % (
                            key_e, zcut, mktimekey, evtclassval, psf_type)
                        linkname_bin = 'bin-%s-%s-%s-%s-%s' % (key_e,
                                                               zcut, mktimekey,
                                                               evtclassval, psf_type)
                        kwargs_bin = kwargs_mktime.copy()
                        kwargs_bin['psftype'] = psf_type
                        kwargs_bin['coordsys'] = coordsys
                        kwargs_bin['evclass'] = evtclassval
                        selectfile_psf = make_full_path(
                            outdir, outkey, NAME_FACTORY.select(**kwargs_bin))
                        binfile_psf = make_full_path(
                            outdir, outkey, NAME_FACTORY.ccube(**kwargs_bin))
                        hpx_order_psf = min(args['hpx_order_max'], psf_dict['hpx_order'])
                        linkname_select = 'select-type-%s-%s-%s-%s-%s' % (key_e, zcut,
                                                                          mktimekey, evtclassval,
                                                                          psf_type)
                        linkname_bin = 'bin-%s-%s-%s-%s-%s' % (key_e, zcut, mktimekey,
                                                               evtclassval, psf_type)

                        self._set_link(linkname_select, Gtlink_select,
                                       infile=selectfile_energy,
                                       outfile=selectfile_psf,
                                       zmax=zmax,
                                       emin=emin,
                                       emax=emax,
                                       evtype=EVT_TYPE_DICT[psf_type],
                                       evclass=NAME_FACTORY.evclassmask(evtclassval),
                                       pfiles=pfiles,
                                       logfile=os.path.join(full_out_dir, "%s.log" % linkname_select))
                        self._set_link(linkname_bin, Gtlink_bin,
                                       coordsys=coordsys,
                                       hpx_order=hpx_order_psf,
                                       evfile=selectfile_psf,
                                       outfile=binfile_psf,
                                       emin=emin,
                                       emax=emax,
                                       enumbins=enumbins,
                                       pfiles=pfiles,
                                       logfile=os.path.join(full_out_dir, "%s.log" % linkname_bin))

class SplitAndMktime_SG(ScatterGather):
    """Small class to generate configurations for SplitAndMktime
    """
    appname = 'fermipy-split-and-mktime-sg'
    usage = "%s [options]" % (appname)
    description = "Prepare data for diffuse all-sky analysis"
    clientclass = SplitAndMktime

    job_time = 1500

    default_options = dict(comp=diffuse_defaults.diffuse['comp'],
                           data=diffuse_defaults.diffuse['data'],
                           hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
                           scratch=diffuse_defaults.diffuse['scratch'],
                           dry_run=diffuse_defaults.diffuse['dry_run'])

    __doc__ += Link.construct_docstring(default_options)

    def build_job_configs(self, args):
        """Hook to build job configurations
        """
        job_configs = {}

        comp_file = args.get('comp', None)
        if comp_file is not None:
            comp_dict = yaml.safe_load(open(comp_file))
            coordsys = comp_dict.pop('coordsys')
            for v in comp_dict.values():
                v['coordsys'] = coordsys
        else:
            return job_configs

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        inputfiles = create_inputlist(args['ft1file'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
        data_ver = NAME_FACTORY.base_dict['data_ver']

        for idx, infile in enumerate(inputfiles):
            key = "%06i" % idx
            key_scfile = "%03i" % (idx + 1)
            output_dir = os.path.join(outdir_base, key)
            try:
                os.mkdir(output_dir)
            except OSError:
                pass
            scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
            logfile = make_nfs_path(os.path.join(output_dir,
                                                 'scatter_mk_%s_%s.log' % (data_ver, key)))

            job_configs[key] = comp_dict.copy()
            job_configs[key].update(dict(ft1file=infile,
                                         scfile=scfile,
                                         comp=args['comp'],
                                         hpx_order_max=args['hpx_order_max'],
                                         outdir=outdir_base,
                                         outkey=key,
                                         logfile=logfile,
                                         pfiles=output_dir))

        return job_configs

class SplitAndMktimeChain(Chain):
    """Chain to run split and mktime and then make livetime and exposure cubes

    This chain consists of:

    split-and-mktime : `SplitAndMkTime_SG`
        Chain to make the binned counts maps for each input file

    coadd-split : `CoaddSplit_SG`
        Link to co-add the binnec counts maps files

    ltsum : `Gtltsum_SG`
        Link to co-add the livetime cube files

    expcube2 : `Gtexpcube2_SG`
        Link to make the corresponding binned exposure maps

    """
    appname = 'fermipy-split-and-mktime-chain'
    linkname_default = 'split-and-mktime-chain'
    usage = '%s [options]' % (appname)
    description = 'Run split-and-mktime, coadd-split and exposure'

    default_options = dict(data=diffuse_defaults.diffuse['data'],
                           comp=diffuse_defaults.diffuse['comp'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           hpx_order_ccube=diffuse_defaults.diffuse['hpx_order_ccube'],
                           hpx_order_expcube=diffuse_defaults.diffuse['hpx_order_expcube'],
                           do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
                           scratch=diffuse_defaults.diffuse['scratch'],
                           dry_run=diffuse_defaults.diffuse['dry_run'])

    __doc__ += Link.construct_docstring(default_options)

    def __init__(self, **kwargs):
        """C'tor
        """
        super(SplitAndMktimeChain, self).__init__(**kwargs)
        self.comp_dict = None

    def _map_arguments(self, args):
        """Map from the top-level arguments to the arguments provided to
        the indiviudal links """
        data = args.get('data')
        comp = args.get('comp')
        ft1file = args.get('ft1file')
        ft2file = args.get('ft2file')
        scratch = args.get('scratch', None)
        dry_run = args.get('dry_run', None)

        self._set_link('split-and-mktime', SplitAndMktime_SG,
                       comp=comp, data=data,
                       hpx_order_max=args.get('hpx_order_ccube', 9),
                       ft1file=ft1file,
                       ft2file=ft2file,
                       do_ltsum=args.get('do_ltsum', False),
                       scratch=scratch,
                       dry_run=dry_run)

        self._set_link('coadd-split', CoaddSplit_SG,
                       comp=comp, data=data,
                       ft1file=ft1file)

        self._set_link('ltsum', Gtltsum_SG,
                       comp=comp, data=data,
                       ft1file=args['ft1file'],
                       dry_run=dry_run)

        self._set_link('expcube2', Gtexpcube2_SG,
                       comp=comp, data=data,
                       hpx_order_max=args.get('hpx_order_expcube', 5),
                       dry_run=dry_run)

def register_classes():
    """Register these classes with the `LinkFactory` """
    SplitAndMktime.register_class()
    SplitAndMktime_SG.register_class()
    SplitAndMktimeChain.register_class()
<commit_msg>Added pfiles option in SplitAndMktime<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Prepare data for diffuse all-sky analysis
"""
from __future__ import absolute_import, division, print_function

import os
import math

import yaml

from fermipy.jobs.utils import is_null
from fermipy.jobs.chain import Link
from fermipy.jobs.chain import Chain
from fermipy.jobs.scatter_gather import ScatterGather
from fermipy.jobs.slac_impl import make_nfs_path

from fermipy.diffuse.utils import create_inputlist
from fermipy.diffuse.name_policy import NameFactory
from fermipy.diffuse import defaults as diffuse_defaults
from fermipy.diffuse.binning import EVT_TYPE_DICT
from fermipy.diffuse.timefilter import MktimeFilterDict

from fermipy.diffuse.job_library import Gtlink_select, Gtlink_mktime,\
    Gtlink_ltcube, Gtlink_bin, Gtltsum_SG, Gtexpcube2_SG

from fermipy.diffuse.gt_coadd_split import CoaddSplit_SG

NAME_FACTORY = NameFactory()
try:
    MKTIME_DICT = MktimeFilterDict.build_from_yamlfile('config/mktime_filters.yaml')
except IOError:
    MKTIME_DICT = MktimeFilterDict(aliases=dict(quality='lat_config==1&&data_qual>0'),
                                   selections=dict(standard='{quality}'))

def make_full_path(basedir, outkey, origname):
    """Make a full file path by combining tokens

    Parameters
    -----------

    basedir : str
        The top level output area

    outkey : str
        The key for the particular instance of the analysis

    origname : str
        Template for the output file name

    Returns
    -------

    outpath : str
        This will be <basedir>:<outkey>:<newname>.fits
        Where newname = origname.replace('.fits', '_<outkey>.fits')

    """
    return os.path.join(basedir, outkey,
                        os.path.basename(origname).replace('.fits',
                                                           '_%s.fits' % outkey))

class SplitAndMktime(Chain):
    """Small class to split, apply mktime and bin data according to some user-provided specification

    This chain consists multiple `Link` objects:

    select-energy-EBIN-ZCUT : `Gtlink_select`
        Initial splitting by energy bin and zenith angle cut

    mktime-EBIN-ZCUT-FILTER : `Gtlink_mktime`
        Application of gtmktime filter for zenith angle cut

    ltcube-EBIN-ZCUT-FILTER : `Gtlink_ltcube`
        Computation of livetime cube for zenith angle cut

    select-type-EBIN-ZCUT-FILTER-TYPE : `Gtlink_select`
        Refinement of selection from event types

    bin-EBIN-ZCUT-FILTER-TYPE : `Gtlink_bin`
        Final binning of the data for each event type

    """
    appname = 'fermipy-split-and-mktime'
    linkname_default = 'split-and-mktime'
    usage = '%s [options]' % (appname)
    description = 'Run gtselect and gtbin together'

    default_options = dict(comp=diffuse_defaults.diffuse['comp'],
                           data=diffuse_defaults.diffuse['data'],
                           hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           evclass=(128, 'Event class bit mask', int),
                           outdir=('counts_cubes', 'Output directory', str),
                           outkey=(None, 'Key for this particular output file', str),
                           pfiles=(None, 'Directory for .par files', str),
                           do_ltsum=(False, 'Sum livetime cube files', bool),
                           scratch=(None, 'Scratch area', str),
                           dry_run=(False, 'Print commands but do not run them', bool))

    __doc__ += Link.construct_docstring(default_options)

    def __init__(self, **kwargs):
        """C'tor
        """
        super(SplitAndMktime, self).__init__(**kwargs)
        self.comp_dict = None

    def _map_arguments(self, args):
        """Map from the top-level arguments to the arguments provided to
        the indiviudal links """
        comp_file = args.get('comp', None)
        datafile = args.get('data', None)
        if is_null(comp_file):
            return
        if is_null(datafile):
            return

        NAME_FACTORY.update_base_dict(args['data'])

        outdir = args.get('outdir')
        outkey = args.get('outkey')
        ft1file = args['ft1file']
        ft2file = args['ft2file']
        if is_null(outdir) or is_null(outkey):
            return
        pfiles = os.path.join(outdir, outkey)

        self.comp_dict = yaml.safe_load(open(comp_file))
        coordsys = self.comp_dict.pop('coordsys')

        full_out_dir = make_nfs_path(os.path.join(outdir, outkey))

        for key_e, comp_e in sorted(self.comp_dict.items()):
            emin = math.pow(10., comp_e['log_emin'])
            emax = math.pow(10., comp_e['log_emax'])
            enumbins = comp_e['enumbins']
            zmax = comp_e['zmax']
            zcut = "zmax%i" % comp_e['zmax']
            evclassstr = NAME_FACTORY.base_dict['evclass']

            kwargs_select = dict(zcut=zcut,
                                 ebin=key_e,
                                 psftype='ALL',
                                 coordsys=coordsys)
            linkname = 'select-energy-%s-%s' % (key_e, zcut)
            selectfile_energy = make_full_path(outdir, outkey, NAME_FACTORY.select(**kwargs_select))
            self._set_link(linkname, Gtlink_select,
                           infile=ft1file,
                           outfile=selectfile_energy,
                           zmax=zmax,
                           emin=emin,
                           emax=emax,
                           evclass=NAME_FACTORY.evclassmask(evclassstr),
                           pfiles=pfiles,
                           logfile=os.path.join(full_out_dir, "%s.log" % linkname))

            if 'mktimefilters' in comp_e:
                mktimefilters = comp_e['mktimefilters']
            else:
                mktimefilters = ['none']

            for mktimekey in mktimefilters:
                kwargs_mktime = kwargs_select.copy()
                kwargs_mktime['mktime'] = mktimekey
                filterstring = MKTIME_DICT[mktimekey]
                mktime_file = make_full_path(outdir, outkey, NAME_FACTORY.mktime(**kwargs_mktime))
                ltcube_file = make_full_path(outdir, outkey, NAME_FACTORY.ltcube(**kwargs_mktime))
                linkname_mktime = 'mktime-%s-%s-%s' % (key_e, zcut, mktimekey)
                linkname_ltcube = 'ltcube-%s-%s-%s' % (key_e, zcut, mktimekey)

                self._set_link(linkname_mktime, Gtlink_mktime,
                               evfile=selectfile_energy,
                               outfile=mktime_file,
                               scfile=ft2file,
                               filter=filterstring,
                               pfiles=pfiles,
                               logfile=os.path.join(full_out_dir, "%s.log" % linkname_mktime))
                self._set_link(linkname_ltcube, Gtlink_ltcube,
                               evfile=mktime_file,
                               outfile=ltcube_file,
                               scfile=ft2file,
                               zmax=zmax,
                               pfiles=pfiles,
                               logfile=os.path.join(full_out_dir, "%s.log" % linkname_ltcube))

                if 'evtclasses' in comp_e:
                    evtclasslist_vals = comp_e['evtclasses']
                else:
                    evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]

                for evtclassval in evtclasslist_vals:
                    for psf_type, psf_dict in sorted(comp_e['psf_types'].items()):
                        linkname_select = 'select-type-%s-%s-%s-%s-%s' % (
                            key_e, zcut, mktimekey, evtclassval, psf_type)
                        linkname_bin = 'bin-%s-%s-%s-%s-%s' % (key_e,
                                                               zcut, mktimekey,
                                                               evtclassval, psf_type)
                        kwargs_bin = kwargs_mktime.copy()
                        kwargs_bin['psftype'] = psf_type
                        kwargs_bin['coordsys'] = coordsys
                        kwargs_bin['evclass'] = evtclassval
                        selectfile_psf = make_full_path(
                            outdir, outkey, NAME_FACTORY.select(**kwargs_bin))
                        binfile_psf = make_full_path(
                            outdir, outkey, NAME_FACTORY.ccube(**kwargs_bin))
                        hpx_order_psf = min(args['hpx_order_max'], psf_dict['hpx_order'])
                        linkname_select = 'select-type-%s-%s-%s-%s-%s' % (key_e, zcut,
                                                                          mktimekey, evtclassval,
                                                                          psf_type)
                        linkname_bin = 'bin-%s-%s-%s-%s-%s' % (key_e, zcut, mktimekey,
                                                               evtclassval, psf_type)

                        self._set_link(linkname_select, Gtlink_select,
                                       infile=selectfile_energy,
                                       outfile=selectfile_psf,
                                       zmax=zmax,
                                       emin=emin,
                                       emax=emax,
                                       evtype=EVT_TYPE_DICT[psf_type],
                                       evclass=NAME_FACTORY.evclassmask(evtclassval),
                                       pfiles=pfiles,
                                       logfile=os.path.join(full_out_dir, "%s.log" % linkname_select))
                        self._set_link(linkname_bin, Gtlink_bin,
                                       coordsys=coordsys,
                                       hpx_order=hpx_order_psf,
                                       evfile=selectfile_psf,
                                       outfile=binfile_psf,
                                       emin=emin,
                                       emax=emax,
                                       enumbins=enumbins,
                                       pfiles=pfiles,
                                       logfile=os.path.join(full_out_dir, "%s.log" % linkname_bin))

class SplitAndMktime_SG(ScatterGather):
    """Small class to generate configurations for SplitAndMktime
    """
    appname = 'fermipy-split-and-mktime-sg'
    usage = "%s [options]" % (appname)
    description = "Prepare data for diffuse all-sky analysis"
    clientclass = SplitAndMktime

    job_time = 1500

    default_options = dict(comp=diffuse_defaults.diffuse['comp'],
                           data=diffuse_defaults.diffuse['data'],
                           hpx_order_max=diffuse_defaults.diffuse['hpx_order_ccube'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
                           scratch=diffuse_defaults.diffuse['scratch'],
                           dry_run=diffuse_defaults.diffuse['dry_run'])

    __doc__ += Link.construct_docstring(default_options)

    def build_job_configs(self, args):
        """Hook to build job configurations
        """
        job_configs = {}

        comp_file = args.get('comp', None)
        if comp_file is not None:
            comp_dict = yaml.safe_load(open(comp_file))
            coordsys = comp_dict.pop('coordsys')
            for v in comp_dict.values():
                v['coordsys'] = coordsys
        else:
            return job_configs

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        inputfiles = create_inputlist(args['ft1file'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
        data_ver = NAME_FACTORY.base_dict['data_ver']

        for idx, infile in enumerate(inputfiles):
            key = "%06i" % idx
            key_scfile = "%03i" % (idx + 1)
            output_dir = os.path.join(outdir_base, key)
            try:
                os.mkdir(output_dir)
            except OSError:
                pass
            scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
            logfile = make_nfs_path(os.path.join(output_dir,
                                                 'scatter_mk_%s_%s.log' % (data_ver, key)))

            job_configs[key] = comp_dict.copy()
            job_configs[key].update(dict(ft1file=infile,
                                         scfile=scfile,
                                         comp=args['comp'],
                                         hpx_order_max=args['hpx_order_max'],
                                         outdir=outdir_base,
                                         outkey=key,
                                         logfile=logfile,
                                         pfiles=output_dir))

        return job_configs

class SplitAndMktimeChain(Chain):
    """Chain to run split and mktime and then make livetime and exposure cubes

    This chain consists of:

    split-and-mktime : `SplitAndMkTime_SG`
        Chain to make the binned counts maps for each input file

    coadd-split : `CoaddSplit_SG`
        Link to co-add the binnec counts maps files

    ltsum : `Gtltsum_SG`
        Link to co-add the livetime cube files

    expcube2 : `Gtexpcube2_SG`
        Link to make the corresponding binned exposure maps

    """
    appname = 'fermipy-split-and-mktime-chain'
    linkname_default = 'split-and-mktime-chain'
    usage = '%s [options]' % (appname)
    description = 'Run split-and-mktime, coadd-split and exposure'

    default_options = dict(data=diffuse_defaults.diffuse['data'],
                           comp=diffuse_defaults.diffuse['comp'],
                           ft1file=diffuse_defaults.diffuse['ft1file'],
                           ft2file=diffuse_defaults.diffuse['ft2file'],
                           hpx_order_ccube=diffuse_defaults.diffuse['hpx_order_ccube'],
                           hpx_order_expcube=diffuse_defaults.diffuse['hpx_order_expcube'],
                           do_ltsum=diffuse_defaults.diffuse['do_ltsum'],
                           scratch=diffuse_defaults.diffuse['scratch'],
                           dry_run=diffuse_defaults.diffuse['dry_run'])

    __doc__ += Link.construct_docstring(default_options)

    def __init__(self, **kwargs):
        """C'tor
        """
        super(SplitAndMktimeChain, self).__init__(**kwargs)
        self.comp_dict = None

    def _map_arguments(self, args):
        """Map from the top-level arguments to the arguments provided to
        the indiviudal links """
        data = args.get('data')
        comp = args.get('comp')
        ft1file = args.get('ft1file')
        ft2file = args.get('ft2file')
        scratch = args.get('scratch', None)
        dry_run = args.get('dry_run', None)

        self._set_link('split-and-mktime', SplitAndMktime_SG,
                       comp=comp, data=data,
                       hpx_order_max=args.get('hpx_order_ccube', 9),
                       ft1file=ft1file,
                       ft2file=ft2file,
                       do_ltsum=args.get('do_ltsum', False),
                       scratch=scratch,
                       dry_run=dry_run)

        self._set_link('coadd-split', CoaddSplit_SG,
                       comp=comp, data=data,
                       ft1file=ft1file)

        self._set_link('ltsum', Gtltsum_SG,
                       comp=comp, data=data,
                       ft1file=args['ft1file'],
                       dry_run=dry_run)

        self._set_link('expcube2', Gtexpcube2_SG,
                       comp=comp, data=data,
                       hpx_order_max=args.get('hpx_order_expcube', 5),
                       dry_run=dry_run)

def register_classes():
    """Register these classes with the `LinkFactory` """
    SplitAndMktime.register_class()
    SplitAndMktime_SG.register_class()
    SplitAndMktimeChain.register_class()
<|endoftext|>


<commit_before>from balcaza.t2types import *
from balcaza.t2activity import BeanshellCode

DecodeBase64toBytes = BeanshellCode(
'''import org.apache.commons.codec.binary.Base64;

bytes = Base64.decodeBase64(base64.getBytes());
''',
	inputs = dict(
		base64 = String
		),
	outputs = dict(
		bytes = String
		),
	name = 'DecodeBase64toBytes'
	)<commit_msg>Complete base64 library<commit_after>from balcaza.t2types import *
from balcaza.t2activity import BeanshellCode

DecodeBase64toBytes = BeanshellCode(
'''import org.apache.commons.codec.binary.Base64;
bytes = Base64.decodeBase64(base64.getBytes());''',
	inputs = dict(
		base64 = String
		),
	outputs = dict(
		bytes = String
		),
	name = 'DecodeBase64toBytes'
	)

EncodeBytesToBase64 = BeanshellCode(
'''import org.apache.commons.codec.binary.Base64;
base64 = new String(Base64.encodeBase64(bytes));''',
	inputs = dict(
		bytes = String
		),
	outputs = dict(
		base64 = String
		),
	name = 'EncodeBytesToBase64'
	)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

from django.template import Library
from django.utils.html import mark_safe, format_html_join
from ..admin.tools import admin_url_params, admin_url_params_encoded

register = Library()

def filer_actions(context):
    """
    Track the number of times the action field has been rendered on the page,
    so we know which value to use.
    """
    context['action_index'] = context.get('action_index', -1) + 1
    return context
filer_actions = register.inclusion_tag(
    "admin/filer/actions.html", takes_context=True)(filer_actions)

@register.simple_tag(takes_context=True)
def filer_admin_context_url_params(context, first_separator='?'):
    return admin_url_params_encoded(
        context['request'], first_separator=first_separator)

@register.simple_tag(takes_context=True)
def filer_admin_context_hidden_formfields(context):
    request = context.get('request')
    return format_html_join(
        '\n',
        '<input type="hidden" name="{0}" value="{1}">',
        admin_url_params(request).items(),
    )
<commit_msg>make flake8 happy<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

from django.template import Library
from django.utils.html import format_html_join
from ..admin.tools import admin_url_params, admin_url_params_encoded

register = Library()

def filer_actions(context):
    """
    Track the number of times the action field has been rendered on the page,
    so we know which value to use.
    """
    context['action_index'] = context.get('action_index', -1) + 1
    return context
filer_actions = register.inclusion_tag(
    "admin/filer/actions.html", takes_context=True)(filer_actions)

@register.simple_tag(takes_context=True)
def filer_admin_context_url_params(context, first_separator='?'):
    return admin_url_params_encoded(
        context['request'], first_separator=first_separator)

@register.simple_tag(takes_context=True)
def filer_admin_context_hidden_formfields(context):
    request = context.get('request')
    return format_html_join(
        '\n',
        '<input type="hidden" name="{0}" value="{1}">',
        admin_url_params(request).items(),
    )
<|endoftext|>


<commit_before># Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
UCI Housing dataset.

This module will paddle.v2.dataset.common.download dataset from
https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and
parse training set and test set into paddle reader creators.
"""

import numpy as np
import os
import paddle.v2.dataset.common

__all__ = ['train', 'test']

URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
MD5 = 'd4accdce7a25600298819f8e28e8d593'
feature_names = [
    'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',
    'PTRATIO', 'B', 'LSTAT', 'convert'
]

UCI_TRAIN_DATA = None
UCI_TEST_DATA = None

def feature_range(maximums, minimums):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots()
    feature_num = len(maximums)
    ax.bar(range(feature_num), maximums - minimums, color='r', align='center')
    ax.set_title('feature scale')
    plt.xticks(range(feature_num), feature_names)
    plt.xlim([-1, feature_num])
    fig.set_figheight(6)
    fig.set_figwidth(10)
    if not os.path.exists('./image'):
        os.makedirs('./image')
    fig.savefig('image/ranges.png', dpi=48)
    plt.close(fig)

def load_data(filename, feature_num=14, ratio=0.8):
    global UCI_TRAIN_DATA, UCI_TEST_DATA
    if UCI_TRAIN_DATA is not None and UCI_TEST_DATA is not None:
        return

    data = np.fromfile(filename, sep=' ')
    data = data.reshape(data.shape[0] / feature_num, feature_num)
    maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum(
        axis=0) / data.shape[0]
    feature_range(maximums[:-1], minimums[:-1])
    for i in xrange(feature_num - 1):
        data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
    offset = int(data.shape[0] * ratio)
    UCI_TRAIN_DATA = data[:offset]
    UCI_TEST_DATA = data[offset:]

def train():
    """
    UCI_HOUSING training set creator.

    It returns a reader creator, each sample in the reader is features after
    normalization and price number.

    :return: Training reader creator
    :rtype: callable
    """
    global UCI_TRAIN_DATA
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))

    def reader():
        for d in UCI_TRAIN_DATA:
            yield d[:-1], d[-1:]

    return reader

def test():
    """
    UCI_HOUSING test set creator.

    It returns a reader creator, each sample in the reader is features after
    normalization and price number.

    :return: Test reader creator
    :rtype: callable
    """
    global UCI_TEST_DATA
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))

    def reader():
        for d in UCI_TEST_DATA:
            yield d[:-1], d[-1:]

    return reader

def fetch():
    paddle.v2.dataset.common.download(URL, 'uci_housing', MD5)

def convert(path):
    """
    Converts dataset to recordio format
    """
    paddle.v2.dataset.common.convert(path, train(), 10, "uci_housing_train")
    paddle.v2.dataset.common.convert(path, test(), 10, "uci_houseing_test")
<commit_msg>fix bugs<commit_after># Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
UCI Housing dataset.

This module will download dataset from
https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and
parse training set and test set into paddle reader creators.
"""

import numpy as np
import os
import paddle.v2.dataset.common

__all__ = ['train', 'test']

URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
MD5 = 'd4accdce7a25600298819f8e28e8d593'
feature_names = [
    'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',
    'PTRATIO', 'B', 'LSTAT', 'convert'
]

UCI_TRAIN_DATA = None
UCI_TEST_DATA = None

def feature_range(maximums, minimums):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots()
    feature_num = len(maximums)
    ax.bar(range(feature_num), maximums - minimums, color='r', align='center')
    ax.set_title('feature scale')
    plt.xticks(range(feature_num), feature_names)
    plt.xlim([-1, feature_num])
    fig.set_figheight(6)
    fig.set_figwidth(10)
    if not os.path.exists('./image'):
        os.makedirs('./image')
    fig.savefig('image/ranges.png', dpi=48)
    plt.close(fig)

def load_data(filename, feature_num=14, ratio=0.8):
    global UCI_TRAIN_DATA, UCI_TEST_DATA
    if UCI_TRAIN_DATA is not None and UCI_TEST_DATA is not None:
        return

    data = np.fromfile(filename, sep=' ')
    data = data.reshape(data.shape[0] / feature_num, feature_num)
    maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum(
        axis=0) / data.shape[0]
    feature_range(maximums[:-1], minimums[:-1])
    for i in xrange(feature_num - 1):
        data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
    offset = int(data.shape[0] * ratio)
    UCI_TRAIN_DATA = data[:offset]
    UCI_TEST_DATA = data[offset:]

def train():
    """
    UCI_HOUSING training set creator.

    It returns a reader creator, each sample in the reader is features after
    normalization and price number.

    :return: Training reader creator
    :rtype: callable
    """
    global UCI_TRAIN_DATA
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))

    def reader():
        for d in UCI_TRAIN_DATA:
            yield d[:-1], d[-1:]

    return reader

def test():
    """
    UCI_HOUSING test set creator.

    It returns a reader creator, each sample in the reader is features after
    normalization and price number.

    :return: Test reader creator
    :rtype: callable
    """
    global UCI_TEST_DATA
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))

    def reader():
        for d in UCI_TEST_DATA:
            yield d[:-1], d[-1:]

    return reader

def fetch():
    paddle.v2.dataset.common.download(URL, 'uci_housing', MD5)

def convert(path):
    """
    Converts dataset to recordio format
    """
    paddle.v2.dataset.common.convert(path, train(), 10, "uci_housing_train")
    paddle.v2.dataset.common.convert(path, test(), 10, "uci_houseing_test")
<|endoftext|>


<commit_before># Copyright 2014 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import itertools
import json
import os
import sys
import time
import unittest

from argus import exceptions
from argus import util

CONF = util.get_config()
# Use this logger to log both to standard output and to argus log file.
LOG = util.get_logger(name=__name__, format_string='%(message)s')
LOG.propagate = False

class _WritelnDecorator(object):
    """Used to decorate file-like objects with a handy 'writeln' method."""
    def __init__(self, stream):
        self.stream = stream

    def __getattr__(self, attr):
        if attr in ('stream', '__getstate__'):
            raise AttributeError(attr)
        return getattr(self.stream, attr)

    def writeln(self, arg=None):
        if arg:
            self.write(arg)
        self.write('\n')  # text-mode streams translate to \r\n if needed

class _TestResult(unittest.TextTestResult):
    def printErrorList(self, flavour, errors):
        for test, err in errors:
            LOG.info(self.separator1)
            LOG.info("%s: %s", flavour, self.getDescription(test))
            LOG.info(self.separator2)
            LOG.info("%s", err)

class Runner(object):
    """Scenarios runner class.

    Given a list of scenarios, this class iterates through each
    one and calls its underlying tests.
    """

    def __init__(self, scenarios, stream=None):
        self._scenarios = scenarios
        self._stream = _WritelnDecorator(stream or sys.stderr)

    def run(self):
        start_time = time.time()
        tests_run = 0
        expected_failures = unexpected_successes = skipped = 0
        failures = errors = 0

        # pylint: disable=redefined-outer-name
        for scenario in self._scenarios:
            try:
                result = scenario.run()
            except exceptions.ArgusError:
                # Something failed internally, report this scenario
                # as failed and continue to the other ones.
                tests = list(scenario.test_names())
                errors += len(tests)
                LOG.exception('Scenario %r failed', scenario.name)
            else:
                result.printErrors()
                tests_run += result.testsRun
                expected_failures += len(result.expectedFailures)
                unexpected_successes += len(result.unexpectedSuccesses)
                skipped += len(result.skipped)
                failures += len(result.failures)
                errors += len(result.errors)

        time_taken = time.time() - start_time

        LOG.info("\nRan %d test%s in %.3fs",
                 tests_run, tests_run != 1 and "s" or "", time_taken)

        if failures or errors:
            head = "FAILED"
        else:
            head = "OK"

        infos = []

        if failures or errors:
            if failures:
                infos.append("failures=%d" % failures)
            if errors:
                infos.append("errors=%d" % errors)

        if skipped:
            infos.append("skipped=%d" % skipped)
        if expected_failures:
            infos.append("expected failures=%d" % expected_failures)
        if unexpected_successes:
            infos.append("unexpected successes=%d" % unexpected_successes)

        if infos:
            LOG.info("%s (%s)", head, ", ".join(infos))
        else:
            LOG.info(head)

        return failures or errors

def _load_userdata(userdata):
    userdata, is_argus, part = userdata.partition("argus.")
    if is_argus:
        userdata = util.get_resource(part.replace(".", "/"))
    else:
        with open(userdata, 'rb') as stream:
            userdata = stream.read()
    return userdata

def _load_metadata(metadata):
    if os.path.isfile(metadata):
        with open(metadata) as stream:
            return json.load(stream)
    return json.loads(metadata)

def _build_scenarios_classes(scenario):
    """Return generic scenarios classes available for further customization."""
    opts = util.parse_cli()
    if opts.instance_output:
        try:
            os.makedirs(opts.instance_output)
        except OSError:
            pass

    test_result = _TestResult(_WritelnDecorator(sys.stdout), None, 0)

    if scenario.userdata:
        userdata = _load_userdata(scenario.userdata)
    else:
        userdata = None
    metadata = _load_metadata(scenario.metadata)
    test_classes = list(map(util.load_qualified_object,
                            scenario.test_classes))
    recipe = util.load_qualified_object(scenario.recipe)
    scenario_class = util.load_qualified_object(scenario.scenario)
    introspection = util.load_qualified_object(scenario.introspection)

    environment_preparer = None
    if scenario.environment:
        environment_preparer = util.load_qualified_object(
            scenario.environment.preparer)
        environment_preparer = environment_preparer(
            scenario.environment.config.config_file,
            scenario.environment.config.values,
            scenario.environment.start_commands,
            scenario.environment.stop_commands,
            scenario.environment.list_services_commands,
            scenario.environment.filter_services_regexes,
            scenario.environment.start_service_command,
            scenario.environment.stop_service_command)

    partial_scenario = functools.partial(
        scenario_class,
        name=scenario.name,
        test_classes=test_classes,
        recipe=recipe,
        metadata=metadata,
        userdata=userdata,
        service_type=scenario.service_type,
        introspection=introspection,
        result=test_result,
        output_directory=opts.instance_output,
        environment_preparer=environment_preparer)
    return [functools.partial(partial_scenario, image=image)
            for image in scenario.images]

def _cloud_scenarios_builder(partial_scenarios):
    """Build cloud specific custom scenarios objects."""
    opts = util.parse_cli()
    builds = set(opts.builds) if opts.builds else {util.BUILDS.Beta}
    arches = set(opts.arches) if opts.arches else {util.ARCHES.x64}
    scenarios = []
    for partial_scenario in partial_scenarios:
        for build in builds:
            for arch in arches:
                scenario = partial_scenario()
                scenario.build = build
                scenario.arch = arch
                scenarios.append(scenario)
    return scenarios

def _filter_scenarios(scenarios):
    """Filter the given scenarios according to some rules.

    The rules are passed at command line and the following
    rules are known:

      * os_type

          Use a scenario only if the image has this OS.
          Multiple OSes can be filtered.

      * test_type: Use a scenario only if it uses a particular test type.
    """
    opts = util.parse_cli()

    # Filter by OS type
    os_types = opts.test_os_types
    if os_types:
        scenarios = [scenario for scenario in scenarios
                     if scenario.image.os_type in os_types]

    # Filter by test_type
    scenario_type = opts.test_scenario_type
    if scenario_type:
        scenarios = [scenario for scenario in scenarios
                     if scenario.type and scenario.type == scenario_type]
    return scenarios

def run_scenarios():
    """Run all the defined scenarios in the configuration file.

    The function will filter all scenarios according to the requested OSes
    and test type. By default, all scenarios are executed.
    """
    opts = util.parse_cli()
    scenarios_builder = SCENARIOS_BUILDERS[opts.scenarios_builder]
    conf_scenarios = _filter_scenarios(CONF.scenarios)
    argus_scenarios = itertools.chain.from_iterable((
        scenarios_builder(_build_scenarios_classes(scenario))
        for scenario in conf_scenarios))
    return Runner(argus_scenarios).run()

SCENARIOS_BUILDERS = {
    "cloud": _cloud_scenarios_builder
}
<commit_msg>scenario runner catch broader exceptions<commit_after># Copyright 2014 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools
import itertools
import json
import os
import sys
import time
import unittest

from argus import exceptions
from argus import util

CONF = util.get_config()
# Use this logger to log both to standard output and to argus log file.
LOG = util.get_logger(name=__name__, format_string='%(message)s')
LOG.propagate = False

class _WritelnDecorator(object):
    """Used to decorate file-like objects with a handy 'writeln' method."""
    def __init__(self, stream):
        self.stream = stream

    def __getattr__(self, attr):
        if attr in ('stream', '__getstate__'):
            raise AttributeError(attr)
        return getattr(self.stream, attr)

    def writeln(self, arg=None):
        if arg:
            self.write(arg)
        self.write('\n')  # text-mode streams translate to \r\n if needed

class _TestResult(unittest.TextTestResult):
    def printErrorList(self, flavour, errors):
        for test, err in errors:
            LOG.info(self.separator1)
            LOG.info("%s: %s", flavour, self.getDescription(test))
            LOG.info(self.separator2)
            LOG.info("%s", err)

class Runner(object):
    """Scenarios runner class.

    Given a list of scenarios, this class iterates through each
    one and calls its underlying tests.
    """

    def __init__(self, scenarios, stream=None):
        self._scenarios = scenarios
        self._stream = _WritelnDecorator(stream or sys.stderr)

    def run(self):
        start_time = time.time()
        tests_run = 0
        expected_failures = unexpected_successes = skipped = 0
        failures = errors = 0

        # pylint: disable=redefined-outer-name
        for scenario in self._scenarios:
            try:
                result = scenario.run()
            except Exception:
                # Something failed internally, report this scenario
                # as failed and continue to the other ones.
                tests = list(scenario.test_names())
                errors += len(tests)
                LOG.exception('Scenario %r failed', scenario.name)
            else:
                result.printErrors()
                tests_run += result.testsRun
                expected_failures += len(result.expectedFailures)
                unexpected_successes += len(result.unexpectedSuccesses)
                skipped += len(result.skipped)
                failures += len(result.failures)
                errors += len(result.errors)

        time_taken = time.time() - start_time

        LOG.info("\nRan %d test%s in %.3fs",
                 tests_run, tests_run != 1 and "s" or "", time_taken)

        if failures or errors:
            head = "FAILED"
        else:
            head = "OK"

        infos = []

        if failures or errors:
            if failures:
                infos.append("failures=%d" % failures)
            if errors:
                infos.append("errors=%d" % errors)

        if skipped:
            infos.append("skipped=%d" % skipped)
        if expected_failures:
            infos.append("expected failures=%d" % expected_failures)
        if unexpected_successes:
            infos.append("unexpected successes=%d" % unexpected_successes)

        if infos:
            LOG.info("%s (%s)", head, ", ".join(infos))
        else:
            LOG.info(head)

        return failures or errors

def _load_userdata(userdata):
    userdata, is_argus, part = userdata.partition("argus.")
    if is_argus:
        userdata = util.get_resource(part.replace(".", "/"))
    else:
        with open(userdata, 'rb') as stream:
            userdata = stream.read()
    return userdata

def _load_metadata(metadata):
    if os.path.isfile(metadata):
        with open(metadata) as stream:
            return json.load(stream)
    return json.loads(metadata)

def _build_scenarios_classes(scenario):
    """Return generic scenarios classes available for further customization."""
    opts = util.parse_cli()
    if opts.instance_output:
        try:
            os.makedirs(opts.instance_output)
        except OSError:
            pass

    test_result = _TestResult(_WritelnDecorator(sys.stdout), None, 0)

    if scenario.userdata:
        userdata = _load_userdata(scenario.userdata)
    else:
        userdata = None
    metadata = _load_metadata(scenario.metadata)
    test_classes = list(map(util.load_qualified_object,
                            scenario.test_classes))
    recipe = util.load_qualified_object(scenario.recipe)
    scenario_class = util.load_qualified_object(scenario.scenario)
    introspection = util.load_qualified_object(scenario.introspection)

    environment_preparer = None
    if scenario.environment:
        environment_preparer = util.load_qualified_object(
            scenario.environment.preparer)
        environment_preparer = environment_preparer(
            scenario.environment.config.config_file,
            scenario.environment.config.values,
            scenario.environment.start_commands,
            scenario.environment.stop_commands,
            scenario.environment.list_services_commands,
            scenario.environment.filter_services_regexes,
            scenario.environment.start_service_command,
            scenario.environment.stop_service_command)

    partial_scenario = functools.partial(
        scenario_class,
        name=scenario.name,
        test_classes=test_classes,
        recipe=recipe,
        metadata=metadata,
        userdata=userdata,
        service_type=scenario.service_type,
        introspection=introspection,
        result=test_result,
        output_directory=opts.instance_output,
        environment_preparer=environment_preparer)
    return [functools.partial(partial_scenario, image=image)
            for image in scenario.images]

def _cloud_scenarios_builder(partial_scenarios):
    """Build cloud specific custom scenarios objects."""
    opts = util.parse_cli()
    builds = set(opts.builds) if opts.builds else {util.BUILDS.Beta}
    arches = set(opts.arches) if opts.arches else {util.ARCHES.x64}
    scenarios = []
    for partial_scenario in partial_scenarios:
        for build in builds:
            for arch in arches:
                scenario = partial_scenario()
                scenario.build = build
                scenario.arch = arch
                scenarios.append(scenario)
    return scenarios

def _filter_scenarios(scenarios):
    """Filter the given scenarios according to some rules.

    The rules are passed at command line and the following
    rules are known:

      * os_type

          Use a scenario only if the image has this OS.
          Multiple OSes can be filtered.

      * test_type: Use a scenario only if it uses a particular test type.
    """
    opts = util.parse_cli()

    # Filter by OS type
    os_types = opts.test_os_types
    if os_types:
        scenarios = [scenario for scenario in scenarios
                     if scenario.image.os_type in os_types]

    # Filter by test_type
    scenario_type = opts.test_scenario_type
    if scenario_type:
        scenarios = [scenario for scenario in scenarios
                     if scenario.type and scenario.type == scenario_type]
    return scenarios

def run_scenarios():
    """Run all the defined scenarios in the configuration file.

    The function will filter all scenarios according to the requested OSes
    and test type. By default, all scenarios are executed.
    """
    opts = util.parse_cli()
    scenarios_builder = SCENARIOS_BUILDERS[opts.scenarios_builder]
    conf_scenarios = _filter_scenarios(CONF.scenarios)
    argus_scenarios = itertools.chain.from_iterable((
        scenarios_builder(_build_scenarios_classes(scenario))
        for scenario in conf_scenarios))
    return Runner(argus_scenarios).run()

SCENARIOS_BUILDERS = {
    "cloud": _cloud_scenarios_builder
}
<|endoftext|>


<commit_before>import sys
import socket
import errno

logprefix = ''
verbose = 0

def log(s):
    try:
        sys.stdout.flush()
        sys.stderr.write(logprefix + s)
        sys.stderr.flush()
    except IOError:
        # this could happen if stderr gets forcibly disconnected, eg. because
        # our tty closes.  That sucks, but it's no reason to abort the program.
        pass

def debug1(s):
    if verbose >= 1:
        log(s)

def debug2(s):
    if verbose >= 2:
        log(s)

def debug3(s):
    if verbose >= 3:
        log(s)

class Fatal(Exception):
    pass

def list_contains_any(l, sub):
    for i in sub:
        if i in l:
            return True
    return False

def resolvconf_nameservers():
    l = []
    for line in open('/etc/resolv.conf'):
        words = line.lower().split()
        if len(words) >= 2 and words[0] == 'nameserver':
            l.append(family_ip_tuple(words[1]))
    return l

def resolvconf_random_nameserver():
    l = resolvconf_nameservers()
    if l:
        if len(l) > 1:
            # don't import this unless we really need it
            import random
            random.shuffle(l)
        return l[0]
    else:
        return (socket.AF_INET, '127.0.0.1')

def islocal(ip, family):
    sock = socket.socket(family)
    try:
        try:
            sock.bind((ip, 0))
        except socket.error as e:
            if e.args[0] == errno.EADDRNOTAVAIL:
                return False  # not a local IP
            else:
                raise
    finally:
        sock.close()
    return True  # it's a local IP, or there would have been an error

def family_ip_tuple(ip):
    if ':' in ip:
        return (socket.AF_INET6, ip)
    else:
        return (socket.AF_INET, ip)

def family_to_string(family):
    if family == socket.AF_INET6:
        return "AF_INET6"
    elif family == socket.AF_INET:
        return "AF_INET"
    else:
        return str(family)
<commit_msg>Remove unused function.<commit_after>import sys
import socket
import errno

logprefix = ''
verbose = 0

def log(s):
    try:
        sys.stdout.flush()
        sys.stderr.write(logprefix + s)
        sys.stderr.flush()
    except IOError:
        # this could happen if stderr gets forcibly disconnected, eg. because
        # our tty closes.  That sucks, but it's no reason to abort the program.
        pass

def debug1(s):
    if verbose >= 1:
        log(s)

def debug2(s):
    if verbose >= 2:
        log(s)

def debug3(s):
    if verbose >= 3:
        log(s)

class Fatal(Exception):
    pass

def resolvconf_nameservers():
    l = []
    for line in open('/etc/resolv.conf'):
        words = line.lower().split()
        if len(words) >= 2 and words[0] == 'nameserver':
            l.append(family_ip_tuple(words[1]))
    return l

def resolvconf_random_nameserver():
    l = resolvconf_nameservers()
    if l:
        if len(l) > 1:
            # don't import this unless we really need it
            import random
            random.shuffle(l)
        return l[0]
    else:
        return (socket.AF_INET, '127.0.0.1')

def islocal(ip, family):
    sock = socket.socket(family)
    try:
        try:
            sock.bind((ip, 0))
        except socket.error as e:
            if e.args[0] == errno.EADDRNOTAVAIL:
                return False  # not a local IP
            else:
                raise
    finally:
        sock.close()
    return True  # it's a local IP, or there would have been an error

def family_ip_tuple(ip):
    if ':' in ip:
        return (socket.AF_INET6, ip)
    else:
        return (socket.AF_INET, ip)

def family_to_string(family):
    if family == socket.AF_INET6:
        return "AF_INET6"
    elif family == socket.AF_INET:
        return "AF_INET"
    else:
        return str(family)
<|endoftext|>


<commit_before>import cv2
from random import randint
from threading import RLock
from server import globals
from board import histogram_util
from board.board_descriptor import BoardDescriptor

class BoardArea(object):
    """
    Represents a description of a board area.

    Field variables:
    board_area_pct -- [x1, y1, x2, y2] in percentage [0..1]
    board_descriptor -- Board descriptor
    area_image -- Extracted area image
    """
    lock = RLock()
    extracted_area_images = {}
    extracted_grayscaled_area_images = {}
    foreground_mask = None

    def __init__(self, area_id=None, board_area_pct=[0.0, 0.0, 1.0, 1.0], board_descriptor=None):
        """
        Initializes a tiled board area.

        :param area_id: Area id. Autogenerated if None.
        :param board_area_pct: Board area in percentage of board [x1, y1, x2, y2]
        :param board_descriptor: Board descriptor. If None, global board descriptor is used.
        """
        self.area_id = area_id if area_id is not None else randint(0, 100000)
        self.board_area_pct = board_area_pct
        self.board_descriptor = board_descriptor if board_descriptor is not None else globals.board_descriptor
        self.background_subtractor = cv2.BackgroundSubtractorMOG2(history=5, varThreshold=16)
        self.current_stability_score = 1.0
        self.current_board_snapshot_id = None

    def area_image(self, snapshot_size=BoardDescriptor.SnapshotSize.SMALL):
        """
        Extracts area image from board snapshot.

        :param snapshot_size: Snapshot size to use
        :return Extracted area image
        """

        with self.lock:

            # Check if board is recognized
            if not self.board_descriptor.is_recognized():
                return None

            # Check if snapshot has changed
            if self.current_board_snapshot_id is not self.board_descriptor.snapshot.id:
                self.extracted_area_images = {}
                self.extracted_grayscaled_area_images = {}

            # Return pre-processed area image
            if snapshot_size in self.extracted_area_images:
                return self.extracted_area_images[snapshot_size]

            # Save snapshot ID
            self.current_board_snapshot_id = self.board_descriptor.snapshot.id

            # Get board canvas image
            board_image = self.board_descriptor.board_canvas()
            image_height, image_width = board_image.shape[:2]

            # Extract area image
            x1 = int(float(image_width) * self.board_area_pct[0])
            y1 = int(float(image_height) * self.board_area_pct[1])
            x2 = int(float(image_width) * self.board_area_pct[2])
            y2 = int(float(image_height) * self.board_area_pct[3])

            self.extracted_area_images[snapshot_size] = board_image[y1:y2, x1:x2]

            return self.extracted_area_images[snapshot_size]

    def grayscaled_area_image(self, snapshot_size=BoardDescriptor.SnapshotSize.SMALL):
        """
        Extracts grayscaled area image from board snapshot.

        :param snapshot_size: Snapshot size to use
        :return Extracted area image
        """

        with self.lock:

            # Check if board is recognized
            if not self.board_descriptor.is_recognized():
                return None

            # Already extracted image
            if self.current_board_snapshot_id is self.board_descriptor.snapshot.id:
                if snapshot_size in self.extracted_grayscaled_area_images:
                    return self.extracted_grayscaled_area_images[snapshot_size]

            # Extract image
            image = self.area_image(snapshot_size)

            # Grayscale image
            self.extracted_grayscaled_area_images[snapshot_size] = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            return self.extracted_grayscaled_area_images[snapshot_size]

    def update_stability_score(self):

        with self.lock:

            # Get area image
            image = self.grayscaled_area_image(BoardDescriptor.SnapshotSize.SMALL)
            if self.foreground_mask is not None:
                mask_height, mask_width = self.foreground_mask.shape[:2]
                image = cv2.resize(image, (mask_width, mask_height))

            # Update foreground mask with area image
            self.foreground_mask = self.background_subtractor.apply(image, learningRate=0.5)

            mask_height, mask_width = self.foreground_mask.shape[:2]

            # Calculate stability score
            histogram = histogram_util.histogram_from_bw_image(self.foreground_mask)
            histogram_median = histogram_util.histogram_median(histogram, mask_width * mask_height)
            self.current_stability_score = 1.0 - (histogram_median / 255.0)

            #image_height, image_width = image.shape[:2]
            #print("Score %i: %f --- %i, %i" % (self.area_id, self.current_stability_score, image_width, image_height))
            #cv2.imwrite("image_%i.png" % self.area_id, image)
            #cv2.imwrite("mask_%i.png" % self.area_id, self.foreground_mask)

    def stability_score(self):
        with self.lock:
            return self.current_stability_score
<commit_msg>Board area size bug fix<commit_after>import cv2
from random import randint
from threading import RLock
from server import globals
from board import histogram_util
from board.board_descriptor import BoardDescriptor

class BoardArea(object):
    """
    Represents a description of a board area.

    Field variables:
    board_area_pct -- [x1, y1, x2, y2] in percentage [0..1]
    board_descriptor -- Board descriptor
    area_image -- Extracted area image
    """
    lock = RLock()
    extracted_area_images = {}
    extracted_grayscaled_area_images = {}
    foreground_mask = None

    def __init__(self, area_id=None, board_area_pct=[0.0, 0.0, 1.0, 1.0], board_descriptor=None):
        """
        Initializes a tiled board area.

        :param area_id: Area id. Autogenerated if None.
        :param board_area_pct: Board area in percentage of board [x1, y1, x2, y2]
        :param board_descriptor: Board descriptor. If None, global board descriptor is used.
        """
        self.area_id = area_id if area_id is not None else randint(0, 100000)
        self.board_area_pct = board_area_pct
        self.board_descriptor = board_descriptor if board_descriptor is not None else globals.board_descriptor
        self.background_subtractor = cv2.BackgroundSubtractorMOG2(history=5, varThreshold=16)
        self.current_stability_score = 1.0
        self.current_board_snapshot_id = None

    def area_image(self, snapshot_size=BoardDescriptor.SnapshotSize.SMALL):
        """
        Extracts area image from board snapshot.

        :param snapshot_size: Snapshot size to use
        :return Extracted area image
        """

        with self.lock:

            # Check if board is recognized
            if not self.board_descriptor.is_recognized():
                return None

            # Check if snapshot has changed
            if self.current_board_snapshot_id is not self.board_descriptor.snapshot.id:
                self.extracted_area_images = {}
                self.extracted_grayscaled_area_images = {}

            # Return pre-processed area image
            if snapshot_size in self.extracted_area_images:
                return self.extracted_area_images[snapshot_size]

            # Save snapshot ID
            self.current_board_snapshot_id = self.board_descriptor.snapshot.id

            # Get board canvas image
            board_image = self.board_descriptor.board_canvas(snapshot_size)
            image_height, image_width = board_image.shape[:2]

            # Extract area image
            x1 = int(float(image_width) * self.board_area_pct[0])
            y1 = int(float(image_height) * self.board_area_pct[1])
            x2 = int(float(image_width) * self.board_area_pct[2])
            y2 = int(float(image_height) * self.board_area_pct[3])

            self.extracted_area_images[snapshot_size] = board_image[y1:y2, x1:x2]

            return self.extracted_area_images[snapshot_size]

    def grayscaled_area_image(self, snapshot_size=BoardDescriptor.SnapshotSize.SMALL):
        """
        Extracts grayscaled area image from board snapshot.

        :param snapshot_size: Snapshot size to use
        :return Extracted area image
        """

        with self.lock:

            # Check if board is recognized
            if not self.board_descriptor.is_recognized():
                return None

            # Already extracted image
            if self.current_board_snapshot_id is self.board_descriptor.snapshot.id:
                if snapshot_size in self.extracted_grayscaled_area_images:
                    return self.extracted_grayscaled_area_images[snapshot_size]

            # Extract image
            image = self.area_image(snapshot_size)

            # Grayscale image
            self.extracted_grayscaled_area_images[snapshot_size] = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            return self.extracted_grayscaled_area_images[snapshot_size]

    def update_stability_score(self):

        with self.lock:

            # Get area image
            image = self.grayscaled_area_image(BoardDescriptor.SnapshotSize.SMALL)
            if self.foreground_mask is not None:
                mask_height, mask_width = self.foreground_mask.shape[:2]
                image = cv2.resize(image, (mask_width, mask_height))

            # Update foreground mask with area image
            self.foreground_mask = self.background_subtractor.apply(image, learningRate=0.5)

            mask_height, mask_width = self.foreground_mask.shape[:2]

            # Calculate stability score
            histogram = histogram_util.histogram_from_bw_image(self.foreground_mask)
            histogram_median = histogram_util.histogram_median(histogram, mask_width * mask_height)
            self.current_stability_score = 1.0 - (histogram_median / 255.0)

            #image_height, image_width = image.shape[:2]
            #print("Score %i: %f --- %i, %i" % (self.area_id, self.current_stability_score, image_width, image_height))
            #cv2.imwrite("image_%i.png" % self.area_id, image)
            #cv2.imwrite("mask_%i.png" % self.area_id, self.foreground_mask)

    def stability_score(self):
        with self.lock:
            return self.current_stability_score
<|endoftext|>