<commit_before>#!/usr/bin/env python
"""Generation of National Mesonet Project CSV File"""

import psycopg2.extras
from metpy.units import units
from pyiem.network import Table as NetworkTable
from pyiem.datatypes import distance, temperature
from pyiem.util import get_dbconn, ssw

def p(val, prec, minv, maxv):
    """rounder"""
    if val is None or val < minv or val > maxv:
        return "null"
    return round(val, prec)

def p2(val, prec, minv, maxv):
    """rounder"""
    if val is None or val < minv or val > maxv:
        return "null"
    return round(temperature(val, "C").value("K"), prec)

def use_table(table, hits):
    """Process for the given table."""
    nt = NetworkTable("ISUSM")
    pgconn = get_dbconn("isuag")
    cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cursor.execute(
        """
    with data as (
        select row_number() OVER (PARTITION by station ORDER by valid DESC),
        * from """
        + table
        + """
        where valid > now() - '48 hours'::interval
        and valid < now())
    SELECT *, valid at time zone 'UTC' as utc_valid from data
    where row_number = 1 ORDER by station ASC"""
    )
    for row in cursor:
        sid = row["station"]
        if sid in hits:
            continue
        hits.append(sid)
        ssw(
            (
                "%s,%.4f,%.4f,%s,%.1f,"
                "%.3f;%.3f;%.3f;%.3f#%s;%s;%s;%s,"
                "%.3f;%.3f;%.3f#%s;%s;%s,"
                "%s,2#%s#%s,%s,"
                "3#%s#%s#%s"
                "\n"
            )
            % (
                sid,
                nt.sts[sid]["lat"],
                nt.sts[sid]["lon"],
                row["utc_valid"].strftime("%Y-%m-%dT%H:%M:%SZ"),
                nt.sts[sid]["elevation"],
                distance(4, "IN").value("M"),
                distance(12, "IN").value("M"),
                distance(24, "IN").value("M"),
                distance(50, "IN").value("M"),
                p2(row["tsoil_c_avg_qc"], 3, -90, 90),
                p2(row["t12_c_avg_qc"], 3, -90, 90),
                p2(row["t24_c_avg_qc"], 3, -90, 90),
                p2(row["t50_c_avg_qc"], 3, -90, 90),
                distance(12, "IN").value("M"),
                distance(24, "IN").value("M"),
                distance(50, "IN").value("M"),
                p(
                    row.get("vwc_12_avg_qc", row.get("calcvwc12_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(
                    row.get("vwc_24_avg_qc", row.get("calcvwc24_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(
                    row.get("vwc_50_avg_qc", row.get("calcvwc50_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(row.get("slrkw_avg_qc"), 1, 0, 1600),
                p2(row["tair_c_avg_qc"], 1, -90, 90),
                p(row.get("rh_qc", row.get("rh_avg_qc")), 1, 0, 1600),
                p(row.get("rain_mm_tot_qc"), 2, 0, 100),
                p(row.get("ws_mps_s_wvt_qc"), 2, 0, 100),
                p(
                    (0 * units("mph")).to(units("meter / second")).m,  # BUG
                    2,
                    0,
                    100,
                ),
                p(row.get("winddir_d1_wvt_qc"), 2, 0, 360),
            )
        )

def do_output():
    """Do as I say"""

    ssw(
        (
            "station_id,LAT [degN],LON [degE],date_time,ELEV [m],"
            "depth [m]#SOILT [K],depth [m]#SOILM [kg/kg],"
            "GSRD[1]H [W/m^2],height [m]#T [K]#RH [%],"
            "PCP[1]H [mm],"
            "height [m]#FF[1]H [m/s]#FFMAX[1]H [m/s]#DD[1]H [degN]\n"
        )
    )

    hits = []
    use_table("sm_minute", hits)
    use_table("sm_hourly", hits)
    ssw(".EOO\n")

def main():
    """Do Something"""
    ssw("Content-type: text/csv;header=present\n\n")
    do_output()

if __name__ == "__main__":
    main()
<commit_msg>fix wind gust info in NMP output<commit_after>#!/usr/bin/env python
"""Generation of National Mesonet Project CSV File"""

import numpy as np
import psycopg2.extras
from metpy.units import units
from pyiem.network import Table as NetworkTable
from pyiem.datatypes import distance, temperature
from pyiem.util import get_dbconn, ssw

def nan(val):
    """Convert to NaN, if necessary."""
    return np.nan if val is None else val

def p(val, prec, minv, maxv):
    """rounder"""
    if val is None or val < minv or val > maxv:
        return "null"
    return round(val, prec)

def p2(val, prec, minv, maxv):
    """rounder"""
    if val is None or val < minv or val > maxv:
        return "null"
    return round(temperature(val, "C").value("K"), prec)

def use_table(table, hits):
    """Process for the given table."""
    nt = NetworkTable("ISUSM")
    pgconn = get_dbconn("isuag")
    cursor = pgconn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cursor.execute(
        """
    with data as (
        select row_number() OVER (PARTITION by station ORDER by valid DESC),
        * from """
        + table
        + """
        where valid > now() - '48 hours'::interval
        and valid < now())
    SELECT *, valid at time zone 'UTC' as utc_valid from data
    where row_number = 1 ORDER by station ASC"""
    )
    for row in cursor:
        sid = row["station"]
        if sid in hits:
            continue
        hits.append(sid)
        ssw(
            (
                "%s,%.4f,%.4f,%s,%.1f,"
                "%.3f;%.3f;%.3f;%.3f#%s;%s;%s;%s,"
                "%.3f;%.3f;%.3f#%s;%s;%s,"
                "%s,2#%s#%s,%s,"
                "3#%s#%s#%s"
                "\n"
            )
            % (
                sid,
                nt.sts[sid]["lat"],
                nt.sts[sid]["lon"],
                row["utc_valid"].strftime("%Y-%m-%dT%H:%M:%SZ"),
                nt.sts[sid]["elevation"],
                distance(4, "IN").value("M"),
                distance(12, "IN").value("M"),
                distance(24, "IN").value("M"),
                distance(50, "IN").value("M"),
                p2(row["tsoil_c_avg_qc"], 3, -90, 90),
                p2(row["t12_c_avg_qc"], 3, -90, 90),
                p2(row["t24_c_avg_qc"], 3, -90, 90),
                p2(row["t50_c_avg_qc"], 3, -90, 90),
                distance(12, "IN").value("M"),
                distance(24, "IN").value("M"),
                distance(50, "IN").value("M"),
                p(
                    row.get("vwc_12_avg_qc", row.get("calcvwc12_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(
                    row.get("vwc_24_avg_qc", row.get("calcvwc24_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(
                    row.get("vwc_50_avg_qc", row.get("calcvwc50_avg_qc")),
                    1,
                    0,
                    100,
                ),
                p(row.get("slrkw_avg_qc"), 1, 0, 1600),
                p2(row["tair_c_avg_qc"], 1, -90, 90),
                p(row.get("rh_qc", row.get("rh_avg_qc")), 1, 0, 1600),
                p(row.get("rain_mm_tot_qc"), 2, 0, 100),
                p(row.get("ws_mps_s_wvt_qc"), 2, 0, 100),
                p(
                    (nan(row.get("ws_mph_max_qc", 0)) * units("mph"))
                    .to(units("meter / second"))
                    .m,  # BUG
                    2,
                    0,
                    100,
                ),
                p(row.get("winddir_d1_wvt_qc"), 2, 0, 360),
            )
        )

def do_output():
    """Do as I say"""

    ssw(
        (
            "station_id,LAT [degN],LON [degE],date_time,ELEV [m],"
            "depth [m]#SOILT [K],depth [m]#SOILM [kg/kg],"
            "GSRD[1]H [W/m^2],height [m]#T [K]#RH [%],"
            "PCP[1]H [mm],"
            "height [m]#FF[1]H [m/s]#FFMAX[1]H [m/s]#DD[1]H [degN]\n"
        )
    )

    hits = []
    use_table("sm_minute", hits)
    use_table("sm_hourly", hits)
    ssw(".EOO\n")

def main():
    """Do Something"""
    ssw("Content-type: text/csv;header=present\n\n")
    do_output()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os

DRIVER = 'src.devops.driver.libvirt.libvirt_driver.LibvirtDriver'
HOME_DIR = os.environ.get('DEVOPS_HOME') or os.environ.get('APPDATA') or os.environ['HOME']
INSTALLED_APPS = ['devops']

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.abspath(os.path.join(HOME_DIR, 'devops.db')),
        'TIMEOUT': 60,
    }
}<commit_msg>change default database to postgres<commit_after>import os

DRIVER = 'src.devops.driver.libvirt.libvirt_driver.LibvirtDriver'
HOME_DIR = os.environ.get('DEVOPS_HOME') or os.environ.get('APPDATA') or os.environ['HOME']
INSTALLED_APPS = ['devops']

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'postgres',
        'USER': 'postgres',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
        'TEST_CHARSET': 'UTF8'
    }
}<|endoftext|>


<commit_before>from flyingpigeon.utils import archive, archiveextract
from flyingpigeon.utils import rename_complexinputs
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, get_values, get_time
from numpy import savetxt, column_stack
from shapely.geometry import Point

from pywps import Process
from pywps import LiteralInput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS
from pywps.app.Common import Metadata

from flyingpigeon.log import init_process_logger

import logging
LOGGER = logging.getLogger("PYWPS")

class PointinspectionProcess(Process):
    """
    TODO: optionally provide point list as file (csv, geojson) and WFS service
    """
    def __init__(self):
        inputs = [
            ComplexInput('resource', 'Resource',
                         abstract='NetCDF Files or archive (tar/zip) containing NetCDF files.',
                         metadata=[Metadata('Info')],
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            LiteralInput("coords", "Coordinates",
                         abstract="A comma-seperated tuple of WGS85 lon,lat decimal coordinates (e.g. 2.356138, 48.846450)",  # noqa
                         default="2.356138, 48.846450",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=100,
                         ),
                         ]
        outputs = [
            ComplexOutput('output_log', 'Logging information',
                          abstract="Collected logs during process run.",
                          as_reference=True,
                          supported_formats=[Format('text/plain')]
                          ),

            ComplexOutput('tarout', 'Subsets',
                          abstract="Tar archive containing the netCDF files",
                          as_reference=True,
                          supported_formats=[Format('application/x-tar')]
                          ),
        ]

        super(PointinspectionProcess, self).__init__(
            self._handler,
            identifier="pointinspection",
            title="Point Inspection",
            abstract='Extract the timeseries at the given coordinates.',
            version="0.10",
            metadata=[
                Metadata('LSCE', 'http://www.lsce.ipsl.fr/en/index.php'),
                Metadata('Doc', 'http://flyingpigeon.readthedocs.io/en/latest/'),
            ],
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        init_process_logger('log.txt')
        response.outputs['output_log'].file = 'log.txt'

        ncs = archiveextract(
            resource=rename_complexinputs(request.inputs['resource']))
        LOGGER.info("ncs: %s " % ncs)

        coords = []
        for coord in request.inputs['coords']:
            coords.append(coord.data)

        LOGGER.info("coords %s", coords)
        filenames = []
        nc_exp = sort_by_filename(ncs, historical_concatination=True)

        for key in nc_exp.keys():
            try:
                LOGGER.info('start calculation for %s ' % key)
                ncs = nc_exp[key]
                times = get_time(ncs, format='%Y-%m-%d_%H:%M:%S')
                concat_vals = times  # ['%s-%02d-%02d_%02d:%02d:%02d' %
                # (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
                header = 'date_time'
                filename = '%s.csv' % key
                filenames.append(filename)

                for p in coords:
                    try:
                        response.update_status('processing point : {0}'.format(p), 20)
                        # define the point:
                        p = p.split(',')
                        point = Point(float(p[0]), float(p[1]))

                        # get the values
                        timeseries = call(resource=ncs, geom=point, select_nearest=True)
                        vals = get_values(timeseries)

                        # concatenation of values
                        header = header + ',%s-%s' % (p[0], p[1])
                        concat_vals = column_stack([concat_vals, vals])
                    except Exception as e:
                        LOGGER.debug('failed for point %s %s' % (p, e))
                response.update_status('*** all points processed for {0} ****'.format(key), 50)
                savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
            except Exception as e:
                LOGGER.debug('failed for %s %s' % (key, e))

    # set the outputs
        response.update_status('*** creating output tar archive ****', 90)
        tarout_file = archive(filenames)
        response.outputs['tarout'].file = tarout_file
        return response
<commit_msg>remove format from get_time<commit_after>from flyingpigeon.utils import archive, archiveextract
from flyingpigeon.utils import rename_complexinputs
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, get_values, get_time
from numpy import savetxt, column_stack
from shapely.geometry import Point

from pywps import Process
from pywps import LiteralInput
from pywps import ComplexInput, ComplexOutput
from pywps import Format, FORMATS
from pywps.app.Common import Metadata

from flyingpigeon.log import init_process_logger

import logging
LOGGER = logging.getLogger("PYWPS")

class PointinspectionProcess(Process):
    """
    TODO: optionally provide point list as file (csv, geojson) and WFS service
    """
    def __init__(self):
        inputs = [
            ComplexInput('resource', 'Resource',
                         abstract='NetCDF Files or archive (tar/zip) containing NetCDF files.',
                         metadata=[Metadata('Info')],
                         min_occurs=1,
                         max_occurs=1000,
                         supported_formats=[
                             Format('application/x-netcdf'),
                             Format('application/x-tar'),
                             Format('application/zip'),
                         ]),

            LiteralInput("coords", "Coordinates",
                         abstract="A comma-seperated tuple of WGS85 lon,lat decimal coordinates (e.g. 2.356138, 48.846450)",  # noqa
                         default="2.356138, 48.846450",
                         data_type='string',
                         min_occurs=1,
                         max_occurs=100,
                         ),
                         ]
        outputs = [
            ComplexOutput('output_log', 'Logging information',
                          abstract="Collected logs during process run.",
                          as_reference=True,
                          supported_formats=[Format('text/plain')]
                          ),

            ComplexOutput('tarout', 'Subsets',
                          abstract="Tar archive containing the netCDF files",
                          as_reference=True,
                          supported_formats=[Format('application/x-tar')]
                          ),
        ]

        super(PointinspectionProcess, self).__init__(
            self._handler,
            identifier="pointinspection",
            title="Point Inspection",
            abstract='Extract the timeseries at the given coordinates.',
            version="0.10",
            metadata=[
                Metadata('LSCE', 'http://www.lsce.ipsl.fr/en/index.php'),
                Metadata('Doc', 'http://flyingpigeon.readthedocs.io/en/latest/'),
            ],
            inputs=inputs,
            outputs=outputs,
            status_supported=True,
            store_supported=True,
        )

    def _handler(self, request, response):
        init_process_logger('log.txt')
        response.outputs['output_log'].file = 'log.txt'

        ncs = archiveextract(
            resource=rename_complexinputs(request.inputs['resource']))
        LOGGER.info("ncs: %s " % ncs)

        coords = []
        for coord in request.inputs['coords']:
            coords.append(coord.data)

        LOGGER.info("coords %s", coords)
        filenames = []
        nc_exp = sort_by_filename(ncs, historical_concatination=True)

        for key in nc_exp.keys():
            try:
                LOGGER.info('start calculation for %s ' % key)
                ncs = nc_exp[key]
                times = get_time(ncs)  # , format='%Y-%m-%d_%H:%M:%S')
                concat_vals = times  # ['%s-%02d-%02d_%02d:%02d:%02d' %
                # (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
                header = 'date_time'
                filename = '%s.csv' % key
                filenames.append(filename)

                for p in coords:
                    try:
                        response.update_status('processing point : {0}'.format(p), 20)
                        # define the point:
                        p = p.split(',')
                        point = Point(float(p[0]), float(p[1]))

                        # get the values
                        timeseries = call(resource=ncs, geom=point, select_nearest=True)
                        vals = get_values(timeseries)

                        # concatenation of values
                        header = header + ',%s-%s' % (p[0], p[1])
                        concat_vals = column_stack([concat_vals, vals])
                    except Exception as e:
                        LOGGER.debug('failed for point %s %s' % (p, e))
                response.update_status('*** all points processed for {0} ****'.format(key), 50)
                savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
            except Exception as e:
                LOGGER.debug('failed for %s %s' % (key, e))

    # set the outputs
        response.update_status('*** creating output tar archive ****', 90)
        tarout_file = archive(filenames)
        response.outputs['tarout'].file = tarout_file
        return response
<|endoftext|>


<commit_before>#!/usr/bin/python

import MySQLdb as mdb
import sys

def get_db_version(con):
    cur = con.cursor()
    cur.execute("SELECT VERSION()")
    ver = cur.fetchone()
    return ver

def create_table(con):
    cur = con.cursor()
    cur.execute("create table bw_internet ( timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, rx bigint unsigned DEFAULT 0, tx bigINT unsigned DEFAULT 0, PRIMARY KEY (timestamp) )")

def add_data(con, data):
    cur = con.cursor()
    cur.execute("INSERT INTO bw_internet(rx, tx) VALUES('%s', '%s')" % (data.get("rx"), data.get("tx")))
    return cur.rowcount

def get_data(con):
    # if you use the following, you can refer to the tuple by the db table column name.
    #cur = con.cursor(mdb.cursors.DictCursor)
    cur = con.cursor()
    #  cur.description   to print header name print "%s %3s" % (desc[0][0], desc[1][0])
    cur.execute("SELECT * FROM bw_internet")
    rows = cur.fetchall()
    return rows

def update_data(con, data):
    cur = con.cursor()
    cur.execute("UPDATE bw_internet SET rx = %s, tx = %s WHERE timestamp = %s", (data.get(""), data.get(""), data.get("")))
    return cur.rowcount

if __name__ == '__main__':
    try:
        #TODO read from config file.
        con = mdb.connect('localhost', 'user', 'test', 'monitor');
        version = get_db_version(con)
        print "Database version: %s " % version

        rows = get_data(con)
        print rows[0]

    except mdb.Error, e:
        print "Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
    finally:
        if con:
            con.close()
<commit_msg>added todo<commit_after>#!/usr/bin/python

import MySQLdb as mdb
import sys

def get_db_version(con):
    cur = con.cursor()
    cur.execute("SELECT VERSION()")
    ver = cur.fetchone()
    return ver

def create_table(con):
    cur = con.cursor()
    cur.execute("create table bw_internet ( timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, rx bigint unsigned DEFAULT 0, tx bigINT unsigned DEFAULT 0, PRIMARY KEY (timestamp) )")

def add_data(con, data):
    cur = con.cursor()
    cur.execute("INSERT INTO bw_internet(rx, tx) VALUES('%s', '%s')" % (data.get("rx"), data.get("tx")))
    return cur.rowcount

def get_data(con):
    # if you use the following, you can refer to the tuple by the db table column name.
    #cur = con.cursor(mdb.cursors.DictCursor)
    cur = con.cursor()
    #  cur.description   to print header name print "%s %3s" % (desc[0][0], desc[1][0])
    cur.execute("SELECT * FROM bw_internet")
    rows = cur.fetchall()
    return rows

def update_data(con, data):
    cur = con.cursor()
    cur.execute("UPDATE bw_internet SET rx = %s, tx = %s WHERE timestamp = %s", (data.get(""), data.get(""), data.get("")))
    return cur.rowcount

#TODO way overdue :D
def get_cpu_from_router():
    pass

#TODO way overdue :D
def get_mem_stat_from_router():
    pass

if __name__ == '__main__':
    con=""
    try:
        #TODO read from config file.
        con = mdb.connect('localhost', 'user', 'test', 'monitor');
        version = get_db_version(con)
        print "Database version: %s " % version

        rows = get_data(con)
        print rows[0]

    except mdb.Error, e:
        print "Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)
    finally:
        if con:
            con.close()
<|endoftext|>


<commit_before>from django.contrib import admin
from django.conf.urls.defaults import patterns, url
from models import MuseumObject, FunctionalCategory
from models import CulturalBloc, ArtefactType, Category
from models import AcquisitionMethod, LoanStatus
from models import AccessStatus, Obtained
from models import RecordStatus
from mediaman.models import ArtefactRepresentation
from common.admin import UndeleteableModelAdmin
from common.adminactions import merge_selected, add_to_collection
from common.adminactions import generate_xls, add_to_opener
from admin_views import search_home, search_xls
from django.core import urlresolvers
from django.utils.datastructures import SortedDict
from django.db.models import F
from location.models import GlobalRegion, Country, StateProvince, RegionDistrict, Locality

class MediaFileInline(admin.TabularInline):
    extra = 0
    search_fields = ['name', ]

    def has_add_permission(self, request):
        return False

class ArtefactRepInline(MediaFileInline):
    model = ArtefactRepresentation
#   position field hidden with CSS
    fields = ('name', 'image', 'thumbnail', 'view_admin_record', 'public', 'position')

    def thumbnail(self, obj):
        try:
            thumb_opts = {'size': (64, 64), 'watermark': ''}
            thumb = obj.image.get_thumbnail(thumb_opts)
            return '' % (obj.image.url, thumb.url)
        except:
            return 'Error generating thumbnail'

    def view_admin_record(self, obj):
        try:
            admin_url = urlresolvers.reverse('admin:mediaman_artefactrepresentation_change', args=(obj.id,))
            return 'View image record' % (admin_url,)
        except:
            return ''
    view_admin_record.allow_tags = True
    thumbnail.allow_tags = True
    readonly_fields = ('name', 'image', 'thumbnail', 'view_admin_record')
    sortable_field_name = "position"

class DocumentInline(MediaFileInline):
    model = MuseumObject.related_documents.through
    fields = ('view_document', 'admin_doc_link', 'is_public')
    readonly_fields = ('view_document', 'admin_doc_link', 'is_public')

    def is_public(self, obj):
        return obj.document.public
    is_public.boolean = True

    def view_document(self, obj):
        try:
            doc = obj.document.document
            return '%s' % (doc.url, doc.name)
        except:
            return ''

    def admin_doc_link(self, obj):
        try:
            doc = obj.document
            admin_url = urlresolvers.reverse('admin:mediaman_document_change', args=(doc.id,))
            return 'View document record' % (admin_url,)
        except:
            return ''
    view_document.allow_tags = True
    admin_doc_link.allow_tags = True
    verbose_name_plural = 'Related documents'

from django.contrib.admin import SimpleListFilter

class HierarchyListFilter(SimpleListFilter):
    """
    An admin list filter for hierarchical data

    Used for the geo-locations. Is not directly usable, must be overridden
    with appropriate class fields set.

    Designed to be used in order of the hierarchy. Will set a flag and stop
    processing children if a higher level filter is set. So checks the children/parents
    for validity.
    """
    def __init__(self, request, *args):
        super(HierarchyListFilter, self).__init__(
            request, *args)

        if self.is_invalid_parent(self.get_parent(request)):
            request.stop_processing_hierarchy_filter = True

    def is_invalid_parent(self, parent):
        current = None

        val = self.value()
        if val:
            current = self.model.objects.get(id=val)
        return current and current.parent != parent

    def get_parent(self, request):
        if hasattr(self, '_parent'):
            return self._parent
        parent_id = request.GET.get(self.parent_parameter_name)
        parent_field = self.model._meta.get_field_by_name('parent')[0]
        parent_type = parent_field.rel.to
        self._parent = parent_type.objects.filter(id=parent_id)[0] if parent_id else None
        return self._parent

    def lookups(self, request, model_admin):
        if hasattr(request, 'stop_processing_hierarchy_filter'):
            return []
        parent = self.get_parent(request)
        options = self.model.objects.filter(parent=parent)
        return [(str(val.id), val.name) for val in options]

    def queryset(self, request, queryset):
        print 'queryset(): %s' % self.title
        if self.value() and not hasattr(request, 'stop_processing_hierarchy_filter'):
            args = {self.parameter_name: self.value()}
            return queryset.filter(**args)
        else:
            return queryset

class CountryListFilter(HierarchyListFilter):
    title = 'country'
    parameter_name = 'country_id'
    parent_parameter_name = 'global_region__id__exact'
    model = Country

class StateProvinceListFilter(HierarchyListFilter):
    title = 'state/province'
    parameter_name = 'state_province_id'
    parent_parameter_name = 'country_id'
    model = StateProvince

class RegionDistrictListFilter(HierarchyListFilter):
    title = 'region/district'
    parameter_name = 'region_district_id'
    parent_parameter_name = 'state_province_id'
    model = RegionDistrict

class LocalityListFilter(HierarchyListFilter):
    title = 'locality'
    parameter_name = 'country_id'
    parent_parameter_name = 'region_district_id'
    model = Locality

class MOAdmin(UndeleteableModelAdmin):
    list_display = ('registration_number',
                    'description', 'comment', 'public', 'is_public_comment')
    readonly_fields = ('place', 'cultural_bloc', 'functional_category',
        'donor_2', 'collector_2', 'old_registration_number',)

    list_filter = ('artefact_type', 'category',
                    'access_status', 'loan_status',
                    'collector', 'donor', 'record_status',
                    'global_region', CountryListFilter, StateProvinceListFilter,
                    RegionDistrictListFilter, LocalityListFilter, 'is_public_comment',
                    'public', 'maker')

    search_fields = ['registration_number', 'description', 'comment',
                     'donor__name', 'collector__name', 'maker__name',
                     'global_region__name', 'country__name', 'state_province__name',
                     'region_district__name', 'locality__name']

    inlines = [
            DocumentInline, ArtefactRepInline
    ]

    raw_id_fields = ('category', 'place', 'collector', 'collector_2',
            'donor', 'donor_2', 'artefact_type', 'maker')
    autocomplete_lookup_fields = {
            'fk': ['place', 'collector', 'donor', 'artefact_type',
                   'maker'],
            'm2m': ['category']
    }
    fieldsets = (
        (None, {
            'fields': ('registration_number', 'old_registration_number',
                       'other_number', 'functional_category', 'category',
                       'artefact_type')
        }),
        ('Registration', {
            'fields': ('registered_by', 'registration_date')
        }),
        ('Old place information', {
            'fields': ('cultural_bloc', 'place',)
        }),
        ('Geo-location', {
            'classes': ('collapse',),
            'fields': ('global_region', 'country', 'state_province',
                'region_district', 'locality')
        }),
        ('Status', {
            'classes': ('collapse',),
            'fields': ('loan_status', 'access_status', 'record_status',)
        }),
        ('Storage location', {
            'classes': ('collapse',),
            'fields': ('storage_section', 'storage_unit',
                'storage_bay', 'storage_shelf_box_drawer')
        }),
        ('Acquisition', {
            'classes': ('collapse',),
            'fields': ('acquisition_date', 'acquisition_method',
                       'reg_info')
        }),
        ('Collector', {
            'classes': ('collapse',),
            'fields': ('collector', 'collector_2',
                'how_collector_obtained', 'when_collector_obtained')
        }),
        ('Donor', {
            'classes': ('collapse',),
            'fields': ('donor', 'donor_2',
                'how_donor_obtained', 'when_donor_obtained')
        }),
        ('Details', {
            'classes': ('collapse',),
            'fields': ('description', 'is_public_comment',
                'comment', 'private_comment', 'significance')
        }),
        ('Extra details', {
            'fields': ('maker', 'manufacture_technique', 'creation_date',
                'site_name_number', 'raw_material', 'indigenous_name',
                'recorded_use', 'assoc_cultural_group',
                'exhibition_history', 'category_illustrated',
                'artefact_illustrated')
        }),
        ('Location', {
            'classes': ('collapse',),
            'fields': (('longitude', 'latitude'),)
        }),
        ('Dimensions', {
            'classes': ('collapse',),
            'fields': (('length', 'width', 'height'),
                ('depth', 'circumference'))
        }),
    )

    #### SETUP ADMIN ACTIONS #####
    actions = [add_to_collection, generate_xls, 'make_comment_public',
        'make_comment_private', 'make_record_public', 'make_record_private',
        'move_comment_to_reg_info']

    def make_comment_public(self, request, queryset):
        queryset.update(is_public_comment=True)
    make_comment_public.short_description = "Make comments public"

    def make_comment_private(self, request, queryset):
        queryset.update(is_public_comment=False)
    make_comment_private.short_description = "Make comments private"

    def make_record_public(self, request, queryset):
        queryset.update(public=True, is_public_comment=True)
    make_record_public.short_description = "Make entire record public"

    def make_record_private(self, request, queryset):
        queryset.update(public=False, is_public_comment=False)
    make_record_private.short_description = "Make entire record private"

    def move_comment_to_reg_info(self, request, queryset):
        for mo in queryset:
            mo.reg_info = mo.reg_info + '\n\n' + mo.comment
            mo.comment = ''
            mo.save()
    move_comment_to_reg_info.short_description = "Move comment field to registration notes"

    def get_urls(self):
        urls = super(MOAdmin, self).get_urls()
        my_urls = patterns('',
            url(r'^search/$', self.admin_site.admin_view(search_home),
                name='search'),
            url(r'^search_results/$', self.admin_site.admin_view(search_home),
                {'template_name': 'common/search_results.html'},
                name='search_results'),
            url(r'^search_xls/$',  self.admin_site.admin_view(search_xls),
                name='search_xls'),
#            (r'^filter/$', self.admin_site.admin_view(self.my_view))
        )
        return my_urls + urls

    def my_view(self, request):
        pass

    def get_actions(self, request):
        """
        Allow selecting multiple items when change-list is opened as a popup
        """
        from django.contrib.admin.views.main import IS_POPUP_VAR
        if IS_POPUP_VAR in request.GET:
            actions = []
            actions.extend(self.get_action(add_to_opener))
            return SortedDict([
                ('add_to_opener', (add_to_opener, 'add_to_opener', 'Select items'))
            ])
        else:
            return super(MOAdmin, self).get_actions(request)

    class Media:
        from django.conf import settings
        static_url = getattr(settings, 'STATIC_URL', '/static/')
        css = {'all': (static_url + 'museumobject-admin-detail.css',)}

admin.site.register(MuseumObject, MOAdmin)

class FunctionCategoryAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition')
    fields = ('name', 'definition')
    search_fields = ['name']
    actions = [merge_selected]
admin.site.register(FunctionalCategory, FunctionCategoryAdmin)

class CulturalBlocAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition')
    search_fields = ['name', ]
    actions = [merge_selected]
admin.site.register(CulturalBloc, CulturalBlocAdmin)

class ArtefactTypeAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition', 'see_also')
    fields = ('name', 'definition', 'see_also')
    search_fields = ['name']
    actions = [merge_selected]
admin.site.register(ArtefactType, ArtefactTypeAdmin)

class CategoryAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}
    search_fields = ('name', 'slug')
    list_display = ('__unicode__', 'description')
    actions = [merge_selected]
admin.site.register(Category, CategoryAdmin)

class AcquisitionMethodAdmin(admin.ModelAdmin):
    list_display = ('method', 'definition')
    fields = ('method', 'preposition', 'definition')
    search_fields = ('method', 'definition')
    actions = [merge_selected]
admin.site.register(AcquisitionMethod, AcquisitionMethodAdmin)

class LoanStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(LoanStatus, LoanStatusAdmin)

class AccessStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(AccessStatus, AccessStatusAdmin)

class ObtainedAdmin(admin.ModelAdmin):
    list_display = ('how', 'definition')
    fields = ('how', 'definition')
    search_fields = ('how', 'definition')
    actions = [merge_selected]
admin.site.register(Obtained, ObtainedAdmin)

class PhotoTypeAdmin(admin.ModelAdmin):
    list_display = ('phototype', 'definition')
    fields = ('phototype', 'definition')
    search_fields = ('phototype', 'definition')
    actions = [merge_selected]
#TODO: Show this properly with photorecords linked to items
#admin.site.register(PhotoType, PhotoTypeAdmin)

class RecordStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(RecordStatus, RecordStatusAdmin)

<commit_msg>Admin filter by acquisition method<commit_after>from django.contrib import admin
from django.conf.urls.defaults import patterns, url
from models import MuseumObject, FunctionalCategory
from models import CulturalBloc, ArtefactType, Category
from models import AcquisitionMethod, LoanStatus
from models import AccessStatus, Obtained
from models import RecordStatus
from mediaman.models import ArtefactRepresentation
from common.admin import UndeleteableModelAdmin
from common.adminactions import merge_selected, add_to_collection
from common.adminactions import generate_xls, add_to_opener
from admin_views import search_home, search_xls
from django.core import urlresolvers
from django.utils.datastructures import SortedDict
from django.db.models import F
from location.models import GlobalRegion, Country, StateProvince, RegionDistrict, Locality

class MediaFileInline(admin.TabularInline):
    extra = 0
    search_fields = ['name', ]

    def has_add_permission(self, request):
        return False

class ArtefactRepInline(MediaFileInline):
    model = ArtefactRepresentation
#   position field hidden with CSS
    fields = ('name', 'image', 'thumbnail', 'view_admin_record', 'public', 'position')

    def thumbnail(self, obj):
        try:
            thumb_opts = {'size': (64, 64), 'watermark': ''}
            thumb = obj.image.get_thumbnail(thumb_opts)
            return '' % (obj.image.url, thumb.url)
        except:
            return 'Error generating thumbnail'

    def view_admin_record(self, obj):
        try:
            admin_url = urlresolvers.reverse('admin:mediaman_artefactrepresentation_change', args=(obj.id,))
            return 'View image record' % (admin_url,)
        except:
            return ''
    view_admin_record.allow_tags = True
    thumbnail.allow_tags = True
    readonly_fields = ('name', 'image', 'thumbnail', 'view_admin_record')
    sortable_field_name = "position"

class DocumentInline(MediaFileInline):
    model = MuseumObject.related_documents.through
    fields = ('view_document', 'admin_doc_link', 'is_public')
    readonly_fields = ('view_document', 'admin_doc_link', 'is_public')

    def is_public(self, obj):
        return obj.document.public
    is_public.boolean = True

    def view_document(self, obj):
        try:
            doc = obj.document.document
            return '%s' % (doc.url, doc.name)
        except:
            return ''

    def admin_doc_link(self, obj):
        try:
            doc = obj.document
            admin_url = urlresolvers.reverse('admin:mediaman_document_change', args=(doc.id,))
            return 'View document record' % (admin_url,)
        except:
            return ''
    view_document.allow_tags = True
    admin_doc_link.allow_tags = True
    verbose_name_plural = 'Related documents'

from django.contrib.admin import SimpleListFilter

class HierarchyListFilter(SimpleListFilter):
    """
    An admin list filter for hierarchical data

    Used for the geo-locations. Is not directly usable, must be overridden
    with appropriate class fields set.

    Designed to be used in order of the hierarchy. Will set a flag and stop
    processing children if a higher level filter is set. So checks the children/parents
    for validity.
    """
    def __init__(self, request, *args):
        super(HierarchyListFilter, self).__init__(
            request, *args)

        if self.is_invalid_parent(self.get_parent(request)):
            request.stop_processing_hierarchy_filter = True

    def is_invalid_parent(self, parent):
        current = None

        val = self.value()
        if val:
            current = self.model.objects.get(id=val)
        return current and current.parent != parent

    def get_parent(self, request):
        if hasattr(self, '_parent'):
            return self._parent
        parent_id = request.GET.get(self.parent_parameter_name)
        parent_field = self.model._meta.get_field_by_name('parent')[0]
        parent_type = parent_field.rel.to
        self._parent = parent_type.objects.filter(id=parent_id)[0] if parent_id else None
        return self._parent

    def lookups(self, request, model_admin):
        if hasattr(request, 'stop_processing_hierarchy_filter'):
            return []
        parent = self.get_parent(request)
        options = self.model.objects.filter(parent=parent)
        return [(str(val.id), val.name) for val in options]

    def queryset(self, request, queryset):
        print 'queryset(): %s' % self.title
        if self.value() and not hasattr(request, 'stop_processing_hierarchy_filter'):
            args = {self.parameter_name: self.value()}
            return queryset.filter(**args)
        else:
            return queryset

class CountryListFilter(HierarchyListFilter):
    title = 'country'
    parameter_name = 'country_id'
    parent_parameter_name = 'global_region__id__exact'
    model = Country

class StateProvinceListFilter(HierarchyListFilter):
    title = 'state/province'
    parameter_name = 'state_province_id'
    parent_parameter_name = 'country_id'
    model = StateProvince

class RegionDistrictListFilter(HierarchyListFilter):
    title = 'region/district'
    parameter_name = 'region_district_id'
    parent_parameter_name = 'state_province_id'
    model = RegionDistrict

class LocalityListFilter(HierarchyListFilter):
    title = 'locality'
    parameter_name = 'country_id'
    parent_parameter_name = 'region_district_id'
    model = Locality

class MOAdmin(UndeleteableModelAdmin):
    list_display = ('registration_number',
                    'description', 'comment', 'public', 'is_public_comment')
    readonly_fields = ('place', 'cultural_bloc', 'functional_category',
        'donor_2', 'collector_2', 'old_registration_number',)

    list_filter = ('artefact_type', 'category',
                    'access_status', 'loan_status',
                    'collector', 'donor', 'record_status',
                    'global_region', CountryListFilter, StateProvinceListFilter,
                    RegionDistrictListFilter, LocalityListFilter, 'is_public_comment',
                    'public', 'maker', 'acquisition_method')

    search_fields = ['registration_number', 'description', 'comment',
                     'donor__name', 'collector__name', 'maker__name',
                     'global_region__name', 'country__name', 'state_province__name',
                     'region_district__name', 'locality__name']

    inlines = [
            DocumentInline, ArtefactRepInline
    ]

    raw_id_fields = ('category', 'place', 'collector', 'collector_2',
            'donor', 'donor_2', 'artefact_type', 'maker')
    autocomplete_lookup_fields = {
            'fk': ['place', 'collector', 'donor', 'artefact_type',
                   'maker'],
            'm2m': ['category']
    }
    fieldsets = (
        (None, {
            'fields': ('registration_number', 'old_registration_number',
                       'other_number', 'functional_category', 'category',
                       'artefact_type')
        }),
        ('Registration', {
            'fields': ('registered_by', 'registration_date')
        }),
        ('Old place information', {
            'fields': ('cultural_bloc', 'place',)
        }),
        ('Geo-location', {
            'classes': ('collapse',),
            'fields': ('global_region', 'country', 'state_province',
                'region_district', 'locality')
        }),
        ('Status', {
            'classes': ('collapse',),
            'fields': ('loan_status', 'access_status', 'record_status',)
        }),
        ('Storage location', {
            'classes': ('collapse',),
            'fields': ('storage_section', 'storage_unit',
                'storage_bay', 'storage_shelf_box_drawer')
        }),
        ('Acquisition', {
            'classes': ('collapse',),
            'fields': ('acquisition_date', 'acquisition_method',
                       'reg_info')
        }),
        ('Collector', {
            'classes': ('collapse',),
            'fields': ('collector', 'collector_2',
                'how_collector_obtained', 'when_collector_obtained')
        }),
        ('Donor', {
            'classes': ('collapse',),
            'fields': ('donor', 'donor_2',
                'how_donor_obtained', 'when_donor_obtained')
        }),
        ('Details', {
            'classes': ('collapse',),
            'fields': ('description', 'is_public_comment',
                'comment', 'private_comment', 'significance')
        }),
        ('Extra details', {
            'fields': ('maker', 'manufacture_technique', 'creation_date',
                'site_name_number', 'raw_material', 'indigenous_name',
                'recorded_use', 'assoc_cultural_group',
                'exhibition_history', 'category_illustrated',
                'artefact_illustrated')
        }),
        ('Location', {
            'classes': ('collapse',),
            'fields': (('longitude', 'latitude'),)
        }),
        ('Dimensions', {
            'classes': ('collapse',),
            'fields': (('length', 'width', 'height'),
                ('depth', 'circumference'))
        }),
    )

    #### SETUP ADMIN ACTIONS #####
    actions = [add_to_collection, generate_xls, 'make_comment_public',
        'make_comment_private', 'make_record_public', 'make_record_private',
        'move_comment_to_reg_info']

    def make_comment_public(self, request, queryset):
        queryset.update(is_public_comment=True)
    make_comment_public.short_description = "Make comments public"

    def make_comment_private(self, request, queryset):
        queryset.update(is_public_comment=False)
    make_comment_private.short_description = "Make comments private"

    def make_record_public(self, request, queryset):
        queryset.update(public=True, is_public_comment=True)
    make_record_public.short_description = "Make entire record public"

    def make_record_private(self, request, queryset):
        queryset.update(public=False, is_public_comment=False)
    make_record_private.short_description = "Make entire record private"

    def move_comment_to_reg_info(self, request, queryset):
        for mo in queryset:
            mo.reg_info = mo.reg_info + '\n\n' + mo.comment
            mo.comment = ''
            mo.save()
    move_comment_to_reg_info.short_description = "Move comment field to registration notes"

    def get_urls(self):
        urls = super(MOAdmin, self).get_urls()
        my_urls = patterns('',
            url(r'^search/$', self.admin_site.admin_view(search_home),
                name='search'),
            url(r'^search_results/$', self.admin_site.admin_view(search_home),
                {'template_name': 'common/search_results.html'},
                name='search_results'),
            url(r'^search_xls/$',  self.admin_site.admin_view(search_xls),
                name='search_xls'),
#            (r'^filter/$', self.admin_site.admin_view(self.my_view))
        )
        return my_urls + urls

    def my_view(self, request):
        pass

    def get_actions(self, request):
        """
        Allow selecting multiple items when change-list is opened as a popup
        """
        from django.contrib.admin.views.main import IS_POPUP_VAR
        if IS_POPUP_VAR in request.GET:
            actions = []
            actions.extend(self.get_action(add_to_opener))
            return SortedDict([
                ('add_to_opener', (add_to_opener, 'add_to_opener', 'Select items'))
            ])
        else:
            return super(MOAdmin, self).get_actions(request)

    class Media:
        from django.conf import settings
        static_url = getattr(settings, 'STATIC_URL', '/static/')
        css = {'all': (static_url + 'museumobject-admin-detail.css',)}

admin.site.register(MuseumObject, MOAdmin)

class FunctionCategoryAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition')
    fields = ('name', 'definition')
    search_fields = ['name']
    actions = [merge_selected]
admin.site.register(FunctionalCategory, FunctionCategoryAdmin)

class CulturalBlocAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition')
    search_fields = ['name', ]
    actions = [merge_selected]
admin.site.register(CulturalBloc, CulturalBlocAdmin)

class ArtefactTypeAdmin(admin.ModelAdmin):
    list_display = ('name', 'definition', 'see_also')
    fields = ('name', 'definition', 'see_also')
    search_fields = ['name']
    actions = [merge_selected]
admin.site.register(ArtefactType, ArtefactTypeAdmin)

class CategoryAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}
    search_fields = ('name', 'slug')
    list_display = ('__unicode__', 'description')
    actions = [merge_selected]
admin.site.register(Category, CategoryAdmin)

class AcquisitionMethodAdmin(admin.ModelAdmin):
    list_display = ('method', 'definition')
    fields = ('method', 'preposition', 'definition')
    search_fields = ('method', 'definition')
    actions = [merge_selected]
admin.site.register(AcquisitionMethod, AcquisitionMethodAdmin)

class LoanStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(LoanStatus, LoanStatusAdmin)

class AccessStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(AccessStatus, AccessStatusAdmin)

class ObtainedAdmin(admin.ModelAdmin):
    list_display = ('how', 'definition')
    fields = ('how', 'definition')
    search_fields = ('how', 'definition')
    actions = [merge_selected]
admin.site.register(Obtained, ObtainedAdmin)

class PhotoTypeAdmin(admin.ModelAdmin):
    list_display = ('phototype', 'definition')
    fields = ('phototype', 'definition')
    search_fields = ('phototype', 'definition')
    actions = [merge_selected]
#TODO: Show this properly with photorecords linked to items
#admin.site.register(PhotoType, PhotoTypeAdmin)

class RecordStatusAdmin(admin.ModelAdmin):
    list_display = ('status', 'definition')
    fields = ('status', 'definition')
    search_fields = ('status', 'definition')
    actions = [merge_selected]
admin.site.register(RecordStatus, RecordStatusAdmin)

<|endoftext|>


<commit_before>import os
import logging
import socket
from dessn.framework.fitter import Fitter
from dessn.framework.models.approx_model import ApproximateModel
from dessn.framework.simulations.snana import SNANASimulationGauss0p3

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    plot_dir = os.path.dirname(os.path.abspath(__file__)) + "/plots/%s/" % os.path.basename(__file__)[:-3]
    dir_name = plot_dir + "output/"
    plot_filename = plot_dir + os.path.basename(__file__)[:-3] + ".png"
    file = os.path.abspath(__file__)
    print(dir_name)

    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

    model = ApproximateModel()
    simulation = SNANASimulationGauss0p3(1000, manual_selection=[22.12, 0.544, None, 1.0])
    print("CORRECTION IS ", simulation.get_approximate_correction())

    fitter = Fitter(dir_name)
    fitter.set_models(model)
    fitter.set_simulations(simulation)
    fitter.set_num_cosmologies(300)
    fitter.set_num_walkers(1)
    fitter.set_max_steps(3000)

    h = socket.gethostname()
    if h != "smp-hk5pn72":  # The hostname of my laptop. Only will work for me, ha!
        fitter.fit(file)
    else:
        from chainconsumer import ChainConsumer
        m, s, chain, truth, weight, old_weight, posterior = fitter.load()
        c = ChainConsumer()
        c.add_chain(chain, weights=weight, posterior=posterior, name="Approx")
        c.configure(spacing=1.0)

        parameters = [r"$\Omega_m$", r"$\alpha$", r"$\beta$", r"$\langle M_B \rangle$"]
        print(c.analysis.get_latex_table(transpose=True))
        c.plotter.plot(filename=plot_filename, truth=truth, parameters=parameters)
        print("Plotting distributions")
        c = ChainConsumer()
        c.add_chain(chain, weights=weight, posterior=posterior, name="Approx")
        c.configure(label_font_size=10, tick_font_size=10, diagonal_tick_labels=False)
        c.plotter.plot_distributions(filename=plot_filename.replace(".png", "_dist.png"), truth=truth, col_wrap=8)
        with open(plot_filename.replace(".png", ".txt"), 'w') as f:
            f.write(c.analysis.get_latex_table(parameters=parameters))
<commit_msg>500 SNe for snana_test<commit_after>import os
import logging
import socket
from dessn.framework.fitter import Fitter
from dessn.framework.models.approx_model import ApproximateModel
from dessn.framework.simulations.snana import SNANASimulationGauss0p3

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    plot_dir = os.path.dirname(os.path.abspath(__file__)) + "/plots/%s/" % os.path.basename(__file__)[:-3]
    dir_name = plot_dir + "output/"
    plot_filename = plot_dir + os.path.basename(__file__)[:-3] + ".png"
    file = os.path.abspath(__file__)
    print(dir_name)

    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

    model = ApproximateModel()
    simulation = SNANASimulationGauss0p3(500, manual_selection=[22.12, 0.544, None, 1.0])
    print("CORRECTION IS ", simulation.get_approximate_correction())

    fitter = Fitter(dir_name)
    fitter.set_models(model)
    fitter.set_simulations(simulation)
    fitter.set_num_cosmologies(300)
    fitter.set_num_walkers(1)
    fitter.set_max_steps(3000)

    h = socket.gethostname()
    if h != "smp-hk5pn72":  # The hostname of my laptop. Only will work for me, ha!
        fitter.fit(file)
    else:
        from chainconsumer import ChainConsumer
        m, s, chain, truth, weight, old_weight, posterior = fitter.load()
        c = ChainConsumer()
        c.add_chain(chain, weights=weight, posterior=posterior, name="Approx")
        c.configure(spacing=1.0)

        parameters = [r"$\Omega_m$", r"$\alpha$", r"$\beta$", r"$\langle M_B \rangle$"]
        print(c.analysis.get_latex_table(transpose=True))
        c.plotter.plot(filename=plot_filename, truth=truth, parameters=parameters)
        print("Plotting distributions")
        c = ChainConsumer()
        c.add_chain(chain, weights=weight, posterior=posterior, name="Approx")
        c.configure(label_font_size=10, tick_font_size=10, diagonal_tick_labels=False)
        c.plotter.plot_distributions(filename=plot_filename.replace(".png", "_dist.png"), truth=truth, col_wrap=8)
        with open(plot_filename.replace(".png", ".txt"), 'w') as f:
            f.write(c.analysis.get_latex_table(parameters=parameters))
<|endoftext|>


<commit_before>"""
=========================================================
Imputing missing values with variants of IterativeImputer
=========================================================

The :class:`sklearn.impute.IterativeImputer` class is very flexible - it can be
used with a variety of estimators to do round-robin regression, treating every
variable as an output in turn.

In this example we compare some estimators for the purpose of missing feature
imputation with :class:`sklearn.imputeIterativeImputer`:

* :class:`~sklearn.linear_model.BayesianRidge`: regularized linear regression
* :class:`~sklearn.tree.DecisionTreeRegressor`: non-linear regression
* :class:`~sklearn.ensemble.ExtraTreesRegressor`: similar to missForest in R
* :class:`~sklearn.neighbors.KNeighborsRegressor`: comparable to other KNN
  imputation approaches

Of particular interest is the ability of
:class:`sklearn.impute.IterativeImputer` to mimic the behavior of missForest, a
popular imputation package for R. In this example, we have chosen to use
:class:`sklearn.ensemble.ExtraTreesRegressor` instead of
:class:`sklearn.ensemble.RandomForestRegressor` (as in missForest) due to its
increased speed.

Note that :class:`sklearn.neighbors.KNeighborsRegressor` is different from KNN
imputation, which learns from samples with missing values by using a distance
metric that accounts for missing values, rather than imputing them.

The goal is to compare different estimators to see which one is best for the
:class:`sklearn.impute.IterativeImputer` when using a
:class:`sklearn.linear_model.BayesianRidge` estimator on the California housing
dataset with a single value randomly removed from each row.

For this particular pattern of missing values we see that
:class:`sklearn.ensemble.ExtraTreesRegressor` and
:class:`sklearn.linear_model.BayesianRidge` give the best results.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.datasets import fetch_california_housing
from sklearn.impute import SimpleImputer
from sklearn.impute import IterativeImputer
from sklearn.linear_model import BayesianRidge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import cross_val_score

N_SPLITS = 5

rng = np.random.RandomState(0)

X_full, y_full = fetch_california_housing(return_X_y=True)
# ~2k samples is enough for the purpose of the example.
# Remove the following two lines for a slower run with different error bars.
X_full = X_full[::10]
y_full = y_full[::10]
n_samples, n_features = X_full.shape

# Estimate the score on the entire dataset, with no missing values
br_estimator = BayesianRidge()
score_full_data = pd.DataFrame(
    cross_val_score(
        br_estimator, X_full, y_full, scoring='neg_mean_squared_error',
        cv=N_SPLITS
    ),
    columns=['Full Data']
)

# Add a single missing value to each row
X_missing = X_full.copy()
y_missing = y_full
missing_samples = np.arange(n_samples)
missing_features = rng.choice(n_features, n_samples, replace=True)
X_missing[missing_samples, missing_features] = np.nan

# Estimate the score after imputation (mean and median strategies)
score_simple_imputer = pd.DataFrame()
for strategy in ('mean', 'median'):
    estimator = make_pipeline(
        SimpleImputer(missing_values=np.nan, strategy=strategy),
        br_estimator
    )
    score_simple_imputer[strategy] = cross_val_score(
        estimator, X_missing, y_missing, scoring='neg_mean_squared_error',
        cv=N_SPLITS
    )

# Estimate the score after iterative imputation of the missing values
# with different estimators
estimators = [
    BayesianRidge(),
    DecisionTreeRegressor(max_features='sqrt', random_state=0),
    ExtraTreesRegressor(n_estimators=10, random_state=0),
    KNeighborsRegressor(n_neighbors=15)
]
score_iterative_imputer = pd.DataFrame()
for impute_estimator in estimators:
    estimator = make_pipeline(
        IterativeImputer(random_state=0, estimator=impute_estimator),
        br_estimator
    )
    score_iterative_imputer[impute_estimator.__class__.__name__] = \
        cross_val_score(
            estimator, X_missing, y_missing, scoring='neg_mean_squared_error',
            cv=N_SPLITS
        )

scores = pd.concat(
    [score_full_data, score_simple_imputer, score_iterative_imputer],
    keys=['Original', 'SimpleImputer', 'IterativeImputer'], axis=1
)

# plot boston results
fig, ax = plt.subplots(figsize=(13, 6))
means = -scores.mean()
errors = scores.std()
means.plot.barh(xerr=errors, ax=ax)
ax.set_title('California Housing Regression with Different Imputation Methods')
ax.set_xlabel('MSE (smaller is better)')
ax.set_yticks(np.arange(means.shape[0]))
ax.set_yticklabels([" w/ ".join(label) for label in means.index.get_values()])
plt.tight_layout(pad=1)
plt.show()
<commit_msg>DOC: fix class ref (#13766)<commit_after>"""
=========================================================
Imputing missing values with variants of IterativeImputer
=========================================================

The :class:`sklearn.impute.IterativeImputer` class is very flexible - it can be
used with a variety of estimators to do round-robin regression, treating every
variable as an output in turn.

In this example we compare some estimators for the purpose of missing feature
imputation with :class:`sklearn.impute.IterativeImputer`:

* :class:`~sklearn.linear_model.BayesianRidge`: regularized linear regression
* :class:`~sklearn.tree.DecisionTreeRegressor`: non-linear regression
* :class:`~sklearn.ensemble.ExtraTreesRegressor`: similar to missForest in R
* :class:`~sklearn.neighbors.KNeighborsRegressor`: comparable to other KNN
  imputation approaches

Of particular interest is the ability of
:class:`sklearn.impute.IterativeImputer` to mimic the behavior of missForest, a
popular imputation package for R. In this example, we have chosen to use
:class:`sklearn.ensemble.ExtraTreesRegressor` instead of
:class:`sklearn.ensemble.RandomForestRegressor` (as in missForest) due to its
increased speed.

Note that :class:`sklearn.neighbors.KNeighborsRegressor` is different from KNN
imputation, which learns from samples with missing values by using a distance
metric that accounts for missing values, rather than imputing them.

The goal is to compare different estimators to see which one is best for the
:class:`sklearn.impute.IterativeImputer` when using a
:class:`sklearn.linear_model.BayesianRidge` estimator on the California housing
dataset with a single value randomly removed from each row.

For this particular pattern of missing values we see that
:class:`sklearn.ensemble.ExtraTreesRegressor` and
:class:`sklearn.linear_model.BayesianRidge` give the best results.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.datasets import fetch_california_housing
from sklearn.impute import SimpleImputer
from sklearn.impute import IterativeImputer
from sklearn.linear_model import BayesianRidge
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import cross_val_score

N_SPLITS = 5

rng = np.random.RandomState(0)

X_full, y_full = fetch_california_housing(return_X_y=True)
# ~2k samples is enough for the purpose of the example.
# Remove the following two lines for a slower run with different error bars.
X_full = X_full[::10]
y_full = y_full[::10]
n_samples, n_features = X_full.shape

# Estimate the score on the entire dataset, with no missing values
br_estimator = BayesianRidge()
score_full_data = pd.DataFrame(
    cross_val_score(
        br_estimator, X_full, y_full, scoring='neg_mean_squared_error',
        cv=N_SPLITS
    ),
    columns=['Full Data']
)

# Add a single missing value to each row
X_missing = X_full.copy()
y_missing = y_full
missing_samples = np.arange(n_samples)
missing_features = rng.choice(n_features, n_samples, replace=True)
X_missing[missing_samples, missing_features] = np.nan

# Estimate the score after imputation (mean and median strategies)
score_simple_imputer = pd.DataFrame()
for strategy in ('mean', 'median'):
    estimator = make_pipeline(
        SimpleImputer(missing_values=np.nan, strategy=strategy),
        br_estimator
    )
    score_simple_imputer[strategy] = cross_val_score(
        estimator, X_missing, y_missing, scoring='neg_mean_squared_error',
        cv=N_SPLITS
    )

# Estimate the score after iterative imputation of the missing values
# with different estimators
estimators = [
    BayesianRidge(),
    DecisionTreeRegressor(max_features='sqrt', random_state=0),
    ExtraTreesRegressor(n_estimators=10, random_state=0),
    KNeighborsRegressor(n_neighbors=15)
]
score_iterative_imputer = pd.DataFrame()
for impute_estimator in estimators:
    estimator = make_pipeline(
        IterativeImputer(random_state=0, estimator=impute_estimator),
        br_estimator
    )
    score_iterative_imputer[impute_estimator.__class__.__name__] = \
        cross_val_score(
            estimator, X_missing, y_missing, scoring='neg_mean_squared_error',
            cv=N_SPLITS
        )

scores = pd.concat(
    [score_full_data, score_simple_imputer, score_iterative_imputer],
    keys=['Original', 'SimpleImputer', 'IterativeImputer'], axis=1
)

# plot boston results
fig, ax = plt.subplots(figsize=(13, 6))
means = -scores.mean()
errors = scores.std()
means.plot.barh(xerr=errors, ax=ax)
ax.set_title('California Housing Regression with Different Imputation Methods')
ax.set_xlabel('MSE (smaller is better)')
ax.set_yticks(np.arange(means.shape[0]))
ax.set_yticklabels([" w/ ".join(label) for label in means.index.get_values()])
plt.tight_layout(pad=1)
plt.show()
<|endoftext|>


<commit_before>from __future__ import absolute_import

import numpy as np

from bokeh.plotting import figure, save
from bokeh.layouts import gridplot

coeffs = [10**0, 10**3, 10**6, 10**9]
V = np.arange(10)

figs = []

for ycoeff in coeffs:
    row = []
    for xcoeff in coeffs:
        fig = figure(plot_height=200, plot_width=200)
        fig.xaxis[0].formatter.use_scientific = False
        fig.yaxis[0].formatter.use_scientific = False
        fig.xaxis[0].major_label_orientation = "vertical"
        fig.yaxis[0].major_label_orientation = "horizontal"
        fig.scatter(V*xcoeff, V*ycoeff)
        row.append(fig)
    figs.append(row)

grid = gridplot(figs, toolbar_location=None)

save(grid)
<commit_msg>Fix grid_axis_alignment_no_toolbar on 32-bit platforms (#8651)<commit_after>from __future__ import absolute_import

import numpy as np

from bokeh.plotting import figure, save
from bokeh.layouts import gridplot

coeffs = [10**0, 10**3, 10**6, 10**9]
V = np.arange(10, dtype="int64")

figs = []

for ycoeff in coeffs:
    row = []
    for xcoeff in coeffs:
        fig = figure(plot_height=200, plot_width=200)
        fig.xaxis[0].formatter.use_scientific = False
        fig.yaxis[0].formatter.use_scientific = False
        fig.xaxis[0].major_label_orientation = "vertical"
        fig.yaxis[0].major_label_orientation = "horizontal"
        fig.scatter(V*xcoeff, V*ycoeff)
        row.append(fig)
    figs.append(row)

grid = gridplot(figs, toolbar_location=None)

save(grid)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This script demonstrates how one can script Mayavi by subclassing the
application, create a new VTK scene and create a few simple modules.

This should be run as::

   $ python test.py

"""
# Author: Prabhu Ramachandran <prabhu_r@users.sf.net>
# Copyright (c) 2005-2007, Enthought, Inc.
# License: BSD Style.

# Standard library imports
from os.path import join, abspath

# Enthought library imports
from enthought.mayavi.plugins.app import Mayavi
from enthought.mayavi.scripts.util import get_data_dir

class MyApp(Mayavi):
    def run(self):
        """This is executed once the application GUI has started.
        *Make sure all other MayaVi specific imports are made here!*
        """
        # Various imports to do different things.
        from enthought.mayavi.sources.vtk_file_reader import VTKFileReader
        from enthought.mayavi.modules.outline import Outline
        from enthought.mayavi.modules.axes import Axes
        from enthought.mayavi.modules.grid_plane import GridPlane
        from enthought.mayavi.modules.image_plane_widget import ImagePlaneWidget
        from enthought.mayavi.modules.text import Text

        script = self.script
        # Create a new scene.
        script.new_scene()

        # Read a VTK (old style) data file.
        r = VTKFileReader()
        r.initialize(join(get_data_dir(abspath(__file__)), 'heart.vtk'))
        script.add_source(r)

        # Put up some text.
        t = Text(text='MayaVi rules!', x_position=0.2,
                 y_position=0.9, width=0.8)
        t.property.color = 1, 1, 0  # Bright yellow, yeah!
        script.add_module(t)

        # Create an outline for the data.
        o = Outline()
        script.add_module(o)

        # Create an axes for the data.
        a = Axes()
        script.add_module(a)

        # Create an orientation axes for the scene.  This only works with
        # VTK-4.5 and above which is why we have the try block.
        try:
            from enthought.mayavi.modules.orientation_axes import OrientationAxes
        except ImportError:
            pass
        else:
            a = OrientationAxes()
            a.marker.set_viewport(0.0, 0.8, 0.2, 1.0)
            script.add_module(a)

        # Create three simple grid plane modules.
        # First normal to 'x' axis.
        gp = GridPlane()
        script.add_module(gp)
        # Second normal to 'y' axis.
        gp = GridPlane()
        gp.grid_plane.axis = 'y'
        script.add_module(gp)
        # Third normal to 'z' axis.
        gp = GridPlane()
        script.add_module(gp)
        gp.grid_plane.axis = 'z'

        # Create one ImagePlaneWidget.
        ipw = ImagePlaneWidget()
        script.add_module(ipw)
        # Set the position to the middle of the data.
        ipw.ipw.slice_position = 16

if __name__ == '__main__':
    a = MyApp()
    a.main()
<commit_msg>BUG: Fix another example with wrong access to data directory<commit_after>#!/usr/bin/env python
"""
This script demonstrates how one can script Mayavi by subclassing the
application, create a new VTK scene and create a few simple modules.

This should be run as::

   $ python test.py

"""
# Author: Prabhu Ramachandran <prabhu_r@users.sf.net>
# Copyright (c) 2005-2007, Enthought, Inc.
# License: BSD Style.

# Standard library imports
from os.path import join, abspath, dirname

# Enthought library imports
from enthought.mayavi.plugins.app import Mayavi
from enthought.mayavi.scripts.util import get_data_dir

class MyApp(Mayavi):
    def run(self):
        """This is executed once the application GUI has started.
        *Make sure all other MayaVi specific imports are made here!*
        """
        # Various imports to do different things.
        from enthought.mayavi.sources.vtk_file_reader import VTKFileReader
        from enthought.mayavi.modules.outline import Outline
        from enthought.mayavi.modules.axes import Axes
        from enthought.mayavi.modules.grid_plane import GridPlane
        from enthought.mayavi.modules.image_plane_widget import ImagePlaneWidget
        from enthought.mayavi.modules.text import Text

        script = self.script
        # Create a new scene.
        script.new_scene()

        # Read a VTK (old style) data file.
        r = VTKFileReader()
        r.initialize(join(get_data_dir(dirname(abspath(__file__))), 'heart.vtk'))
        script.add_source(r)

        # Put up some text.
        t = Text(text='MayaVi rules!', x_position=0.2,
                 y_position=0.9, width=0.8)
        t.property.color = 1, 1, 0  # Bright yellow, yeah!
        script.add_module(t)

        # Create an outline for the data.
        o = Outline()
        script.add_module(o)

        # Create an axes for the data.
        a = Axes()
        script.add_module(a)

        # Create an orientation axes for the scene.  This only works with
        # VTK-4.5 and above which is why we have the try block.
        try:
            from enthought.mayavi.modules.orientation_axes import OrientationAxes
        except ImportError:
            pass
        else:
            a = OrientationAxes()
            a.marker.set_viewport(0.0, 0.8, 0.2, 1.0)
            script.add_module(a)

        # Create three simple grid plane modules.
        # First normal to 'x' axis.
        gp = GridPlane()
        script.add_module(gp)
        # Second normal to 'y' axis.
        gp = GridPlane()
        gp.grid_plane.axis = 'y'
        script.add_module(gp)
        # Third normal to 'z' axis.
        gp = GridPlane()
        script.add_module(gp)
        gp.grid_plane.axis = 'z'

        # Create one ImagePlaneWidget.
        ipw = ImagePlaneWidget()
        script.add_module(ipw)
        # Set the position to the middle of the data.
        ipw.ipw.slice_position = 16

if __name__ == '__main__':
    a = MyApp()
    a.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Try to identify a running sshd process in two ways: (1) look for a process
# with the name "sshd" or "in.sshd"; (2) look for a process listening on port
# 22. If neither of those is found, then assume that sshd is stopped.
#
# Separately, check the sshd configuration for password-based logins by
# invoking sshd with the "-T" argument to get its effective configuration. If
# we were able to identify a running sshd process as described above, then use
# that process to determine the path to the sshd binary and/or the non-default
# configuration file name. Otherwise, just do "sshd -T", hoping that it'll be
# in our search path.

import json
import psutil
import re
import subprocess

sshd_name_re = re.compile(r'\bsshd\b|\bin\.sshd\b')

try:
    sshd_process = next(p for p in psutil.process_iter()
                        if sshd_name_re.search(p.exe()) or
                        any(c for c in p.connections('tcp')
                            if c.laddr[1] == 22 and not len(c.raddr)))
    sshd_config_command = [sshd_process.exe(), '-T']
    sshd_cmdline = sshd_process.cmdline()
    try:
        sshd_config_file = sshd_cmdline[sshd_cmdline.index('-f') + 1]
        sshd_config_command.extend(['-f', sshd_config_file])
    except:
        pass
except:
    sshd_process = None
    sshd_config_command = ['sshd', '-T']

try:
    sshd_config = subprocess.check_output(sshd_config_command).decode('ascii')
except:
    sshd_config = ''

results = {}

if sshd_process:
    results['status'] = 'running'
elif sshd_config:
    results['status'] = 'stopped'
else:
    results['status'] = 'missing'

results['config'] = {}
for config in sshd_config.strip().split('\n'):
    key, value = config.split(' ', 1)
    results['config'][key] = value

print(json.dumps(results))
<commit_msg>Refactor sshd plugin, make it work with partially installed sshd<commit_after>#!/usr/bin/env python3

# Try to identify a running sshd process in two ways: (1) look for a process
# with the name "sshd" or "in.sshd"; (2) look for a process listening on port
# 22. If neither of those is found, then assume that sshd is stopped.
#
# Separately, check the sshd configuration for password-based logins by
# invoking sshd with the "-T" argument to get its effective configuration. If
# we were able to identify a running sshd process as described above, then use
# that process to determine the path to the sshd binary and/or the non-default
# configuration file name. Otherwise, just do "sshd -T", hoping that it'll be
# in our search path.

import json
import psutil
import re
import subprocess

sshd_name_re = re.compile(r'\bsshd\b|\bin\.sshd\b')

results = {}

try:
    sshd_process = next(p for p in psutil.process_iter()
                        if sshd_name_re.search(p.exe()) or
                        any(c for c in p.connections('tcp')
                            if c.laddr[1] == 22 and not len(c.raddr)))
except:
    sshd_process = None
    results['status'] = 'stopped'
    sshd_config_command = ['sshd', '-T']
else:
    results['status'] = 'running'
    sshd_config_command = [sshd_process.exe(), '-T']
    sshd_cmdline = sshd_process.cmdline()
    try:
        sshd_config_file = sshd_cmdline[sshd_cmdline.index('-f') + 1]
        sshd_config_command.extend(['-f', sshd_config_file])
    except:
        pass

try:
    sshd_config = subprocess.check_output(
        sshd_config_command, stderr=open('/dev/null', 'w')).decode('ascii')
except FileNotFoundError:
    if not sshd_process:
        results['status'] = 'missing'
    sshd_config = ''
except:
    sshd_config = ''

results['config'] = {}
if sshd_config:
    for config in sshd_config.strip().split('\n'):
        key, value = config.split(' ', 1)
        results['config'][key] = value

print(json.dumps(results))
<|endoftext|>