<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

from __future__ import print_function

"""
BibStat reports some interesting numbers on the bibliographic record set.
"""

__revision__ = "$Id$"

import getopt
import sys
import time

from invenio.base.factory import with_app_context

def report_table_status(tablename):
    """Report stats for the table TABLENAME.  If TABLENAME does not
       exists, return empty string.
    """
    from invenio.legacy.dbquery import get_table_status_info
    out = ""
    table_info = get_table_status_info(tablename)
    if table_info:
        out =  "%14s %17d %17d %17d" % (table_info['Name'],
                                        table_info['Rows'],
                                        table_info['Data_length'],
                                        table_info['Max_data_length']
                                        )
    return out

def report_definitions_of_physical_tags():
    """
    Report definitions of physical MARC tags.
    """
    from invenio.legacy.dbquery import run_sql
    print("### 1 - PHYSICAL TAG DEFINITIONS")
    print()
    print("# MARC tag ... description")
    res = run_sql('SELECT id,value,name FROM tag ORDER BY value')
    for row in res:
        (dummytagid, tagvalue, tagname) = row
        print("%s ... %s" % (tagvalue, tagname,))

def report_definitions_of_logical_fields():
    """
    Report definitions of logical fields.
    """
    from invenio.legacy.dbquery import run_sql
    print()
    print("### 2 - LOGICAL FIELD DEFINITIONS")
    print()
    print("# logical field: associated physical tags", end=' ')
    res = run_sql('SELECT id,name,code FROM field ORDER BY code')
    for row in res:
        (fieldid, dummyfieldname, fieldcode) = row
        print()
        print("%s:" % (fieldcode,), end=' ')
        res2 = run_sql("""SELECT value FROM tag, field_tag
                           WHERE id_field=%s AND id_tag=id
                       """, (fieldid,))
        for row2 in res2:
            tag = row2[0]
            print(tag, end=' ')
    print()

def report_definitions_of_indexes():
    """
    Report definitions of indexes.
    """
    from invenio.legacy.dbquery import run_sql
    print()
    print("### 3 - INDEX DEFINITIONS")
    print()
    print("# index (stemming): associated logical fields", end=' ')
    res = run_sql("""SELECT id,name,stemming_language FROM idxINDEX
                     ORDER BY name""")
    for row in res:
        (indexid, indexname, indexstem) = row
        if indexstem:
            indexname += ' (%s)' % indexstem
        print()
        print("%s:" % (indexname,), end=' ')
        res2 = run_sql("""SELECT code FROM field, idxINDEX_field
                           WHERE id_idxINDEX=%s AND id_field=id
                       """, (indexid,))
        for row2 in res2:
            code = row2[0]
            print(code, end=' ')
    print()

def report_on_all_bibliographic_tables():
    """Report stats for all the interesting bibliographic tables."""
    print()
    print("### 4 -  TABLE SPACE AND SIZE INFO")
    print('')
    print("# %12s %17s %17s %17s" % ("TABLE", "ROWS", "DATA SIZE", "INDEX SIZE"))
    for i in range(0, 10):
        for j in range(0, 10):
            print(report_table_status("bib%1d%1dx" % (i, j)))
            print(report_table_status("bibrec_bib%1d%1dx" % (i, j)))
    for i in range(0, 11):
        print(report_table_status("idxWORD%02dF" % i))
        print(report_table_status("idxWORD%02dR" % i))
    for i in range(0, 11):
        print(report_table_status("idxPHRASE%02dF" % i))
        print(report_table_status("idxPHRASE%02dR" % i))
    return

def report_tag_usage():
    """Analyze bibxxx tables and report info on usage of various tags."""
    print('')
    print("### 5 -  TAG USAGE INFO")
    print('')
    print("# TAG     NB_RECORDS\t# recID1 recID2 ... recID9 (example records)")
    from invenio.legacy.dbquery import run_sql
    for i in range(0, 10):
        for j in range(0, 10):
            bibxxx = "bib%1d%1dx" % (i, j)
            bibrec_bibxxx = 'bibrec_' + bibxxx
            # detect all the various tags in use:
            res = run_sql("SELECT DISTINCT(tag) FROM %s" % (bibxxx,))
            for row in res:
                tag = row[0]
                # detect how many records have this tag in use:
                res_usage = run_sql("""SELECT DISTINCT(b.id) FROM bibrec AS b,
                                                           %s AS bb, %s AS bx
                                            WHERE b.id=bb.id_bibrec
                                              AND bb.id_bibxxx=bx.id
                                              AND bx.tag=%%s
                                        """ % (bibrec_bibxxx, bibxxx),
                                    (tag,))
                # print results
                print(tag, (8-len(tag))*' ', len(res_usage), \
                      '\t\t', '#', " ".join([str(row[0]) for row in
                                             res_usage[:9]]))

def report_header():
    """
    Start reporting.
    """
    from invenio.legacy.dbquery import CFG_DATABASE_HOST, \
        CFG_DATABASE_PORT, CFG_DATABASE_NAME
    print('### BIBSTAT REPORT FOR DB %s:%s.%s RUN AT %s' % (CFG_DATABASE_HOST,
                                                         CFG_DATABASE_PORT,
                                                         CFG_DATABASE_NAME,
                                                         time.asctime()))
    print('')

def report_footer():
    """
    Stop reporting.
    """
    print()
    print()
    print('### END OF BIBSTAT REPORT')

def usage(exitcode=1, msg=""):
    """Prints usage info."""
    if msg:
        sys.stderr.write("Error: %s.\n" % msg)
    sys.stderr.write("Usage: %s [options]\n" % sys.argv[0])
    sys.stderr.write("General options:\n")
    sys.stderr.write("  -h, --help      \t\t Print this help.\n")
    sys.stderr.write("  -V, --version   \t\t Print version information.\n")
    sys.exit(exitcode)

@with_app_context()
def main():
    """Report stats on the Invenio bibliographic tables."""
    try:
        opts, dummyargs = getopt.getopt(sys.argv[1:], "hV", ["help", "version"])
    except getopt.GetoptError as err:
        usage(1, err)
    if opts:
        for opt in opts:
            if opt[0] in ["-h", "--help"]:
                usage(0)
            elif opt[0] in ["-V", "--version"]:
                print(__revision__)
                sys.exit(0)
            else:
                usage(1)
    else:
        report_header()
        report_definitions_of_physical_tags()
        report_definitions_of_logical_fields()
        report_definitions_of_indexes()
        report_on_all_bibliographic_tables()
        report_tag_usage()
        report_footer()
<commit_msg>legacy: fix import in bibstat cli<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

from __future__ import print_function

"""
BibStat reports some interesting numbers on the bibliographic record set.
"""

__revision__ = "$Id$"

import getopt
import sys
import time

from invenio.base.factory import with_app_context

def report_table_status(tablename):
    """Report stats for the table TABLENAME.  If TABLENAME does not
       exists, return empty string.
    """
    from invenio.legacy.dbquery import get_table_status_info
    out = ""
    table_info = get_table_status_info(tablename)
    if table_info:
        out =  "%14s %17d %17d %17d" % (table_info['Name'],
                                        table_info['Rows'],
                                        table_info['Data_length'],
                                        table_info['Max_data_length']
                                        )
    return out

def report_definitions_of_physical_tags():
    """
    Report definitions of physical MARC tags.
    """
    from invenio.legacy.dbquery import run_sql
    print("### 1 - PHYSICAL TAG DEFINITIONS")
    print()
    print("# MARC tag ... description")
    res = run_sql('SELECT id,value,name FROM tag ORDER BY value')
    for row in res:
        (dummytagid, tagvalue, tagname) = row
        print("%s ... %s" % (tagvalue, tagname,))

def report_definitions_of_logical_fields():
    """
    Report definitions of logical fields.
    """
    from invenio.legacy.dbquery import run_sql
    print()
    print("### 2 - LOGICAL FIELD DEFINITIONS")
    print()
    print("# logical field: associated physical tags", end=' ')
    res = run_sql('SELECT id,name,code FROM field ORDER BY code')
    for row in res:
        (fieldid, dummyfieldname, fieldcode) = row
        print()
        print("%s:" % (fieldcode,), end=' ')
        res2 = run_sql("""SELECT value FROM tag, field_tag
                           WHERE id_field=%s AND id_tag=id
                       """, (fieldid,))
        for row2 in res2:
            tag = row2[0]
            print(tag, end=' ')
    print()

def report_definitions_of_indexes():
    """
    Report definitions of indexes.
    """
    from invenio.legacy.dbquery import run_sql
    print()
    print("### 3 - INDEX DEFINITIONS")
    print()
    print("# index (stemming): associated logical fields", end=' ')
    res = run_sql("""SELECT id,name,stemming_language FROM idxINDEX
                     ORDER BY name""")
    for row in res:
        (indexid, indexname, indexstem) = row
        if indexstem:
            indexname += ' (%s)' % indexstem
        print()
        print("%s:" % (indexname,), end=' ')
        res2 = run_sql("""SELECT code FROM field, idxINDEX_field
                           WHERE id_idxINDEX=%s AND id_field=id
                       """, (indexid,))
        for row2 in res2:
            code = row2[0]
            print(code, end=' ')
    print()

def report_on_all_bibliographic_tables():
    """Report stats for all the interesting bibliographic tables."""
    print()
    print("### 4 -  TABLE SPACE AND SIZE INFO")
    print('')
    print("# %12s %17s %17s %17s" % ("TABLE", "ROWS", "DATA SIZE", "INDEX SIZE"))
    for i in range(0, 10):
        for j in range(0, 10):
            print(report_table_status("bib%1d%1dx" % (i, j)))
            print(report_table_status("bibrec_bib%1d%1dx" % (i, j)))
    for i in range(0, 11):
        print(report_table_status("idxWORD%02dF" % i))
        print(report_table_status("idxWORD%02dR" % i))
    for i in range(0, 11):
        print(report_table_status("idxPHRASE%02dF" % i))
        print(report_table_status("idxPHRASE%02dR" % i))
    return

def report_tag_usage():
    """Analyze bibxxx tables and report info on usage of various tags."""
    print('')
    print("### 5 -  TAG USAGE INFO")
    print('')
    print("# TAG     NB_RECORDS\t# recID1 recID2 ... recID9 (example records)")
    from invenio.legacy.dbquery import run_sql
    for i in range(0, 10):
        for j in range(0, 10):
            bibxxx = "bib%1d%1dx" % (i, j)
            bibrec_bibxxx = 'bibrec_' + bibxxx
            # detect all the various tags in use:
            res = run_sql("SELECT DISTINCT(tag) FROM %s" % (bibxxx,))
            for row in res:
                tag = row[0]
                # detect how many records have this tag in use:
                res_usage = run_sql("""SELECT DISTINCT(b.id) FROM bibrec AS b,
                                                           %s AS bb, %s AS bx
                                            WHERE b.id=bb.id_bibrec
                                              AND bb.id_bibxxx=bx.id
                                              AND bx.tag=%%s
                                        """ % (bibrec_bibxxx, bibxxx),
                                    (tag,))
                # print results
                print(tag, (8-len(tag))*' ', len(res_usage), \
                      '\t\t', '#', " ".join([str(row[0]) for row in
                                             res_usage[:9]]))

def report_header():
    """
    Start reporting.
    """
    from invenio.config import CFG_DATABASE_HOST, \
        CFG_DATABASE_PORT, CFG_DATABASE_NAME
    print('### BIBSTAT REPORT FOR DB %s:%s.%s RUN AT %s' % (CFG_DATABASE_HOST,
                                                         CFG_DATABASE_PORT,
                                                         CFG_DATABASE_NAME,
                                                         time.asctime()))
    print('')

def report_footer():
    """
    Stop reporting.
    """
    print()
    print()
    print('### END OF BIBSTAT REPORT')

def usage(exitcode=1, msg=""):
    """Prints usage info."""
    if msg:
        sys.stderr.write("Error: %s.\n" % msg)
    sys.stderr.write("Usage: %s [options]\n" % sys.argv[0])
    sys.stderr.write("General options:\n")
    sys.stderr.write("  -h, --help      \t\t Print this help.\n")
    sys.stderr.write("  -V, --version   \t\t Print version information.\n")
    sys.exit(exitcode)

@with_app_context()
def main():
    """Report stats on the Invenio bibliographic tables."""
    try:
        opts, dummyargs = getopt.getopt(sys.argv[1:], "hV", ["help", "version"])
    except getopt.GetoptError as err:
        usage(1, err)
    if opts:
        for opt in opts:
            if opt[0] in ["-h", "--help"]:
                usage(0)
            elif opt[0] in ["-V", "--version"]:
                print(__revision__)
                sys.exit(0)
            else:
                usage(1)
    else:
        report_header()
        report_definitions_of_physical_tags()
        report_definitions_of_logical_fields()
        report_definitions_of_indexes()
        report_on_all_bibliographic_tables()
        report_tag_usage()
        report_footer()
<|endoftext|>


<commit_before># script to stress test the X-Drive
import os
import time
import shutil

path_big_file=R"X:\Data\SCOTT\2017-05-10 GCaMP6f\GCaMP6f PFC GABA cre\2017-05-10-23 misc\2017-05-11 cell3_annotated.tif"
path_path_to_walk=R"X:\Data\DIC2\2014"

def timeIt(timeStart=None):
    if timeStart:
        return time.time()-timeStart
    return time.time()

def test_listing(path,runs=5):
    print("walking the entire tree of:",path)
    times=[]
    for run in range(runs):
        t1=timeIt()
        abfs,folders=0,0
        for dirpath, subdirs, files in os.walk(path):
            folders+=1
            for x in files:
                if x.endswith(".abf"):
                    abfs+=1
        times.append(timeIt(t1))
        print("(%d/%d) scanned %d folders in %.03f seconds"%(run+1,runs,folders,times[-1]))
    print("Average time for %d runs: %.03f seconds"%(runs,sum(times)/len(times)))

def test_copying(path,runs=5):
    print("copying a HUGE file:",path)
    times=[]
    for run in range(runs):
        t1=timeIt()
        shutil.copy(path,'./test.data')
        time.sleep(1)
        times.append(timeIt(t1))
        mbs=os.stat('./test.data').st_size/times[-1]/1024/1024
        os.remove('./test.data')
        print("(%d/%d) copy took %.03f seconds (%.03f MB/s)"%(run+1,runs,times[-1],mbs))
    print("Average time for %d runs: %.03f seconds"%(runs,sum(times)/len(times)))
    return

if __name__=="__main__":
    test_listing(path_path_to_walk)
    #test_copying(path_big_file)
    print("DONE")<commit_msg>Update networkTest.py<commit_after># script to stress test the X-Drive
import os
import time
import shutil

path_big_file=R"X:\Data\SCOTT\2017-05-10 GCaMP6f\GCaMP6f PFC GABA cre\2017-05-10-23 misc\2017-05-11 cell3_annotated.tif"
path_path_to_walk=R"X:\Data\DIC2\2014"

def timeIt(timeStart=None):
    if timeStart:
        return time.time()-timeStart
    return time.time()

def test_listing(path,runs=5):
    print("walking the entire tree of:",path)
    times=[]
    for run in range(runs):
        t1=timeIt()
        abfs,folders=0,0
        for dirpath, subdirs, files in os.walk(path):
            folders+=1
            for x in files:
                if x.endswith(".abf"):
                    abfs+=1
        times.append(timeIt(t1))
        print("(%d/%d) scanned %d folders in %.03f seconds"%(run+1,runs,folders,times[-1]))
    print("Average time for %d runs: %.03f seconds"%(runs,sum(times)/len(times)))

def test_copying(path,runs=5):
    print("copying a HUGE file:",path)
    times=[]
    for run in range(runs):
        t1=timeIt()
        shutil.copy(path,'./test.data')
        time.sleep(1)
        times.append(timeIt(t1))
        mbs=os.stat('./test.data').st_size/times[-1]/1024/1024
        os.remove('./test.data')
        print("(%d/%d) copy took %.03f seconds (%.03f MB/s)"%(run+1,runs,times[-1],mbs))
    print("Average time for %d runs: %.03f seconds"%(runs,sum(times)/len(times)))
    return

if __name__=="__main__":
    test_listing(path_path_to_walk)
    test_copying(path_big_file)
    print("DONE")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import subprocess
path_base = "c:/00_GOMOGI/geodata/"
# path for book code
# path_base = "../geodata/"
# for windows users
command_gdal_translate = "c:/OSGeo4W/bin/gdal_translate.exe"

command_gdalwarp = "c:/OSGeo4W/bin/gdal_translate.exe"
command_gdalinfo = "c:/OSGeo4W/bin/gdalinfo.exe"
command_gdaldem = "c:/OSGeo4w/bin/gdaldem.exe"

# for linux users
#command_gdal_translate = "gdal_translate"

orig_dem_asc = path_base + "ALS_DGM_10m.asc"
# input arcascii format DEM data
input_dem = path_base + "small_elev.asc"

temp_vrt = path_base + "new_virtual_file.vrt"

temp_tiff = path_base + "temp_image.tif"

threejs_webgl_dem = path_base + "webgl_dem.bin"

new_clip_ktn_dem = path_base + "ktn_clip-dhm5.asc"

final_heightmap = path_base + "final_heightmap.png"

output_envi = path_base + "final_envi.bin"

ktn_ortho_2012 = path_base + "ktn-ortho-2012.tif"

ktn_ortho_2012_clip = path_base + "ktn-ortho-2012_clip.jpg"

####subprocess.call(["gdalbuildvrt", temp_vrt, orig_dem_asc])
# gdalbuildvrt jotunheimen.vrt my_elevation.dem

# 5000m x 5000m square
# clip_command = command_gdal_translate + " -projwin 477200 189600 482200 184600 -of AAIGrid " \
#                + orig_dem_asc + " " + new_clip_ktn_dem

# [-projwin ulx uly lrx lry] upper left(ul) x
# 500m x 500m DEM  or 50px x 50px  because DEM pixel is 10m x 10m
clip_command = command_gdal_translate + " -projwin 478000 185200 478500 184700 -of AAIGrid " \
               + orig_dem_asc + " " + new_clip_ktn_dem

#print (clip_command.split())
print ("now executing this command: " + clip_command)
print ("so be patient this could take some time...now clipping...")
#subprocess.call(clip_command.split(), shell=False)

dem2tiff = command_gdalwarp + " " + new_clip_ktn_dem + " "  + temp_tiff
print ("now executing this command: " + dem2tiff)
#subprocess.call(dem2tiff.split(), shell=False)

show_tif_info = command_gdalinfo + " -mm " + temp_tiff
tif_info = subprocess.call(show_tif_info.split(), shell=False)
print type(tif_info)
print "foo"

# max_min = re.compile(r'Min/Max=')

# if max_min.match(str(max_min)):
#     print "yes match"
# else:
#     print "no match"

# outputs an ENVI image file .bin with height values from 0 to 65535 in 16 bit format
tif_2_envi = command_gdal_translate + " -scale 700 851 0 65535 -ot UInt16 -outsize 200 200 -of ENVI " + temp_tiff + " " + output_envi

#subprocess.call(tif_2_envi.split(),shell=False)

####################
####################

# outputs a PNG with only 0-255 height values
translate2png = command_gdal_translate + " -scale 700 851 0 255 -outsize 200 200 -of PNG " + temp_tiff + " " + final_heightmap
#subprocess.call(translate2png.split(), shell=False)

# subprocess.call(['gdalwarp',"-te", "","","","", temp_vrt, temp_tiff])

### section creates a color relief texture for my terrain
relief_colors = path_base + "color-relief.txt"
output_relief = path_base + "new_relief.tif"
run_color_relief = command_gdaldem + " color-relief " + temp_tiff + " " + relief_colors + " " + output_relief

run_clip_ortho = command_gdal_translate + " -projwin 478000 185200 478500 184700 -of JPEG " \
               + ktn_ortho_2012 + " " + ktn_ortho_2012_clip

subprocess.call(run_clip_ortho.split(),shell=False)<commit_msg>remove old code<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import subprocess

path_base = "c:/00_GOMOGI/geodata/"
# path for book code
# path_base = "../geodata/"
# for windows users
command_gdal_translate = "c:/OSGeo4W/bin/gdal_translate.exe"
command_gdalwarp = "c:/OSGeo4W/bin/gdal_translate.exe"
command_gdalinfo = "c:/OSGeo4W/bin/gdalinfo.exe"
command_gdaldem = "c:/OSGeo4w/bin/gdaldem.exe"

# for linux users
#command_gdal_translate = "gdal_translate"

orig_dem_asc = path_base + "ALS_DGM_10m.asc"
# input arcascii format DEM data
input_dem = path_base + "small_elev.asc"

temp_vrt = path_base + "new_virtual_file.vrt"

temp_tiff = path_base + "temp_image.tif"

threejs_webgl_dem = path_base + "webgl_dem.bin"

new_clip_ktn_dem = path_base + "ktn_clip-dhm5.asc"

final_heightmap = path_base + "final_heightmap.png"

output_envi = path_base + "final_envi.bin"

ktn_ortho_2012 = path_base + "ktn-ortho-2012.tif"

ktn_ortho_2012_clip = path_base + "ktn-ortho-2012_clip.jpg"

# create temp vrt file from ascii
subprocess.call(["gdalbuildvrt", temp_vrt, orig_dem_asc])

# 5000m x 5000m square
# clip_command = command_gdal_translate + " -projwin 477200 189600 482200 184600 -of AAIGrid " \
#                + orig_dem_asc + " " + new_clip_ktn_dem

# [-projwin ulx uly lrx lry] upper left(ul) x
# 500m x 500m DEM  or 50px x 50px  because DEM pixel is 10m x 10m
clip_command = command_gdal_translate + " -projwin 478000 185200 478500 184700 -of AAIGrid " \
               + orig_dem_asc + " " + new_clip_ktn_dem

print ("now executing this command: " + clip_command)
print ("so be patient this could take some time...now clipping...")
subprocess.call(clip_command.split(), shell=False)

# transform dem to tiff
dem2tiff = command_gdalwarp + " " + new_clip_ktn_dem + " "  + temp_tiff
print ("now executing this command: " + dem2tiff)
subprocess.call(dem2tiff.split(), shell=False)

show_tif_info = command_gdalinfo + " -mm " + temp_tiff
tif_info = subprocess.call(show_tif_info.split(), shell=False)

# outputs an ENVI image file .bin with height values from 0 to 65535 in 16 bit format
tif_2_envi = command_gdal_translate + " -scale 700 851 0 65535 -ot UInt16 -outsize 200 200 -of ENVI " + temp_tiff + " " + output_envi

subprocess.call(tif_2_envi.split(),shell=False)

####################
##   extras
####################

# outputs a PNG with only 0-255 height values
# translate2png = command_gdal_translate + " -scale 700 851 0 255 -outsize 200 200 -of PNG " + temp_tiff + " " + final_heightmap
#subprocess.call(translate2png.split(), shell=False)

# subprocess.call(['gdalwarp',"-te", "","","","", temp_vrt, temp_tiff])

### section creates a color relief texture for my terrain
# relief_colors = path_base + "color-relief.txt"
# output_relief = path_base + "new_relief.tif"
# run_color_relief = command_gdaldem + " color-relief " + temp_tiff + " " + relief_colors + " " + output_relief
#
#
# run_clip_ortho = command_gdal_translate + " -projwin 478000 185200 478500 184700 -of JPEG " \
#                + ktn_ortho_2012 + " " + ktn_ortho_2012_clip
#
# subprocess.call(run_clip_ortho.split(),shell=False)<|endoftext|>


<commit_before>""" Pandas extension for chemical analysis """
from __future__ import absolute_import
import pandas as pd

import oddt

pd.set_option("display.max_colwidth", 999999)

def _mol_dict_reader(fmt='sdf',
                     filepath_or_buffer=None,
                     names=None,
                     molecule_column='mol',
                     smiles_column=None):
    out = []
    for mol in oddt.toolkit.readfile(fmt, filepath_or_buffer):
        if names:
            mol_data = dict((k, v) for k, v in mol.data.items() if k in names)
        else:
            mol_data = dict(mol.data)
        if molecule_column:
            mol_data[molecule_column] = mol
        if smiles_column:
            mol_data[smiles_column] = mol.write('smi').split()[0]
        out.append(mol_data)
    return ChemDataFrame(out)

def read_sdf(filepath_or_buffer=None,
             names=None,
             molecule_column='mol',
             smiles_column=None):
    return _mol_dict_reader(fmt='sdf',
                            filepath_or_buffer=filepath_or_buffer,
                            names=names,
                            molecule_column=molecule_column,
                            smiles_column=smiles_column)

class ChemSeries(pd.Series):
    @property
    def _constructor(self):
        return ChemSeries

class ChemDataFrame(pd.DataFrame):
    """

    Note:
    Thanks to: http://blog.snapdragon.cc/2015/05/05/subclass-pandas-dataframe-to-save-custom-attributes/
    """
    def to_sdf(self,
               filepath_or_buffer=None,
               update_properties=True,
               molecule_column='mol',
               names=None):
        out = oddt.toolkit.Outputfile('sdf', filepath_or_buffer, overwrite=True)
        for ix, row in self.iterrows():
            mol = row[molecule_column].clone
            if update_properties:
                new_data = row.to_dict()
                del new_data[molecule_column]
                mol.data.update(new_data)
            if names:
                for k in mol.data.keys():
                    if k not in names:
                        del mol.data[k]
            out.write(mol)
        out.close()

    def to_html(self, *args, **kwargs):
        kwargs['escape'] = False
        return super(ChemDataFrame, self).to_html(*args, **kwargs)

    @property
    def _constructor(self):
        """ Force new class to be usead as sconstructor when slicing """
        return ChemDataFrame

    _constructor_sliced = ChemSeries
<commit_msg>Add mol_name column nd allow skipping empty molecules<commit_after>""" Pandas extension for chemical analysis """
from __future__ import absolute_import
import pandas as pd

import oddt

pd.set_option("display.max_colwidth", 999999)

def _mol_dict_reader(fmt='sdf',
                     filepath_or_buffer=None,
                     names=None,
                     molecule_column='mol',
                     molecule_name='mol_name',
                     smiles_column=None,
                     skip_bad_mols=False,
                     **kwargs):
    out = []
    for mol in oddt.toolkit.readfile(fmt, filepath_or_buffer):
        if skip_bad_mols and mol is None:
            continue  # add warning with number of skipped molecules
        if names:
            mol_data = dict((k, v) for k, v in mol.data.items() if k in names)
        else:
            mol_data = dict(mol.data)
        if molecule_column:
            mol_data[molecule_column] = mol
        if molecule_name:
            mol_data[molecule_name] = mol.title
        if smiles_column:
            mol_data[smiles_column] = mol.write('smi').split()[0]
        out.append(mol_data)
    return ChemDataFrame(out, **kwargs)

def read_sdf(filepath_or_buffer=None,
             names=None,
             molecule_column='mol',
             molecule_name='mol_name',
             smiles_column=None,
             skip_bad_mols=False,
             **kwargs):
    return _mol_dict_reader(fmt='sdf',
                            filepath_or_buffer=filepath_or_buffer,
                            names=names,
                            molecule_column=molecule_column,
                            molecule_name=molecule_name,
                            smiles_column=smiles_column,
                            skip_bad_mols=skip_bad_mols,
                            **kwargs)

class ChemSeries(pd.Series):
    @property
    def _constructor(self):
        return ChemSeries

class ChemDataFrame(pd.DataFrame):
    """

    Note:
    Thanks to: http://blog.snapdragon.cc/2015/05/05/subclass-pandas-dataframe-to-save-custom-attributes/
    """
    def to_sdf(self,
               filepath_or_buffer=None,
               update_properties=True,
               molecule_column='mol',
               names=None):
        out = oddt.toolkit.Outputfile('sdf', filepath_or_buffer, overwrite=True)
        for ix, row in self.iterrows():
            mol = row[molecule_column].clone
            if update_properties:
                new_data = row.to_dict()
                del new_data[molecule_column]
                mol.data.update(new_data)
            if names:
                for k in mol.data.keys():
                    if k not in names:
                        del mol.data[k]
            out.write(mol)
        out.close()

    def to_html(self, *args, **kwargs):
        kwargs['escape'] = False
        return super(ChemDataFrame, self).to_html(*args, **kwargs)

    @property
    def _constructor(self):
        """ Force new class to be usead as sconstructor when slicing """
        return ChemDataFrame

    _constructor_sliced = ChemSeries
<|endoftext|>


<commit_before>from applib import dbStore

if __name__ == "__main__":
    rr = dbStore.query_sql()

    dbStore.add_sql(6, "hahahah", 123)
    print(list(rr))

    rr = dbStore.query_sql(1)
    print(list(rr))
<commit_msg>add query db test<commit_after>import sqlite3
from applib import dbStore

if __name__ == "__main__":
    strr = dbStore.query_sql()
    conn = sqlite3.connect(dbStore.dataDir)
    rr = dbStore.do_sql(strr, conn)

    for i in list(rr):
        print(i)
<|endoftext|>


<commit_before>#TODO: Fully comment, constrain imports.

import datetime
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base

engine = create_engine('mysql+pymysql://thehub:cas0iWur@thehub:3306/hubdb')
Base = declarative_base()

class Code(Base):
    __tablename__ = 'codes'
    id = Column(Integer, primary_key=True, nullable=False)
    code = Column(CHAR(4), nullable=False)
    value = Column(String(256), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by error table fields.
        """
        return { 'id' : self.id, 'value' : self.value }

class Device(Base):
    __tablename__ = 'devices'
    id = Column(Integer, primary_key=True, nullable=False)
    serial_port = Column(String(256), unique=True, nullable=False)
    description = Column(String(256), nullable=False)
    model_number = Column(
        Integer, ForeignKey('device_models.model_number'), nullable=False
    )
    model = relationship("DeviceModel", foreign_keys=[model_number])
    serial_number = Column(Integer, nullable=False)
    sensors = relationship("Sensor")

    def to_dictionary(self):
        """
        Returns a dictionary indexed by device table fields.
        """
        return { 'id' : self.id,
                 'serial_port' : self.serial_port,
                 'description' : self.description,
                 'type' : self.type }

class DeviceModel(Base):
    __tablename__ = 'device_models'
    model_number = Column(
        Integer, primary_key=True, unique=True,
        autoincrement=False, nullable=False
    )
    number_of_sensors = Column(Integer, nullable=False)
    description = Column(String(256), nullable=False)
    wireless = Column(Boolean(create_constraint=False), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by device model fields.
        """
        return { 'model_number' : self.model_number,
                 'number_of_sensors' : self.number_of_sensors,
                 'description' : self.description,
                 'wireless' : self.wireless }

class Sensor(Base):
    __tablename__ = 'sensors'
    id = Column(Integer, primary_key=True, nullable=False)
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    description = Column(String(256), nullable=False)
    model_number = Column(
        Integer, ForeignKey('sensor_models.model_number'), nullable=False
    )
    model = relationship("SensorModel", foreign_keys=[model_number])
    serial_number = Column(Integer, nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by sensor table fields.
        """
        return { 'id' : self.id,
                 'device' : self.device.id,
                 'description' : self.description,
                 'type' : self.type }

class SensorModel(Base):
    __tablename__ = 'sensor_models'
    model_number = Column(
        Integer, primary_key=True, unique=True,
        autoincrement=False, nullable=False
    )
    minimum_value = Column(Integer, nullable=False)
    maximum_value = Column(Integer, nullable=False)
    description = Column(String(256), nullable=False)
    wireless = Column(Boolean(create_constraint=False), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by sensor model fields.
        """
        return { 'id' : self.id,
                 'minimum_value' : self.minimum_value,
                 'maximum_value' : self.maximum_value,
                 'description' : self.description,
                 'wireless' : self.wireless }

class DataEvent(Base):
    __tablename__ = 'data_events'
    id = Column(Integer, primary_key=True, nullable=False)
    timestamp = Column(
        DateTime, default=datetime.datetime.utcnow, nullable=False
    )
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    sensor_id = Column(Integer, ForeignKey('sensors.id'), nullable=False)
    sensor = relationship("Sensor", foreign_keys=[sensor_id])
    value = Column(Integer, nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by event table fields.
        """
        return { 'id' : self.id,
                 'timestamp' : self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
                 'device' : self.device.id,
                 'sensor' : self.sensor.id,
                 'value' : self.value }

class CodeEvent(Base):
    __tablename__ = 'code_events'
    id = Column(Integer, primary_key=True, nullable=False)
    timestamp = Column(
        DateTime, default=datetime.datetime.utcnow, nullable=False
    )
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    code_id = Column(Integer, ForeignKey('codes.id'), nullable=False)
    code = relationship("Code", foreign_keys=[code_id])

    def to_dictionary(self):
        """
        Returns a dictionary indexed by code log fields.
        """
        return { 'id' : self.id,
                 'timestamp' : self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
                 'device' : self.device.id,
                 'code' : self.code.code }

Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session=Session()
<commit_msg>Change db connection string so database works from localhost.<commit_after>#TODO: Fully comment, constrain imports.

import datetime
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base

engine = create_engine('mysql+pymysql://thehub:cas0iWur@localhost:3306/hubdb')
Base = declarative_base()

class Code(Base):
    __tablename__ = 'codes'
    id = Column(Integer, primary_key=True, nullable=False)
    code = Column(CHAR(4), nullable=False)
    value = Column(String(256), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by error table fields.
        """
        return { 'id' : self.id, 'value' : self.value }

class Device(Base):
    __tablename__ = 'devices'
    id = Column(Integer, primary_key=True, nullable=False)
    serial_port = Column(String(256), unique=True, nullable=False)
    description = Column(String(256), nullable=False)
    model_number = Column(
        Integer, ForeignKey('device_models.model_number'), nullable=False
    )
    model = relationship("DeviceModel", foreign_keys=[model_number])
    serial_number = Column(Integer, nullable=False)
    sensors = relationship("Sensor")

    def to_dictionary(self):
        """
        Returns a dictionary indexed by device table fields.
        """
        return { 'id' : self.id,
                 'serial_port' : self.serial_port,
                 'description' : self.description,
                 'type' : self.type }

class DeviceModel(Base):
    __tablename__ = 'device_models'
    model_number = Column(
        Integer, primary_key=True, unique=True,
        autoincrement=False, nullable=False
    )
    number_of_sensors = Column(Integer, nullable=False)
    description = Column(String(256), nullable=False)
    wireless = Column(Boolean(create_constraint=False), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by device model fields.
        """
        return { 'model_number' : self.model_number,
                 'number_of_sensors' : self.number_of_sensors,
                 'description' : self.description,
                 'wireless' : self.wireless }

class Sensor(Base):
    __tablename__ = 'sensors'
    id = Column(Integer, primary_key=True, nullable=False)
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    description = Column(String(256), nullable=False)
    model_number = Column(
        Integer, ForeignKey('sensor_models.model_number'), nullable=False
    )
    model = relationship("SensorModel", foreign_keys=[model_number])
    serial_number = Column(Integer, nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by sensor table fields.
        """
        return { 'id' : self.id,
                 'device' : self.device.id,
                 'description' : self.description,
                 'type' : self.type }

class SensorModel(Base):
    __tablename__ = 'sensor_models'
    model_number = Column(
        Integer, primary_key=True, unique=True,
        autoincrement=False, nullable=False
    )
    minimum_value = Column(Integer, nullable=False)
    maximum_value = Column(Integer, nullable=False)
    description = Column(String(256), nullable=False)
    wireless = Column(Boolean(create_constraint=False), nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by sensor model fields.
        """
        return { 'id' : self.id,
                 'minimum_value' : self.minimum_value,
                 'maximum_value' : self.maximum_value,
                 'description' : self.description,
                 'wireless' : self.wireless }

class DataEvent(Base):
    __tablename__ = 'data_events'
    id = Column(Integer, primary_key=True, nullable=False)
    timestamp = Column(
        DateTime, default=datetime.datetime.utcnow, nullable=False
    )
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    sensor_id = Column(Integer, ForeignKey('sensors.id'), nullable=False)
    sensor = relationship("Sensor", foreign_keys=[sensor_id])
    value = Column(Integer, nullable=False)

    def to_dictionary(self):
        """
        Returns a dictionary indexed by event table fields.
        """
        return { 'id' : self.id,
                 'timestamp' : self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
                 'device' : self.device.id,
                 'sensor' : self.sensor.id,
                 'value' : self.value }

class CodeEvent(Base):
    __tablename__ = 'code_events'
    id = Column(Integer, primary_key=True, nullable=False)
    timestamp = Column(
        DateTime, default=datetime.datetime.utcnow, nullable=False
    )
    device_id = Column(Integer, ForeignKey('devices.id'), nullable=False)
    device = relationship("Device", foreign_keys=[device_id])
    code_id = Column(Integer, ForeignKey('codes.id'), nullable=False)
    code = relationship("Code", foreign_keys=[code_id])

    def to_dictionary(self):
        """
        Returns a dictionary indexed by code log fields.
        """
        return { 'id' : self.id,
                 'timestamp' : self.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
                 'device' : self.device.id,
                 'code' : self.code.code }

Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session=Session()
<|endoftext|>


<commit_before>from takeyourmeds.utils.test import TestCase

from .actions import make_call, send_sms

class TestActions(TestCase):
    def test_call(self):
        sid = make_call(
            '+441324430099',
            'https://api.twilio.com/cowbell.mp3'
        )

        self.assertEqual(len(sid), 34)

    def test_sms(self):
        sid = send_sms('+441324430099', "Hello!")

        self.assertEqual(len(sid), 34)

<commit_msg>Single line.<commit_after>from takeyourmeds.utils.test import TestCase

from .actions import make_call, send_sms

class TestActions(TestCase):
    def test_call(self):
        sid = make_call('+441324430099', 'https://api.twilio.com/cowbell.mp3')

        self.assertEqual(len(sid), 34)

    def test_sms(self):
        sid = send_sms('+441324430099', "Hello!")

        self.assertEqual(len(sid), 34)

<|endoftext|>


<commit_before>from uwsgiconf import Section
from uwsgiconf.exceptions import ConfigurationError

import pytest

def test_monitoring_snmp(assert_lines):

    assert_lines([
        'snmp = 192.168.1.1:2222',
        'snmp-community = mymetr',
    ], Section().monitoring.enable_snmp('192.168.1.1:2222', 'mymetr'))

def test_monitoring_stats(assert_lines):

    assert_lines([
        'stats-server = 127.0.0.1:1717',
        'stats-http = true',
    ], Section().monitoring.set_stats_params('127.0.0.1:1717', enable_http=True))

def test_monitoring_metrics(assert_lines):

    assert_lines([
        'enable-metrics = true',
        'metrics-dir = /here',
    ], Section().monitoring.set_metrics_params(enable=True, store_dir='/here'))

    assert_lines([
        'metric-threshold = alarm=some,key=mycounter,reset=0,value=1000',
    ], Section().monitoring.set_metrics_threshold('mycounter', 1000, reset_to=0, alarm='some'))

    section = Section()
    assert_lines([
        'alarm = pinger cmd:ping 127.0.0.1',
        'metric-threshold = alarm=pinger,key=mycounter,value=2000',
    ], section.monitoring.set_metrics_threshold(
        'mycounter', 2000, alarm=section.alarms.alarm_types.command('pinger', 'ping 127.0.0.1')
    ))

def test_monitoring_register_metric(assert_lines):

    monitoring = Section().monitoring

    my_abs = monitoring.metric_types.absolute('myabs', initial_value=100)
    my_abs2 = monitoring.metric_types.alias('myabs2', initial_value=200, alias_for=my_abs)

    with pytest.raises(ConfigurationError):
        # Alias without alias_for
        monitoring.metric_types.alias('myabs3', initial_value=200)

    assert_lines([
        'metric = name=myabs,initial_value=100,type=absolute',
        'metric = name=myabs2,alias=myabs,initial_value=200,type=alias',
        'metric = name=mymet,collector=sum,children=worker.1.requests;myabs,type=gauge',

    ], monitoring.register_metric([
        my_abs,
        my_abs2,
        monitoring.metric_types.gauge(
            'mymet',
            collector=monitoring.collectors.sum(['worker.1.requests', my_abs]),
        )
    ]))

def test_monitoring_collectors(assert_lines):

    collectors = Section().monitoring.collectors

    assert str(collectors.manual()) == 'manual'
    assert str(collectors.pointer()) == 'ptr'
    assert str(collectors.file('/here/a', get_slot=2)) == 'file,arg1=/here/a,arg1n=2'
    assert str(collectors.function('some')) == 'func,arg1=some'
    assert str(collectors.sum(['a', 'b'])) == 'sum,children=a;b'
    assert str(collectors.avg(['a', 'b'])) == 'avg,children=a;b'
    assert str(collectors.accumulator(['a', 'b'])) == 'accumulator,children=a;b'
    assert str(collectors.adder(['a', 'b'], 3)) == 'adder,arg1n=3,children=a;b'
    assert str(collectors.multiplier(['a', 'b'], 4)) == 'multiplier,arg1n=4,children=a;b'

def test_monitoring_pushers(assert_lines):

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = socket:127.0.0.1:8125,myinstance',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.socket('127.0.0.1:8125', 'myinstance')
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.rrdtool('/here')
    pusher.set_basic_params('libmyrdd.so')
    assert_lines([
        'stats-push = rrdtool:/here',
        'rrdtool-lib = libmyrdd.so',

    ], monitoring.register_stats_pusher(pusher))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.statsd('127.0.0.1:8125', 'myinstance')
    pusher.set_basic_params(no_workers=True)
    assert_lines([
        'stats-push = statsd:127.0.0.1:8125,myinstance',
        'statsd-no-workers = true',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.carbon('127.0.0.1:2003')
    pusher.set_basic_params(root_node='myroot')
    pusher.set_connection_params(retries=5)
    assert_lines([
        'stats-push = carbon:127.0.0.1:2003',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.zabbix('127.0.0.1:10051')
    pusher.set_basic_params('/put/here')
    assert_lines([
        'stats-push = zabbix:127.0.0.1:10051',
        'zabbix-template = /put/here',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = mongodb:collection=mycol,freq=10',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.mongo(collection='mycol', interval=10)
    ))

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = file:path=/here/a.json,freq=15',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.file('/here/a.json', interval=15)
    ))
<commit_msg>kwarg fix<commit_after>from uwsgiconf import Section
from uwsgiconf.exceptions import ConfigurationError

import pytest

def test_monitoring_snmp(assert_lines):

    assert_lines([
        'snmp = 192.168.1.1:2222',
        'snmp-community = mymetr',
    ], Section().monitoring.enable_snmp('192.168.1.1:2222', 'mymetr'))

def test_monitoring_stats(assert_lines):

    assert_lines([
        'stats-server = 127.0.0.1:1717',
        'stats-http = true',
    ], Section().monitoring.set_stats_params('127.0.0.1:1717', enable_http=True))

def test_monitoring_metrics(assert_lines):

    assert_lines([
        'enable-metrics = true',
        'metrics-dir = /here',
    ], Section().monitoring.set_metrics_params(enable=True, store_dir='/here'))

    assert_lines([
        'metric-threshold = alarm=some,key=mycounter,reset=0,value=1000',
    ], Section().monitoring.set_metrics_threshold('mycounter', 1000, reset_to=0, alarm='some'))

    section = Section()
    assert_lines([
        'alarm = pinger cmd:ping 127.0.0.1',
        'metric-threshold = alarm=pinger,key=mycounter,value=2000',
    ], section.monitoring.set_metrics_threshold(
        'mycounter', 2000, alarm=section.alarms.alarm_types.command('pinger', 'ping 127.0.0.1')
    ))

def test_monitoring_register_metric(assert_lines):

    monitoring = Section().monitoring

    my_abs = monitoring.metric_types.absolute('myabs', initial_value=100)
    my_abs2 = monitoring.metric_types.alias('myabs2', initial_value=200, alias_for=my_abs)

    with pytest.raises(ConfigurationError):
        # Alias without alias_for
        monitoring.metric_types.alias('myabs3', initial_value=200)

    assert_lines([
        'metric = name=myabs,initial_value=100,type=absolute',
        'metric = name=myabs2,alias=myabs,initial_value=200,type=alias',
        'metric = name=mymet,collector=sum,children=worker.1.requests;myabs,type=gauge',

    ], monitoring.register_metric([
        my_abs,
        my_abs2,
        monitoring.metric_types.gauge(
            'mymet',
            collector=monitoring.collectors.sum(['worker.1.requests', my_abs]),
        )
    ]))

def test_monitoring_collectors(assert_lines):

    collectors = Section().monitoring.collectors

    assert str(collectors.manual()) == 'manual'
    assert str(collectors.pointer()) == 'ptr'
    assert str(collectors.file('/here/a', get_slot=2)) == 'file,arg1=/here/a,arg1n=2'
    assert str(collectors.function('some')) == 'func,arg1=some'
    assert str(collectors.sum(['a', 'b'])) == 'sum,children=a;b'
    assert str(collectors.avg(['a', 'b'])) == 'avg,children=a;b'
    assert str(collectors.accumulator(['a', 'b'])) == 'accumulator,children=a;b'
    assert str(collectors.adder(['a', 'b'], 3)) == 'adder,arg1n=3,children=a;b'
    assert str(collectors.multiplier(['a', 'b'], 4)) == 'multiplier,arg1n=4,children=a;b'

def test_monitoring_pushers(assert_lines):

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = socket:127.0.0.1:8125,myinstance',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.socket('127.0.0.1:8125', 'myinstance')
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.rrdtool('/here')
    pusher.set_basic_params('libmyrdd.so')
    assert_lines([
        'stats-push = rrdtool:/here',
        'rrdtool-lib = libmyrdd.so',

    ], monitoring.register_stats_pusher(pusher))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.statsd('127.0.0.1:8125', 'myinstance')
    pusher.set_basic_params(no_workers=True)
    assert_lines([
        'stats-push = statsd:127.0.0.1:8125,myinstance',
        'statsd-no-workers = true',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.carbon('127.0.0.1:2003')
    pusher.set_basic_params(root_node='myroot')
    pusher.set_connection_params(retries=5)
    assert_lines([
        'stats-push = carbon:127.0.0.1:2003',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    pusher = monitoring.pushers.zabbix('127.0.0.1:10051')
    pusher.set_basic_params('/put/here')
    assert_lines([
        'stats-push = zabbix:127.0.0.1:10051',
        'zabbix-template = /put/here',

    ], monitoring.register_stats_pusher(
        pusher
    ))

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = mongodb:collection=mycol,freq=10',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.mongo(collection='mycol', push_interval=10)
    ))

    monitoring = Section().monitoring
    assert_lines([
        'stats-push = file:path=/here/a.json,freq=15',

    ], monitoring.register_stats_pusher(
        monitoring.pushers.file('/here/a.json', push_interval=15)
    ))
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding:utf-8 -*-
"""
Conversion: perform any conversions needed (or leave the data as it is, if not needed)

All exposed WS return JSON with a status OK/KO, along with an error
description if that's the case.
"""

import logging
import os
import glob
import json
import sys
import math
import ConfigParser
import re
import base64
import mimetypes
import cherrypy
import cv2
import numpy as np
from ipolimage import Image
from ipolevaluator import evaluate

from errors import IPOLConvertInputError
from errors import IPOLCropInputError

def authenticate(func):
    """
    Wrapper to authenticate before using an exposed function
    """

    def authenticate_and_call(*args, **kwargs):
        """
        Invokes the wrapped function if authenticated
        """
        if not is_authorized_ip(cherrypy.request.remote.ip) and not (\
                        "X-Real-IP" in cherrypy.request.headers and is_authorized_ip(\
                    cherrypy.request.headers["X-Real-IP"])):
            error = {'status': 'KO', 'error': "Authentication Failed"}
            return json.dumps(error)
        return func(*args, **kwargs)

    def is_authorized_ip(ip):
        """
        Validates the given IP
        """
        conversion = Conversion.get_instance()
        patterns = []
        # Creates the patterns  with regular expressions
        for authorized_pattern in conversion.authorized_patterns:
            patterns.append(re.compile(authorized_pattern.replace(".", r"\.").replace("*", "[0-9]*")))
        # Compare the IP with the patterns
        for pattern in patterns:
            if pattern.match(ip) is not None:
                return True
        return False

    return authenticate_and_call

class Conversion(object):
    """
    The Conversion module
    """

    instance = None

    @staticmethod
    def get_instance():
        """
        Singleton pattern
        """
        if Conversion.instance is None:
            Conversion.instance = Conversion()
        return Conversion.instance

    def __init__(self):
        """
        Initialize Conversion class
        """
        self.base_directory = os.getcwd()
        self.logs_dir = cherrypy.config.get("logs_dir")
        self.host_name = cherrypy.config.get("server.socket_host")
        self.config_common_dir = cherrypy.config.get("config_common_dir")

        # Logs
        try:
            if not os.path.exists(self.logs_dir):
                os.makedirs(self.logs_dir)
            self.logger = self.init_logging()
        except Exception as ex:
            self.logger.exception("Failed to create log dir (using file dir). {}: {}".format(type(ex).__name__, str(ex)))

        # Security: authorized IPs
        self.authorized_patterns = self.read_authorized_patterns()

    def read_authorized_patterns(self):
        """
        Read from the IPs conf file
        """
        # Check if the config file exists
        authorized_patterns_path = os.path.join(self.config_common_dir, "authorized_patterns.conf")
        if not os.path.isfile(authorized_patterns_path):
            self.logger.error("read_authorized_patterns: Can't open {}".format(authorized_patterns_path))
            return []

        # Read config file
        try:
            cfg = ConfigParser.ConfigParser()
            cfg.read([authorized_patterns_path])
            patterns = []
            for item in cfg.items('Patterns'):
                patterns.append(item[1])
            return patterns
        except ConfigParser.Error:
            self.logger.exception("Bad format in {}".format(authorized_patterns_path))
            return []

    def init_logging(self):
        """
        Initialize the error logs of the module.
        """
        logger = logging.getLogger("conversion_log")

        logger.setLevel(logging.DEBUG)
        handler = logging.FileHandler(os.path.join(self.logs_dir, 'error.log'))
        formatter = logging.Formatter(
            '%(asctime)s; [%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
        handler.setFormatter(formatter)

        logger.addHandler(handler)
        return logger

    @staticmethod
    @cherrypy.expose
    def index():
        """
        index of the module.
        """
        return "Conversion module"

    @staticmethod
    @cherrypy.expose
    def default(attr):
        """
        Default method invoked when asked for non-existing service.
        """
        data = {'status': 'KO', "message": "Unknown service '{}'".format(attr)}
        return json.dumps(data)

    @staticmethod
    @cherrypy.expose
    def ping():
        """
        Ping service: answer with a PONG.
        """
        data = {'status': 'OK', "ping": "pong"}
        return json.dumps(data)

    @cherrypy.expose
    @authenticate
    def shutdown(self):
        """
        Shutdown the module.
        """
        data = {'status': 'KO'}
        try:
            cherrypy.engine.exit()
            data['status'] = 'OK'
        except Exception as ex:
            self.logger.exception("Failed to shutdown. {}: {}".format(type(ex).__name__, str(ex)))
            sys.exit(1)
        return json.dumps(data)

    @cherrypy.expose
    def convert(self, work_dir, inputs_description, crop_info=None):
        # inputs_desc seems more consistent accross modules, should be changed here and in core.py and test.py
        """
        Pre-process the input data in their working directory, according to the DDL specs and optional crop.
        For each file processed, return a code
        -1: exception, response KO
        0:  No conversion needed, returned by convert_image()
        1:  Conversion performed (ex: image resizing), returned by convert_image()
        2:  Conversion is not authorized by ddl, returned by convert_image()
        """
        # Info is a dictionary for each input, with return code and possible error
        info = {}
        # Global try, will stop at first exception in an input
        try:
            inputs_desc = json.loads(inputs_description)
            if crop_info is not None:
                crop_info = json.loads(crop_info)
            # loop on ddl inputs
            inputs_len = len(inputs_desc)
            for i in range(inputs_len):
                input_desc = inputs_desc[i]
                # before transformation success, default return code is failure
                info[i] = {'code': -1}
                # Search for a file for this input
                pattern = os.path.join(work_dir, 'input_{}'.format(i)) + '.*'
                input_files = glob.glob(pattern)
                # no file found, is this input optional?
                if len(input_files) < 1:
                    # optional is said by {"required": False}, absence of required field means: required
                    if not input_desc.get('required', True):
                        del info[i] # input[i] not present but not required, say nothing
                        continue
                    # An input is required and is absent, warn but no exception
                    info[i]['error'] = "Input required, but file not found in: {}".format(pattern)
                    continue
                input_file = input_files[0]
                # Is file too large for expected input in DDL?
                if input_desc.get('max_weight') and os.path.getsize(input_file) > input_desc.get('max_weight'):
                    info[i]['error'] = "File too large: {}".format(input_file)
                    continue
                # check input type
                input_type = input_desc['type']
                if input_type not in ['image', 'data', 'video']:
                    info[i]['error'] = "{}: unknown input type".format(input_type)
                    continue
                # do conversion
                if input_desc.get('type') == 'image':
                    info[i]['code'] = self.convert_image(input_file, input_desc, crop_info)
                elif input_desc.get('type') == "data":
                    info[i]['code'] = self.add_ext_to_data(input_file, input_desc)
                elif input_desc.get('type') == "video":
                    info[i]['code'] = 1

        except IPOLConvertInputError as ex:
            message = "Input #{}. {}".format(i, str(ex))
            return self.make_KO_response(message, work_dir)
        except IPOLCropInputError as ex:
            message = "Input #{}. {}".format(i, str(ex))
            return self.make_KO_response(message, work_dir)
        except (OSError, IOError) as ex:
            message = "Input #{}. {}: {}".format(i, type(ex).__name__, str(ex))
            return self.make_KO_response(message, work_dir)
        except Exception as ex:
            message = "Input #{}, unexpected error. {}: {}. file: {}".format(i, type(ex).__name__, str(ex), input_file)
            return self.make_KO_response(message, work_dir)
        # globally OK (no exception), but for some input, a return code could be -1
        return json.dumps({'status': 'OK', 'info': info})

    def make_KO_response(self, message, work_dir):
        """
        Return a JSON KO response with an error message.
        """
        response = {'status':'KO'}
        self.logger.exception(message)
        # do not send full path to client
        response['error'] = message.replace(work_dir, '<work_dir>')
        print json.dumps(response, indent=2, ensure_ascii=False)
        print message
        return json.dumps(response)

    def convert_image(self, input_file, input_desc, crop_info=None):
        """
        Convert image if needed
        """
        # Method could be a function (no-self-use)
        code = 0
        # Image has to be always loaded to test width and size
        im = Image(input_file)

        # convert image matrix if needed (before resize)
        if im.convert(input_desc.get('dtype')):
            code = 1

        # crop before reducing image to max_pixels (or the crop box will be outside of scope)
        if crop_info is not None:
            # Crop is needed
            x = int(round(crop_info.get('x')))
            y = int(round(crop_info.get('y')))
            width = int(round(crop_info.get('width')))
            height = int(round(crop_info.get('height')))
            im.crop(x=x, y=y, width=width, height=height)
            code = 1

        w_h = re.compile(' *[x*] *').split(input_desc.get('max_pixels'))
        w_h = list(map(int, w_h)) # cast to int, py3 compat
        # demo may wait a containing box
        if len(w_h) == 2 and w_h[0] > 0 and w_h[1] > 0:
            if im.width > w_h[0] or im.height > w_h[1]:
                im.resize(width=w_h[0], height=w_h[1])
                code = 1
        else:
            max_pixels = evaluate(input_desc.get('max_pixels'))
            input_pixels = im.width * im.height
            if input_pixels > max_pixels:
                fxy = math.sqrt(float(max_pixels - 100) / float(input_pixels))
                im.resize(fx=fxy, fy=fxy)
                code = 1

        input_file_type, _ = mimetypes.guess_type(input_file)
        input_desc_type, _ = mimetypes.guess_type("dummy" + input_desc.get('ext'))
        if input_file_type != input_desc_type:
            # Another encoding (extension) expected, rename file with right extension, will be done on saving
            input_file = os.path.splitext(input_file)[0] + input_desc.get('ext')
            code = 1

        if code == 1: # image data have been modified, save it
            if input_desc.get("forbid_preprocess", False):
                return 2 # Conversion needed but forbidden
            im.write(input_file, force=True) # force overwrites if needed
        return code

    @staticmethod
    def add_ext_to_data(input_file, input_desc):
        """
        Add the specified extension to the data file
        """
        ext = input_desc.get('ext')
        if ext is None: # Why not an IPOLDDL error or something like that ?
            raise IPOLConvertInputError('DDL Error, missing extension field (to convert the input in the expected format by demo)')
        filename_no_ext, _ = os.path.splitext(input_file)
        input_with_extension = filename_no_ext + ext
        os.rename(input_file, input_with_extension)
        return 0

    @cherrypy.expose
    def convert_tiff_to_png(self, img):
        """
        Converts the input TIFF to PNG.
        This is used by the web interface for visualization purposes
        """
        data = {"status": "KO"}
        try:
            buf = base64.b64decode(img)
            # cv2.IMREAD_COLOR option is 7x faster than load asis and convert to uint8
            # cv2 loads bytes as np vector
            cvim = cv2.imdecode(np.fromstring(buf, dtype=np.uint8), cv2.IMREAD_COLOR) # cv2 l
            _, buf = cv2.imencode(".png", cvim) # [cv2.IMWRITE_PNG_COMPRESSION, 9]
            data["img"] = base64.b64encode(buf) # reencode bytes
            data["status"] = "OK"
        except Exception as ex:
            print "Failed to convert image from TIFF to PNG", ex
            self.logger.exception("Failed to convert image from TIFF to PNG")
        return json.dumps(data)
<commit_msg>[modules/conversion] only evaluator<commit_after>#!/usr/bin/python
# -*- coding:utf-8 -*-
"""
Conversion: perform any conversions needed (or leave the data as it is, if not needed)

All exposed WS return JSON with a status OK/KO, along with an error
description if that's the case.
"""

import logging
import os
import glob
import json
import sys
import math
import ConfigParser
import re
import base64
import mimetypes
import traceback
import cherrypy
import cv2
import numpy as np
from ipolimage import Image
from ipolevaluator import evaluate

from errors import IPOLConvertInputError
from errors import IPOLCropInputError

def authenticate(func):
    """
    Wrapper to authenticate before using an exposed function
    """

    def authenticate_and_call(*args, **kwargs):
        """
        Invokes the wrapped function if authenticated
        """
        if not is_authorized_ip(cherrypy.request.remote.ip) and not (\
                        "X-Real-IP" in cherrypy.request.headers and is_authorized_ip(\
                    cherrypy.request.headers["X-Real-IP"])):
            error = {'status': 'KO', 'error': "Authentication Failed"}
            return json.dumps(error)
        return func(*args, **kwargs)

    def is_authorized_ip(ip):
        """
        Validates the given IP
        """
        conversion = Conversion.get_instance()
        patterns = []
        # Creates the patterns  with regular expressions
        for authorized_pattern in conversion.authorized_patterns:
            patterns.append(re.compile(authorized_pattern.replace(".", r"\.").replace("*", "[0-9]*")))
        # Compare the IP with the patterns
        for pattern in patterns:
            if pattern.match(ip) is not None:
                return True
        return False

    return authenticate_and_call

class Conversion(object):
    """
    The Conversion module
    """

    instance = None

    @staticmethod
    def get_instance():
        """
        Singleton pattern
        """
        if Conversion.instance is None:
            Conversion.instance = Conversion()
        return Conversion.instance

    def __init__(self):
        """
        Initialize Conversion class
        """
        self.base_directory = os.getcwd()
        self.logs_dir = cherrypy.config.get("logs_dir")
        self.host_name = cherrypy.config.get("server.socket_host")
        self.config_common_dir = cherrypy.config.get("config_common_dir")

        # Logs
        try:
            if not os.path.exists(self.logs_dir):
                os.makedirs(self.logs_dir)
            self.logger = self.init_logging()
        except Exception as ex:
            self.logger.exception("Failed to create log dir (using file dir). {}: {}".format(type(ex).__name__, str(ex)))

        # Security: authorized IPs
        self.authorized_patterns = self.read_authorized_patterns()

    def read_authorized_patterns(self):
        """
        Read from the IPs conf file
        """
        # Check if the config file exists
        authorized_patterns_path = os.path.join(self.config_common_dir, "authorized_patterns.conf")
        if not os.path.isfile(authorized_patterns_path):
            self.logger.error("read_authorized_patterns: Can't open {}".format(authorized_patterns_path))
            return []

        # Read config file
        try:
            cfg = ConfigParser.ConfigParser()
            cfg.read([authorized_patterns_path])
            patterns = []
            for item in cfg.items('Patterns'):
                patterns.append(item[1])
            return patterns
        except ConfigParser.Error:
            self.logger.exception("Bad format in {}".format(authorized_patterns_path))
            return []

    def init_logging(self):
        """
        Initialize the error logs of the module.
        """
        logger = logging.getLogger("conversion_log")

        logger.setLevel(logging.DEBUG)
        handler = logging.FileHandler(os.path.join(self.logs_dir, 'error.log'))
        formatter = logging.Formatter(
            '%(asctime)s; [%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')
        handler.setFormatter(formatter)

        logger.addHandler(handler)
        return logger

    @staticmethod
    @cherrypy.expose
    def index():
        """
        index of the module.
        """
        return "Conversion module"

    @staticmethod
    @cherrypy.expose
    def default(attr):
        """
        Default method invoked when asked for non-existing service.
        """
        data = {'status': 'KO', "message": "Unknown service '{}'".format(attr)}
        return json.dumps(data)

    @staticmethod
    @cherrypy.expose
    def ping():
        """
        Ping service: answer with a PONG.
        """
        data = {'status': 'OK', "ping": "pong"}
        return json.dumps(data)

    @cherrypy.expose
    @authenticate
    def shutdown(self):
        """
        Shutdown the module.
        """
        data = {'status': 'KO'}
        try:
            cherrypy.engine.exit()
            data['status'] = 'OK'
        except Exception as ex:
            self.logger.exception("Failed to shutdown. {}: {}".format(type(ex).__name__, str(ex)))
            sys.exit(1)
        return json.dumps(data)

    @cherrypy.expose
    def convert(self, work_dir, inputs_description, crop_info=None):
        # inputs_desc seems more consistent accross modules, should be changed here and in core.py and test.py
        """
        Pre-process the input data in their working directory, according to the DDL specs and optional crop.
        For each file processed, return a code
        -1: exception, response KO
        0:  No conversion needed, returned by convert_image()
        1:  Conversion performed (ex: image resizing), returned by convert_image()
        2:  Conversion is not authorized by ddl, returned by convert_image()
        """
        # Info is a dictionary for each input, with return code and possible error
        info = {}
        # Global try, will stop at first exception in an input
        try:
            inputs_desc = json.loads(inputs_description)
            if crop_info is not None:
                crop_info = json.loads(crop_info)
            # loop on ddl inputs
            inputs_len = len(inputs_desc)
            for i in range(inputs_len):
                input_desc = inputs_desc[i]
                # before transformation success, default return code is failure
                info[i] = {'code': -1}
                # Search for a file for this input
                pattern = os.path.join(work_dir, 'input_{}'.format(i)) + '.*'
                input_files = glob.glob(pattern)
                # no file found, is this input optional?
                if len(input_files) < 1:
                    # optional is said by {"required": False}, absence of required field means: required
                    if not input_desc.get('required', True):
                        del info[i] # input[i] not present but not required, say nothing
                        continue
                    # An input is required and is absent, warn but no exception
                    info[i]['error'] = "Input required, but file not found in: {}".format(pattern)
                    continue
                input_file = input_files[0]
                # Is file too large for expected input in DDL?
                if input_desc.get('max_weight') and os.path.getsize(input_file) > input_desc.get('max_weight'):
                    info[i]['error'] = "File too large: {}".format(input_file)
                    continue
                # check input type
                input_type = input_desc['type']
                if input_type not in ['image', 'data', 'video']:
                    info[i]['error'] = "{}: unknown input type".format(input_type)
                    continue
                # do conversion
                if input_desc.get('type') == 'image':
                    info[i]['code'] = self.convert_image(input_file, input_desc, crop_info)
                elif input_desc.get('type') == "data":
                    info[i]['code'] = self.add_ext_to_data(input_file, input_desc)
                elif input_desc.get('type') == "video":
                    info[i]['code'] = 1

        except IPOLConvertInputError as ex:
            message = "Input #{}. {}".format(i, str(ex))
            return self.make_KO_response(message, work_dir, ex)
        except IPOLCropInputError as ex:
            message = "Input #{}. {}".format(i, str(ex))
            return self.make_KO_response(message, work_dir, ex)
        except (OSError, IOError) as ex:
            message = "Input #{}. {}: {}".format(i, type(ex).__name__, str(ex))
            return self.make_KO_response(message, work_dir, ex)
        except Exception as ex:
            message = "Input #{}, unexpected error. {}: {}. file: {}".format(i, type(ex).__name__, str(ex), input_file)
            return self.make_KO_response(message, work_dir, ex)
        # globally OK (no exception), but for some input, a return code could be -1
        return json.dumps({'status': 'OK', 'info': info})

    def make_KO_response(self, message, work_dir, ex):
        """
        Return a JSON KO response with an error message.
        """
        response = {'status':'KO'}
        self.logger.exception(message)
        # do not send full path to client
        response['error'] = message.replace(work_dir, '<work_dir>')
        print(traceback.format_exc())
        print json.dumps(response, indent=2, ensure_ascii=False)
        print message
        return json.dumps(response)

    def convert_image(self, input_file, input_desc, crop_info=None):
        """
        Convert image if needed
        """
        # Method could be a function (no-self-use)
        code = 0
        # Image has to be always loaded to test width and size
        im = Image(input_file)

        # convert image matrix if needed (before resize)
        if im.convert(input_desc.get('dtype')):
            code = 1

        # crop before reducing image to max_pixels (or the crop box will be outside of scope)
        if crop_info is not None:
            # Crop is needed
            x = int(round(crop_info.get('x')))
            y = int(round(crop_info.get('y')))
            width = int(round(crop_info.get('width')))
            height = int(round(crop_info.get('height')))
            im.crop(x=x, y=y, width=width, height=height)
            code = 1

        max_pixels = evaluate(input_desc.get('max_pixels'))
        input_pixels = im.width * im.height
        if input_pixels > max_pixels:
            fxy = math.sqrt(float(max_pixels - 1) / float(input_pixels))
            im.resize(fx=fxy, fy=fxy)
            code = 1

        input_file_type, _ = mimetypes.guess_type(input_file)
        input_desc_type, _ = mimetypes.guess_type("dummy" + input_desc.get('ext'))
        if input_file_type != input_desc_type:
            # Another encoding (extension) expected, rename file with right extension, will be done on saving
            input_file = os.path.splitext(input_file)[0] + input_desc.get('ext')
            code = 1

        if code == 1: # image data have been modified, save it
            if input_desc.get("forbid_preprocess", False):
                return 2 # Conversion needed but forbidden
            im.write(input_file, force=True) # force overwrites if needed
        return code

    @staticmethod
    def add_ext_to_data(input_file, input_desc):
        """
        Add the specified extension to the data file
        """
        ext = input_desc.get('ext')
        if ext is None: # Why not an IPOLDDL error or something like that ?
            raise IPOLConvertInputError('DDL Error, missing extension field (to convert the input in the expected format by demo)')
        filename_no_ext, _ = os.path.splitext(input_file)
        input_with_extension = filename_no_ext + ext
        os.rename(input_file, input_with_extension)
        return 0

    @cherrypy.expose
    def convert_tiff_to_png(self, img):
        """
        Converts the input TIFF to PNG.
        This is used by the web interface for visualization purposes
        """
        data = {"status": "KO"}
        try:
            buf = base64.b64decode(img)
            # cv2.IMREAD_COLOR option is 7x faster than load asis and convert to uint8
            # cv2 loads bytes as np vector
            cvim = cv2.imdecode(np.fromstring(buf, dtype=np.uint8), cv2.IMREAD_COLOR) # cv2 l
            _, buf = cv2.imencode(".png", cvim) # [cv2.IMWRITE_PNG_COMPRESSION, 9]
            data["img"] = base64.b64encode(buf) # reencode bytes
            data["status"] = "OK"
        except Exception as ex:
            print "Failed to convert image from TIFF to PNG", ex
            self.logger.exception("Failed to convert image from TIFF to PNG")
        return json.dumps(data)
<|endoftext|>


<commit_before>import numpy

import chainer
from chainer.backends import cuda
from chainer import distribution
from chainer.functions.array import expand_dims
from chainer.functions.math import digamma
from chainer.functions.math import exponential
from chainer.functions.math import lgamma
from chainer.functions.math import sum as sum_mod

def _lbeta(x):
    return sum_mod.sum(lgamma.lgamma(x), axis=-1) \
        - lgamma.lgamma(sum_mod.sum(x, axis=-1))

class Dirichlet(distribution.Distribution):

    """Dirichlet Distribution.

    The probability density function of the distribution is expressed as

    .. math::
        p(x) = \\frac{\\Gamma(\\sum_{i=1}^{K} \\alpha_i)}
            {\\prod_{i=1}^{K} \\Gamma (\\alpha_i)}
            \\prod_{i=1}^{K} {x_i}^{\\alpha_i-1}

    Args:
        alpha(:class:`~chainer.Variable` or :ref:`ndarray`): Parameter of
            distribution.
    """

    def __init__(self, alpha):
        self.__alpha = chainer.as_variable(alpha)

    @property
    def alpha(self):
        return self.__alpha

    @property
    def alpha0(self):
        return sum_mod.sum(self.alpha, axis=-1)

    @property
    def batch_shape(self):
        return self.alpha.shape[:-1]

    @property
    def entropy(self):
        return _lbeta(self.alpha) \
            + (self.alpha0 - self.event_shape[0]) \
            * digamma.digamma(self.alpha0) \
            - sum_mod.sum((self.alpha - 1)
                          * digamma.digamma(self.alpha), axis=-1)

    @property
    def event_shape(self):
        return self.alpha.shape[-1:]

    def log_prob(self, x):
        return - _lbeta(self.alpha) \
            + sum_mod.sum((self.alpha - 1) * exponential.log(x), axis=-1)

    @property
    def mean(self):
        alpha0 = expand_dims.expand_dims(self.alpha0, axis=-1)
        return self.alpha / alpha0

    def sample_n(self, n):
        obo_alpha = self.alpha.data.reshape(-1, self.event_shape[0])
        xp = cuda.get_array_module(self.alpha)
        if xp is numpy:
            eps = [xp.random.dirichlet(
                one_alpha, size=(n,)).astype(numpy.float32)
                for one_alpha in obo_alpha]
        else:
            eps = [xp.random.dirichlet(
                one_alpha, size=(n,)).astype(numpy.float32)
                for one_alpha in obo_alpha]
        eps = [xp.expand_dims(eps_, 0) for eps_ in eps]
        eps = xp.swapaxes(xp.vstack(eps), 0, 1)
        eps = eps.reshape((n,) + self.alpha.shape)
        noise = chainer.Variable(eps)
        return noise

    @property
    def support(self):
        return '[0, 1]'

    @property
    def variance(self):
        alpha0 = expand_dims.expand_dims(self.alpha0, axis=-1)
        return self.alpha * (alpha0 - self.alpha) \
            / alpha0 ** 2 / (alpha0 + 1)

@distribution.register_kl(Dirichlet, Dirichlet)
def _kl_dirichlet_dirichlet(dist1, dist2):
    return - _lbeta(dist1.alpha) + _lbeta(dist2.alpha) \
        + sum_mod.sum((dist1.alpha - dist2.alpha) * (
            digamma.digamma(dist1.alpha)
            - expand_dims.expand_dims(digamma.digamma(
                dist1.alpha0), axis=-1)), axis=-1)
<commit_msg>Cache attributes of Dirichlet<commit_after>import numpy

import chainer
from chainer.backends import cuda
from chainer import distribution
from chainer.functions.array import expand_dims
from chainer.functions.math import digamma
from chainer.functions.math import exponential
from chainer.functions.math import lgamma
from chainer.functions.math import sum as sum_mod
from chainer.utils import cache

def _lbeta(x):
    return sum_mod.sum(lgamma.lgamma(x), axis=-1) \
        - lgamma.lgamma(sum_mod.sum(x, axis=-1))

class Dirichlet(distribution.Distribution):

    """Dirichlet Distribution.

    The probability density function of the distribution is expressed as

    .. math::
        p(x) = \\frac{\\Gamma(\\sum_{i=1}^{K} \\alpha_i)}
            {\\prod_{i=1}^{K} \\Gamma (\\alpha_i)}
            \\prod_{i=1}^{K} {x_i}^{\\alpha_i-1}

    Args:
        alpha(:class:`~chainer.Variable` or :ref:`ndarray`): Parameter of
            distribution.
    """

    def __init__(self, alpha):
        self.__alpha = alpha

    @cache.cached_property
    def alpha(self):
        return chainer.as_variable(self.__alpha)

    @cache.cached_property
    def alpha0(self):
        return sum_mod.sum(self.alpha, axis=-1)

    @property
    def batch_shape(self):
        return self.alpha.shape[:-1]

    @cache.cached_property
    def entropy(self):
        return _lbeta(self.alpha) \
            + (self.alpha0 - self.event_shape[0]) \
            * digamma.digamma(self.alpha0) \
            - sum_mod.sum((self.alpha - 1)
                          * digamma.digamma(self.alpha), axis=-1)

    @property
    def event_shape(self):
        return self.alpha.shape[-1:]

    def log_prob(self, x):
        return - _lbeta(self.alpha) \
            + sum_mod.sum((self.alpha - 1) * exponential.log(x), axis=-1)

    @cache.cached_property
    def mean(self):
        alpha0 = expand_dims.expand_dims(self.alpha0, axis=-1)
        return self.alpha / alpha0

    def sample_n(self, n):
        obo_alpha = self.alpha.data.reshape(-1, self.event_shape[0])
        xp = cuda.get_array_module(self.alpha)
        if xp is numpy:
            eps = [xp.random.dirichlet(
                one_alpha, size=(n,)).astype(numpy.float32)
                for one_alpha in obo_alpha]
        else:
            eps = [xp.random.dirichlet(
                one_alpha, size=(n,)).astype(numpy.float32)
                for one_alpha in obo_alpha]
        eps = [xp.expand_dims(eps_, 0) for eps_ in eps]
        eps = xp.swapaxes(xp.vstack(eps), 0, 1)
        eps = eps.reshape((n,) + self.alpha.shape)
        noise = chainer.Variable(eps)
        return noise

    @property
    def support(self):
        return '[0, 1]'

    @cache.cached_property
    def variance(self):
        alpha0 = expand_dims.expand_dims(self.alpha0, axis=-1)
        return self.alpha * (alpha0 - self.alpha) \
            / alpha0 ** 2 / (alpha0 + 1)

@distribution.register_kl(Dirichlet, Dirichlet)
def _kl_dirichlet_dirichlet(dist1, dist2):
    return - _lbeta(dist1.alpha) + _lbeta(dist2.alpha) \
        + sum_mod.sum((dist1.alpha - dist2.alpha) * (
            digamma.digamma(dist1.alpha)
            - expand_dims.expand_dims(digamma.digamma(
                dist1.alpha0), axis=-1)), axis=-1)
<|endoftext|>


<commit_before>"""Handle commands."""

import random

from ..commands import COMMANDS
from ..handler import Handler
from ..packets import MessagePacket

class CommandHandler(Handler):
    """Command handler."""

    ARGN_EXPR = r'%ARG(\d+)(?:=([^|]+))?(?:((?:\|\w+)+))?%'
    ARGS_EXPR = r'%ARGS(?:=([^|]+))?(?:((?:\|\w+)+))?%'
    MODIFIERS = {
        "upper": str.upper,
        "lower": str.lower,
        "title": str.title,
        "reverse": lambda text: text[::-1],
        "tag": lambda tag: tag[1:] if tag[0] == '@' and len(tag) > 1 else tag,
        "shuffle": lambda text: ''.join(random.sample(text, len(text)))
    }

    def __init__(self, channel, api):
        super().__init__()

        self.channel = channel
        self.api = api

        self.magics = {command.COMMAND: command(api) for command in COMMANDS}

    async def on_message(self, packet):
        """Handle message events."""

        if packet.target and packet.text == "/cry":
            return MessagePacket(
                "cries with ", ("tag", packet.user), action=True)

        if len(packet) > 1 and packet[0] == "!" and packet[1] != ' ':

            command, *args = packet[1:].text.split()

            data = {
                "username": packet.user,
                "channel": self.channel,
                "packet": packet
            }

            if command in self.magics:

                response = await self.magics[command](*args, **data)

                if packet.target and response:
                    if not isinstance(response, MessagePacket):
                        response = MessagePacket(response)
                    response.target = packet.user

                return response

            else:

                split = packet.split()
                hyphenated_options = (((split[:index]), split[index:])
                                      for index in range(len(split), 0, -1))

                for command, args in hyphenated_options:

                    command = MessagePacket.join(*command, separator='-').text[1:]
                    args = tuple(arg.text for arg in args)

                    response = await self.custom_response(
                        packet, command, *args, **data)

                    if response is not None:
                        return response

                return MessagePacket("Command not found.", target=packet.user)

    async def custom_response(self, _packet, command, *args, **data):

        response = await self.api.get_command(command)
        if response.status == 404:
            response = await self.api.get_command_alias(command)

            if response.status == 404:
                return None
            else:
                json = await response.json()

                args = MessagePacket(
                    *json["data"]["attributes"]["arguments"]
                ).text.split() + args

                json = json["data"]["attributes"]["command"][
                    "response"]
        else:
            json = await response.json()
            json = json["data"]["attributes"]["response"]

        json["target"] = _packet.user if _packet.target else None

        return self._inject(MessagePacket.from_json(json),
                            command, *args, **data)

    def _inject(self, _packet, *args, **data):
        """Inject targets into a packet."""

        def sub_argn(match):
            """Substitute an argument in place of an ARGN target."""

            argn, default, modifiers = match.groups()
            argn = int(argn)

            if default is None:
                result = args[argn]
            else:
                result = args[argn] if argn < len(args) else default

            if modifiers is not None:
                result = self._modify(result, *modifiers.split('|')[1:])

            return result

        try:
            _packet.sub(self.ARGN_EXPR, sub_argn)
        except IndexError:
            return MessagePacket("Not enough arguments!")

        def sub_args(match):
            """Substitute all arguments in place of the ARGS target."""

            default, modifiers = match.groups()

            if not args[1:] and default is not None:
                result = default
            else:
                result = ' '.join(args[1:])

            if modifiers is not None:
                result = self._modify(result, *modifiers.split('|')[1:])

            return result

        if "%ARGS%" in _packet and len(args) < 2:
            return MessagePacket("Not enough arguments!")

        _packet.sub(self.ARGS_EXPR, sub_args)

        _packet.replace(**{
            "%USER%": data.get("username"),
            "%COUNT%": "%COUNT%",  # TODO
            "%CHANNEL%": data.get("channel")
        })

        return _packet

    def _modify(self, argument, *modifiers):
        """Apply modifiers to an argument."""

        for modifier in modifiers:
            if modifier in self.MODIFIERS:
                argument = self.MODIFIERS[modifier](argument)

        return argument

    async def on_repeat(self, packet):
        return packet
<commit_msg>Fix linting<commit_after>"""Handle commands."""

import random

from ..commands import COMMANDS
from ..handler import Handler
from ..packets import MessagePacket

class CommandHandler(Handler):
    """Command handler."""

    ARGN_EXPR = r'%ARG(\d+)(?:=([^|]+))?(?:((?:\|\w+)+))?%'
    ARGS_EXPR = r'%ARGS(?:=([^|]+))?(?:((?:\|\w+)+))?%'
    MODIFIERS = {
        "upper": str.upper,
        "lower": str.lower,
        "title": str.title,
        "reverse": lambda text: text[::-1],
        "tag": lambda tag: tag[1:] if tag[0] == '@' and len(tag) > 1 else tag,
        "shuffle": lambda text: ''.join(random.sample(text, len(text)))
    }

    def __init__(self, channel, api):
        super().__init__()

        self.channel = channel
        self.api = api

        self.magics = {command.COMMAND: command(api) for command in COMMANDS}

    async def on_message(self, packet):
        """Handle message events."""

        if packet.target and packet.text == "/cry":
            return MessagePacket(
                "cries with ", ("tag", packet.user), action=True)

        if len(packet) > 1 and packet[0] == "!" and packet[1] != ' ':

            command, *args = packet[1:].text.split()

            data = {
                "username": packet.user,
                "channel": self.channel,
                "packet": packet
            }

            if command in self.magics:

                response = await self.magics[command](*args, **data)

                if packet.target and response:
                    if not isinstance(response, MessagePacket):
                        response = MessagePacket(response)
                    response.target = packet.user

                return response

            else:

                split = packet.split()
                hyphenated_options = (((split[:index]), split[index:])
                                      for index in range(len(split), 0, -1))

                for command, args in hyphenated_options:

                    command = MessagePacket.join(*command,
                                                 separator='-').text[1:]
                    args = tuple(arg.text for arg in args)

                    response = await self.custom_response(
                        packet, command, *args, **data)

                    if response is not None:
                        return response

                return MessagePacket("Command not found.", target=packet.user)

    async def custom_response(self, _packet, command, *args, **data):

        response = await self.api.get_command(command)
        if response.status == 404:
            response = await self.api.get_command_alias(command)

            if response.status == 404:
                return None
            else:
                json = await response.json()

                args = MessagePacket(
                    *json["data"]["attributes"]["arguments"]
                ).text.split() + args

                json = json["data"]["attributes"]["command"][
                    "response"]
        else:
            json = await response.json()
            json = json["data"]["attributes"]["response"]

        json["target"] = _packet.user if _packet.target else None

        return self._inject(MessagePacket.from_json(json),
                            command, *args, **data)

    def _inject(self, _packet, *args, **data):
        """Inject targets into a packet."""

        def sub_argn(match):
            """Substitute an argument in place of an ARGN target."""

            argn, default, modifiers = match.groups()
            argn = int(argn)

            if default is None:
                result = args[argn]
            else:
                result = args[argn] if argn < len(args) else default

            if modifiers is not None:
                result = self._modify(result, *modifiers.split('|')[1:])

            return result

        try:
            _packet.sub(self.ARGN_EXPR, sub_argn)
        except IndexError:
            return MessagePacket("Not enough arguments!")

        def sub_args(match):
            """Substitute all arguments in place of the ARGS target."""

            default, modifiers = match.groups()

            if not args[1:] and default is not None:
                result = default
            else:
                result = ' '.join(args[1:])

            if modifiers is not None:
                result = self._modify(result, *modifiers.split('|')[1:])

            return result

        if "%ARGS%" in _packet and len(args) < 2:
            return MessagePacket("Not enough arguments!")

        _packet.sub(self.ARGS_EXPR, sub_args)

        _packet.replace(**{
            "%USER%": data.get("username"),
            "%COUNT%": "%COUNT%",  # TODO
            "%CHANNEL%": data.get("channel")
        })

        return _packet

    def _modify(self, argument, *modifiers):
        """Apply modifiers to an argument."""

        for modifier in modifiers:
            if modifier in self.MODIFIERS:
                argument = self.MODIFIERS[modifier](argument)

        return argument

    async def on_repeat(self, packet):
        return packet
<|endoftext|>


<commit_before>
import sys
import os

'''
Easier searching for good RFI flagging values
'''

try:
    ms_name = sys.argv[1]
except IndexError:
    ms_name = raw_input("Input vis? : ")

# Just want the number of SPWs
tb.open(os.path.join(ms_name, "SPECTRAL_WINDOW"))
nchans = tb.getcol('NUM_CHAN')
tb.close()

spws = range(len(nchans))

default('flagdata')

for spw in spws:
    print("On spw "+str(spw)+" of "+str(len(nchans)))
    freqdevscale = 4.0
    timedevscale = 4.0
    print("Starting at freqdevscale = %s and timedevscale = %s" % (freqdevscale, timedevscale))
    while True:
        flagdata(vis=ms_name, mode='rflag', field='3C48*',
                 spw=str(spw), datacolumn='corrected',
                 action='calculate', display='both',
                 freqdevscale=freqdevscale, timedevscale=timedevscale,
                 flagbackup=False)

        adjust = True if raw_input("New thresholds? : ") == "T" else False

        if adjust:
            print("Current freqdevscale and timedevscale: %s %s" % (freqdevscale, timedevscale))
            freqdevscale = float(raw_input("New freqdevscale : "))
            timedevscale = float(raw_input("New timedevscale : "))
        else:
            break
<commit_msg>Allow for applying of flagging<commit_after>
import sys
import os

'''
Easier searching for good RFI flagging values
'''

try:
    ms_name = sys.argv[1]
    apply_flagging = True if sys.argv[2] == "True" else False
    extend_pol = True if sys.argv[3] == "True" else False
except IndexError:
    ms_name = raw_input("Input vis? : ")
    apply_flagging = True if raw_input("Apply the flagging? : ") == "True" else False
    extend_pol = True if raw_input("Extend across pols?") == "True" else False

# Just want the number of SPWs
tb.open(os.path.join(ms_name, "SPECTRAL_WINDOW"))
nchans = tb.getcol('NUM_CHAN')
tb.close()

spws = range(len(nchans))

default('flagdata')

for spw in spws:
    print("On spw "+str(spw)+" of "+str(len(nchans)))
    freqdevscale = 4.0
    timedevscale = 4.0
    growtime=99.0
    growfreq=99.0
    print("Starting at freqdevscale = %s and timedevscale = %s" % (freqdevscale, timedevscale))
    print("Starting at growfreq = %s and growtime = %s" % (growfreq, growtime))
    while True:
        flagdata(vis=ms_name, mode='rflag', field='3C48*',
                 spw=str(spw), datacolumn='corrected',
                 action='calculate', display='both',
                 freqdevscale=freqdevscale, timedevscale=timedevscale,
                 growtime=growtime, growfreq=growfreq,
                 flagbackup=False)

        adjust = True if raw_input("New thresholds? : ") == "True" else False

        if adjust:
            print("Current freqdevscale and timedevscale: %s %s" % (freqdevscale, timedevscale))
            freqdevscale = float(raw_input("New freqdevscale : "))
            timedevscale = float(raw_input("New timedevscale : "))
            growfreq = float(raw_input("New growfreq : "))
            growtime = float(raw_input("New growtime : "))
        else:
            break

    # Now apply the flagging
    if apply_flagging:
        if extend_pol:
            flagdata(vis=vis, mode='extend',
                     spw=str(spw), extendpols=True,
                     action='apply', display='')
        else:
            flagdata(vis=vos, spw=str(spw),
                     action='apply', display='')

        obliterate = True if raw_input("Flag whole SPW? : ") == "True" else False

        if obliterate:
            flagdata(vis=vis, spw=str(spw))
            print("Flagged all of SPW " + str(spw))
    else:
        print("Applying not enabled.")

<|endoftext|>