<commit_before>'''Example script to run a full analysis on telescope data. The original data can be found in the example folder of the EuTelescope framework.

The residuals are calculated with different cuts on prealigned and aligned data for demonstration purpose:

When only prealigning the DUTs and using all DUT hits and cutting on the chi2:
The residuals are very dependent if the prealignment is sufficient. Residuals are usually rather high (several 10 um)

When aligning the DUTs and only interpolating the tracks from 2 DUTs:
The residual for the planes 2 - 4 (DUT 1 - DUT 3) are about 6.5 um in x/y and comparable to the residuals from the EuTelescope software (6 um).

When aligning the DUTs and using all DUT hits and cutting on the chi2:
The residuals and selected number of tracks are highly dependent on the chi2 cut and are at least 6 um and usually < 10 um depending on the plane position.
This is an effect of multiple scattering. The outer most plans have a rather high residual (~ 18 um)

SETUP:

The telescope consists of 6 planes with 15 cm clearance between the planes.
The data was taken at Desy with ~ 3-4 GeV/c (to be checked).

The Mimosa26 has an active area of 21.2mm x 10.6mm and the pixel matrix consists of 1152 columns and 576 rows (18.4um x 18.4um pixel size).
The total size of the chip is 21.5mm x 13.7mm x 0.036mm (radiation length 9.3660734)

The matrix is divided into 4 areas. For each area the threshold can be set up individually.
The quartes are from column 0-287, 288,575, 576-863 and 864-1151.

The Mimosa26 detects ionizing particle with a density of up to 10^6 hits / cm^2 / s. The hit rate for a beam telescope is ~5 hits / frame.
'''

import os
import logging
from multiprocessing import Pool

import testbeam_analysis
from testbeam_analysis import hit_analysis
from testbeam_analysis import dut_alignment
from testbeam_analysis import track_analysis
from testbeam_analysis import result_analysis

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")

if __name__ == '__main__':  # Main entry point is needed for multiprocessing under windows

    # Get the absolute example path
    tests_data_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(os.path.dirname(testbeam_analysis.__file__))) + r'/examples/data'))

    # The location of the example data files, one file per DUT
    data_files = [(os.path.join(tests_data_folder, r'TestBeamData_Mimosa26_DUT%d' % i + '.h5')) for i in range(6)]  # The first device is the reference for the coordinate system

    # Pixel dimesions and matrix size of the DUTs
    pixel_size = [(18.4, 18.4)] * 6  # Column, row pixel pitch in um
    n_pixels = [(1152, 576)] * 6  # Number of pixel on column, row

    z_positions = [0., 15000, 30000, 45000, 60000, 75000]  # z position in um
    dut_names = ("Tel_0", "Tel_1", "Tel_2", "Tel_3", "Tel_4", "Tel_5")  # Friednly names for plotting

    # Create output subfolder where all output data and plots are stored
    output_folder = os.path.join(os.path.split(data_files[0])[0], 'output')
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # The following shows a complete test beam analysis by calling the seperate function in correct order

    # Remove hot pixel, only needed for devices wih noisy pixel like Mimosa 26
    # A pool of workers to remove the noisy pixels in all files in parallel
    kwargs = [{
        'input_hits_file': data_files[i],
        'n_pixel': n_pixels[i],
        'pixel_size': pixel_size[i],
        'threshold': 5,
        'dut_name': dut_names[i]} for i in range(0, len(data_files))]
    pool = Pool()
    for kwarg in kwargs:
        pool.apply_async(hit_analysis.remove_noisy_pixels, kwds=kwarg)
    pool.close()
    pool.join()

    # Cluster hits off all DUTs
    # A pool of workers to cluster hits in all files in parallel
    kwargs = [{
        'input_hits_file': data_files[i][:-3] + '_noisy_pixels.h5',
        'max_x_distance': 3,
        'max_y_distance': 3,
        'max_time_distance': 2,
        'max_cluster_hits': 1000000,
        'dut_name': dut_names[i]} for i in range(0, len(data_files))]
    pool = Pool()
    for kwarg in kwargs:
        pool.apply_async(hit_analysis.cluster_hits, kwds=kwarg)
    pool.close()
    pool.join()

    # Correlate the row / column of each DUT
    dut_alignment.correlate_cluster(input_cluster_files=[data_file[:-3] + '_noisy_pixels_cluster.h5' for data_file in data_files],
                                    output_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
                                    n_pixels=n_pixels,
                                    pixel_size=pixel_size,
                                    dut_names=dut_names)

    # Create prealignment relative to the first DUT from the correlation data
    dut_alignment.prealignment(input_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
                               output_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               z_positions=z_positions,
                               pixel_size=pixel_size,
                               dut_names=dut_names,
                               fit_background=True,  # This data has several tracks per event and noisy pixel, thus fit existing background
                               non_interactive=True)  # Tries to find cuts automatically; deactivate to do this manualy

    # Merge the cluster tables to one merged table aligned at the event number
    dut_alignment.merge_cluster_data(input_cluster_files=[data_file[:-3] + '_noisy_pixels_cluster.h5' for data_file in data_files],
                                     output_merged_file=os.path.join(output_folder, 'Merged.h5'),
                                     pixel_size=pixel_size)

    # Apply the prealignment to the merged cluster table to create tracklets
    dut_alignment.apply_alignment(input_hit_file=os.path.join(output_folder, 'Merged.h5'),
                                  input_alignment=os.path.join(output_folder, 'Alignment.h5'),
                                  output_hit_aligned_file=os.path.join(output_folder, 'Tracklets_prealigned.h5'),
                                  force_prealignment=True)

    # Find tracks from the prealigned tracklets and stores the with quality indicator into track candidates table
    track_analysis.find_tracks(input_tracklets_file=os.path.join(output_folder, 'Tracklets_prealigned.h5'),
                               input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               output_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'))

    # The following two steps are for demonstration only. They show track fitting and residual calculation on
    # prealigned hits. Usually you are not interessted in this and will use the aligned hits directly.

    # Step 1.: Fit the track candidates and create new track table (using the prealignment!)
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_prealigned.h5'),
                              exclude_dut_hit=True,  # To get unconstrained residuals do not use DUT hit for track fit
                              force_prealignment=True,  # This is just for demonstration purpose, you usually fully aligned hits
                              selection_track_quality=0)  # We will cut on chi2

    # Step 2.:  Calculate the residuals to check the alignment (using the prealignment!)
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_prealigned.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_prealigned.h5'),
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size,
                                        max_chi2=2000,
                                        force_prealignment=True)  # This is just for demonstration purpose, you usually use fully aligned hits

    # Do an alignment step with the track candidates, corrects rotations and is therefore much more precise than simple prealignment
    dut_alignment.alignment(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'),
                            input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                            n_pixels=n_pixels,
                            pixel_size=pixel_size)

    # Apply the alignment to the merged cluster table to create tracklets
    dut_alignment.apply_alignment(input_hit_file=os.path.join(output_folder, 'Merged.h5'),
                                  input_alignment=os.path.join(output_folder, 'Alignment.h5'),
                                  output_hit_aligned_file=os.path.join(output_folder, 'Tracklets.h5'))

    # Find tracks from the tracklets and stores the with quality indicator into track candidates table
    track_analysis.find_tracks(input_tracklets_file=os.path.join(output_folder, 'Tracklets.h5'),
                               input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               output_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'))

    # Example 1: use all DUTs in fit and cut on chi2
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_all.h5'),
                              exclude_dut_hit=True,  # To get unconstrained residuals do not use DUT hit for track fit
                              selection_track_quality=0)    # We do not cut on track quality but on chi2 later

    # Create unconstrained residuals
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_all.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_all.h5'),
                                        max_chi2=500,  # The chi2 cut has a large influence on the residuals and number of tracks, since the resolution is dominated by multiple scattering
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size)

    # Example 2: Use only 2 DUTs next to the fit DUT and cut on track quality. Thus the track fit is just a track interpolation with chi2 = 0.
    # This is better here due to heavily scatterd tracks, where a straight line assumption for all DUTs is wrong.
    # This leads to symmetric residuals in x and y for all DUTs between 2 fits DUTs: 1, 2, 3, 4
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_some.h5'),
                              selection_hit_duts=[[1, 2],  # Only select DUTs next to the DUT to fit
                                                  [0, 2],
                                                  [1, 3],
                                                  [2, 4],
                                                  [3, 5],
                                                  [3, 4]],
                              selection_track_quality=1)    # We cut on track quality

    # Create unconstrained residuals
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_some.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_some.h5'),
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size)
<commit_msg>MAINT: raw string not necessary<commit_after>'''Example script to run a full analysis on telescope data. The original data can be found in the example folder of the EuTelescope framework.

The residuals are calculated with different cuts on prealigned and aligned data for demonstration purpose:

When only prealigning the DUTs and using all DUT hits and cutting on the chi2:
The residuals are very dependent if the prealignment is sufficient. Residuals are usually rather high (several 10 um)

When aligning the DUTs and only interpolating the tracks from 2 DUTs:
The residual for the planes 2 - 4 (DUT 1 - DUT 3) are about 6.5 um in x/y and comparable to the residuals from the EuTelescope software (6 um).

When aligning the DUTs and using all DUT hits and cutting on the chi2:
The residuals and selected number of tracks are highly dependent on the chi2 cut and are at least 6 um and usually < 10 um depending on the plane position.
This is an effect of multiple scattering. The outer most plans have a rather high residual (~ 18 um)

SETUP:

The telescope consists of 6 planes with 15 cm clearance between the planes.
The data was taken at Desy with ~ 3-4 GeV/c (to be checked).

The Mimosa26 has an active area of 21.2mm x 10.6mm and the pixel matrix consists of 1152 columns and 576 rows (18.4um x 18.4um pixel size).
The total size of the chip is 21.5mm x 13.7mm x 0.036mm (radiation length 9.3660734)

The matrix is divided into 4 areas. For each area the threshold can be set up individually.
The quartes are from column 0-287, 288,575, 576-863 and 864-1151.

The Mimosa26 detects ionizing particle with a density of up to 10^6 hits / cm^2 / s. The hit rate for a beam telescope is ~5 hits / frame.
'''

import os
import logging
from multiprocessing import Pool

import testbeam_analysis
from testbeam_analysis import hit_analysis
from testbeam_analysis import dut_alignment
from testbeam_analysis import track_analysis
from testbeam_analysis import result_analysis

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")

if __name__ == '__main__':  # Main entry point is needed for multiprocessing under windows

    # Get the absolute example path
    tests_data_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(os.path.dirname(testbeam_analysis.__file__))) + r'/examples/data'))

    # The location of the example data files, one file per DUT
    data_files = [(os.path.join(tests_data_folder, 'TestBeamData_Mimosa26_DUT%d' % i + '.h5')) for i in range(6)]  # The first device is the reference for the coordinate system

    # Pixel dimesions and matrix size of the DUTs
    pixel_size = [(18.4, 18.4)] * 6  # Column, row pixel pitch in um
    n_pixels = [(1152, 576)] * 6  # Number of pixel on column, row

    z_positions = [0., 15000, 30000, 45000, 60000, 75000]  # z position in um
    dut_names = ("Tel_0", "Tel_1", "Tel_2", "Tel_3", "Tel_4", "Tel_5")  # Friednly names for plotting

    # Create output subfolder where all output data and plots are stored
    output_folder = os.path.join(os.path.split(data_files[0])[0], 'output')
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # The following shows a complete test beam analysis by calling the seperate function in correct order

    # Remove hot pixel, only needed for devices wih noisy pixel like Mimosa 26
    # A pool of workers to remove the noisy pixels in all files in parallel
    kwargs = [{
        'input_hits_file': data_files[i],
        'n_pixel': n_pixels[i],
        'pixel_size': pixel_size[i],
        'threshold': 5,
        'dut_name': dut_names[i]} for i in range(0, len(data_files))]
    pool = Pool()
    for kwarg in kwargs:
        pool.apply_async(hit_analysis.remove_noisy_pixels, kwds=kwarg)
    pool.close()
    pool.join()

    # Cluster hits off all DUTs
    # A pool of workers to cluster hits in all files in parallel
    kwargs = [{
        'input_hits_file': data_files[i][:-3] + '_noisy_pixels.h5',
        'max_x_distance': 3,
        'max_y_distance': 3,
        'max_time_distance': 2,
        'max_cluster_hits': 1000000,
        'dut_name': dut_names[i]} for i in range(0, len(data_files))]
    pool = Pool()
    for kwarg in kwargs:
        pool.apply_async(hit_analysis.cluster_hits, kwds=kwarg)
    pool.close()
    pool.join()

    # Correlate the row / column of each DUT
    dut_alignment.correlate_cluster(input_cluster_files=[data_file[:-3] + '_noisy_pixels_cluster.h5' for data_file in data_files],
                                    output_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
                                    n_pixels=n_pixels,
                                    pixel_size=pixel_size,
                                    dut_names=dut_names)

    # Create prealignment relative to the first DUT from the correlation data
    dut_alignment.prealignment(input_correlation_file=os.path.join(output_folder, 'Correlation.h5'),
                               output_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               z_positions=z_positions,
                               pixel_size=pixel_size,
                               dut_names=dut_names,
                               fit_background=True,  # This data has several tracks per event and noisy pixel, thus fit existing background
                               non_interactive=True)  # Tries to find cuts automatically; deactivate to do this manualy

    # Merge the cluster tables to one merged table aligned at the event number
    dut_alignment.merge_cluster_data(input_cluster_files=[data_file[:-3] + '_noisy_pixels_cluster.h5' for data_file in data_files],
                                     output_merged_file=os.path.join(output_folder, 'Merged.h5'),
                                     pixel_size=pixel_size)

    # Apply the prealignment to the merged cluster table to create tracklets
    dut_alignment.apply_alignment(input_hit_file=os.path.join(output_folder, 'Merged.h5'),
                                  input_alignment=os.path.join(output_folder, 'Alignment.h5'),
                                  output_hit_aligned_file=os.path.join(output_folder, 'Tracklets_prealigned.h5'),
                                  force_prealignment=True)

    # Find tracks from the prealigned tracklets and stores the with quality indicator into track candidates table
    track_analysis.find_tracks(input_tracklets_file=os.path.join(output_folder, 'Tracklets_prealigned.h5'),
                               input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               output_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'))

    # The following two steps are for demonstration only. They show track fitting and residual calculation on
    # prealigned hits. Usually you are not interessted in this and will use the aligned hits directly.

    # Step 1.: Fit the track candidates and create new track table (using the prealignment!)
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_prealigned.h5'),
                              exclude_dut_hit=True,  # To get unconstrained residuals do not use DUT hit for track fit
                              force_prealignment=True,  # This is just for demonstration purpose, you usually fully aligned hits
                              selection_track_quality=0)  # We will cut on chi2

    # Step 2.:  Calculate the residuals to check the alignment (using the prealignment!)
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_prealigned.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_prealigned.h5'),
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size,
                                        max_chi2=2000,
                                        force_prealignment=True)  # This is just for demonstration purpose, you usually use fully aligned hits

    # Do an alignment step with the track candidates, corrects rotations and is therefore much more precise than simple prealignment
    dut_alignment.alignment(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates_prealignment.h5'),
                            input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                            n_pixels=n_pixels,
                            pixel_size=pixel_size)

    # Apply the alignment to the merged cluster table to create tracklets
    dut_alignment.apply_alignment(input_hit_file=os.path.join(output_folder, 'Merged.h5'),
                                  input_alignment=os.path.join(output_folder, 'Alignment.h5'),
                                  output_hit_aligned_file=os.path.join(output_folder, 'Tracklets.h5'))

    # Find tracks from the tracklets and stores the with quality indicator into track candidates table
    track_analysis.find_tracks(input_tracklets_file=os.path.join(output_folder, 'Tracklets.h5'),
                               input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                               output_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'))

    # Example 1: use all DUTs in fit and cut on chi2
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_all.h5'),
                              exclude_dut_hit=True,  # To get unconstrained residuals do not use DUT hit for track fit
                              selection_track_quality=0)    # We do not cut on track quality but on chi2 later

    # Create unconstrained residuals
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_all.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_all.h5'),
                                        max_chi2=500,  # The chi2 cut has a large influence on the residuals and number of tracks, since the resolution is dominated by multiple scattering
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size)

    # Example 2: Use only 2 DUTs next to the fit DUT and cut on track quality. Thus the track fit is just a track interpolation with chi2 = 0.
    # This is better here due to heavily scatterd tracks, where a straight line assumption for all DUTs is wrong.
    # This leads to symmetric residuals in x and y for all DUTs between 2 fits DUTs: 1, 2, 3, 4
    track_analysis.fit_tracks(input_track_candidates_file=os.path.join(output_folder, 'TrackCandidates.h5'),
                              input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                              output_tracks_file=os.path.join(output_folder, 'Tracks_some.h5'),
                              selection_hit_duts=[[1, 2],  # Only select DUTs next to the DUT to fit
                                                  [0, 2],
                                                  [1, 3],
                                                  [2, 4],
                                                  [3, 5],
                                                  [3, 4]],
                              selection_track_quality=1)    # We cut on track quality

    # Create unconstrained residuals
    result_analysis.calculate_residuals(input_tracks_file=os.path.join(output_folder, 'Tracks_some.h5'),
                                        input_alignment_file=os.path.join(output_folder, 'Alignment.h5'),
                                        output_residuals_file=os.path.join(output_folder, 'Residuals_some.h5'),
                                        n_pixels=n_pixels,
                                        pixel_size=pixel_size)
<|endoftext|>


<commit_before>__version__ = '0.0.24'
<commit_msg>Release 0.0.25<commit_after>__version__ = '0.0.25'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Created on Sept 26, 2013

@author: Rafael Nunes
'''
import logging as log
from webapp2 import redirect_to, redirect
from google.appengine.api import images
from google.appengine.ext.webapp import blobstore_handlers

from core.base import BaseHandler, user_required
from core.models import Partido, Candidato

from core import settings
import urllib2

class CandidatosHandler(BaseHandler):
	def filter_candidatos(self):
		estado = self.request.get('estado')
		cargo = self.request.get('cargo')
		reeleicao = self.request.get('reeleicao')
		media_sessoes = self.request.get('media_sessoes')
		media_comissoes = self.request.get('media_comissoes')
		instrucao = self.request.get('instrucao')

		json_response = []
		candidatos_db = Candidato.query().fetch()

		grau = {"ENSINO FUNDAMENTAL INCOMPLETO":1,
		"ENSINO FUNDAMENTAL COMPLETO": 2,
		"ENSINO MÉDIO INCOMPLETO": 3,
		"ENSINO MÉDIO COMPLETO": 4,
		"SUPERIOR INCOMPLETO": 5,
		"SUPERIOR COMPLETO": 6
		}

		candidatos = Candidato.query(Candidato.estado == estado,
				# Candidato.cargo == cargo,
				# Candidato.reeleicao == reeleicao,
				# Candidato.media_sessoes <= media_sessoes,
				# Candidato.media_comissoes <= media_comissoes,
				# grau[Candidato.instrucao] >= instrucao
				).fetch()

		candidatos = [candidato.to_dict() for candidato in candidatos]

		return self.render('show_candidatos.html', candidatos=candidatos)<commit_msg>filter<commit_after># -*- coding: utf-8 -*-
'''
Created on Sept 26, 2013

@author: Rafael Nunes
'''
import logging as log
from webapp2 import redirect_to, redirect
from google.appengine.api import images
from google.appengine.ext.webapp import blobstore_handlers

from core.base import BaseHandler, user_required
from core.models import Partido, Candidato

from core import settings
import urllib2

class CandidatosHandler(BaseHandler):
	def filter_candidatos(self):
		estado = self.request.get('estado')
		cargo = self.request.get('cargo')
		reeleicao = self.request.get('reeleicao')
		faltas_sessoes = self.request.get('faltas_sessoes')
		faltas_comissoes = self.request.get('faltas_comissoes')
		instrucao = self.request.get('instrucao')

		json_response = []
		candidatos_db = Candidato.query().fetch()

		grau = {"ENSINO FUNDAMENTAL INCOMPLETO":1,
		"ENSINO FUNDAMENTAL COMPLETO": 2,
		"ENSINO MÉDIO INCOMPLETO": 3,
		"ENSINO MÉDIO COMPLETO": 4,
		"SUPERIOR INCOMPLETO": 5,
		"SUPERIOR COMPLETO": 6
		}

		candidatos = Candidato.query(Candidato.estado == estado,
				# ndb.AND(cargo!="",Candidato.cargo == cargo),
				# ndb.AND(reeleicao!="",Candidato.reeleicao == reeleicao),
				# ndb.AND(faltas_sessoes!="",Candidato.faltas_sessoes <= faltas_sessoes),
				# ndb.AND(faltas_comissoes!="",Candidato.faltas_comissoes <= faltas_comissoes),
				# ndb.AND(instrucao!="", grau[Candidato.instrucao] >= instrucao)
				).fetch()

		candidatos = [candidato.to_dict() for candidato in candidatos]

		return self.render('show_candidatos.html', candidatos=candidatos)<|endoftext|>


<commit_before>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import shutil
import subprocess
import sys
import tempfile
import unittest
import benchexec.util as util
sys.dont_write_bytecode = True # prevent creation of .pyc files

here = os.path.dirname(__file__)
base_dir = os.path.join(here, '..', '..', '..')
bin_dir = os.path.join(base_dir, 'bin')
tablegenerator = os.path.join(bin_dir, 'table-generator')

class TableGeneratorIntegrationTests(unittest.TestCase):

    # Tests compare the generated CSV files and ignore the HTML files
    # because we assume the HTML files change more often on purpose.

    @classmethod
    def setUpClass(cls):
        cls.longMessage = True
        cls.maxDiff = None

    def setUp(self):
        self.tmp = tempfile.mkdtemp(prefix="BenchExec.tablegenerator.integration_test")

    def tearDown(self):
        shutil.rmtree(self.tmp)

    def run_cmd(self, *args):
        output = subprocess.check_output(args=args).decode()
        print(output)
        return output

    def generate_tables_and_compare_csv(self, args, table_prefix, result_prefix=None, diff_prefix=None, expected_counts=None):
        output = self.run_cmd(*[tablegenerator] + list(args) + ['--outputpath', self.tmp])
        generated_files = set(map(lambda x : os.path.join(self.tmp, x), os.listdir(self.tmp)))

        csv_file = os.path.join(self.tmp, table_prefix + '.csv')
        html_file = os.path.join(self.tmp, table_prefix + '.html')
        expected_files = {csv_file, html_file}
        if diff_prefix:
            csv_diff_file = os.path.join(self.tmp, diff_prefix + '.csv')
            html_diff_file = os.path.join(self.tmp, diff_prefix + '.html')
            expected_files |= {csv_diff_file, html_diff_file}

        self.assertSetEqual(generated_files, expected_files, 'Set of generated files differs from set of expected files')

        generated = util.read_file(csv_file)
        expected = util.read_file(here, 'expected', (result_prefix or table_prefix) + '.csv')
        self.assertMultiLineEqual(generated, expected)

        if diff_prefix:
            generated_diff = util.read_file(csv_diff_file)
            expected_diff = util.read_file(here, 'expected', diff_prefix + '.csv')
            self.assertMultiLineEqual(generated_diff, expected_diff)

        if expected_counts:
            # output of table-generator should end with statistics about regressions
            counts = output[output.find('REGRESSIONS'):].strip()
            self.assertMultiLineEqual(expected_counts, counts)
        else:
            self.assertNotIn('REGRESSIONS', output)
            self.assertNotIn('STATS', output)

    def test_simple_table(self):
        self.generate_tables_and_compare_csv(
            [os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            )

    def test_simple_table_correct_only(self):
        self.generate_tables_and_compare_csv(
            ['--correct-only', os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            'test.2015-03-03_1613.results.predicateAnalysis.correct-only',
            )

    def test_simple_table_all_columns(self):
        self.generate_tables_and_compare_csv(
            ['--all-columns', os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            'test.2015-03-03_1613.results.predicateAnalysis.all-columns',
            )

    def test_simple_table_xml(self):
        self.generate_tables_and_compare_csv(
            ['-x', os.path.join(here, 'simple-table.xml')],
            'simple-table.table',
            'test.2015-03-03_1613.results.predicateAnalysis',
            )

    def test_simple_table_xml_with_columns(self):
        self.generate_tables_and_compare_csv(
            ['-x', os.path.join(here, 'simple-table-with-columns.xml')],
            'simple-table-with-columns.table',
            )

    def test_multi_table(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'predicateAnalysis',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1815.results.predicateAnalysis.xml'),
            ],
            table_prefix='predicateAnalysis.table',
            )

    def test_multi_table_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'predicateAnalysis-reverse',
             os.path.join(here, 'results', 'test.2015-03-03_1815.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='predicateAnalysis-reverse.table',
            )

    def test_multi_table_no_diff(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613', '--no-diff',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            )

    def test_multi_table_differing_files(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            diff_prefix='test.2015-03-03_1613.diff',
            )

    def test_multi_table_differing_files_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-reverse',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-reverse.table',
            diff_prefix='test.2015-03-03_1613-reverse.diff',
            )

    def test_multi_table_differing_files_correct_only(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-correct-only', '--correct-only',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-correct-only.table',
            diff_prefix='test.2015-03-03_1613-correct-only.diff',
            )

    def test_dump_count_single_table(self):
        self.generate_tables_and_compare_csv(
            ['--dump',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.results.predicateAnalysis',
            expected_counts='REGRESSIONS 0\nSTATS\n3 1 0', # 3 correct, 1 incorrect, 0 unknown (1 without property is ignored)
            )

    def test_dump_count_single_table2(self):
        self.generate_tables_and_compare_csv(
            ['--dump',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.results.valueAnalysis',
            expected_counts='REGRESSIONS 0\nSTATS\n2 0 1', # 2 correct, 0 incorrect, 1 unknown
            )

    def test_dump_count_multi_table(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613', '--dump',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            diff_prefix='test.2015-03-03_1613.diff',
            expected_counts='REGRESSIONS 2\nSTATS\n3 1 0\n2 0 2',
            )

    def test_dump_count_multi_table_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-reverse', '--dump',
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.valueAnalysis.xml'),
             os.path.join(here, 'results', 'test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-reverse.table',
            diff_prefix='test.2015-03-03_1613-reverse.diff',
            expected_counts='REGRESSIONS 1\nSTATS\n2 0 2\n3 1 0',
            )<commit_msg>Refactor tests, add output in case of non-zero exit code.<commit_after>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import shutil
import subprocess
import sys
import tempfile
import unittest
import benchexec.util as util
from subprocess import CalledProcessError
sys.dont_write_bytecode = True # prevent creation of .pyc files

here = os.path.dirname(__file__)
base_dir = os.path.join(here, '..', '..', '..')
bin_dir = os.path.join(base_dir, 'bin')
tablegenerator = os.path.join(bin_dir, 'table-generator')

def result_file(name):
    return os.path.join(here, 'results', name)

class TableGeneratorIntegrationTests(unittest.TestCase):

    # Tests compare the generated CSV files and ignore the HTML files
    # because we assume the HTML files change more often on purpose.

    @classmethod
    def setUpClass(cls):
        cls.longMessage = True
        cls.maxDiff = None

    def setUp(self):
        self.tmp = tempfile.mkdtemp(prefix="BenchExec.tablegenerator.integration_test")

    def tearDown(self):
        shutil.rmtree(self.tmp)

    def run_cmd(self, *args):
        try:
            output = subprocess.check_output(args=args, stderr=subprocess.STDOUT).decode()
        except CalledProcessError as e:
            print(e.output.decode())
            raise e
        print(output)
        return output

    def generate_tables_and_compare_csv(self, args, table_prefix, result_prefix=None, diff_prefix=None, expected_counts=None):
        output = self.run_cmd(*[tablegenerator] + list(args) + ['--outputpath', self.tmp])
        generated_files = set(map(lambda x : os.path.join(self.tmp, x), os.listdir(self.tmp)))

        csv_file = os.path.join(self.tmp, table_prefix + '.csv')
        html_file = os.path.join(self.tmp, table_prefix + '.html')
        expected_files = {csv_file, html_file}
        if diff_prefix:
            csv_diff_file = os.path.join(self.tmp, diff_prefix + '.csv')
            html_diff_file = os.path.join(self.tmp, diff_prefix + '.html')
            expected_files |= {csv_diff_file, html_diff_file}

        self.assertSetEqual(generated_files, expected_files, 'Set of generated files differs from set of expected files')

        generated = util.read_file(csv_file)
        expected = util.read_file(here, 'expected', (result_prefix or table_prefix) + '.csv')
        self.assertMultiLineEqual(generated, expected)

        if diff_prefix:
            generated_diff = util.read_file(csv_diff_file)
            expected_diff = util.read_file(here, 'expected', diff_prefix + '.csv')
            self.assertMultiLineEqual(generated_diff, expected_diff)

        if expected_counts:
            # output of table-generator should end with statistics about regressions
            counts = output[output.find('REGRESSIONS'):].strip()
            self.assertMultiLineEqual(expected_counts, counts)
        else:
            self.assertNotIn('REGRESSIONS', output)
            self.assertNotIn('STATS', output)

    def test_simple_table(self):
        self.generate_tables_and_compare_csv(
            [result_file('test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            )

    def test_simple_table_correct_only(self):
        self.generate_tables_and_compare_csv(
            ['--correct-only', result_file('test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            'test.2015-03-03_1613.results.predicateAnalysis.correct-only',
            )

    def test_simple_table_all_columns(self):
        self.generate_tables_and_compare_csv(
            ['--all-columns', result_file('test.2015-03-03_1613.results.predicateAnalysis.xml')],
            'test.2015-03-03_1613.results.predicateAnalysis',
            'test.2015-03-03_1613.results.predicateAnalysis.all-columns',
            )

    def test_simple_table_xml(self):
        self.generate_tables_and_compare_csv(
            ['-x', os.path.join(here, 'simple-table.xml')],
            'simple-table.table',
            'test.2015-03-03_1613.results.predicateAnalysis',
            )

    def test_simple_table_xml_with_columns(self):
        self.generate_tables_and_compare_csv(
            ['-x', os.path.join(here, 'simple-table-with-columns.xml')],
            'simple-table-with-columns.table',
            )

    def test_multi_table(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'predicateAnalysis',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1815.results.predicateAnalysis.xml'),
            ],
            table_prefix='predicateAnalysis.table',
            )

    def test_multi_table_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'predicateAnalysis-reverse',
             result_file('test.2015-03-03_1815.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='predicateAnalysis-reverse.table',
            )

    def test_multi_table_no_diff(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613', '--no-diff',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            )

    def test_multi_table_differing_files(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            diff_prefix='test.2015-03-03_1613.diff',
            )

    def test_multi_table_differing_files_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-reverse',
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-reverse.table',
            diff_prefix='test.2015-03-03_1613-reverse.diff',
            )

    def test_multi_table_differing_files_correct_only(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-correct-only', '--correct-only',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-correct-only.table',
            diff_prefix='test.2015-03-03_1613-correct-only.diff',
            )

    def test_dump_count_single_table(self):
        self.generate_tables_and_compare_csv(
            ['--dump',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.results.predicateAnalysis',
            expected_counts='REGRESSIONS 0\nSTATS\n3 1 0', # 3 correct, 1 incorrect, 0 unknown (1 without property is ignored)
            )

    def test_dump_count_single_table2(self):
        self.generate_tables_and_compare_csv(
            ['--dump',
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.results.valueAnalysis',
            expected_counts='REGRESSIONS 0\nSTATS\n2 0 1', # 2 correct, 0 incorrect, 1 unknown
            )

    def test_dump_count_multi_table(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613', '--dump',
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613.table',
            diff_prefix='test.2015-03-03_1613.diff',
            expected_counts='REGRESSIONS 2\nSTATS\n3 1 0\n2 0 2',
            )

    def test_dump_count_multi_table_reverse(self):
        self.generate_tables_and_compare_csv(
            ['--name', 'test.2015-03-03_1613-reverse', '--dump',
             result_file('test.2015-03-03_1613.results.valueAnalysis.xml'),
             result_file('test.2015-03-03_1613.results.predicateAnalysis.xml'),
            ],
            table_prefix='test.2015-03-03_1613-reverse.table',
            diff_prefix='test.2015-03-03_1613-reverse.diff',
            expected_counts='REGRESSIONS 1\nSTATS\n2 0 2\n3 1 0',
            )<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test staking in neblio
"""

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
from test_framework.messages import *

# Create one-input, one-output, no-fee transaction:
class RawTransactionsTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 6
        self.extra_args = [[], [], [], [], [], []]

    def setup_network(self, split=False):
        super().setup_network()

    def progress_mock_time(self, by_how_many_seconds):
        assert self.curr_time is not None
        self.curr_time += by_how_many_seconds
        for n in self.nodes:
            n.setmocktime(self.curr_time)

    def reset_mock_time(self, value=None):
        for n in self.nodes:
            if value is None:
                self.curr_time = int(time.time())
                n.setmocktime(self.curr_time)
            else:
                assert type(value) == int
                self.curr_time = value
                n.setmocktime(self.curr_time)

    def gen_pos_block(self, node_number, max_retries=10, average_block_time=STAKE_TARGET_SPACING, block_time_spread=10):
        r = random.randrange(-block_time_spread, block_time_spread + 1)
        self.progress_mock_time(average_block_time - self.last_random_time_offset + r)
        self.last_random_time_offset = r
        balance = self.nodes[node_number].getbalance()
        if balance == 0:
            raise ValueError("Node has no balance to stake")
        for i in range(max_retries):
            hashes = self.nodes[node_number].generatepos(1)
            if len(hashes) > 0:
                return hashes[0]
            else:
                # we progress in time to provide some window for nSearchTime vs nLastCoinStakeSearchTime
                self.progress_mock_time(1)
        raise CalledProcessError("Failed to stake. Max tries limit reached.")

    def gen_pow_block(self, node_number, average_block_time, block_time_spread):
        hashes = self.nodes[node_number].generate(1)
        assert_equal(len(hashes), 1)
        r = random.randrange(-block_time_spread, block_time_spread + 1)
        self.progress_mock_time(average_block_time - self.last_random_time_offset + r)
        self.last_random_time_offset = r
        return hashes[0]

    def create_tx_with_output_amounts(self, available_outputs, addresses_vs_amounts, fee=Decimal('0.1')):
        total_output_amount = fee
        for addr in addresses_vs_amounts:
            total_output_amount += addresses_vs_amounts[addr]
        total_input = 0
        utxos_to_be_used = []
        for input in available_outputs:
            if total_input < total_output_amount:
                if input['confirmations'] > COINBASE_MATURITY:
                    utxos_to_be_used.append(input)
                    total_input += input['amount']
            else:
                break
        if total_input < total_output_amount:
            logger.info("Attempting to reach value: {}".format(total_output_amount))
            logger.info("Available outputs: {}".format(available_outputs))
            raise ValueError("Total input could not reach the required output. Find available outputs above.")
        tx_inputs = []
        for input in utxos_to_be_used:
            tx_inputs.append({"txid": input['txid'], "vout": input['vout']})
        tx_outputs = addresses_vs_amounts
        return self.nodes[0].createrawtransaction(tx_inputs, tx_outputs)

    def run_test(self):
        self.sync_all()
        self.reset_mock_time()
        self.last_random_time_offset = 0
        block_time_spread = 10
        average_block_time = 30
        for i in range(100):  # mine 100 blocks, we reduce the amount per call to avoid timing out
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        # find the output that has the genesis block reward
        listunspent = self.nodes[0].listunspent()

        genesis_utxo = None
        for utxo_data in listunspent:
            if utxo_data['amount'] == Decimal('124000000.00000000'):
                genesis_utxo = utxo_data
                break
        assert genesis_utxo is not None

        # Create outputs in nodes[1] to stake them
        inputs = [{"txid": genesis_utxo['txid'], "vout": genesis_utxo['vout']}]

        outputs = {}
        outputs_count = 220
        for i in range(outputs_count):
            outputs[self.nodes[1].getnewaddress()] = 500
        rawTx = self.nodes[0].createrawtransaction(inputs, outputs)
        signedRawTx = self.nodes[0].signrawtransaction(rawTx)
        self.nodes[0].sendrawtransaction(signedRawTx['hex'])

        # Create outputs in nodes[2] to stake them
        for i in range(100):  # mine 100 blocks
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        self.sync_all()

        # Here we create the outputs in nodes[2] that should be combined
        n2_addr = self.nodes[2].getnewaddress()
        n2_utxos_to_combine_in_stake = 10
        n2_amount_per_address = Decimal('110')
        # the condition for combination; utxos will be added until we reach 'STAKE_COMBINE_THRESHOLD' nebls
        # note: The outcome can be > STAKE_COMBINE_THRESHOLD
        assert (n2_utxos_to_combine_in_stake - 1) * n2_amount_per_address <= STAKE_COMBINE_THRESHOLD
        for i in range(n2_utxos_to_combine_in_stake):
            addresses_vs_amounts_node2 = {n2_addr: n2_amount_per_address}
            tx_for_n2 = self.create_tx_with_output_amounts(self.nodes[0].listunspent(), addresses_vs_amounts_node2)
            signed_tx_for_n2 = self.nodes[0].signrawtransaction(tx_for_n2)
            self.nodes[0].sendrawtransaction(signed_tx_for_n2['hex'])

        # Here we create the outputs in nodes[2] that should be combined, except for one output
        n3_addr = self.nodes[3].getnewaddress()
        n3_utxos_to_combine_in_stake = 10  # the amount we expect to be combined
        n3_amount_per_address = Decimal('120')
        # the condition for combination; utxos will be added until we reach 'STAKE_COMBINE_THRESHOLD' nebls
        # note: The outcome can be > STAKE_COMBINE_THRESHOLD
        assert (n3_utxos_to_combine_in_stake - 1) * n3_amount_per_address > STAKE_COMBINE_THRESHOLD
        for i in range(n3_utxos_to_combine_in_stake):
            addresses_vs_amounts_node3 = {n3_addr: n3_amount_per_address}
            tx_for_n3 = self.create_tx_with_output_amounts(self.nodes[0].listunspent(), addresses_vs_amounts_node3)
            signed_tx_for_n3 = self.nodes[0].signrawtransaction(tx_for_n3)
            self.nodes[0].sendrawtransaction(signed_tx_for_n3['hex'])

        for i in range(800):  # mine 800 blocks
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        self.sync_all()

        n1_balance_before = self.nodes[1].getbalance()
        n2_balance_before = self.nodes[2].getbalance()
        n3_balance_before = self.nodes[3].getbalance()

        # move to the future to make coins stakable (this is not needed because 10 minutes is too short)
        # self.progress_mock_time(60*10)

        LAST_POW_BLOCK = 1000

        votes = self.nodes[0].listvotes()
        assert_equal(len(votes), 0)

        block_count_to_stake = 30
        # We can't stake more than the outputs we have
        assert block_count_to_stake <= outputs_count
        for i in range(block_count_to_stake):
            hash = self.gen_pos_block(1)

        self.sync_all()

        # check that the total number of blocks is the PoW mined + PoS mined
        for n in self.nodes:
            assert_equal(n.getblockcount(), LAST_POW_BLOCK + block_count_to_stake)

        # check that votes are empty
        for i in range(block_count_to_stake):
            blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + i + 1)
            assert_equal(blk_data['votevalue'], None)

        self.nodes[0].castvote(LAST_POW_BLOCK + block_count_to_stake + 5,
                               LAST_POW_BLOCK + block_count_to_stake + 10,
                               999,
                               11)
        self.nodes[0].castvote(LAST_POW_BLOCK + 5,
                               LAST_POW_BLOCK + 10,
                               888,
                               15)
        self.nodes[0].castvote(LAST_POW_BLOCK + block_count_to_stake + 500,
                               LAST_POW_BLOCK + block_count_to_stake + 510,
                               777,
                               33)

        next_block_count_to_stake = 15
        for i in range(next_block_count_to_stake):
            hash = self.gen_pos_block(0)

        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 1)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 2)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 3)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 4)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 5)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 6)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 7)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 8)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 9)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 10)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 11)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 12)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 13)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 14)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 15)
        assert_equal(blk_data['votevalue'], None)

if __name__ == '__main__':
    RawTransactionsTest().main()
<commit_msg>Minor comment on test<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test staking in neblio
"""

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
from test_framework.messages import *

# Create one-input, one-output, no-fee transaction:
class RawTransactionsTest(BitcoinTestFramework):
    def set_test_params(self):
        self.setup_clean_chain = True
        self.num_nodes = 6
        self.extra_args = [[], [], [], [], [], []]

    def setup_network(self, split=False):
        super().setup_network()

    def progress_mock_time(self, by_how_many_seconds):
        assert self.curr_time is not None
        self.curr_time += by_how_many_seconds
        for n in self.nodes:
            n.setmocktime(self.curr_time)

    def reset_mock_time(self, value=None):
        for n in self.nodes:
            if value is None:
                self.curr_time = int(time.time())
                n.setmocktime(self.curr_time)
            else:
                assert type(value) == int
                self.curr_time = value
                n.setmocktime(self.curr_time)

    def gen_pos_block(self, node_number, max_retries=10, average_block_time=STAKE_TARGET_SPACING, block_time_spread=10):
        r = random.randrange(-block_time_spread, block_time_spread + 1)
        self.progress_mock_time(average_block_time - self.last_random_time_offset + r)
        self.last_random_time_offset = r
        balance = self.nodes[node_number].getbalance()
        if balance == 0:
            raise ValueError("Node has no balance to stake")
        for i in range(max_retries):
            hashes = self.nodes[node_number].generatepos(1)
            if len(hashes) > 0:
                return hashes[0]
            else:
                # we progress in time to provide some window for nSearchTime vs nLastCoinStakeSearchTime
                self.progress_mock_time(1)
        raise CalledProcessError("Failed to stake. Max tries limit reached.")

    def gen_pow_block(self, node_number, average_block_time, block_time_spread):
        hashes = self.nodes[node_number].generate(1)
        assert_equal(len(hashes), 1)
        r = random.randrange(-block_time_spread, block_time_spread + 1)
        self.progress_mock_time(average_block_time - self.last_random_time_offset + r)
        self.last_random_time_offset = r
        return hashes[0]

    def create_tx_with_output_amounts(self, available_outputs, addresses_vs_amounts, fee=Decimal('0.1')):
        total_output_amount = fee
        for addr in addresses_vs_amounts:
            total_output_amount += addresses_vs_amounts[addr]
        total_input = 0
        utxos_to_be_used = []
        for input in available_outputs:
            if total_input < total_output_amount:
                if input['confirmations'] > COINBASE_MATURITY:
                    utxos_to_be_used.append(input)
                    total_input += input['amount']
            else:
                break
        if total_input < total_output_amount:
            logger.info("Attempting to reach value: {}".format(total_output_amount))
            logger.info("Available outputs: {}".format(available_outputs))
            raise ValueError("Total input could not reach the required output. Find available outputs above.")
        tx_inputs = []
        for input in utxos_to_be_used:
            tx_inputs.append({"txid": input['txid'], "vout": input['vout']})
        tx_outputs = addresses_vs_amounts
        return self.nodes[0].createrawtransaction(tx_inputs, tx_outputs)

    def run_test(self):
        self.sync_all()
        self.reset_mock_time()
        self.last_random_time_offset = 0
        block_time_spread = 10
        average_block_time = 30
        for i in range(100):  # mine 100 blocks, we reduce the amount per call to avoid timing out
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        # find the output that has the genesis block reward
        listunspent = self.nodes[0].listunspent()

        genesis_utxo = None
        for utxo_data in listunspent:
            if utxo_data['amount'] == Decimal('124000000.00000000'):
                genesis_utxo = utxo_data
                break
        assert genesis_utxo is not None

        # Create outputs in nodes[1] to stake them
        inputs = [{"txid": genesis_utxo['txid'], "vout": genesis_utxo['vout']}]

        outputs = {}
        outputs_count = 220
        for i in range(outputs_count):
            outputs[self.nodes[1].getnewaddress()] = 500
        rawTx = self.nodes[0].createrawtransaction(inputs, outputs)
        signedRawTx = self.nodes[0].signrawtransaction(rawTx)
        self.nodes[0].sendrawtransaction(signedRawTx['hex'])

        # Create outputs in nodes[2] to stake them
        for i in range(100):  # mine 100 blocks
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        self.sync_all()

        # Here we create the outputs in nodes[2] that should be combined
        n2_addr = self.nodes[2].getnewaddress()
        n2_utxos_to_combine_in_stake = 10
        n2_amount_per_address = Decimal('110')
        # the condition for combination; utxos will be added until we reach 'STAKE_COMBINE_THRESHOLD' nebls
        # note: The outcome can be > STAKE_COMBINE_THRESHOLD
        assert (n2_utxos_to_combine_in_stake - 1) * n2_amount_per_address <= STAKE_COMBINE_THRESHOLD
        for i in range(n2_utxos_to_combine_in_stake):
            addresses_vs_amounts_node2 = {n2_addr: n2_amount_per_address}
            tx_for_n2 = self.create_tx_with_output_amounts(self.nodes[0].listunspent(), addresses_vs_amounts_node2)
            signed_tx_for_n2 = self.nodes[0].signrawtransaction(tx_for_n2)
            self.nodes[0].sendrawtransaction(signed_tx_for_n2['hex'])

        # Here we create the outputs in nodes[2] that should be combined, except for one output
        n3_addr = self.nodes[3].getnewaddress()
        n3_utxos_to_combine_in_stake = 10  # the amount we expect to be combined
        n3_amount_per_address = Decimal('120')
        # the condition for combination; utxos will be added until we reach 'STAKE_COMBINE_THRESHOLD' nebls
        # note: The outcome can be > STAKE_COMBINE_THRESHOLD
        assert (n3_utxos_to_combine_in_stake - 1) * n3_amount_per_address > STAKE_COMBINE_THRESHOLD
        for i in range(n3_utxos_to_combine_in_stake):
            addresses_vs_amounts_node3 = {n3_addr: n3_amount_per_address}
            tx_for_n3 = self.create_tx_with_output_amounts(self.nodes[0].listunspent(), addresses_vs_amounts_node3)
            signed_tx_for_n3 = self.nodes[0].signrawtransaction(tx_for_n3)
            self.nodes[0].sendrawtransaction(signed_tx_for_n3['hex'])

        for i in range(800):  # mine 800 blocks
            hash = self.gen_pow_block(0, average_block_time, block_time_spread)
        self.sync_all()

        n1_balance_before = self.nodes[1].getbalance()
        n2_balance_before = self.nodes[2].getbalance()
        n3_balance_before = self.nodes[3].getbalance()

        # move to the future to make coins stakable (this is not needed because 10 minutes is too short)
        # self.progress_mock_time(60*10)

        LAST_POW_BLOCK = 1000

        votes = self.nodes[0].listvotes()
        assert_equal(len(votes), 0)

        block_count_to_stake = 30
        # We can't stake more than the outputs we have
        assert block_count_to_stake <= outputs_count
        for i in range(block_count_to_stake):
            hash = self.gen_pos_block(1)

        self.sync_all()

        # check that the total number of blocks is the PoW mined + PoS mined
        for n in self.nodes:
            assert_equal(n.getblockcount(), LAST_POW_BLOCK + block_count_to_stake)

        # check that votes are empty
        for i in range(block_count_to_stake):
            blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + i + 1)
            assert_equal(blk_data['votevalue'], None)

        self.nodes[0].castvote(LAST_POW_BLOCK + block_count_to_stake + 5,
                               LAST_POW_BLOCK + block_count_to_stake + 10,
                               999,
                               11)
        self.nodes[0].castvote(LAST_POW_BLOCK + 5,
                               LAST_POW_BLOCK + 10,
                               888,
                               15)
        self.nodes[0].castvote(LAST_POW_BLOCK + block_count_to_stake + 500,
                               LAST_POW_BLOCK + block_count_to_stake + 510,
                               777,
                               33)

        next_block_count_to_stake = 15
        for i in range(next_block_count_to_stake):
            hash = self.gen_pos_block(0)

        # now we ensure that only the correct block range has the vote with the correct values
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 1)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 2)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 3)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 4)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 5)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 6)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 7)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 8)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 9)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 10)
        assert_equal(blk_data['votevalue'], {'VoteValue': 11, 'ProposalID': 999})
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 11)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 12)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 13)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 14)
        assert_equal(blk_data['votevalue'], None)
        blk_data = self.nodes[0].getblockbynumber(LAST_POW_BLOCK + block_count_to_stake + 15)
        assert_equal(blk_data['votevalue'], None)

if __name__ == '__main__':
    RawTransactionsTest().main()
<|endoftext|>


<commit_before>
from spectral_cube import SpectralCube
import astropy.units as u
import numpy as np
from astropy.table import Table
import matplotlib.pyplot as p
from scipy import ndimage as nd
from astropy.wcs.utils import proj_plane_pixel_scales
from astropy.utils.console import ProgressBar
from astropy.wcs.utils import proj_plane_pixel_area
from astropy.io import fits
from radio_beam import Beam
from astropy.visualization import AsinhStretch
from astropy.visualization.mpl_normalize import ImageNormalize
from astropy.visualization import hist
from reproject import reproject_interp
import seaborn as sb

from rotation_curves.vrot_fit import return_smooth_model
from paths import (fourteenB_HI_data_path, iram_co21_data_path,
                   paper1_figures_path)
from galaxy_params import gal
from plotting_styles import onecolumn_figure, default_figure, twocolumn_figure
from constants import moment0_name, hi_freq, moment1_name

try:
    from corner import hist2d
except ImportError:
    raise ImportError("You need to install the corner package!")

'''
Is bright CO better correlated w/ the HI velocity?

Compares Tpeak_CO vs |V_peak,CO - V_rot,HI|
'''

table_name = fourteenB_HI_data_path("diskfit_noasymm_noradial_nowarp_output/rad.out.csv")
tab = Table.read(table_name)

cube = SpectralCube.read(iram_co21_data_path("m33.co21_iram.fits"))
del cube._header[""]

# Galactic radii
radii = gal.radius(header=cube.header)

hi_model = return_smooth_model(tab, cube.header, gal) * u.m / u.s

# Now load in the HI centroids
moment1 = fits.open(fourteenB_HI_data_path(moment1_name))[0]

moment1_reproj = reproject_interp(moment1, cube.wcs.celestial,
                                  shape_out=cube.shape[1:])[0] * u.m / u.s

# Mask everything at the edges of the map.
r_max = 6. * u.kpc
hi_model[radii > r_max] = np.NaN * u.m / u.s
cube = cube.with_mask(radii < r_max)

# Use avg noise reported in Druard+14
co_avg_noise = 20.33 * u.mK

# Only keep peaks above 3 sigma.
min_snr = 5.
min_pts = 2

Tpeak = np.zeros(cube.shape[1:]) * u.K
vel_Tpeak = np.zeros(cube.shape[1:]) * u.m / u.s

# Compute the first moment array for the whole cube.
vel_centroid = cube.with_mask(cube > co_avg_noise * 3).moment1()
# We'll zero out any noise dominated regions below, so don't
# worry about the masking.

good_posns = np.where(cube.mask.include().sum(0) > 0)

for y, x in ProgressBar(zip(*good_posns)):
    specind = cube[:, y, x].argmax()
    Tpeak[y, x] = cube[:, y, x][specind]

    if Tpeak[y, x] < min_snr * co_avg_noise:
        Tpeak[y, x] = np.NaN * u.K
        vel_Tpeak[y, x] = np.NaN * u.m / u.s
        vel_centroid[y, x] = np.NaN * u.m / u.s
    else:
        vel_Tpeak[y, x] = cube.spectral_axis[specind]

Tpeak[Tpeak == 0.0 * u.K] = np.NaN * u.K
vel_Tpeak[vel_Tpeak == 0.0 * u.m / u.s] = np.NaN * u.m / u.s

vel_diff = np.abs(hi_model - vel_Tpeak).to(u.km / u.s)

# Clean-up the points by requiring they be part of a region larger than the
# beam
Tpeak_mask = np.isfinite(Tpeak)
pixscale = proj_plane_pixel_scales(cube.wcs)[0]
beam_kern = cube.beam.as_tophat_kernel(pixscale)

# Open and close
Tpeak_mask = nd.binary_opening(Tpeak_mask, beam_kern)
Tpeak_mask = nd.binary_closing(Tpeak_mask, beam_kern)

vel_diff_values = vel_diff.value[Tpeak_mask & np.isfinite(Tpeak)]
Tpeak_values = Tpeak.value[Tpeak_mask & np.isfinite(Tpeak)]

onecolumn_figure(font_scale=1.1)
hist2d(vel_diff_values, Tpeak_values, bins=16, data_kwargs={"alpha": 0.6},
       range=[(0.0, 30.0),
              (0.0, np.max(Tpeak_values))])
p.hlines(co_avg_noise.to(u.K).value * min_snr, 0.0, 30.0, color='r',
         linestyle='--')
p.ylabel(r"T$_\mathrm{peak, CO}$ (K)")
p.xlabel(r"|V$_\mathrm{peak, CO}$ - V$_\mathrm{rot, HI}$| (km/s)")
p.grid()
p.tight_layout()

p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset.pdf"))
p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset.png"))
p.close()

# Now do the difference between HI and CO centroids
# vel_diff = (hi_model - vel_Tpeak).to(u.km / u.s)
# vel_diff_values = vel_diff[Tpeak_mask & np.isfinite(Tpeak)]

vel_diff_centroids = \
    (moment1_reproj - vel_centroid.quantity).to(u.km / u.s)
vel_diff_centroid_values = \
    vel_diff_centroids.value[Tpeak_mask & np.isfinite(Tpeak)]

# vel_diff_hicentroid_copeak = \
#     (moment1_reproj - vel_Tpeak).to(u.km / u.s)
# vel_diff_hicentroid_copeak_values = \
#     vel_diff_hicentroid_copeak.value[Tpeak_mask & np.isfinite(Tpeak)]
# vel_diff_hicentroid_copeak_values = \
#     vel_diff_hicentroid_copeak_values[np.isfinite(vel_diff_hicentroid_copeak_values)]

# one = hist(vel_diff_values, bins='scott', color='r', normed=True,
#            label='V$_\mathrm{peak, CO}$ - V$_\mathrm{rot, HI}', histtype='step')
# two = hist(vel_diff_centroid_values, bins='scott', color='g', normed=True,
#            label='V$_\mathrm{cent, CO}$ - V$_\mathrm{cent, HI}', histtype='step')
# three = hist(vel_diff_hicentroid_copeak_values, bins='scott', color='b', normed=True,
#              label='V$_\mathrm{peak, CO}$ - V$_\mathrm{cent, HI}', histtype='step')
# p.legend(frameon=True)
# p.grid()

hist2d(np.abs(vel_diff_centroid_values), Tpeak_values, bins=12,
       data_kwargs={"alpha": 0.6},
       range=[(0.0, 30.0),
              (0.0, np.max(Tpeak_values))])
p.hlines(co_avg_noise.to(u.K).value * min_snr, 0.0, 120.0, color='r',
         linestyle='--')
p.ylabel(r"T$_\mathrm{peak, CO}$ (K)")
p.xlabel(r"|V$_\mathrm{cent, CO}$ - V$_\mathrm{cemt, HI}$| (km/s)")
p.grid()
p.tight_layout()

p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset_centroids.pdf"))
p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset_centroids.png"))
p.close()

# Where do the outliers occur
moment0 = fits.open(fourteenB_HI_data_path(moment0_name))[0]

moment0_reproj = reproject_interp(moment0, cube.wcs.celestial,
                                  shape_out=vel_diff.shape)[0]

beam = Beam.from_fits_header(moment0.header)

moment0_Kkm_s = beam.jtok(hi_freq).value * moment0_reproj / 1000.

pixscale = np.sqrt(proj_plane_pixel_area(cube.wcs.celestial))

twocolumn_figure(fig_ratio=0.95, font_scale=1.2)
sb.set_palette("colorblind")

ax = p.subplot(111, projection=cube.wcs.celestial)
im = ax.imshow(moment0_Kkm_s,
               origin='lower',
               interpolation='nearest',
               norm=ImageNormalize(vmin=-0.001,
                                   vmax=np.nanmax(moment0_Kkm_s),
                                   stretch=AsinhStretch()))
ax.set_ylabel("DEC (J2000)")
ax.set_xlabel("RA (J2000)")
ax.set_ylim([260, 900])
ax.set_xlim([130, 620])
# ax.add_patch(beam.ellipse_to_plot(int(0.05 * moment0.shape[0]),
#                                   int(0.05 * moment0.shape[1]), pixscale))

# Where are the velocity outliers spatially?
# Assign colours to points based on their percentile in the distribution

percs = np.percentile(vel_diff_values[vel_diff_values < 50],
                      [85, 95, 99, 99.5])
labels = ["85%", "95%", "99%", "99.5%"]
labels = ["> {0} km/s ({1})".format(round(perc, 1), label)
          for perc, label in zip(percs, labels)]

cols = ["D", "s", "o", "^"]

nonan_vel_diff = vel_diff.copy()
nonan_vel_diff[np.isnan(nonan_vel_diff)] = 0.0 * u.km / u.s
nonan_vel_diff *= Tpeak_mask * np.isfinite(Tpeak)

for perc, col, label in zip(percs, cols, labels):
    ypos, xpos = np.where(np.abs(nonan_vel_diff.value) > perc)
    p.plot(xpos, ypos, col, label=label)

p.legend(loc='lower left', frameon=True)

cbar = p.colorbar(im)
cbar.set_label(r"Integrated Intensity (K km s$^{-1}$)")

p.savefig(paper1_figures_path("co21_HI_velocity_offset_outliers.pdf"))
p.savefig(paper1_figures_path("co21_HI_velocity_offset_outliers.png"))
p.close()

default_figure()
<commit_msg>Add in peak HI vs peak CO plot<commit_after>
from spectral_cube import SpectralCube
import astropy.units as u
import numpy as np
from astropy.table import Table
import matplotlib.pyplot as p
from scipy import ndimage as nd
from astropy.wcs.utils import proj_plane_pixel_scales
from astropy.utils.console import ProgressBar
from astropy.wcs.utils import proj_plane_pixel_area
from astropy.io import fits
from radio_beam import Beam
from astropy.visualization import AsinhStretch
from astropy.visualization.mpl_normalize import ImageNormalize
from astropy.visualization import hist
from reproject import reproject_interp
import seaborn as sb

from rotation_curves.vrot_fit import return_smooth_model
from paths import (fourteenB_HI_data_path, iram_co21_data_path,
                   paper1_figures_path)
from galaxy_params import gal
from plotting_styles import onecolumn_figure, default_figure, twocolumn_figure
from constants import moment0_name, hi_freq, moment1_name, peakvels_name

try:
    from corner import hist2d
except ImportError:
    raise ImportError("You need to install the corner package!")

'''
Is bright CO better correlated w/ the HI velocity?

Compares Tpeak_CO vs |V_peak,CO - V_rot,HI|
'''

table_name = fourteenB_HI_data_path("diskfit_noasymm_noradial_nowarp_output/rad.out.csv")
tab = Table.read(table_name)

cube = SpectralCube.read(iram_co21_data_path("m33.co21_iram.fits"))
del cube._header[""]

# Galactic radii
radii = gal.radius(header=cube.header)

hi_model = return_smooth_model(tab, cube.header, gal) * u.m / u.s

# Now load in the HI centroids
moment1 = fits.open(fourteenB_HI_data_path(moment1_name))[0]

# And the peak HI velocities
peakvel = fits.open(fourteenB_HI_data_path(peakvels_name))[0]

# Reproject to the CO
moment1_reproj = reproject_interp(moment1, cube.wcs.celestial,
                                  shape_out=cube.shape[1:])[0] * u.m / u.s
peakvel_reproj = reproject_interp(peakvel, cube.wcs.celestial,
                                  shape_out=cube.shape[1:])[0] * u.m / u.s

# Mask everything at the edges of the map.
r_max = 6. * u.kpc
hi_model[radii > r_max] = np.NaN * u.m / u.s
cube = cube.with_mask(radii < r_max)

# Use avg noise reported in Druard+14
co_avg_noise = 20.33 * u.mK

# Only keep peaks above 3 sigma.
min_snr = 5.
min_pts = 2

Tpeak = np.zeros(cube.shape[1:]) * u.K
vel_Tpeak = np.zeros(cube.shape[1:]) * u.m / u.s

# Compute the first moment array for the whole cube.
vel_centroid = cube.with_mask(cube > co_avg_noise * 3).moment1()
# We'll zero out any noise dominated regions below, so don't
# worry about the masking.

good_posns = np.where(cube.mask.include().sum(0) > 0)

for y, x in ProgressBar(zip(*good_posns)):
    specind = cube[:, y, x].argmax()
    Tpeak[y, x] = cube[:, y, x][specind]

    if Tpeak[y, x] < min_snr * co_avg_noise:
        Tpeak[y, x] = np.NaN * u.K
        vel_Tpeak[y, x] = np.NaN * u.m / u.s
        vel_centroid[y, x] = np.NaN * u.m / u.s
    else:
        vel_Tpeak[y, x] = cube.spectral_axis[specind]

Tpeak[Tpeak == 0.0 * u.K] = np.NaN * u.K
vel_Tpeak[vel_Tpeak == 0.0 * u.m / u.s] = np.NaN * u.m / u.s

vel_diff = np.abs(hi_model - vel_Tpeak).to(u.km / u.s)

# Clean-up the points by requiring they be part of a region larger than the
# beam
Tpeak_mask = np.isfinite(Tpeak)
pixscale = proj_plane_pixel_scales(cube.wcs)[0]
beam_kern = cube.beam.as_tophat_kernel(pixscale)

# Open and close
Tpeak_mask = nd.binary_opening(Tpeak_mask, beam_kern)
Tpeak_mask = nd.binary_closing(Tpeak_mask, beam_kern)

vel_diff_values = vel_diff.value[Tpeak_mask & np.isfinite(Tpeak)]
Tpeak_values = Tpeak.value[Tpeak_mask & np.isfinite(Tpeak)]

onecolumn_figure(font_scale=1.1)
hist2d(vel_diff_values, Tpeak_values, bins=16, data_kwargs={"alpha": 0.6},
       range=[(0.0, 30.0),
              (0.0, np.max(Tpeak_values))])
p.hlines(co_avg_noise.to(u.K).value * min_snr, 0.0, 30.0, color='r',
         linestyle='--')
p.ylabel(r"T$_\mathrm{peak, CO}$ (K)")
p.xlabel(r"|V$_\mathrm{peak, CO}$ - V$_\mathrm{rot, HI}$| (km/s)")
p.grid()
p.tight_layout()

p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset.pdf"))
p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset.png"))
p.close()

# HI vs CO peak velocities
vel_diff_peaks = \
    (peakvel_reproj - vel_Tpeak).to(u.km / u.s)
vel_diff_peaks_values = \
    vel_diff_peaks.value[Tpeak_mask & np.isfinite(Tpeak)]

hist2d(vel_diff_peaks_values, Tpeak_values, bins=16, data_kwargs={"alpha": 0.6},
       range=[(0.0, 30.0),
              (0.0, np.max(Tpeak_values))])
p.hlines(co_avg_noise.to(u.K).value * min_snr, 0.0, 30.0, color='r',
         linestyle='--')
p.ylabel(r"T$_\mathrm{peak, CO}$ (K)")
p.xlabel(r"|V$_\mathrm{peak, CO}$ - V$_\mathrm{peak, HI}$| (km/s)")
p.grid()
p.tight_layout()

p.savefig(paper1_figures_path("co21_Tpeak_peakvelocity_offset.pdf"))
p.savefig(paper1_figures_path("co21_Tpeak_peakvelocity_offset.png"))
p.close()

# Now do the difference between HI and CO centroids
# vel_diff = (hi_model - vel_Tpeak).to(u.km / u.s)
# vel_diff_values = vel_diff[Tpeak_mask & np.isfinite(Tpeak)]

vel_diff_centroids = \
    (moment1_reproj - vel_centroid.quantity).to(u.km / u.s)
vel_diff_centroid_values = \
    vel_diff_centroids.value[Tpeak_mask & np.isfinite(Tpeak)]

# vel_diff_hicentroid_copeak = \
#     (moment1_reproj - vel_Tpeak).to(u.km / u.s)
# vel_diff_hicentroid_copeak_values = \
#     vel_diff_hicentroid_copeak.value[Tpeak_mask & np.isfinite(Tpeak)]
# vel_diff_hicentroid_copeak_values = \
#     vel_diff_hicentroid_copeak_values[np.isfinite(vel_diff_hicentroid_copeak_values)]

# one = hist(vel_diff_values, bins='scott', color='r', normed=True,
#            label='V$_\mathrm{peak, CO}$ - V$_\mathrm{rot, HI}', histtype='step')
# two = hist(vel_diff_centroid_values, bins='scott', color='g', normed=True,
#            label='V$_\mathrm{cent, CO}$ - V$_\mathrm{cent, HI}', histtype='step')
# three = hist(vel_diff_hicentroid_copeak_values, bins='scott', color='b', normed=True,
#              label='V$_\mathrm{peak, CO}$ - V$_\mathrm{cent, HI}', histtype='step')
# p.legend(frameon=True)
# p.grid()

hist2d(np.abs(vel_diff_centroid_values), Tpeak_values, bins=12,
       data_kwargs={"alpha": 0.6},
       range=[(0.0, 30.0),
              (0.0, np.max(Tpeak_values))])
p.hlines(co_avg_noise.to(u.K).value * min_snr, 0.0, 120.0, color='r',
         linestyle='--')
p.ylabel(r"T$_\mathrm{peak, CO}$ (K)")
p.xlabel(r"|V$_\mathrm{cent, CO}$ - V$_\mathrm{cent, HI}$| (km/s)")
p.grid()
p.tight_layout()

p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset_centroids.pdf"))
p.savefig(paper1_figures_path("co21_Tpeak_velocity_offset_centroids.png"))
p.close()

# Where do the outliers occur
moment0 = fits.open(fourteenB_HI_data_path(moment0_name))[0]

moment0_reproj = reproject_interp(moment0, cube.wcs.celestial,
                                  shape_out=vel_diff.shape)[0]

beam = Beam.from_fits_header(moment0.header)

moment0_Kkm_s = beam.jtok(hi_freq).value * moment0_reproj / 1000.

pixscale = np.sqrt(proj_plane_pixel_area(cube.wcs.celestial))

twocolumn_figure(fig_ratio=0.95, font_scale=1.2)
sb.set_palette("colorblind")

ax = p.subplot(111, projection=cube.wcs.celestial)
im = ax.imshow(moment0_Kkm_s,
               origin='lower',
               interpolation='nearest',
               norm=ImageNormalize(vmin=-0.001,
                                   vmax=np.nanmax(moment0_Kkm_s),
                                   stretch=AsinhStretch()))
ax.set_ylabel("DEC (J2000)")
ax.set_xlabel("RA (J2000)")
ax.set_ylim([260, 900])
ax.set_xlim([130, 620])
# ax.add_patch(beam.ellipse_to_plot(int(0.05 * moment0.shape[0]),
#                                   int(0.05 * moment0.shape[1]), pixscale))

# Where are the velocity outliers spatially?
# Assign colours to points based on their percentile in the distribution

percs = np.percentile(vel_diff_values[vel_diff_values < 50],
                      [85, 95, 99, 99.5])
labels = ["85%", "95%", "99%", "99.5%"]
labels = ["> {0} km/s ({1})".format(round(perc, 1), label)
          for perc, label in zip(percs, labels)]

cols = ["D", "s", "o", "^"]

nonan_vel_diff = vel_diff.copy()
nonan_vel_diff[np.isnan(nonan_vel_diff)] = 0.0 * u.km / u.s
nonan_vel_diff *= Tpeak_mask * np.isfinite(Tpeak)

for perc, col, label in zip(percs, cols, labels):
    ypos, xpos = np.where(np.abs(nonan_vel_diff.value) > perc)
    p.plot(xpos, ypos, col, label=label)

p.legend(loc='lower left', frameon=True)

cbar = p.colorbar(im)
cbar.set_label(r"Integrated Intensity (K km s$^{-1}$)")

p.savefig(paper1_figures_path("co21_HI_velocity_offset_outliers.pdf"))
p.savefig(paper1_figures_path("co21_HI_velocity_offset_outliers.png"))
p.close()

default_figure()
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Tool import Tool
from UM.Event import Event, MouseEvent, KeyEvent
from UM.Application import Application
from UM.Message import Message
from UM.Scene.ToolHandle import ToolHandle
from UM.Scene.Selection import Selection

from UM.Math.Plane import Plane
from UM.Math.Vector import Vector
from UM.Math.Quaternion import Quaternion
from UM.Math.Float import Float

from UM.Operations.RotateOperation import RotateOperation
from UM.Operations.TranslateOperation import TranslateOperation
from UM.Operations.GroupedOperation import GroupedOperation
from UM.Operations.SetTransformOperation import SetTransformOperation
from UM.Operations.LayFlatOperation import LayFlatOperation

from . import RotateToolHandle

import math
import time

class RotateTool(Tool):
    def __init__(self):
        super().__init__()
        self._handle = RotateToolHandle.RotateToolHandle()

        self._snap_rotation = True
        self._snap_angle = math.radians(15)

        self._angle = None

        self._angle_update_time = None

        self.setExposedProperties("Rotation", "RotationSnap", "RotationSnapAngle")

    def event(self, event):
        super().event(event)

        if event.type == Event.KeyPressEvent and event.key == KeyEvent.ShiftKey:
            self._snap_rotation = (not self._snap_rotation)
            self.propertyChanged.emit()

        if event.type == Event.KeyReleaseEvent and event.key == KeyEvent.ShiftKey:
            self._snap_rotation = (not self._snap_rotation)
            self.propertyChanged.emit()

        if event.type == Event.MousePressEvent:
            if MouseEvent.LeftButton not in event.buttons:
                return False

            id = self._selection_pass.getIdAtPosition(event.x, event.y)
            if not id:
                return

            if ToolHandle.isAxis(id):
                self.setLockedAxis(id)
                handle_position = self._handle.getWorldPosition()

                if id == ToolHandle.XAxis:
                    self.setDragPlane(Plane(Vector(1, 0, 0), handle_position.x))
                elif id == ToolHandle.YAxis:
                    self.setDragPlane(Plane(Vector(0, 1, 0), handle_position.y))
                elif self._locked_axis == ToolHandle.ZAxis:
                    self.setDragPlane(Plane(Vector(0, 0, 1), handle_position.z))

                self.setDragStart(event.x, event.y)
                self._angle = 0
                self.operationStarted.emit(self)

        if event.type == Event.MouseMoveEvent:
            if not self.getDragPlane():
                return False

            if not self.getDragStart():
                self.setDragStart(event.x, event.y)

            handle_position = self._handle.getWorldPosition()

            drag_start = (self.getDragStart() - handle_position).normalize()
            drag_position = self.getDragPosition(event.x, event.y)
            if not drag_position:
                return
            drag_end = (drag_position - handle_position).normalize()

            try:
                angle = math.acos(drag_start.dot(drag_end))
            except ValueError:
                angle = 0

            if self._snap_rotation:
                angle = int(angle / self._snap_angle) * self._snap_angle
                if angle == 0:
                    return

            rotation = None
            if self.getLockedAxis() == ToolHandle.XAxis:
                direction = 1 if Vector.Unit_X.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_X)
            elif self.getLockedAxis() == ToolHandle.YAxis:
                direction = 1 if Vector.Unit_Y.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_Y)
            elif self.getLockedAxis() == ToolHandle.ZAxis:
                direction = 1 if Vector.Unit_Z.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_Z)

            self._angle += direction * angle

            # Rate-limit the angle change notification
            # This is done to prevent the UI from being flooded with property change notifications,
            # which in turn would trigger constant repaints.
            new_time = time.monotonic()
            if not self._angle_update_time or new_time - self._angle_update_time > 0.01:
                self.propertyChanged.emit()
                self._angle_update_time = new_time

            Selection.applyOperation(RotateOperation, rotation)

            self.setDragStart(event.x, event.y)

        if event.type == Event.MouseReleaseEvent:
            if self.getDragPlane():
                self.setDragPlane(None)
                self.setLockedAxis(None)
                self._angle = None
                self.propertyChanged.emit()
                self.operationStopped.emit(self)
                return True

    def getRotation(self):
        return round(math.degrees(self._angle)) if self._angle else None

    def getRotationSnap(self):
        return self._snap_rotation

    def setRotationSnap(self, snap):
        if snap != self._snap_rotation:
            self._snap_rotation = snap
            self.propertyChanged.emit()

    def getRotationSnapAngle(self):
        return self._snap_angle

    def setRotationSnapAngle(self, angle):
        if angle != self._snap_angle:
            self._snap_angle = angle
            self.propertyChanged.emit()

    def resetRotation(self):
        Selection.applyOperation(SetTransformOperation, None, Quaternion(), None)

    def layFlat(self):
        self.operationStarted.emit(self)
        progress_message = Message("Laying object flat on buildplate...", lifetime = 0, dismissable = False)
        progress_message.setProgress(-1)

        total_iterations = 0
        for selected_object in Selection.getAllSelectedObjects():
            total_iterations += len(selected_object.getMeshDataTransformed().getVertices()) * 2

        progress_message.setMaxProgress(total_iterations)
        progress_message.show()

        Selection.applyOperation(LayFlatOperation)

        progress_message.hide()
        self.operationStopped.emit(self)<commit_msg>Show progress window<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

# TODO: find a better way to let the gui thread process events
from PyQt5.QtWidgets import QApplication

from UM.Tool import Tool
from UM.Event import Event, MouseEvent, KeyEvent
from UM.Application import Application
from UM.Message import Message
from UM.Scene.ToolHandle import ToolHandle
from UM.Scene.Selection import Selection

from UM.Math.Plane import Plane
from UM.Math.Vector import Vector
from UM.Math.Quaternion import Quaternion
from UM.Math.Float import Float

from UM.Operations.RotateOperation import RotateOperation
from UM.Operations.TranslateOperation import TranslateOperation
from UM.Operations.GroupedOperation import GroupedOperation
from UM.Operations.SetTransformOperation import SetTransformOperation
from UM.Operations.LayFlatOperation import LayFlatOperation

from . import RotateToolHandle

import math
import time

class RotateTool(Tool):
    def __init__(self):
        super().__init__()
        self._handle = RotateToolHandle.RotateToolHandle()

        self._snap_rotation = True
        self._snap_angle = math.radians(15)

        self._angle = None

        self._angle_update_time = None

        self.setExposedProperties("Rotation", "RotationSnap", "RotationSnapAngle")

    def event(self, event):
        super().event(event)

        if event.type == Event.KeyPressEvent and event.key == KeyEvent.ShiftKey:
            self._snap_rotation = (not self._snap_rotation)
            self.propertyChanged.emit()

        if event.type == Event.KeyReleaseEvent and event.key == KeyEvent.ShiftKey:
            self._snap_rotation = (not self._snap_rotation)
            self.propertyChanged.emit()

        if event.type == Event.MousePressEvent:
            if MouseEvent.LeftButton not in event.buttons:
                return False

            id = self._selection_pass.getIdAtPosition(event.x, event.y)
            if not id:
                return

            if ToolHandle.isAxis(id):
                self.setLockedAxis(id)
                handle_position = self._handle.getWorldPosition()

                if id == ToolHandle.XAxis:
                    self.setDragPlane(Plane(Vector(1, 0, 0), handle_position.x))
                elif id == ToolHandle.YAxis:
                    self.setDragPlane(Plane(Vector(0, 1, 0), handle_position.y))
                elif self._locked_axis == ToolHandle.ZAxis:
                    self.setDragPlane(Plane(Vector(0, 0, 1), handle_position.z))

                self.setDragStart(event.x, event.y)
                self._angle = 0
                self.operationStarted.emit(self)

        if event.type == Event.MouseMoveEvent:
            if not self.getDragPlane():
                return False

            if not self.getDragStart():
                self.setDragStart(event.x, event.y)

            handle_position = self._handle.getWorldPosition()

            drag_start = (self.getDragStart() - handle_position).normalize()
            drag_position = self.getDragPosition(event.x, event.y)
            if not drag_position:
                return
            drag_end = (drag_position - handle_position).normalize()

            try:
                angle = math.acos(drag_start.dot(drag_end))
            except ValueError:
                angle = 0

            if self._snap_rotation:
                angle = int(angle / self._snap_angle) * self._snap_angle
                if angle == 0:
                    return

            rotation = None
            if self.getLockedAxis() == ToolHandle.XAxis:
                direction = 1 if Vector.Unit_X.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_X)
            elif self.getLockedAxis() == ToolHandle.YAxis:
                direction = 1 if Vector.Unit_Y.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_Y)
            elif self.getLockedAxis() == ToolHandle.ZAxis:
                direction = 1 if Vector.Unit_Z.dot(drag_start.cross(drag_end)) > 0 else -1
                rotation = Quaternion.fromAngleAxis(direction * angle, Vector.Unit_Z)

            self._angle += direction * angle

            # Rate-limit the angle change notification
            # This is done to prevent the UI from being flooded with property change notifications,
            # which in turn would trigger constant repaints.
            new_time = time.monotonic()
            if not self._angle_update_time or new_time - self._angle_update_time > 0.01:
                self.propertyChanged.emit()
                self._angle_update_time = new_time

            Selection.applyOperation(RotateOperation, rotation)

            self.setDragStart(event.x, event.y)

        if event.type == Event.MouseReleaseEvent:
            if self.getDragPlane():
                self.setDragPlane(None)
                self.setLockedAxis(None)
                self._angle = None
                self.propertyChanged.emit()
                self.operationStopped.emit(self)
                return True

    def getRotation(self):
        return round(math.degrees(self._angle)) if self._angle else None

    def getRotationSnap(self):
        return self._snap_rotation

    def setRotationSnap(self, snap):
        if snap != self._snap_rotation:
            self._snap_rotation = snap
            self.propertyChanged.emit()

    def getRotationSnapAngle(self):
        return self._snap_angle

    def setRotationSnapAngle(self, angle):
        if angle != self._snap_angle:
            self._snap_angle = angle
            self.propertyChanged.emit()

    def resetRotation(self):
        Selection.applyOperation(SetTransformOperation, None, Quaternion(), None)

    def layFlat(self):
        self.operationStarted.emit(self)
        progress_message = Message("Laying object flat on buildplate...", lifetime = 0, dismissable = False)
        progress_message.setProgress(-1)

        # TODO: find a better way to let the gui thread process events
        QApplication.processEvents()

        total_iterations = 0
        for selected_object in Selection.getAllSelectedObjects():
            total_iterations += len(selected_object.getMeshDataTransformed().getVertices()) * 2

        progress_message.setMaxProgress(total_iterations)
        progress_message.show()

        # TODO: find a way to show progress during applyOperation
        Selection.applyOperation(LayFlatOperation)

        progress_message.hide()
        self.operationStopped.emit(self)<|endoftext|>