<commit_before>#!/usr/bin/env python
import socket, sys
import subprocess, shlex
from struct import unpack
from encode_protocol import encode_protocol

NUM_FIELDS = 43
SEPARATOR = ','

# parsing one line of data from tshark
# 1) encode categorial data to numbers
# 2) encode hex data using unpack:
#    http://stackoverflow.com/questions/3531723/unpack-from-hex-to-double-in-python
#    the fields to encode: ip.dsfield.dscp, ip.dsfield.ecn, ip.id, ip.flags, tcp.flags
# 3) encode NULL to 0;
def parse_sample(line):
  fields = line.lower().split(SEPARATOR, NUM_FIELDS)

  # THE FIRST FIELD (frame.time_relative) CONTAINS THE UNIQUE IDENTIFIER OF EACH PACKET AS IT WILL BE UNIQUE

  '''Hex fields present in the data: Entire feature set present at FeatureSet.txt
  fields[1] - ip.id
  fields[14] - ip.dsfield.dscp
  fields[15] - ip.dsfield.ecn
  '''

  # Consider only TCP packets: read the ip.proto to check for TCP (ip.proto = 6)
  # Send only TCP data to EC2
  try:
	fields[0]
  except:
	return ""

  if fields[2] != "6":
	return ""

  # 1) encode categorial data to numbers: fields[2] - ip.proto
  fields[2] = str(encode_protocol(fields[2]))

  # 2) convert hex data to string:
  for i in [1,14,15]:
    fields[i] = '' + str(int(fields[i],16))
    #if len(fields[i]) >= 8:
    #  fields[i] = unpack('d', ''.join(fields[i]))

  # 3) encode NULL to 0;
  for i in range(len(fields)):
    if fields[i] == "":
      fields[i] = "0"

  # return the parsed data
  return ','.join(fields)

"""
connect as a TCP *client*
should ideally be a server, because Spark should be the client
but making this part a client is easier than letting EC2 connect
to a machine inside university network
"""
# hard-coded IP and port for testing
TCP_IP = '127.0.0.1'
TCP_PORT = 3000

# EC2 instance socket address taken as cmd line arguments
#TCP_IP = sys.argv[1]
#TCP_PORT = int(sys.argv[2])

BUFFER_SIZE = 1024
#soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#soc.connect((TCP_IP, TCP_PORT))

# RUN tshark COMMAND
cmd = shlex.split('sudo tshark -r AD1.pcap.gz -T fields -e frame.time_relative -e ip.id -e ip.proto -e frame.interface_id -e frame.encap_type -e frame.offset_shift -e frame.time_epoch -e frame.time_delta -e frame.len -e frame.cap_len -e frame.marked -e frame.ignored -e ip.hdr_len -e ip.dsfield -e ip.dsfield.dscp -e ip.dsfield.ecn -e ip.len -e ip.flags.rb -e ip.flags.df -e ip.flags.mf -e ip.frag_offset -e ip.ttl -e tcp.stream -e tcp.hdr_len -e tcp.flags.res -e tcp.flags.ns -e tcp.flags.cwr -e tcp.flags.ecn -e tcp.flags.urg -e tcp.flags.ack -e tcp.flags.push -e tcp.flags.reset -e tcp.flags.syn -e tcp.flags.fin -e tcp.window_size_value -e tcp.window_size -e tcp.window_size_scalefactor -e tcp.option_len -e tcp.options.timestamp.tsval -e tcp.options.timestamp.tsecr -e tcp.analysis.bytes_in_flight -e eth.lg -e eth.ig -E separator=,')

proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# proc = subprocess.Popen(['ping', 'google.com'], shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

# skip the first line, it is not data
#proc.stdout.readline()

# keep reading and sending
f = open("test.txt", "w")
while 1:
  item = parse_sample(proc.stdout.readline())
  if item == "":
	continue

  f.write(item)

  # item CONTAINS THE PACKET INFO TO BE SENT TO EC2 WHEREIN THE 1ST FIELD CONTAINS THE ip.id FIELD THAT UNIQUELY IDENTIFIES THIS PACKET
  #soc.send(item)

soc.close()

<commit_msg>Uncommented the hard-coded lines used for testing<commit_after>#!/usr/bin/env python
import socket, sys
import subprocess, shlex
from struct import unpack
from encode_protocol import encode_protocol

NUM_FIELDS = 43
SEPARATOR = ','

# parsing one line of data from tshark
# 1) encode categorial data to numbers
# 2) encode hex data using unpack:
#    http://stackoverflow.com/questions/3531723/unpack-from-hex-to-double-in-python
#    the fields to encode: ip.dsfield.dscp, ip.dsfield.ecn, ip.id, ip.flags, tcp.flags
# 3) encode NULL to 0;
def parse_sample(line):
  fields = line.lower().split(SEPARATOR, NUM_FIELDS)

  # THE FIRST FIELD (frame.time_relative) CONTAINS THE UNIQUE IDENTIFIER OF EACH PACKET AS IT WILL BE UNIQUE

  '''Hex fields present in the data: Entire feature set present at FeatureSet.txt
  fields[1] - ip.id
  fields[14] - ip.dsfield.dscp
  fields[15] - ip.dsfield.ecn
  '''

  # Consider only TCP packets: read the ip.proto to check for TCP (ip.proto = 6)
  # Send only TCP data to EC2
  try:
	fields[0]
  except:
	return ""

  if fields[2] != "6":
	return ""

  # 1) encode categorial data to numbers: fields[2] - ip.proto
  fields[2] = str(encode_protocol(fields[2]))

  # 2) convert hex data to string:
  for i in [1,14,15]:
    fields[i] = '' + str(int(fields[i],16))
    #if len(fields[i]) >= 8:
    #  fields[i] = unpack('d', ''.join(fields[i]))

  # 3) encode NULL to 0;
  for i in range(len(fields)):
    if fields[i] == "":
      fields[i] = "0"

  # return the parsed data
  return ','.join(fields)

"""
connect as a TCP *client*
should ideally be a server, because Spark should be the client
but making this part a client is easier than letting EC2 connect
to a machine inside university network
"""
# hard-coded IP and port for testing
#TCP_IP = '127.0.0.1'
#TCP_PORT = 3000

# EC2 instance socket address taken as cmd line arguments
TCP_IP = sys.argv[1]
TCP_PORT = int(sys.argv[2])

BUFFER_SIZE = 1024
#soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#soc.connect((TCP_IP, TCP_PORT))

# RUN tshark COMMAND
cmd = shlex.split('sudo tshark -i vboxnet0 -T fields -e frame.time_relative -e ip.id -e ip.proto -e frame.interface_id -e frame.encap_type -e frame.offset_shift -e frame.time_epoch -e frame.time_delta -e frame.len -e frame.cap_len -e frame.marked -e frame.ignored -e ip.hdr_len -e ip.dsfield -e ip.dsfield.dscp -e ip.dsfield.ecn -e ip.len -e ip.flags.rb -e ip.flags.df -e ip.flags.mf -e ip.frag_offset -e ip.ttl -e tcp.stream -e tcp.hdr_len -e tcp.flags.res -e tcp.flags.ns -e tcp.flags.cwr -e tcp.flags.ecn -e tcp.flags.urg -e tcp.flags.ack -e tcp.flags.push -e tcp.flags.reset -e tcp.flags.syn -e tcp.flags.fin -e tcp.window_size_value -e tcp.window_size -e tcp.window_size_scalefactor -e tcp.option_len -e tcp.options.timestamp.tsval -e tcp.options.timestamp.tsecr -e tcp.analysis.bytes_in_flight -e eth.lg -e eth.ig -E separator=,')

proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# proc = subprocess.Popen(['ping', 'google.com'], shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

# skip the first line, it is not data
#proc.stdout.readline()

# keep reading and sending
f = open("test.txt", "w")
while 1:
  item = parse_sample(proc.stdout.readline())
  if item == "":
	continue

  f.write(item)

  # item CONTAINS THE PACKET INFO TO BE SENT TO EC2 WHEREIN THE 1ST FIELD CONTAINS THE ip.id FIELD THAT UNIQUELY IDENTIFIES THIS PACKET
  #soc.send(item)

soc.close()

<|endoftext|>


<commit_before>from __future__ import unicode_literals
from boto.ec2.connection import EC2Connection
from boto.ec2 import get_region
from sqs import (make_SQS_connection, get_queue, build_job_message, send_message,
                 queue_size,)
from operator import attrgetter

JOBS_QUEUE = 'landsat_jobs_queue'
STATE_CODES = {'pending': 0,
               'runnning': 16,
               'shutting-down': 32,
               'terminated': 48,
               'stopping': 64,
               'stopped': 80
               }
LIMITS = {'low': 20, 'med': 50}
TEAMS = {'A': 5, 'B': 10}
NEW_WORKER_STATS = {'AMI': 'ami-517b5561',
                    'KEY_PAIR': 'landsatproject',
                    'SECURITY_GROUP': 'launch-wizard-1',
                    'INSTANCE_TYPE': 't2.medium',
                    'AVAILIBILITY_ZONE': 'us-west-2a',
                    'SUBNET_ID': 'subnet-85ff7ce0'
                    }

def make_EC2_connection(region_name, aws_access_key_id, aws_secret_access_key):
    '''Make EC2 connection to AWS'''
    return EC2Connection(aws_access_key_id=aws_access_key_id,
                         aws_secret_access_key=aws_secret_access_key,
                         region=get_region(region_name))

def foreman(conn, region_name, aws_access_key_id, aws_secret_access_key):
    '''Check status of queue and spawn/kill workers as needed'''
    make_SQS_connection(region_name, aws_access_key_id, aws_secret_access_key)
    jobs_queue = get_queue(conn, JOBS_QUEUE)
    number_queued_jobs = queue_size(jobs_queue)

    workers = list_worker_instances(conn, 'landsatAWS_worker')
    running_workers = list_running_workers(workers)
    running_parttime_workers = list_running_parttime_workers(workers)
    stopped_parttime_workers = list_stopped_parttime_workers(workers)

    if number_queued_jobs > LIMITS['med']:
        worker_deficit = TEAMS['B'] - len(running_workers)
        if worker_deficit > 0:
            if len(stopped_parttime_workers) > 0:
                for stopped in stopped_parttime_workers:
                    stopped.start()
                workers = list_worker_instances(conn, 'landsatAWS_worker')
                number_running = len(list_running_workers(workers))
                number_pending = len(list_pending_instances(workers))
                worker_deficit = TEAMS['B'] - number_pending + number_running
                if worker_deficit > 0:
                    spawn_workers(conn, worker_deficit)
    elif LIMITS['med'] > number_queued_jobs > LIMITS['low']:
        worker_deficit = TEAMS['A'] - len(running_workers)
        if worker_deficit > 0:
            for stopped in stopped_parttime_workers:
                stopped.start()

def spawn_workers(conn, count):
    return conn.run_instances(NEW_WORKER_STATS['AMI'],
                              in_count=count,
                              max_count=count,
                              key_name=NEW_WORKER_STATS['KEY_PAIR'],
                              security_groups=[NEW_WORKER_STATS['SECURITY_GROUP']],
                              instance_type=NEW_WORKER_STATS['INSTANCE_TYPE'],
                              placement=NEW_WORKER_STATS['AVAILIBILITY_ZONE'],
                              subnet_id=NEW_WORKER_STATS['SUBNET_ID']
                              )

def kill_workers(conn):
    pass

def list_worker_instances(conn, worker_type):
    '''Return list of worker instances in order of lauch time, most recent
       first.'''
    filters = {'tag:Name': worker_type}
    instances = conn.get_only_instances(filters=filters)
    instances.sort(key=attrgetter('launch_time'), reverse=True)
    return instances

def list_running_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['runnning']]

def list_running_parttime_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['runnning']
            and worker.tags['Schedule'] == 'parttime']

def list_stopped_parttime_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['stopped']
            and worker.tags['Schedule'] == 'parttime']

def list_pending_instances(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['pending']]

if __name__ == '__main__':
    import os

    AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
    AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
    REGION = 'us-west-2'

    conn = make_EC2_connection(REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
    worker_type = 'landsatAWS_worker'
    workers = list_worker_instances(conn, worker_type)
    running = list_running_parttime_workers(workers)
    stopped = list_stopped_parttime_workers(workers)
    import ipdb; ipdb.set_trace()
    print(workers)
<commit_msg>#3, #15 fix breakout queue checking and worker spawning code from foreman into new function<commit_after>from __future__ import unicode_literals
from boto.ec2.connection import EC2Connection
from boto.ec2 import get_region
from sqs import (make_SQS_connection, get_queue, build_job_message, send_message,
                 queue_size,)
from operator import attrgetter

JOBS_QUEUE = 'landsat_jobs_queue'
STATE_CODES = {'pending': 0,
               'runnning': 16,
               'shutting-down': 32,
               'terminated': 48,
               'stopping': 64,
               'stopped': 80
               }
LIMITS = {'low': 20, 'med': 50}
TEAMS = {'A': 5, 'B': 10}
NEW_WORKER_STATS = {'AMI': 'ami-517b5561',
                    'KEY_PAIR': 'landsatproject',
                    'SECURITY_GROUP': 'launch-wizard-1',
                    'INSTANCE_TYPE': 't2.medium',
                    'AVAILIBILITY_ZONE': 'us-west-2a',
                    'SUBNET_ID': 'subnet-85ff7ce0'
                    }

def make_EC2_connection(region_name, aws_access_key_id, aws_secret_access_key):
    '''Make EC2 connection to AWS'''
    return EC2Connection(aws_access_key_id=aws_access_key_id,
                         aws_secret_access_key=aws_secret_access_key,
                         region=get_region(region_name))

def foreman(conn, region_name, aws_access_key_id, aws_secret_access_key):
    '''Check status of queue and spawn/kill workers as needed'''
    make_SQS_connection(region_name, aws_access_key_id, aws_secret_access_key)
    jobs_queue = get_queue(conn, JOBS_QUEUE)
    number_queued_jobs = queue_size(jobs_queue)

    workers = list_worker_instances(conn, 'landsatAWS_worker')
    running_workers = list_running_workers(workers)
    running_parttime_workers = list_running_parttime_workers(workers)
    stopped_parttime_workers = list_stopped_parttime_workers(workers)

    adjust_team_size(workers,
                     running_workers,
                     running_parttime_workers,
                     stopped_parttime_workers,
                     number_queued_jobs)

def adjust_team_size(workers, running_workers, running_parttime_workers,
                     stopped_parttime_workers, number_queued_jobs):
    if number_queued_jobs > LIMITS['med']:
        worker_deficit = TEAMS['B'] - len(running_workers)
        if worker_deficit > 0:
            if len(stopped_parttime_workers) > 0:
                for stopped in stopped_parttime_workers:
                    stopped.start()
                workers = list_worker_instances(conn, 'landsatAWS_worker')
                number_running = len(list_running_workers(workers))
                number_pending = len(list_pending_instances(workers))
                worker_deficit = TEAMS['B'] - number_pending + number_running
                if worker_deficit > 0:
                    spawn_workers(conn, worker_deficit)
    elif LIMITS['med'] > number_queued_jobs > LIMITS['low']:
        worker_deficit = TEAMS['A'] - len(running_workers)
        if worker_deficit > 0:
            for stopped in stopped_parttime_workers:
                stopped.start()

def spawn_workers(conn, count):
    return conn.run_instances(NEW_WORKER_STATS['AMI'],
                              in_count=count,
                              max_count=count,
                              key_name=NEW_WORKER_STATS['KEY_PAIR'],
                              security_groups=[NEW_WORKER_STATS['SECURITY_GROUP']],
                              instance_type=NEW_WORKER_STATS['INSTANCE_TYPE'],
                              placement=NEW_WORKER_STATS['AVAILIBILITY_ZONE'],
                              subnet_id=NEW_WORKER_STATS['SUBNET_ID']
                              )

def kill_workers(conn):
    pass

def list_worker_instances(conn, worker_type):
    '''Return list of worker instances in order of lauch time, most recent
       first.'''
    filters = {'tag:Name': worker_type}
    instances = conn.get_only_instances(filters=filters)
    instances.sort(key=attrgetter('launch_time'), reverse=True)
    return instances

def list_running_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['runnning']]

def list_running_parttime_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['runnning']
            and worker.tags['Schedule'] == 'parttime']

def list_stopped_parttime_workers(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['stopped']
            and worker.tags['Schedule'] == 'parttime']

def list_pending_instances(workers):
    return [worker for worker in workers
            if worker.state_code == STATE_CODES['pending']]

if __name__ == '__main__':
    import os

    AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
    AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
    REGION = 'us-west-2'

    conn = make_EC2_connection(REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
    worker_type = 'landsatAWS_worker'
    workers = list_worker_instances(conn, worker_type)
    running = list_running_parttime_workers(workers)
    stopped = list_stopped_parttime_workers(workers)
    import ipdb; ipdb.set_trace()
    print(workers)
<|endoftext|>


<commit_before>"""empty message

Revision ID: 59c9813e6f67
Revises: fa7e12c88c5
Create Date: 2015-11-03 22:20:51.669027

"""

# revision identifiers, used by Alembic.
revision = '59c9813e6f67'
down_revision = 'fa7e12c88c5'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('chat_users', sa.Column('show_timestamps', sa.Boolean(), nullable=False))
    op.add_column('users', sa.Column('show_timestamps', sa.Boolean(), nullable=False))
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('users', 'show_timestamps')
    op.drop_column('chat_users', 'show_timestamps')
    ### end Alembic commands ###
<commit_msg>server_default on the above.<commit_after>"""empty message

Revision ID: 59c9813e6f67
Revises: fa7e12c88c5
Create Date: 2015-11-03 22:20:51.669027

"""

# revision identifiers, used by Alembic.
revision = '59c9813e6f67'
down_revision = 'fa7e12c88c5'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('chat_users', sa.Column('show_timestamps', sa.Boolean(), nullable=False, server_default='false'))
    op.add_column('users', sa.Column('show_timestamps', sa.Boolean(), nullable=False, server_default='false'))
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('users', 'show_timestamps')
    op.drop_column('chat_users', 'show_timestamps')
    ### end Alembic commands ###
<|endoftext|>


<commit_before>### Joins tiles from the model together and converts them to tsvs so that they can be put through Hadoop
### for zonal statistics

import utilities
import subprocess
import sys
sys.path.append('../')
import constants_and_names

### git clone https://github.com/wri/raster-to-tsv
### git pull https://github.com/wri/raster-to-tsv float_output
### git clone https://github.com/wri/carbon-budget
### cd carbon-budget/analyses/

print "Making list of biomass tiles..."
# biomass_tile_list = list_tiles()
# biomass_tile_list = ['10N_080W', '40N_120E', '00N_000E'] # test tiles
biomass_tile_list = ['00N_000E'] # test tiles
print "  Biomass tile list retrieved. There are", len(biomass_tile_list), "biomass tiles total."

# Input files will be downloaded to here
local_dir = r'/home/ubuntu/data/'

# # For downloading all tiles in the input folders
# download_list = [constants_and_names.net_emis_dir, constants_and_names.emissions_total_dir,
#                  constants_and_names.annual_gain_combo_dir, constants_and_names.cumul_gain_combo_dir,
#                  constants_and_names.tcd_dir, constants_and_names.loss_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), local_dir)

# For copying individual tiles to spot machine for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.net_emis_dir, constants_and_names.pattern_net_emis, tile), local_dir)  # cumulative aboveand belowground carbon gain for all forest types
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.emissions_total_dir, tile, constants_and_names.pattern_emissions_total), local_dir)  # emissions from all drivers
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.annual_gain_combo_dir, constants_and_names.pattern_annual_gain_combo, tile), local_dir)  # annual gain rate
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.cumul_gain_combo_dir, constants_and_names.pattern_cumul_gain_combo, tile), local_dir)  # cumulative gain
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.tcd_dir, constants_and_names.pattern_tcd, tile), local_dir)  # tree cover density
    utilities.s3_file_download('{0}{1}.tif'.format(constants_and_names.loss_dir, tile), local_dir)  # tree cover loss

out_locn = 's3://gfw2-data/climate/carbon_model/test_output_tsvs/'

# Iterates through tiles to join the desired rasters and convert them to tsvs
for tile in biomass_tile_list:

    print "Processing tile", tile

    # Names of the files that are used for this analysis
    net_emis = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_net_emis, tile)
    gross_emis = '{0}{1}_{2}.tif'.format(local_dir, tile, constants_and_names.pattern_emissions_total)
    annual_gain = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_annual_gain_combo, tile)
    cumul_gain = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_cumul_gain_combo, tile)
    tcd = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_tcd, tile)
    tcl = '{0}{1}.tif'.format(local_dir, tile)

    # Location of write-tsv.py
    ras_cwd = r'/home/ubuntu/raster-to-tsv'

    print "Joining annual gain and tcd2000 for", tile
    # # Code to run write-tsv.py directly on one annual gain tile
    # python write-tsv.py --datasets annual_gain_rate_AGB_BGB_t_ha_all_forest_types_00N_000E.tif Hansen_GFC2014_treecover2000_00N_000E.tif - -s3 - output
    # s3://gfw2-data/climate/carbon_model/test_output_tsvs/annualGain_tcd2000_v2/ --threads 3 --csv-process emissions_gain --prefix annualGain_tcd2000_00N_000E --separate
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', annual_gain, tcd, '--s3-output', '{}annualGain_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'annualGain_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining cumulative gain and tcd2000 for", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', cumul_gain, tcd, '--s3-output', '{}cumulGain_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'cumulGain_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining net emissions and tcd2000", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', net_emis, tcd, '--s3-output', '{}netEmis_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'netEmis_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining gross emissions and tcd2000 and tree cover loss for", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', gross_emis, tcd, tcl, '--s3-output', '{}grossEmis_tcd2000_tcl/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'grossEmis_tcd2000_treeCoverLoss_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)
<commit_msg>Regenerating 00N_000E tsvs for model output to Hadoop.<commit_after>### Joins tiles from the model together and converts them to tsvs so that they can be put through Hadoop
### for zonal statistics

import utilities
import subprocess
import sys
sys.path.append('../')
import constants_and_names

### git clone https://github.com/wri/raster-to-tsv
### git pull https://github.com/wri/raster-to-tsv float_output
### git clone https://github.com/wri/carbon-budget
### cd carbon-budget/analyses/

print "Making list of biomass tiles..."
# biomass_tile_list = list_tiles()
# biomass_tile_list = ['10N_080W', '40N_120E', '00N_000E'] # test tiles
biomass_tile_list = ['00N_000E'] # test tiles
print "  Biomass tile list retrieved. There are", len(biomass_tile_list), "biomass tiles total."

# Input files will be downloaded to here
local_dir = r'/home/ubuntu/data/'

# # For downloading all tiles in the input folders
# download_list = [constants_and_names.net_emis_dir, constants_and_names.emissions_total_dir,
#                  constants_and_names.annual_gain_combo_dir, constants_and_names.cumul_gain_combo_dir,
#                  constants_and_names.tcd_dir, constants_and_names.loss_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), local_dir)

# For copying individual tiles to spot machine for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.net_emis_dir, constants_and_names.pattern_net_emis, tile), local_dir)  # cumulative aboveand belowground carbon gain for all forest types
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.emissions_total_dir, tile, constants_and_names.pattern_emissions_total), local_dir)  # emissions from all drivers
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.annual_gain_combo_dir, constants_and_names.pattern_annual_gain_combo, tile), local_dir)  # annual gain rate
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.cumul_gain_combo_dir, constants_and_names.pattern_cumul_gain_combo, tile), local_dir)  # cumulative gain
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(constants_and_names.tcd_dir, constants_and_names.pattern_tcd, tile), local_dir)  # tree cover density
    utilities.s3_file_download('{0}{1}.tif'.format(constants_and_names.loss_dir, tile), local_dir)  # tree cover loss

out_locn = 's3://gfw2-data/climate/carbon_model/test_output_tsvs/'

# Iterates through tiles to join the desired rasters and convert them to tsvs
for tile in biomass_tile_list:

    print "Processing tile", tile

    # Names of the files that are used for this analysis
    net_emis = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_net_emis, tile)
    gross_emis = '{0}{1}_{2}.tif'.format(local_dir, tile, constants_and_names.pattern_emissions_total)
    annual_gain = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_annual_gain_combo, tile)
    cumul_gain = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_cumul_gain_combo, tile)
    tcd = '{0}{1}_{2}.tif'.format(local_dir, constants_and_names.pattern_tcd, tile)
    tcl = '{0}{1}.tif'.format(local_dir, tile)

    # Location of write-tsv.py
    ras_cwd = r'/home/ubuntu/raster-to-tsv'

    print "Joining annual gain and tcd2000 for", tile
    # # Code to run write-tsv.py directly on one annual gain tile
    # python write-tsv.py --datasets annual_gain_rate_AGB_BGB_t_ha_all_forest_types_00N_000E.tif Hansen_GFC2014_treecover2000_00N_000E.tif --s3-output
    # s3://gfw2-data/climate/carbon_model/test_output_tsvs/annualGain_tcd2000_v2/ --threads 3 --csv-process emissions_gain --prefix annualGain_tcd2000_00N_000E --separate
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', annual_gain, tcd, '--s3-output', '{}annualGain_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'annualGain_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining cumulative gain and tcd2000 for", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', cumul_gain, tcd, '--s3-output', '{}cumulGain_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'cumulGain_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining net emissions and tcd2000", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', net_emis, tcd, '--s3-output', '{}netEmis_tcd2000/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'netEmis_tcd2000_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)

    print "Joining gross emissions and tcd2000 and tree cover loss for", tile
    ras_to_vec_cmd = ['python', 'write-tsv.py', '--datasets', gross_emis, tcd, tcl, '--s3-output', '{}grossEmis_tcd2000_tcl/'.format(out_locn)]
    ras_to_vec_cmd += ['--threads', '20', '--csv-process', 'emissions_gain', '--prefix', 'grossEmis_tcd2000_treeCoverLoss_{}'.format(tile), '--separate']
    subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)
<|endoftext|>


<commit_before>
from builtins import object
import configparser
import numpy as np
from astropy.time import Time
from .QueueManager import ListQueueManager, GreedyQueueManager, GurobiQueueManager
from .ObsLogger import ObsLogger
from .configuration import SchedulerConfiguration
from .constants import BASE_DIR
from .utils import block_index

class Scheduler(object):

    def __init__(self, scheduler_config_file_fullpath,
            run_config_file_fullpath, other_queue_configs = None):

        self.scheduler_config = SchedulerConfiguration(
            scheduler_config_file_fullpath)
        self.queue_configs = self.scheduler_config.build_queue_configs()
        self.queues = self.scheduler_config.build_queues(self.queue_configs)
        self.timed_queues_tonight = []

        self.set_queue('default')

        self.run_config = configparser.ConfigParser()
        self.run_config.read(run_config_file_fullpath)

        if 'log_name' in self.run_config['scheduler']:
            log_name = self.run_config['scheduler']['log_name']
        else:
            log_name = self.scheduler_config.config['run_name']

        # initialize sqlite history
        self.obs_log = ObsLogger(log_name,
                clobber=self.run_config['scheduler'].getboolean('clobber_db'))

    def set_queue(self, queue_name):

        # TODO: log the switch

        if queue_name not in self.queues:
            raise ValueError(f'Requested queue {queue_name} not available!')

        self.Q = self.queues[queue_name]

    def add_queue(self,  queue_name, queue, clobber=True):

        if clobber or (queue_name not in self.queues):
            self.queues[queue_name] = queue
        else:
            raise ValueError(f"Queue {queue_name} already exists!")

    def delete_queue(self, queue_name):

        if (queue_name in self.queues):
            if self.Q.queue_name == queue_name:
                self.set_queue('default')
            del self.queues[queue_name]
        else:
            raise ValueError(f"Queue {queue_name} does not exist!")

    def find_excluded_blocks_tonight(self, time_now):
        # also sets up timed_queues_tonight

        # start of the night
        mjd_today = np.floor(time_now.mjd).astype(int)

        # Look for timed queues that will be valid tonight,
        # to exclude from the nightly solution
        self.timed_queues_tonight = []
        block_start = block_index(Time(mjd_today, format='mjd'))
        block_stop = block_index(Time(mjd_today + 1, format='mjd'))
        exclude_blocks = []
        for qq_name, qq in self.queues.items():
            if qq.queue_name in ['default', 'fallback']:
                continue
            if qq.validity_window is not None:
                valid_blocks = qq.valid_blocks(complete_only=True)
                valid_blocks_tonight = [b for b in valid_blocks if
                        (block_start <= b <= block_stop)]
                if len(valid_blocks_tonight):
                    self.timed_queues_tonight.append(qq_name)
                exclude_blocks.extend(valid_blocks_tonight)
        return exclude_blocks

    def check_for_TOO_queue_and_switch(self, time_now):
        # check if a TOO queue is now valid
        for qq_name, qq in self.queues.items():
            if qq.is_TOO:
                if qq.is_valid(time_now):
                    # only switch if we don't have an active TOO queue
                    if not self.Q.is_TOO and len(qq.queue):
                        self.set_queue(qq_name)

    def check_for_timed_queue_and_switch(self, time_now):
            # drop out of a timed queue if it's no longer valid
            if self.Q.queue_name != 'default':
                if not self.Q.is_valid(time_now):
                    self.set_queue('default')

            # check if a timed queue is now valid
            for qq_name in self.timed_queues_tonight:
                qq = self.queues[qq_name]
                if qq.is_valid(time_now) and len(qq.queue):
                    # only switch if we are in the default or fallback queue
                    if self.Q.queue_name in ['default', 'fallback']:
                        self.set_queue(qq_name)

    def remove_empty_and_expired_queues(self, time_now):
        for qq_name, qq in self.queues.items():
            if qq.queue_name in ['default', 'fallback']:
                continue
            if qq.validity_window is not None:
                if qq.validity_window[1] < time_now:
                    self.delete_queue(qq_name)
            if len(qq.queue) == 0:
                self.delete_queue(qq_name)
<commit_msg>queue removal must be staged<commit_after>
from builtins import object
import configparser
import numpy as np
from astropy.time import Time
from .QueueManager import ListQueueManager, GreedyQueueManager, GurobiQueueManager
from .ObsLogger import ObsLogger
from .configuration import SchedulerConfiguration
from .constants import BASE_DIR
from .utils import block_index

class Scheduler(object):

    def __init__(self, scheduler_config_file_fullpath,
            run_config_file_fullpath, other_queue_configs = None):

        self.scheduler_config = SchedulerConfiguration(
            scheduler_config_file_fullpath)
        self.queue_configs = self.scheduler_config.build_queue_configs()
        self.queues = self.scheduler_config.build_queues(self.queue_configs)
        self.timed_queues_tonight = []

        self.set_queue('default')

        self.run_config = configparser.ConfigParser()
        self.run_config.read(run_config_file_fullpath)

        if 'log_name' in self.run_config['scheduler']:
            log_name = self.run_config['scheduler']['log_name']
        else:
            log_name = self.scheduler_config.config['run_name']

        # initialize sqlite history
        self.obs_log = ObsLogger(log_name,
                clobber=self.run_config['scheduler'].getboolean('clobber_db'))

    def set_queue(self, queue_name):

        # TODO: log the switch

        if queue_name not in self.queues:
            raise ValueError(f'Requested queue {queue_name} not available!')

        self.Q = self.queues[queue_name]

    def add_queue(self,  queue_name, queue, clobber=True):

        if clobber or (queue_name not in self.queues):
            self.queues[queue_name] = queue
        else:
            raise ValueError(f"Queue {queue_name} already exists!")

    def delete_queue(self, queue_name):

        if (queue_name in self.queues):
            if self.Q.queue_name == queue_name:
                self.set_queue('default')
            del self.queues[queue_name]
        else:
            raise ValueError(f"Queue {queue_name} does not exist!")

    def find_excluded_blocks_tonight(self, time_now):
        # also sets up timed_queues_tonight

        # start of the night
        mjd_today = np.floor(time_now.mjd).astype(int)

        # Look for timed queues that will be valid tonight,
        # to exclude from the nightly solution
        self.timed_queues_tonight = []
        block_start = block_index(Time(mjd_today, format='mjd'))
        block_stop = block_index(Time(mjd_today + 1, format='mjd'))
        exclude_blocks = []
        for qq_name, qq in self.queues.items():
            if qq.queue_name in ['default', 'fallback']:
                continue
            if qq.validity_window is not None:
                valid_blocks = qq.valid_blocks(complete_only=True)
                valid_blocks_tonight = [b for b in valid_blocks if
                        (block_start <= b <= block_stop)]
                if len(valid_blocks_tonight):
                    self.timed_queues_tonight.append(qq_name)
                exclude_blocks.extend(valid_blocks_tonight)
        return exclude_blocks

    def check_for_TOO_queue_and_switch(self, time_now):
        # check if a TOO queue is now valid
        for qq_name, qq in self.queues.items():
            if qq.is_TOO:
                if qq.is_valid(time_now):
                    # only switch if we don't have an active TOO queue
                    if not self.Q.is_TOO and len(qq.queue):
                        self.set_queue(qq_name)

    def check_for_timed_queue_and_switch(self, time_now):
            # drop out of a timed queue if it's no longer valid
            if self.Q.queue_name != 'default':
                if not self.Q.is_valid(time_now):
                    self.set_queue('default')

            # check if a timed queue is now valid
            for qq_name in self.timed_queues_tonight:
                qq = self.queues[qq_name]
                if qq.is_valid(time_now) and len(qq.queue):
                    # only switch if we are in the default or fallback queue
                    if self.Q.queue_name in ['default', 'fallback']:
                        self.set_queue(qq_name)

    def remove_empty_and_expired_queues(self, time_now):
        queues_for_deletion = []
        for qq_name, qq in self.queues.items():
            if qq.queue_name in ['default', 'fallback']:
                continue
            if qq.validity_window is not None:
                if qq.validity_window[1] < time_now:
                    queues_for_deletion.append(qq_name)
            if len(qq.queue) == 0:
                    queues_for_deletion.append(qq_name)

        for qq_name in queues_for_deletion:
                self.delete_queue(qq_name)
<|endoftext|>


<commit_before>
import os
import inspect
import numpy as np
from astropy.time import Time
import astropy.coordinates as coords
import astropy.units as u
import astroplan

BASE_DIR = os.path.dirname(os.path.abspath(inspect.getfile(
                inspect.currentframe()))) + '/'

P48_loc = coords.EarthLocation(lat=coords.Latitude('33d21m26.35s'),
                               lon=coords.Longitude('-116d51m32.04s'),
                               height=1707.)

# use UTC only
P48_Observer = astroplan.Observer(location=P48_loc)

# HA and Dec from http://www.oir.caltech.edu/twiki_oir/bin/view/Palomar/ZTF/TelescopeSpecifications v5
# Dome estimate from Jeff Z email, 9/21/15
# Requirements/goals info from Jeff Z email, 12/12/16
P48_slew_pars = {
    'ha': {'coord': 'ra', 'accel': 0.27 * u.deg * u.second**(-2.),
           'decel': 0.27 * u.deg * u.second**(-2.),
           'vmax': 1.18 * u.deg / u.second},
    'dec': {'coord': 'dec', 'accel': 0.41 * u.deg * u.second**(-2.),
            'decel': 0.41 * u.deg * u.second**(-2.),
            'vmax': 1.50 * u.deg / u.second},
    'dome': {'coord': 'az', 'accel': 0.5 * u.deg * u.second**(-2.),
             'decel': 0.5 * u.deg * u.second**(-2.),
             'vmax': 5. * u.deg / u.second}}
P48_slew_pars_goal = {
    'ha': {'coord': 'ra', 'accel': 0.50 * u.deg * u.second**(-2.),
           'decel': 0.50 * u.deg * u.second**(-2.),
           'vmax': 3.00 * u.deg / u.second},
    'dec': {'coord': 'dec', 'accel': 0.5 * u.deg * u.second**(-2.),
            'decel': 0.5 * u.deg * u.second**(-2.),
            'vmax': 3.00 * u.deg / u.second},
    'dome': {'coord': 'az', 'accel': 0.5 * u.deg * u.second**(-2.),
             'decel': 0.5 * u.deg * u.second**(-2.),
             'vmax': 5. * u.deg / u.second}}
# PTF configuration
# P48_slew_pars = {
#    'ha': {'coord': 'ra', 'accel': 0.27 * u.deg * u.second**(-2.),
#           'decel': 0.27 * u.deg * u.second**(-2.),
#           'vmax': 1.6 * u.deg / u.second},
#    'dec': {'coord': 'dec', 'accel': 0.2 * u.deg * u.second**(-2.),
#            'decel': 0.15 * u.deg * u.second**(-2.),
#            'vmax': 1.2 * u.deg / u.second},
#    'dome': {'coord': 'az', 'accel': 0.17 * u.deg * u.second**(-2.),
#             'decel': 0.6 * u.deg * u.second**(-2.),
#             'vmax': 3. * u.deg / u.second}}

EXPOSURE_TIME = 30. * u.second
READOUT_TIME = 10. * u.second
FILTER_CHANGE_TIME = 90. * u.second
SETTLE_TIME = 2. * u.second

MAX_AIRMASS = 2.5

TIME_BLOCK_SIZE = 30. * u.min

PROGRAM_NAME_TO_ID = {'collaboration': 1, 'MSIP': 2, 'Caltech': 3}
PROGRAM_NAMES = list(PROGRAM_NAME_TO_ID.keys())
PROGRAM_ID_TO_NAME = {v: k for k, v in list(PROGRAM_NAME_TO_ID.items())}
PROGRAM_IDS = list(PROGRAM_ID_TO_NAME.keys())

PROGRAM_BLOCK_SEQUENCE = [1, 2, 1, 2, 3]
LEN_BLOCK_SEQUENCE = len(PROGRAM_BLOCK_SEQUENCE)

FILTER_NAME_TO_ID = {'g': 1, 'r': 2, 'i': 3}
FILTER_NAMES = list(FILTER_NAME_TO_ID.keys())
FILTER_ID_TO_NAME = {v: k for k, v in list(FILTER_NAME_TO_ID.items())}
FILTER_IDS = list(FILTER_ID_TO_NAME.keys())

PIXEL_SCALE = 1.006  # arcsec/pixel

def slew_time(axis, angle):
    vmax = P48_slew_pars[axis]['vmax']
    acc = P48_slew_pars[axis]['accel']
    dec = P48_slew_pars[axis]['decel']

    t_acc = vmax / acc
    t_dec = vmax / dec
    # TODO: if needed, include conditional for scalars
#    if 0.5*vmax*(t_acc+t_dec)>=angle:
#        return np.sqrt(2*angle*(1./acc+1./dec))
#    else:
#        return 0.5*(2.*angle/vmax+t_acc+t_dec)
    slew_time = 0.5 * (2. * angle / vmax + t_acc + t_dec)
    w = 0.5 * vmax * (t_acc + t_dec) >= angle
    slew_time[w] = np.sqrt(2 * angle[w] * (1. / acc + 1. / dec))

    wnonzero = slew_time > 0
    slew_time[wnonzero] += SETTLE_TIME
    return slew_time
<commit_msg>update slew speeds from Telescope Drive Performance Assessment v1.2<commit_after>
import os
import inspect
import numpy as np
from astropy.time import Time
import astropy.coordinates as coords
import astropy.units as u
import astroplan

BASE_DIR = os.path.dirname(os.path.abspath(inspect.getfile(
                inspect.currentframe()))) + '/'

P48_loc = coords.EarthLocation(lat=coords.Latitude('33d21m26.35s'),
                               lon=coords.Longitude('-116d51m32.04s'),
                               height=1707.)

# use UTC only
P48_Observer = astroplan.Observer(location=P48_loc)

# HA and Dec from http://www.oir.caltech.edu/twiki_oir/bin/view/Palomar/ZTF/TelescopeSpecifications v5
# Dome estimate from Jeff Z email, 9/21/15
# goals info from Jeff Z email, 12/12/16
# Ha/Dec from Telescope Drive Performance Assessment v1.2; dome estimate from
# Jeff Z. email, 9/27/17
P48_slew_pars = {
    'ha': {'coord': 'ra', 'accel': 0.4 * u.deg * u.second**(-2.),
           'decel': 0.4 * u.deg * u.second**(-2.),
           'vmax': 2.5 * u.deg / u.second},
    'dec': {'coord': 'dec', 'accel': 0.5 * u.deg * u.second**(-2.),
            'decel': 0.5 * u.deg * u.second**(-2.),
            'vmax': 3.0 * u.deg / u.second},
    'dome': {'coord': 'az', 'accel': 0.5 * u.deg * u.second**(-2.),
             'decel': 0.5 * u.deg * u.second**(-2.),
             'vmax': 3. * u.deg / u.second}}
P48_slew_pars_goal = {
    'ha': {'coord': 'ra', 'accel': 0.50 * u.deg * u.second**(-2.),
           'decel': 0.50 * u.deg * u.second**(-2.),
           'vmax': 3.00 * u.deg / u.second},
    'dec': {'coord': 'dec', 'accel': 0.5 * u.deg * u.second**(-2.),
            'decel': 0.5 * u.deg * u.second**(-2.),
            'vmax': 3.00 * u.deg / u.second},
    'dome': {'coord': 'az', 'accel': 0.5 * u.deg * u.second**(-2.),
             'decel': 0.5 * u.deg * u.second**(-2.),
             'vmax': 5. * u.deg / u.second}}
# PTF configuration
# P48_slew_pars = {
#    'ha': {'coord': 'ra', 'accel': 0.27 * u.deg * u.second**(-2.),
#           'decel': 0.27 * u.deg * u.second**(-2.),
#           'vmax': 1.6 * u.deg / u.second},
#    'dec': {'coord': 'dec', 'accel': 0.2 * u.deg * u.second**(-2.),
#            'decel': 0.15 * u.deg * u.second**(-2.),
#            'vmax': 1.2 * u.deg / u.second},
#    'dome': {'coord': 'az', 'accel': 0.17 * u.deg * u.second**(-2.),
#             'decel': 0.6 * u.deg * u.second**(-2.),
#             'vmax': 3. * u.deg / u.second}}

EXPOSURE_TIME = 30. * u.second
READOUT_TIME = 10. * u.second
FILTER_CHANGE_TIME = 90. * u.second
SETTLE_TIME = 2. * u.second

MAX_AIRMASS = 2.5

TIME_BLOCK_SIZE = 30. * u.min

PROGRAM_NAME_TO_ID = {'collaboration': 1, 'MSIP': 2, 'Caltech': 3}
PROGRAM_NAMES = list(PROGRAM_NAME_TO_ID.keys())
PROGRAM_ID_TO_NAME = {v: k for k, v in list(PROGRAM_NAME_TO_ID.items())}
PROGRAM_IDS = list(PROGRAM_ID_TO_NAME.keys())

PROGRAM_BLOCK_SEQUENCE = [1, 2, 1, 2, 3]
LEN_BLOCK_SEQUENCE = len(PROGRAM_BLOCK_SEQUENCE)

FILTER_NAME_TO_ID = {'g': 1, 'r': 2, 'i': 3}
FILTER_NAMES = list(FILTER_NAME_TO_ID.keys())
FILTER_ID_TO_NAME = {v: k for k, v in list(FILTER_NAME_TO_ID.items())}
FILTER_IDS = list(FILTER_ID_TO_NAME.keys())

PIXEL_SCALE = 1.006  # arcsec/pixel

def slew_time(axis, angle):
    vmax = P48_slew_pars[axis]['vmax']
    acc = P48_slew_pars[axis]['accel']
    dec = P48_slew_pars[axis]['decel']

    t_acc = vmax / acc
    t_dec = vmax / dec
    # TODO: if needed, include conditional for scalars
#    if 0.5*vmax*(t_acc+t_dec)>=angle:
#        return np.sqrt(2*angle*(1./acc+1./dec))
#    else:
#        return 0.5*(2.*angle/vmax+t_acc+t_dec)
    slew_time = 0.5 * (2. * angle / vmax + t_acc + t_dec)
    w = 0.5 * vmax * (t_acc + t_dec) >= angle
    slew_time[w] = np.sqrt(2 * angle[w] * (1. / acc + 1. / dec))

    wnonzero = slew_time > 0
    slew_time[wnonzero] += SETTLE_TIME
    return slew_time
<|endoftext|>


<commit_before>import numpy as np
import numpy.fft as fft
import re

from urllib2 import urlopen, HTTPError
from scipy.signal import get_window
from scipy.stats import pearsonr
from giapy.data_tools.abstractDataClasses import AbsGeoTimeSeries, \
                                                AbsGeoTimeSeriesContainer

def calcRSL(sim, rsldata, smooth=True, noise=0):
    #TODO This function shouldn't have to know what's inside any object, let
    # alone a complicated one, like sim. Consider calculating rsl first and
    # passing it and calculated times in (ts, rsl, emergedata).
    # To reference to present day
    #u0 = sim.inputs.harmTrans.spectogrd(sim['topo'][-1])
    u0 = sim['sstopo'][-1]

    uAtLocs = []
    for ut in sim['sstopo']:
        ut = u0 - ut
        interpfunc = sim.inputs.grid.create_interper(ut.T*1000)
        uAtLocs.append(interpfunc.ev(rsldata.lons, rsldata.lats))

    uAtLocs = np.array(uAtLocs).T

    calcTimes = sim['sstopo'].outTimes
    if np.all(np.diff(calcTimes)<0):
        reverse = True
    else:
        reverse=False

    data = {}
    for uAtLoc, loc in zip(uAtLocs, rsldata):
        if smooth:
            ts = np.union1d(np.sort(loc.ts), np.linspace(0, loc.ts.max()))
        else:
            ts = np.sort(loc.ts)

        # Sim objects output times in thousands of years relative to present
        tsint = (2015 - ts)/1000.

        if reverse:
            timeseries = np.array([ts,
                    np.interp(tsint,
                                sim['sstopo'].outTimes[::-1], uAtLoc[::-1])]).T
        else:
            timeseries = np.array([ts,
                    np.interp(tsint,
                                sim['sstopo'].outTimes, uAtLoc)]).T

        #Apply the noise (default 0)
        scatter = noise*np.random.randn(len(timeseries[:,1]))
        timeseries[:,1] = timeseries[:,1] + scatter
        data[loc.stid] = RSLDatum(timeseries,
                                        lat=loc.lat,
                                        lon=loc.lon,
                                        sitename=loc.sitename,
                                        stid=loc.stid)

    return RSLData(data=data)

class RLR(np.ndarray):
    """Download a Revised Local Reference for sea level from the PSMSL.

    Data is stored as an ndarray, with the first column ([:,0]) storing the
    dates of the records (in yearly decimal format, e.g., 1947.23), the second
    ([:,1]) the sea level records, and the third and fourth error information.

    Missing data points are stored as -99999, and a list of the indices of
    non-absent data are stored in RLR.ind.

    Parameters
    ----------
    num : int
        The unique locator code for the RLR data to download
    typ : 'monthly' or 'annual'
        Download either monthly or annual data

    Reference
    ---------
    Simon J. Holgate, Andrew Matthews, Philip L. Woodworth, Lesley J. Rickards,
    Mark E. Tamisiea, Elizabeth Bradshaw, Peter R. Foden, Kathleen M. Gordon,
    Svetlana Jevrejeva, and Jeff Pugh (2013) New Data Systems and Products at
    the Permanent Service for Mean Sea Level. Journal of Coastal Research:
    Volume 29, Issue 3: pp. 493 - 504. doi:10.2112/JCOASTRES-D-12-00175.1.
    """
    def __new__(cls, num, typ='monthly', *args):
        # option parsing
        if typ not in ['monthly', 'annual']:
            raise ValueError("typ must be 'monthly' or 'annual'")

        # the data array
        _url = 'http://www.psmsl.org/data/obtaining/rlr.'+typ+'.data/'
        try:
            _response = urlopen(_url+str(num)+'.rlrdata')
        except HTTPError:
            raise ValueError("{0} is an invalid station id".format(num))

        # Format of data is: date | mean sea level | missing record | flag
        if typ=='annual':
            # In annual data, third column indicates whether the record was
            # missing 30 days ('Y') or not ('N')
            converters = {2: lambda s: (0 if s=='N' else 1)}
        else:
            converters = None
        _data = np.loadtxt(_response, delimiter=';', converters=converters)
        # Protect against sites with only one data point
        _data = np.atleast_2d(_data)

        # the metadata
        _murl = 'http://www.psmsl.org/data/obtaining/stations/'
        _meta = urlopen(_murl+str(num)+'.php')
        _txt = _meta.read()
        _sitename = re.findall('
(.*)
', _txt, flags=re.DOTALL)[0]
        _tmarker = '(.*)'
        _m = re.findall(_tmarker, _txt, flags=re.DOTALL)
        _rows = re.findall('
.*?
', _m[0], flags=re.DOTALL)
        _table = [[cell[4:-5] for cell in
                    re.findall('.*?	', row, flags=re.DOTALL)]
                    for row in _rows]

        if _table[3][0] in ['GLOSS ID', 'GLOSS ID:']:       # some locs have a
            _metadata = {'sitename' : _sitename,            # gloss id
                         'stid' : int(_table[0][1]),
                         'lat'  : float(_table[1][1]),
                         'lon'  : float(_table[2][1]),
                         #'gloss': float(_table[3][1][-7:-4]),
                         'coastcode' : int(_table[4][1]),
                         'stcode' : int(_table[5][1]),
                         'country' : _table[6][1],
                         'yrmin' : int(_table[7][1][:4]),
                         'yrmax' : int(_table[7][1][-4:]),
                         'comp' : float(_table[8][1]),
                         'frcode' : _table[9][1],
                         'update' : _table[10][1],
                         'type' : typ
                         }
        else:                                               # some don't
            _metadata = {'sitename' : _sitename,
                         'stid' : int(_table[0][1]),
                         'lat'  : float(_table[1][1]),
                         'lon'  : float(_table[2][1]),
                         'coastcode' : int(_table[3][1]),
                         'stcode' : int(_table[4][1]),
                         'country' : _table[5][1],
                         'yrmin' : int(_table[6][1][:4]),
                         'yrmax' : int(_table[6][1][-4:]),
                         'comp' : float(_table[7][1]),
                         'frcode' : _table[8][1],
                         'update' : _table[9][1],
                         'type' : typ
                         }

        # store indices of time where data are absent
        _inds = np.where(_data[:,1]!=-99999)[0]

        obj = np.asarray(_data).view(cls)
        obj.metadata = _metadata
        obj.inds = _inds

        return obj

    def __array_finalize__(self, obj):
        if obj is None: return
        self.metadata = getattr(obj, 'metadata', None)
        self.inds = getattr(obj, 'inds', None)

    def __reduce__(self):
        # Get the parent's __reduce__ tuple
        pickled_state = super(RLR, self).__reduce__()
        # Create our own tuple to pass to __setstate__
        new_state = pickled_state[2] + (self.inds,self.metadata)
        # Return a tuple that replaces the
        # parent's __setstate__ tuple with our
        # own
        return (pickled_state[0], pickled_state[1], new_state)

    def __setstate__(self, state):
        self.inds, self.metadata = state[-2], state[-1]  # Set the info attribute
        # Call the parent's __setstate__ with the other tuple elements.
        super(RLR, self).__setstate__(state[0:-2])

    def __str__(self):
        return 'RLR( {0}, {1} )'.format(self.metadata['sitename'],
                        self.metadata['type'])

    def trend(self, coeffs=False):
        """Calculate the best linear fit for data, returns coeffs or line.
        """
        inds = self.inds
        A = np.array([self[inds,0], np.ones(len(inds))])
        w = np.linalg.lstsq(A.T, self[inds,1])[0]
        if coeffs:
            return w
        else:
            return w[0]*self[:,0]+w[1]

    def detrended(self):
        """Remove the linear trend from the RLR data.
        """
        inds = self.inds
        result = self.copy()#[self.inds,:]
        result[inds,1] = result[inds,1]-self.trend()[inds]#[self.inds, :]
        return result

    def window_filter(self, window):
        """Filters the data using a window in Fourier space.

        See docstring for scipy.signal.window
        """
        if self.metadata['type'] == 'monthly':
            d = 0.08333
        else:
            d = 1

        inds = self.inds
        result = self.copy()#[self.inds,:]

        fs = fft.fftshift(fft.fftfreq(len(self.inds), d=d))
        self_fft = fft.fftshift(fft.fft(self.detrended()[inds, 1]))
        win = get_window(window, len(inds))
        result[inds,1] = fft.ifft(fft.ifftshift(win*self_fft))
        result[inds,1] = result[inds,1] + self.trend()[inds]#[self.inds]

        return result

    def runavg_filter(self, w, nmiss=0, trimmed=True, align='left'):
        """Computes the running average of w samples.

        When a gap occurs in the data, it is given zero weight in the filter,
        and the weighting is renormalized. Thus, some gaps can be filled. If
        the gap cannot be filled, it is flagged as missing.

        Mitchum 1987 (cited by Clarke 1992)
        """
        result = self.copy()

        if nmiss<0:
            raise ValueError('nmiss must be non-negative')

        if align=='left':
            start = w
            finish = self.shape[0]
        elif align=='cent':
            start = w/2
            finish = self.shape[0]-w/2
        elif align=='right':
            start = 0
            finish = self.shape[0]-w

        for i in range(start, finish):
            if align=='left':
                block = self[i-w:i, 1]
            elif align=='cent':
                block = self[i-w/2:i+w/2, 1]
            elif align=='right':
                block = self[i:i+w, 1]
            block = block[block!=-99999]
            if len(block)<=nmiss:
                result[i, 1] = -99999
            else:
                result[i, 1] = block.mean()

        # reset the data not absent indices
        if trimmed:
            if align=='left':
                result = result[w:,:]
            elif align=='cent':
                result = result[w/2:-w/2,:]
            elif align=='right':
                result = result[:-w,:]

        result.inds = np.where(result[:,1]!=-99999)[0]
        return result

    def time_filter(self, tmin, tmax):
        """Filter the data to a specific time window [tmin, tmax]"""
        filtinds = np.where(np.logical_and(self[:,0]>tmin, self[:,0]<=tmax))
        result = self[filtinds]
        # correct data not absent indices
        result.inds = np.where(result[:,1]!=-99999)[0]

        return result

    def plot(self, ax, *args, **kwargs):
        """Plots the data on ax, skipping missing points"""
        ax.plot(self[self.inds, 0], self[self.inds, 1], *args, **kwargs)
        return ax

class RSLDatum(AbsGeoTimeSeries):
    def __repr__(self):
        return 'RLR( {0}, {1} )'.format(self.sitename, self.type)

class RSLData(AbsGeoTimeSeriesContainer):
    def __init__(self, nbrs=None, data=None, typ='monthly'):
        if nbrs is not None:
            self.data = {}
            self.download(nbrs, typ)
        elif data is not None:
            self.data = data
        self.form_long_vectors()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __iter__(self):
        return self.data.itervalues()

    def __len__(self):
        return len(self.data)

    def download(self, nbrs, typ):
        for n in nbrs:
            try:
                loc = RLR(n, typ=typ)
                self.data[n] = RSLDatum(loc, **loc.metadata)
            except ValueError as e:
                print e.message
                continue

    def filter_by_time(self, tmin, tmax, replace=False):
        result = {}
        for n, loc in self.data.iteritems():
            if loc.metadata['yrmin']<=tmin and loc.metadata['yrmax']>=tmax:
                result[n] = loc
        if replace:
            self.data = result
        else:
            return RSLData(nbrs=None, data=result)

    def form_long_vectors(self):
        """Update the long lists: long_data, long_time, and locs with currently
        encapsulated data.

        Certain numbers are stored twice, for convenience in calculating the
        residuals, in the form of long lists. These are every emergence height,
        every emergence time, and every lon,lat pair. The one-time storage
        overhead is worth the time saved from not recreating these lists on
        every iteration of an inversion.
        """

        self.long_data = np.array([None])
        self.long_time = np.array([None])
        self.locs = np.array([None, None])
        for loc in self:
            self.locs = np.vstack((self.locs, loc.loc))
            self.long_time = np.concatenate((self.long_time, loc.ts))
            self.long_data = np.concatenate((self.long_data, loc.ys))
        self.long_data = self.long_data[1:]
        self.long_time = self.long_time[1:]
        self.locs = self.locs[1:]

def download_all_rlr(ns=range(1,2400), typ='monthly'):
    return RSLData(ns, typ=typ)

def argmaxcontsect(inds):
    """Given indices, returns start index and end index for largest continuous
    section
    """
    diff = inds[1:]-inds[:-1]
    cumsum = cumsummax = icurr = imax = 0

    for i, x in enumerate(diff):
        if x == 1:
            cumsum += 1
            if cumsum > cumsummax:
                cumsummax = cumsum
                imax = icurr
        else:
            cumsum = 0
            icurr = i+1

    return inds[imax], inds[imax+cumsummax]

def corrcoef(loc1, loc2, shift=0):
    """Calculate the correlation between two RLR locations. Can shift one
    location in time by shift keyword.
    """

    filt1 = loc1.copy()
    filt2 = loc2.copy()

    filt1[:,0] += shift

    maxyr = min(filt1[:,0].max(), filt2[:,0].max())
    minyr = max(filt1[:,0].min(), filt2[:,0].min())
    filt1 = filt1.time_filter(minyr, maxyr)
    filt2 = filt2.time_filter(minyr, maxyr)

    inds = np.intersect1d(filt1.inds, filt2.inds)
    return pearsonr(filt1[inds,1], filt2[inds,1])[0]
<commit_msg>Mean referenced predections to data. Consider including inds for missing data in AbstractDatum so it doesn't have to be everywhere else.<commit_after>import numpy as np
import numpy.fft as fft
import re

from urllib2 import urlopen, HTTPError
from scipy.signal import get_window
from scipy.stats import pearsonr
from giapy.data_tools.abstractDataClasses import AbsGeoTimeSeries, \
                                                AbsGeoTimeSeriesContainer

def calcRSL(sim, rsldata, smooth=True, noise=0):
    #TODO This function shouldn't have to know what's inside any object, let
    # alone a complicated one, like sim. Consider calculating rsl first and
    # passing it and calculated times in (ts, rsl).
    # To reference to present day
    #u0 = sim.inputs.harmTrans.spectogrd(sim['topo'][-1])
    u0 = sim['sstopo'][-1]

    uAtLocs = []
    for ut in sim['sstopo']:
        ut = u0 - ut
        interpfunc = sim.inputs.grid.create_interper(ut.T*1000)
        uAtLocs.append(interpfunc.ev(rsldata.lons, rsldata.lats))

    uAtLocs = np.array(uAtLocs).T

    calcTimes = sim['sstopo'].outTimes
    if np.all(np.diff(calcTimes)<0):
        reverse = True
    else:
        reverse=False

    data = {}
    for uAtLoc, loc in zip(uAtLocs, rsldata):
        if smooth:
            ts = np.union1d(np.sort(loc.ts), np.linspace(0, loc.ts.max()))
        else:
            ts = np.sort(loc.ts)

        # Sim objects output times in thousands of years relative to present
        tsint = (2015 - ts)/1000.

        if reverse:
            timeseries = np.array([ts,
                    np.interp(tsint,
                                sim['sstopo'].outTimes[::-1], uAtLoc[::-1])]).T
        else:
            timeseries = np.array([ts,
                    np.interp(tsint,
                                sim['sstopo'].outTimes, uAtLoc)]).T

        # Mean reference, so that mean of data and prediction are same.
        timeseries[:,1] += np.mean(loc.ys[loc.inds]) - np.mean(timeseries[:,1])

        #Apply the noise (default 0)
        scatter = noise*np.random.randn(len(timeseries[:,1]))
        timeseries[:,1] = timeseries[:,1] + scatter
        data[loc.stid] = RSLDatum(timeseries,
                                        lat=loc.lat,
                                        lon=loc.lon,
                                        sitename=loc.sitename,
                                        stid=loc.stid)

    return RSLData(data=data)

class RLR(np.ndarray):
    """Download a Revised Local Reference for sea level from the PSMSL.

    Data is stored as an ndarray, with the first column ([:,0]) storing the
    dates of the records (in yearly decimal format, e.g., 1947.23), the second
    ([:,1]) the sea level records, and the third and fourth error information.

    Missing data points are stored as -99999, and a list of the indices of
    non-absent data are stored in RLR.ind.

    Parameters
    ----------
    num : int
        The unique locator code for the RLR data to download
    typ : 'monthly' or 'annual'
        Download either monthly or annual data

    Reference
    ---------
    Simon J. Holgate, Andrew Matthews, Philip L. Woodworth, Lesley J. Rickards,
    Mark E. Tamisiea, Elizabeth Bradshaw, Peter R. Foden, Kathleen M. Gordon,
    Svetlana Jevrejeva, and Jeff Pugh (2013) New Data Systems and Products at
    the Permanent Service for Mean Sea Level. Journal of Coastal Research:
    Volume 29, Issue 3: pp. 493 - 504. doi:10.2112/JCOASTRES-D-12-00175.1.
    """
    def __new__(cls, num, typ='monthly', *args):
        # option parsing
        if typ not in ['monthly', 'annual']:
            raise ValueError("typ must be 'monthly' or 'annual'")

        # the data array
        _url = 'http://www.psmsl.org/data/obtaining/rlr.'+typ+'.data/'
        try:
            _response = urlopen(_url+str(num)+'.rlrdata')
        except HTTPError:
            raise ValueError("{0} is an invalid station id".format(num))

        # Format of data is: date | mean sea level | missing record | flag
        if typ=='annual':
            # In annual data, third column indicates whether the record was
            # missing 30 days ('Y') or not ('N')
            converters = {2: lambda s: (0 if s=='N' else 1)}
        else:
            converters = None
        _data = np.loadtxt(_response, delimiter=';', converters=converters)
        # Protect against sites with only one data point
        _data = np.atleast_2d(_data)

        # the metadata
        _murl = 'http://www.psmsl.org/data/obtaining/stations/'
        _meta = urlopen(_murl+str(num)+'.php')
        _txt = _meta.read()
        _sitename = re.findall('
(.*)
', _txt, flags=re.DOTALL)[0]
        _tmarker = '(.*)'
        _m = re.findall(_tmarker, _txt, flags=re.DOTALL)
        _rows = re.findall('
.*?
', _m[0], flags=re.DOTALL)
        _table = [[cell[4:-5] for cell in
                    re.findall('.*?	', row, flags=re.DOTALL)]
                    for row in _rows]

        if _table[3][0] in ['GLOSS ID', 'GLOSS ID:']:       # some locs have a
            _metadata = {'sitename' : _sitename,            # gloss id
                         'stid' : int(_table[0][1]),
                         'lat'  : float(_table[1][1]),
                         'lon'  : float(_table[2][1]),
                         #'gloss': float(_table[3][1][-7:-4]),
                         'coastcode' : int(_table[4][1]),
                         'stcode' : int(_table[5][1]),
                         'country' : _table[6][1],
                         'yrmin' : int(_table[7][1][:4]),
                         'yrmax' : int(_table[7][1][-4:]),
                         'comp' : float(_table[8][1]),
                         'frcode' : _table[9][1],
                         'update' : _table[10][1],
                         'type' : typ
                         }
        else:                                               # some don't
            _metadata = {'sitename' : _sitename,
                         'stid' : int(_table[0][1]),
                         'lat'  : float(_table[1][1]),
                         'lon'  : float(_table[2][1]),
                         'coastcode' : int(_table[3][1]),
                         'stcode' : int(_table[4][1]),
                         'country' : _table[5][1],
                         'yrmin' : int(_table[6][1][:4]),
                         'yrmax' : int(_table[6][1][-4:]),
                         'comp' : float(_table[7][1]),
                         'frcode' : _table[8][1],
                         'update' : _table[9][1],
                         'type' : typ
                         }

        # store indices of time where data are absent
        _inds = np.where(_data[:,1]!=-99999)[0]

        obj = np.asarray(_data).view(cls)
        obj.metadata = _metadata
        obj.inds = _inds

        return obj

    def __array_finalize__(self, obj):
        if obj is None: return
        self.metadata = getattr(obj, 'metadata', None)
        self.inds = getattr(obj, 'inds', None)

    def __reduce__(self):
        # Get the parent's __reduce__ tuple
        pickled_state = super(RLR, self).__reduce__()
        # Create our own tuple to pass to __setstate__
        new_state = pickled_state[2] + (self.inds,self.metadata)
        # Return a tuple that replaces the
        # parent's __setstate__ tuple with our
        # own
        return (pickled_state[0], pickled_state[1], new_state)

    def __setstate__(self, state):
        self.inds, self.metadata = state[-2], state[-1]  # Set the info attribute
        # Call the parent's __setstate__ with the other tuple elements.
        super(RLR, self).__setstate__(state[0:-2])

    def __str__(self):
        return 'RLR( {0}, {1} )'.format(self.metadata['sitename'],
                        self.metadata['type'])

    def trend(self, coeffs=False):
        """Calculate the best linear fit for data, returns coeffs or line.
        """
        inds = self.inds
        A = np.array([self[inds,0], np.ones(len(inds))])
        w = np.linalg.lstsq(A.T, self[inds,1])[0]
        if coeffs:
            return w
        else:
            return w[0]*self[:,0]+w[1]

    def detrended(self):
        """Remove the linear trend from the RLR data.
        """
        inds = self.inds
        result = self.copy()#[self.inds,:]
        result[inds,1] = result[inds,1]-self.trend()[inds]#[self.inds, :]
        return result

    def window_filter(self, window):
        """Filters the data using a window in Fourier space.

        See docstring for scipy.signal.window
        """
        if self.metadata['type'] == 'monthly':
            d = 0.08333
        else:
            d = 1

        inds = self.inds
        result = self.copy()#[self.inds,:]

        fs = fft.fftshift(fft.fftfreq(len(self.inds), d=d))
        self_fft = fft.fftshift(fft.fft(self.detrended()[inds, 1]))
        win = get_window(window, len(inds))
        result[inds,1] = fft.ifft(fft.ifftshift(win*self_fft))
        result[inds,1] = result[inds,1] + self.trend()[inds]#[self.inds]

        return result

    def runavg_filter(self, w, nmiss=0, trimmed=True, align='left'):
        """Computes the running average of w samples.

        When a gap occurs in the data, it is given zero weight in the filter,
        and the weighting is renormalized. Thus, some gaps can be filled. If
        the gap cannot be filled, it is flagged as missing.

        Mitchum 1987 (cited by Clarke 1992)
        """
        result = self.copy()

        if nmiss<0:
            raise ValueError('nmiss must be non-negative')

        if align=='left':
            start = w
            finish = self.shape[0]
        elif align=='cent':
            start = w/2
            finish = self.shape[0]-w/2
        elif align=='right':
            start = 0
            finish = self.shape[0]-w

        for i in range(start, finish):
            if align=='left':
                block = self[i-w:i, 1]
            elif align=='cent':
                block = self[i-w/2:i+w/2, 1]
            elif align=='right':
                block = self[i:i+w, 1]
            block = block[block!=-99999]
            if len(block)<=nmiss:
                result[i, 1] = -99999
            else:
                result[i, 1] = block.mean()

        # reset the data not absent indices
        if trimmed:
            if align=='left':
                result = result[w:,:]
            elif align=='cent':
                result = result[w/2:-w/2,:]
            elif align=='right':
                result = result[:-w,:]

        result.inds = np.where(result[:,1]!=-99999)[0]
        return result

    def time_filter(self, tmin, tmax):
        """Filter the data to a specific time window [tmin, tmax]"""
        filtinds = np.where(np.logical_and(self[:,0]>tmin, self[:,0]<=tmax))
        result = self[filtinds]
        # correct data not absent indices
        result.inds = np.where(result[:,1]!=-99999)[0]

        return result

    def plot(self, ax, *args, **kwargs):
        """Plots the data on ax, skipping missing points"""
        ax.plot(self[self.inds, 0], self[self.inds, 1], *args, **kwargs)
        return ax

class RSLDatum(AbsGeoTimeSeries):
    def __repr__(self):
        return 'RLR( {0}, {1} )'.format(self.sitename, self.type)

    @property
    def inds(self):
        return self.timeseries.inds

class RSLData(AbsGeoTimeSeriesContainer):
    def __init__(self, nbrs=None, data=None, typ='monthly'):
        if nbrs is not None:
            self.data = {}
            self.download(nbrs, typ)
        elif data is not None:
            self.data = data
        self.form_long_vectors()

    def __getitem__(self, key):
        return self.data.__getitem__(key)

    def __iter__(self):
        return self.data.itervalues()

    def __len__(self):
        return len(self.data)

    def download(self, nbrs, typ):
        for n in nbrs:
            try:
                loc = RLR(n, typ=typ)
                self.data[n] = RSLDatum(loc, **loc.metadata)
            except ValueError as e:
                print e.message
                continue

    def filter_by_time(self, tmin, tmax, replace=False):
        result = {}
        for n, loc in self.data.iteritems():
            if loc.metadata['yrmin']<=tmin and loc.metadata['yrmax']>=tmax:
                result[n] = loc
        if replace:
            self.data = result
        else:
            return RSLData(nbrs=None, data=result)

    def form_long_vectors(self):
        """Update the long lists: long_data, long_time, and locs with currently
        encapsulated data.

        Certain numbers are stored twice, for convenience in calculating the
        residuals, in the form of long lists. These are every emergence height,
        every emergence time, and every lon,lat pair. The one-time storage
        overhead is worth the time saved from not recreating these lists on
        every iteration of an inversion.
        """

        self.long_data = np.array([None])
        self.long_time = np.array([None])
        self.locs = np.array([None, None])
        for loc in self:
            self.locs = np.vstack((self.locs, loc.loc))
            self.long_time = np.concatenate((self.long_time, loc.ts))
            self.long_data = np.concatenate((self.long_data, loc.ys))
        self.long_data = self.long_data[1:]
        self.long_time = self.long_time[1:]
        self.locs = self.locs[1:]

def download_all_rlr(ns=range(1,2400), typ='monthly'):
    return RSLData(ns, typ=typ)

def argmaxcontsect(inds):
    """Given indices, returns start index and end index for largest continuous
    section
    """
    diff = inds[1:]-inds[:-1]
    cumsum = cumsummax = icurr = imax = 0

    for i, x in enumerate(diff):
        if x == 1:
            cumsum += 1
            if cumsum > cumsummax:
                cumsummax = cumsum
                imax = icurr
        else:
            cumsum = 0
            icurr = i+1

    return inds[imax], inds[imax+cumsummax]

def corrcoef(loc1, loc2, shift=0):
    """Calculate the correlation between two RLR locations. Can shift one
    location in time by shift keyword.
    """

    filt1 = loc1.copy()
    filt2 = loc2.copy()

    filt1[:,0] += shift

    maxyr = min(filt1[:,0].max(), filt2[:,0].max())
    minyr = max(filt1[:,0].min(), filt2[:,0].min())
    filt1 = filt1.time_filter(minyr, maxyr)
    filt2 = filt2.time_filter(minyr, maxyr)

    inds = np.intersect1d(filt1.inds, filt2.inds)
    return pearsonr(filt1[inds,1], filt2[inds,1])[0]
<|endoftext|>


<commit_before># Do not mess with the default types (0-5). This is passed to the Object
# model's 'choices' argument.
OBJECT_TYPES = (
   (0, 'NOTHING'),
   (1, 'PLAYER'),
   (2, 'ROOM'),
   (3, 'THING'),
   (4, 'EXIT'),
   (5, 'GOING'),
   (6, 'GARBAGE'),
)

# Hate to duplicate the above, but it's the easiest way.
OTYPE_NOTHING = 0
OTYPE_PLAYER = 1
OTYPE_ROOM = 2
OTYPE_THING = 3
OTYPE_EXIT = 4
OTYPE_GOING = 5
OTYPE_GARBAGE = 6

# This is a list of flags that the server actually uses. Anything not in this
# list is a custom flag.
SERVER_FLAGS = ["CONNECTED", "DARK", "FLOATING", "GAGGED", "HAVEN", "OPAQUE", "SAFE", "SLAVE", "SUSPECT", "TRANSPARENT"]

# These flags are not saved.
NOSAVE_FLAGS = ["CONNECTED"]

# These flags can't be modified by players.
NOSET_FLAGS = ["CONNECTED"]

# These attribute names can't be modified by players.
NOSET_ATTRIBS = ["MONEY", "ALIAS", "LASTPAGED", "CHANLIST", "LAST", "LASTSITE"]

# These attributes don't show up on objects when examined.
HIDDEN_ATTRIBS = ["CHANLIST"]

# Server version number.
EVENNIA_REVISION = '$Rev$'
EVENNIA_VERSION = 'Alpha'

# The message to show when the user lacks permissions for something.
NOPERMS_MSG = "You do not have the necessary permissions to do that."

# Message seen when object doesn't control the other object.
NOCONTROL_MSG = "You don't have authority over that object."
<commit_msg>Finalized EVENNIA_VERSION to include svn revision<commit_after># Do not mess with the default types (0-5). This is passed to the Object
# model's 'choices' argument.
OBJECT_TYPES = (
   (0, 'NOTHING'),
   (1, 'PLAYER'),
   (2, 'ROOM'),
   (3, 'THING'),
   (4, 'EXIT'),
   (5, 'GOING'),
   (6, 'GARBAGE'),
)

# Hate to duplicate the above, but it's the easiest way.
OTYPE_NOTHING = 0
OTYPE_PLAYER = 1
OTYPE_ROOM = 2
OTYPE_THING = 3
OTYPE_EXIT = 4
OTYPE_GOING = 5
OTYPE_GARBAGE = 6

# This is a list of flags that the server actually uses. Anything not in this
# list is a custom flag.
SERVER_FLAGS = ["CONNECTED", "DARK", "FLOATING", "GAGGED", "HAVEN", "OPAQUE", "SAFE", "SLAVE", "SUSPECT", "TRANSPARENT"]

# These flags are not saved.
NOSAVE_FLAGS = ["CONNECTED"]

# These flags can't be modified by players.
NOSET_FLAGS = ["CONNECTED"]

# These attribute names can't be modified by players.
NOSET_ATTRIBS = ["MONEY", "ALIAS", "LASTPAGED", "CHANLIST", "LAST", "LASTSITE"]

# These attributes don't show up on objects when examined.
HIDDEN_ATTRIBS = ["CHANLIST"]

# Server version number.
EVENNIA_REVISION = '$Rev$'
# Clip out the SVN keyword information
EVENNIA_VERSION = 'Alpha-' + EVENNIA_REVISION[6:-2]

# The message to show when the user lacks permissions for something.
NOPERMS_MSG = "You do not have the necessary permissions to do that."

# Message seen when object doesn't control the other object.
NOCONTROL_MSG = "You don't have authority over that object."
<|endoftext|>


<commit_before>import subprocess

from gitflow import utils
from gitflow.context import Context
from gitflow.procedures import get_command_context

def call(context: Context):
    command_context = get_command_context(
        context=context,
        object_arg=utils.get_or_default(context.args, '<object>', None)
    )

    for stage in context.config.build_stages:
        for step in stage.steps:
            for command in step.commands:
                proc = subprocess.Popen(args=command,
                                 stdin=subprocess.PIPE,
                                 cwd=context.root)
                proc.wait()

    return command_context.result
<commit_msg>error handling<commit_after>import os
import subprocess

from gitflow import utils, _
from gitflow.context import Context
from gitflow.procedures import get_command_context

def call(context: Context):
    command_context = get_command_context(
        context=context,
        object_arg=utils.get_or_default(context.args, '<object>', None)
    )

    for stage in context.config.build_stages:
        for step in stage.steps:
            for command in step.commands:
                try:
                    proc = subprocess.Popen(args=command,
                                            stdin=subprocess.PIPE,
                                            cwd=context.root)
                    proc.wait()
                    if proc.returncode != os.EX_OK:
                        command_context.fail(os.EX_DATAERR,
                                             _("Build failed."),
                                             _("Stage {stage}:{step} returned with an error.")
                                             .format(stage=stage.name, step=step.name))
                except FileNotFoundError as e:
                    command_context.fail(os.EX_DATAERR,
                                         _("Build failed."),
                                         _("Stage {stage}:{step} could not be executed.")
                                         .format(stage=stage.name, step=step.name))

    return command_context.result
<|endoftext|>