<commit_before>import re
from rest_framework import renderers

class KeyStringIOSRenderer(renderers.BaseRenderer):

	media_type	= 'text/plain'
	format		= 'ios'
	charset		= 'utf-8'
	key_name	= 'key'
	value_name	= 'value'
	attribution = '/* Generated by Localizr. \n(https://github.com/michaelhenry/Localizr)\n */\n\n'

	def process_key(self, key):
		return str(key)

	def process_value(self, value):
		v 	= str(value)
		v 	= v.replace("%s", "%@")
		return v

	def render(self, data, media_type=None, renderer_context=None):
		r 	=	 self.attribution
		r   +=  '\n'.join('\"%s\" = \"%s\";' % (self.process_value(val[self.key_name]),
			self.process_value(val[self.value_name])) for (val) in data)
		return r.encode(self.charset)

class KeyStringAndroidRenderer(renderers.BaseRenderer):

	media_type	= 'application/xml'
	format		= 'android'
	charset		= 'utf-8'
	key_name	= 'key'
	value_name	= 'value'
	attribution = '\n<!-- Generated by Localizr. \n(https://github.com/michaelhenry/Localizr) \n-->\n\n'

	def process_key(self, key):
		pattern 	= 	'[^0-9a-zA-Z]+'

		k 	= str(key).lower()
		k 	= re.sub(pattern, '_', k).lower()
		return k

	def process_value(self, value):
		v = str(value)
		v = v.replace("%@", "%s")
		return v

	def render(self, data, media_type=None, renderer_context=None):
		r 	=  '<resources>'
		r 	+= self.attribution
		r 	+= '\n'.join('\t<string name=\"%s\">%s</string>' % (self.process_key(val[self.key_name]),
			self.process_value(val[self.value_name])) for (val) in data)
		r 	+= '\n'
		r 	+= '</resources>'
		return r.encode(self.charset)
<commit_msg>support sequence pattern like %n$@ and %n$ns<commit_after>import re
from rest_framework import renderers

class KeyStringIOSRenderer(renderers.BaseRenderer):

	media_type	= 'text/plain'
	format		= 'ios'
	charset		= 'utf-8'
	key_name	= 'key'
	value_name	= 'value'
	attribution = '/* Generated by Localizr. \n(https://github.com/michaelhenry/Localizr)\n */\n\n'

	def process_key(self, key):
		return str(key)

	def process_value(self, value):

		v 	= str(value)
		v 	= v.replace("%s", "%@")

		#search for existing android sequence pattern and convert it to ios.
		sequence_pattern_regex_for_android = re.compile('([%]\d+[$]\d+[s])', re.S)
		v = sequence_pattern_regex_for_android.sub(lambda m: re.sub(r'\d+s', '@', m.group()), v)

		return v

	def render(self, data, media_type=None, renderer_context=None):
		r 	=	 self.attribution
		r 	+=  '\n'.join('\"%s\" = \"%s\";' % (self.process_value(val[self.key_name]),
			self.process_value(val[self.value_name])) for (val) in data)
		return r.encode(self.charset)

class KeyStringAndroidRenderer(renderers.BaseRenderer):

	media_type	= 'application/xml'
	format		= 'android'
	charset		= 'utf-8'
	key_name	= 'key'
	value_name	= 'value'
	attribution = '\n<!-- Generated by Localizr. \n(https://github.com/michaelhenry/Localizr) \n-->\n\n'

	def process_key(self, key):
		pattern 	= 	'[^0-9a-zA-Z]+'

		k 	= str(key).lower()
		k 	= re.sub(pattern, '_', k).lower()
		return k

	def process_value(self, value):

		v = str(value)
		v = v.replace("%@", "%s")

		#search for existing ios sequence pattern and convert it to android.
		sequence_pattern_regex_for_ios = re.compile('([%]\d[$][@])', re.S)
		v = sequence_pattern_regex_for_ios.sub(lambda m: m.group().replace('@',"1s"), v)

		return v

	def render(self, data, media_type=None, renderer_context=None):
		r 	=  '<resources>'
		r 	+= self.attribution
		r 	+= '\n'.join('\t<string name=\"%s\">%s</string>' % (self.process_key(val[self.key_name]),
			self.process_value(val[self.value_name])) for (val) in data)
		r 	+= '\n'
		r 	+= '</resources>'
		return r.encode(self.charset)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
"""
Usage:
  run.py --bucket-name=<BUCKET_NAME> --key=<KEY> [--delete-original]
  run.py -h | --help

Options:
  -h --help                     Show this message.
  --bucket-name=<BUCKET_NAME>   Name of the bucket where the job json file resides.
  --key=<KEY>                   Key of the job json file.
  --delete-original             Delete the original source key.

"""

import json
import subprocess
import os
import tempfile

import boto3
import docopt

def download_in_parallel(tasks):
    print(f"Starting download of {len(tasks)} images from s3")
    processes = [start_download_process(task) for task in tasks]
    images = [check_status(local_image_path, process, task) for process, task, local_image_path in processes]
    return images

def upload_in_parallel(images):
    print(f"Starting upload of {len(images)} images to s3")
    upload_processes = [
        start_upload_process(local_image_path, task)
        for task, local_image_path in images
    ]
    for process, local_image_path, upload_location in upload_processes:
        failure_message = f"Uploading from {local_image_path} to {upload_location} failed!"
        success_message = f"Uploading from {local_image_path} to {upload_location} succeeded"
        wait(process,
             success_message,
             failure_message)
        # TODO delete original file if --delete-original flag is set

def check_status(local_image_path, process, task):
    s3_source = build_s3_source(task)
    success_message = f"Downloading of {s3_source} to {local_image_path} succeeded"
    failure_message = f"Downloading of {s3_source} to {local_image_path} failed!"
    wait(process, success_message, failure_message)
    return task, local_image_path

def start_download_process(task):
    _, tmp_fp = tempfile.mkstemp()
    s3_source = build_s3_source(task)
    p = subprocess.Popen(["aws", "s3", "cp", s3_source, tmp_fp])
    return p, task, tmp_fp

def start_upload_process(local_image_path, task):
    s3_location = build_s3_destination(task)
    p = subprocess.Popen(["aws", "s3", "cp", local_image_path, s3_location])
    return p, local_image_path, s3_location

def build_s3_destination(task):
    s3_destination_bucket = task['destination']['bucket_name']
    s3_destination_path = task['destination']['object_path']
    s3_location = f"s3://{s3_destination_bucket}/{s3_destination_path}"
    return s3_location

def build_s3_source(task):
    s3_source_bucket = task["source"]["bucket_name"]
    s3_source_path = task["source"]["object_path"]
    s3_source = f"s3://{s3_source_bucket}/{s3_source_path}"
    return s3_source

def wait(process, success_message, failure_message):
    if process.wait() != 0:
        raise Exception(failure_message)
    else:
        print(success_message)

def embed_image_metadata(task, local_image_path):
    xmp_metadata = task["metadata"]["xmp"]
    arguments = [build_exiftool_argument(key, value) for key, value in xmp_metadata.items() if value is not None]
    sep_arguments = ["exiftool", "-m", "-sep", ","] + arguments
    sep_arguments.append(local_image_path)

    if subprocess.call(sep_arguments) != 0:
        raise Exception("Failed embedding metadata!")
    else:
        print(f"Metadata embedded successfully for {build_s3_source(task)}")

def build_exiftool_argument(key, value):
    if isinstance(value, list):
        value = ",".join(value)
    return f"-xmp:{key}={value}"

def split(parallelism, tasks):
    return [tasks[i:i + parallelism] for i in range(0, len(tasks), parallelism)]

def main():
    args = docopt.docopt(__doc__)

    job_bucket = args["--bucket-name"]
    job_s3_key = args["--key"]
    delete_original = bool(args['--delete-original'])
    parallelism = 10

    print(f"Downloading {job_s3_key}")
    client = boto3.client("s3")
    job_filename = os.path.basename(job_s3_key)
    client.download_file(Bucket=job_bucket,
                         Key=job_s3_key,
                         Filename=job_filename)

    with open(job_filename, "r") as job_file:
        job = json.load(job_file)

    images = []
    try:
        for split_tasks in split(parallelism, job['task_list']):
            images += download_in_parallel(split_tasks)

        print("Finished downloading s3 files!")

        for task, local_image_path in images:
            embed_image_metadata(task, local_image_path)

        for split_images in split(parallelism, images):
            upload_in_parallel(split_images)
    finally:
        for _, local_image_path in images:
            os.unlink(local_image_path)

    print("Finished uploading files to s3!")

if __name__ == '__main__':
    main()
<commit_msg>Space to trigger a rebuild<commit_after>#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
"""
Usage:
  run.py --bucket-name=<BUCKET_NAME> --key=<KEY> [--delete-original]
  run.py -h | --help

Options:
  -h --help                     Show this message.
  --bucket-name=<BUCKET_NAME>   Name of the bucket where the job json file resides.
  --key=<KEY>                   Key of the job json file.
  --delete-original             Delete the original source key.

"""

import json
import subprocess
import os
import tempfile

import boto3
import docopt

def download_in_parallel(tasks):
    print(f"Starting download of {len(tasks)} images from s3")
    processes = [start_download_process(task) for task in tasks]
    images = [check_status(local_image_path, process, task) for process, task, local_image_path in processes]
    return images

def upload_in_parallel(images):
    print(f"Starting upload of {len(images)} images to s3")
    upload_processes = [
        start_upload_process(local_image_path, task)
        for task, local_image_path in images
    ]
    for process, local_image_path, upload_location in upload_processes:
        failure_message = f"Uploading from {local_image_path} to {upload_location} failed!"
        success_message = f"Uploading from {local_image_path} to {upload_location} succeeded"
        wait(process,
             success_message,
             failure_message)
        # TODO delete original file if --delete-original flag is set

def check_status(local_image_path, process, task):
    s3_source = build_s3_source(task)
    success_message = f"Downloading of {s3_source} to {local_image_path} succeeded"
    failure_message = f"Downloading of {s3_source} to {local_image_path} failed!"
    wait(process, success_message, failure_message)
    return task, local_image_path

def start_download_process(task):
    _, tmp_fp = tempfile.mkstemp()
    s3_source = build_s3_source(task)
    p = subprocess.Popen(["aws", "s3", "cp", s3_source, tmp_fp])
    return p, task, tmp_fp

def start_upload_process(local_image_path, task):
    s3_location = build_s3_destination(task)
    p = subprocess.Popen(["aws", "s3", "cp", local_image_path, s3_location])
    return p, local_image_path, s3_location

def build_s3_destination(task):
    s3_destination_bucket = task['destination']['bucket_name']
    s3_destination_path = task['destination']['object_path']
    s3_location = f"s3://{s3_destination_bucket}/{s3_destination_path}"
    return s3_location

def build_s3_source(task):
    s3_source_bucket = task["source"]["bucket_name"]
    s3_source_path = task["source"]["object_path"]
    s3_source = f"s3://{s3_source_bucket}/{s3_source_path}"
    return s3_source

def wait(process, success_message, failure_message):
    if process.wait() != 0:
        raise Exception(failure_message)
    else:
        print(success_message)

def embed_image_metadata(task, local_image_path):
    xmp_metadata = task["metadata"]["xmp"]
    arguments = [build_exiftool_argument(key, value) for key, value in xmp_metadata.items() if value is not None]
    sep_arguments = ["exiftool", "-m", "-sep", ","] + arguments
    sep_arguments.append(local_image_path)

    if subprocess.call(sep_arguments) != 0:
        raise Exception("Failed embedding metadata!")
    else:
        print(f"Metadata embedded successfully for {build_s3_source(task)}")

def build_exiftool_argument(key, value):
    if isinstance(value, list):
        value = ",".join(value)
    return f"-xmp:{key}={value}"

def split(parallelism, tasks):
    return [tasks[i:i + parallelism] for i in range(0, len(tasks), parallelism)]

def main():
    args = docopt.docopt(__doc__)

    job_bucket = args["--bucket-name"]
    job_s3_key = args["--key"]
    delete_original = bool(args['--delete-original'])
    parallelism = 10

    print(f"Downloading {job_s3_key}")
    client = boto3.client("s3")

    job_filename = os.path.basename(job_s3_key)
    client.download_file(Bucket=job_bucket,
                         Key=job_s3_key,
                         Filename=job_filename)

    with open(job_filename, "r") as job_file:
        job = json.load(job_file)

    images = []
    try:
        for split_tasks in split(parallelism, job['task_list']):
            images += download_in_parallel(split_tasks)

        print("Finished downloading s3 files!")

        for task, local_image_path in images:
            embed_image_metadata(task, local_image_path)

        for split_images in split(parallelism, images):
            upload_in_parallel(split_images)
    finally:
        for _, local_image_path in images:
            os.unlink(local_image_path)

    print("Finished uploading files to s3!")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#coding=utf-8
from docker import Client
import time
import logging
from envir import config

class DockerOpt:
    def __init__(self):
        self.log = logging.getLogger(self.__class__.__name__)
        self.log.setLevel(logging.DEBUG)
        sh = logging.StreamHandler()
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        sh.setFormatter(formatter)
        # sh.setLevel(logging.INFO)
        self.log.addHandler(sh)
        app_config = config.read_app_config()
        self.app = app_config
        self.url = app_config['docker']['url']
        self.log.debug("create docker with %s", self.url)

    def gen_tag(self, branch, app_version, api_version):
        now = time.localtime()
        now_str = time.strftime("%Y%m%d%H%M%S", now)
        if str(branch).startswith("develop"):
            tag_name = api_version + "-" + app_version + "-d" + now_str
        elif str(branch).startswith("feature/"):
            tag_name = api_version + "-" + app_version + "-f" + now_str
        elif str(branch).startswith("release/"):
            tag_name = api_version + "-" + app_version + "-r" + now_str
        elif str(branch).startswith("hotfix/"):
            tag_name = api_version + "-" + app_version + "-h" + now_str

        return tag_name

    def gen_repository(self, registry, project_key, app_name):
        return str(registry) + "/" + str(project_key) + "/" + str(app_name)

    def build(self, path, tag):
        """
        Similar to the `docker interpreter`, interpreter a docker image
        :param path: context path include Dockerfile
        :param tag: image's tag
        :return: None
        """
        version = self.app['docker']['api']['version']
        cli = Client(base_url=self.url, version=str(version))
        response = [line for line in cli.build(path, tag)]
        self.log.info(response)

    def push_images(self, repository, tag=None):
        version = self.app['docker']['api']['version']
        cli = Client(base_url=self.url, version=str(version))
        response = [line for line in cli.push(repository, tag=tag, stream=True)]
        self.log.info(response)<commit_msg>added app configuration<commit_after>#coding=utf-8
from docker import Client
import time
import logging
from envir import config

class DockerOpt:
    def __init__(self):
        self.log = logging.getLogger(self.__class__.__name__)
        self.log.setLevel(logging.DEBUG)
        sh = logging.StreamHandler()
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        sh.setFormatter(formatter)
        # sh.setLevel(logging.INFO)
        self.log.addHandler(sh)
        app_config = config.read_app_config()
        self.app = app_config
        self.url = app_config['docker']['url']
        self.log.debug("create docker with %s", self.url)

    def gen_tag(self, branch, app_version, api_version):
        now = time.localtime()
        now_str = time.strftime("%Y%m%d%H%M%S", now)
        if str(branch).startswith("develop"):
            tag_name = api_version + "-" + app_version + "-d" + now_str
        elif str(branch).startswith("feature/"):
            tag_name = api_version + "-" + app_version + "-f" + now_str
        elif str(branch).startswith("release/"):
            tag_name = api_version + "-" + app_version + "-r" + now_str
        elif str(branch).startswith("hotfix/"):
            tag_name = api_version + "-" + app_version + "-h" + now_str

        return tag_name

    def gen_repository(self, registry, project_key, app_name):
        return str(registry) + "/" + str(project_key) + "/" + str(app_name)

    def build(self, path, tag):
        """
        Similar to the `docker interpreter`, interpreter a docker image
        :param path: context path include Dockerfile
        :param tag: image's tag
        :return: None
        """
        version = self.app['docker']['api']['version']
        cli = Client(base_url=self.url, version=str(version))
        response = [line for line in cli.build(path, tag)]
        self.log.info(response)
        self.log.info("successful build image with dockerImageTag=%s", str(tag).split(':')[1])

    def push_images(self, repository, tag=None):
        version = self.app['docker']['api']['version']
        cli = Client(base_url=self.url, version=str(version))
        response = [line for line in cli.push(repository, tag=tag, stream=True)]
        self.log.info(response)<|endoftext|>


<commit_before>from django import forms

from .models import Satellite

class DataSelectionForm(forms.Form):
    two_column_format = True

    title = forms.CharField(required=False, widget=forms.HiddenInput(attrs={'class': 'hidden_form_title'}))
    description = forms.CharField(required=False, widget=forms.HiddenInput(attrs={'class': 'hidden_form_description'}))
    satellite = forms.ModelChoiceField(
        queryset=Satellite.objects.all(), widget=forms.HiddenInput(attrs={'class': 'hidden_form_satellite'}))
    area_id = forms.CharField(widget=forms.HiddenInput(attrs={'class': 'hidden_form_id'}))

    latitude_min = forms.FloatField(
        label='Min Latitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    latitude_max = forms.FloatField(
        label='Max Latitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    longitude_min = forms.FloatField(
        label='Min Longitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    longitude_max = forms.FloatField(
        label='Max Longitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    time_start = forms.DateField(
        label='Start Date',
        widget=forms.DateInput(
            attrs={'class': 'datepicker field-divided',
                   'placeholder': '01/01/2010',
                   'required': 'required'}))
    time_end = forms.DateField(
        label='End Date',
        widget=forms.DateInput(
            attrs={'class': 'datepicker field-divided',
                   'placeholder': '01/02/2010',
                   'required': 'required'}))

    def __init__(self, *args, **kwargs):
        time_start = kwargs.pop('time_start', None)
        time_end = kwargs.pop('time_end', None)
        area = kwargs.pop('area', None)
        self.user_id = kwargs.pop('user_id', None)
        self.user_history = kwargs.pop('user_history', None)
        self.task_model_class = kwargs.pop('task_model_class', None)
        super(DataSelectionForm, self).__init__(*args, **kwargs)
        #meant to prevent this routine from running if trying to init from querydict.
        if time_start and time_end:
            self.fields['time_start'] = forms.DateField(
                initial=time_start.strftime("%m/%d/%Y"),
                label='Start Date',
                widget=forms.DateInput(attrs={'class': 'datepicker field-divided',
                                              'required': 'required'}))
            self.fields['time_end'] = forms.DateField(
                initial=time_end.strftime("%m/%d/%Y"),
                label='End Date',
                widget=forms.DateInput(attrs={'class': 'datepicker field-divided',
                                              'required': 'required'}))
        if area:
            self.fields['latitude_min'].widget.attrs.update({'min': area.latitude_min, 'max': area.latitude_max})
            self.fields['latitude_max'].widget.attrs.update({'min': area.latitude_min, 'max': area.latitude_max})
            self.fields['longitude_min'].widget.attrs.update({'min': area.longitude_min, 'max': area.longitude_max})
            self.fields['longitude_max'].widget.attrs.update({'min': area.longitude_min, 'max': area.longitude_max})

    def clean(self):
        cleaned_data = super(DataSelectionForm, self).clean()

        if cleaned_data.get('latitude_min') >= cleaned_data.get('latitude_max'):
            self.add_error(
                'latitude_min',
                "Please enter a valid pair of latitude values where the lower bound is less than the upper bound.")

        if cleaned_data.get('longitude_min') >= cleaned_data.get('longitude_max'):
            self.add_error(
                'longitude_min',
                "Please enter a valid pair of longitude values where the lower bound is less than the upper bound.")

        # The start and end time fields are removed in some apps (e.g. spectral_anomaly),
        # so we may not perform checks on these fields.
        time_start, time_end = cleaned_data.get('time_start'), cleaned_data.get('time_end')
        if time_start is not None and time_end is not None and (time_start >= time_end):
            self.add_error('time_start',
                           "Please enter a valid start and end time range where the start is before the end.")

        area = (cleaned_data.get('latitude_max') - cleaned_data.get('latitude_min')) * (
                cleaned_data.get('longitude_max') - cleaned_data.get('longitude_min'))

        # Limit the area allowed.
        max_area = 1
        if area > max_area:
            self.add_error('latitude_min', 'Tasks over an area greater than {} '
                                           'square degree(s) are not permitted.'.format(max_area))

        # Limit the time range allowed.
        if time_start is not None and time_end is not None:
            # For some apps, the time extent is not relevant to resource consumption
            # (e.g. if data is only loaded for the first and last years).
            from apps.coastal_change.models import CoastalChangeTask
            if self.task_model_class not in [CoastalChangeTask]:
                max_num_years = 5
                if self.check_time_range(time_start, time_end, max_num_years):
                    self.add_error('time_start', 'Tasks over a time range greater than {} '
                                                 'year(s) are not permitted.'.format(max_num_years))

        # Limit each user to 1 running task per app.
        num_running_tasks = self.task_model_class.get_queryset_from_history(
            self.user_history, complete=False).count()
        if num_running_tasks > 0:
            self.add_error(None, 'You may only run one task at a time.')

        return cleaned_data

    def check_time_range(self, time_start, time_end, max_num_years=5):
        """
        Determines if the time range [time_start, time_end] exceeds an upper bound.
        If so, returns `True`.

        Parameters
        ----------
        time_start, time_end: datetime.date
        max_num_years: float (default 5)
            The maximum number of years the time range can have.
        """
        year_diff = time_end.year - time_start.year
        month_diff = time_end.month - time_start.month
        day_diff = time_end.day - time_start.day
        return (year_diff > max_num_years) or \
               (year_diff == max_num_years and month_diff > 0) or \
               (year_diff == max_num_years and month_diff == 0 and day_diff > 0)<commit_msg>Now the max number of tasks per user defaults to infinite (no limit). This can be changed by setting the `MAX_NUM_TASKS_PER_USER` variable in `dc_algorithm.forms.DataSelectionForm.clean()` to a positive integer.<commit_after>from django import forms

from .models import Satellite

class DataSelectionForm(forms.Form):
    two_column_format = True

    title = forms.CharField(required=False, widget=forms.HiddenInput(attrs={'class': 'hidden_form_title'}))
    description = forms.CharField(required=False, widget=forms.HiddenInput(attrs={'class': 'hidden_form_description'}))
    satellite = forms.ModelChoiceField(
        queryset=Satellite.objects.all(), widget=forms.HiddenInput(attrs={'class': 'hidden_form_satellite'}))
    area_id = forms.CharField(widget=forms.HiddenInput(attrs={'class': 'hidden_form_id'}))

    latitude_min = forms.FloatField(
        label='Min Latitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    latitude_max = forms.FloatField(
        label='Max Latitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    longitude_min = forms.FloatField(
        label='Min Longitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    longitude_max = forms.FloatField(
        label='Max Longitude',
        widget=forms.NumberInput(attrs={'class': 'field-divided',
                                        'step': "any",
                                        'required': 'required'}))
    time_start = forms.DateField(
        label='Start Date',
        widget=forms.DateInput(
            attrs={'class': 'datepicker field-divided',
                   'placeholder': '01/01/2010',
                   'required': 'required'}))
    time_end = forms.DateField(
        label='End Date',
        widget=forms.DateInput(
            attrs={'class': 'datepicker field-divided',
                   'placeholder': '01/02/2010',
                   'required': 'required'}))

    def __init__(self, *args, **kwargs):
        time_start = kwargs.pop('time_start', None)
        time_end = kwargs.pop('time_end', None)
        area = kwargs.pop('area', None)
        self.user_id = kwargs.pop('user_id', None)
        self.user_history = kwargs.pop('user_history', None)
        self.task_model_class = kwargs.pop('task_model_class', None)
        super(DataSelectionForm, self).__init__(*args, **kwargs)
        #meant to prevent this routine from running if trying to init from querydict.
        if time_start and time_end:
            self.fields['time_start'] = forms.DateField(
                initial=time_start.strftime("%m/%d/%Y"),
                label='Start Date',
                widget=forms.DateInput(attrs={'class': 'datepicker field-divided',
                                              'required': 'required'}))
            self.fields['time_end'] = forms.DateField(
                initial=time_end.strftime("%m/%d/%Y"),
                label='End Date',
                widget=forms.DateInput(attrs={'class': 'datepicker field-divided',
                                              'required': 'required'}))
        if area:
            self.fields['latitude_min'].widget.attrs.update({'min': area.latitude_min, 'max': area.latitude_max})
            self.fields['latitude_max'].widget.attrs.update({'min': area.latitude_min, 'max': area.latitude_max})
            self.fields['longitude_min'].widget.attrs.update({'min': area.longitude_min, 'max': area.longitude_max})
            self.fields['longitude_max'].widget.attrs.update({'min': area.longitude_min, 'max': area.longitude_max})

    def clean(self):
        cleaned_data = super(DataSelectionForm, self).clean()

        if cleaned_data.get('latitude_min') >= cleaned_data.get('latitude_max'):
            self.add_error(
                'latitude_min',
                "Please enter a valid pair of latitude values where the lower bound is less than the upper bound.")

        if cleaned_data.get('longitude_min') >= cleaned_data.get('longitude_max'):
            self.add_error(
                'longitude_min',
                "Please enter a valid pair of longitude values where the lower bound is less than the upper bound.")

        # The start and end time fields are removed in some apps (e.g. spectral_anomaly),
        # so we may not perform checks on these fields.
        time_start, time_end = cleaned_data.get('time_start'), cleaned_data.get('time_end')
        if time_start is not None and time_end is not None and (time_start >= time_end):
            self.add_error('time_start',
                           "Please enter a valid start and end time range where the start is before the end.")

        area = (cleaned_data.get('latitude_max') - cleaned_data.get('latitude_min')) * (
                cleaned_data.get('longitude_max') - cleaned_data.get('longitude_min'))

        # Limit the area allowed.
        max_area = 1
        if area > max_area:
            self.add_error('latitude_min', 'Tasks over an area greater than {} '
                                           'square degree(s) are not permitted.'.format(max_area))

        # Limit the time range allowed.
        if time_start is not None and time_end is not None:
            # For some apps, the time extent is not relevant to resource consumption
            # (e.g. if data is only loaded for the first and last years).
            from apps.coastal_change.models import CoastalChangeTask
            if self.task_model_class not in [CoastalChangeTask]:
                max_num_years = 5
                if self.check_time_range(time_start, time_end, max_num_years):
                    self.add_error('time_start', 'Tasks over a time range greater than {} '
                                                 'year(s) are not permitted.'.format(max_num_years))

        # Limit each user to 1 running task per app.
        # This is the maximum number of tasks per user across all apps.
        # A value of `None` indicates no limit.
        MAX_NUM_TASKS_PER_USER = None
        if MAX_NUM_TASKS_PER_USER is not None:
            num_running_tasks = self.task_model_class.get_queryset_from_history(
                self.user_history, complete=False).count()
            if num_running_tasks >= MAX_NUM_TASKS_PER_USER:
                self.add_error(None, 'You may only run {} task(s) at a time.'.format(MAX_NUM_TASKS_PER_USER))

        return cleaned_data

    def check_time_range(self, time_start, time_end, max_num_years=5):
        """
        Determines if the time range [time_start, time_end] exceeds an upper bound.
        If so, returns `True`.

        Parameters
        ----------
        time_start, time_end: datetime.date
        max_num_years: float (default 5)
            The maximum number of years the time range can have.
        """
        year_diff = time_end.year - time_start.year
        month_diff = time_end.month - time_start.month
        day_diff = time_end.day - time_start.day
        return (year_diff > max_num_years) or \
               (year_diff == max_num_years and month_diff > 0) or \
               (year_diff == max_num_years and month_diff == 0 and day_diff > 0)<|endoftext|>


<commit_before>#Author : jeonghoonkang https://github.com/jeonghoonkang

import sys
sys.path.append("./lib")
from lcd import *

if __name__== "__main__" :
  lcd_init()
  LCDoff()
  pinkLCDon()
  yellowLCDon()
  str = "Yellow?"
  while True :
    lcd_string('Print Test' ,LCD_LINE_1,1)
    lcd_string('%s' % (str),LCD_LINE_2,1)
    time.sleep(1.5)

<commit_msg>Update lcd_app.py<commit_after>#Author : jeonghoonkang https://github.com/jeonghoonkang
# -*- coding: utf-8 -*-

import sys
sys.path.append("./lib")
from lcd import *

if __name__== "__main__" :
  lcd_init()
  LCDoff()
  pinkLCDon()
  yellowLCDon()
  str = "Yellow?"
  while True :
    lcd_string('Print Test' ,LCD_LINE_1,1)
    lcd_string('%s' % (str),LCD_LINE_2,1)
    time.sleep(1.5)

<|endoftext|>


<commit_before># Copyright 2016 United States Government as represented by the Administrator
# of the National Aeronautics and Space Administration. All Rights Reserved.
#
# Portion of this code is Copyright Geoscience Australia, Licensed under the
# Apache License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the License
# at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# The CEOS 2 platform is licensed under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from django import forms

import datetime

from data_cube_ui.models import Baseline

from data_cube_ui.forms import GeospatialForm as GeospatialFormBase

"""
File designed to house the different forms for taking in user input in the web application.  Forms
allow for input validation and passing of data.  Includes forms for creating Queries to be ran.
"""

# Author: AHDS
# Creation date: 2016-06-23
# Modified by:
# Last modified date:

years_range = list(range(1990, datetime.datetime.now().year+1))

class DataSelectForm(forms.Form):
    """
    Django form to be created for selecting information and validating input for:
        result_type
        band_selection
        title
        description
    """

    title = forms.CharField(widget=forms.HiddenInput())
    description = forms.CharField(widget=forms.HiddenInput())

    months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
    #index+1 so jan==1
    months_sel = [(index+1, month) for index, month in enumerate(months)]
    baseline_selection = forms.MultipleChoiceField(help_text='Select the month(s) that will be used to create the baseline. Scenes from the selected months that occur before the selected scene will be used to form a baseline.', label="Baseline Period:", choices=months_sel, widget=forms.SelectMultiple(attrs={'class': 'field-long tooltipped'}))

class GeospatialForm(GeospatialFormBase):
    def __init__(self, acquisition_list=None, *args, **kwargs):
        super(GeospatialForm, self).__init__(*args, **kwargs)
        self.fields.pop('time_start')
        self.fields.pop('time_end')
        scene_sel = reversed([(str(index)+"-"+date.strftime("%Y/%m/%d %H:%M UTC"), date.strftime("%Y/%m/%d %H:%M UTC")) for index, date in enumerate(acquisition_list)])
        self.fields['scene_selection'] = forms.ChoiceField(help_text='Select a scene to perform NDVI differencing on.', label="Scene Selection:", choices=scene_sel, widget=forms.Select(attrs={'class': 'field-long tooltipped'}))
<commit_msg>Fix for NDVI anomaly creation issue found by mplatt<commit_after># Copyright 2016 United States Government as represented by the Administrator
# of the National Aeronautics and Space Administration. All Rights Reserved.
#
# Portion of this code is Copyright Geoscience Australia, Licensed under the
# Apache License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the License
# at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# The CEOS 2 platform is licensed under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from django import forms

import datetime

from data_cube_ui.models import Baseline

from data_cube_ui.forms import GeospatialForm as GeospatialFormBase

"""
File designed to house the different forms for taking in user input in the web application.  Forms
allow for input validation and passing of data.  Includes forms for creating Queries to be ran.
"""

# Author: AHDS
# Creation date: 2016-06-23
# Modified by:
# Last modified date:

years_range = list(range(1990, datetime.datetime.now().year+1))

class DataSelectForm(forms.Form):
    """
    Django form to be created for selecting information and validating input for:
        result_type
        band_selection
        title
        description
    """

    title = forms.CharField(widget=forms.HiddenInput())
    description = forms.CharField(widget=forms.HiddenInput())

    months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
    #index+1 so jan==1
    months_sel = [(index+1, month) for index, month in enumerate(months)]
    baseline_selection = forms.MultipleChoiceField(help_text='Select the month(s) that will be used to create the baseline. Scenes from the selected months that occur before the selected scene will be used to form a baseline.', label="Baseline Period:", choices=months_sel, widget=forms.SelectMultiple(attrs={'class': 'field-long tooltipped'}))

class GeospatialForm(GeospatialFormBase):
    def __init__(self, acquisition_list=None, *args, **kwargs):
        super(GeospatialForm, self).__init__(*args, **kwargs)
        self.fields.pop('time_start')
        self.fields.pop('time_end')
        scene_sel = [(str(index)+"-"+date.strftime("%Y/%m/%d %H:%M UTC"), date.strftime("%Y/%m/%d %H:%M UTC")) for index, date in enumerate(acquisition_list)]
        self.fields['scene_selection'] = forms.ChoiceField(help_text='Select a scene to perform NDVI differencing on.', label="Scene Selection:", choices=scene_sel, widget=forms.Select(attrs={'class': 'field-long tooltipped'}))
<|endoftext|>


<commit_before>import Tkinter as tk
import ProjectileMotion as pm
import matplotlib.pyplot as plt
from PMPlots import *

class ProjectileMotionGUI:

    PLOT_TYPES = ['Y-position vs. X-position', 'X-Position vs. Time', \
                    'Y-Position vs. Time', 'X-Force vs. Time', 'Y-Force vs. Time', \
                    'X-Momentum vs. Time', 'Y-Momentum vs. Time', 'Kinetic Energy vs. Time', \
                    'Potential Energy vs. Time', 'Total Energy vs. Time']

    PLOT_NAME_SHORTCUTS = ['y_vs_x', 'x_vs_t', 'y_vs_t', 'fx_vs_t', 'fy_vs_t', \
                           'px_vs_t', 'py_vs_t', 'K_vs_t', 'U_vs_t', 'E_vs_t']

    def __init__(self, master):
        self.master = master
        master.title("Projectile Motion Dynamics") # set title for main window

        ### BASIC PARAMETER FRAME ###
	label = tk.Label(text="Basic Parameters", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # projectile motion object
        self.pm = pm.ProjectileMotion()

        # Populate basic parameter fields for projectile motiom
        self.basicParamEntries = self.loadFields(self.pm.basicParams, self.pm.basicParamsUnits)

        ### AIR RESISTANCE FRAME ###
	label = tk.Label(text="Drag Parameters", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # Button to include air resistance (or not)
        self.var = tk.IntVar()
        cb = tk.Checkbutton(text="Include air resistance", justify=tk.LEFT, variable=self.var, command=self.revealOptions)
        cb.pack(side=tk.TOP, anchor=tk.W)

        # Populate drag/air-resistance fields with a "disabled" default
        self.dragParamEntries = self.loadFields(self.pm.dragParams, self.pm.dragParamsUnits, 'disabled')

        label = tk.Label(text="Generate Plots", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # Add plot type check buttons
        self.plotChoices()

        # run simulation button (the command should load final choices and then run sim)
        rb = tk.Button(master, text='Run Simulation', command=self.runSimulation)
        rb.pack(side=tk.LEFT, padx=5, pady=5)

        # quit buttons
        qb = tk.Button(master, text='Quit Simulation', command=master.quit)
        qb.pack(side=tk.RIGHT, padx=5, pady=5)

    def loadFields(self, fields, unitFields, currState='normal'):
        #frame.bind()

        entries = []
        for key in fields:
            # define new frame (row) for each field
            frame = tk.Frame(self.master)
            frame.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5)

            # label for the current parameter
            label = tk.Label(frame, width=15, text=key, anchor='w')
            label.pack(side=tk.LEFT)

            # entry object which ingests a value for current parameter
            entry = tk.Entry(frame)
            entry.insert(tk.END, fields[key])
            entry.pack(side=tk.LEFT, expand=tk.YES, fill=tk.X)
            entry.config(state=currState)

            # set units
            unitsLabel = tk.Label(frame, width=8, text=unitFields[key], anchor='w')
            unitsLabel.pack(side=tk.RIGHT)

            # append values to entries list
            entries.append( (key, entry) )

        return entries

    def revealOptions(self):
        for ent in self.dragParamEntries:
            if (self.var.get() == 0):
                ent[1].config(state='disabled')
            else:
                ent[1].config(state='normal')

    def plotChoices(self):
        # ordering of self.checkVarList same as PLOT_OPTIONS
        self.checkVarList = []
        for plotType in self.PLOT_TYPES:
            CheckVar = tk.IntVar()
            self.checkVarList.append(CheckVar)

            cb = tk.Checkbutton(text=plotType, variable=CheckVar, justify=tk.LEFT)
            cb.pack(side=tk.TOP, anchor=tk.W)

    def runSimulation(self):
        # This method will take last recorded values
        # and check box parameters and pass them
        # to the projectile motion object in order
        # for it to do computations. Then it will make plots.
        # Essentially, it is the workhorse method

        # append drag force fields to normal ones
        #allEntries = self.basicParamEntries + self.dragParamEntries

        # read in and set user values
        self.pm.setValues(self.basicParamEntries, 'basic')
        self.pm.setValues(self.dragParamEntries, 'drag')

        # ask if we are using drag force
        maxTime, maxRange, maxHeight = self.pm.evolve(self.var.get())
        print("Maximum Time = %.3f (s)" % maxTime)
        print("Maximum Range = %.3f (m)" % maxRange)
        print("Maximum Height = %.3f (m)" % maxHeight)

        ani_list = []
        for kk in np.arange(0, len(self.checkVarList)):
            if (self.checkVarList[kk].get() == 1):
                my_plots = PMPlots(self.pm, self.PLOT_NAME_SHORTCUTS[kk])
                ani_list.append(my_plots.runSimulation())

        # show all relevant plots and then clear params when finished
        plt.show()
        self.pm.clear()

### MAIN EXECUTABLE ###
if __name__ == "__main__":
    root = tk.Tk()
    my_gui = ProjectileMotionGUI(root)
    root.mainloop()
<commit_msg>Add files via upload<commit_after>import Tkinter as tk
import ProjectileMotion as pm
import matplotlib.pyplot as plt
from PMPlots import *

class ProjectileMotionGUI:

    PLOT_TYPES = ['Y-position vs. X-position', 'X-Position vs. Time', \
                    'Y-Position vs. Time', 'X-Force vs. Time', 'Y-Force vs. Time', \
                    'X-Momentum vs. Time', 'Y-Momentum vs. Time', 'Kinetic Energy vs. Time', \
                    'Potential Energy vs. Time', 'Total Energy vs. Time']

    PLOT_NAME_SHORTCUTS = ['y_vs_x', 'x_vs_t', 'y_vs_t', 'fx_vs_t', 'fy_vs_t', \
                           'px_vs_t', 'py_vs_t', 'K_vs_t', 'U_vs_t', 'E_vs_t']

    def __init__(self, master):
        self.master = master
        master.title("Projectile Motion Dynamics") # set title for main window

        ### BASIC PARAMETER FRAME ###
        label = tk.Label(text="Basic Parameters", fg="blue", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # projectile motion object
        self.pm = pm.ProjectileMotion()

        # Populate basic parameter fields for projectile motiom
        self.basicParamEntries = self.loadFields(self.pm.basicParams, self.pm.basicParamsUnits)

        ### AIR RESISTANCE FRAME ###
        label = tk.Label(text="Drag Parameters", fg="blue", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # Button to include air resistance (or not)
        self.var = tk.IntVar()
        cb = tk.Checkbutton(text="Include air resistance", justify=tk.LEFT, variable=self.var, command=self.revealOptions)
        cb.pack(side=tk.TOP, anchor=tk.W)

        # Populate drag/air-resistance fields with a "disabled" default
        self.dragParamEntries = self.loadFields(self.pm.dragParams, self.pm.dragParamsUnits, 'disabled')

        label = tk.Label(text="Generate Plots", fg="blue", width=15, anchor='w')
        label.pack(side=tk.TOP)
        label.config(font=("Arial", 18))

        # Add plot type check buttons
        self.plotChoices()

        # run simulation button (the command should load final choices and then run sim)
        rb = tk.Button(master, text='Run Simulation', command=self.runSimulation)
        rb.pack(side=tk.LEFT, padx=5, pady=5)

        # quit buttons
        qb = tk.Button(master, text='Quit Simulation', command=master.quit)
        qb.pack(side=tk.RIGHT, padx=5, pady=5)

    def loadFields(self, fields, unitFields, currState='normal'):
        #frame.bind()

        entries = []
        for key in fields:
            # define new frame (row) for each field
            frame = tk.Frame(self.master)
            frame.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5)

            # label for the current parameter
            label = tk.Label(frame, width=15, text=key, anchor='w')
            label.pack(side=tk.LEFT)

            # entry object which ingests a value for current parameter
            entry = tk.Entry(frame)
            entry.insert(tk.END, fields[key])
            entry.pack(side=tk.LEFT, expand=tk.YES, fill=tk.X)
            entry.config(state=currState)

            # set units
            unitsLabel = tk.Label(frame, width=8, text=unitFields[key], anchor='w')
            unitsLabel.pack(side=tk.RIGHT)

            # append values to entries list
            entries.append( (key, entry) )

        return entries

    def revealOptions(self):
        for ent in self.dragParamEntries:
            if (self.var.get() == 0):
                ent[1].config(state='disabled')
            else:
                ent[1].config(state='normal')

    def plotChoices(self):
        # ordering of self.checkVarList same as PLOT_OPTIONS
        self.checkVarList = []
        for plotType in self.PLOT_TYPES:
            CheckVar = tk.IntVar()
            self.checkVarList.append(CheckVar)

            cb = tk.Checkbutton(text=plotType, variable=CheckVar, justify=tk.LEFT)
            cb.pack(side=tk.TOP, anchor=tk.W)

    def runSimulation(self):
        # This method will take last recorded values
        # and check box parameters and pass them
        # to the projectile motion object in order
        # for it to do computations. Then it will make plots.
        # Essentially, it is the workhorse method

        # append drag force fields to normal ones
        #allEntries = self.basicParamEntries + self.dragParamEntries

        # read in and set user values
        self.pm.setValues(self.basicParamEntries, 'basic')
        self.pm.setValues(self.dragParamEntries, 'drag')

        # ask if we are using drag force
        maxTime, maxRange, maxHeight = self.pm.evolve(self.var.get())
        print("Maximum Time = %.3f (s)" % maxTime)
        print("Maximum Range = %.3f (m)" % maxRange)
        print("Maximum Height = %.3f (m)" % maxHeight)

        ani_list = []
        for kk in np.arange(0, len(self.checkVarList)):
            if (self.checkVarList[kk].get() == 1):
                my_plots = PMPlots(self.pm, self.PLOT_NAME_SHORTCUTS[kk])
                ani_list.append(my_plots.runSimulation())

        # show all relevant plots and then clear params when finished
        plt.show()
        self.pm.clear()

### MAIN EXECUTABLE ###
if __name__ == "__main__":
    root = tk.Tk()
    my_gui = ProjectileMotionGUI(root)
    root.mainloop()
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import print_function
from astropy.extern.six import BytesIO
from astropy.table import Table
from astropy.io import fits
from astropy import coordinates
from ..query import BaseQuery
from ..utils import commons
from ..utils import async_to_sync
from . import conf

__all__ = ['Heasarc', 'HeasarcClass']

@async_to_sync
class HeasarcClass(BaseQuery):

    """HEASARC query class.

    For a list of available HEASARC mission tables, visit:
        https://heasarc.gsfc.nasa.gov/cgi-bin/W3Browse/w3catindex.pl
    """

    URL = conf.server
    TIMEOUT = conf.timeout

    def query_async(self, request_payload, cache=True):
        """
        Submit a query based on a given request_payload. This allows detailed
        control of the query to be submitted.
        """
        response = self._request('GET', self.URL, params=request_payload,
                                 timeout=self.TIMEOUT, cache=cache)
        return response

    def query_mission_cols(self, mission,
                           cache=True, get_query_payload=False,
                           **kwargs):
        """
        Returns a list containing the names of columns that will be returned for
        a given mission table. By default, all column names are returned. A list
        of the default columns can be obtained by setting `fields='Standard'`

        Parameters
        ----------
        mission : str
            Mission table (short name) to search from
        fields : str, optional
            Return format for columns from the server available options:
            * Standard      : Return default table columns
            * All (default) : Return all table columns
            * <custom>      : User defined csv list of columns to be returned
        All other parameters have no effect
        """
        # Query fails if nothing is found, so set search radius very large and
        # only take a single value (all we care about is the column names)
        kwargs['radius'] = 100000
        kwargs['resultmax'] = 1

        # By default, return all column names
        fields = kwargs.get('fields', None)
        if fields is None:
            kwargs['fields'] = 'All'

        table = self.query_position(position='0.0 0.0', mission=mission,
                                cache=cache, get_query_payload=get_query_payload,
                                **kwargs)

        # Return payload if requested
        if get_query_payload:
            return table

        return table.colnames

    def query_object_async(self, object_name, mission,
                           cache=True, get_query_payload=False,
                           **kwargs):
        """ Query around a specific object within a given mission catalog

        Parameters
        ----------
        object_name : str
            Object to query around. To set search radius use the hidden 'radius'
            parameter.
        mission : str
            Mission table to search from
        **kwargs :
            see :func:`_args_to_payload` for list of additional parameters that
            can be used to refine search query
        """
        request_payload = self._args_to_payload(
            mission = mission,
            entry   = object_name,
            **kwargs
        )

        # Return payload if requested
        if get_query_payload:
            return request_payload

        return self.query_async(request_payload, cache)

    def query_position_async(self, position, mission,
                             cache=True, get_query_payload=False,
                             **kwargs):
        """
        Query around a specific set of coordinates within a given mission catalog.
        This method first converts the supplied coordinates into the FK5
        reference frame and searches for sources from there. Because of this,
        the returned offset coordinates may look different than the ones supplied.

        Parameters
        ----------
        position : `astropy.coordinates` or str
            The position around which to search. It may be specified as a
            string in which case it is resolved using online services or as
            the appropriate `astropy.coordinates` object. ICRS coordinates
            may also be entered as a string.
            (adapted from nrao module)
        mission : str
            Mission table to search from
        **kwargs :
            see :func:`_args_to_payload` for list of additional parameters that
            can be used to refine search query
        """
        # Convert the coordinates to FK5
        c = commons.parse_coordinates(position).transform_to(coordinates.FK5)
        kwargs['coordsys'] = 'fk5'
        kwargs['equinox'] = 2000

        # Generate the request
        request_payload = self._args_to_payload(
            mission = mission,
            entry   = "{},{}".format(c.ra.degree, c.dec.degree),
            **kwargs
        )

        # Return payload if requested
        if get_query_payload:
            return request_payload

        # Submit the request
        return self.query_async(request_payload, cache)

    def _fallback(self, content):
        """
        Blank columns which have to be converted to float or in fail so
        lets fix that by replacing with -1's
        """

        data = BytesIO(content)
        header = fits.getheader(data, 1)  # Get header for column info
        colstart = [y for x, y in header.items() if "TBCOL" in x]
        collens = [int(float(y[1:]))
                   for x, y in header.items() if "TFORM" in x]
        new_table = []

        old_table = content.split("END")[-1].strip()
        for line in old_table.split("\n"):
            newline = []
            for n, tup in enumerate(zip(colstart, collens), start=1):
                cstart, clen = tup
                part = line[cstart - 1:cstart + clen]
                newline.append(part)
                if len(part.strip()) == 0:
                    if header["TFORM%i" % n][0] in ["F", "I"]:
                        # extra space is required to sperate column
                        newline[-1] = "-1".rjust(clen) + " "
            new_table.append("".join(newline))

        data = BytesIO(content.replace(old_table, "\n".join(new_table)))
        return Table.read(data, hdu=1)

    def _parse_result(self, response, verbose=False):
        # if verbose is False then suppress any VOTable related warnings
        if not verbose:
            commons.suppress_vo_warnings()
        try:
            data = BytesIO(response.content)
            table = Table.read(data, hdu=1)
            return table
        except ValueError:
            return self._fallback(response.content)

    def _args_to_payload(self, **kwargs):
        """
        Generates the payload based on user supplied arguments

        Parameters
        ----------
        mission : str
            Mission table to query
        entry : str, optional
            Object or position for center of query. A blank value will return
            all entries in the mission table. Acceptable formats:
            * Object name : Name of object, e.g. 'Crab'
            * Coordinates : X,Y coordinates, either as 'degrees,degrees' or
              'hh mm ss,dd mm ss'
        fields : str, optional
            Return format for columns from the server available options:
            * Standard (default) : Return default table columns
            * All                : Return all table columns
            * <custom>           : User defined csv list of columns to be returned
        radius : float (arcmin), optional
            Return all mission entries within a given distance of search query.
            If not supplied, the server default will be used (~ 60 arcmin)
        coordsys: str, optional
            If 'entry' is a set of coordinates, this specifies the coordinate
            system used to interpret them. By default, equatorial coordinates
            are assumed. Possible values:
            * 'fk5' <default> (FK5 J2000 equatorial coordinates)
            * 'fk4'           (FK4 B1950 equatorial coordinates)
            * 'equatorial'    (equatorial coordinates, `equinox` param determines epoch)
            * 'galactic'      (Galactic coordinates)
        equinox : int, optional
            Epoch by which to interpret supplied equatorial coordinates (defaults
            to 2000, ignored if `coordsys` is not 'equatorial')
        resultmax : int, optional
            Set maximum query results to be returned
        sortvar : str, optional
            Set the name of the column by which to sort the results. By default
            the results are sorted by distance from queried object/position

        displaymode : str, optional
            Return format from server. Since the user does not interact with
            this directly, it's best to leave this alone
        action : str, optional
            Type of action to be taken (defaults to 'Query')
        """
        # Define the basic query for this object
        request_payload = dict(
            tablehead   = ('name=BATCHRETRIEVALCATALOG_2.0 {}'
                          .format(kwargs.get('mission'))),
            Entry       = kwargs.get('entry', 'none'),
            Action      = kwargs.get('action', 'Query'),
            displaymode = kwargs.get('displaymode','FitsDisplay')
        )

        # Fill in optional information for refined queries

        # Handle queries involving coordinates
        coordsys = kwargs.get('coordsys', 'fk5')
        if coordsys.lower() == 'fk5':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'

        elif coordsys.lower() == 'fk4':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'
            request_payload['equinox']     = 1950

        elif coordsys.lower() == 'equatorial':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'

            equinox = kwargs.get('equinox', None)
            if equinox is not None:
                request_payload['Equinox'] = str(equinox)

        elif coordsys.lower() == 'galactic':
            request_payload['Coordinates'] = 'Galactic: LII BII'

        # Specify which table columns are to be returned
        fields = kwargs.get('fields', None)
        if fields is not None:
            if fields.lower() == 'standard':
                request_payload['Fields'] = 'Standard'
            elif fields.lower() == 'all':
                request_payload['Fields'] = 'All'
            else:
                request_payload['varon'] = fields.lower().split(',')

        # Set search radius (arcmin)
        radius = kwargs.get('radius', None)
        if radius is not None:
            request_payload['Radius'] = radius

        # Maximum number of results to be returned
        resultmax = kwargs.get('resultmax', None)
        if resultmax is not None:
            request_payload['ResultMax'] = int(resultmax)

        # Set variable for sorting results
        sortvar = kwargs.get('sortvar', None)
        if sortvar is not None:
            request_payload['sortvar'] = sortvar.lower()

        return request_payload

Heasarc = HeasarcClass()
<commit_msg>Add method to return list of available mission tables<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import print_function
from astropy.extern.six import BytesIO
from astropy.table import Table
from astropy.io import fits
from astropy import coordinates
from ..query import BaseQuery
from ..utils import commons
from ..utils import async_to_sync
from . import conf

__all__ = ['Heasarc', 'HeasarcClass']

@async_to_sync
class HeasarcClass(BaseQuery):

    """HEASARC query class.

    For a list of available HEASARC mission tables, visit:
        https://heasarc.gsfc.nasa.gov/cgi-bin/W3Browse/w3catindex.pl
    """

    URL = conf.server
    TIMEOUT = conf.timeout

    def query_async(self, request_payload, cache=True, url=conf.server):
        """
        Submit a query based on a given request_payload. This allows detailed
        control of the query to be submitted.
        """
        response = self._request('GET', url, params=request_payload,
                                 timeout=self.TIMEOUT, cache=cache)
        return response

    def query_mission_list(self,
                           cache=True, get_query_payload=False):
        """
        Returns a list of all available mission tables with descriptions
        """
        request_payload = self._args_to_payload(
            Entry   = 'none',
            mission = 'xxx',
            displaymode = 'BatchDisplay'
        )

        if get_query_payload:
            return request_payload

        # Parse the results specially (it's ascii format, not fits)
        response = self.query_async(request_payload,
                   url='https://heasarc.gsfc.nasa.gov/db-perl/W3Browse/w3query.pl',
                   cache=cache)
        data = BytesIO(response.content)
        table = Table.read(data, format='ascii.fixed_width_two_line',delimiter='+',
                           header_start=1, position_line=2, data_start=3, data_end=-1)
        return table

    def query_mission_cols(self, mission,
                           cache=True, get_query_payload=False,
                           **kwargs):
        """
        Returns a list containing the names of columns that will be returned for
        a given mission table. By default, all column names are returned. A list
        of the default columns can be obtained by setting `fields='Standard'`

        Parameters
        ----------
        mission : str
            Mission table (short name) to search from
        fields : str, optional
            Return format for columns from the server available options:
            * Standard      : Return default table columns
            * All (default) : Return all table columns
            * <custom>      : User defined csv list of columns to be returned
        All other parameters have no effect
        """
        # Query fails if nothing is found, so set search radius very large and
        # only take a single value (all we care about is the column names)
        kwargs['radius'] = 100000
        kwargs['resultmax'] = 1

        # By default, return all column names
        fields = kwargs.get('fields', None)
        if fields is None:
            kwargs['fields'] = 'All'

        table = self.query_position(position='0.0 0.0', mission=mission,
                                cache=cache, get_query_payload=get_query_payload,
                                **kwargs)

        # Return payload if requested
        if get_query_payload:
            return table

        return table.colnames

    def query_object_async(self, object_name, mission,
                           cache=True, get_query_payload=False,
                           **kwargs):
        """ Query around a specific object within a given mission catalog

        Parameters
        ----------
        object_name : str
            Object to query around. To set search radius use the hidden 'radius'
            parameter.
        mission : str
            Mission table to search from
        **kwargs :
            see :func:`_args_to_payload` for list of additional parameters that
            can be used to refine search query
        """
        request_payload = self._args_to_payload(
            mission = mission,
            entry   = object_name,
            **kwargs
        )

        # Return payload if requested
        if get_query_payload:
            return request_payload

        return self.query_async(request_payload, cache=cache)

    def query_position_async(self, position, mission,
                             cache=True, get_query_payload=False,
                             **kwargs):
        """
        Query around a specific set of coordinates within a given mission catalog.
        This method first converts the supplied coordinates into the FK5
        reference frame and searches for sources from there. Because of this,
        the returned offset coordinates may look different than the ones supplied.

        Parameters
        ----------
        position : `astropy.coordinates` or str
            The position around which to search. It may be specified as a
            string in which case it is resolved using online services or as
            the appropriate `astropy.coordinates` object. ICRS coordinates
            may also be entered as a string.
            (adapted from nrao module)
        mission : str
            Mission table to search from
        **kwargs :
            see :func:`_args_to_payload` for list of additional parameters that
            can be used to refine search query
        """
        # Convert the coordinates to FK5
        c = commons.parse_coordinates(position).transform_to(coordinates.FK5)
        kwargs['coordsys'] = 'fk5'
        kwargs['equinox'] = 2000

        # Generate the request
        request_payload = self._args_to_payload(
            mission = mission,
            entry   = "{},{}".format(c.ra.degree, c.dec.degree),
            **kwargs
        )

        # Return payload if requested
        if get_query_payload:
            return request_payload

        # Submit the request
        return self.query_async(request_payload, cache=cache)

    def _fallback(self, content):
        """
        Blank columns which have to be converted to float or in fail so
        lets fix that by replacing with -1's
        """

        data = BytesIO(content)
        header = fits.getheader(data, 1)  # Get header for column info
        colstart = [y for x, y in header.items() if "TBCOL" in x]
        collens = [int(float(y[1:]))
                   for x, y in header.items() if "TFORM" in x]
        new_table = []

        old_table = content.split("END")[-1].strip()
        for line in old_table.split("\n"):
            newline = []
            for n, tup in enumerate(zip(colstart, collens), start=1):
                cstart, clen = tup
                part = line[cstart - 1:cstart + clen]
                newline.append(part)
                if len(part.strip()) == 0:
                    if header["TFORM%i" % n][0] in ["F", "I"]:
                        # extra space is required to sperate column
                        newline[-1] = "-1".rjust(clen) + " "
            new_table.append("".join(newline))

        data = BytesIO(content.replace(old_table, "\n".join(new_table)))
        return Table.read(data, hdu=1)

    def _parse_result(self, response, verbose=False):
        # if verbose is False then suppress any VOTable related warnings
        if not verbose:
            commons.suppress_vo_warnings()
        try:
            data = BytesIO(response.content)
            table = Table.read(data, hdu=1)
            return table
        except ValueError:
            return self._fallback(response.content)

    def _args_to_payload(self, **kwargs):
        """
        Generates the payload based on user supplied arguments

        Parameters
        ----------
        mission : str
            Mission table to query
        entry : str, optional
            Object or position for center of query. A blank value will return
            all entries in the mission table. Acceptable formats:
            * Object name : Name of object, e.g. 'Crab'
            * Coordinates : X,Y coordinates, either as 'degrees,degrees' or
              'hh mm ss,dd mm ss'
        fields : str, optional
            Return format for columns from the server available options:
            * Standard (default) : Return default table columns
            * All                : Return all table columns
            * <custom>           : User defined csv list of columns to be returned
        radius : float (arcmin), optional
            Return all mission entries within a given distance of search query.
            If not supplied, the server default will be used (~ 60 arcmin)
        coordsys: str, optional
            If 'entry' is a set of coordinates, this specifies the coordinate
            system used to interpret them. By default, equatorial coordinates
            are assumed. Possible values:
            * 'fk5' <default> (FK5 J2000 equatorial coordinates)
            * 'fk4'           (FK4 B1950 equatorial coordinates)
            * 'equatorial'    (equatorial coordinates, `equinox` param determines epoch)
            * 'galactic'      (Galactic coordinates)
        equinox : int, optional
            Epoch by which to interpret supplied equatorial coordinates (defaults
            to 2000, ignored if `coordsys` is not 'equatorial')
        resultmax : int, optional
            Set maximum query results to be returned
        sortvar : str, optional
            Set the name of the column by which to sort the results. By default
            the results are sorted by distance from queried object/position

        displaymode : str, optional
            Return format from server. Since the user does not interact with
            this directly, it's best to leave this alone
        action : str, optional
            Type of action to be taken (defaults to 'Query')
        """
        # Define the basic query for this object
        request_payload = dict(
            tablehead   = ('name=BATCHRETRIEVALCATALOG_2.0 {}'
                          .format(kwargs.get('mission'))),
            Entry       = kwargs.get('entry', 'none'),
            Action      = kwargs.get('action', 'Query'),
            displaymode = kwargs.get('displaymode','FitsDisplay')
        )

        # Fill in optional information for refined queries

        # Handle queries involving coordinates
        coordsys = kwargs.get('coordsys', 'fk5')
        if coordsys.lower() == 'fk5':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'

        elif coordsys.lower() == 'fk4':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'
            request_payload['equinox']     = 1950

        elif coordsys.lower() == 'equatorial':
            request_payload['Coordinates'] = 'Equatorial: R.A. Dec'

            equinox = kwargs.get('equinox', None)
            if equinox is not None:
                request_payload['Equinox'] = str(equinox)

        elif coordsys.lower() == 'galactic':
            request_payload['Coordinates'] = 'Galactic: LII BII'

        # Specify which table columns are to be returned
        fields = kwargs.get('fields', None)
        if fields is not None:
            if fields.lower() == 'standard':
                request_payload['Fields'] = 'Standard'
            elif fields.lower() == 'all':
                request_payload['Fields'] = 'All'
            else:
                request_payload['varon'] = fields.lower().split(',')

        # Set search radius (arcmin)
        radius = kwargs.get('radius', None)
        if radius is not None:
            request_payload['Radius'] = radius

        # Maximum number of results to be returned
        resultmax = kwargs.get('resultmax', None)
        if resultmax is not None:
            request_payload['ResultMax'] = int(resultmax)

        # Set variable for sorting results
        sortvar = kwargs.get('sortvar', None)
        if sortvar is not None:
            request_payload['sortvar'] = sortvar.lower()

        return request_payload

Heasarc = HeasarcClass()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import logging
import time
import os
import codecs
from tools import io

# Configure logging
logger = logging.getLogger('EEEE')
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(message)s',
                              datefmt='%Y-%m-%d %I:%M:%S')
ch.setFormatter(formatter)
logger.addHandler(ch)

# get current time and inform about start
total_time = time.time()
logger.info('eGo data processing started...')

# list of sql_snippets that process the data in correct order
snippet_dir = 'sql_snippets'
script_dir = 'python_scripts'
sql_snippets = [
    # 'scenario_eGo_data_processing.sql',
    # 'setup_bkg_vg250.sql',
    # 'process_eGo_substation.sql',
    # 'process_eGo_grid_district.sql',
    # 'setup_osm_landuse.sql',
    # 'process_eGo_osm_loads_industry.sql',
    # 'process_eGo_osm_loads.sql',
    # 'setup_zensus_population_per_ha.sql',
    # 'process_eGo_loads_melted.sql',
    # 'process_eGo_loads_per_grid_district.sql',
    # 'process_eGo_consumption.sql',
    # 'analyse_eGo_paper_result.sql',

    # 'setup_eGo_wpa_per_grid_district.sql',
    'setup_eGo_lattice_per_area.sql',
    # 'process_eGo_dea_allocation_methods.sql'
    ]

python_scripts = [
   # 'demand_per_mv_grid_district.py',
   # 'peak_load_per_load_area.py'
]

# get database connection
conn = io.oedb_session(section='oedb')

# iterate over list of sql snippets and execute them
for snippet in sql_snippets:
    # timing and logging
    snippet_time = time.time()
    logger.info("Execute '{}' ...".format(snippet))
    snippet_str = open(os.path.join(snippet_dir, snippet)).read()

    # execute desired sql snippet
    conn.execution_options(autocommit=True).execute(snippet_str)

    # inform the user
    logger.info('...successfully done in {:.2f} seconds.'.format(
        time.time() - snippet_time))

# close database connection
conn.close()

# iterate over list of python scripts and execute
for script in python_scripts:
    # timing and logging
    script_time = time.localtime()
    logger.info("Execute '{}' ...".format(script))
    filename = os.path.join(script_dir, script)
    script_str = open(filename, "rb").read()

    # execute desired sql snippet
    exec(compile(script_str, filename, 'exec'), globals, locals)

    # inform the user
    logger.info('...successfully done in {:.2f} seconds.'.format(
        time.time() - script_time))

logger.info('Data processing script successfully executed in {:.2f} seconds'.format(
    time.time() - total_time))
<commit_msg>First dev version with python script execution<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import logging
import time
import os
import codecs
from tools import io

# Configure logging
logger = logging.getLogger('EEEE')
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(message)s',
                              datefmt='%Y-%m-%d %I:%M:%S')
ch.setFormatter(formatter)
logger.addHandler(ch)

# get current time and inform about start
total_time = time.time()
logger.info('eGo data processing started...')

# list of sql_snippets that process the data in correct order
snippet_dir = 'sql_snippets'
script_dir = 'python_scripts'
sql_snippets = [
    # 'scenario_eGo_data_processing.sql',
    # 'setup_bkg_vg250.sql',
    # 'process_eGo_substation.sql',
    # 'process_eGo_grid_district.sql',
    # 'setup_osm_landuse.sql',
    # 'process_eGo_osm_loads_industry.sql',
    # 'process_eGo_osm_loads.sql',
    # 'setup_zensus_population_per_ha.sql',
    # 'process_eGo_loads_melted.sql',
    # 'process_eGo_loads_per_grid_district.sql',
    # 'process_eGo_consumption.sql',
    # 'analyse_eGo_paper_result.sql',

    # 'setup_eGo_wpa_per_grid_district.sql',
    # 'setup_eGo_lattice_per_area.sql',
    # 'process_eGo_dea_allocation_methods.sql'
    ]

python_scripts = [
   'demand_per_mv_grid_district.py',
   # 'peak_load_per_load_area.py'
]

# get database connection
conn = io.oedb_session(section='oedb')

# iterate over list of sql snippets and execute them
for snippet in sql_snippets:
    # timing and logging
    snippet_time = time.time()
    logger.info("Execute '{}' ...".format(snippet))
    snippet_str = open(os.path.join(snippet_dir, snippet)).read()

    # execute desired sql snippet
    conn.execution_options(autocommit=True).execute(snippet_str)

    # inform the user
    logger.info('...successfully done in {:.2f} seconds.'.format(
        time.time() - snippet_time))

# close database connection
conn.close()

# iterate over list of python scripts and execute
for script in python_scripts:
    # timing and logging
    script_time = time.time()
    logger.info("Execute '{}' ...".format(script))
    filename = os.path.join(script_dir, script)
    script_str = open(filename, "rb").read()

    # execute desired sql snippet
    exec(compile(script_str, filename, 'exec'))

    # inform the user
    logger.info('...successfully done in {:.2f} seconds.'.format(
        time.time() - script_time))

logger.info('Data processing script successfully executed in {:.2f} seconds'.format(
    time.time() - total_time))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
:Author Patrik Valkovic
:Created 23.06.2017 16:39
:Licence GNUv3
Part of grammpy

"""

class Rule:
    pass
<commit_msg>Add base interface for rule<commit_after>#!/usr/bin/env python
"""
:Author Patrik Valkovic
:Created 23.06.2017 16:39
:Licence GNUv3
Part of grammpy

"""

from grammpy import EPSILON

class Rule:
    right = [EPSILON]
    left = [EPSILON]
    rule = ([EPSILON], [EPSILON])
    rules = [([EPSILON], [EPSILON])]

    def is_regular(self):
        return False

    def is_contextfree(self):
        return False

    def is_context(self):
        return False

    def is_unrestricted(self):
        return False
<|endoftext|>


<commit_before>import os
import boto
import glob
import logging
import re
from collections import namedtuple

import pandas as pd

from dataactvalidator.app import createApp
from dataactvalidator.scripts.loaderUtils import LoaderUtils
from dataactcore.interfaces.db import GlobalDB
from dataactcore.models.domainModels import SF133
from dataactcore.config import CONFIG_BROKER

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def load_all_sf133(sf133_path=None):
    """Load any SF-133 files that are not yet in the database."""
    # get a list of SF 133 files to load
    sf133_list = get_sf133_list(sf133_path)
    SF_RE = re.compile(r'sf_133_(?P<year>\d{4})_(?P<period>\d{2})\.csv')
    for sf133 in sf133_list:
        # for each SF file, parse out fiscal year and period
        # and call the SF 133 loader
        file_match = SF_RE.match(sf133.file)
        if not file_match:
            logger.info('{}Skipping SF 133 file with invalid name: {}'.format(
                os.linesep, sf133.full_file))
            continue
        logger.info('{}Starting {}...'.format(os.linesep, sf133.full_file))
        load_sf133(
            sf133.full_file, file_match.group('year'), file_match.group('period'))

def load_sf133(filename, fiscal_year, fiscal_period, force_load=False):
    """Load SF 133 (budget execution report) lookup table."""
    model = SF133

    with createApp().app_context():
        sess = GlobalDB.db().session

        existing_records = sess.query(model).filter(
            model.fiscal_year == fiscal_year, model.period == fiscal_period)
        if force_load:
            # force a reload of this period's current data
            logger.info('Force SF 133 load: deleting existing records for {} {}'.format(
                fiscal_year, fiscal_period))
            delete_count = existing_records.delete()
            logger.info('{} records deleted'.format(delete_count))
        elif existing_records.count():
            # if there's existing data & we're not forcing a load, skip
            logger.info('SF133 {} {} already in database ({} records). Skipping file.'.format(
                fiscal_year, fiscal_period, existing_records.count()))
            return

        data = pd.read_csv(filename, dtype=str)
        data = LoaderUtils.cleanData(
            data,
            model,
            {"ata": "allocation_transfer_agency",
             "aid": "agency_identifier",
             "availability_type_code": "availability_type_code",
             "bpoa": "beginning_period_of_availa",
             "epoa": "ending_period_of_availabil",
             "main_account": "main_account_code",
             "sub_account": "sub_account_code",
             "fiscal_year": "fiscal_year",
             "period": "period",
             "line_num": "line",
             "amount_summed":
            "amount"},
            {"allocation_transfer_agency": {"pad_to_length": 3},
             "agency_identifier": {"pad_to_length": 3},
             "main_account_code": {"pad_to_length": 4},
             "sub_account_code": {"pad_to_length": 3},
             # next 3 lines handle the TAS fields that shouldn't
             # be padded but should still be empty spaces rather
             # than NULLs. this ensures that the downstream pivot & melt
             # (which insert the missing 0-value SF-133 lines)
             # will work as expected (values used in the pivot
             # index cannot be NULL).
             # the "pad_to_length: 0" works around the fact
             # that sometimes the incoming data for these columns
             # is a single space and sometimes it is blank/NULL.
             "beginning_period_of_availa": {"pad_to_length": 0},
             "ending_period_of_availabil": {"pad_to_length": 0},
             "availability_type_code": {"pad_to_length": 0},
             "amount": {"strip_commas": True}}
        )

        # todo: find out how to handle dup rows (e.g., same tas/period/line number)
        # line numbers 2002 and 2012 are the only duped SF 133 report line numbers,
        # and they are not used by the validation rules, so for now
        # just remove them before loading our SF-133 table
        dupe_line_numbers = ['2002', '2102']
        data = data[~data.line.isin(dupe_line_numbers)]

        # add concatenated TAS field for internal use (i.e., joining to staging tables)
        data['tas'] = data.apply(lambda row: format_internal_tas(row), axis=1)

        # incoming .csv does not always include rows for zero-value SF-133 lines
        # so we add those here because they're needed for the SF-133 validations.
        # 1. "pivot" the sf-133 dataset to explode it horizontally, creating one
        # row for each tas/fiscal year/period, with columns for each SF-133 line.
        # the "fill_value=0" parameter puts a 0 into any Sf-133 line number cell
        # with a missing value for a specific tas/fiscal year/period.
        # 2. Once the zeroes are filled in, "melt" the pivoted data back to its normal
        # format of one row per tas/fiscal year/period.
        # NOTE: fields used for the pivot in step #1 (i.e., items in pivot_idx) cannot
        # have NULL values, else they will be silently dropped by pandas :(
        pivot_idx = ['created_at', 'updated_at', 'agency_identifier', 'allocation_transfer_agency',
                     'availability_type_code', 'beginning_period_of_availa', 'ending_period_of_availabil',
                     'main_account_code', 'sub_account_code', 'tas', 'fiscal_year', 'period']
        data.amount = data.amount.astype(float)
        data = pd.pivot_table(data, values='amount', index=pivot_idx, columns=['line'], fill_value=0).reset_index()
        data = pd.melt(data, id_vars=pivot_idx, value_name='amount')

        # Now that we've added zero lines for EVERY tas and SF 133 line number, get rid of the ones
        # we don't actually use in the validations. Arguably, it would be better just to include
        # everything, but that drastically increases the number of records we're inserting to the
        # sf_133 table. If we ever decide that we need *all* SF 133 lines that are zero value,
        # remove the next two lines.
        sf_133_validation_lines = [
            '1000', '1010', '1011', '1012', '1013', '1020', '1021', '1022',
            '1023', '1024', '1025', '1026', '1029', '1030', '1031', '1032',
            '1033', '1040', '1041', '1042', '1160', '1180', '1260', '1280',
            '1340', '1440', '1540', '1640', '1750', '1850', '1910', '2190',
            '2490', '2500', '3020', '4801', '4802', '4881', '4882', '4901',
            '4902', '4908', '4981', '4982'
        ]
        data = data[(data.line.isin(sf_133_validation_lines)) | (data.amount != 0)]

        # we didn't use the the 'keep_null' option when padding allocation transfer agency,
        # because nulls in that column break the pivot (see above comments).
        # so, replace the ata '000' with an empty value before inserting to db
        data['allocation_transfer_agency'] = data['allocation_transfer_agency'].str.replace('000', '')
        # make a pass through the dataframe, changing any empty values to None, to ensure
        # that those are represented as NULL in the db.
        data = data.applymap(lambda x: str(x).strip() if len(str(x).strip()) else None)

        # insert to db
        table_name = model.__table__.name
        num = LoaderUtils.insertDataframe(data, table_name, sess.connection())
        sess.commit()

    logger.info('{} records inserted to {}'.format(num, table_name))

def format_internal_tas(row):
    """Concatenate TAS components into a single field for internal use."""
    # This formatting should match formatting in dataactcore.models.stagingModels concatTas
    tas = ''.join([
        row['allocation_transfer_agency'] if row['allocation_transfer_agency'] else '000',
        row['agency_identifier'] if row['agency_identifier'] else '000',
        row['beginning_period_of_availa'] if row['beginning_period_of_availa'].strip() else '0000',
        row['ending_period_of_availabil'] if row['ending_period_of_availabil'].strip() else '0000',
        row['availability_type_code'].strip() if row['availability_type_code'].strip() else ' ',
        row['main_account_code'] if row['main_account_code'] else '0000',
        row['sub_account_code'] if row['sub_account_code'] else '000'
    ])
    return tas

def get_sf133_list(sf133_path):
    """Return info about existing SF133 files as a list of named tuples."""
    SF133File = namedtuple('SF133', ['full_file', 'file'])
    if sf133_path is not None:
        logger.info('Loading local SF-133')
        # get list of SF 133 files in the specified local directory
        sf133_files = glob.glob(os.path.join(sf133_path, 'sf_133*.csv'))
        sf133_list = [SF133File(sf133, os.path.basename(sf133)) for sf133 in sf133_files]
    else:
        logger.info("Loading SF-133")
        if CONFIG_BROKER["use_aws"]:
            # get list of SF 133 files in the config bucket on S3
            s3connection = boto.s3.connect_to_region(CONFIG_BROKER['aws_region'])
            s3bucket = s3connection.lookup(CONFIG_BROKER['aws_bucket'])
            # get bucketlistresultset with all sf_133 files
            sf133_files = s3bucket.list(
                prefix='{}/sf_133'.format(CONFIG_BROKER['sf_133_folder']))
            sf133_list = [SF133File(sf133, os.path.basename(sf133.name)) for sf133 in sf133_files]
        else:
            sf133_list = []

    return sf133_list

if __name__ == '__main__':
    load_all_sf133(
        os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config")
    )
<commit_msg>Replace model variable with SF133<commit_after>import os
import boto
import glob
import logging
import re
from collections import namedtuple

import pandas as pd

from dataactvalidator.app import createApp
from dataactvalidator.scripts.loaderUtils import LoaderUtils
from dataactcore.interfaces.db import GlobalDB
from dataactcore.models.domainModels import SF133
from dataactcore.config import CONFIG_BROKER

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def load_all_sf133(sf133_path=None):
    """Load any SF-133 files that are not yet in the database."""
    # get a list of SF 133 files to load
    sf133_list = get_sf133_list(sf133_path)
    SF_RE = re.compile(r'sf_133_(?P<year>\d{4})_(?P<period>\d{2})\.csv')
    for sf133 in sf133_list:
        # for each SF file, parse out fiscal year and period
        # and call the SF 133 loader
        file_match = SF_RE.match(sf133.file)
        if not file_match:
            logger.info('{}Skipping SF 133 file with invalid name: {}'.format(
                os.linesep, sf133.full_file))
            continue
        logger.info('{}Starting {}...'.format(os.linesep, sf133.full_file))
        load_sf133(
            sf133.full_file, file_match.group('year'), file_match.group('period'))

def load_sf133(filename, fiscal_year, fiscal_period, force_load=False):
    """Load SF 133 (budget execution report) lookup table."""

    with createApp().app_context():
        sess = GlobalDB.db().session

        existing_records = sess.query(SF133).filter(
            SF133.fiscal_year == fiscal_year, SF133.period == fiscal_period)
        if force_load:
            # force a reload of this period's current data
            logger.info('Force SF 133 load: deleting existing records for {} {}'.format(
                fiscal_year, fiscal_period))
            delete_count = existing_records.delete()
            logger.info('{} records deleted'.format(delete_count))
        elif existing_records.count():
            # if there's existing data & we're not forcing a load, skip
            logger.info('SF133 {} {} already in database ({} records). Skipping file.'.format(
                fiscal_year, fiscal_period, existing_records.count()))
            return

        data = pd.read_csv(filename, dtype=str)
        data = LoaderUtils.cleanData(
            data,
            SF133,
            {"ata": "allocation_transfer_agency",
             "aid": "agency_identifier",
             "availability_type_code": "availability_type_code",
             "bpoa": "beginning_period_of_availa",
             "epoa": "ending_period_of_availabil",
             "main_account": "main_account_code",
             "sub_account": "sub_account_code",
             "fiscal_year": "fiscal_year",
             "period": "period",
             "line_num": "line",
             "amount_summed":
            "amount"},
            {"allocation_transfer_agency": {"pad_to_length": 3},
             "agency_identifier": {"pad_to_length": 3},
             "main_account_code": {"pad_to_length": 4},
             "sub_account_code": {"pad_to_length": 3},
             # next 3 lines handle the TAS fields that shouldn't
             # be padded but should still be empty spaces rather
             # than NULLs. this ensures that the downstream pivot & melt
             # (which insert the missing 0-value SF-133 lines)
             # will work as expected (values used in the pivot
             # index cannot be NULL).
             # the "pad_to_length: 0" works around the fact
             # that sometimes the incoming data for these columns
             # is a single space and sometimes it is blank/NULL.
             "beginning_period_of_availa": {"pad_to_length": 0},
             "ending_period_of_availabil": {"pad_to_length": 0},
             "availability_type_code": {"pad_to_length": 0},
             "amount": {"strip_commas": True}}
        )

        # todo: find out how to handle dup rows (e.g., same tas/period/line number)
        # line numbers 2002 and 2012 are the only duped SF 133 report line numbers,
        # and they are not used by the validation rules, so for now
        # just remove them before loading our SF-133 table
        dupe_line_numbers = ['2002', '2102']
        data = data[~data.line.isin(dupe_line_numbers)]

        # add concatenated TAS field for internal use (i.e., joining to staging tables)
        data['tas'] = data.apply(lambda row: format_internal_tas(row), axis=1)

        # incoming .csv does not always include rows for zero-value SF-133 lines
        # so we add those here because they're needed for the SF-133 validations.
        # 1. "pivot" the sf-133 dataset to explode it horizontally, creating one
        # row for each tas/fiscal year/period, with columns for each SF-133 line.
        # the "fill_value=0" parameter puts a 0 into any Sf-133 line number cell
        # with a missing value for a specific tas/fiscal year/period.
        # 2. Once the zeroes are filled in, "melt" the pivoted data back to its normal
        # format of one row per tas/fiscal year/period.
        # NOTE: fields used for the pivot in step #1 (i.e., items in pivot_idx) cannot
        # have NULL values, else they will be silently dropped by pandas :(
        pivot_idx = ['created_at', 'updated_at', 'agency_identifier', 'allocation_transfer_agency',
                     'availability_type_code', 'beginning_period_of_availa', 'ending_period_of_availabil',
                     'main_account_code', 'sub_account_code', 'tas', 'fiscal_year', 'period']
        data.amount = data.amount.astype(float)
        data = pd.pivot_table(data, values='amount', index=pivot_idx, columns=['line'], fill_value=0).reset_index()
        data = pd.melt(data, id_vars=pivot_idx, value_name='amount')

        # Now that we've added zero lines for EVERY tas and SF 133 line number, get rid of the ones
        # we don't actually use in the validations. Arguably, it would be better just to include
        # everything, but that drastically increases the number of records we're inserting to the
        # sf_133 table. If we ever decide that we need *all* SF 133 lines that are zero value,
        # remove the next two lines.
        sf_133_validation_lines = [
            '1000', '1010', '1011', '1012', '1013', '1020', '1021', '1022',
            '1023', '1024', '1025', '1026', '1029', '1030', '1031', '1032',
            '1033', '1040', '1041', '1042', '1160', '1180', '1260', '1280',
            '1340', '1440', '1540', '1640', '1750', '1850', '1910', '2190',
            '2490', '2500', '3020', '4801', '4802', '4881', '4882', '4901',
            '4902', '4908', '4981', '4982'
        ]
        data = data[(data.line.isin(sf_133_validation_lines)) | (data.amount != 0)]

        # we didn't use the the 'keep_null' option when padding allocation transfer agency,
        # because nulls in that column break the pivot (see above comments).
        # so, replace the ata '000' with an empty value before inserting to db
        data['allocation_transfer_agency'] = data['allocation_transfer_agency'].str.replace('000', '')
        # make a pass through the dataframe, changing any empty values to None, to ensure
        # that those are represented as NULL in the db.
        data = data.applymap(lambda x: str(x).strip() if len(str(x).strip()) else None)

        # insert to db
        table_name = SF133.__table__.name
        num = LoaderUtils.insertDataframe(data, table_name, sess.connection())
        sess.commit()

    logger.info('{} records inserted to {}'.format(num, table_name))

def format_internal_tas(row):
    """Concatenate TAS components into a single field for internal use."""
    # This formatting should match formatting in dataactcore.models.stagingModels concatTas
    tas = ''.join([
        row['allocation_transfer_agency'] if row['allocation_transfer_agency'] else '000',
        row['agency_identifier'] if row['agency_identifier'] else '000',
        row['beginning_period_of_availa'] if row['beginning_period_of_availa'].strip() else '0000',
        row['ending_period_of_availabil'] if row['ending_period_of_availabil'].strip() else '0000',
        row['availability_type_code'].strip() if row['availability_type_code'].strip() else ' ',
        row['main_account_code'] if row['main_account_code'] else '0000',
        row['sub_account_code'] if row['sub_account_code'] else '000'
    ])
    return tas

def get_sf133_list(sf133_path):
    """Return info about existing SF133 files as a list of named tuples."""
    SF133File = namedtuple('SF133', ['full_file', 'file'])
    if sf133_path is not None:
        logger.info('Loading local SF-133')
        # get list of SF 133 files in the specified local directory
        sf133_files = glob.glob(os.path.join(sf133_path, 'sf_133*.csv'))
        sf133_list = [SF133File(sf133, os.path.basename(sf133)) for sf133 in sf133_files]
    else:
        logger.info("Loading SF-133")
        if CONFIG_BROKER["use_aws"]:
            # get list of SF 133 files in the config bucket on S3
            s3connection = boto.s3.connect_to_region(CONFIG_BROKER['aws_region'])
            s3bucket = s3connection.lookup(CONFIG_BROKER['aws_bucket'])
            # get bucketlistresultset with all sf_133 files
            sf133_files = s3bucket.list(
                prefix='{}/sf_133'.format(CONFIG_BROKER['sf_133_folder']))
            sf133_list = [SF133File(sf133, os.path.basename(sf133.name)) for sf133 in sf133_files]
        else:
            sf133_list = []

    return sf133_list

if __name__ == '__main__':
    load_all_sf133(
        os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config")
    )
<|endoftext|>