<commit_before>import json
import datetime

from copy import copy
from flask import request
from flask.ext.cors import cross_origin
from dateutil.parser import parse as parse_date

from alerta.app import app, db
from alerta.alert import Alert
from alerta.app.utils import jsonify, jsonp, process_alert
from alerta.app.metrics import Timer
from alerta.plugins import RejectException

LOG = app.logger

webhook_timer = Timer('alerts', 'webhook', 'Web hook alerts', 'Total time to process number of web hook alerts')
duplicate_timer = Timer('alerts', 'duplicate', 'Duplicate alerts', 'Total time to process number of duplicate alerts')
correlate_timer = Timer('alerts', 'correlate', 'Correlated alerts', 'Total time to process number of correlated alerts')
create_timer = Timer('alerts', 'create', 'Newly created alerts', 'Total time to process number of new alerts')

def cw_state_to_severity(state):

    if state == 'ALARM':
        return 'major'
    elif state == 'INSUFFICIENT_DATA':
        return 'warning'
    elif state == 'OK':
        return 'normal'
    else:
        return 'unknown'

def parse_notification(notification):

    notification = json.loads(notification)

    if notification['Type'] == 'SubscriptionConfirmation':

        return Alert(
            resource=notification['TopicArn'],
            event=notification['Type'],
            environment='Production',
            severity='informational',
            service=['Unknown'],
            group='AWS/CloudWatch',
            text='%s SubscribeURL' % (notification['Message'], notification['SubscribeURL']),
            origin=notification['TopicArn'],
            event_type='cloudwatchAlarm',
            create_time=datetime.datetime.strptime(notification['Timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ'),
            raw_data=notification,
        )

    elif notification['Type'] == 'Notification':

        alarm = json.loads(notification['Message'])

        return Alert(
            resource='%s:%s' % (alarm['Trigger']['Dimensions'][0]['name'], alarm['Trigger']['Dimensions'][0]['value']),
            event=alarm['AlarmName'],
            environment='Production',
            severity=cw_state_to_severity(alarm['NewStateValue']),
            service=[alarm['AWSAccountId']],
            group=alarm['Trigger']['Namespace'],
            value=alarm['NewStateValue'],
            text=alarm['AlarmDescription'],
            tags=[alarm['Region']],
            attributes={
                'incidentKey': alarm['AlarmName'],
                'thresholdInfo': alarm['Trigger']
            },
            origin=notification['TopicArn'],
            event_type='cloudwatchAlarm',
            create_time=datetime.datetime.strptime(notification['Timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ'),
            raw_data=alarm
        )

@app.route('/webhooks/cloudwatch', methods=['OPTIONS', 'POST'])
@cross_origin()
@jsonp
def cloudwatch():

    hook_started = webhook_timer.start_timer()
    try:
        incomingAlert = parse_notification(request.data)
    except ValueError as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 400

    try:
        alert = process_alert(incomingAlert)
    except RejectException as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 403
    except Exception as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 500

    webhook_timer.stop_timer(hook_started)

    if alert:
        body = alert.get_body()
        body['href'] = "%s/%s" % (request.base_url, alert.id)
        return jsonify(status="ok", id=alert.id, alert=body), 201, {'Location': '%s/%s' % (request.base_url, alert.id)}
    else:
        return jsonify(status="error", message="insert or update of cloudwatch alarm failed"), 500

def parse_pingdom(check):

    check = json.loads(check)

    if check['action'] == 'assign':
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down'],
            environment='Production',
            severity='critical',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )
    elif check['action'] == 'notify_of_close':
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down'],
            environment='Production',
            severity='normal',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )
    else:
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down', check['description']],
            environment='Production',
            severity='indeterminate',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )

@app.route('/webhooks/pingdom', methods=['OPTIONS', 'GET'])
@cross_origin()
@jsonp
def pingdom():

    hook_started = webhook_timer.start_timer()
    try:
        incomingAlert = parse_pingdom(request.args.get('message'))
    except ValueError as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 400

    try:
        alert = process_alert(incomingAlert)
    except RejectException as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 403
    except Exception as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 500

    webhook_timer.stop_timer(hook_started)

    if alert:
        body = alert.get_body()
        body['href'] = "%s/%s" % (request.base_url, alert.id)
        return jsonify(status="ok", id=alert.id, alert=body), 201, {'Location': '%s/%s' % (request.base_url, alert.id)}
    else:
        return jsonify(status="error", message="insert or update of pingdom check failed"), 500

def parse_pagerduty(message):

    incident_key = message['data']['incident']['incident_key']
    incident_number = message['data']['incident']['incident_number']
    html_url = message['data']['incident']['html_url']
    incident_url = '#%s' % (html_url, incident_number)

    try:
        alert = db.get_alerts(query={'attributes.incidentKey': incident_key}, limit=1)[0]
    except IndexError:
        raise

    from alerta.app import status_code

    if message['type'] == 'incident.trigger':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s assigned to %s' % (incident_url, user)
    elif message['type'] == 'incident.acknowledge':
        status = status_code.ACK
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s acknowledged by %s' % (incident_url, user)
    elif message['type'] == 'incident.unacknowledge':
        status = status_code.OPEN
        text = 'Incident %s unacknowledged due to timeout' % incident_url
    elif message['type'] == 'incident.resolve':
        status = status_code.CLOSED
        if message['data']['incident']['resolved_by_user']:
            user = message['data']['incident']['resolved_by_user']['name']
        else:
            user = 'n/a'
        text = 'Incident %s resolved by %s' % (incident_url, user)
    elif message['type'] == 'incident.assign':
        status = status_code.ASSIGN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s manually assigned to %s' % (incident_url, user)
    elif message['type'] == 'incident.escalate':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s escalated to %s' % (incident_url, user)
    elif message['type'] == 'incident.delegate':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s reassigned due to escalation to %s' % (incident_url, user)
    else:
        status = status_code.UNKNOWN
        text = message['type']

    return alert.id, status, text

@app.route('/webhooks/pagerduty', methods=['OPTIONS', 'POST'])
@cross_origin()
def pagerduty():

    hook_started = webhook_timer.start_timer()
    data = request.json

    if data and 'messages' in data:
        for message in data['messages']:
            try:
                id, status, text = parse_pagerduty(message)
            except IndexError as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 400

            try:
                alert = db.set_status(id=id, status=status, text=text)
            except Exception as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 500
    else:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message="no messages in PagerDuty data payload"), 400

    webhook_timer.stop_timer(hook_started)

    if alert:
        return jsonify(status="ok"), 200
    else:
        return jsonify(status="error", message="update PagerDuty incident status failed"), 500

def parse_prometheus(status, alert):

    labels = copy(alert['labels'])
    annotations = copy(alert['annotations'])

    starts_at = parse_date(alert['startsAt'])
    if alert['endsAt'] == '0001-01-01T00:00:00Z':
        ends_at = None
    else:
        ends_at = parse_date(alert['endsAt'])

    if status == 'firing':
        severity = labels.pop('severity', 'warning')
        create_time = starts_at
    elif status == 'resolved':
        severity = 'normal'
        create_time = ends_at
    else:
        severity = 'unknown'
        create_time = ends_at or starts_at

    summary = annotations.pop('summary', None)
    description = annotations.pop('description', None)
    text = description or summary or '%s: %s on %s' % (labels['job'], labels['alertname'], labels['instance'])

    if 'generatorURL' in alert:
        annotations['moreInfo'] = 'Prometheus Graph' % alert['generatorURL']

    return Alert(
        resource=labels.pop('exported_instance', None) or labels.pop('instance'),
        event=labels.pop('alertname'),
        environment=labels.pop('environment', 'Production'),
        severity=severity,
        correlate=labels.pop('correlate').split(',') if 'correlate' in labels else None,
        service=labels.pop('service', '').split(','),
        group=labels.pop('group', None),
        value=labels.pop('value', None),
        text=text,
        customer=labels.pop('customer', None),
        tags=["%s=%s" % t for t in labels.items()],
        attributes=annotations,
        origin='prometheus/' + labels.get('job', '-'),
        event_type='prometheusAlert',
        create_time=create_time,
        raw_data=alert
    )

@app.route('/webhooks/prometheus', methods=['OPTIONS', 'POST'])
@cross_origin()
def prometheus():

    if request.json and 'alert' in request.json:
        hook_started = webhook_timer.start_timer()
        status = request.json['status']
        for alert in request.json['alert']:
            try:
                incomingAlert = parse_prometheus(status, alert)
            except ValueError as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 400

            try:
                process_alert(incomingAlert)
            except RejectException as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 403
            except Exception as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 500

        webhook_timer.stop_timer(hook_started)
    else:
        return jsonify(status="error", message="no alerts in Prometheus notification payload"), 400

    return jsonify(status="ok"), 200
<commit_msg>Change alert parser to support alertmanager API v3<commit_after>import json
import datetime

from copy import copy
from flask import request
from flask.ext.cors import cross_origin
from dateutil.parser import parse as parse_date

from alerta.app import app, db
from alerta.alert import Alert
from alerta.app.utils import jsonify, jsonp, process_alert
from alerta.app.metrics import Timer
from alerta.plugins import RejectException

LOG = app.logger

webhook_timer = Timer('alerts', 'webhook', 'Web hook alerts', 'Total time to process number of web hook alerts')
duplicate_timer = Timer('alerts', 'duplicate', 'Duplicate alerts', 'Total time to process number of duplicate alerts')
correlate_timer = Timer('alerts', 'correlate', 'Correlated alerts', 'Total time to process number of correlated alerts')
create_timer = Timer('alerts', 'create', 'Newly created alerts', 'Total time to process number of new alerts')

def cw_state_to_severity(state):

    if state == 'ALARM':
        return 'major'
    elif state == 'INSUFFICIENT_DATA':
        return 'warning'
    elif state == 'OK':
        return 'normal'
    else:
        return 'unknown'

def parse_notification(notification):

    notification = json.loads(notification)

    if notification['Type'] == 'SubscriptionConfirmation':

        return Alert(
            resource=notification['TopicArn'],
            event=notification['Type'],
            environment='Production',
            severity='informational',
            service=['Unknown'],
            group='AWS/CloudWatch',
            text='%s SubscribeURL' % (notification['Message'], notification['SubscribeURL']),
            origin=notification['TopicArn'],
            event_type='cloudwatchAlarm',
            create_time=datetime.datetime.strptime(notification['Timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ'),
            raw_data=notification,
        )

    elif notification['Type'] == 'Notification':

        alarm = json.loads(notification['Message'])

        return Alert(
            resource='%s:%s' % (alarm['Trigger']['Dimensions'][0]['name'], alarm['Trigger']['Dimensions'][0]['value']),
            event=alarm['AlarmName'],
            environment='Production',
            severity=cw_state_to_severity(alarm['NewStateValue']),
            service=[alarm['AWSAccountId']],
            group=alarm['Trigger']['Namespace'],
            value=alarm['NewStateValue'],
            text=alarm['AlarmDescription'],
            tags=[alarm['Region']],
            attributes={
                'incidentKey': alarm['AlarmName'],
                'thresholdInfo': alarm['Trigger']
            },
            origin=notification['TopicArn'],
            event_type='cloudwatchAlarm',
            create_time=datetime.datetime.strptime(notification['Timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ'),
            raw_data=alarm
        )

@app.route('/webhooks/cloudwatch', methods=['OPTIONS', 'POST'])
@cross_origin()
@jsonp
def cloudwatch():

    hook_started = webhook_timer.start_timer()
    try:
        incomingAlert = parse_notification(request.data)
    except ValueError as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 400

    try:
        alert = process_alert(incomingAlert)
    except RejectException as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 403
    except Exception as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 500

    webhook_timer.stop_timer(hook_started)

    if alert:
        body = alert.get_body()
        body['href'] = "%s/%s" % (request.base_url, alert.id)
        return jsonify(status="ok", id=alert.id, alert=body), 201, {'Location': '%s/%s' % (request.base_url, alert.id)}
    else:
        return jsonify(status="error", message="insert or update of cloudwatch alarm failed"), 500

def parse_pingdom(check):

    check = json.loads(check)

    if check['action'] == 'assign':
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down'],
            environment='Production',
            severity='critical',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )
    elif check['action'] == 'notify_of_close':
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down'],
            environment='Production',
            severity='normal',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )
    else:
        return Alert(
            resource=check['host'],
            event=check['description'],
            correlate=['up', 'down', check['description']],
            environment='Production',
            severity='indeterminate',
            service=[check['checkname']],
            group='Network',
            text='%s is %s.' % (check['checkname'], check['description']),
            attributes={'incidentKey': check['incidentid']},
            origin='Pingdom',
            event_type='availabilityAlert',
            raw_data=check,
        )

@app.route('/webhooks/pingdom', methods=['OPTIONS', 'GET'])
@cross_origin()
@jsonp
def pingdom():

    hook_started = webhook_timer.start_timer()
    try:
        incomingAlert = parse_pingdom(request.args.get('message'))
    except ValueError as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 400

    try:
        alert = process_alert(incomingAlert)
    except RejectException as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 403
    except Exception as e:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message=str(e)), 500

    webhook_timer.stop_timer(hook_started)

    if alert:
        body = alert.get_body()
        body['href'] = "%s/%s" % (request.base_url, alert.id)
        return jsonify(status="ok", id=alert.id, alert=body), 201, {'Location': '%s/%s' % (request.base_url, alert.id)}
    else:
        return jsonify(status="error", message="insert or update of pingdom check failed"), 500

def parse_pagerduty(message):

    incident_key = message['data']['incident']['incident_key']
    incident_number = message['data']['incident']['incident_number']
    html_url = message['data']['incident']['html_url']
    incident_url = '#%s' % (html_url, incident_number)

    try:
        alert = db.get_alerts(query={'attributes.incidentKey': incident_key}, limit=1)[0]
    except IndexError:
        raise

    from alerta.app import status_code

    if message['type'] == 'incident.trigger':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s assigned to %s' % (incident_url, user)
    elif message['type'] == 'incident.acknowledge':
        status = status_code.ACK
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s acknowledged by %s' % (incident_url, user)
    elif message['type'] == 'incident.unacknowledge':
        status = status_code.OPEN
        text = 'Incident %s unacknowledged due to timeout' % incident_url
    elif message['type'] == 'incident.resolve':
        status = status_code.CLOSED
        if message['data']['incident']['resolved_by_user']:
            user = message['data']['incident']['resolved_by_user']['name']
        else:
            user = 'n/a'
        text = 'Incident %s resolved by %s' % (incident_url, user)
    elif message['type'] == 'incident.assign':
        status = status_code.ASSIGN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s manually assigned to %s' % (incident_url, user)
    elif message['type'] == 'incident.escalate':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s escalated to %s' % (incident_url, user)
    elif message['type'] == 'incident.delegate':
        status = status_code.OPEN
        user = message['data']['incident']['assigned_to_user']['name']
        text = 'Incident %s reassigned due to escalation to %s' % (incident_url, user)
    else:
        status = status_code.UNKNOWN
        text = message['type']

    return alert.id, status, text

@app.route('/webhooks/pagerduty', methods=['OPTIONS', 'POST'])
@cross_origin()
def pagerduty():

    hook_started = webhook_timer.start_timer()
    data = request.json

    if data and 'messages' in data:
        for message in data['messages']:
            try:
                id, status, text = parse_pagerduty(message)
            except IndexError as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 400

            try:
                alert = db.set_status(id=id, status=status, text=text)
            except Exception as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 500
    else:
        webhook_timer.stop_timer(hook_started)
        return jsonify(status="error", message="no messages in PagerDuty data payload"), 400

    webhook_timer.stop_timer(hook_started)

    if alert:
        return jsonify(status="ok"), 200
    else:
        return jsonify(status="error", message="update PagerDuty incident status failed"), 500

def parse_prometheus(status, alert):

    labels = copy(alert['labels'])
    annotations = copy(alert['annotations'])

    starts_at = parse_date(alert['startsAt'])
    if alert['endsAt'] == '0001-01-01T00:00:00Z':
        ends_at = None
    else:
        ends_at = parse_date(alert['endsAt'])

    if status == 'firing':
        severity = labels.pop('severity', 'warning')
        create_time = starts_at
    elif status == 'resolved':
        severity = 'normal'
        create_time = ends_at
    else:
        severity = 'unknown'
        create_time = ends_at or starts_at

    summary = annotations.pop('summary', None)
    description = annotations.pop('description', None)
    text = description or summary or '%s: %s on %s' % (labels['job'], labels['alertname'], labels['instance'])

    if 'generatorURL' in alert:
        annotations['moreInfo'] = 'Prometheus Graph' % alert['generatorURL']

    return Alert(
        resource=labels.pop('exported_instance', None) or labels.pop('instance'),
        event=labels.pop('alertname'),
        environment=labels.pop('environment', 'Production'),
        severity=severity,
        correlate=labels.pop('correlate').split(',') if 'correlate' in labels else None,
        service=labels.pop('service', '').split(','),
        group=labels.pop('group', None),
        value=labels.pop('value', None),
        text=text,
        customer=labels.pop('customer', None),
        tags=["%s=%s" % t for t in labels.items()],
        attributes=annotations,
        origin='prometheus/' + labels.get('job', '-'),
        event_type='prometheusAlert',
        create_time=create_time,
        raw_data=alert
    )

@app.route('/webhooks/prometheus', methods=['OPTIONS', 'POST'])
@cross_origin()
def prometheus():

    if request.json and 'alerts' in request.json:
        hook_started = webhook_timer.start_timer()
        status = request.json['status']
        for alert in request.json['alerts']:
            try:
                incomingAlert = parse_prometheus(status, alert)
            except ValueError as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 400

            try:
                process_alert(incomingAlert)
            except RejectException as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 403
            except Exception as e:
                webhook_timer.stop_timer(hook_started)
                return jsonify(status="error", message=str(e)), 500

        webhook_timer.stop_timer(hook_started)
    else:
        return jsonify(status="error", message="no alerts in Prometheus notification payload"), 400

    return jsonify(status="ok"), 200
<|endoftext|>


<commit_before>

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    docstring for DeltaVariance

    """

    def __init__(self, img, header, weights, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.weights = weights
        self.diam_ratio = diam_ratio

        self.nanflag = False
        if np.isnan(self.img).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to avoid noisy pixels
                # Can't be bigger than half the image (deals with issue from sim cubes)
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except KeyError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size), np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, nsamples=5000, alpha=0.05):

        for i, (convolved_array, convolved_weight) in enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            bootstrap_delvar = np.empty((nsamples, ))
            for n in range(nsamples):
                resample = bootstrap_resample(convolved_array)
                bootstrap_delvar[n] = _delvar(resample, convolved_weight)

            stat = np.sort(bootstrap_delvar)
            error = (stat[int((alpha/2.0)*nsamples)],
                    stat[int((1-alpha/2.0)*nsamples)])

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar()

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            p.errorbar(self.lags, self.delta_var, yerr=[np.abs(self.delta_var - self.delta_var_error[0, :]),
                       np.abs(self.delta_var + self.delta_var_error[1, :])], fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    INPUTS
    ------

    lag : float

    OUTPUTS
    -------

    kernel : numpy.ndarray
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    INPUTS
    ------

    lag - float
          size of kernel, same as width

    diam_ratio - float
                 ratio between kernel diameters

    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """

    """

    def __init__(self, dataset1, weights1, dataset2, weights2, diam_ratio=1.5,
                 lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1], weights1, diam_ratio=diam_ratio,
                                         lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1], weights2, diam_ratio=diam_ratio,
                                     lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):

        self.distance = np.linalg.norm(np.log10(self.delvar1.delta_var) -
                                       np.log10(self.delvar2.delta_var))

        if verbose:
            import matplotlib.pyplot as p

            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=[np.abs(self.delvar1.delta_var - self.delvar1.delta_var_error[0, :]),
                       np.abs(self.delvar1.delta_var + self.delvar1.delta_var_error[1, :])], fmt="bD-",
                       label="Delta Var 1")
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=[np.abs(self.delvar2.delta_var - self.delvar2.delta_var_error[0, :]),
                       np.abs(self.delvar2.delta_var + self.delvar2.delta_var_error[1, :])], fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
          np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None
    Results
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<commit_msg>Changed Delta Variance distance to weighted euclidean norm.<commit_after>

'''

Implementation of the Delta-Variance Method from Stutzki et al. 1998.

'''

import numpy as np
from scipy.stats import nanmean
from scipy.spatial.distance import euclidean
from astropy.convolution import convolve_fft
from astropy import units as u

class DeltaVariance(object):

    """

    docstring for DeltaVariance

    """

    def __init__(self, img, header, weights, diam_ratio=1.5, lags=None):
        super(DeltaVariance, self).__init__()

        self.img = img
        self.header = header
        self.weights = weights
        self.diam_ratio = diam_ratio

        self.nanflag = False
        if np.isnan(self.img).any():
            self.nanflag = True

        if lags is None:
            try:
                self.ang_size = self.header["CDELT2"] * u.deg
                min_size = (0.1 * u.arcmin) / self.ang_size.to(u.arcmin)
                # Can't be smaller than one pixel, set to 3 to
                # avoid noisy pixels. Can't be bigger than half the image
                #(deals with issue from sim cubes).
                if min_size < 3.0 or min_size > min(self.img.shape) / 2.:
                    min_size = 3.0
            except KeyError:
                print "No CDELT2 in header. Using pixel scales."
                self.ang_size = 1.0 * u.astrophys.pixel
                min_size = 3.0
            self.lags = np.logspace(np.log10(min_size),
                                    np.log10(min(self.img.shape) / 2.), 25)
        else:
            self.lags = lags

        self.convolved_arrays = []
        self.convolved_weights = []
        self.delta_var = np.empty((len(self.lags, )))
        self.delta_var_error = np.empty((2, len(self.lags)))

    def do_convolutions(self):
        for i, lag in enumerate(self.lags):
            core = core_kernel(lag, self.img.shape[0], self.img.shape[1])
            annulus = annulus_kernel(
                lag, self.diam_ratio, self.img.shape[0], self.img.shape[1])

            # Extend to avoid boundary effects from non-periodicity
            pad_weights = np.pad(self.weights, int(lag), padwithzeros)
            pad_img = np.pad(self.img, int(lag), padwithzeros) * pad_weights

            interpolate_nan = False
            if self.nanflag:
                interpolate_nan = True

            img_core = convolve_fft(
                pad_img, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            img_annulus = convolve_fft(
                pad_img, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_core = convolve_fft(
                pad_weights, core, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)
            weights_annulus = convolve_fft(
                pad_weights, annulus, normalize_kernel=True,
                interpolate_nan=interpolate_nan,
                ignore_edge_zeros=True)

            weights_core[np.where(weights_core == 0)] = np.NaN
            weights_annulus[np.where(weights_annulus == 0)] = np.NaN

            self.convolved_arrays.append(
                (img_core / weights_core) - (img_annulus / weights_annulus))
            self.convolved_weights.append(weights_core * weights_annulus)

        return self

    def compute_deltavar(self, nsamples=5000, alpha=0.05):

        for i, (convolved_array, convolved_weight) in \
         enumerate(zip(self.convolved_arrays, self.convolved_weights)):

            delta_var_val = _delvar(convolved_array, convolved_weight)

            # bootstrap to find an error
            bootstrap_delvar = np.empty((nsamples, ))
            for n in range(nsamples):
                resample = bootstrap_resample(convolved_array)
                bootstrap_delvar[n] = _delvar(resample, convolved_weight)

            stat = np.sort(bootstrap_delvar)
            error = (stat[int((alpha/2.0)*nsamples)],
                     stat[int((1-alpha/2.0)*nsamples)])

            self.delta_var[i] = delta_var_val
            self.delta_var_error[:, i] = error

        return self

    def run(self, verbose=False, ang_units=True):

        self.do_convolutions()
        self.compute_deltavar()

        if ang_units:
            self.lags *= self.ang_size

        if verbose:
            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar = [np.abs(self.delta_var -
                                self.delta_var_error[0, :]),
                         np.abs(self.delta_var +
                                self.delta_var_error[1, :])]
            p.errorbar(self.lags, self.delta_var, yerr=error_bar, fmt="bD-")
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def core_kernel(lag, x_size, y_size):
    '''
    Core Kernel for convolution.

    INPUTS
    ------

    lag : float

    OUTPUTS
    -------

    kernel : numpy.ndarray
    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))
    kernel = ((4 / np.pi * lag)) * \
        np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)

    return kernel / np.sum(kernel)

def annulus_kernel(lag, diam_ratio, x_size, y_size):
    '''

    Annulus Kernel for convolution.

    INPUTS
    ------

    lag - float
          size of kernel, same as width

    diam_ratio - float
                 ratio between kernel diameters

    '''

    x, y = np.meshgrid(np.arange(-x_size / 2, x_size / 2 + 1, 1),
                       np.arange(-y_size / 2, y_size / 2 + 1, 1))

    inner = np.exp(-(x ** 2. + y ** 2.) / (lag / 2.) ** 2.)
    outer = np.exp(-(x ** 2. + y ** 2.) / (diam_ratio * lag / 2.) ** 2.)

    kernel = 4 / (np.pi * lag * (diam_ratio ** 2. - 1)) * (outer - inner)

    return kernel / np.sum(kernel)

def padwithzeros(vector, pad_width, iaxis, kwargs):
    vector[:pad_width[0]] = 0
    vector[-pad_width[1]:] = 0
    return vector

class DeltaVariance_Distance(object):

    """

    """

    def __init__(self, dataset1, weights1, dataset2, weights2, diam_ratio=1.5,
                 lags=None, fiducial_model=None):
        super(DeltaVariance_Distance, self).__init__()

        if fiducial_model is not None:
            self.delvar1 = fiducial_model
        else:
            self.delvar1 = DeltaVariance(dataset1[0], dataset1[1], weights1,
                                         diam_ratio=diam_ratio, lags=lags)
            self.delvar1.run()

        self.delvar2 = DeltaVariance(dataset2[0], dataset2[1], weights2,
                                     diam_ratio=diam_ratio, lags=lags)
        self.delvar2.run()

        self.distance = None

    def distance_metric(self, verbose=False):

        errors1 = np.abs(self.delvar1.delta_var_error[1, :] -
                         self.delvar1.delta_var_error[0, :])
        errors2 = np.abs(self.delvar2.delta_var_error[1, :] -
                         self.delvar2.delta_var_error[0, :])

        self.distance = euclidean(np.log10(self.delvar1.delta_var / errors1),
                                  np.log10(self.delvar2.delta_var / errors2))

        if verbose:
            import matplotlib.pyplot as p

            import matplotlib.pyplot as p
            ax = p.subplot(111)
            ax.set_xscale("log", nonposx="clip")
            ax.set_yscale("log", nonposx="clip")
            error_bar1 = [np.abs(self.delvar1.delta_var -
                                 self.delvar1.delta_var_error[0, :]),
                          np.abs(self.delvar1.delta_var +
                                 self.delvar1.delta_var_error[1, :])]
            p.errorbar(self.delvar1.lags, self.delvar1.delta_var,
                       yerr=error_bar1, fmt="bD-",
                       label="Delta Var 1")
            error_bar2 = [np.abs(self.delvar2.delta_var -
                                 self.delvar2.delta_var_error[0, :]),
                          np.abs(self.delvar2.delta_var +
                                 self.delvar2.delta_var_error[1, :])]
            p.errorbar(self.delvar2.lags, self.delvar2.delta_var,
                       yerr=error_bar2, fmt="gD-",
                       label="Delta Var 2")
            ax.legend(loc='best')
            ax.grid(True)
            ax.set_xlabel("Lag (arcmin)")
            ax.set_ylabel(r"$\sigma^{2}_{\Delta}$")
            p.show()

        return self

def _delvar(array, weight):
    '''
    Computes the delta variance of the given array.
    '''
    avg_value = nanmean(array, axis=None)

    val = np.nansum((array - avg_value) ** 2. * weight) /\
        np.nansum(weight)

    return val

def bootstrap_resample(X, n=None):
    """
    Code based on: http://nbviewer.ipython.org/gist/aflaxman/6871948
    Bootstrap resample an array_like
    Parameters
    ----------
    X : array_like
      data to resample
    n : int, optional
      length of resampled array, equal to len(X) if n==None
    Results
    -------
    returns X_resamples
    """
    if n is None:
        n = len(X)

    resample_i = np.floor(np.random.rand(n)*len(X)).astype(int)
    X_resample = X[resample_i]
    return X_resample
<|endoftext|>


<commit_before>#
#   Copyright (c) 2013-2014, Scott J Maddox
#
#   This file is part of openbandparams.
#
#   openbandparams is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published
#   by the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   openbandparams is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with openbandparams.  If not, see <http://www.gnu.org/licenses/>.
#
#############################################################################

from openbandparams.iii_v.quaternary import (Quaternary1, Quaternary2,
    Quaternary3)
from openbandparams.iii_v.zinc_blende.binary import (AlN, AlP, AlAs, AlSb,
    GaN, GaP, GaAs, GaSb, InN, InP, InAs, InSb)
from openbandparams.iii_v.zinc_blende.ternary import (AlNP, AlNAs, AlPAs,
    AlPSb, AlAsSb, GaNP, GaNAs, GaPAs, GaPSb, GaAsSb, InNP, InNAs, InPAs,
    InPSb, InAsSb, AlGaN, AlInN, GaInN, AlGaP, AlInP, GaInP, AlGaAs, AlInAs,
    GaInAs, AlGaSb, AlInSb, GaInSb)

# Type 1: AB_{x}C_{y}D_{1-x-y}
# binaryies = (AB, AC, AD)
# ternaries = (ABC, ABD ,ACD)

class AlNPAs(Quaternary1):
    name = 'AlNPAs'
    elements = ('Al', 'N', 'P', 'As')
    binaries = (AlN, AlP, AlAs)
    ternaries = (AlNP, AlNAs, AlPAs)

# class AlNPSb(Quaternary1):
#    name = 'AlNPSb'
#    elements = ('Al', 'N', 'P', 'Sb')
#    binaries = (AlN, AlP, AlSb)
#    ternaries = (AlNP, AlNSb, AlPSb)

# class AlNAsSb(Quaternary1):
#    name = 'AlNAsSb'
#    elements = ('Al', 'N', 'As', 'Sb')
#    binaries = (AlN, AlAs, AlSb)
#    ternaries = (AlNAs, AlNSb, AlAsSb)

class AlPAsSb(Quaternary1):
    name = 'AlPAsSb'
    elements = ('Al', 'P', 'As', 'Sb')
    binaries = (AlP, AlAs, AlSb)
    ternaries = (AlPAs, AlPSb, AlAsSb)

class GaNPAs(Quaternary1):
    name = 'GaNPAs'
    elements = ('Ga', 'N', 'P', 'As')
    binaries = (GaN, GaP, GaAs)
    ternaries = (GaNP, GaNAs, GaPAs)

# class GaNPSb(Quaternary1):
#    name = 'GaNPSb'
#    elements = ('Ga', 'N', 'P', 'Sb')
#    binaries = (GaN, GaP, GaSb)
#    ternaries = (GaNP, GaNSb, GaPSb)

# class GaNAsSb(Quaternary1):
#    name = 'GaNAsSb'
#    elements = ('Ga', 'N', 'As', 'Sb')
#    binaries = (GaN, GaAs, GaSb)
#    ternaries = (GaNAs, GaNSb, GaAsSb)

class GaPAsSb(Quaternary1):
    name = 'GaPAsSb'
    elements = ('Ga', 'P', 'As', 'Sb')
    binaries = (GaP, GaAs, GaSb)
    ternaries = (GaPAs, GaPSb, GaAsSb)

class InNPAs(Quaternary1):
    name = 'InNPAs'
    elements = ('In', 'N', 'P', 'As')
    binaries = (InN, InP, InAs)
    ternaries = (InNP, InNAs, InPAs)

# class InNPSb(Quaternary1):
#    name = 'InNPSb'
#    elements = ('In', 'N', 'P', 'Sb')
#    binaries = (InN, InP, InSb)
#    ternaries = (InNP, InNSb, InPSb)

# class InNAsSb(Quaternary1):
#    name = 'InNAsSb'
#    elements = ('In', 'N', 'As', 'Sb')
#    binaries = (InN, InAs, InSb)
#    ternaries = (InNAs, InNSb, InAsSb)

class InPAsSb(Quaternary1):
    name = 'InPAsSb'
    elements = ('In', 'P', 'As', 'Sb')
    binaries = (InP, InAs, InSb)
    ternaries = (InPAs, InPSb, InAsSb)

# Type 2: A_{x}B_{y}C_{1-x-y}D
# binaries = (AD, BD, CD)
# ternaries = (ABD, ACD, BCD)

class AlGaInN(Quaternary2):
    name = 'AlGaInN'
    elements = ('Al', 'Ga', 'In', 'N')
    binaries = (AlN, GaN, InN)
    ternaries = (AlGaN, AlInN, GaInN)

class AlGaInP(Quaternary2):
    name = 'AlGaInP'
    elements = ('Al', 'Ga', 'In', 'P')
    binaries = (AlP, GaP, InP)
    ternaries = (AlGaP, AlInP, GaInP)

class AlGaInAs(Quaternary2):
    name = 'AlGaInAs'
    elements = ('Al', 'Ga', 'In', 'As')
    binaries = (AlAs, GaAs, InAs)
    ternaries = (AlGaAs, AlInAs, GaInAs)

class AlGaInSb(Quaternary2):
    name = 'AlGaInSb'
    elements = ('Al', 'Ga', 'In', 'Sb')
    binaries = (AlSb, GaSb, InSb)
    ternaries = (AlGaSb, AlInSb, GaInSb)

# Type 3: A_{x}B_{1-x}C_{y}D_{1-y}
# binaries = (AC, AD, BC, BD)
# ternaries = (ABC, ABD, ACD, BCD)

class AlGaNP(Quaternary3):
    name = 'AlGaNP'
    elements = ('Al', 'Ga', 'N', 'P')
    binaries = (AlN, AlP, GaN, GaP)
    ternaries = (AlGaN, AlGaP, AlNP, GaNP)

class AlGaNAs(Quaternary3):
    name = 'AlGaNAs'
    elements = ('Al', 'Ga', 'N', 'As')
    binaries = (AlN, AlAs, GaN, GaAs)
    ternaries = (AlGaN, AlGaAs, AlNAs, GaNAs)

# class AlGaNSb(Quaternary3):
#    name = 'AlGaNSb'
#    elements = ('Al', 'Ga', 'N', 'Sb')
#    binaries = (AlN, AlSb, GaN, GaSb)
#    ternaries = (AlGaN, AlGaSb, AlNSb, GaNSb)

class AlGaPAs(Quaternary3):
    name = 'AlGaPAs'
    elements = ('Al', 'Ga', 'P', 'As')
    binaries = (AlP, AlAs, GaP, GaAs)
    ternaries = (AlGaP, AlGaAs, AlPAs, GaPAs)

class AlGaPSb(Quaternary3):
    name = 'AlGaPSb'
    elements = ('Al', 'Ga', 'P', 'Sb')
    binaries = (AlP, AlSb, GaP, GaSb)
    ternaries = (AlGaP, AlGaSb, AlPSb, GaPSb)

class AlGaAsSb(Quaternary3):
    name = 'AlGaAsSb'
    elements = ('Al', 'Ga', 'As', 'Sb')
    binaries = (AlAs, AlSb, GaAs, GaSb)
    ternaries = (AlGaAs, AlGaSb, AlAsSb, GaAsSb)

class AlInNP(Quaternary3):
    name = 'AlInNP'
    elements = ('Al', 'In', 'N', 'P')
    binaries = (AlN, AlP, InN, InP)
    ternaries = (AlInN, AlInP, AlNP, InNP)

class AlInNAs(Quaternary3):
    name = 'AlInNAs'
    elements = ('Al', 'In', 'N', 'As')
    binaries = (AlN, AlAs, InN, InAs)
    ternaries = (AlInN, AlInAs, AlNAs, InNAs)

# class AlInNSb(Quaternary3):
#    name = 'AlInNSb'
#    elements = ('Al', 'In', 'N', 'Sb')
#    binaries = (AlN, AlSb, InN, InSb)
#    ternaries = (AlInN, AlInSb, AlNSb, InNSb)

class AlInPAs(Quaternary3):
    name = 'AlInPAs'
    elements = ('Al', 'In', 'P', 'As')
    binaries = (AlP, AlAs, InP, InAs)
    ternaries = (AlInP, AlInAs, AlPAs, InPAs)

class AlInPSb(Quaternary3):
    name = 'AlInPSb'
    elements = ('Al', 'In', 'P', 'Sb')
    binaries = (AlP, AlSb, InP, InSb)
    ternaries = (AlInP, AlInSb, AlPSb, InPSb)

class AlInAsSb(Quaternary3):
    name = 'AlInAsSb'
    elements = ('Al', 'In', 'As', 'Sb')
    binaries = (AlAs, AlSb, InAs, InSb)
    ternaries = (AlInAs, AlInSb, AlAsSb, InAsSb)

class GaInNP(Quaternary3):
    name = 'GaInNP'
    elements = ('Ga', 'In', 'N', 'P')
    binaries = (GaN, GaP, InN, InP)
    ternaries = (GaInN, GaInP, GaNP, InNP)

class GaInNAs(Quaternary3):
    name = 'GaInNAs'
    elements = ('Ga', 'In', 'N', 'As')
    binaries = (GaN, GaAs, InN, InAs)
    ternaries = (GaInN, GaInAs, GaNAs, InNAs)

# class GaInNSb(Quaternary3):
#    name = 'GaInNSb'
#    elements = ('Ga', 'In', 'N', 'Sb')
#    binaries = (GaN, GaSb, InN, InSb)
#    ternaries = (GaInN, GaInSb, GaNSb, InNSb)

class GaInPAs(Quaternary3):
    name = 'GaInPAs'
    elements = ('Ga', 'In', 'P', 'As')
    binaries = (GaP, GaAs, InP, InAs)
    ternaries = (GaInP, GaInAs, GaPAs, InPAs)

class GaInPSb(Quaternary3):
    name = 'GaInPSb'
    elements = ('Ga', 'In', 'P', 'Sb')
    binaries = (GaP, GaSb, InP, InSb)
    ternaries = (GaInP, GaInSb, GaPSb, InPSb)

class GaInAsSb(Quaternary3):
    name = 'GaInAsSb'
    elements = ('Ga', 'In', 'As', 'Sb')
    binaries = (GaAs, GaSb, InAs, InSb)
    ternaries = (GaInAs, GaInSb, GaAsSb, InAsSb)

quaternaries = [  # Type 1: AB_{x}C_{y}D_{1-x-y}
                AlNPAs,  # AlNPSb, AlNAsSb,
                AlPAsSb,
                GaNPAs,  # GaNPSb, GaNAsSb,
                GaPAsSb,
                InNPAs,  # InNPSb, InNAsSb,
                InPAsSb,
                # Type 2: B_{x}C_{y}D_{1-x-y}A
                AlGaInN, AlGaInP, AlGaInAs, AlGaInSb,
                # Type 3: A_{x}B_{1-x}C_{y}D_{1-y}
                AlGaNP, AlInNP, GaInNP,
                AlGaNAs, AlInNAs, GaInNAs,
                AlGaPAs, AlInPAs, GaInPAs,
                AlGaPSb, AlInPSb, GaInPSb,
                AlGaAsSb, AlInAsSb, GaInAsSb]

__all__ = ['quaternaries']
__all__ += [quaternary.name for quaternary in quaternaries]
<commit_msg>remove `iii_v.zinc_blende.quaternary`<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import urllib2
import datetime
import xml.etree.ElementTree as ET

from pytz import UTC
from bs4 import BeautifulSoup
from django.db import transaction
from django.db.models import Max

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mlbslicedb.settings")
import django
django.setup()

from gd2.models import Pitch, Runner, Atbat, Action, Inning, Game, Player, Team

action_fields = set(f.name for f in Action._meta.get_fields())
atbat_fields = set(f.name for f in Atbat._meta.get_fields())
pitch_fields = set(f.name for f in Pitch._meta.get_fields())
runner_fields = set(f.name for f in Runner._meta.get_fields())

def clean_zero_len_string(obj_dict):
    for k,v in obj_dict.items():
        if v == '':
            obj_dict[k] = None
    return obj_dict

def add_team(team_dict):
    try:
        team = Team.objects.get(id=team_dict['id'])
        team.name = team_dict['name']
        team.abbrev = team_dict['abbrev']
    except Team.DoesNotExist:
        team = Team(**team_dict)
    team.save()
    return team

def add_player(player_dict):
    try:
        player = Player.objects.get(id=player_dict['id'])
        player.first = player_dict['first']
        player.last = player_dict['last']
    except Player.DoesNotExist:
        player = Player(id=player_dict['id'],
                        first=player_dict['first'],
                        last=player_dict['last'],)
    player.save()
    return player

@transaction.atomic
def add_game(game_url):
    gamecenter_url = game_url + 'gamecenter.xml'
    game_game_url = game_url + 'game.xml'
    gamecenter_xml = urllib2.urlopen(gamecenter_url).read()
    gamecenter_root = ET.fromstring(gamecenter_xml)
    gamecenter_attrib = gamecenter_root.attrib
    game_id = gamecenter_attrib['id']
    game_status = gamecenter_attrib['status']
    game_type = gamecenter_attrib['type']
    probables_element = gamecenter_root.findall('probables')[0]
    home_probable_element = probables_element.findall('home')[0]
    home_prob_id = home_probable_element.findall('player_id')[0].text
    home_prob_first = home_probable_element.findall('useName')[0].text
    home_prob_last = home_probable_element.findall('lastName')[0].text

    away_probable_element = probables_element.findall('away')[0]
    away_prob_id = home_probable_element.findall('player_id')[0].text
    away_prob_first = home_probable_element.findall('useName')[0].text
    away_prob_last = home_probable_element.findall('lastName')[0].text

    game_game_xml = urllib2.urlopen(game_game_url).read()
    game_game_root = ET.fromstring(game_game_xml)
    game_game_attrib = game_game_root.attrib
    team_elements = game_game_root.findall('team')
    for team_element in team_elements:
        team_attrib = team_element.attrib
        team_dict = {'id': team_attrib['id'],
                     'name': team_attrib['name'],
                     'abbrev': team_attrib['abbrev'],}
        if team_attrib['type'] == 'home':
            home_team = add_team(team_dict)
        elif team_attrib['type'] == 'away':
            away_team = add_team(team_dict)
        else:
            pass

    try:
        game = Game.objects.get(id=game_id)
        print 'game already exists: %s' % game_id
        return
    except Game.DoesNotExist:
        game = Game(id=game_id,)

    if home_prob_id:
        home_prob_player = add_player({'id': int(home_prob_id),
                                       'first': home_prob_first,
                                       'last': home_prob_last,})
        game.home_probable = home_prob_player

    if away_prob_id:
        away_prob_player = add_player({'id': int(away_prob_id),
                                       'first': away_prob_first,
                                       'last': away_prob_last,})
        game.away_probable = away_prob_player

    game.home_team = home_team
    game.away_team = away_team
    game.status = game_status
    game.type = game_type

    if game.status == 'F':
        box_url = game_url + 'boxscore.xml'
        box_xml = urllib2.urlopen(box_url).read()
        box_root = ET.fromstring(box_xml)
        box_attrib = box_root.attrib
        linescore_element = box_root.findall('linescore')[0]
        linescore_attrib = linescore_element.attrib
        away_runs = linescore_attrib['away_team_runs']
        away_hits = linescore_attrib['away_team_hits']
        home_runs = linescore_attrib['home_team_runs']
        home_hits = linescore_attrib['home_team_hits']
        game.away_runs = away_runs
        game.away_hits = away_hits
        game.home_runs = home_runs
        game.home_hits = home_hits
        game.save()

        players_url = game_url + 'players.xml'
        players_xml = urllib2.urlopen(players_url).read()
        players_root = ET.fromstring(players_xml)

        for team_elem in players_root.findall('team'):
            for player_elem in team_elem.findall('player'):
                add_player(player_elem.attrib)

        innings_url = game_url + 'inning/'
        innings_html = urllib2.urlopen(innings_url).read()
        innings_soup = BeautifulSoup(innings_html, 'html.parser')
        inning_links = [link.get('href') for
                            link in innings_soup.find_all('a')
                                if re.search('inning_\d',link.get('href'))]

        for inning_link in inning_links:
            inning_url = innings_url + inning_link
            inning_xml = urllib2.urlopen(inning_url).read()
            inning_root = ET.fromstring(inning_xml)

            for half_elem in inning_root.getchildren():
                inning_dict = {'game': Game.objects.get(id=game_id),
                               'top_bottom': half_elem.tag,
                               'num': int(inning_root.attrib['num']),
                               'away_team': away_team,
                               'home_team': home_team,
                               'next': inning_root.attrib['next'],}
                inning_dict = clean_zero_len_string(inning_dict)
                inning = Inning(**inning_dict)
                inning.save()

                for event_elem in half_elem.getchildren():
                    if event_elem.tag.lower() == 'atbat':
                        atbat_dict = dict(event_elem.items())
                        atbat_dict['inning'] = inning
                        atbat_dict['batter'] = Player.objects.get(
                                                id=atbat_dict['batter'])
                        atbat_dict['pitcher'] = Player.objects.get(
                                                id=atbat_dict['pitcher'])
                        atbat_dict['description'] = \
                                        atbat_dict.pop('des', None)
                        atbat_dict['description_spanish'] = \
                                        atbat_dict.pop('des_es', None)

                        atbat_dict = {k: v for k, v in atbat_dict.items()
                                        if k in atbat_fields}
                        atbat_dict = clean_zero_len_string(atbat_dict)
                        atbat = Atbat(**atbat_dict)
                        atbat.save()

                        for pitch_runner_elem in event_elem.getchildren():
                            if pitch_runner_elem.tag.lower() == 'pitch':
                                pitch_elem = pitch_runner_elem
                                pitch_dict = dict(pitch_elem.items())
                                pitch_dict['atbat'] = atbat
                                pitch_dict['local_id'] = pitch_dict.pop(
                                                                'id', None)
                                pitch_dict['description'] = \
                                            pitch_dict.pop('des', None)
                                pitch_dict['description_spanish'] = \
                                            pitch_dict.pop('des_es', None)
                                pitch_dict = {k: v for k, v in
                                                pitch_dict.items() if
                                                    k in pitch_fields}
                                pitch_dict = clean_zero_len_string(
                                                pitch_dict)
                                pitch = Pitch(**pitch_dict)
                                pitch.save()

                            elif pitch_runner_elem.tag.lower() == 'runner':
                                runner_elem = pitch_runner_elem
                                runner_dict = dict(runner_elem.items())
                                runner_dict['atbat'] = atbat
                                runner_dict['runner'] = \
                                    Player.objects.get(id=runner_dict['id'])
                                runner_dict = {k: v for k, v in
                                                runner_dict.items() if
                                                    k in runner_fields}
                                runner_dict = clean_zero_len_string(
                                                runner_dict)
                                runner = Runner(**runner_dict)
                                runner.save()

                    elif event_elem.tag.lower() == 'action':
                        action_dict = dict(event_elem.items())
                        action_dict['inning'] = inning
                        action_dict['description'] = \
                                        action_dict.pop('des', None)
                        action_dict['description_spanish'] = \
                                        action_dict.pop('des_es', None)

                        action_dict = {k: v for k, v in action_dict.items()
                                        if k in action_fields}
                        action_dict = clean_zero_len_string(action_dict)
                        action = Action(**action_dict)
                        action.save()
        game.start_datetime = game.inning_set.first(
                                ).atbat_set.first(
                                ).start_tfs_zulu
        game.save()
    return

base_url = 'http://gd2.mlb.com/components/game/mlb/'

recent_datetime_dict = Game.objects.all().aggregate(Max('start_datetime'))
recent_datetime = recent_datetime_dict['start_datetime__max']
if recent_datetime:
    start_datetime = recent_datetime - datetime.timedelta(days=1)
else:
    start_datetime = datetime.datetime(2011, 3, 1, 0, 0, 0).replace(tzinfo=UTC)
cur_datetime = start_datetime
end_datetime = datetime.datetime.now().replace(tzinfo=UTC) + \
               datetime.timedelta(days=1)

while cur_datetime < end_datetime:
    print 'loading games from ' + cur_datetime.date().isoformat()
    day_url = base_url + 'year_%d/month_%02d/day_%02d' % (cur_datetime.year,
                                                           cur_datetime.month,
                                                           cur_datetime.day,)

    try:
        day_html = urllib2.urlopen(day_url).read()
        day_soup = BeautifulSoup(day_html, 'html.parser')
        game_links = [link.get('href')
                        for link in day_soup.find_all('a')
                          if link.get('href').find('gid_') >= 0]
    except urllib2.HTTPError:
        game_links = []

    for game_link in game_links:
        game_url = day_url + '/' + game_link[game_link.find('/')+1:]
        print 'game url: %s' % game_url
        try:
            add_game(game_url)
        except:
            print 'error reading game at: %s' % game_url
            continue

    cur_datetime += datetime.timedelta(days=1)
<commit_msg>made a main function<commit_after>#!/usr/bin/env python

import os
import re
import urllib2
import datetime
import xml.etree.ElementTree as ET

from pytz import UTC
from bs4 import BeautifulSoup
from django.db import transaction
from django.db.models import Max

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mlbslicedb.settings")
import django
django.setup()

from gd2.models import Pitch, Runner, Atbat, Action, Inning, Game, Player, Team

action_fields = set(f.name for f in Action._meta.get_fields())
atbat_fields = set(f.name for f in Atbat._meta.get_fields())
pitch_fields = set(f.name for f in Pitch._meta.get_fields())
runner_fields = set(f.name for f in Runner._meta.get_fields())

def clean_zero_len_string(obj_dict):
    for k,v in obj_dict.items():
        if v == '':
            obj_dict[k] = None
    return obj_dict

def add_team(team_dict):
    try:
        team = Team.objects.get(id=team_dict['id'])
        team.name = team_dict['name']
        team.abbrev = team_dict['abbrev']
    except Team.DoesNotExist:
        team = Team(**team_dict)
    team.save()
    return team

def add_player(player_dict):
    try:
        player = Player.objects.get(id=player_dict['id'])
        player.first = player_dict['first']
        player.last = player_dict['last']
    except Player.DoesNotExist:
        player = Player(id=player_dict['id'],
                        first=player_dict['first'],
                        last=player_dict['last'],)
    player.save()
    return player

@transaction.atomic
def add_game(game_url):
    gamecenter_url = game_url + 'gamecenter.xml'
    game_game_url = game_url + 'game.xml'
    gamecenter_xml = urllib2.urlopen(gamecenter_url).read()
    gamecenter_root = ET.fromstring(gamecenter_xml)
    gamecenter_attrib = gamecenter_root.attrib
    game_id = gamecenter_attrib['id']
    game_status = gamecenter_attrib['status']
    game_type = gamecenter_attrib['type']
    probables_element = gamecenter_root.findall('probables')[0]
    home_probable_element = probables_element.findall('home')[0]
    home_prob_id = home_probable_element.findall('player_id')[0].text
    home_prob_first = home_probable_element.findall('useName')[0].text
    home_prob_last = home_probable_element.findall('lastName')[0].text

    away_probable_element = probables_element.findall('away')[0]
    away_prob_id = home_probable_element.findall('player_id')[0].text
    away_prob_first = home_probable_element.findall('useName')[0].text
    away_prob_last = home_probable_element.findall('lastName')[0].text

    game_game_xml = urllib2.urlopen(game_game_url).read()
    game_game_root = ET.fromstring(game_game_xml)
    game_game_attrib = game_game_root.attrib
    team_elements = game_game_root.findall('team')
    for team_element in team_elements:
        team_attrib = team_element.attrib
        team_dict = {'id': team_attrib['id'],
                     'name': team_attrib['name'],
                     'abbrev': team_attrib['abbrev'],}
        if team_attrib['type'] == 'home':
            home_team = add_team(team_dict)
        elif team_attrib['type'] == 'away':
            away_team = add_team(team_dict)
        else:
            pass

    try:
        game = Game.objects.get(id=game_id)
        print 'game already exists: %s' % game_id
        return
    except Game.DoesNotExist:
        game = Game(id=game_id,)

    if home_prob_id:
        home_prob_player = add_player({'id': int(home_prob_id),
                                       'first': home_prob_first,
                                       'last': home_prob_last,})
        game.home_probable = home_prob_player

    if away_prob_id:
        away_prob_player = add_player({'id': int(away_prob_id),
                                       'first': away_prob_first,
                                       'last': away_prob_last,})
        game.away_probable = away_prob_player

    game.home_team = home_team
    game.away_team = away_team
    game.status = game_status
    game.type = game_type

    if game.status == 'F':
        box_url = game_url + 'boxscore.xml'
        box_xml = urllib2.urlopen(box_url).read()
        box_root = ET.fromstring(box_xml)
        box_attrib = box_root.attrib
        linescore_element = box_root.findall('linescore')[0]
        linescore_attrib = linescore_element.attrib
        away_runs = linescore_attrib['away_team_runs']
        away_hits = linescore_attrib['away_team_hits']
        home_runs = linescore_attrib['home_team_runs']
        home_hits = linescore_attrib['home_team_hits']
        game.away_runs = away_runs
        game.away_hits = away_hits
        game.home_runs = home_runs
        game.home_hits = home_hits
        game.save()

        players_url = game_url + 'players.xml'
        players_xml = urllib2.urlopen(players_url).read()
        players_root = ET.fromstring(players_xml)

        for team_elem in players_root.findall('team'):
            for player_elem in team_elem.findall('player'):
                add_player(player_elem.attrib)

        innings_url = game_url + 'inning/'
        innings_html = urllib2.urlopen(innings_url).read()
        innings_soup = BeautifulSoup(innings_html, 'html.parser')
        inning_links = [link.get('href') for
                            link in innings_soup.find_all('a')
                                if re.search('inning_\d',link.get('href'))]

        for inning_link in inning_links:
            inning_url = innings_url + inning_link
            inning_xml = urllib2.urlopen(inning_url).read()
            inning_root = ET.fromstring(inning_xml)

            for half_elem in inning_root.getchildren():
                inning_dict = {'game': Game.objects.get(id=game_id),
                               'top_bottom': half_elem.tag,
                               'num': int(inning_root.attrib['num']),
                               'away_team': away_team,
                               'home_team': home_team,
                               'next': inning_root.attrib['next'],}
                inning_dict = clean_zero_len_string(inning_dict)
                inning = Inning(**inning_dict)
                inning.save()

                for event_elem in half_elem.getchildren():
                    if event_elem.tag.lower() == 'atbat':
                        atbat_dict = dict(event_elem.items())
                        atbat_dict['inning'] = inning
                        atbat_dict['batter'] = Player.objects.get(
                                                id=atbat_dict['batter'])
                        atbat_dict['pitcher'] = Player.objects.get(
                                                id=atbat_dict['pitcher'])
                        atbat_dict['description'] = \
                                        atbat_dict.pop('des', None)
                        atbat_dict['description_spanish'] = \
                                        atbat_dict.pop('des_es', None)

                        atbat_dict = {k: v for k, v in atbat_dict.items()
                                        if k in atbat_fields}
                        atbat_dict = clean_zero_len_string(atbat_dict)
                        atbat = Atbat(**atbat_dict)
                        atbat.save()

                        for pitch_runner_elem in event_elem.getchildren():
                            if pitch_runner_elem.tag.lower() == 'pitch':
                                pitch_elem = pitch_runner_elem
                                pitch_dict = dict(pitch_elem.items())
                                pitch_dict['atbat'] = atbat
                                pitch_dict['local_id'] = pitch_dict.pop(
                                                                'id', None)
                                pitch_dict['description'] = \
                                            pitch_dict.pop('des', None)
                                pitch_dict['description_spanish'] = \
                                            pitch_dict.pop('des_es', None)
                                pitch_dict = {k: v for k, v in
                                                pitch_dict.items() if
                                                    k in pitch_fields}
                                pitch_dict = clean_zero_len_string(
                                                pitch_dict)
                                pitch = Pitch(**pitch_dict)
                                pitch.save()

                            elif pitch_runner_elem.tag.lower() == 'runner':
                                runner_elem = pitch_runner_elem
                                runner_dict = dict(runner_elem.items())
                                runner_dict['atbat'] = atbat
                                runner_dict['runner'] = \
                                    Player.objects.get(id=runner_dict['id'])
                                runner_dict = {k: v for k, v in
                                                runner_dict.items() if
                                                    k in runner_fields}
                                runner_dict = clean_zero_len_string(
                                                runner_dict)
                                runner = Runner(**runner_dict)
                                runner.save()

                    elif event_elem.tag.lower() == 'action':
                        action_dict = dict(event_elem.items())
                        action_dict['inning'] = inning
                        action_dict['description'] = \
                                        action_dict.pop('des', None)
                        action_dict['description_spanish'] = \
                                        action_dict.pop('des_es', None)

                        action_dict = {k: v for k, v in action_dict.items()
                                        if k in action_fields}
                        action_dict = clean_zero_len_string(action_dict)
                        action = Action(**action_dict)
                        action.save()
        game.start_datetime = game.inning_set.first(
                                ).atbat_set.first(
                                ).start_tfs_zulu
        game.save()
    return

if __name__ == "__main__":
  base_url = 'http://gd2.mlb.com/components/game/mlb/'

  recent_datetime_dict = Game.objects.all().aggregate(Max('start_datetime'))
  recent_datetime = recent_datetime_dict['start_datetime__max']
  if recent_datetime:
      start_datetime = recent_datetime - datetime.timedelta(days=1)
  else:
      start_datetime = datetime.datetime(2011, 3, 1, 0, 0, 0).replace(
                                                                tzinfo=UTC)
  cur_datetime = start_datetime
  end_datetime = datetime.datetime.now().replace(tzinfo=UTC) + \
                 datetime.timedelta(days=1)

  while cur_datetime < end_datetime:
      print 'loading games from ' + cur_datetime.date().isoformat()
      day_url = base_url + 'year_%d/month_%02d/day_%02d' % (cur_datetime.year,
                                                             cur_datetime.month,
                                                             cur_datetime.day,)

      try:
          day_html = urllib2.urlopen(day_url).read()
          day_soup = BeautifulSoup(day_html, 'html.parser')
          game_links = [link.get('href')
                          for link in day_soup.find_all('a')
                            if link.get('href').find('gid_') >= 0]
      except urllib2.HTTPError:
          game_links = []

      for game_link in game_links:
          game_url = day_url + '/' + game_link[game_link.find('/')+1:]
          print 'game url: %s' % game_url
          try:
              add_game(game_url)
          except:
              print 'error reading game at: %s' % game_url
              continue

      cur_datetime += datetime.timedelta(days=1)
<|endoftext|>


<commit_before># This script reads input from the Google Form:
# https://docs.google.com/forms/d/e/1FAIpQLScuM_jGV5MO2K6CCurHlC7s_YwIVQcOuLPExobCq9uDnX4p5Q/viewform
# Each submission is converted into an article of one of the contribution types supported by the site https://bssw.io

# required install package: Run those two lines below in the Terminal
# pip install oauth2client==1.5.2
# pip install gspread==0.6.2

import gspread
from oauth2client.client import SignedJwtAssertionCredentials
import json
import os
from datetime import datetime
from dateutil.parser import *

# Find the correct contribution type
CONTRIBUTION_LOOKUP = {
    "Original experience": "Articles",
    "Curated links": "CuratedContent",
    "Event": "Events",
    "Blog Article": "Site",
}

# Below are the indexes of necessary information in the Google Sheet.
# Column "A" in Tab "Form Responses 1" will be index 0, "B" is index 1 and so on.

TIMESTAMP = 0 # Date and time of contribution submission
FIRST_NAME = 3  # first name
LAST_NAME = 4  # last name
EMAIL_ADDRESS = 5 # contributor's email address
GITHUB_ID = 6 # contributor's GitHub ID
CATEGORY = 7  # category (Planning, Development, Performance, Reliability, Collaboration, Skills)
TOPICS = range(8, 13)  # each category has a collection of possible topic.  We scan all since only the selected category will have topics.
CONTRIBUTION_TYPE = 14  # Possible values: Original experience, Blog article, Curated Links, Event
FILE_KEY = 15 # 20 or fewer characters used to construct a unique file name
ARTICLE_TITLE = [ 16, 18, 20] # Contribution's title
ARTICLE_TEXT = [17, 19, 21]  # indexes of file content
EVENT_TITLE = 22
EVENT_START = 23
EVENT_END = 24
EVENT_LOCATION = 25
EVENT_WEBSITE = 26
EVENT_SHORT_DESC = 27
EVENT_LONG_DESC = 28
PREFIX = "BSSW_GENERATED_FILES"  # Folder for Generated Files

# get the google client - no need to change
def get_google_client():
    with open('google_key.json') as f:
        config = json.load(f)

    creds = SignedJwtAssertionCredentials(
        config['client_email'],
        config['private_key'],
        ['https://spreadsheets.google.com/feeds']
    )
    return gspread.authorize(creds)

# create appropriate file path for the new files
def generate_filepath(values):
    # decide the contribution type for build file prefix
    contribution_type = CONTRIBUTION_LOOKUP[values[CONTRIBUTION_TYPE]]

    # name of the file and filepath
    name = values[LAST_NAME] + values[FILE_KEY]

    filepath = "{}/{}/{}.md".format(PREFIX, contribution_type, name)

    # check if the file already exist
    if os.path.isfile(filepath):
        suffix = 1
        while True:
            filepath = "{}/{}/{}{}.md".format(PREFIX, contribution_type, name, suffix)
            if not os.path.isfile(filepath):
                return filepath
            suffix += 1
    return filepath

# convert date the the correct format
def convert_date(date_str1, date_str2):
    # convert string date to date
    date1 = datetime.strptime(date_str1, '%m/%d/%Y')
    date2 = datetime.strptime(date_str2, '%m/%d/%Y')

    # convert date bback to string in the right format
    if date1.year == date2.year:
        return "{} - {}".format(datetime.strftime(date1, "%B %d"), datetime.strftime(date2, "%B %d, %Y"))
    return "{} - {}".format(datetime.strftime(date1, "%B %d, %Y"), datetime.strftime(date2, "%B %d, %Y"))

# get the topic from Google sheet
def get_topic(values):
    for i in TOPICS:
        if values[i]:
            return values[i]
    return ""

# write content to file
def write_file(values, fo):

    timestamp = parse(values[TIMESTAMP])
    # build contributor string.  Define as first/last name.
    contributor = values[FIRST_NAME] + " " + values[LAST_NAME]
    # convert to GitHub link if available
    if values[GITHUB_ID]:
        contributor = "[" + contributor + "](https://github.com/" + values[GITHUB_ID] + ")"

# check if the event
    if values[EVENT_LOCATION] != "":
        fo.write("# {}".format(values[EVENT_TITLE]))
        fo.write("\n\n- Dates: " + convert_date(values[EVENT_START], values[EVENT_END]))
        fo.write("\n- Location: {}".format(values[EVENT_LOCATION]))
        fo.write("\n- Event Website: {}".format(values[EVENT_WEBSITE]))
        fo.write("\n\n{}".format(values[EVENT_SHORT_DESC]))
        fo.write("\n\n**Description**: {}".format(values[EVENT_LONG_DESC]))
        fo.write("\n\n#### Contributed by {}".format(contributor))
        fo.write("\n\n#### Publication date: {}".format(timestamp.strftime('%B %d, %Y')))

    # for blog, article and curated links
    else:
        for i in ARTICLE_TITLE:
            if values[i] != "":
                fo.write("\n# "+values[i])

        fo.write("\n\n#### Contributed by {}".format(contributor))
        fo.write("\n\n#### Publication date: {}".format(timestamp.strftime('%B %d, %Y')))

        for i in ARTICLE_TEXT:
            if values[i] != "":
                fo.write("\n\n"+values[i])

# write content to files
def process_value(values):
    # get topic and file path
    topic = get_topic(values)
    filepath = generate_filepath(values)

    print("\nGenerating {}\n".format(filepath))

    # open file and write
    with open(filepath, "w") as fo:
        write_file(values, fo)
        fo.write("\n\n<!---")
        fo.write("\nPublish: no")
        fo.write("\nCategories: {}".format(values[CATEGORY]))
        fo.write("\nTopics: {}".format(topic))
        fo.write("\nLevel: 2")
        fo.write("\nPrerequisites: defaults")
        fo.write("\nAggregate: none")
        fo.write("\n--->\n")

def run():
    print("Read Google Form")

    gc = get_google_client()

    # Open file: key in the Google Sheet URL
#   remember to share the sheet with duong-13@bssw-195403.iam.gserviceaccount.com
    doc = gc.open_by_key('1u6CLr6LbVbbWnpUVij1_z_ugWwAy-5RI7d6QwyFxK00')

    # Open tab: tab name in the Google Sheet
    sheet = doc.worksheet("Form Responses 1")

    row_count = sheet.row_count
    # run through all rows of the Google Sheet
    for row in range(2, row_count + 1):
        values = sheet.row_values(row)
        # Blank row
        if not values[0]:
            break

        # process the values collected from Google Sheet
        process_value(values)

if __name__ == '__main__':
    run()
<commit_msg>Add detailed user instructions<commit_after># This script reads input from the Google Form:
# https://docs.google.com/forms/d/e/1FAIpQLScuM_jGV5MO2K6CCurHlC7s_YwIVQcOuLPExobCq9uDnX4p5Q/viewform
# using the key string '1u6CLr6LbVbbWnpUVij1_z_ugWwAy-5RI7d6QwyFxK00' (ID of the generated Google spreadsheet).
# Each submission is converted into an article of one of the contribution types supported by the site https://bssw.io
# This script should be executed on any platform that supports python.
#
# Steps:
# 1. Clone or update the betterscientificsoftware.github.io repo onto your local system.
#
# 2. Execute this script.
#
# 3. After executing this script, any contributions found in the spreadsheet will be found in properly formatted markdown
# files with the publish field set to 'no'.
# Files will be found in:
#   BSSW_GENERATED_FILES/Articles
#   BSSW_GENERATED_FILES/CuratedContent
#   BSSW_GENERATED_FILES/Events
#   BSSW_GENERATED_FILES/Site
#
# 4. Review the generated files. Some submissions may have non-ASCII characters if the author copied and pasted from
#    an editor that supports such symbols.  For example, MS Word will convert standard quotes to non-standard quotes.
#    The spreadsheet cells can be directly edited to remove these anomolies.
#
# 5. After reviewing the content, these files can be moved to the main folders.  For example, files in
#    BSSW_GENERATED_FILES/Articles should be moved to the top-level folder Articles.
#
# 6. Commit the new files back to the GitHub repo.
#
# 7. From the GitHub web interface to betterscientificsoftware.github.io, confirm that the new files are present
#    and render properly in the web preview.
#
# 8. Edit the file from the web interface. Fix any rendering errors and change the Publish field in the meta-date
#    block at the bottom of the article to yes.
# 9. You are done! The new files should appear on bssw.io after the next auto-generate cycle.

# required install package: Run those two lines below in the Terminal
# pip install oauth2client==1.5.2
# pip install gspread==0.6.2

import gspread
from oauth2client.client import SignedJwtAssertionCredentials
import json
import os
from datetime import datetime
from dateutil.parser import *

# Find the correct contribution type
CONTRIBUTION_LOOKUP = {
    "Original experience": "Articles",
    "Curated links": "CuratedContent",
    "Event": "Events",
    "Blog Article": "Site",
}

# Below are the indexes of necessary information in the Google Sheet.
# Column "A" in Tab "Form Responses 1" will be index 0, "B" is index 1 and so on.

TIMESTAMP = 0 # Date and time of contribution submission
FIRST_NAME = 3  # first name
LAST_NAME = 4  # last name
EMAIL_ADDRESS = 5 # contributor's email address
GITHUB_ID = 6 # contributor's GitHub ID
CATEGORY = 7  # category (Planning, Development, Performance, Reliability, Collaboration, Skills)
TOPICS = range(8, 13)  # each category has a collection of possible topic.  We scan all since only the selected category will have topics.
CONTRIBUTION_TYPE = 14  # Possible values: Original experience, Blog article, Curated Links, Event
FILE_KEY = 15 # 20 or fewer characters used to construct a unique file name
ARTICLE_TITLE = [ 16, 18, 20] # Contribution's title
ARTICLE_TEXT = [17, 19, 21]  # indexes of file content
EVENT_TITLE = 22
EVENT_START = 23
EVENT_END = 24
EVENT_LOCATION = 25
EVENT_WEBSITE = 26
EVENT_SHORT_DESC = 27
EVENT_LONG_DESC = 28
PREFIX = "BSSW_GENERATED_FILES"  # Folder for Generated Files

# get the google client - no need to change
def get_google_client():
    with open('google_key.json') as f:
        config = json.load(f)

    creds = SignedJwtAssertionCredentials(
        config['client_email'],
        config['private_key'],
        ['https://spreadsheets.google.com/feeds']
    )
    return gspread.authorize(creds)

# create appropriate file path for the new files
def generate_filepath(values):
    # decide the contribution type for build file prefix
    contribution_type = CONTRIBUTION_LOOKUP[values[CONTRIBUTION_TYPE]]

    # name of the file and filepath
    name = values[LAST_NAME] + values[FILE_KEY]

    filepath = "{}/{}/{}.md".format(PREFIX, contribution_type, name)

    # check if the file already exist
    if os.path.isfile(filepath):
        suffix = 1
        while True:
            filepath = "{}/{}/{}{}.md".format(PREFIX, contribution_type, name, suffix)
            if not os.path.isfile(filepath):
                return filepath
            suffix += 1
    return filepath

# convert date the the correct format
def convert_date(date_str1, date_str2):
    # convert string date to date
    date1 = datetime.strptime(date_str1, '%m/%d/%Y')
    date2 = datetime.strptime(date_str2, '%m/%d/%Y')

    # convert date bback to string in the right format
    if date1.year == date2.year:
        return "{} - {}".format(datetime.strftime(date1, "%B %d"), datetime.strftime(date2, "%B %d, %Y"))
    return "{} - {}".format(datetime.strftime(date1, "%B %d, %Y"), datetime.strftime(date2, "%B %d, %Y"))

# get the topic from Google sheet
def get_topic(values):
    for i in TOPICS:
        if values[i]:
            return values[i]
    return ""

# write content to file
def write_file(values, fo):

    timestamp = parse(values[TIMESTAMP])
    # build contributor string.  Define as first/last name.
    contributor = values[FIRST_NAME] + " " + values[LAST_NAME]
    # convert to GitHub link if available
    if values[GITHUB_ID]:
        contributor = "[" + contributor + "](https://github.com/" + values[GITHUB_ID] + ")"

# check if the event
    if values[EVENT_LOCATION] != "":
        fo.write("# {}".format(values[EVENT_TITLE]))
        fo.write("\n\n- Dates: " + convert_date(values[EVENT_START], values[EVENT_END]))
        fo.write("\n- Location: {}".format(values[EVENT_LOCATION]))
        fo.write("\n- Event Website: {}".format(values[EVENT_WEBSITE]))
        fo.write("\n\n{}".format(values[EVENT_SHORT_DESC]))
        fo.write("\n\n**Description**: {}".format(values[EVENT_LONG_DESC]))
        fo.write("\n\n#### Contributed by {}".format(contributor))
        fo.write("\n\n#### Publication date: {}".format(timestamp.strftime('%B %d, %Y')))

    # for blog, article and curated links
    else:
        for i in ARTICLE_TITLE:
            if values[i] != "":
                fo.write("\n# "+values[i])

        fo.write("\n\n#### Contributed by {}".format(contributor))
        fo.write("\n\n#### Publication date: {}".format(timestamp.strftime('%B %d, %Y')))

        for i in ARTICLE_TEXT:
            if values[i] != "":
                fo.write("\n\n"+values[i])

# write content to files
def process_value(values):
    # get topic and file path
    topic = get_topic(values)
    filepath = generate_filepath(values)

    print("\nGenerating {}\n".format(filepath))

    # open file and write
    with open(filepath, "w") as fo:
        write_file(values, fo)
        fo.write("\n\n<!---")
        fo.write("\nPublish: no")
        fo.write("\nCategories: {}".format(values[CATEGORY]))
        fo.write("\nTopics: {}".format(topic))
        fo.write("\nLevel: 2")
        fo.write("\nPrerequisites: defaults")
        fo.write("\nAggregate: none")
        fo.write("\n--->\n")

def run():
    print("Read Google Form")

    gc = get_google_client()

    # Open file: key in the Google Sheet URL
#   remember to share the sheet with duong-13@bssw-195403.iam.gserviceaccount.com
    doc = gc.open_by_key('1u6CLr6LbVbbWnpUVij1_z_ugWwAy-5RI7d6QwyFxK00')

    # Open tab: tab name in the Google Sheet
    sheet = doc.worksheet("Form Responses 1")

    row_count = sheet.row_count
    # run through all rows of the Google Sheet
    for row in range(2, row_count + 1):
        values = sheet.row_values(row)
        # Blank row
        if not values[0]:
            break

        # process the values collected from Google Sheet
        process_value(values)

if __name__ == '__main__':
    run()
<|endoftext|>