<commit_before>import picamera
from RPi import GPIO
import os
import sendgrid
from sendgrid.helpers.mail import Email, Content, Mail
from gpiozero import MotionSensor
from datetime import datetime
import time

SERVO_PIN = 13

def provision_pi_camera(hflip=False, vflip=False, zoom=(0.0, 0.0, 1.0, 1.0), video_stabilization=True):
    camera = picamera.PiCamera()
    camera.hflip = hflip
    camera.vflip = vflip
    camera.video_stabilization = video_stabilization
    camera.zoom = zoom
    return camera

def rotate_servo(angle):
    DC = 0
    if angle == 90:
        DC = 7
    elif angle == 0:
        DC = 2
    elif angle == 180:
        DC = 12
    else:
        raise ValueError("Accepted values for servo rotation are 0, 90, and 180.")

    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.ChangeDutyCycle(DC)   # turn towards 90 degree
    time.sleep(1)            # sleep 1 second

def reset_servo():
    rotate_servo(90)

def send_email():
    # This function will need to take in pictures and add them as attachments.
    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    from_email = Email("SpyPi@no-reply")
    subject = "Motion Detected"
    to_email = Email("sendToUser@example.com")
    content = Content("text/plain", "We have detected motion from your pi!\n\n")
    mail = Mail(from_email, subject, to_email, content)
    response = sg.client.mail.send.post(request_body=mail.get())
    print(response.status_code)
    print(response.body)
    print(response.headers)

def main():
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(SERVO_PIN, GPIO.OUT)
    # pwm object on servo pin with 50Hz signal
    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.start(7)
    # ----------End of Servo Setup -------------
    pir = MotionSensor(4)
    with provision_pi_camera() as camera:
        while True:
            if pir.motion_detected:
                now = datetime.now()
                camera.capture("{}-middle.jpg".format(now.iso_format()))
                rotate_servo(0)
                camera.capture("{}-left.jpg".format(now.iso_format()))
                rotate_servo(180)
                camera.capture("{}-right.jpg".format(now.iso_format()))
                reset_servo()
                send_email()

if __name__ == "__main__":
    main()
<commit_msg>Working Correctly<commit_after>import picamera
from RPi import GPIO
import os
import sendgrid
from sendgrid.helpers.mail import *
from gpiozero import MotionSensor
from datetime import datetime
import time

SERVO_PIN = 13
MOTION_PIN = 11

def provision_pi_camera(hflip=False, vflip=False, zoom=(0.0, 0.0, 1.0, 1.0), video_stabilization=True):
    camera = picamera.PiCamera()
    camera.hflip = hflip
    camera.vflip = vflip
    camera.video_stabilization = video_stabilization
    camera.zoom = zoom
    return camera

def rotate_servo(angle):
    DC = 0
    if angle == 90:
        DC = 7
    elif angle == 0:
        DC = 2
    elif angle == 180:
        DC = 12
    else:
        raise ValueError("Accepted values for servo rotation are 0, 90, and 180.")

    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.ChangeDutyCycle(DC)   # turn towards 90 degree
    time.sleep(1)            # sleep 1 second

def reset_servo():
    rotate_servo(90)

def send_email():
    # This function will need to take in pictures and add them as attachments.
    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    from_email = Email("SpyPi@no-reply")
    subject = "Motion Detected"
    to_email = Email("gagnej3@wit.edu")
    content = Content("text/plain", "We have detected motion from your pi!\n\n")
    mail = Mail(from_email, subject, to_email, content)
    response = sg.client.mail.send.post(request_body=mail.get())
    print(response.status_code)
    print(response.body)
    print(response.headers)

def main():
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(SERVO_PIN, GPIO.OUT)

    # pwm object on servo pin with 50Hz signal
    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.start(7)
    # ----------End of Servo Setup -------------

    GPIO.setup(MOTION_PIN,GPIO.IN)
    with provision_pi_camera() as camera:
        while True:
            i = GPIO.input(MOTION_PIN)
	    if i==1:
		print 'motion detected'
                #pwm.start(7)
		pwm.ChangeDutyCycle(7)
		time.sleep(1)
		#rotate_servo(90)
		#time.sleep(1)
		now = datetime.now()
                camera.capture("{}-middle.jpg".format(now.isoformat()))
                pwm.ChangeDutyCycle(2)
		time.sleep(1)
		#rotate_servo(0)
		#time.sleep(1)
                camera.capture("{}-left.jpg".format(now.isoformat()))
          	pwm.ChangeDutyCycle(7)
		time.sleep(1)
		#rotate_servo(90)
		#rotate_servo(180)
		#time.sleep(1)
                camera.capture("{}-right.jpg".format(now.isoformat()))
                pwm.ChangeDutyCycle(12)
		time.sleep(1)
		pwm.ChangeDutyCycle(7)
		time.sleep(1)
		#rotate_servo(90)
		#reset_servo()
                #send_email()
		#time.sleep(4.1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
import logging.config
import jinja2
import sys
import os
import yaml
from flask import Flask, url_for, Blueprint, render_template

from cla_public.views.index import base_blueprint
from cla_public.views.checker import checker_blueprint, result_blueprint

log = logging.getLogger(__name__)

# Name of the environment variable that we read the config file from
CONFIG_FILE_ENV_NAME = 'CLA_PUBLIC_CONFIG'

# Name of the environment variable that we determine the verbosity of
# the logger.
VERBOSE_LOGGING_ENV_NAME = 'CLA_PUBLIC_VERBOSE'

def setup_logging(verbose=False):
    try:
        with open('logging.conf') as f:
            logging.config.dictConfig(yaml.load(f.read()))
    except (yaml.YAMLError, yaml.MarkedYAMLError, IOError):
        if verbose:
            level = logging.DEBUG
        else:
            level = logging.INFO

        logging.basicConfig(
            level=level,
            format='%(asctime)s %(levelname)-8s %(name)s %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')

        # logging for the cla_public module (top level)
        logger = logging.getLogger('cla_public')
        logger.setLevel(level)

def setup_config(app, config_name):
    """Initialises the configuration for a Flask application.

    After that, the environment variable CONFIG_FILE_ENV_NAME is
    consulted, and its value, which must point to a Python file either
    absolutely or relative to the root of the project."""
    try:
        user_config_file = os.environ[CONFIG_FILE_ENV_NAME]
        if not os.path.isfile(user_config_file):
            log.critical(
                'Config file %s is not a valid file. Exiting...',
                user_config_file)
            sys.exit(1)
        else:
            try:
                with open(user_config_file) as f:
                    config_data = yaml.load(f.read())
                app.config.update(config_data[config_name])
                log.info('Loaded configuration file %s', user_config_file)
            except (yaml.YAMLError, yaml.MarkedYAMLError):
                log.exception(
                    'Parsing YAML file %s failed. Exiting...',
                    user_config_file)
                sys.exit(1)
    except KeyError:
        log.critical(
            'Environment variable %s must be set. Exiting...',
            CONFIG_FILE_ENV_NAME)
        sys.exit(1)

def change_jinja_templates(app):
    # Change the template loader so it will seek out the MOJ Jinja
    # base templates.
    moj_loader = jinja2.ChoiceLoader([
        app.jinja_loader,
        jinja2.PackageLoader('moj_template', 'templates'),
    ])

    app.jinja_loader = moj_loader

    # we need to load a special method called "static" to mimic
    # Django; ideally we would not rely on Django-isms but the MOJ
    # template assumes you're using jinja with Django.
    import moj_template
    root_template_dir = moj_template.__path__[0]
    static_dir = os.path.join(root_template_dir, 'static')
    template_dir = os.path.join(root_template_dir,
                                'templates', 'moj_template')

    moj_template_blueprint = Blueprint('moj_template', 'moj_template',
                                       static_folder=static_dir,
                                       static_url_path='/moj-static',
                                       template_folder=template_dir)
    app.register_blueprint(moj_template_blueprint)

    @app.context_processor
    def utility_processor():
        def static(filename):
            return url_for('moj_template.static', filename=filename)
        return {'static': static}

    # Expose user variables
    @app.context_processor
    def moj_variables():
        try:
            return app.config['APP_SETTINGS']
        except KeyError:
            log.critical(
                'Cannot find APP_SETTINGS group in the configuration file.')
            sys.exit(1)
    return app

def register_error_handlers(app):
    @app.errorhandler(404)
    def http_page_not_found(e):
        return render_template('404.html'), 404

    @app.errorhandler(500)
    def http_server_error(e):
        return render_template('500.html'), 500

    return app

def create_app(config_name='FLASK'):
    app = Flask(__name__)
    # This should happen before other things
    app.register_blueprint(base_blueprint)
    app.register_blueprint(checker_blueprint)
    app.register_blueprint(result_blueprint)
    setup_logging(bool(os.environ.get(VERBOSE_LOGGING_ENV_NAME, False)))
    setup_config(app, config_name)
    app = change_jinja_templates(app)
    app = register_error_handlers(app)
    return app
<commit_msg>Use reorganised blueprints<commit_after># -*- coding: utf-8 -*-
"CLA Public app"

import logging
import logging.config
import jinja2
import sys
import os
import yaml
from flask import Blueprint, Flask, url_for, render_template

from cla_public.apps.base.views import base
from cla_public.apps.checker.views import checker

log = logging.getLogger(__name__)

# Name of the environment variable that we read the config file from
CONFIG_FILE_ENV_NAME = 'CLA_PUBLIC_CONFIG'

# Name of the environment variable that we determine the verbosity of
# the logger.
VERBOSE_LOGGING_ENV_NAME = 'CLA_PUBLIC_VERBOSE'

def setup_logging(verbose=False):
    try:
        with open('logging.conf') as f:
            logging.config.dictConfig(yaml.load(f.read()))
    except (yaml.YAMLError, yaml.MarkedYAMLError, IOError):
        if verbose:
            level = logging.DEBUG
        else:
            level = logging.INFO

        logging.basicConfig(
            level=level,
            format='%(asctime)s %(levelname)-8s %(name)s %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S')

        # logging for the cla_public module (top level)
        logger = logging.getLogger('cla_public')
        logger.setLevel(level)

def setup_config(app, config_name):
    """Initialises the configuration for a Flask application.

    After that, the environment variable CONFIG_FILE_ENV_NAME is
    consulted, and its value, which must point to a Python file either
    absolutely or relative to the root of the project."""
    try:
        user_config_file = os.environ[CONFIG_FILE_ENV_NAME]
        if not os.path.isfile(user_config_file):
            log.critical(
                'Config file %s is not a valid file. Exiting...',
                user_config_file)
            sys.exit(1)
        else:
            try:
                with open(user_config_file) as f:
                    config_data = yaml.load(f.read())
                app.config.update(config_data[config_name])
                log.info('Loaded configuration file %s', user_config_file)
            except (yaml.YAMLError, yaml.MarkedYAMLError):
                log.exception(
                    'Parsing YAML file %s failed. Exiting...',
                    user_config_file)
                sys.exit(1)
    except KeyError:
        log.critical(
            'Environment variable %s must be set. Exiting...',
            CONFIG_FILE_ENV_NAME)
        sys.exit(1)

def change_jinja_templates(app):
    # Change the template loader so it will seek out the MOJ Jinja
    # base templates.
    moj_loader = jinja2.ChoiceLoader([
        app.jinja_loader,
        jinja2.PackageLoader('moj_template', 'templates'),
    ])

    app.jinja_loader = moj_loader

    # we need to load a special method called "static" to mimic
    # Django; ideally we would not rely on Django-isms but the MOJ
    # template assumes you're using jinja with Django.
    import moj_template
    root_template_dir = moj_template.__path__[0]
    static_dir = os.path.join(root_template_dir, 'static')
    template_dir = os.path.join(root_template_dir,
                                'templates', 'moj_template')

    moj_template_blueprint = Blueprint('moj_template', 'moj_template',
                                       static_folder=static_dir,
                                       static_url_path='/moj-static',
                                       template_folder=template_dir)
    app.register_blueprint(moj_template_blueprint)

    @app.context_processor
    def utility_processor():
        def static(filename):
            return url_for('moj_template.static', filename=filename)
        return {'static': static}

    # Expose user variables
    @app.context_processor
    def moj_variables():
        try:
            return app.config['APP_SETTINGS']
        except KeyError:
            log.critical(
                'Cannot find APP_SETTINGS group in the configuration file.')
            sys.exit(1)
    return app

def register_error_handlers(app):
    @app.errorhandler(404)
    def http_page_not_found(e):
        return render_template('404.html'), 404

    @app.errorhandler(500)
    def http_server_error(e):
        return render_template('500.html'), 500

    return app

def create_app(config_name='FLASK'):
    app = Flask(__name__)
    app.register_blueprint(base)
    app.register_blueprint(checker)
    setup_logging(bool(os.environ.get(VERBOSE_LOGGING_ENV_NAME, False)))
    setup_config(app, config_name)
    app = change_jinja_templates(app)
    app = register_error_handlers(app)
    return app
<|endoftext|>


<commit_before>'''This script is experimental.

Try pre-training the CNN component of the text categorizer using a cheap
language modelling-like objective. Specifically, we load pre-trained vectors
(from something like word2vec, GloVe, FastText etc), and use the CNN to
predict the tokens' pre-trained vectors. This isn't as easy as it sounds:
we're not merely doing compression here, because heavy dropout is applied,
including over the input words. This means the model must often (50% of the time)
use the context in order to predict the word.

To evaluate the technique, we're pre-training with the 50k texts from the IMDB
corpus, and then training with only 100 labels. Note that it's a bit dirty to
pre-train with the development data, but also not *so* terrible: we're not using
the development labels, after all --- only the unlabelled text.
'''
from __future__ import print_function, unicode_literals
import plac
import random
import numpy
import time
import ujson as json
from pathlib import Path
import sys

import spacy
from spacy.attrs import ID
from spacy.util import minibatch_by_words, use_gpu, compounding, ensure_path
from spacy._ml import Tok2Vec, flatten, chain, zero_init, create_default_optimizer
from thinc.v2v import Affine

def prefer_gpu():
    used = spacy.util.use_gpu(0)
    if used is None:
        return False
    else:
        import cupy.random
        cupy.random.seed(0)
        return True

def load_texts(path):
    '''Load inputs from a jsonl file.

    Each line should be a dict like {"text": "..."}
    '''
    path = ensure_path(path)
    with path.open('r', encoding='utf8') as file_:
        texts = [json.loads(line)['text'] for line in file_]
    random.shuffle(texts)
    return texts

def stream_texts():
    for line in sys.stdin:
        yield json.loads(line)['text']

def make_update(model, docs, optimizer, drop=0.):
    """Perform an update over a single batch of documents.

    docs (iterable): A batch of `Doc` objects.
    drop (float): The droput rate.
    optimizer (callable): An optimizer.
    RETURNS loss: A float for the loss.
    """
    predictions, backprop = model.begin_update(docs, drop=drop)
    loss, gradients = get_vectors_loss(model.ops, docs, predictions)
    backprop(gradients, sgd=optimizer)
    return loss

def get_vectors_loss(ops, docs, prediction):
    """Compute a mean-squared error loss between the documents' vectors and
    the prediction.

    Note that this is ripe for customization! We could compute the vectors
    in some other word, e.g. with an LSTM language model, or use some other
    type of objective.
    """
    # The simplest way to implement this would be to vstack the
    # token.vector values, but that's a bit inefficient, especially on GPU.
    # Instead we fetch the index into the vectors table for each of our tokens,
    # and look them up all at once. This prevents data copying.
    ids = ops.flatten([doc.to_array(ID).ravel() for doc in docs])
    target = docs[0].vocab.vectors.data[ids]
    d_scores = (prediction - target) / prediction.shape[0]
    loss = (d_scores**2).sum()
    return loss, d_scores

def create_pretraining_model(nlp, tok2vec):
    '''Define a network for the pretraining. We simply add an output layer onto
    the tok2vec input model. The tok2vec input model needs to be a model that
    takes a batch of Doc objects (as a list), and returns a list of arrays.
    Each array in the output needs to have one row per token in the doc.
    '''
    output_size = nlp.vocab.vectors.data.shape[1]
    output_layer = zero_init(Affine(output_size, drop_factor=0.0))
    model = chain(
        tok2vec,
        flatten,
        output_layer
    )
    model.output_layer = output_layer
    model.begin_training([nlp.make_doc('Give it a doc to infer shapes')])
    return model

class ProgressTracker(object):
    def __init__(self, frequency=100000):
        self.loss = 0.
        self.nr_word = 0
        self.words_per_epoch = Counter()
        self.frequency = frequency
        self.last_time = time.time()
        self.last_update = 0

    def update(self, epoch, loss, docs):
        self.loss += loss
        words_in_batch = sum(len(doc) for doc in docs)
        self.words_per_epoch[epoch] += words_in_batch
        self.nr_word += words_in_batch
        words_since_update = self.nr_word - self.last_update
        if words_since_update >= self.frequency:
            wps = words_since_update / (time.time() - self.last_time)
            self.last_update = self.nr_word
            self.last_time = time.time()
            status = (epoch, self.nr_word, '%.5f' % self.loss, int(wps))
            return status
        else:
            return None

@plac.annotations(
    texts_loc=("Path to jsonl file with texts to learn from", "positional", None, str),
    vectors_model=("Name or path to vectors model to learn from"),
    output_dir=("Directory to write models each epoch", "positional", None, str),
    width=("Width of CNN layers", "option", "cw", int),
    depth=("Depth of CNN layers", "option", "cd", int),
    embed_rows=("Embedding rows", "option", "er", int),
    dropout=("Dropout", "option", "d", float),
    seed=("Seed for random number generators", "option", "s", float),
    nr_iter=("Number of iterations to pretrain", "option", "i", int),
)
def pretrain(texts_loc, vectors_model, output_dir, width=128, depth=4,
        embed_rows=1000, dropout=0.2, nr_iter=1, seed=0):
    """
    Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components,
    using an approximate language-modelling objective. Specifically, we load
    pre-trained vectors, and train a component like a CNN, BiLSTM, etc to predict
    vectors which match the pre-trained ones. The weights are saved to a directory
    after each epoch. You can then pass a path to one of these pre-trained weights
    files to the 'spacy train' command.

    This technique may be especially helpful if you have little labelled data.
    However, it's still quite experimental, so your mileage may vary.

    To load the weights back in during 'spacy train', you need to ensure
    all settings are the same between pretraining and training. The API and
    errors around this need some improvement.
    """
    config = dict(locals())
    output_dir = ensure_path(output_dir)
    random.seed(seed)
    numpy.random.seed(seed)
    if not output_dir.exists():
        output_dir.mkdir()
    with (output_dir / 'config.json').open('w') as file_:
        file_.write(json.dumps(config))
    has_gpu = prefer_gpu()
    nlp = spacy.load(vectors_model)
    tok2vec = Tok2Vec(width, embed_rows,
                conv_depth=depth,
                pretrained_vectors=nlp.vocab.vectors.name,
                bilstm_depth=0, # Requires PyTorch. Experimental.
                cnn_maxout_pieces=2, # You can try setting this higher
                subword_features=True) # Set to False for character models, e.g. Chinese
    model = create_pretraining_model(nlp, tok2vec)
    optimizer = create_default_optimizer(model.ops)
    tracker = ProgressTracker()
    print('Epoch', '#Words', 'Loss', 'w/s')
    texts = stream_texts() if text_loc == '-' else load_texts(texts_loc)
    for epoch in range(nr_iter):
        for batch in minibatch_by_words(texts, tuples=False, size=50000):
            docs = [nlp.make_doc(text) for text in batch]
            loss = make_update(model, docs, optimizer, drop=dropout)
            progress = tracker.update(epoch, loss, docs)
            if progress:
                print(*progress)
            if texts_loc == '-' and progress.words_per_epoch[epoch] >= 10**7:
                break
        with model.use_params(optimizer.averages):
            with (output_dir / ('model%d.bin' % epoch)).open('wb') as file_:
                file_.write(tok2vec.to_bytes())
            with (output_dir / 'log.jsonl').open('a') as file_:
                file_.write(json.dumps({'nr_word': tracker.nr_word,
                    'loss': tracker.loss, 'epoch': epoch}))
        if texts_loc != '-':
            texts = load_texts(texts_loc)
<commit_msg>Fix pretrain<commit_after>'''This script is experimental.

Try pre-training the CNN component of the text categorizer using a cheap
language modelling-like objective. Specifically, we load pre-trained vectors
(from something like word2vec, GloVe, FastText etc), and use the CNN to
predict the tokens' pre-trained vectors. This isn't as easy as it sounds:
we're not merely doing compression here, because heavy dropout is applied,
including over the input words. This means the model must often (50% of the time)
use the context in order to predict the word.

To evaluate the technique, we're pre-training with the 50k texts from the IMDB
corpus, and then training with only 100 labels. Note that it's a bit dirty to
pre-train with the development data, but also not *so* terrible: we're not using
the development labels, after all --- only the unlabelled text.
'''
from __future__ import print_function, unicode_literals
import plac
import random
import numpy
import time
import ujson as json
from pathlib import Path
import sys
from collections import Counter

import spacy
from spacy.attrs import ID
from spacy.util import minibatch_by_words, use_gpu, compounding, ensure_path
from spacy._ml import Tok2Vec, flatten, chain, zero_init, create_default_optimizer
from thinc.v2v import Affine

def prefer_gpu():
    used = spacy.util.use_gpu(0)
    if used is None:
        return False
    else:
        import cupy.random
        cupy.random.seed(0)
        return True

def load_texts(path):
    '''Load inputs from a jsonl file.

    Each line should be a dict like {"text": "..."}
    '''
    path = ensure_path(path)
    with path.open('r', encoding='utf8') as file_:
        texts = [json.loads(line)['text'] for line in file_]
    random.shuffle(texts)
    return texts

def stream_texts():
    for line in sys.stdin:
        yield json.loads(line)['text']

def make_update(model, docs, optimizer, drop=0.):
    """Perform an update over a single batch of documents.

    docs (iterable): A batch of `Doc` objects.
    drop (float): The droput rate.
    optimizer (callable): An optimizer.
    RETURNS loss: A float for the loss.
    """
    predictions, backprop = model.begin_update(docs, drop=drop)
    loss, gradients = get_vectors_loss(model.ops, docs, predictions)
    backprop(gradients, sgd=optimizer)
    return loss

def get_vectors_loss(ops, docs, prediction):
    """Compute a mean-squared error loss between the documents' vectors and
    the prediction.

    Note that this is ripe for customization! We could compute the vectors
    in some other word, e.g. with an LSTM language model, or use some other
    type of objective.
    """
    # The simplest way to implement this would be to vstack the
    # token.vector values, but that's a bit inefficient, especially on GPU.
    # Instead we fetch the index into the vectors table for each of our tokens,
    # and look them up all at once. This prevents data copying.
    ids = ops.flatten([doc.to_array(ID).ravel() for doc in docs])
    target = docs[0].vocab.vectors.data[ids]
    d_scores = (prediction - target) / prediction.shape[0]
    loss = (d_scores**2).sum()
    return loss, d_scores

def create_pretraining_model(nlp, tok2vec):
    '''Define a network for the pretraining. We simply add an output layer onto
    the tok2vec input model. The tok2vec input model needs to be a model that
    takes a batch of Doc objects (as a list), and returns a list of arrays.
    Each array in the output needs to have one row per token in the doc.
    '''
    output_size = nlp.vocab.vectors.data.shape[1]
    output_layer = zero_init(Affine(output_size, drop_factor=0.0))
    model = chain(
        tok2vec,
        flatten,
        output_layer
    )
    model.output_layer = output_layer
    model.begin_training([nlp.make_doc('Give it a doc to infer shapes')])
    return model

class ProgressTracker(object):
    def __init__(self, frequency=100000):
        self.loss = 0.
        self.nr_word = 0
        self.words_per_epoch = Counter()
        self.frequency = frequency
        self.last_time = time.time()
        self.last_update = 0

    def update(self, epoch, loss, docs):
        self.loss += loss
        words_in_batch = sum(len(doc) for doc in docs)
        self.words_per_epoch[epoch] += words_in_batch
        self.nr_word += words_in_batch
        words_since_update = self.nr_word - self.last_update
        if words_since_update >= self.frequency:
            wps = words_since_update / (time.time() - self.last_time)
            self.last_update = self.nr_word
            self.last_time = time.time()
            status = (epoch, self.nr_word, '%.5f' % self.loss, int(wps))
            return status
        else:
            return None

@plac.annotations(
    texts_loc=("Path to jsonl file with texts to learn from", "positional", None, str),
    vectors_model=("Name or path to vectors model to learn from"),
    output_dir=("Directory to write models each epoch", "positional", None, str),
    width=("Width of CNN layers", "option", "cw", int),
    depth=("Depth of CNN layers", "option", "cd", int),
    embed_rows=("Embedding rows", "option", "er", int),
    dropout=("Dropout", "option", "d", float),
    seed=("Seed for random number generators", "option", "s", float),
    nr_iter=("Number of iterations to pretrain", "option", "i", int),
)
def pretrain(texts_loc, vectors_model, output_dir, width=128, depth=4,
        embed_rows=1000, dropout=0.2, nr_iter=1, seed=0):
    """
    Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components,
    using an approximate language-modelling objective. Specifically, we load
    pre-trained vectors, and train a component like a CNN, BiLSTM, etc to predict
    vectors which match the pre-trained ones. The weights are saved to a directory
    after each epoch. You can then pass a path to one of these pre-trained weights
    files to the 'spacy train' command.

    This technique may be especially helpful if you have little labelled data.
    However, it's still quite experimental, so your mileage may vary.

    To load the weights back in during 'spacy train', you need to ensure
    all settings are the same between pretraining and training. The API and
    errors around this need some improvement.
    """
    config = dict(locals())
    output_dir = ensure_path(output_dir)
    random.seed(seed)
    numpy.random.seed(seed)
    if not output_dir.exists():
        output_dir.mkdir()
    with (output_dir / 'config.json').open('w') as file_:
        file_.write(json.dumps(config))
    has_gpu = prefer_gpu()
    nlp = spacy.load(vectors_model)
    tok2vec = Tok2Vec(width, embed_rows,
                conv_depth=depth,
                pretrained_vectors=nlp.vocab.vectors.name,
                bilstm_depth=0, # Requires PyTorch. Experimental.
                cnn_maxout_pieces=2, # You can try setting this higher
                subword_features=True) # Set to False for character models, e.g. Chinese
    model = create_pretraining_model(nlp, tok2vec)
    optimizer = create_default_optimizer(model.ops)
    tracker = ProgressTracker()
    print('Epoch', '#Words', 'Loss', 'w/s')
    texts = stream_texts() if texts_loc == '-' else load_texts(texts_loc)
    for epoch in range(nr_iter):
        for batch in minibatch_by_words(texts, tuples=False, size=50000):
            docs = [nlp.make_doc(text) for text in batch]
            loss = make_update(model, docs, optimizer, drop=dropout)
            progress = tracker.update(epoch, loss, docs)
            if progress:
                print(*progress)
            if texts_loc == '-' and progress.words_per_epoch[epoch] >= 10**7:
                break
        with model.use_params(optimizer.averages):
            with (output_dir / ('model%d.bin' % epoch)).open('wb') as file_:
                file_.write(tok2vec.to_bytes())
            with (output_dir / 'log.jsonl').open('a') as file_:
                file_.write(json.dumps({'nr_word': tracker.nr_word,
                    'loss': tracker.loss, 'epoch': epoch}))
        if texts_loc != '-':
            texts = load_texts(texts_loc)
<|endoftext|>


<commit_before>import picamera
from RPi import GPIO
import os
import sendgrid
from sendgrid.helpers.mail import *
from gpiozero import MotionSensor
from datetime import datetime
import time

SERVO_PIN = 13
MOTION_PIN = 11

def provision_pi_camera(hflip=False, vflip=False, zoom=(0.0, 0.0, 1.0, 1.0), video_stabilization=True):
    camera = picamera.PiCamera()
    camera.hflip = hflip
    camera.vflip = vflip
    camera.video_stabilization = video_stabilization
    camera.zoom = zoom
    return camera

def rotate_servo(angle):
    DC = 0
    if angle == 90:
        DC = 7
    elif angle == 0:
        DC = 2
    elif angle == 180:
        DC = 12
    else:
        raise ValueError("Accepted values for servo rotation are 0, 90, and 180.")

    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.ChangeDutyCycle(DC)   # turn towards 90 degree
    time.sleep(1.5)

def reset_servo():
    rotate_servo(90)

def send_email(middle, left, right):
    # This function will need to take in pictures and add them as attachments.
    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    from_email = Email("SpyPi@no-reply")
    subject = "Motion Detected"
    to_email = Email("gagnej3@wit.edu")
    content = Content("text/plain", "We have detected motion from your pi!\n\n")
    mail = Mail(from_email, subject, to_email, content)
    middle_attachment = Attachment()
    middle_attachment.set_filename(middle)
    middle_attachment.set_content("Test1")
    middle_attachment.set_type('image/jpeg')
    middle_attachment.set_disposition("attachment")
    middle_attachment.set_content_id("Middle")
    mail.add_attachment(middle_attachment)
    left_attachment = Attachment()
    left_attachment.set_filename(left)
    left_attachment.set_content("Test2")
    left_attachment.set_type('image/jpeg')
    left_attachment.set_disposition("attachment")
    left_attachment.set_content_id("Left")
    mail.add_attachment(left_attachment)
    right_attachment = Attachment()
    right_attachment.set_filename(right)
    right_attachment.set_content("Test3")
    right_attachment.set_type('image/jpeg')
    right_attachment.set_disposition("attachment")
    right_attachment.set_content_id("Right")
    mail.add_attachment(right_attachment)
    response = sg.client.mail.send.post(request_body=mail.get())
    print(response.status_code)
    print(response.body)
    print(response.headers)

def main():
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(SERVO_PIN, GPIO.OUT)

    # pwm object on servo pin with 50Hz signal
    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.start(7)
    # ----------End of Servo Setup -------------
    GPIO.setup(MOTION_PIN, GPIO.IN)
    with provision_pi_camera() as camera:
        while True:
            i = GPIO.input(MOTION_PIN)
        if i == 1:
            print 'motion detected'
            # pwm.start(7)
        pwm.ChangeDutyCycle(7)
        time.sleep(1)
        # rotate_servo(90)
        # time.sleep(1)
        now = datetime.now()
        middle_filename = "{}-middle.jpg".format(now.isoformat())
        camera.capture(middle_filename)
        pwm.ChangeDutyCycle(2)
        time.sleep(1.5)
        # rotate_servo(0)
        # time.sleep(1.5)
        left_filename = "{}-left.jpg".format(now.isoformat())
        camera.capture(left_filename)
        pwm.ChangeDutyCycle(7)
        time.sleep(1.5)
        # rotate_servo(90)
        # rotate_servo(180)
        # time.sleep(1.5)
        right_filename = "{}-right.jpg".format(now.isoformat())
        camera.capture(right_filename)
        pwm.ChangeDutyCycle(12)
        time.sleep(1.5)
        pwm.ChangeDutyCycle(7)
        time.sleep(1.5)
        # rotate_servo(90)
        # reset_servo()
        send_email(middle_filename, left_filename, right_filename)
        # time.sleep(4.1)

if __name__ == "__main__":
    main()
<commit_msg>testing api key<commit_after>import picamera
from RPi import GPIO
import os
import sendgrid
from sendgrid.helpers.mail import *
from gpiozero import MotionSensor
from datetime import datetime
import time

SERVO_PIN = 13
MOTION_PIN = 11

def provision_pi_camera(hflip=False, vflip=False, zoom=(0.0, 0.0, 1.0, 1.0), video_stabilization=True):
    camera = picamera.PiCamera()
    camera.hflip = hflip
    camera.vflip = vflip
    camera.video_stabilization = video_stabilization
    camera.zoom = zoom
    return camera

def rotate_servo(angle):
    DC = 0
    if angle == 90:
        DC = 7
    elif angle == 0:
        DC = 2
    elif angle == 180:
        DC = 12
    else:
        raise ValueError("Accepted values for servo rotation are 0, 90, and 180.")

    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.ChangeDutyCycle(DC)   # turn towards 90 degree
    time.sleep(1.5)

def reset_servo():
    rotate_servo(90)

def send_email(middle, left, right):
    # This function will need to take in pictures and add them as attachments.
    sg = sendgrid.SendGridAPIClient(apikey=os.environ.get('SENDGRID_API_KEY'))
    print os.environ.get('SENDGRID_API_KEY')
    # from_email = Email("SpyPi@no-reply")
    # subject = "Motion Detected"
    # to_email = Email("gagnej3@wit.edu")
    # content = Content("text/plain", "We have detected motion from your pi!\n\n")
    # mail = Mail(from_email, subject, to_email, content)
    # middle_attachment = Attachment()
    # middle_attachment.set_filename(middle)
    # middle_attachment.set_content("Test1")
    # middle_attachment.set_type('image/jpeg')
    # middle_attachment.set_disposition("attachment")
    # middle_attachment.set_content_id("Middle")
    # mail.add_attachment(middle_attachment)
    # left_attachment = Attachment()
    # left_attachment.set_filename(left)
    # left_attachment.set_content("Test2")
    # left_attachment.set_type('image/jpeg')
    # left_attachment.set_disposition("attachment")
    # left_attachment.set_content_id("Left")
    # mail.add_attachment(left_attachment)
    # right_attachment = Attachment()
    # right_attachment.set_filename(right)
    # right_attachment.set_content("Test3")
    # right_attachment.set_type('image/jpeg')
    # right_attachment.set_disposition("attachment")
    # right_attachment.set_content_id("Right")
    # mail.add_attachment(right_attachment)
    # response = sg.client.mail.send.post(request_body=mail.get())
    # print(response.status_code)
    # print(response.body)
    # print(response.headers)

def main():
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(SERVO_PIN, GPIO.OUT)

    # pwm object on servo pin with 50Hz signal
    pwm = GPIO.PWM(SERVO_PIN, 50)
    pwm.start(7)
    # ----------End of Servo Setup -------------
    GPIO.setup(MOTION_PIN, GPIO.IN)
    with provision_pi_camera() as camera:
        while True:
            i = GPIO.input(MOTION_PIN)
        if i == 1:
            print 'motion detected'
            # pwm.start(7)
        pwm.ChangeDutyCycle(7)
        time.sleep(1)
        # rotate_servo(90)
        # time.sleep(1)
        now = datetime.now()
        middle_filename = "{}-middle.jpg".format(now.isoformat())
        camera.capture(middle_filename)
        pwm.ChangeDutyCycle(2)
        time.sleep(1.5)
        # rotate_servo(0)
        # time.sleep(1.5)
        left_filename = "{}-left.jpg".format(now.isoformat())
        camera.capture(left_filename)
        pwm.ChangeDutyCycle(7)
        time.sleep(1.5)
        # rotate_servo(90)
        # rotate_servo(180)
        # time.sleep(1.5)
        right_filename = "{}-right.jpg".format(now.isoformat())
        camera.capture(right_filename)
        pwm.ChangeDutyCycle(12)
        time.sleep(1.5)
        pwm.ChangeDutyCycle(7)
        time.sleep(1.5)
        # rotate_servo(90)
        # reset_servo()
        send_email(middle_filename, left_filename, right_filename)
        # time.sleep(4.1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>'''
Created on 18 Dec 2015

@author: Lauri
'''
import csv
import codecs
import traceback
import sys
import logging
import os

"""
Different formatted csvs made with the csv_formatter are used to
make html tables. The different formatted csv paths are given as parameters.

"""

HIGHLIGHT_HIGHER = (1, 3, 5, 6, 7, 8)
HIGHLIGHT_LOWER = (2, 4, 9)

logging.basicConfig(filename='log/error.log',
                    level=logging.ERROR,
                    format='%(asctime)s: %(levelname)s: %(message)s')

def main():
    args = sys.argv
    if len(args) < 2:
        logging.critical('Wrong number of arguments.' +
                         'Usage: "python csv_to_html.py testData*"')
        sys.exit()
    data = []
    software = []
    for csv in args[1:]:
        csv_name = os.path.basename(csv)
        software.append(csv_name.split("_")[0])
        data.append(read_csv_row(csv, 1))
    titles = read_csv_row(args[1], 0)
    html = buildHTML(software, data, titles)
    save_to_html(html, 'siegeresults.html')

def is_neutral(i):
    """
    Checks if the current row in the table should be highlited or not.

    """

    return not (i in HIGHLIGHT_HIGHER or i in HIGHLIGHT_LOWER)

def save_to_html(html, to_path):
    try:
        with codecs.open(to_path, 'w', 'utf-8') as output:
            output.write(html)

    except:
        logging.critical(traceback.format_exc() +
                         ' exception while saving html to: ' +
                         str(to_path))
        sys.exit

def build_row(row_data, title, i):
    """
    Returns row html built with the given row_data, title of row and
    row index.

    """

    column_html = '%s	\n' % title
    if not is_neutral(i) and all(row_data[0] == data for data in row_data):
        for d in row_data:
            column_html += ('%s	\n' % d)
    elif not is_neutral(i):
        val = max(row_data)
        if i in HIGHLIGHT_LOWER:
            val = min(row_data)
        for d in row_data:
            if d == val:
                column_html += ('%s	\n' % d)
            else:
                column_html += ('%s	\n' % d)
    else:
        for d in row_data:
            column_html += ('%s	\n' % d)
    return ('
%s
\n' % column_html)

def buildHTML(software, data, titles):
    """
    Builds the html to display the given data in a table.

    """

    html_table = '''
\n

\n
                  	\n'''
    for s in software:
        html_table += '%s	\n''' % s
    html_table += '
\n'

    for i in range(len(titles)):
        row_data = []
        for d in data:
            row_data.append(d[i])
        html_table += build_row(row_data, titles[i], i)

    html =   '''<!DOCTYPE html>\n
                \n
                \n
                <title>Results</title>\n
                <link rel="stylesheet"
                type="text/css"
                href="tablestyle.css">
                \n
                \n
                %s

                \n
                 ''' % html_table
    return html

def read_csv_row(from_path, row_number):
    """
    Reads a row in a csv file formatted with the csv_formatter script.

    """

    try:
        with codecs.open(from_path, 'r', 'utf-8') as inp:
            reader = csv.reader(inp, dialect='excel', lineterminator='\n')
            i = 0
            for l in reader:
                if i == row_number and i != 0:
                    data = [x.strip() for x in l]
                    data = [l[0], int(l[1]),
                            float(l[2]), int(l[3]),
                            float(l[4]), float(l[5]),
                            float(l[6]), float(l[7]),
                            int(l[8]), int(l[9])]
                    return data
                elif i == row_number:
                    return l
                i += 1

    except Exception:
        logging.critical(traceback.format_exc() +
                         ' exception while reading csv from: ' +
                          str(from_path))
        sys.exit

main()
<commit_msg>html formatter support for multiple competitors<commit_after>"""
Created on 18 Dec 2015

@author: Lauri
@modified-by: Kristian

"""
import csv
import codecs
import traceback
import sys
import logging
import os

"""
Different formatted csvs made with the csv_formatter are used to
make html tables. The different formatted csv paths are given as parameters.

"""

HIGHLIGHT_HIGHER = (1, 3, 5, 6, 8)
HIGHLIGHT_LOWER = (2, 4, 7, 9)
HTML_DOC = '''
<!DOCTYPE html>

<title>Results</title>
<link rel="stylesheet" type="text/css" href="tablestyle.css">

{0}

'''

logging.basicConfig(
    filename='log/error.log',
    level=logging.ERROR,
    format='%(asctime)s: %(levelname)s: %(message)s'
)

def main(args):
    if len(args) < 2:
        logging.critical('Wrong number of arguments.\n' +
                         'Usage: "python csv_to_html.py testData*"')
        sys.exit(1)

    software = []
    titles = None
    for i, csv in enumerate(args[1:]):
        csv_name = os.path.basename(csv)
        name = csv_name.split("_")[0]
        if titles is None:
            titles = read_csv_row(csv, 0)
        data = read_csv_row(csv, 1)
        software.append({'name': name, 'data': data})
    html = buildHTML(software, titles)
    sys.exit(save_to_html(html, 'siegeresults.html'))

def is_neutral(i):
    """
    Checks if the current row in the table should be highlited or not.

    """
    return i not in HIGHLIGHT_HIGHER and i not in HIGHLIGHT_LOWER

def save_to_html(html, to_path):
    try:
        with codecs.open(to_path, 'w', 'utf-8') as output:
            output.write(html)
    except Exception:
        msg = '{0} exception while saving html to: {1}'
        logging.critical(msg.format(traceback.format_exc(), to_path))
        return 1
    return 0

def build_row(row_data, neutral):
    """
    Returns row html built with the given row_data, title of row and
    row index.

    """
    column = '{0}	'
    column_cls = '{1}	'
    if neutral:
        return ''.join([column.format(d) for d in row_data])
    if min(row_data) == max(row_data):
        return ''.join([column_cls.format('even', d) for d in row_data])
    val = max(row_data)
    return ''.join([
        column_cls.format('best' if d == val else 'neutral', d)
        for d in row_data
    ])

def buildHTML(software, titles):
    """
    Builds the html to display the given data in a table.

    """
    row = '
{0}
'
    head = '{0}	'
    column = '{0}	'
    headers = [head.format('')]
    headers.extend([head.format(s['name']) for s in software])
    html_table = row.format(''.join(headers))

    for i, title in enumerate(titles):
        row_data = [app['data'][i] for app in software]
        neutral = is_neutral(i)
        row_head = column.format(title)
        row_content = build_row(row_data, neutral)
        html_table += row.format(row_head + row_content)

    return HTML_DOC.format(html_table)

def read_csv_row(from_path, row_number=None):
    """
    Reads a row in a csv file formatted with the csv_formatter script.

    """
    forms = [str, int, float, int, float, float, float, float, int, int]
    try:
        with codecs.open(from_path, 'r', 'utf-8') as inp:
            reader = csv.reader(inp, dialect='excel', lineterminator='\n')
            if row_number is None:
                return [row for row in reader]
            if row_number == 0:
                return next(reader)
            f = lambda x: x[1] if (x[0] == row_number) else None
            iterated = map(f, enumerate(reader))
            row = filter(lambda x: x is not None, iterated)[0]
            return [a(b.strip()) for (a,b) in zip(forms, row)]
    except Exception as excp:
        print(excp)
        msg = '{0} exception while reading csv from: {1}'
        logging.critical(msg.format(traceback.format_exc(), from_path))
        sys.exit(1)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import argparse
import logging
import os
import os.path
import setuptools
import sys
from contextlib import contextmanager
from setuptools.command.install import install

from clamp.build import create_singlejar, build_jar, copy_included_jars

logging.basicConfig()
log = logging.getLogger("clamp")

@contextmanager
def honor_verbosity(verbose):
    if verbose > 1:
        old_level = log.getEffectiveLevel()
        log.setLevel(logging.DEBUG)
    yield
    if verbose > 1:
        log.setLevel(old_level)

class ClampSetup(object):

    # FIXME include such things as excluded/included jars, etc

    def __init__(self, modules):
        self.modules = modules

def parse_clamp_keyword(distribution, keyword, values):
    if keyword != "clamp":
        raise DistutilsSetupError("invalid keyword: {}".format(keyword))
    if "modules" not in values:
        raise DistutilsSetupError("clamp keyword must specify modules: {}".format(keyword))
    distribution.zip_safe = False  # given the use of jars
    try:
        invalid = []
        clamped_modules = list(values["modules"])
        for v in clamped_modules:
            if not isinstance(v, basestring):
                invalid.append(v)
        if invalid:
            raise DistutilsSetupError(
                "clamp.modules={} is invalid, must be an iterable of importable module names".format(
                    values))
    except TypeError, ex:
        log.error("Invalid clamp", exc_info=True)
        raise DistutilsSetupError("clamp={} is invalid: {}".format(values, ex))
    distribution.clamp = ClampSetup(clamped_modules)

class build_jar_command(setuptools.Command):

    description = "create a jar for all clamped Python classes for this package"
    user_options = [
        ("output=",   "o", "write jar to output path"),
    ]

    def initialize_options(self):
        self.output = None

    def finalize_options(self):
        if self.output is not None:
            dir_path = os.path.split(self.output)[0]
            if dir_path and not os.path.exists(dir_path):
                raise DistutilsOptionError("Directory {} to write jar must exist".format(dir_path))
            if os.path.splitext(self.output)[1] != ".jar":
                raise DistutilsOptionError("Path must be to a valid jar name, not {}".format(self.output))

    def get_jar_name(self):
        metadata = self.distribution.metadata
        return "{}-{}.jar".format(metadata.get_name(), metadata.get_version())

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            if not self.distribution.clamp:
                raise DistutilsOptionError("Specify the modules to be built into a jar  with the 'clamp' setup keyword")
            build_jar(self.distribution.metadata.get_name(),
                      self.get_jar_name(), self.distribution.clamp, self.output)

class clamp_command(install):

    description = "install required jars, run usual install, and clamp modules into jar"

    def get_jar_name(self):
        metadata = self.distribution.metadata
        return "{}-{}.jar".format(metadata.get_name(), metadata.get_version())

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            if not self.distribution.clamp:
                raise DistutilsOptionError("Specify the modules to be built into a jar  with the 'clamp' setup keyword")

            # 1. Ensure any included jars are immediately available
            available_paths = set(sys.path)
            jar_paths = copy_included_jars(self.distribution.metadata.get_name(), self.distribution.packages)
            for path in jar_paths:
                if path not in available_paths:
                    print "Adding jar to sys.path", path
                    sys.path.append(path)  # make these jars are available

            # 2. Compile Python classes, which may depend on included jars
            def run_commands():
                def run_commands2():
                    install.run(self)
                run_commands2()

            # NB: This stack logic plus resetting the apparent module
            # mirrors and thus works around the "slightly kludgy, but
            # seems to work" (!)  frame inspection logic that
            # setuptools uses to determine whether it should use the
            # legacy-supporting .egg-info or use .egg.
            #
            # We want the latter to be built, hence this workaround.

            run_commands.func_globals["__name__"] = "distutils.dist"
            run_commands()

            # 3. Building clamped jar relies on both included jars and Python classes
            build_jar(self.distribution.metadata.get_name(),
                      self.get_jar_name(), self.distribution.clamp)

class singlejar_command(setuptools.Command):

    description = "create a singlejar of all Jython dependencies, including clamped jars"
    user_options = [
        ("output=",    "o",  "write jar to output path"),
        ("classpath=", None, "jars to include in addition to Jython runtime and site-packages jars"),  # FIXME take a list?
        ("runpy=",     "r",  "path to __run__.py to make a runnable jar"),
    ]

    def initialize_options(self):
        metadata = self.distribution.metadata
        self.output = os.path.join(os.getcwd(), "{}-{}-single.jar".format(metadata.get_name(), metadata.get_version()))
        self.classpath = []
        self.runpy = os.path.join(os.getcwd(), "__run__.py")

    def finalize_options(self):
        # could validate self.output is a valid path FIXME
        if self.classpath:
            self.classpath = self.classpath.split(":")

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            create_singlejar(self.output, self.classpath, self.runpy)

def singlejar_script_command():
    parser = argparse.ArgumentParser(description="create a singlejar of all Jython dependencies, including clamped jars")
    parser.add_argument("--output", "-o", default="jython-single.jar", metavar="PATH",
                        help="write jar to output path")
    parser.add_argument("--classpath", default=None,
                        help="jars to include in addition to Jython runtime and site-packages jars")
    parser.add_argument("--runpy", "-r", default=os.path.join(os.getcwd(), "__run__.py"), metavar="PATH",
                        help="path to __run__.py to make a runnable jar")
    args = parser.parse_args()
    if args.classpath:
        args.classpath = args.classpath.split(":")
    else:
        args.classpath = []
    create_singlejar(args.output, args.classpath, args.runpy)
<commit_msg>Remove extra clever logic and call do_egg_install directly<commit_after>import argparse
import logging
import os
import os.path
import setuptools
import sys
from contextlib import contextmanager
from setuptools.command.install import install

from clamp.build import create_singlejar, build_jar, copy_included_jars

logging.basicConfig()
log = logging.getLogger("clamp")

@contextmanager
def honor_verbosity(verbose):
    if verbose > 1:
        old_level = log.getEffectiveLevel()
        log.setLevel(logging.DEBUG)
    yield
    if verbose > 1:
        log.setLevel(old_level)

class ClampSetup(object):

    # FIXME include such things as excluded/included jars, etc

    def __init__(self, modules):
        self.modules = modules

def parse_clamp_keyword(distribution, keyword, values):
    if keyword != "clamp":
        raise DistutilsSetupError("invalid keyword: {}".format(keyword))
    if "modules" not in values:
        raise DistutilsSetupError("clamp keyword must specify modules: {}".format(keyword))
    distribution.zip_safe = False  # given the use of jars
    try:
        invalid = []
        clamped_modules = list(values["modules"])
        for v in clamped_modules:
            if not isinstance(v, basestring):
                invalid.append(v)
        if invalid:
            raise DistutilsSetupError(
                "clamp.modules={} is invalid, must be an iterable of importable module names".format(
                    values))
    except TypeError, ex:
        log.error("Invalid clamp", exc_info=True)
        raise DistutilsSetupError("clamp={} is invalid: {}".format(values, ex))
    distribution.clamp = ClampSetup(clamped_modules)

class build_jar_command(setuptools.Command):

    description = "create a jar for all clamped Python classes for this package"
    user_options = [
        ("output=",   "o", "write jar to output path"),
    ]

    def initialize_options(self):
        self.output = None

    def finalize_options(self):
        if self.output is not None:
            dir_path = os.path.split(self.output)[0]
            if dir_path and not os.path.exists(dir_path):
                raise DistutilsOptionError("Directory {} to write jar must exist".format(dir_path))
            if os.path.splitext(self.output)[1] != ".jar":
                raise DistutilsOptionError("Path must be to a valid jar name, not {}".format(self.output))

    def get_jar_name(self):
        metadata = self.distribution.metadata
        return "{}-{}.jar".format(metadata.get_name(), metadata.get_version())

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            if not self.distribution.clamp:
                raise DistutilsOptionError("Specify the modules to be built into a jar  with the 'clamp' setup keyword")
            build_jar(self.distribution.metadata.get_name(),
                      self.get_jar_name(), self.distribution.clamp, self.output)

class clamp_command(install):

    description = "install required jars, run usual install, and clamp modules into jar"

    def get_jar_name(self):
        metadata = self.distribution.metadata
        return "{}-{}.jar".format(metadata.get_name(), metadata.get_version())

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            if not self.distribution.clamp:
                raise DistutilsOptionError("Specify the modules to be built into a jar  with the 'clamp' setup keyword")

            # 1. Ensure any included jars are immediately available
            available_paths = set(sys.path)
            jar_paths = copy_included_jars(self.distribution.metadata.get_name(), self.distribution.packages)
            for path in jar_paths:
                if path not in available_paths:
                    print "Adding jar to sys.path", path
                    sys.path.append(path)  # make these jars are available

            # 2. Compile Python classes, which may depend on included jars.
            #
            # Use the underlying do_egg_install, which is invoked by
            # install.run in setuptools if it detects it is not in
            # legacy mode (using "slightly kludgy, but seems to work"
            # (!)  frame inspection logic). We want to install an egg,
            # which supports pth, not legacy-supporting .egg-info.
            self.do_egg_install()

            # 3. Building clamped jar relies on both included jars and Python classes
            build_jar(self.distribution.metadata.get_name(),
                      self.get_jar_name(), self.distribution.clamp)

class singlejar_command(setuptools.Command):

    description = "create a singlejar of all Jython dependencies, including clamped jars"
    user_options = [
        ("output=",    "o",  "write jar to output path"),
        ("classpath=", None, "jars to include in addition to Jython runtime and site-packages jars"),  # FIXME take a list?
        ("runpy=",     "r",  "path to __run__.py to make a runnable jar"),
    ]

    def initialize_options(self):
        metadata = self.distribution.metadata
        self.output = os.path.join(os.getcwd(), "{}-{}-single.jar".format(metadata.get_name(), metadata.get_version()))
        self.classpath = []
        self.runpy = os.path.join(os.getcwd(), "__run__.py")

    def finalize_options(self):
        # could validate self.output is a valid path FIXME
        if self.classpath:
            self.classpath = self.classpath.split(":")

    def run(self):
        with honor_verbosity(self.distribution.verbose):
            create_singlejar(self.output, self.classpath, self.runpy)

def singlejar_script_command():
    parser = argparse.ArgumentParser(description="create a singlejar of all Jython dependencies, including clamped jars")
    parser.add_argument("--output", "-o", default="jython-single.jar", metavar="PATH",
                        help="write jar to output path")
    parser.add_argument("--classpath", default=None,
                        help="jars to include in addition to Jython runtime and site-packages jars")
    parser.add_argument("--runpy", "-r", default=os.path.join(os.getcwd(), "__run__.py"), metavar="PATH",
                        help="path to __run__.py to make a runnable jar")
    args = parser.parse_args()
    if args.classpath:
        args.classpath = args.classpath.split(":")
    else:
        args.classpath = []
    create_singlejar(args.output, args.classpath, args.runpy)
<|endoftext|>


<commit_before>import numpy

class FuncBase(object):
    """Base object for performing network analysis on a
    ``spaghetti.Network`` object.

    Parameters
    ----------

    ntw : spaghetti.Network
        A spaghetti network object.

    pointpattern : spaghetti.network.PointPattern
        A spaghetti point pattern object.

    nsteps : int
        The number of steps at which the count of the nearest
        neighbors is computed. Default is 10.

    permutations : int
        The number of permutations to perform. Default is 99.

    threshold : float
        The level at which significance is computed.
        (0.5 would be 97.5% and 2.5%). Default is 0.5.

    distribution : str
        The distribution from which random points are sampled.
        Currently, the only supported distribution is uniform.

    upperbound : float
        The upper bound at which the `K`-function is computed.
        Defaults to the maximum observed nearest neighbor distance.

    Attributes
    ----------

    sim : numpy.ndarray
        A simulated distance matrix.

    npts : int
        The number of points (``pointpattern.npoints``).

    xaxis : numpy.ndarray
        The observed x-axis of values.

    observed : numpy.ndarray
        The observed y-axis of values.

    """

    def __init__(
        self,
        ntw,
        pointpattern,
        nsteps=10,
        permutations=99,
        threshold=0.5,
        distribution="uniform",
        upperbound=None,
    ):

        # set initial class attributes
        self.ntw = ntw
        self.pointpattern = pointpattern
        self.nsteps = nsteps
        self.permutations = permutations
        self.threshold = threshold

        # set and validate the distribution
        self.distribution = distribution.lower()
        self.validatedistribution()

        # create an empty array to store the simulated points
        self.sim = numpy.empty((permutations, nsteps))
        self.pts = self.pointpattern
        self.npts = self.pts.npoints

        # set the upper bounds
        self.upperbound = upperbound

        # compute the statistic K
        self.computeobserved()
        self.computepermutations()

        # compute the envelope vectors
        self.computeenvelope()

    def validatedistribution(self):
        """enusure statistical distribution is supported
        """

        valid_distributions = ["uniform"]
        if not self.distribution in valid_distributions:
            msg = "%s distribution not currently supported." % self.distribution
            raise RuntimeError(msg)

    def computeenvelope(self):
        """compute upper and lower bounds of envelope
        """

        upper = 1.0 - self.threshold / 2.0
        lower = self.threshold / 2.0

        self.upperenvelope = numpy.nanmax(self.sim, axis=0) * upper
        self.lowerenvelope = numpy.nanmin(self.sim, axis=0) * lower

    def setbounds(self, distances):
        """set upper bound
        """
        if self.upperbound is None:
            self.upperbound = numpy.nanmax(distances)

class GlobalAutoK(FuncBase):
    """See full description in ``network.Network.GlobalAutoK()``.

    Attributes
    ----------

    lam : float
        The ``lambda`` value.

    """

    def computeobserved(self):
        """Compute the observed nearest.
        """

        # pairwise distances
        distances = self.ntw.allneighbordistances(self.pointpattern, fill_diagonal=0.0)

        self.setbounds(distances)

        # set the intensity (lambda)
        self.lam = self.npts / sum(self.ntw.arc_lengths.values())

        # compute a Global Auto K-Function
        observedx, observedy = global_auto_k(
            self.npts, distances, self.upperbound, self.lam, nsteps=self.nsteps
        )

        # set observed values
        self.observed = observedy
        self.xaxis = observedx

    def computepermutations(self):
        """Compute permutations (Monte Carlo simulation) of the points.
        """

        # for each round of permutations
        for p in range(self.permutations):

            # simulate a point pattern
            sim = self.ntw.simulate_observations(
                self.npts, distribution=self.distribution
            )

            # distances
            distances = self.ntw.allneighbordistances(sim, fill_diagonal=0.0)

            # compute a Global Auto K-Function
            simx, simy = global_auto_k(
                self.npts, distances, self.upperbound, self.lam, nsteps=self.nsteps
            )

            # label the permutation
            self.sim[p] = simy

def global_auto_k(n_obs, dists, upperbound, intensity, nsteps=10):
    """Compute a `K`-function.

    Parameters
    ----------

    n_obs : int
        The number of observations. See ``self.npts``.

    dists : numpy.ndarray
        A matrix of pairwise distances.

    upperbound : int or float
        The end value of the sequence.

    intensity : float
        lambda value

    nsteps : int
        The number of distance bands. Default is 10. Must be
        non-negative.

    Returns
    -------

    x : numpy.ndarray
        The x-axis of values.

    y : numpy.ndarray
        The y-axis of values.

    """

    # create interval for x-axis
    x = numpy.linspace(0, upperbound, nsteps)

    # create empty y-axis vector and iterate over x-axis interval
    # -- here we should also not consider self-neighbors (the diagonal)
    y = numpy.array([dists[dists <= r].shape[0] - n_obs for r in x]).astype(float)

    # compute k for y-axis vector
    y /= n_obs * intensity

    return x, y
<commit_msg>optimized kfunc thnx to ljwolf<commit_after>import numpy

class FuncBase(object):
    """Base object for performing network analysis on a
    ``spaghetti.Network`` object.

    Parameters
    ----------

    ntw : spaghetti.Network
        A spaghetti network object.

    pointpattern : spaghetti.network.PointPattern
        A spaghetti point pattern object.

    nsteps : int
        The number of steps at which the count of the nearest
        neighbors is computed. Default is 10.

    permutations : int
        The number of permutations to perform. Default is 99.

    threshold : float
        The level at which significance is computed.
        (0.5 would be 97.5% and 2.5%). Default is 0.5.

    distribution : str
        The distribution from which random points are sampled.
        Currently, the only supported distribution is uniform.

    upperbound : float
        The upper bound at which the `K`-function is computed.
        Defaults to the maximum observed nearest neighbor distance.

    Attributes
    ----------

    sim : numpy.ndarray
        A simulated distance matrix.

    npts : int
        The number of points (``pointpattern.npoints``).

    xaxis : numpy.ndarray
        The observed x-axis of values.

    observed : numpy.ndarray
        The observed y-axis of values.

    """

    def __init__(
        self,
        ntw,
        pointpattern,
        nsteps=10,
        permutations=99,
        threshold=0.5,
        distribution="uniform",
        upperbound=None,
    ):

        # set initial class attributes
        self.ntw = ntw
        self.pointpattern = pointpattern
        self.nsteps = nsteps
        self.permutations = permutations
        self.threshold = threshold

        # set and validate the distribution
        self.distribution = distribution.lower()
        self.validatedistribution()

        # create an empty array to store the simulated points
        self.sim = numpy.empty((permutations, nsteps))
        self.pts = self.pointpattern
        self.npts = self.pts.npoints

        # set the upper bounds
        self.upperbound = upperbound

        # compute the statistic K
        self.computeobserved()
        self.computepermutations()

        # compute the envelope vectors
        self.computeenvelope()

    def validatedistribution(self):
        """enusure statistical distribution is supported
        """

        valid_distributions = ["uniform"]
        if not self.distribution in valid_distributions:
            msg = "%s distribution not currently supported." % self.distribution
            raise RuntimeError(msg)

    def computeenvelope(self):
        """compute upper and lower bounds of envelope
        """

        upper = 1.0 - self.threshold / 2.0
        lower = self.threshold / 2.0

        self.upperenvelope = numpy.nanmax(self.sim, axis=0) * upper
        self.lowerenvelope = numpy.nanmin(self.sim, axis=0) * lower

    def setbounds(self, distances):
        """set upper bound
        """
        if self.upperbound is None:
            self.upperbound = numpy.nanmax(distances)

class GlobalAutoK(FuncBase):
    """See full description in ``network.Network.GlobalAutoK()``.

    Attributes
    ----------

    lam : float
        The ``lambda`` value.

    """

    def computeobserved(self):
        """Compute the observed nearest.
        """

        # pairwise distances
        distances = self.ntw.allneighbordistances(self.pointpattern)
        distances = upper_triangle_as_vector(distances)

        self.setbounds(distances)

        # set the intensity (lambda)
        self.lam = self.npts / sum(self.ntw.arc_lengths.values())

        # compute a Global Auto K-Function
        observedx, observedy = global_auto_k(
            self.npts, distances, self.upperbound, self.lam, nsteps=self.nsteps
        )

        # set observed values
        self.observed = observedy
        self.xaxis = observedx

    def computepermutations(self):
        """Compute permutations (Monte Carlo simulation) of the points.
        """

        # for each round of permutations
        for p in range(self.permutations):

            # simulate a point pattern
            sim = self.ntw.simulate_observations(
                self.npts, distribution=self.distribution
            )

            # distances
            distances = self.ntw.allneighbordistances(sim)
            distances = upper_triangle_as_vector(distances)

            # compute a Global Auto K-Function
            simx, simy = global_auto_k(
                self.npts, distances, self.upperbound, self.lam, nsteps=self.nsteps
            )

            # label the permutation
            self.sim[p] = simy

def upper_triangle_as_vector(matrix):
    """Return the upper triangle of a symmetric matrix without the diagonal."""
    return matrix[numpy.triu_indices_from(matrix, k=1)]

def global_auto_k(n_obs, dists, upperbound, intensity, nsteps=10):
    """Compute a `K`-function.

    Parameters
    ----------

    n_obs : int
        The number of observations. See ``self.npts``.

    dists : numpy.ndarray
        A vector (the upper triangle of a symmetric matrix)
        of pairwise distances.

    upperbound : int or float
        The end value of the sequence.

    intensity : float
        lambda value

    nsteps : int
        The number of distance bands. Default is 10. Must be
        non-negative.

    Returns
    -------

    x : numpy.ndarray
        The x-axis of values.

    y : numpy.ndarray
        The y-axis of values.

    """

    # create interval for x-axis
    x = numpy.linspace(0, upperbound, num=nsteps).reshape(-1, 1)

    # "iterate" over the x-axis interval, slice out and count neighbors within
    # each step radius, and multiply x2 to account for the lower triangle
    y = (dists < x).sum(axis=1) * 2.0

    # finalize the K computation for the denominator if the y-axis vector
    y /= n_obs * intensity

    # reset the shape of the x-axis
    x = x.squeeze()

    return x, y
<|endoftext|>


<commit_before>"""Classes for tracking statistics in Apairy.

Each statistic is represented as an object of type Tally, Level, or Series.
When apiary gets a data point for a statistic, it calls the add() method.
Periodically, apairy calls report() to retrieve information from the statistic.

Statistics should aggregate information between calls to report().  They should
also produce information about the change in each value since the last call
to report().

Types of stats:

    tally  - Tracks the total number of a certain kind of event.  Reports the
            number of events in each interval, and the total number of
            events.
    level  - Tracks a quantity that increments and decrements.  Reports the
            current level and the high/low/median/mean during the interval.
    series - Tracks a value as it changes over time.  Reports the mean/median/
            min/max/stdev in the interval.
"""

import numpy
import math
from collections import OrderedDict
from .table import ALIGN_LEFT, ALIGN_CENTER, ALIGN_RIGHT

class Statistic(object):
    def __init__(self):
        self._last = {}

    def add(self, *args, **kwargs):
        """Consume a value and store it for a later calculate() call."""
        raise NotImplementedError()

    def calculate(self):
        """Crunch numbers received since the last reset() call.

        Produce a dict of named aggregate stats.  Value type for each aggregate
        stat is expected to be the same between calls to calculate().
        """
        raise NotImplementedError()

    def reset(self):
        """Clear all stored values."""
        raise NotImplementedError()

    def format_number(self, value):
        raise NotImplementedError()

    def format_change(self, value):
        raise NotImplementedError()

    def report(self):
        current = self.calculate()
        self.reset()

        row = []

        for name, value in current.iteritems():
            row.append((ALIGN_RIGHT, name.lower() + ":"))
            row.append((ALIGN_RIGHT, self.format_number(value)))

            if name in self._last:
                row.append((ALIGN_LEFT, self.format_change(value - self._last[name])))
            else:
                row.append((ALIGN_LEFT, ""))

        self._last = current

        return row

class IntegerStatistic(Statistic):
    def format_number(self, value):
        return "%d" % value

    def format_change(self, value):
        if value == 0:
            return ""
        else:
            return "(%+d)" % value

class FloatStatistic(Statistic):
    def _get_precision(self, f, significant_figures, min_precision=3):
        # I don't want to print in scientific notation.  It's harder to
        # quickly visually compare numbers with exponents.  Instead, I'd like to
        # show a minimum number of significant figures, and print
        # as many digits as are necessary to make this happen.

        if f == 0:
            # just protecting against taking the log of 0 here, so comparing
            # a float to 0 is okay
            exponent = 0
        else:
            exponent = int(math.ceil(math.log10(abs(f))))
        return max(significant_figures - exponent, min_precision)

    def format_number(self, value):
        return "%.*f" % (self._get_precision(value, 6), value)

    def format_change(self, value):
        return "(%+.3g)" % value

class Tally(IntegerStatistic):
    def __init__(self):
        super(IntegerStatistic,self).__init__()

        self._grand_total = 0
        self._total = 0

    def add(self):
        self._total += 1

    def calculate(self):
        self._grand_total += self._total

        stats = OrderedDict()

        stats["Current"] = self._total
        stats["Total"] = self._grand_total

        return stats

    def reset(self):
        self._total = 0

class Level(IntegerStatistic):
    def __init__(self):
        super(Level,self).__init__()

        self._level = 0
        self._levels = [0]

    def add(self, direction):
        if direction == "+":
            self._level += 1
        elif direction == "-":
            self._level -= 1

        self._levels.append(self._level)

    def reset(self):
        self._levels = []

    def calculate(self):
        stats = OrderedDict()

        stats['Current'] = self._level

        if self._levels:
            add_series_stats(stats, self._levels)

        return stats

class Series(FloatStatistic):
    def __init__(self):
        super(Series, self).__init__()

        self._values = []

    def add(self, value):
        self._values.append(value)

    def reset(self):
        self._values = []

    def calculate(self):
        stats = OrderedDict()

        if self._values:
            stats['Current'] = self._values[-1]
            add_series_stats(stats, self._values)

        return stats

def add_series_stats(stats, series):
    series = numpy.array(series)

    stats['Min'] = numpy.min(series)
    stats['Max'] = numpy.max(series)
    stats['Median'] = numpy.median(series)
    stats['Mean'] = numpy.mean(series)
    stats['Stdev'] = numpy.std(series)
<commit_msg>add rate to tally stats<commit_after>"""Classes for tracking statistics in Apairy.

Each statistic is represented as an object of type Tally, Level, or Series.
When apiary gets a data point for a statistic, it calls the add() method.
Periodically, apairy calls report() to retrieve information from the statistic.

Statistics should aggregate information between calls to report().  They should
also produce information about the change in each value since the last call
to report().

Types of stats:

    tally  - Tracks the total number of a certain kind of event.  Reports the
            number of events in each interval, and the total number of
            events.
    level  - Tracks a quantity that increments and decrements.  Reports the
            current level and the high/low/median/mean during the interval.
    series - Tracks a value as it changes over time.  Reports the mean/median/
            min/max/stdev in the interval.
"""

import numpy
import math
import time
from collections import OrderedDict
from .table import ALIGN_LEFT, ALIGN_CENTER, ALIGN_RIGHT

class Statistic(object):
    def __init__(self):
        self._last = {}

    def add(self, *args, **kwargs):
        """Consume a value and store it for a later calculate() call."""
        raise NotImplementedError()

    def calculate(self):
        """Crunch numbers received since the last reset() call.

        Produce a dict of named aggregate stats.  Value type for each aggregate
        stat is expected to be the same between calls to calculate().
        """
        raise NotImplementedError()

    def reset(self):
        """Clear all stored values."""
        raise NotImplementedError()

    def format_number(self, value):
        raise NotImplementedError()

    def format_change(self, value):
        raise NotImplementedError()

    def report(self):
        current = self.calculate()
        self.reset()

        row = []

        for name, value in current.iteritems():
            row.append((ALIGN_RIGHT, name.lower() + ":"))
            row.append((ALIGN_RIGHT, self.format_number(value)))

            if name in self._last:
                row.append((ALIGN_LEFT, self.format_change(value - self._last[name])))
            else:
                row.append((ALIGN_LEFT, ""))

        self._last = current

        return row

class IntegerStatistic(Statistic):
    def format_number(self, value):
        return "%d" % value

    def format_change(self, value):
        if value == 0:
            return ""
        else:
            return "(%+d)" % value

class FloatStatistic(Statistic):
    def _get_precision(self, f, significant_figures, min_precision=3):
        # I don't want to print in scientific notation.  It's harder to
        # quickly visually compare numbers with exponents.  Instead, I'd like to
        # show a minimum number of significant figures, and print
        # as many digits as are necessary to make this happen.

        if f == 0:
            # just protecting against taking the log of 0 here, so comparing
            # a float to 0 is okay
            exponent = 0
        else:
            exponent = int(math.ceil(math.log10(abs(f))))
        return max(significant_figures - exponent, min_precision)

    def format_number(self, value):
        return "%.*f" % (self._get_precision(value, 6), value)

    def format_change(self, value):
        return "(%+.3g)" % value

class Tally(IntegerStatistic):
    def __init__(self):
        super(IntegerStatistic,self).__init__()

        self._grand_total = 0
        self.reset()

    def add(self):
        self._total += 1

    def calculate(self):
        self._grand_total += self._total

        stats = OrderedDict()

        stats["Current"] = self._total
        stats["Total"] = self._grand_total
        stats["Rate"] = self._total / (time.time() - self._start_time)

        return stats

    def reset(self):
        self._total = 0
        self._start_time = time.time()

class Level(IntegerStatistic):
    def __init__(self):
        super(Level,self).__init__()

        self._level = 0
        self._levels = [0]

    def add(self, direction):
        if direction == "+":
            self._level += 1
        elif direction == "-":
            self._level -= 1

        self._levels.append(self._level)

    def reset(self):
        self._levels = []

    def calculate(self):
        stats = OrderedDict()

        stats['Current'] = self._level

        if self._levels:
            add_series_stats(stats, self._levels)

        return stats

class Series(FloatStatistic):
    def __init__(self):
        super(Series, self).__init__()

        self._values = []

    def add(self, value):
        self._values.append(value)

    def reset(self):
        self._values = []

    def calculate(self):
        stats = OrderedDict()

        if self._values:
            stats['Current'] = self._values[-1]
            add_series_stats(stats, self._values)

        return stats

def add_series_stats(stats, series):
    series = numpy.array(series)

    stats['Min'] = numpy.min(series)
    stats['Max'] = numpy.max(series)
    stats['Median'] = numpy.median(series)
    stats['Mean'] = numpy.mean(series)
    stats['Stdev'] = numpy.std(series)
<|endoftext|>


<commit_before>###############################################################################
# apogee.spec.window: routines for dealing with the individual element windows
###############################################################################
import os, os.path
import numpy
from apogee.tools.read import modelspecOnApStarWavegrid

@modelspecOnApStarWavegrid
def read(elem,apStarWavegrid=True):
    """
    NAME:
       read
    PURPOSE:
       read the window weights for a given element
    INPUT:
       elem - element
       apStarWavegrid= (True) if True, output the window onto the apStar wavelength grid, otherwise just give the ASPCAP version (blue+green+red directly concatenated)
    OUTPUT:
       Array with window weights
    HISTORY:
       2015-01-25 - Written - Bovy (IAS)
    """
    win=\
        numpy.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                   'filter/%s.filt' \
                                       % ((elem.lower().capitalize()))))
    return win

def num(elem):
    """
    NAME:
       num
    PURPOSE:
       return the number of windows for a given element or window array
    INPUT:
       elem - element
    OUTPUT:
       Number of windows
    HISTORY:
       2015-01-25 - Written - Bovy (IAS)
    """
    # Calculate the wavelength regions
    si, ei= waveregions(elem,asIndex=True)
    return len(si)

def waveregions(elem,asIndex=False):
    """
    NAME:
       waveregions
    PURPOSE:
       return the wavelength regions corresponding to different elements
    INPUT:
       elem - element
       asIndx= (False) if yes, return the indices into an apStar-like wavelength grid rather than the wavelengths directly
    OUTPUT:
       (startlams,endlams) or (startindxs, endindxs)
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    # Load the window
    win= read(elem,apStarWavegrid=True)
    # Calculate number of contiguous regions, assume this is not at the edge
    mask= ((win > 0.)*(True-numpy.isnan(win))).astype('int')
    dmaskp= numpy.roll(mask,-1)-mask
    dmaskn= numpy.roll(mask,1)-mask
    # Calculate the distance between adjacent windows and combine them if close
    import apogee.spec.plot as splot
    l10wavs= numpy.log10(splot.apStarWavegrid())
    indices= numpy.arange(len(l10wavs))
    if asIndex:
        startindxs= indices[dmaskp == 1.]
        endindxs= indices[dmaskn == 1.]
    startl10lams= l10wavs[dmaskp == 1.]
    endl10lams= l10wavs[dmaskn == 1.]
    diff= numpy.roll(startl10lams,-1)-endl10lams
    if asIndex:
        newStartindxs, newEndindxs= [startindxs[0]], [endindxs[0]]
    newStartl10lams, newEndl10lams= [startl10lams[0]], [endl10lams[0]]
    winIndx= 0
    for ii in range(len(startl10lams)-1):
        if diff[ii] < 10.*splot._DLOG10LAMBDA:
            if asIndex:
                newEndindxs[winIndx]= endindxs[ii+1]
            newEndl10lams[winIndx]= endl10lams[ii+1]
        else:
            if asIndex:
                newStartindxs.append(startindxs[ii+1])
                newEndindxs.append(endindxs[ii+1])
            newStartl10lams.append(startl10lams[ii+1])
            newEndl10lams.append(endl10lams[ii+1])
            winIndx+= 1
    if asIndex:
        return (newStartindxs,newEndindxs)
    else:
        return (10.**numpy.array(newStartl10lams),
                10.**numpy.array(newEndl10lams))

def tophat(elem):
    """
    NAME:
       tophat
    PURPOSE:
       return an array with True in the window of a given element and False otherwise
    INPUT:
       elem - element
    OUTPUT:
       array on apStar grid
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    import apogee.spec.plot as splot
    out= numpy.zeros(splot._NLAMBDA,dtype='bool')
    for si,ei in zip(*waveregions(elem,asIndex=True)):
        out[si+1:ei]= True
    return out

def total_dlambda(elem):
    """
    NAME:
       total_dlambda
    PURPOSE:
       return the total wavelength span covered by the windows of a given element
    INPUT:
       elem - element
    OUTPUT:
       total width in \AA
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    si,ei= waveregions(elem,asIndex=False)
    return numpy.sum(ei-si)
<commit_msg>allow padding of windows<commit_after>###############################################################################
# apogee.spec.window: routines for dealing with the individual element windows
###############################################################################
import os, os.path
import numpy
from apogee.tools.read import modelspecOnApStarWavegrid

@modelspecOnApStarWavegrid
def read(elem,apStarWavegrid=True):
    """
    NAME:
       read
    PURPOSE:
       read the window weights for a given element
    INPUT:
       elem - element
       apStarWavegrid= (True) if True, output the window onto the apStar wavelength grid, otherwise just give the ASPCAP version (blue+green+red directly concatenated)
    OUTPUT:
       Array with window weights
    HISTORY:
       2015-01-25 - Written - Bovy (IAS)
    """
    win=\
        numpy.loadtxt(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                   'filter/%s.filt' \
                                       % ((elem.lower().capitalize()))))
    return win

def num(elem):
    """
    NAME:
       num
    PURPOSE:
       return the number of windows for a given element or window array
    INPUT:
       elem - element
    OUTPUT:
       Number of windows
    HISTORY:
       2015-01-25 - Written - Bovy (IAS)
    """
    # Calculate the wavelength regions
    si, ei= waveregions(elem,asIndex=True)
    return len(si)

def waveregions(elem,asIndex=False,pad=0):
    """
    NAME:
       waveregions
    PURPOSE:
       return the wavelength regions corresponding to different elements
    INPUT:
       elem - element
       asIndx= (False) if yes, return the indices into an apStar-like wavelength grid rather than the wavelengths directly
       pad= (0) pad on each side by this many log10 wavelengths in 6e-6 (changes how windows are combined)
    OUTPUT:
       (startlams,endlams) or (startindxs, endindxs)
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    # Load the window
    win= read(elem,apStarWavegrid=True)
    # Calculate number of contiguous regions, assume this is not at the edge
    mask= ((win > 0.)*(True-numpy.isnan(win))).astype('int')
    dmaskp= numpy.roll(mask,-1)-mask
    dmaskn= numpy.roll(mask,1)-mask
    # Calculate the distance between adjacent windows and combine them if close
    import apogee.spec.plot as splot
    l10wavs= numpy.log10(splot.apStarWavegrid())
    indices= numpy.arange(len(l10wavs))
    if asIndex:
        startindxs= indices[dmaskp == 1.]
        endindxs= indices[dmaskn == 1.]
    startl10lams= l10wavs[dmaskp == 1.]
    endl10lams= l10wavs[dmaskn == 1.]
    if pad > 0:
        if asIndex:
            startindxs= [si-pad for si in startindxs]
            endindxs= [ei+pad for ei in endindxs]
        startl10lams-= pad*splot._DLOG10LAMBDA
        endl10lams+= pad*splot._DLOG10LAMBDA
    diff= numpy.roll(startl10lams,-1)-endl10lams
    if asIndex:
        newStartindxs, newEndindxs= [startindxs[0]], [endindxs[0]]
    newStartl10lams, newEndl10lams= [startl10lams[0]], [endl10lams[0]]
    winIndx= 0
    for ii in range(len(startl10lams)-1):
        if diff[ii] < 10.*splot._DLOG10LAMBDA:
            if asIndex:
                newEndindxs[winIndx]= endindxs[ii+1]
            newEndl10lams[winIndx]= endl10lams[ii+1]
        else:
            if asIndex:
                newStartindxs.append(startindxs[ii+1])
                newEndindxs.append(endindxs[ii+1])
            newStartl10lams.append(startl10lams[ii+1])
            newEndl10lams.append(endl10lams[ii+1])
            winIndx+= 1
    if asIndex:
        return (newStartindxs,newEndindxs)
    else:
        return (10.**numpy.array(newStartl10lams),
                10.**numpy.array(newEndl10lams))

def tophat(elem):
    """
    NAME:
       tophat
    PURPOSE:
       return an array with True in the window of a given element and False otherwise
    INPUT:
       elem - element
    OUTPUT:
       array on apStar grid
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    import apogee.spec.plot as splot
    out= numpy.zeros(splot._NLAMBDA,dtype='bool')
    for si,ei in zip(*waveregions(elem,asIndex=True)):
        out[si+1:ei]= True
    return out

def total_dlambda(elem,pad=0):
    """
    NAME:
       total_dlambda
    PURPOSE:
       return the total wavelength span covered by the windows of a given element
    INPUT:
       elem - element
       pad= (0) pad on each side by this many log10 wavelengths in 6e-6 (changes how windows are combined)
    OUTPUT:
       total width in \AA
    HISTORY:
       2015-01-26 - Written - Bovy (IAS@KITP)
    """
    si,ei= waveregions(elem,asIndex=False,pad=pad)
    return numpy.sum(ei-si)
<|endoftext|>