<commit_before>from sklearn.feature_extraction.text import CountVectorizer
from Data_Preprocessing import Data_Preprocessor
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import Perceptron
from sklearn.naive_bayes import BernoulliNB
from sklearn.learning_curve import learning_curve
from sklearn import cross_validation
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import numpy as np
import time

full_dataset_path = "20news"

random_samples = 10
train_sizes = np.arange(0.01, 1.00, 0.01)
jobs = 4

dp = Data_Preprocessor()
vectorizer = CountVectorizer(tokenizer=dp.tokenize, stop_words='english', lowercase=True, min_df=2, analyzer="word")
clfMNB = MultinomialNB(alpha=.01)
clfBNB = BernoulliNB(alpha=.01)
clfPer = Perceptron(alpha=.0001)

try:
    data = joblib.load("saved/vectorized_data.plk")
    labels = joblib.load("saved/labels.plk")
    print "Data vector found... loaded..."
except:
    print "Loading data..."
    start = time.time()
    data, labels = dp.raw_to_vector(full_dataset_path, ["header"])
    done = time.time()
    elapsed = done - start
    print "Elapsed: " + str(elapsed)
    start = time.time()
    print "Data vector not found... vectorizing..."
    data = vectorizer.fit_transform(data)
    joblib.dump(data, "saved/vectorized_data.plk")
    joblib.dump(labels, "saved/labels.plk")
    done = time.time()
    elapsed = done - start
    print "Elapsed: " + str(elapsed)

print "Number of Samples: " + str(data.shape[0])
print "Number of Features: " + str(data.shape[1])

print "Learning Curve:"
shuffle = cross_validation.ShuffleSplit(data.shape[0], n_iter=random_samples, random_state=0)

MNB_train_size, train_scores, test_scores = \
    learning_curve(clfMNB, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

MNB_avg_test_scores = [np.average(ts) for ts in test_scores]
MNB_avg_train_scores = [np.average(ts) for ts in train_scores]
MNB_std_deviation = [np.std(ts) for ts in test_scores]

BNB_train_size, train_scores, test_scores = \
    learning_curve(clfBNB, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

BNB_avg_test_scores = [np.average(ts) for ts in test_scores]
BNB_avg_train_scores = [np.average(ts) for ts in train_scores]
BNB_std_deviation = [np.std(ts) for ts in test_scores]

Per_train_size, train_scores, test_scores = \
    learning_curve(clfPer, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

Per_avg_test_scores = [np.average(ts) for ts in test_scores]
Per_avg_train_scores = [np.average(ts) for ts in train_scores]
Per_std_deviation = [np.std(ts) for ts in test_scores]

plt.plot(MNB_train_size, MNB_avg_test_scores, label="Multinomial Naive Bayes")
plt.plot(BNB_train_size, BNB_avg_test_scores, label="Bernoulli Naive Bayes")
plt.plot(Per_train_size, Per_avg_test_scores, label="Perceptron")

plt.fill_between(MNB_train_size, [x[0]+x[1] for x in zip(MNB_avg_test_scores, MNB_std_deviation)],
                 [x[0]-x[1] for x in zip(MNB_avg_test_scores, MNB_std_deviation)], color='blue', alpha=0.9)
plt.fill_between(BNB_train_size, [x[0]+x[1] for x in zip(BNB_avg_test_scores, BNB_std_deviation)],
                 [x[0]-x[1] for x in zip(BNB_avg_test_scores, BNB_std_deviation)], color='green', alpha=0.9)
plt.fill_between(Per_train_size, [x[0]+x[1] for x in zip(Per_avg_test_scores, Per_std_deviation)],
                 [x[0]-x[1] for x in zip(Per_avg_test_scores, Per_std_deviation)], color='red', alpha=0.9)

plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1, ncol=1, mode="expand", borderaxespad=0.)

plt.axis([0, 20000, 0.1, 1])
plt.xticks(range(0, 20000, 1000))
plt.yticks(np.arange(0, 1, 0.1))

for k in range(0, len(MNB_train_size), 5):
    plt.text(MNB_train_size[k], MNB_avg_test_scores[k], str(MNB_avg_test_scores[k]))
    plt.text(Per_train_size[k], Per_avg_test_scores[k], str(Per_avg_test_scores[k]))
    plt.text(BNB_train_size[k], BNB_avg_test_scores[k], str(BNB_avg_test_scores[k]))
plt.show()
<commit_msg>Graph<commit_after>from sklearn.feature_extraction.text import CountVectorizer
from Data_Preprocessing import Data_Preprocessor
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import Perceptron
from sklearn.naive_bayes import BernoulliNB
from sklearn.learning_curve import learning_curve
from sklearn import cross_validation
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import numpy as np
import time

full_dataset_path = "20news"

random_samples = 10
train_sizes = np.arange(0.01, 1.00, 0.01)
jobs = 4

dp = Data_Preprocessor()
vectorizer = CountVectorizer(tokenizer=dp.tokenize, stop_words='english', lowercase=True, min_df=2, analyzer="word")
clfMNB = MultinomialNB(alpha=.01)
clfBNB = BernoulliNB(alpha=.01)
clfPer = Perceptron(alpha=.0001)

try:
    data = joblib.load("saved/vectorized_data.plk")
    labels = joblib.load("saved/labels.plk")
    print "Data vector found... loaded..."
except:
    print "Loading data..."
    start = time.time()
    data, labels = dp.raw_to_vector(full_dataset_path, ["header"])
    done = time.time()
    elapsed = done - start
    print "Elapsed: " + str(elapsed)
    start = time.time()
    print "Data vector not found... vectorizing..."
    data = vectorizer.fit_transform(data)
    joblib.dump(data, "saved/vectorized_data.plk")
    joblib.dump(labels, "saved/labels.plk")
    done = time.time()
    elapsed = done - start
    print "Elapsed: " + str(elapsed)

print "Number of Samples: " + str(data.shape[0])
print "Number of Features: " + str(data.shape[1])

print "Learning Curve:"
start = time.time()
shuffle = cross_validation.ShuffleSplit(data.shape[0], n_iter=random_samples, random_state=0)

MNB_train_size, train_scores, test_scores = \
    learning_curve(clfMNB, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

MNB_avg_test_scores = [np.average(ts) for ts in test_scores]
MNB_avg_train_scores = [np.average(ts) for ts in train_scores]
MNB_std_deviation = [np.std(ts) for ts in test_scores]

BNB_train_size, train_scores, test_scores = \
    learning_curve(clfBNB, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

BNB_avg_test_scores = [np.average(ts) for ts in test_scores]
BNB_avg_train_scores = [np.average(ts) for ts in train_scores]
BNB_std_deviation = [np.std(ts) for ts in test_scores]

Per_train_size, train_scores, test_scores = \
    learning_curve(clfPer, data, labels, cv=shuffle, n_jobs=4, train_sizes=train_sizes)

Per_avg_test_scores = [np.average(ts) for ts in test_scores]
Per_avg_train_scores = [np.average(ts) for ts in train_scores]
Per_std_deviation = [np.std(ts) for ts in test_scores]

done = time.time()
elapsed = done - start
print "Elapsed: " + str(elapsed)

plt.plot(MNB_train_size, MNB_avg_test_scores, label="Multinomial Naive Bayes")
plt.plot(BNB_train_size, BNB_avg_test_scores, label="Bernoulli Naive Bayes")
plt.plot(Per_train_size, Per_avg_test_scores, label="Perceptron")

plt.fill_between(MNB_train_size, [x[0]+x[1] for x in zip(MNB_avg_test_scores, MNB_std_deviation)],
                 [x[0]-x[1] for x in zip(MNB_avg_test_scores, MNB_std_deviation)], color='blue', alpha=0.2)
plt.fill_between(BNB_train_size, [x[0]+x[1] for x in zip(BNB_avg_test_scores, BNB_std_deviation)],
                 [x[0]-x[1] for x in zip(BNB_avg_test_scores, BNB_std_deviation)], color='green', alpha=0.2)
plt.fill_between(Per_train_size, [x[0]+x[1] for x in zip(Per_avg_test_scores, Per_std_deviation)],
                 [x[0]-x[1] for x in zip(Per_avg_test_scores, Per_std_deviation)], color='red', alpha=0.2)

plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=1, ncol=1, mode="expand", borderaxespad=0.)

plt.axis([0, 20000, 0.25, 0.9])
plt.xticks(range(0, 20000, 1000))
plt.yticks(np.arange(0.25, 0.9, 0.1))

for k in range(0, len(MNB_train_size), 5):
    plt.text(MNB_train_size[k], MNB_avg_test_scores[k], str(MNB_avg_test_scores[k]))
    plt.text(Per_train_size[k], Per_avg_test_scores[k], str(Per_avg_test_scores[k]))
    plt.text(BNB_train_size[k], BNB_avg_test_scores[k], str(BNB_avg_test_scores[k]))
plt.show()
<|endoftext|>


<commit_before>import os
import json
import boto
import urllib2
from boto.s3.key import Key
import boto3
import click
import logging
from copy import copy
from collections import OrderedDict
from datetime import date, timedelta
from elasticsearch import Elasticsearch, RequestError, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import requests

from reader import csv_reader

logger = logging.getLogger('landsat8.meta')
bucket_name = os.getenv('BUCKETNAME', 'landsat8-meta')
thumbs_bucket_name = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
s3 = boto3.resource('s3')
es_index = 'sat-api'
es_type = 'landsat8'

def get_url(url, j=False):
    req = urllib2.Request(url)
    r = urllib2.urlopen(req).read()
    if j:
        obj = json.loads(r)
    else:
        obj = r
    return obj

def get_role():
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
    obj = get_url(url)
    return obj

def get_credentials():
    role = get_role()
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/' + role
    obj = get_url(url, True)
    return obj

def connection_to_es(es_host, es_port, aws=False):

    args = {}

    if aws:
        cred = get_credentials()

        access_key = cred['AccessKeyId']
        secret_access = cred['SecretAccessKey']
        region = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
        awsauth = AWS4Auth(access_key, secret_access, region, 'es')

        args = {
            'http_auth': awsauth,
            'use_ssl': True,
            'verify_certs': True,
            'connection_class': RequestsHttpConnection
        }

    es = Elasticsearch(hosts=[{
        'host': es_host,
        'port': es_port
    }], **args)

    return es

def create_index(index_name, doc_type, es_host, es_port, **kwargs):

    body = {
        doc_type: {
            'properties': {
                'scene_id': {'type': 'string', 'index': 'not_analyzed'},
                'satellite_name': {'type': 'string'},
                'cloud_coverage': {'type': 'float'},
                'date': {'type': 'date'},
                'data_geometry': {
                    'type': 'geo_shape',
                    'tree': 'quadtree',
                    'precision': '5mi'}
            }
        }
    }
    es = connection_to_es(es_host, es_port, kwargs['aws'])

    es.indices.create(index=index_name, ignore=400)

    es.indices.put_mapping(
        doc_type=doc_type,
        body=body,
        index=index_name
    )

def meta_constructor(metadata):
    internal_meta = copy(metadata)

    data_geometry = {
        'type': 'Polygon',
        'crs': {
            'type': 'name',
            'properties': {
                'name': 'urn:ogc:def:crs:EPSG:8.9:4326'
            }
        },
        'coordinates': [[
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')],
            [metadata.get('upperLeftCornerLongitude'), metadata.get('upperLeftCornerLatitude')],
            [metadata.get('lowerLeftCornerLongitude'), metadata.get('lowerLeftCornerLatitude')],
            [metadata.get('lowerRightCornerLongitude'), metadata.get('lowerRightCornerLatitude')],
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')]
        ]]
    }

    body = OrderedDict([
        ('scene_id', metadata.get('sceneID')),
        ('satellite_name', 'landsat-8'),
        ('cloud_coverage', metadata.get('cloudCoverFull', 100)),
        ('date', metadata.get('acquisitionDate')),
        ('thumbnail', metadata.get('browseURL')),
        ('data_geometry', data_geometry)
    ])

    body.update(internal_meta)

    return body

def elasticsearch_updater(product_dir, metadata, **kwargs):

    try:
        es = connection_to_es(kwargs['es_host'], kwargs['es_port'], kwargs['aws'])

        body = meta_constructor(metadata)

        logger.info('Pushing to Elasticsearch')

        try:
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)
        except RequestError as e:
            body['data_geometry'] = None
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)

    except Exception as e:
        logger.error('Unhandled error occured while writing to elasticsearch')
        logger.error('Details: %s' % e.__str__())

def thumbnail_writer(product_dir, metadata, **kwargs):
    """
    Extra function to download images from USGS, then upload to S3 and call
    the ES metadata writer afterwards.
    """

    # Download original thumbnail
    output_file = metadata['sceneID'] + '.jpg'
    thumbnail = 'https://' + thumbs_bucket_name + '.s3.amazonaws.com/' + output_file

    new_thumb = requests.get(thumbnail)
    if new_thumb.status_code != 200:

        # Upload thumbnail to S3
        thumbs_bucket_name2 = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
        try:
            print('uploading %s' % output_file)
            c = boto.connect_s3()
            b = c.get_bucket(thumbs_bucket_name2)
            k = Key(b, name=output_file)
            k.set_metadata('Content-Type', 'image/jpeg')
            r = requests.get(metadata['browseURL'])
            k.set_contents_from_string(r.content, policy='public-read')
        except Exception as e:
            print(e)

    # Update metadata record
    metadata['thumbnail'] = thumbnail

    elasticsearch_updater(product_dir, metadata, **kwargs)
    return

def file_writer(product_dir, metadata):
    body = meta_constructor(metadata)

    if not os.path.exists(product_dir):
        os.makedirs(product_dir)

    f = open(os.path.join(product_dir, body['scene_id'] + '.json'), 'w')
    f.write(json.dumps(body))
    logger.info('saving to disk at %s' % product_dir)
    f.close()

def s3_writer(product_dir, metadata):
    # make sure product_dir doesn't start with slash (/) or dot (.)
    if product_dir.startswith('.'):
        product_dir = product_dir[1:]

    if product_dir.startswith('/'):
        product_dir = product_dir[1:]

    body = meta_constructor(metadata)

    key = os.path.join(product_dir, body['scene_id'] + '.json')
    s3.Object(bucket_name, key).put(Body=json.dumps(body), ACL='public-read', ContentType='application/json')

    logger.info('saving to s3 at %s')

def last_updated(today):
    """ Gets the latest time a product added to Elasticsearch """

    bucket = s3.Bucket(bucket_name)

    start_day = today.day
    start_month = today.month

    yr_counter = 0
    while True:
        m_counter = 0
        year = today.year - yr_counter
        if year < 2015:
            break
        while True:
            month = start_month - m_counter
            if month == 0:
                start_month = 12
                break
            d_counter = 0
            while True:
                day = start_day - d_counter
                if day == 0:
                    start_day = 31
                    break
                path = os.path.join(str(year), str(month), str(day))
                logger.info('checking %s' % path)
                objs = bucket.objects.filter(Prefix=path).limit(1)
                if list(objs):
                    return date(year, month, day)
                d_counter += 1
            m_counter += 1
        yr_counter += 1

    return None

@click.command()
@click.argument('ops', metavar='<operations: choices: s3 | es | disk | thumbs>', nargs=-1)
@click.option('--start', default=None, help='Start Date. Format: YYYY-MM-DD')
@click.option('--end', default=None, help='End Date. Format: YYYY-MM-DD')
@click.option('--es-host', default='localhost', help='Elasticsearch host address')
@click.option('--es-port', default=9200, type=int, help='Elasticsearch port number')
@click.option('--folder', default='.', help='Destination folder if is written to disk')
@click.option('--download', is_flag=True,
              help='Sets the updater to download the metadata file first instead of streaming it')
@click.option('--aws', is_flag=True, default=False,
              help='Uses AWS STS to obtain aws credentials for accessing the various services')
@click.option('--download-folder', default=None,
              help='The folder to save the downloaded metadata to. Defaults to a temp folder')
@click.option('-v', '--verbose', is_flag=True)
@click.option('--concurrency', default=20, type=int, help='Process concurrency. Default=20')
def main(ops, start, end, es_host, es_port, folder, download,
         aws, download_folder, verbose, concurrency):

    if not ops:
        raise click.UsageError('No Argument provided. Use --help if you need help')

    accepted_args = {
        'es': elasticsearch_updater,
        's3': s3_writer,
        'disk': file_writer,
        'thumbs': thumbnail_writer
    }

    writers = []
    for op in ops:
        if op in accepted_args.keys():
            writers.append(accepted_args[op])
        else:
            raise click.UsageError('Operation (%s) is not supported' % op)

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()

    if verbose:
        ch.setLevel(logging.INFO)
    else:
        ch.setLevel(logging.ERROR)

    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if 'es' in ops or 'thumbs' in ops:
        create_index(es_index, es_type, es_host, es_port, aws=aws)

    if not start and not end:
        delta = timedelta(days=3)
        start = date.today() - delta
        start = '{0}-{1}-{2}'.format(start.year, start.month, start.day)

    csv_reader(folder, writers, start_date=start, end_date=end, download=download, download_path=download_folder,
               num_worker_threads=concurrency, es_host=es_host, es_port=es_port, aws=aws)

if __name__ == '__main__':
    main()
<commit_msg>add token<commit_after>import os
import json
import boto
import urllib2
from boto.s3.key import Key
import boto3
import click
import logging
from copy import copy
from collections import OrderedDict
from datetime import date, timedelta
from elasticsearch import Elasticsearch, RequestError, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import requests

from reader import csv_reader

logger = logging.getLogger('landsat8.meta')
bucket_name = os.getenv('BUCKETNAME', 'landsat8-meta')
thumbs_bucket_name = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
s3 = boto3.resource('s3')
es_index = 'sat-api'
es_type = 'landsat8'

def get_url(url, j=False):
    req = urllib2.Request(url)
    r = urllib2.urlopen(req).read()
    if j:
        obj = json.loads(r)
    else:
        obj = r
    return obj

def get_role():
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
    obj = get_url(url)
    return obj

def get_credentials():
    role = get_role()
    url = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/' + role
    obj = get_url(url, True)
    return obj

def connection_to_es(es_host, es_port, aws=False):

    args = {}

    if aws:
        cred = get_credentials()

        access_key = cred['AccessKeyId']
        secret_access = cred['SecretAccessKey']
        token = cred['token']
        region = os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
        awsauth = AWS4Auth(access_key, secret_access, region, 'es',
                           session_token=token)

        args = {
            'http_auth': awsauth,
            'use_ssl': True,
            'verify_certs': True,
            'connection_class': RequestsHttpConnection
        }

    es = Elasticsearch(hosts=[{
        'host': es_host,
        'port': es_port
    }], **args)

    return es

def create_index(index_name, doc_type, es_host, es_port, **kwargs):

    body = {
        doc_type: {
            'properties': {
                'scene_id': {'type': 'string', 'index': 'not_analyzed'},
                'satellite_name': {'type': 'string'},
                'cloud_coverage': {'type': 'float'},
                'date': {'type': 'date'},
                'data_geometry': {
                    'type': 'geo_shape',
                    'tree': 'quadtree',
                    'precision': '5mi'}
            }
        }
    }
    es = connection_to_es(es_host, es_port, kwargs['aws'])

    es.indices.create(index=index_name, ignore=400)

    es.indices.put_mapping(
        doc_type=doc_type,
        body=body,
        index=index_name
    )

def meta_constructor(metadata):
    internal_meta = copy(metadata)

    data_geometry = {
        'type': 'Polygon',
        'crs': {
            'type': 'name',
            'properties': {
                'name': 'urn:ogc:def:crs:EPSG:8.9:4326'
            }
        },
        'coordinates': [[
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')],
            [metadata.get('upperLeftCornerLongitude'), metadata.get('upperLeftCornerLatitude')],
            [metadata.get('lowerLeftCornerLongitude'), metadata.get('lowerLeftCornerLatitude')],
            [metadata.get('lowerRightCornerLongitude'), metadata.get('lowerRightCornerLatitude')],
            [metadata.get('upperRightCornerLongitude'), metadata.get('upperRightCornerLatitude')]
        ]]
    }

    body = OrderedDict([
        ('scene_id', metadata.get('sceneID')),
        ('satellite_name', 'landsat-8'),
        ('cloud_coverage', metadata.get('cloudCoverFull', 100)),
        ('date', metadata.get('acquisitionDate')),
        ('thumbnail', metadata.get('browseURL')),
        ('data_geometry', data_geometry)
    ])

    body.update(internal_meta)

    return body

def elasticsearch_updater(product_dir, metadata, **kwargs):

    try:
        es = connection_to_es(kwargs['es_host'], kwargs['es_port'], kwargs['aws'])

        body = meta_constructor(metadata)

        logger.info('Pushing to Elasticsearch')

        try:
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)
        except RequestError as e:
            body['data_geometry'] = None
            es.index(index=es_index, doc_type=es_type, id=body['scene_id'],
                     body=body, timeout=10)

    except Exception as e:
        logger.error('Unhandled error occured while writing to elasticsearch')
        logger.error('Details: %s' % e.__str__())

def thumbnail_writer(product_dir, metadata, **kwargs):
    """
    Extra function to download images from USGS, then upload to S3 and call
    the ES metadata writer afterwards.
    """

    # Download original thumbnail
    output_file = metadata['sceneID'] + '.jpg'
    thumbnail = 'https://' + thumbs_bucket_name + '.s3.amazonaws.com/' + output_file

    new_thumb = requests.get(thumbnail)
    if new_thumb.status_code != 200:

        # Upload thumbnail to S3
        thumbs_bucket_name2 = os.getenv('THUMBS_BUCKETNAME', 'ad-thumbnails')
        try:
            print('uploading %s' % output_file)
            c = boto.connect_s3()
            b = c.get_bucket(thumbs_bucket_name2)
            k = Key(b, name=output_file)
            k.set_metadata('Content-Type', 'image/jpeg')
            r = requests.get(metadata['browseURL'])
            k.set_contents_from_string(r.content, policy='public-read')
        except Exception as e:
            print(e)

    # Update metadata record
    metadata['thumbnail'] = thumbnail

    elasticsearch_updater(product_dir, metadata, **kwargs)
    return

def file_writer(product_dir, metadata):
    body = meta_constructor(metadata)

    if not os.path.exists(product_dir):
        os.makedirs(product_dir)

    f = open(os.path.join(product_dir, body['scene_id'] + '.json'), 'w')
    f.write(json.dumps(body))
    logger.info('saving to disk at %s' % product_dir)
    f.close()

def s3_writer(product_dir, metadata):
    # make sure product_dir doesn't start with slash (/) or dot (.)
    if product_dir.startswith('.'):
        product_dir = product_dir[1:]

    if product_dir.startswith('/'):
        product_dir = product_dir[1:]

    body = meta_constructor(metadata)

    key = os.path.join(product_dir, body['scene_id'] + '.json')
    s3.Object(bucket_name, key).put(Body=json.dumps(body), ACL='public-read', ContentType='application/json')

    logger.info('saving to s3 at %s')

def last_updated(today):
    """ Gets the latest time a product added to Elasticsearch """

    bucket = s3.Bucket(bucket_name)

    start_day = today.day
    start_month = today.month

    yr_counter = 0
    while True:
        m_counter = 0
        year = today.year - yr_counter
        if year < 2015:
            break
        while True:
            month = start_month - m_counter
            if month == 0:
                start_month = 12
                break
            d_counter = 0
            while True:
                day = start_day - d_counter
                if day == 0:
                    start_day = 31
                    break
                path = os.path.join(str(year), str(month), str(day))
                logger.info('checking %s' % path)
                objs = bucket.objects.filter(Prefix=path).limit(1)
                if list(objs):
                    return date(year, month, day)
                d_counter += 1
            m_counter += 1
        yr_counter += 1

    return None

@click.command()
@click.argument('ops', metavar='<operations: choices: s3 | es | disk | thumbs>', nargs=-1)
@click.option('--start', default=None, help='Start Date. Format: YYYY-MM-DD')
@click.option('--end', default=None, help='End Date. Format: YYYY-MM-DD')
@click.option('--es-host', default='localhost', help='Elasticsearch host address')
@click.option('--es-port', default=9200, type=int, help='Elasticsearch port number')
@click.option('--folder', default='.', help='Destination folder if is written to disk')
@click.option('--download', is_flag=True,
              help='Sets the updater to download the metadata file first instead of streaming it')
@click.option('--aws', is_flag=True, default=False,
              help='Uses AWS STS to obtain aws credentials for accessing the various services')
@click.option('--download-folder', default=None,
              help='The folder to save the downloaded metadata to. Defaults to a temp folder')
@click.option('-v', '--verbose', is_flag=True)
@click.option('--concurrency', default=20, type=int, help='Process concurrency. Default=20')
def main(ops, start, end, es_host, es_port, folder, download,
         aws, download_folder, verbose, concurrency):

    if not ops:
        raise click.UsageError('No Argument provided. Use --help if you need help')

    accepted_args = {
        'es': elasticsearch_updater,
        's3': s3_writer,
        'disk': file_writer,
        'thumbs': thumbnail_writer
    }

    writers = []
    for op in ops:
        if op in accepted_args.keys():
            writers.append(accepted_args[op])
        else:
            raise click.UsageError('Operation (%s) is not supported' % op)

    logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()

    if verbose:
        ch.setLevel(logging.INFO)
    else:
        ch.setLevel(logging.ERROR)

    formatter = logging.Formatter('%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if 'es' in ops or 'thumbs' in ops:
        create_index(es_index, es_type, es_host, es_port, aws=aws)

    if not start and not end:
        delta = timedelta(days=3)
        start = date.today() - delta
        start = '{0}-{1}-{2}'.format(start.year, start.month, start.day)

    csv_reader(folder, writers, start_date=start, end_date=end, download=download, download_path=download_folder,
               num_worker_threads=concurrency, es_host=es_host, es_port=es_port, aws=aws)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import asyncio
import curses
import random
import math

import async_curses
import network_tools

import time

DEFAULT = 1 # DEFAULT_COLOR
RED = 2
GREEN = 3

SELECT_DEFAULT = 11
SELECT_RED = 12
SELECT_GREEN = 13

BLACK_BACKGROUND = 0
WHITE_BACKGROUND = 1

class HostInfoWindow(async_curses.Window):
	def __init__(self, parent, height=-1, width=-1, y_start=0, x_start=0):
		super().__init__(parent, height, width, y_start, x_start)
		self.Active = False
		self.PingResults = None
		self.lable = '-:'
		self.lable_color = DEFAULT
		self.info = 'Unknown'
		self.info_color = RED

	@property
	def ping_results(self):
		return

	@ping_results.setter
	def ping_results(self, value):
		# get the colors
		self.lable_color = DEFAULT
		self.info_color = DEFAULT
		if self.active:
			self.lable_color = SELECT_DEFAULT
			self.info_color = SELECT_DEFAULT
			if value.state == network_tools.STATE_UP:
				self.info_color = SELECT_GREEN
			else:
				self.info_color = SELECT_RED

		else:
			if value.state == network_tools.STATE_UP:
				self.info_color = GREEN
			else:
				self.info_color = RED

		self.lable = '-:'
		host = 'Unknown'
		r_dns = None
		if value.ip is not None:
			self.lable = value.ip.split('.')[3] + ':'
			r_dns = network_tools.reverse_dns(value.ip)
			if r_dns is not None:
				host = r_dns.split('.')[0]
			width = self.maxyx[1] - 4
			if len(host)>width:
				host = host[0:width-1]

		self.info = host

		self.update_contents()

	@property
	def active(self):
		return self.Active

	@active.setter
	def active(self, value):
		if value == True or value == False:
			self.Active = value
			if self.active == True:
				self.lable_color = SELECT_DEFAULT
				if self.info_color == RED:
					self.info_color = SELECT_RED
				elif self.info_color == GREEN:
					self.info_color = SELECT_GREEN
			else:
				self.lable_color = DEFAULT
				if self.info_color == SELECT_RED:
					self.info_color = RED
				elif self.info_color == SELECT_GREEN:
					self.info_color = GREEN
			self.update_contents()
	'''
	@property
	def lable(self):
		return self.Lable

	@lable.setter
	def lable(self, value):
		old = self.Lable
		self.Lable = str(value)
		if self.lable != old:
			self.update_contents()

	@property
	def state(self):
		return self.State

	@state.setter
	def state(self, value):
		old = self.state
		if value == 'UP' or value == 'DOWN':
			self.State = value
		elif value == network_tools.STATE_UP:
			self.State = 'UP'
		elif value == network_tools.STATE_DOWN:
			self.State = 'DOWN'
		if self.state != old:
			self.update_contents()
	'''

	def update_contents(self):
		self.clear_text()
		self.add(self.lable, color=self.lable_color)
		self.add(self.info, color=self.info_color)
		self.text_area.noutrefresh()

class UI(async_curses.BaseUI):
	def setup(self):
		curses.init_pair(DEFAULT, curses.COLOR_CYAN, curses.COLOR_BLACK)
		curses.init_pair(RED, curses.COLOR_RED, curses.COLOR_BLACK)
		curses.init_pair(GREEN, curses.COLOR_GREEN, curses.COLOR_BLACK)

		curses.init_pair(SELECT_DEFAULT, curses.COLOR_CYAN, curses.COLOR_WHITE)
		curses.init_pair(SELECT_RED, curses.COLOR_RED, curses.COLOR_WHITE)
		curses.init_pair(SELECT_GREEN, curses.COLOR_GREEN, curses.COLOR_WHITE)

		maxy, maxx = self.maxyx
		self.title = async_curses.BorderedWindow(self.main_window, 3, maxx, 0, 0)
		title_text = "Network 10.10.1.0/24"
		num_spaces = (maxx -2 - len(title_text))//2
		self.title.contents = ' '*num_spaces + title_text
		self.body = async_curses.BorderedWindow(self.main_window, maxy - 3, maxx, 3, 0)
		self.rows = 43
		self.cols = 6

		self.layout = async_curses.TableLayout(self.body.text_area, self.rows, self.cols, HostInfoWindow)

		self.active_cell = [0,0]
		self.ping_results = {}

	async def ping_worker(self, rate):
		tasks = []
		num = 0
		while True:
			await asyncio.sleep(rate)
			task = asyncio.ensure_future(network_tools.ping(f'10.10.8.{num}'))
			tasks.append(task)

			#check for results
			for task in tasks:
				await asyncio.sleep(0)
				if task.done():
					tasks.remove(task)
					result = task.result()
					result = network_tools.parse_ping_output(*result)
					ip_num = None
					if result.ip is not None:
						ip_num = int(result.ip.split('.')[3])
					else:
						continue
					c = math.floor(ip_num/self.rows)
					r = ip_num % self.rows

					textarea = self.layout.sub_windows[r][c]
					textarea.ping_results = result

			num += 1
			if num > 255:
				num = 0

	def key_stroke_handler(self, key):
		last_active = self.layout.sub_windows[self.active_cell[0]][self.active_cell[1]]
		last_active.active = False

		if key == curses.KEY_UP:
			self.active_cell[0] -= 1
			if self.active_cell[0] < 0:
				self.active_cell[0] = 0

		elif key == curses.KEY_DOWN:
			self.active_cell[0] += 1
			if self.active_cell[0] > self.rows - 1:
				self.active_cell[0] = self.rows - 1

		elif key == curses.KEY_RIGHT:
			self.active_cell[1] += 1
			if self.active_cell[1] > self.cols -1:
				self.active_cell[1] = self.cols -1

		elif key == curses.KEY_LEFT:
			self.active_cell[1] -= 1
			if self.active_cell[1] < 0:
				self.active_cell[1] = 0

		current_active = self.layout.sub_windows[self.active_cell[0]][self.active_cell[1]]
		current_active.active = True

if __name__=='__main__':
	try:
		with UI(frame_rate=10) as ui:
			loop = asyncio.get_event_loop()
			asyncio.ensure_future(ui.keyboard_listener())
			asyncio.ensure_future(ui.ping_worker(0.1))
			loop.run_until_complete(ui.screen_updater())
	except KeyboardInterrupt:
		pass
<commit_msg>fixed display bug related to scrolling of windows and long host names.<commit_after>import asyncio
import curses
import random
import math

import async_curses
import network_tools

import time

DEFAULT = 1 # DEFAULT_COLOR
RED = 2
GREEN = 3

SELECT_DEFAULT = 11
SELECT_RED = 12
SELECT_GREEN = 13

BLACK_BACKGROUND = 0
WHITE_BACKGROUND = 1

class HostInfoWindow(async_curses.Window):
	def __init__(self, parent, height=-1, width=-1, y_start=0, x_start=0):
		super().__init__(parent, height, width, y_start, x_start)
		self.Active = False
		self.PingResults = None
		self.lable = '-:'
		self.lable_color = DEFAULT
		self.info = 'Unknown'
		self.info_color = RED

	@property
	def ping_results(self):
		return

	@ping_results.setter
	def ping_results(self, value):
		# get the colors
		self.lable_color = DEFAULT
		self.info_color = DEFAULT
		if self.active:
			self.lable_color = SELECT_DEFAULT
			self.info_color = SELECT_DEFAULT
			if value.state == network_tools.STATE_UP:
				self.info_color = SELECT_GREEN
			else:
				self.info_color = SELECT_RED

		else:
			if value.state == network_tools.STATE_UP:
				self.info_color = GREEN
			else:
				self.info_color = RED

		self.lable = '-:'
		host = 'Unknown'
		r_dns = None
		if value.ip is not None:
			self.lable = value.ip.split('.')[3] + ':'
			r_dns = network_tools.reverse_dns(value.ip)
			if r_dns is not None:
				host = r_dns.split('.')[0]
			width = self.maxyx[1] - 5
			if len(host)>width:
				host = host[0:width-1]

		self.info = host

		self.update_contents()

	@property
	def active(self):
		return self.Active

	@active.setter
	def active(self, value):
		if value == True or value == False:
			self.Active = value
			if self.active == True:
				self.lable_color = SELECT_DEFAULT
				if self.info_color == RED:
					self.info_color = SELECT_RED
				elif self.info_color == GREEN:
					self.info_color = SELECT_GREEN
			else:
				self.lable_color = DEFAULT
				if self.info_color == SELECT_RED:
					self.info_color = RED
				elif self.info_color == SELECT_GREEN:
					self.info_color = GREEN
			self.update_contents()
	'''
	@property
	def lable(self):
		return self.Lable

	@lable.setter
	def lable(self, value):
		old = self.Lable
		self.Lable = str(value)
		if self.lable != old:
			self.update_contents()

	@property
	def state(self):
		return self.State

	@state.setter
	def state(self, value):
		old = self.state
		if value == 'UP' or value == 'DOWN':
			self.State = value
		elif value == network_tools.STATE_UP:
			self.State = 'UP'
		elif value == network_tools.STATE_DOWN:
			self.State = 'DOWN'
		if self.state != old:
			self.update_contents()
	'''

	def update_contents(self):
		self.clear_text()
		self.add(self.lable, color=self.lable_color)
		self.add(self.info, color=self.info_color)
		self.text_area.noutrefresh()

class UI(async_curses.BaseUI):
	def setup(self):
		curses.init_pair(DEFAULT, curses.COLOR_CYAN, curses.COLOR_BLACK)
		curses.init_pair(RED, curses.COLOR_RED, curses.COLOR_BLACK)
		curses.init_pair(GREEN, curses.COLOR_GREEN, curses.COLOR_BLACK)

		curses.init_pair(SELECT_DEFAULT, curses.COLOR_CYAN, curses.COLOR_WHITE)
		curses.init_pair(SELECT_RED, curses.COLOR_RED, curses.COLOR_WHITE)
		curses.init_pair(SELECT_GREEN, curses.COLOR_GREEN, curses.COLOR_WHITE)

		maxy, maxx = self.maxyx
		self.title = async_curses.BorderedWindow(self.main_window, 3, maxx, 0, 0)
		title_text = "Network 10.10.1.0/24"
		num_spaces = (maxx -2 - len(title_text))//2
		self.title.contents = ' '*num_spaces + title_text
		self.body = async_curses.BorderedWindow(self.main_window, maxy - 3, maxx, 3, 0)
		self.rows = 43
		self.cols = 6

		self.layout = async_curses.TableLayout(self.body.text_area, self.rows, self.cols, HostInfoWindow)

		self.active_cell = [0,0]
		self.ping_results = {}

	async def ping_worker(self, rate):
		tasks = []
		num = 0
		while True:
			await asyncio.sleep(rate)
			task = asyncio.ensure_future(network_tools.ping(f'10.10.8.{num}'))
			tasks.append(task)

			#check for results
			for task in tasks:
				await asyncio.sleep(0)
				if task.done():
					tasks.remove(task)
					result = task.result()
					result = network_tools.parse_ping_output(*result)
					ip_num = None
					if result.ip is not None:
						ip_num = int(result.ip.split('.')[3])
					else:
						continue
					c = math.floor(ip_num/self.rows)
					r = ip_num % self.rows

					textarea = self.layout.sub_windows[r][c]
					textarea.ping_results = result

			num += 1
			if num > 255:
				num = 0

	def key_stroke_handler(self, key):
		last_active = self.layout.sub_windows[self.active_cell[0]][self.active_cell[1]]
		last_active.active = False

		if key == curses.KEY_UP:
			self.active_cell[0] -= 1
			if self.active_cell[0] < 0:
				self.active_cell[0] = 0

		elif key == curses.KEY_DOWN:
			self.active_cell[0] += 1
			if self.active_cell[0] > self.rows - 1:
				self.active_cell[0] = self.rows - 1

		elif key == curses.KEY_RIGHT:
			self.active_cell[1] += 1
			if self.active_cell[1] > self.cols -1:
				self.active_cell[1] = self.cols -1

		elif key == curses.KEY_LEFT:
			self.active_cell[1] -= 1
			if self.active_cell[1] < 0:
				self.active_cell[1] = 0

		current_active = self.layout.sub_windows[self.active_cell[0]][self.active_cell[1]]
		current_active.active = True

if __name__=='__main__':
	try:
		with UI(frame_rate=10) as ui:
			loop = asyncio.get_event_loop()
			asyncio.ensure_future(ui.keyboard_listener())
			asyncio.ensure_future(ui.ping_worker(0.1))
			loop.run_until_complete(ui.screen_updater())
	except KeyboardInterrupt:
		pass
<|endoftext|>


<commit_before>"""
Nest-WFH is a project that aims to prevent your Nest thermostat from going into
auto-away mode if your phone is also at home.

The program works by querying to see if your phone's MAC address appears on the
network. If so, it will manually set all thermostats to "home". This is meant
to be run as a cron job.
"""

__author__ = 'ajay@roopakalu.com (Ajay Roopakalu)'

import re
import subprocess
import sys

import dateutil.parser
import requests
import nmap

from datetime import datetime
from datetime import timedelta
from dateutil import tz

import calendar_client
import keys

STRUCTURE_URL = 'https://developer-api.nest.com/structures/'
THERMOSTATS_URL = 'https://developer-api.nest.com/devices/thermostats/'

STATUS_HOME = '"home"'
STATUS_AWAY = '"away"'

ENTER_WORK_REGEX = re.compile('I entered work')
EXIT_WORK_REGEX = re.compile('I exited work')
WFH_REGEX = re.compile('WFH')
MIDDAY = 12 # Hour of the day to indicate noon

def IsReferenceDeviceOnNetwork(ip_subnet, mac_address):
  # Refresh the ARP cache by doing a fast scan of hosts on this subnet
  nm = nmap.PortScanner()
  nm.scan(hosts=ip_subnet, arguments='-sP')
  # Query the ARP cache for the specified MAC address
  proc = subprocess.Popen(['arp', '-n'], stdout=subprocess.PIPE)
  output, _ = proc.communicate()
  for line in output.split('\n'):
    if line.find(mac_address) > 0:
      return True
  return False

def GetAllThermostats(access_token):
  response = requests.get(THERMOSTATS_URL, params={'auth': access_token})
  return response.json()

def GetStructureIds(thermostat_model):
  structure_ids = []
  for thermostat in thermostat_model.itervalues():
    structure_ids.append(thermostat['structure_id'])
  return structure_ids

def GetAwayStatus(access_token, structure_ids):
  params = {'auth': access_token}
  headers = {'Content-Type': 'application/json'}
  results = {}
  for structure_id in structure_ids:
    response = requests.get(
        STRUCTURE_URL + structure_id + '/away',
        params=params, headers=headers)
    results[structure_id] = response.text
  return results

def SetAwayStatus(access_token, structure_ids, status):
  params = {'auth': access_token}
  headers = {'Content-Type': 'application/json'}
  results = {}
  existing_statuses = GetAwayStatus(access_token, structure_ids)

  for structure_id in structure_ids:
    if existing_statuses[structure_id] != status:
      print 'Setting status of', structure_id, 'to:', status
      response = requests.put(
          STRUCTURE_URL + structure_id + '/away',
          params=params, headers=headers, data=status)
      results[structure_id] = (response.text == status)
    else:
      print 'Target status of', status, 'for', structure_id, 'already set.'
  return results

def main(argv):
  now = datetime.now(tz=tz.tzlocal())
  localized_now = now.astimezone(tz.gettz(keys.WORK_HOURS_CALENDAR_TZ))
  today = localized_now.replace(hour=0, minute=0, second=0, microsecond=0)
  tomorrow = today + timedelta(days=1)

  thermostat_model = GetAllThermostats(keys.ACCESS_TOKEN)
  structure_ids = GetStructureIds(thermostat_model)
  calendar_instance = calendar_client.Calendar(argv)

  for event in calendar_instance.GetEvents(
      keys.WORK_HOURS_CALENDAR_ID, today, tomorrow):
    try:
      if WFH_REGEX.match(event.get('summary')):
        print SetAwayStatus(
            keys.ACCESS_TOKEN, structure_ids, status=STATUS_HOME)
      startEntity = event.get('start')
      # Ignore full-day events here.
      if not startEntity.get('dateTime'):
        continue
      startTime = dateutil.parser.parse(startEntity.get('dateTime'))
      if today < startTime and startTime < tomorrow:
          if (localized_now.hour <= MIDDAY and
              ENTER_WORK_REGEX.match(event.get('summary'))):
            print 'User is at work..'
            print SetAwayStatus(
                keys.ACCESS_TOKEN, structure_ids, status=STATUS_AWAY)
          if (localized_now.hour > MIDDAY and
              EXIT_WORK_REGEX.match(event.get('summary'))):
            print 'User is coming home..'
            print SetAwayStatus(
                keys.ACCESS_TOKEN, structure_ids, status=STATUS_HOME)
    except Exception as e:
      print 'Error while performing operation:', e

if __name__ == '__main__':
  main(sys.argv)
<commit_msg>Use Python logging library instead of print.<commit_after>"""
Nest-WFH is a project that aims to prevent your Nest thermostat from going into
auto-away mode if your phone is also at home.

The program works by querying to see if your phone's MAC address appears on the
network. If so, it will manually set all thermostats to "home". This is meant
to be run as a cron job.
"""

__author__ = 'ajay@roopakalu.com (Ajay Roopakalu)'

import logging
import logging.handlers
import re
import subprocess
import sys

import dateutil.parser
import requests
import nmap

from datetime import datetime
from datetime import timedelta
from dateutil import tz

import calendar_client
import keys

LOG_FILENAME = '/var/log/nest-wfh/nest-wfh.INFO'

STRUCTURE_URL = 'https://developer-api.nest.com/structures/'
THERMOSTATS_URL = 'https://developer-api.nest.com/devices/thermostats/'

STATUS_HOME = '"home"'
STATUS_AWAY = '"away"'

ENTER_WORK_REGEX = re.compile('I entered work')
EXIT_WORK_REGEX = re.compile('I exited work')
WFH_REGEX = re.compile('WFH')
MIDDAY = 12 # Hour of the day to indicate noon

LOGGER = None

def Init():
  global LOGGER
  fmt = '%(asctime)-15s %(threadName)s %(filename)s:%(lineno)d %(message)s'
  LOGGER = logging.getLogger(__name__)
  LOGGER.setLevel(logging.INFO)
  # Rotate at 1M bytes, store 5 old versions
  handler = logging.handlers.RotatingFileHandler(
      LOG_FILENAME, maxBytes=1>>20, backupCount=5)
  handler.setFormatter(logging.Formatter(fmt))
  LOGGER.addHandler(handler)

def IsReferenceDeviceOnNetwork(ip_subnet, mac_address):
  # Refresh the ARP cache by doing a fast scan of hosts on this subnet
  nm = nmap.PortScanner()
  nm.scan(hosts=ip_subnet, arguments='-sP')
  # Query the ARP cache for the specified MAC address
  proc = subprocess.Popen(['arp', '-n'], stdout=subprocess.PIPE)
  output, _ = proc.communicate()
  for line in output.split('\n'):
    if line.find(mac_address) > 0:
      return True
  return False

def GetAllThermostats(access_token):
  response = requests.get(THERMOSTATS_URL, params={'auth': access_token})
  return response.json()

def GetStructureIds(thermostat_model):
  structure_ids = []
  for thermostat in thermostat_model.itervalues():
    structure_ids.append(thermostat['structure_id'])
  return structure_ids

def GetAwayStatus(access_token, structure_ids):
  params = {'auth': access_token}
  headers = {'Content-Type': 'application/json'}
  results = {}
  for structure_id in structure_ids:
    response = requests.get(
        STRUCTURE_URL + structure_id + '/away',
        params=params, headers=headers)
    results[structure_id] = response.text
  return results

def SetAwayStatus(access_token, structure_ids, status):
  params = {'auth': access_token}
  headers = {'Content-Type': 'application/json'}
  results = {}
  existing_statuses = GetAwayStatus(access_token, structure_ids)

  for structure_id in structure_ids:
    if existing_statuses[structure_id] != status:
      LOGGER.info('Setting status of %s to : %s',structure_id, status)
      response = requests.put(
          STRUCTURE_URL + structure_id + '/away',
          params=params, headers=headers, data=status)
      results[structure_id] = (response.text == status)
    else:
      LOGGER.info(
          'Target status of %s for %s already set.', status, structure_id)
  return results

def GetWorkStatusEvents(service, today, tomorrow):
  try:
    events = service.events().list(
        calendarId=keys.WORK_HOURS_CALENDAR_ID, orderBy='startTime',
        singleEvents=True, timeMin=today.isoformat(),
        timeMax=tomorrow.isoformat()).execute()
    return events.get('items')
  except client.AccessTokenRefreshError:
    LOGGER.warning(
        'The credentials have been revoked or expired, please re-run the '
        'application to re-authorize.')
    sys.exit(-1)

def main(argv):
  Init()

  now = datetime.now(tz=tz.tzlocal())
  localized_now = now.astimezone(tz.gettz(keys.WORK_HOURS_CALENDAR_TZ))
  today = localized_now.replace(hour=0, minute=0, second=0, microsecond=0)
  tomorrow = today + timedelta(days=1)

  LOGGER.info('Retrieving relevant calendar events.')
  calendar_instance = calendar_client.Calendar(argv)
  events = calendar_instance.GetEvents(
      keys.WORK_HOURS_CALENDAR_ID, today, tomorrow)
  if not events:
    LOGGER.info('No events found.')
    exit(0)

  LOGGER.info('Retrieving known thermostats.')
  thermostat_model = GetAllThermostats(keys.ACCESS_TOKEN)
  structure_ids = GetStructureIds(thermostat_model)

  for event in events:
    try:
      # If WFH, always set status to HOME.
      if WFH_REGEX.match(event.get('summary')):
        LOGGER.info(
            SetAwayStatus(keys.ACCESS_TOKEN, structure_ids, status=STATUS_HOME))

      startEntity = event.get('start')
      # Ignore full-day events here.
      if not startEntity.get('dateTime'):
        continue
      startTime = dateutil.parser.parse(startEntity.get('dateTime'))

      if today < startTime < tomorrow:
          if (localized_now.hour <= MIDDAY and
              ENTER_WORK_REGEX.match(event.get('summary'))):
            LOGGER.info('User is at work..')
            LOGGER.info(
                SetAwayStatus(
                    keys.ACCESS_TOKEN, structure_ids, status=STATUS_AWAY))
          if (localized_now.hour > MIDDAY and
              EXIT_WORK_REGEX.match(event.get('summary'))):
            LOGGER.info('User is coming home..')
            LOGGER.info(
                SetAwayStatus(
                    keys.ACCESS_TOKEN, structure_ids, status=STATUS_HOME))
    except Exception as e:
      LOGGER.exception('Error while performing operation: %s', e)

if __name__ == '__main__':
  main(sys.argv)
<|endoftext|>


<commit_before>import csv
from operator import itemgetter
from tabulate import tabulate
import random
import math

DRAW = 0
WIN = 1
LOSS = 2

output = []

class Ladder:

    def __init__(self, rounds, teams, points={WIN: 2, DRAW: 1, LOSS: 0}):
        self.rounds = rounds
        self.points = points
        self.init_ladder(teams)

    def init_ladder(self, teams):
        # Create a blank ladder with an entry for each team, including W/L/D/P
        # statistics
        self.ladder = []
        for team in teams:
            entry = {}
            entry['Name'] = team['Name']
            entry['Win'] = 0
            entry['Loss'] = 0
            entry['Draw'] = 0
            entry['Points'] = 0
            self.ladder.append(entry)

    def record_result(self, result):
        if result[0] == WIN:
            self.record_win(result[1])
            self.record_loss(result[2])
        elif result[0] == DRAW:
            self.record_draw(result[1])
            self.record_draw(result[2])
        else:
            raise ValueError('Result type not supported!')

    def team_index(self, name):
        # Get the index of a team in the ladder by a given name.
        for index, team in enumerate(self.ladder):
            if team['Name'] == name:
                return index
        else:
            raise KeyError('Team ' + str(name) + ' not found in ladder!')

    def record_win(self, team):
        self.ladder[self.team_index(team)]['Win'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[WIN]

    def record_loss(self, team):
        self.ladder[self.team_index(team)]['Loss'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[LOSS]

    def record_draw(self, team):
        self.ladder[self.team_index(team)]['Draw'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[DRAW]

    def sort_ladder(self):
        self.ladder.sort(key=itemgetter('Points'), reverse=True)

    def top(self, n):
        # Return top n teams in ladder.
        self.sort_ladder()
        return self.ladder[:n]

    def print_ladder(self):
        self.sort_ladder()
        printable = [['Name', 'Win', 'Loss', 'Draw', 'Points']]

        for row in self.ladder:
            printable.append([row['Name'],
                              row['Win'],
                              row['Loss'],
                              row['Draw'],
                              row['Points']])

        print(tabulate(printable, headers='firstrow'))

def basic_game(team1, team2):
    # A template for all games. The game must take two team dictionaries.

    # If the game is won by a team, the result is set to the WIN constant.
    # If the game is drawn, the result is set to the DRAW constant.
    result = WIN

    # If the game has a winner, the winner is set to the name of the winning
    # team and the loser to the name of the losing team. In the event of a
    # draw, order does not matter, but both variables must still be filled.
    winner = team1['Name']
    loser = team2['Name']

    # All extra game statistics are stored in the stats dictionary. The key for
    # each entry is the name of the statistic.
    stats = {}

    # The game must return this information in the following tuple format.
    return (result, winner, loser, stats)

def cricket(team1, team2):
    # Simulate a game of cricket.
    if team1['Strength'] == team2['Strength']:
        result = DRAW
        winner = team1['Name']
        loser = team2['Name']
    else:
        result = WIN
        if team1['Strength'] > team2['Strength']:
            winner = team1['Name']
            loser = team2['Name']
        else:
            winner = team2['Name']
            loser = team1['Name']

    # Generate winner statistics
    winning_runs = random.randint(120, 180)
    winning_wickets = random.randint(5, 10)
    winning_score = str(winning_wickets) + '/' + str(winning_runs)

    if result == DRAW:
        losing_runs = winning_runs
    else:
        losing_runs = random.randint(80, winning_runs)

    losing_wickets = random.randint(5, 10)
    losing_score = str(losing_wickets) + '/' + str(losing_runs)

    stats = {'Winning Score': winning_score,
             'Losing Score': losing_score}

    return (result, winner, loser, stats)

def play(team1, team2, game, ladder=None):
    # Return the result of a match played between two teams according to
    # the rules of a given game. If a ladder is supplied, the result is
    # recorded in the ladder.
    result = game(team1, team2)

    if ladder is not None:
        ladder.record_result(result)

    return result

def rotate_except_first(l):
    # Rotate a list of teams excluding the first team.
    # For example,
    # 1 2 3 4 5 6 -> 1 6 2 3 4 5
    new = [l[0], l[-1]]
    for i in range(2, len(l)):
        new.append(l[i-1])
    return new

def elimination(teams):
    # Play a simple elimination fixture for the given teams.

    number_of_rounds = int(math.log(len(teams), 2))

    for n in range(number_of_rounds):
        matches = loop_matches(teams)

        # Play all matches in round
        results = [play(match[0], match[1], cricket, None) for match in matches]

        for result in results:
            store(result, "Finals " + str(n + 1))

            # Eliminate losing teams
            teams = [team for team in teams if team['Name'] != result[2]]

    return teams

def loop_matches(teams):
    # Split the list of teams into two halves and them zip them in matches.
    # For example, 1 2 3 4 5 6, or:
    # 1 2 3
    # 6 5 4
    # Would become,
    # (1, 6), (2, 5), (3, 4)
    return list(zip(teams[:len(teams)//2], reversed(teams[len(teams)//2:])))

def round_robin(teams):
    # Generate a round-robin fixture using the algorithm from
    # https://en.wikipedia.org/wiki/Round-robin_tournament

    # Add a dummy team to support byes in competitions with an uneven number
    # of teams.
    if len(teams) % 2 != 0:
        teams.append('BYE')

    number_of_rounds = len(teams) - 1
    rounds = []

    for n in range(number_of_rounds):
        matches = loop_matches(teams)
        rounds.append(matches)
        teams = rotate_except_first(teams)

    return rounds

def convert_to_int(n):
    try:
        return int(n)
    except ValueError:
        return n

def load_teams(filename):
    with open(filename) as f:
        reader = csv.DictReader(f)
        teams = [team for team in reader]

    # Convert data to integer if possible.
    for team in teams:
        for key in team:
            team[key] = convert_to_int(team[key])

    return teams

def store(result, round_no):
    # Format result into list and store in output buffer.
    row = {'Round': round_no}

    if result[0] == DRAW:
        row['Winner'] = result[1] + ', ' + result[2]
        row['Loser'] = '-'
    else:
        row['Winner'] = result[1]
        row['Loser'] = result[2]

    # Add all statistics.
    for stat, value in result[3].items():
        row[stat] = value

    output.append(row)

def output_data(filename):
    # Flush output buffer to filename.
    with open(filename, 'w') as f:
        # Add mandatory field names
        fieldnames = ['Round', 'Winner', 'Loser']

        # Add all other field names
        fieldnames.extend([field for field in output[0].keys()
                           if field not in fieldnames])

        writer = csv.DictWriter(f, fieldnames=fieldnames)

        writer.writeheader()
        writer.writerows(output)

def play_fixture(fixture, ladder, game):
    # Take a given fixture of matches (a list of lists of matches).
    # Play each round, getting the result using the specified game
    # and store the result.
    for round_number, round_matches in enumerate(fixture):
        for match in round_matches:
            result = play(match[0], match[1], game, ladder)
            store(result, round_number + 1)

def simple_simulate(teams, structure, finals_structure, final_n=4):
    # Simulate a league using the given teams, structure, finals structure,
    # and number of league winners to move on to the finals.
    fixture = structure(teams)
    ladder = Ladder(len(fixture), teams)

    play_fixture(fixture, ladder, cricket)

    ladder.print_ladder()

    finalist_names = [team['Name'] for team in ladder.top(final_n)]
    finalists = [team for team in teams if team['Name'] in finalist_names]

    print(finals_structure(finalists))
    output_data('out.csv')

teams = load_teams('data.csv')
simple_simulate(teams, round_robin, elimination)
<commit_msg>Generalise simple_simulate<commit_after>import csv
from operator import itemgetter
from tabulate import tabulate
import random
import math

DRAW = 0
WIN = 1
LOSS = 2

output = []

class Ladder:

    def __init__(self, rounds, teams, points={WIN: 2, DRAW: 1, LOSS: 0}):
        self.rounds = rounds
        self.points = points
        self.init_ladder(teams)

    def init_ladder(self, teams):
        # Create a blank ladder with an entry for each team, including W/L/D/P
        # statistics
        self.ladder = []
        for team in teams:
            entry = {}
            entry['Name'] = team['Name']
            entry['Win'] = 0
            entry['Loss'] = 0
            entry['Draw'] = 0
            entry['Points'] = 0
            self.ladder.append(entry)

    def record_result(self, result):
        if result[0] == WIN:
            self.record_win(result[1])
            self.record_loss(result[2])
        elif result[0] == DRAW:
            self.record_draw(result[1])
            self.record_draw(result[2])
        else:
            raise ValueError('Result type not supported!')

    def team_index(self, name):
        # Get the index of a team in the ladder by a given name.
        for index, team in enumerate(self.ladder):
            if team['Name'] == name:
                return index
        else:
            raise KeyError('Team ' + str(name) + ' not found in ladder!')

    def record_win(self, team):
        self.ladder[self.team_index(team)]['Win'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[WIN]

    def record_loss(self, team):
        self.ladder[self.team_index(team)]['Loss'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[LOSS]

    def record_draw(self, team):
        self.ladder[self.team_index(team)]['Draw'] += 1
        self.ladder[self.team_index(team)]['Points'] += self.points[DRAW]

    def sort_ladder(self):
        self.ladder.sort(key=itemgetter('Points'), reverse=True)

    def top(self, n):
        # Return top n teams in ladder.
        self.sort_ladder()
        return self.ladder[:n]

    def print_ladder(self):
        self.sort_ladder()
        printable = [['Name', 'Win', 'Loss', 'Draw', 'Points']]

        for row in self.ladder:
            printable.append([row['Name'],
                              row['Win'],
                              row['Loss'],
                              row['Draw'],
                              row['Points']])

        print(tabulate(printable, headers='firstrow'))

def basic_game(team1, team2):
    # A template for all games. The game must take two team dictionaries.

    # If the game is won by a team, the result is set to the WIN constant.
    # If the game is drawn, the result is set to the DRAW constant.
    result = WIN

    # If the game has a winner, the winner is set to the name of the winning
    # team and the loser to the name of the losing team. In the event of a
    # draw, order does not matter, but both variables must still be filled.
    winner = team1['Name']
    loser = team2['Name']

    # All extra game statistics are stored in the stats dictionary. The key for
    # each entry is the name of the statistic.
    stats = {}

    # The game must return this information in the following tuple format.
    return (result, winner, loser, stats)

def cricket(team1, team2):
    # Simulate a game of cricket.
    if team1['Strength'] == team2['Strength']:
        result = DRAW
        winner = team1['Name']
        loser = team2['Name']
    else:
        result = WIN
        if team1['Strength'] > team2['Strength']:
            winner = team1['Name']
            loser = team2['Name']
        else:
            winner = team2['Name']
            loser = team1['Name']

    # Generate winner statistics
    winning_runs = random.randint(120, 180)
    winning_wickets = random.randint(5, 10)
    winning_score = str(winning_wickets) + '/' + str(winning_runs)

    if result == DRAW:
        losing_runs = winning_runs
    else:
        losing_runs = random.randint(80, winning_runs)

    losing_wickets = random.randint(5, 10)
    losing_score = str(losing_wickets) + '/' + str(losing_runs)

    stats = {'Winning Score': winning_score,
             'Losing Score': losing_score}

    return (result, winner, loser, stats)

def play(team1, team2, game, ladder=None):
    # Return the result of a match played between two teams according to
    # the rules of a given game. If a ladder is supplied, the result is
    # recorded in the ladder.
    result = game(team1, team2)

    if ladder is not None:
        ladder.record_result(result)

    return result

def rotate_except_first(l):
    # Rotate a list of teams excluding the first team.
    # For example,
    # 1 2 3 4 5 6 -> 1 6 2 3 4 5
    new = [l[0], l[-1]]
    for i in range(2, len(l)):
        new.append(l[i-1])
    return new

def elimination(teams):
    # Play a simple elimination fixture for the given teams.

    number_of_rounds = int(math.log(len(teams), 2))

    for n in range(number_of_rounds):
        matches = loop_matches(teams)

        # Play all matches in round
        results = [play(match[0], match[1], cricket, None) for match in matches]

        for result in results:
            store(result, "Finals " + str(n + 1))

            # Eliminate losing teams
            teams = [team for team in teams if team['Name'] != result[2]]

    return teams

def loop_matches(teams):
    # Split the list of teams into two halves and them zip them in matches.
    # For example, 1 2 3 4 5 6, or:
    # 1 2 3
    # 6 5 4
    # Would become,
    # (1, 6), (2, 5), (3, 4)
    return list(zip(teams[:len(teams)//2], reversed(teams[len(teams)//2:])))

def round_robin(teams):
    # Generate a round-robin fixture using the algorithm from
    # https://en.wikipedia.org/wiki/Round-robin_tournament

    # Add a dummy team to support byes in competitions with an uneven number
    # of teams.
    if len(teams) % 2 != 0:
        teams.append('BYE')

    number_of_rounds = len(teams) - 1
    rounds = []

    for n in range(number_of_rounds):
        matches = loop_matches(teams)
        rounds.append(matches)
        teams = rotate_except_first(teams)

    return rounds

def convert_to_int(n):
    try:
        return int(n)
    except ValueError:
        return n

def load_teams(filename):
    with open(filename) as f:
        reader = csv.DictReader(f)
        teams = [team for team in reader]

    # Convert data to integer if possible.
    for team in teams:
        for key in team:
            team[key] = convert_to_int(team[key])

    return teams

def store(result, round_no):
    # Format result into list and store in output buffer.
    row = {'Round': round_no}

    if result[0] == DRAW:
        row['Winner'] = result[1] + ', ' + result[2]
        row['Loser'] = '-'
    else:
        row['Winner'] = result[1]
        row['Loser'] = result[2]

    # Add all statistics.
    for stat, value in result[3].items():
        row[stat] = value

    output.append(row)

def output_data(filename):
    # Flush output buffer to filename.
    with open(filename, 'w') as f:
        # Add mandatory field names
        fieldnames = ['Round', 'Winner', 'Loser']

        # Add all other field names
        fieldnames.extend([field for field in output[0].keys()
                           if field not in fieldnames])

        writer = csv.DictWriter(f, fieldnames=fieldnames)

        writer.writeheader()
        writer.writerows(output)

def play_fixture(fixture, ladder, game):
    # Take a given fixture of matches (a list of lists of matches).
    # Play each round, getting the result using the specified game
    # and store the result.
    for round_number, round_matches in enumerate(fixture):
        for match in round_matches:
            result = play(match[0], match[1], game, ladder)
            store(result, round_number + 1)

def simple_simulate(teams, game, structure, finals_structure, final_n=4):
    # Simulate a league using the given teams, game, structure, finals
    # structure, and number of league winners to move on to the finals.
    fixture = structure(teams)
    ladder = Ladder(len(fixture), teams)

    play_fixture(fixture, ladder, game)

    ladder.print_ladder()

    finalist_names = [team['Name'] for team in ladder.top(final_n)]
    finalists = [team for team in teams if team['Name'] in finalist_names]

    print(finals_structure(finalists))
    output_data('out.csv')

teams = load_teams('data.csv')
simple_simulate(teams, cricket, round_robin, elimination)
<|endoftext|>


<commit_before>import gzip
import logging

from itertools import zip_longest
from typing import List

from overrides import overrides
import numpy

from sklearn.neighbors import LSHForest

from ..data.dataset import TextDataset
from ..data.instance import TextInstance, BackgroundTextInstance
from .memory_network import MemoryNetworkSolver

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class DifferentiableSearchSolver(MemoryNetworkSolver):
    """
    A DifferentiableSearchSolver is a MemoryNetworkSolver that does its own search over a corpus to
    find relevant background knowledge for a given input sentence, instead of being reliant on some
    external code (such as Lucene) to do the search for us.

    The only thing we have to change here is re-computing the background info in
    self.pre_epoch_hook(), along with adding a few command-line arguments.

    To do the search, we encode a corpus of sentences using a sentence encoder (e.g., as LSTM),
    then use nearest-neighbor search on the sentence encodings.

    We perform the nearest-neighbor search using a scikit-learn's locality sensitive hash (LSH).
    All variable names involving "lsh" in this class refer to "locality sensitive hash".

    Note that as this is currently implemented, we will take initial background sentences from a
    file, using the standard MemoryNetworkSolver code.  It is only in subsequent epochs that we
    will override that and use our differentiable search to find background knowledge.
    """
    def __init__(self, **kwargs):
        super(DifferentiableSearchSolver, self).__init__(**kwargs)
        self.corpus_path = kwargs['corpus_path']
        self.num_background = kwargs['num_background']
        self.num_epochs_delay = kwargs['num_epochs_delay']
        self.num_epochs_per_encoding = kwargs['num_epochs_per_encoding']

        self.lsh = LSHForest(random_state=12345)
        self.instance_index = {}  # type: Dict[int, str]

    @classmethod
    @overrides
    def update_arg_parser(cls, parser):
        super(DifferentiableSearchSolver, cls).update_arg_parser(parser)
        parser.add_argument('--corpus_path', type=str,
                            help="Location of corpus to use for background knowledge search. "
                            "This corpus is assumed to be gzipped, one sentence per line.")
        parser.add_argument('--num_background', type=int, default=10,
                            help="Number of background sentences to collect for each input")
        parser.add_argument('--num_epochs_delay', type=int, default=10,
                            help="Wait this many epochs before running differentiable search. "
                            "This lets you train with the base memory network code using external "
                            "background knowledge for a time, then, once the encoder is trained "
                            "sufficiently, you can turn on the differentiable search.")
        # TODO(matt): consider only re-encoding at early stopping, instead of a
        # number-of-epoch-based parameter.
        parser.add_argument('--num_epochs_per_encoding', type=int, default=2,
                            help="Number of epochs we wait in between re-encoding the corpus")

    @overrides
    def load_model(self, epoch: int=None):
        """
        Other than loading the model (which we leave to the super class), we also initialize the
        LSH, assuming that if you're loading a model you probably want access to search over the
        input corpus, either for making predictions or for finding nearest neighbors.
        """
        super(DifferentiableSearchSolver, self).load_model(epoch)
        self._initialize_lsh()

    def get_nearest_neighbors(self, instance: TextInstance) -> List[TextInstance]:
        '''
        Search in the corpus for the nearest neighbors to `instance`.  The corpus we search, how
        many neighbors to return, and the specifics of the encoder model are all defined in
        parameters passed to the constructor of this object.
        '''
        sentence_vector = self.get_sentence_vector(instance.text)
        _, nearest_neighbor_indices = self.lsh.kneighbors([sentence_vector],
                                                          n_neighbors=self.num_background)
        return [self.instance_index[neighbor_index] for neighbor_index in nearest_neighbor_indices[0]]

    @overrides
    def _pre_epoch_hook(self, epoch: int):
        """
        Here is where we re-encode the background knowledge and re-do the nearest neighbor search,
        recreating the train and validation datasets with new background data.

        We wait self.num_epochs_delay before starting this, and then we do it every
        self.num_epochs_per_encoding epochs.
        """
        super(DifferentiableSearchSolver, self)._pre_epoch_hook(epoch)
        if epoch >= self.num_epochs_delay and epoch % self.num_epochs_per_encoding == 0:
            # First we encode the corpus and (re-)build an LSH.
            self._initialize_lsh()

            # Finally, we update both self.training_dataset and self.validation_dataset with new
            # background information, taken from a nearest neighbor search over the corpus.
            logger.info("Updating the training data background")
            self.training_dataset = self._update_background_dataset(self.training_dataset)
            self.train_input, self.train_labels = self.prep_labeled_data(
                    self.training_dataset, for_train=False)
            logger.info("Updating the validation data background")
            self.validation_dataset = self._update_background_dataset(self.validation_dataset)
            self.validation_input, self.validation_labels = self._prep_question_dataset(
                    self.validation_dataset)

    def _initialize_lsh(self, batch_size=100):
        """
        This method encodes the corpus in batches, using encoder_model initialized above.  After
        the whole corpus is encoded, we pass the vectors off to sklearn's LSHForest.fit() method.
        """
        if self._sentence_encoder_model is None:
            self._build_sentence_encoder_model()
        logger.info("Reading corpus file")
        corpus_file = gzip.open(self.corpus_path)
        corpus_lines = [line.decode('utf-8') for line in corpus_file.readlines()]

        # Because we're calling as_training_data() on the instances, we need them to have a label,
        # so we pass label=True here.  TODO(matt): make it so that we can get just the input from
        # an instance without the label somehow.
        logger.info("Creating dataset")
        dataset = TextDataset.read_from_lines(corpus_lines, label=True)
        max_lengths = [self.max_sentence_length, self.max_knowledge_length]
        logger.info("Indexing and padding dataset")
        indexed_dataset = self._index_and_pad_dataset(dataset, max_lengths)

        def _get_generator():
            instances = zip(dataset.instances, indexed_dataset.instances)
            grouped_instances = zip_longest(*(iter(instances),) * batch_size)
            for batch in grouped_instances:
                batch = [x for x in batch if x is not None]
                yield batch

        generator = _get_generator()
        encoded_sentences = []
        logger.info("Encoding the background corpus")
        num_batches = len(dataset.instances) / batch_size
        log_every = max(1, int(num_batches / 100))
        batch_num = 0
        for batch in generator:
            batch_num += 1
            if batch_num % log_every == 0:
                logger.info("Processing batch %d / %d", batch_num, num_batches)
            instances, indexed_instances = zip(*batch)

            for instance in instances:
                self.instance_index[len(self.instance_index)] = instance

            encoder_input = [instance.as_training_data()[0] for instance in indexed_instances]
            encoder_input = numpy.asarray(encoder_input, dtype='int32')
            current_batch_encoded_sentences = self._sentence_encoder_model.predict(encoder_input)
            for encoded_sentence in current_batch_encoded_sentences:
                encoded_sentences.append(encoded_sentence)
        encoded_sentences = numpy.asarray(encoded_sentences)
        logger.info("Fitting the LSH")
        self.lsh.fit(encoded_sentences)

    def _update_background_dataset(self, dataset: TextDataset) -> TextDataset:
        """
        Given a dataset, this method takes all of the instance in that dataset and finds new
        background knowledge for them, returning a new dataset with updated background knowledge.
        """
        new_instances = []
        for instance in dataset.instances:  # type: BackgroundTextInstance
            text_instance = TextInstance(instance.text, label=True)
            new_background = self.get_nearest_neighbors(text_instance)
            background_text = [background.text for background in new_background]
            new_instances.append(BackgroundTextInstance(instance.text,
                                                        background_text,
                                                        instance.label,
                                                        instance.index))
        return TextDataset(new_instances)
<commit_msg>Made loading a saved LSH an option<commit_after>import gzip
import logging
import pickle

from itertools import zip_longest
from typing import List

from overrides import overrides
import numpy

from sklearn.neighbors import LSHForest

from ..data.dataset import TextDataset
from ..data.instance import TextInstance, BackgroundTextInstance
from .memory_network import MemoryNetworkSolver

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class DifferentiableSearchSolver(MemoryNetworkSolver):
    """
    A DifferentiableSearchSolver is a MemoryNetworkSolver that does its own search over a corpus to
    find relevant background knowledge for a given input sentence, instead of being reliant on some
    external code (such as Lucene) to do the search for us.

    The only thing we have to change here is re-computing the background info in
    self.pre_epoch_hook(), along with adding a few command-line arguments.

    To do the search, we encode a corpus of sentences using a sentence encoder (e.g., as LSTM),
    then use nearest-neighbor search on the sentence encodings.

    We perform the nearest-neighbor search using a scikit-learn's locality sensitive hash (LSH).
    All variable names involving "lsh" in this class refer to "locality sensitive hash".

    Note that as this is currently implemented, we will take initial background sentences from a
    file, using the standard MemoryNetworkSolver code.  It is only in subsequent epochs that we
    will override that and use our differentiable search to find background knowledge.
    """
    def __init__(self, **kwargs):
        super(DifferentiableSearchSolver, self).__init__(**kwargs)
        self.corpus_path = kwargs['corpus_path']
        self.num_background = kwargs['num_background']
        self.num_epochs_delay = kwargs['num_epochs_delay']
        self.num_epochs_per_encoding = kwargs['num_epochs_per_encoding']
        self.load_saved_lsh = kwargs['load_saved_lsh']

        self.lsh = LSHForest(random_state=12345)
        self.instance_index = {}  # type: Dict[int, str]

    @classmethod
    @overrides
    def update_arg_parser(cls, parser):
        super(DifferentiableSearchSolver, cls).update_arg_parser(parser)
        parser.add_argument('--corpus_path', type=str,
                            help="Location of corpus to use for background knowledge search. "
                            "This corpus is assumed to be gzipped, one sentence per line.")
        parser.add_argument('--load_saved_lsh', action='store_true',
                            help="Only meaningful if you are loading a model.  When loading, "
                            "should we load a pickled LSH, or should we re-initialize the LSH "
                            "from the input corpus?  Note that if you give a corpus path, and you "
                            "load a saved LSH that was constructed from a _different_ corpus, you "
                            "could end up with really weird behavior.")
        parser.add_argument('--num_background', type=int, default=10,
                            help="Number of background sentences to collect for each input")
        parser.add_argument('--num_epochs_delay', type=int, default=10,
                            help="Wait this many epochs before running differentiable search. "
                            "This lets you train with the base memory network code using external "
                            "background knowledge for a time, then, once the encoder is trained "
                            "sufficiently, you can turn on the differentiable search.")
        # TODO(matt): consider only re-encoding at early stopping, instead of a
        # number-of-epoch-based parameter.
        parser.add_argument('--num_epochs_per_encoding', type=int, default=2,
                            help="Number of epochs we wait in between re-encoding the corpus")

    @overrides
    def load_model(self, epoch: int=None):
        """
        Other than loading the model (which we leave to the super class), we also initialize the
        LSH, assuming that if you're loading a model you probably want access to search over the
        input corpus, either for making predictions or for finding nearest neighbors.

        We can either load a saved LSH, or re-initialize it with a new corpus, depending on
        self.load_saved_lsh.
        """
        super(DifferentiableSearchSolver, self).load_model(epoch)
        if self.load_saved_lsh:
            lsh_file = open("%s_lsh.pkl" % self.model_prefix, "rb")
            sentence_index_file = open("%s_index.pkl" % self.model_prefix, "rb")
            self.lsh = pickle.load(lsh_file)
            self.instance_index = pickle.load(sentence_index_file)
            lsh_file.close()
            sentence_index_file.close()
        else:
            self._initialize_lsh()

    def get_nearest_neighbors(self, instance: TextInstance) -> List[TextInstance]:
        '''
        Search in the corpus for the nearest neighbors to `instance`.  The corpus we search, how
        many neighbors to return, and the specifics of the encoder model are all defined in
        parameters passed to the constructor of this object.
        '''
        sentence_vector = self.get_sentence_vector(instance.text)
        _, nearest_neighbor_indices = self.lsh.kneighbors([sentence_vector],
                                                          n_neighbors=self.num_background)
        return [self.instance_index[neighbor_index] for neighbor_index in nearest_neighbor_indices[0]]

    @overrides
    def _pre_epoch_hook(self, epoch: int):
        """
        Here is where we re-encode the background knowledge and re-do the nearest neighbor search,
        recreating the train and validation datasets with new background data.

        We wait self.num_epochs_delay before starting this, and then we do it every
        self.num_epochs_per_encoding epochs.
        """
        super(DifferentiableSearchSolver, self)._pre_epoch_hook(epoch)
        if epoch >= self.num_epochs_delay and epoch % self.num_epochs_per_encoding == 0:
            # First we encode the corpus and (re-)build an LSH.
            self._initialize_lsh()

            # Finally, we update both self.training_dataset and self.validation_dataset with new
            # background information, taken from a nearest neighbor search over the corpus.
            logger.info("Updating the training data background")
            self.training_dataset = self._update_background_dataset(self.training_dataset)
            self.train_input, self.train_labels = self.prep_labeled_data(
                    self.training_dataset, for_train=False)
            logger.info("Updating the validation data background")
            self.validation_dataset = self._update_background_dataset(self.validation_dataset)
            self.validation_input, self.validation_labels = self._prep_question_dataset(
                    self.validation_dataset)

    @overrides
    def _save_model(self, epoch: int):
        """
        In addition to whatever superclasses do, here we need to save the LSH, so we can load it
        later if desired.
        """
        super(DifferentiableSearchSolver, self)._save_model(epoch)
        lsh_file = open("%s_lsh.pkl" % self.model_prefix, "wb")
        sentence_index_file = open("%s_index.pkl" % self.model_prefix, "wb")
        pickle.dump(self.lsh, lsh_file)
        pickle.dump(self.instance_index, sentence_index_file)
        lsh_file.close()
        sentence_index_file.close()

    def _initialize_lsh(self, batch_size=100):
        """
        This method encodes the corpus in batches, using encoder_model initialized above.  After
        the whole corpus is encoded, we pass the vectors off to sklearn's LSHForest.fit() method.
        """
        if self._sentence_encoder_model is None:
            self._build_sentence_encoder_model()
        logger.info("Reading corpus file")
        corpus_file = gzip.open(self.corpus_path)
        corpus_lines = [line.decode('utf-8') for line in corpus_file.readlines()]

        # Because we're calling as_training_data() on the instances, we need them to have a label,
        # so we pass label=True here.  TODO(matt): make it so that we can get just the input from
        # an instance without the label somehow.
        logger.info("Creating dataset")
        dataset = TextDataset.read_from_lines(corpus_lines, label=True)
        max_lengths = [self.max_sentence_length, self.max_knowledge_length]
        logger.info("Indexing and padding dataset")
        indexed_dataset = self._index_and_pad_dataset(dataset, max_lengths)

        def _get_generator():
            instances = zip(dataset.instances, indexed_dataset.instances)
            grouped_instances = zip_longest(*(iter(instances),) * batch_size)
            for batch in grouped_instances:
                batch = [x for x in batch if x is not None]
                yield batch

        generator = _get_generator()
        encoded_sentences = []
        logger.info("Encoding the background corpus")
        num_batches = len(dataset.instances) / batch_size
        log_every = max(1, int(num_batches / 100))
        batch_num = 0
        for batch in generator:
            batch_num += 1
            if batch_num % log_every == 0:
                logger.info("Processing batch %d / %d", batch_num, num_batches)
            instances, indexed_instances = zip(*batch)

            for instance in instances:
                self.instance_index[len(self.instance_index)] = instance

            encoder_input = [instance.as_training_data()[0] for instance in indexed_instances]
            encoder_input = numpy.asarray(encoder_input, dtype='int32')
            current_batch_encoded_sentences = self._sentence_encoder_model.predict(encoder_input)
            for encoded_sentence in current_batch_encoded_sentences:
                encoded_sentences.append(encoded_sentence)
        encoded_sentences = numpy.asarray(encoded_sentences)
        logger.info("Fitting the LSH")
        self.lsh.fit(encoded_sentences)

    def _update_background_dataset(self, dataset: TextDataset) -> TextDataset:
        """
        Given a dataset, this method takes all of the instance in that dataset and finds new
        background knowledge for them, returning a new dataset with updated background knowledge.
        """
        new_instances = []
        for instance in dataset.instances:  # type: BackgroundTextInstance
            text_instance = TextInstance(instance.text, label=True)
            new_background = self.get_nearest_neighbors(text_instance)
            background_text = [background.text for background in new_background]
            new_instances.append(BackgroundTextInstance(instance.text,
                                                        background_text,
                                                        instance.label,
                                                        instance.index))
        return TextDataset(new_instances)
<|endoftext|>


<commit_before>import inoreaderapi
import delugeapi

def go_process():
    items = inoreaderapi.get_downloadable_items()
    if len(items['items']) == 0:
        print "No items to retreive. Exiting."
    else:
        print "Retreived " + str(len(items['items'])) + " starred items"
        client = delugeapi.connect_to_deluge()
        for item in items['items']:
            print "Saving -> '" + item['title'] + "'"
            tor_id = client.add_torr_url(inoreaderapi.get_enclosure_url(item))
            if tor_id: #URL added, otherwise it's probably already in deluge!
                label = item['origin']['title']

                if not client.label_exist(label):
                    client.add_label(label, {'move_completed_path': inoreaderapi.generate_save_path(item)})

                client.add_tor_label(tor_id, label )

        inoreaderapi.toggle_labels([i['id'] for i in items['items']])

if __name__ == "__main__":
    go_process()<commit_msg>Added exception handeling to remove the added items from inoreader, even if exception occurs in the middle.<commit_after>import inoreaderapi
import delugeapi

def go_process():
    items = inoreaderapi.get_downloadable_items()
    if len(items['items']) == 0:
        print "No items to retreive. Exiting."
    else:
        print "Retreived " + str(len(items['items'])) + " starred items"
        client = delugeapi.connect_to_deluge()
        for item in items['items']:
            try:
                print "Saving -> '" + item['title'] + "'"
                tor_id = client.add_torr_url(inoreaderapi.get_enclosure_url(item))
                if tor_id: #URL added, otherwise it's probably already in deluge!
                    label = item['origin']['title']

                    if not client.label_exist(label):
                        client.add_label(label, {'move_completed_path': inoreaderapi.generate_save_path(item)})

                    client.add_tor_label(tor_id, label )
                item['saved'] = True
            except:
                print 'Failed on processing: ' + item['title']
                raise
                #print 'Debug item content: '
                #print item
            finally:
                inoreaderapi.toggle_labels([i['id'] for i in items['items'] if i.has_key('saved')])

if __name__ == "__main__":
    go_process()<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Soft Bill is a virtual, RS-232 bill validator
@auth@or: me@corytodd.us
@version: 0.3

TODO: Check ACK number, resend last message if required
"""
import acceptor, autopilot
import sys

### Main  Routine ###
def main(portname):
    """
    Application to simulate hardware bill validator

    Args:
        portname -- string portname e.g. COM2, /dev/tty.*
    """

    slave = acceptor.Acceptor()

    cmd_table = '''

    H or ? to show Help
    Q or CTRL+C to Quit

    Bill position to simulate bill insertions:
    1 - $1   or 1st note
    2 - $2   or 2nd note
    3 - $5   or 3rd note
    4 - $10  or 4th note
    5 - $20  or 5th note
    6 - $50  or 6th note
    7 - $100 or 7th note

    Note:
    Software automatically changes states once mock bill insertion begins
    Idling->Accepting->Escrowed->{Stacking,Returning}->{Stacked,Returned}

    Toggle Events:
    C - Cheated
    R - Rejected (We think note is invalid)
    J - Jammed
    F - Stacker Full
    P - LRC present (cashbox: set to 1 means it's there)

    Extra Stuff:
    W - Powering up
    I - Invalid Command was received
    X - Failure (This BA has failed)
    Y - Empty Cashbox
    A - AutoPilot

    Bill Disable and Enables:
    Dx - where x is the index to disable (e.g. D1 disables $1)
    Ex - where x in the index to enable  (e.g. E1 enables $1)

    '''

    print "Starting software BA on port {:s}".format(portname)
    slave.start(portname)

    # Acquire a pilot
    pilot = autopilot.AutoPilot(slave)

    # Loop until we are to exit
    try:
        print cmd_table
        while slave.running:

            cmd = raw_input()
            result = slave.parse_cmd(cmd)
            if result is 0:
                pass
            elif result is 1:
                slave.stop()
            elif result is 2:
                print cmd_table
            elif result is 3:
                if not pilot.running:
                    pilot.start()
                else:
                    pilot.stop()

    except KeyboardInterrupt:
        slave.running = False

    print '\n\nGoodbye!'
    print 'Port {:s} closed'.format(portname)

if __name__ == "__main__":
    main(sys.argv[1])
<commit_msg>auto pilot feature added released as 0.4<commit_after>#!/usr/bin/env python
"""
Soft Bill is a virtual, RS-232 bill validator
@auth@or: me@corytodd.us
@version: 0.4

TODO: Check ACK number, resend last message if required
"""
import acceptor, autopilot
import sys

### Main  Routine ###
def main(portname):
    """
    Application to simulate hardware bill validator

    Args:
        portname -- string portname e.g. COM2, /dev/tty.*
    """

    slave = acceptor.Acceptor()

    cmd_table = '''

    H or ? to show Help
    Q or CTRL+C to Quit

    Bill position to simulate bill insertions:
    1 - $1   or 1st note
    2 - $2   or 2nd note
    3 - $5   or 3rd note
    4 - $10  or 4th note
    5 - $20  or 5th note
    6 - $50  or 6th note
    7 - $100 or 7th note

    Note:
    Software automatically changes states once mock bill insertion begins
    Idling->Accepting->Escrowed->{Stacking,Returning}->{Stacked,Returned}

    Toggle Events:
    C - Cheated
    R - Rejected (We think note is invalid)
    J - Jammed
    F - Stacker Full
    P - LRC present (cashbox: set to 1 means it's there)

    Extra Stuff:
    W - Powering up
    I - Invalid Command was received
    X - Failure (This BA has failed)
    Y - Empty Cashbox
    A - AutoPilot

    Bill Disable and Enables:
    Dx - where x is the index to disable (e.g. D1 disables $1)
    Ex - where x in the index to enable  (e.g. E1 enables $1)

    '''

    print "Starting software BA on port {:s}".format(portname)
    slave.start(portname)

    # Acquire a pilot
    pilot = autopilot.AutoPilot(slave)

    # Loop until we are to exit
    try:
        print cmd_table
        while slave.running:

            cmd = raw_input()
            result = slave.parse_cmd(cmd)
            if result is 0:
                pass
            elif result is 1:
                slave.stop()
            elif result is 2:
                print cmd_table
            elif result is 3:
                if not pilot.running:
                    pilot.start()
                else:
                    pilot.stop()

    except KeyboardInterrupt:
        slave.running = False

    print '\n\nGoodbye!'
    print 'Port {:s} closed'.format(portname)

if __name__ == "__main__":
    main(sys.argv[1])
<|endoftext|>