<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Author: Daniel Roschka <daniel@smaato.com>
Copyright: Smaato Inc. 2014
URL: https://github.com/smaato/ebs-snapshot-automation

A small script meant to be run as a cronjob to regularly snapshot EBS volumes attached to EC2
instances and handle rolling backups for these snapshots.

Instances to backup are selected by a configured tag, so the instances to backup can be easily
selected by adding or removing the appropriate tag to the EC2 instance, without any configuration
change needed for this script.
'''

import argparse
from boto.ec2 import connect_to_region
from boto.exception import BotoServerError
from datetime import datetime
from itertools import chain
import logging
import sys

def make_snapshots(connection, tag):
    '''
    Creates a snapshot for every EBS volume attached to an instance which contains the tag used
    to filter for instances to backup.
    '''
    tag_key_value = tag.split(':')
    if len(tag_key_value) != 2:
        logging.error("Given tag key value: \"%s\" is invalid.", tag)
        sys.exit(1)

    tag_filter = {'tag:%s' % tag_key_value[0] : '%s' % tag_key_value[1]}
    reservations = connection.get_all_reservations(filters=tag_filter)
    instances = list(chain.from_iterable([x.instances for x in reservations]))
    if not instances:
        logging.warning("Couldn't find any instances whose volumes need snapshotting. Aborting …")
        sys.exit(0)

    # filter the volumes for the ones attached to the tagged instances
    volumes = [v for v in connection.get_all_volumes() if v.status == 'in-use' and
	                    v.attach_data.instance_id in [i.id for i in instances]]
    if not volumes:
        logging.warning("Found instances to backup, but no attached volumes. Something is fishy \
                         here. Aborting …")
        sys.exit(1)

    for instance in instances:
        volumes_tmp = [v for v in volumes if v.attach_data.instance_id == instance.id]
        for volume in volumes_tmp:
            try:
                if 'Name' in instance.tags:
                    snapshot = volume.create_snapshot(description='automated snapshot of volume \
                                                      %s attached as %s to %s (%s)' %
                                                      (volume.id,
                                                       volume.attach_data.device,
                                                       instance.tags['Name'],
                                                       volume.attach_data.instance_id))
                else:
                    snapshot = volume.create_snapshot(description='automated snapshot of volume \
                                                      %s attached as %s to %s' %
                                                      (volume.id,
                                                       volume.attach_data.device,
                                                       volume.attach_data.instance_id))
            except BotoServerError as exc:
                logging.error("Creating a snapshot of volume %s failed:\n%s", volume.id, exc)
            else:
                logging.info("Creating snapshot %s of volume %s", snapshot.id, volume.id)
            try:
                if 'Name' in instance.tags:
                    snapshot.add_tag('Name', '%s %s %s' % (instance.tags['Name'],
                                     volume.attach_data.device,
                                     datetime.now().strftime('%Y-%m-%d %H:%M')))
                else:
                    snapshot.add_tag('Name', '%s %s %s' % (volume.attach_data.instance_id,
                                     volume.attach_data.device,
                                     datetime.now().strftime('%Y-%m-%d %H:%M')))
                snapshot.add_tag('Creator', 'ebs_snapshot_automation')
                snapshot.add_tag('Origin-Instance', volume.attach_data.instance_id)
            except BotoServerError as exc:
                logging.error("Tagging the snapshot %s of volume %s failed:\n%s",
                               snapshot.id,
                               volume.id,
                               exc)

def delete_old_snapshots(connection, num_backups):
    '''
    Removes the oldest snapshots for a volume if there are more snapshots than configured in
    num_backups
    '''
    try:
        snapshots = connection.get_all_snapshots(filters={'tag:Creator' : 'ebs_snapshot_automation'})
    except BotoServerError as exc:
         logging.error("Getting all previously created snapshots failed:\n%s", exc)
         sys.exit(1)

    for volume_id in list(set([snapshot.volume_id for snapshot in snapshots])):
        snapshots_for_volume = [s for s in snapshots if s.volume_id == volume_id]
        snapshots_for_volume = sorted(snapshots_for_volume, key=lambda x: x.start_time)

        while len(snapshots_for_volume) > num_backups:
            try:
                snapshot = snapshots_for_volume.pop(0)
                snapshot.delete()
            except BotoServerError as exc:
                logging.error("Deleting snapshot %s (%s) belonging to volume %s failed:\n%s",
                             snapshot.id,
                             snapshot.start_time,
                             snapshot.volume_id,
                             exc)
            else:
                logging.info("Deleting snapshot %s (%s) belonging to volume %s",
                             snapshot.id,
                             snapshot.start_time,
                             snapshot.volume_id)

def main():
    '''
    Setup the commandline parsing, the AWS connection and call the functions containing the actual
    logic.
    '''
    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)

    parser = argparse.ArgumentParser(description='Script to automate \
                                     snapshotting of EBS volumes')
    parser.add_argument('-a', '--access-key', dest='aws_access_key',
                        required=True, help='The AWS access key to use')
    parser.add_argument('-s', '--secret-key', dest='aws_secret_key',
                        required=True, help='The AWS secret key to use')
    parser.add_argument('-r', '--region', dest='region', default='us-east-1',
                        help='The AWS region to connect to')
    parser.add_argument('-n', '--num-backups', dest='num_backups', type=int, default=14,
                        help='The number of backups for each volume to keep')
    parser.add_argument('-t', '--tag', dest='tag', default='Lifecycle:legacy',
                        help='Key and value (separated by a colon) of a tag attached to instances \
                              whose EBS volumes should be backed up')
    args = parser.parse_args()

    try:
        connection = connect_to_region(args.region,
                                       aws_access_key_id=args.aws_access_key,
                                       aws_secret_access_key=args.aws_secret_key)
    except BotoServerError as exc:
        logging.error("Establishing a connection to AWS failed:\n%s", exc)

    make_snapshots(connection, args.tag)
    delete_old_snapshots(connection, args.num_backups)

if __name__ == "__main__":
    main()
<commit_msg>Fixes some python styling issues.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Author: Daniel Roschka <daniel@smaato.com>
Copyright: Smaato Inc. 2014-2015
URL: https://github.com/smaato/ebs-snapshot-automation

A small script meant to be run as a cronjob to regularly snapshot EBS volumes
attached to EC2 instances and handle rolling backups for these snapshots.

Instances to backup are selected by a configured tag, so the instances to
backup can be easily selected by adding or removing the appropriate tag to the
EC2 instance, without any configuration change needed for this script.
"""

import argparse
from boto.ec2 import connect_to_region
from boto.exception import BotoServerError
from datetime import datetime
from itertools import chain
import logging
import sys

def make_snapshots(connection, tag):
    """Create snapshots

    Create a snapshot for every EBS volume attached to an instance which
    contains the tag used to filter for instances to backup.
    """
    tag_key_value = tag.split(':')
    if len(tag_key_value) != 2:
        logging.error("Given tag key value: \"%s\" is invalid.", tag)
        sys.exit(1)

    tag_filter = {'tag:%s' % tag_key_value[0]: '%s' % tag_key_value[1]}
    reservations = connection.get_all_reservations(filters=tag_filter)
    instances = list(chain.from_iterable([x.instances for x in reservations]))
    if not instances:
        logging.warning("Couldn't find any instances whose volumes need snapshotting. Aborting …")
        sys.exit(0)

    # filter the volumes for the ones attached to the tagged instances
    volumes = [v for v in connection.get_all_volumes() if v.status == 'in-use' and
               v.attach_data.instance_id in [i.id for i in instances]]
    if not volumes:
        logging.warning("Found instances to backup, but no attached volumes. Something is fishy \
                         here. Aborting …")
        sys.exit(1)

    for instance in instances:
        volumes_tmp = [v for v in volumes if v.attach_data.instance_id == instance.id]
        for volume in volumes_tmp:
            try:
                if 'Name' in instance.tags:
                    snapshot = volume.create_snapshot(description='automated snapshot of volume \
                                                      %s attached as %s to %s (%s)' %
                                                      (volume.id,
                                                       volume.attach_data.device,
                                                       instance.tags['Name'],
                                                       volume.attach_data.instance_id))
                else:
                    snapshot = volume.create_snapshot(description='automated snapshot of volume \
                                                      %s attached as %s to %s' %
                                                      (volume.id,
                                                       volume.attach_data.device,
                                                       volume.attach_data.instance_id))
            except BotoServerError as exc:
                logging.error("Creating a snapshot of volume %s failed:\n%s",
                              volume.id,
                              exc)
            else:
                logging.info("Creating snapshot %s of volume %s",
                             snapshot.id,
                             volume.id)
            try:
                snapshot_date = datetime.now().strftime('%Y-%m-%d %H:%M')
                if 'Name' in instance.tags:
                    snapshot.add_tag('Name', '%s %s %s' % (instance.tags['Name'],
                                                           volume.attach_data.device,
                                                           snapshot_date))
                else:
                    snapshot.add_tag('Name', '%s %s %s' % (volume.attach_data.instance_id,
                                                           volume.attach_data.device,
                                                           snapshot_date))
                snapshot.add_tag('Creator', 'ebs_snapshot_automation')
                snapshot.add_tag('Origin-Instance',
                                 volume.attach_data.instance_id)
            except BotoServerError as exc:
                logging.error("Tagging the snapshot %s of volume %s failed:\n%s",
                              snapshot.id,
                              volume.id,
                              exc)

def delete_old_snapshots(connection, num_backups):
    """Remove old snapshots

    Remove the oldest snapshots for a volume if there are more snapshots than
    configured in num_backups
    """
    try:
        snapshots = connection.get_all_snapshots(filters={'tag:Creator': 'ebs_snapshot_automation'})
    except BotoServerError as exc:
        logging.error("Getting all previously created snapshots failed:\n%s", exc)
        sys.exit(1)

    for volume_id in list(set([snapshot.volume_id for snapshot in snapshots])):
        snapshots_for_volume = [s for s in snapshots if s.volume_id == volume_id]
        snapshots_for_volume = sorted(snapshots_for_volume, key=lambda x: x.start_time)

        while len(snapshots_for_volume) > num_backups:
            try:
                snapshot = snapshots_for_volume.pop(0)
                snapshot.delete()
            except BotoServerError as exc:
                logging.error("Deleting snapshot %s (%s) belonging to volume %s failed:\n%s",
                              snapshot.id,
                              snapshot.start_time,
                              snapshot.volume_id,
                              exc)
            else:
                logging.info("Deleting snapshot %s (%s) belonging to volume %s",
                             snapshot.id,
                             snapshot.start_time,
                             snapshot.volume_id)

def main():
    """
    Setup command line parsing, the AWS connection and call the functions
    containing the actual logic.
    """
    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                        level=logging.INFO)

    parser = argparse.ArgumentParser(description='Script to automate \
                                     snapshotting of EBS volumes')
    parser.add_argument('-a', '--access-key', dest='aws_access_key',
                        required=True, help='The AWS access key to use')
    parser.add_argument('-s', '--secret-key', dest='aws_secret_key',
                        required=True, help='The AWS secret key to use')
    parser.add_argument('-r', '--region', dest='region', default='us-east-1',
                        help='The AWS region to connect to')
    parser.add_argument('-n', '--num-backups', dest='num_backups', type=int,
                        default=14,
                        help='The number of backups for each volume to keep')
    parser.add_argument('-t', '--tag', dest='tag', default='Lifecycle:legacy',
                        help='Key and value (separated by a colon) of a tag '
                        'attached to instances whose EBS volumes should be '
                        'backed up')
    args = parser.parse_args()

    try:
        connection = connect_to_region(args.region,
                                       aws_access_key_id=args.aws_access_key,
                                       aws_secret_access_key=args.aws_secret_key)
    except BotoServerError as exc:
        logging.error("Establishing a connection to AWS failed:\n%s", exc)

    make_snapshots(connection, args.tag)
    delete_old_snapshots(connection, args.num_backups)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
edbo_connector
Author: Eldar Aliiev
Email: e.aliiev@vnmu.edu.ua
"""

from .client import EDBOWebApiClient

__name__ = 'python-edbo-connector'
__author__ = 'Eldar Aliiev'
__copyright__ = 'Copyright 2017, National Pirogov Memorial Medical University, Vinnytsya'
__credits__ = ['Eldar Aliiev']
__license__ = 'MIT'
__version__ = '1.0.0'
__maintainer__ = 'Eldar Aliiev'
__email__ = 'e.aliiev@vnmu.edu.ua'
__status__ = 'Production'
__all__ = ['EDBOWebApiClient']
<commit_msg>Some release improvement<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
edbo_connector
Author: Eldar Aliiev
Email: e.aliiev@vnmu.edu.ua
"""

from .client import EDBOWebApiClient

__name__ = 'python-edbo-connector'
__author__ = 'Eldar Aliiev'
__copyright__ = 'Copyright 2017, National Pirogov Memorial Medical University, Vinnytsya'
__credits__ = ['Eldar Aliiev']
__license__ = 'MIT'
__version__ = '1.0.1'
__maintainer__ = 'Eldar Aliiev'
__email__ = 'e.aliiev@vnmu.edu.ua'
__status__ = 'Production'
__all__ = ['EDBOWebApiClient']
<|endoftext|>


<commit_before>import os, sys
from netCDF4 import Dataset
from numpy.linalg import norm
import numpy

numpy.set_printoptions(threshold='nan')

def main(netcdf_filename):

    rootgrp = Dataset(netcdf_filename, "r", format="NETCDF4")

    # lons = rootgrp.variables['lon'][:]
    # lats = rootgrp.variables['lat'][:]
    time = rootgrp.variables['time']

    yield_whe_frame_prev = None
    yield_whe_frame_next = None
    for t in range(time.size):
        yield_whe_frame_prev = yield_whe_frame_next
        yield_whe_frame_next = rootgrp.variables['yield_whe'][t]
        if yield_whe_frame_prev is not None and yield_whe_frame_next is not None:
            print yield_whe_frame_next
            print "Euclidean difference: %f" % norm(yield_whe_frame_next-yield_whe_frame_next)

    # print "number of longitudes: %d" % lons.size
    # print "number of latitudes: %d" % lats.size
    # print "number of yield_whe's: %d" % yield_whe.size

    rootgrp.close()

if __name__ == "__main__":
    netcdf_filename = "/var/www/ede_unstacked/ede/data/atlas/papsim_wfdei.cru_hist_harmnon_noirr_yield_whe_annual_1979_2012.nc4"
    main(netcdf_filename)
<commit_msg>analyze data script<commit_after>import os, sys
from netCDF4 import Dataset
from numpy.linalg import norm
import numpy

numpy.set_printoptions(threshold='nan')

def main(netcdf_filename):

    rootgrp = Dataset(netcdf_filename, "r", format="NETCDF4")

    lons = rootgrp.variables['lon']
    lats = rootgrp.variables['lat']
    yield_whe = rootgrp.variables['yield_whe']
    for lat in lats.size:
        for lon in lons.size:
            print yield_whe[:, lat, lon]

    # time = rootgrp.variables['time']
    # yield_whe_frame_prev = None
    # yield_whe_frame_next = None
    # for t in range(time.size):
    #     yield_whe_frame_prev = yield_whe_frame_next
    #     yield_whe_frame_next = rootgrp.variables['yield_whe'][t]
    #     if yield_whe_frame_prev is not None and yield_whe_frame_next is not None:
    #         print yield_whe_frame_next
    #         print "Euclidean difference: %f" % norm(yield_whe_frame_next-yield_whe_frame_next)

    rootgrp.close()

if __name__ == "__main__":
    netcdf_filename = "/var/www/ede_unstacked/ede/data/atlas/papsim_wfdei.cru_hist_harmnon_noirr_yield_whe_annual_1979_2012.nc4"
    main(netcdf_filename)
<|endoftext|>


<commit_before>"""
The exam proctoring subsystem for the Open edX platform.
"""

from __future__ import absolute_import

__version__ = '1.3.3'

default_app_config = 'edx_proctoring.apps.EdxProctoringConfig'  # pylint: disable=invalid-name
<commit_msg>Bump version to 1.3.4<commit_after>"""
The exam proctoring subsystem for the Open edX platform.
"""

from __future__ import absolute_import

__version__ = '1.3.4'

default_app_config = 'edx_proctoring.apps.EdxProctoringConfig'  # pylint: disable=invalid-name
<|endoftext|>


<commit_before>"""
    This file contains filters which are used for pre and post
    processing various kinds of text within LifeFlow.

    Which values are applied is controlled by a number of global
    variables within the project's settings.py file. These vars
    are:

    LIFEFLOW_ENTRY_FILTERS
    LIFEFLOW_COMMENT_FILTERS

    If you wish to add your own filters, you don't
    have to add them to this file, they can exist anywhere, and
    simply import them into the settings.py file and add them
    to the appropriate global variable.

    The API for these processing functions is very simple:
    they accept two parameters, a string to process,
    and optionally a related model.
"""

import cgi, re
from django.conf import settings
from lifeflow.markdown.markdown import Markdown
from lifeflow.markdown import mdx_lifeflow
from lifeflow.markdown import mdx_code
from lifeflow.markdown import mdx_footnotes
from lifeflow.markdown import mdx_foreign_formats

def comment_markup(txt,obj=None):
    filters = getattr(settings,'LIFEFLOW_COMMENT_FILTERS', DEFAULT_COMMENT_FILTERS)
    for filter in filters:
        txt = filter(txt)
    return txt

def entry_markup(txt,obj=None):
    filters = getattr(settings,'LIFEFLOW_ENTRY_FILTERS', DEFAULT_ENTRY_FILTERS)
    for filter in filters:
        txt = filter(txt)
    return txt

def comment_markdown(txt,obj=None):
    exts = (mdx_code,)
    md = Markdown(txt,extensions=exts,safe_mode=True)
    return md.convert()

def entry_markdown(txt,obj=None):
    exts = (mdx_code, mdx_footnotes,mdx_foreign_formats, mdx_lifeflow)
    md = Markdown(txt,extensions=exts,extension_configs={'lifeflow':obj})
    return md.convert()

DEFAULT_COMMENT_FILTERS = (comment_markdown,)
DEFAULT_ENTRY_FILTERS = (entry_markdown,)
<commit_msg>Removed unused cgi and re imports.<commit_after>"""
    This file contains filters which are used for pre and post
    processing various kinds of text within LifeFlow.

    Which values are applied is controlled by a number of global
    variables within the project's settings.py file. These vars
    are:

    LIFEFLOW_ENTRY_FILTERS
    LIFEFLOW_COMMENT_FILTERS

    If you wish to add your own filters, you don't
    have to add them to this file, they can exist anywhere, and
    simply import them into the settings.py file and add them
    to the appropriate global variable.

    The API for these processing functions is very simple:
    they accept two parameters, a string to process,
    and optionally a related model.
"""

from django.conf import settings
from lifeflow.markdown.markdown import Markdown
from lifeflow.markdown import mdx_lifeflow
from lifeflow.markdown import mdx_code
from lifeflow.markdown import mdx_footnotes
from lifeflow.markdown import mdx_foreign_formats

def comment_markup(txt,obj=None):
    filters = getattr(settings,'LIFEFLOW_COMMENT_FILTERS', DEFAULT_COMMENT_FILTERS)
    for filter in filters:
        txt = filter(txt)
    return txt

def entry_markup(txt,obj=None):
    filters = getattr(settings,'LIFEFLOW_ENTRY_FILTERS', DEFAULT_ENTRY_FILTERS)
    for filter in filters:
        txt = filter(txt)
    return txt

def comment_markdown(txt,obj=None):
    exts = (mdx_code,)
    md = Markdown(txt,extensions=exts,safe_mode=True)
    return md.convert()

def entry_markdown(txt,obj=None):
    exts = (mdx_code, mdx_footnotes,mdx_foreign_formats, mdx_lifeflow)
    md = Markdown(txt,extensions=exts,extension_configs={'lifeflow':obj})
    return md.convert()

DEFAULT_COMMENT_FILTERS = (comment_markdown,)
DEFAULT_ENTRY_FILTERS = (entry_markdown,)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os, sys, time, math

import pymongo
import tensorflow as tf
from tensorflow.contrib.learn import NanLossDuringTrainingError
import gridfs

class Saver(tf.train.Saver):

    def __init__(self,
                 sess,
                 host='localhost',
                 port=31001,
                 dbname=None,
                 collname='test',
                 exp_id=None,
                 restore_vars=False,
                 restore_var_file='',
                 save_filters_to_db=False,
                 start_step=None,
                 save_vars=True,
                 save_vars_freq=3000,
                 save_path='',
                 save_loss=True,
                 save_loss_freq=5,
                 tensorboard=False,
                 tensorboard_dir='',
                 *args, **kwargs):
        """
        Output printer, logger and saver to a database

        NOTE: Not working yet
        """
        super(Saver, self).__init__(*args, **kwargs)
        if dbname is not None:
            self.conn = pymongo.MongoClient(host=host, port=port)
            self.coll = self.conn[dbname][collname + '.files']
            self.collfs = gridfs.GridFS(self.conn[dbname], collname)
            self.collfs_recent = gridfs.GridFS(self.conn[dbname + '__RECENT'],collname)
            self.save_filters_to_db = save_filters_to_db
        else:
            # TODO: save locally
            raise ValueError('Please specify database name for storing data')
        self.sess = sess
        self.exp_id = exp_id
        self.save_vars = save_vars
        self.save_vars_freq = save_vars_freq
        self.save_path = save_path
        self.save_loss = save_loss
        self.save_loss_freq = save_loss_freq
        if restore_vars:
            self.restore_model(restore_var_file)
            print('Variables restored')

        if tensorboard:  # save graph to tensorboard
            tf.train.SummaryWriter(tensorboard_dir, tf.get_default_graph())

        self.start_time_step = time.time()  # start timer

    def restore_model(self):
        """
        Fetches record, saves locally, then uses tf's saver.restore
        """
        # fetch record from database and get the filename info from record
        rec = self.load_from_db(self.exp_id)
        # should be of form *-1000 (step)
        filename_suffix = rec.filename.split('/')[-1]
        loaded_filename = os.path.join(self.save_path, 'retrieved_' + filename_suffix)

        # create new file to write from gridfs
        load_dest = open(loaded_filename, "w+")
        load_dest.close()
        load_dest = open(loaded_filename, 'rwb+')
        fsbucket = gridfs.GridFSBucket(rec._GridOut__files.database,
                            bucket_name=rec._GridOut__files.name.split('.')[0])
        # save to local disk
        fsbucket.download_to_stream(rec._id, load_dest)

        # tf restore
        self.restore(self.sess, loaded_filename)
        print('Model variables restored.')

    def load_from_db(self, query):
        """
        Loads checkpoint from the database

        Checks the recent and regular checkpoint fs to find the latest one
        matching the query. Returns the GridOut obj corresponding to the
        record.

        Args:
            query: dict of Mongo queries
        """
        count = self.collfs.find(query).count()
        if count > 0:  # get latest that matches query
            ckpt_record = self.collfs._GridFS__files.find(query,
                            sort=[('uploadDate', -1)])[0]
            loading_from = 'long-term storage'

        count_recent = self.collfs_recents.find(query).count()
        if count_recent > 0:  # get latest that matches query
            ckpt_record_rec = self.collfs_recent._GridFS__files.find(query,
                                sort=[('uploadDate', -1)])[0]
            # use the record with latest timestamp
            if ckpt_record is None or ckpt_record_rec['uploadDate'] > ckpt_record['uploadDate']:
                loading_from = 'recent storage'
                ckpt_record = ckpt_record_rec

        if count + count_recent == 0:  # no matches for query
            raise Exception('No matching checkpoint for query "{}"'.format(repr(query)))

        print('Loading checkpoint from ', loading_from)
        return ckpt_record

    def save(self, step, results):
        elapsed_time_step = time.time() - self.start_time_step
        self.start_time_step = time.time()

        if math.isnan(results['loss']):
            raise NanLossDuringTrainingError

        rec = {'exp_id': self.exp_id,
            #    'cfg': preprocess_config(cfg),
            #    'saved_filters': saved_filters,
                'kind': 'train',
                'step': step,
                'loss': float(results['loss']),
                'learning_rate': float(results['lr']),
                'duration': int(1000 * elapsed_time_step)}

        if step > 0:
            # write loss to db
            if self.save_loss and step % self.save_loss_freq == 0:
                self.coll.insert_one(rec)

        if self.save_vars and step % self.save_vars_freq == 0 and step > 0:
            saved_path = super(Saver, self).save(self.sess,
                                    save_path=self.save_path,
                                    global_step=step,
                                    write_meta_graph=False)
            # save the saved file to 'recent' checkpoint fs
            if self.save_filters_to_db:
                self.collfs_recent.put(open(saved_path, 'rb'),
                                       filename=saved_path,
                                       saved_filters=True,
                                       **rec)
                # TODO: when do we move from recent to non recent...
                print('Saved variable checkpoint to recent fs.')

        print('Step {} -- loss: {:.6f}, lr: {:.6f}, time: {:.0f}'
              'ms'.format(rec['step'], rec['loss'], rec['learning_rate'], rec['duration']))
        sys.stdout.flush()  # flush the stdout buffer

    def valid(self, step, results):
        elapsed_time_step = time.time() - self.start_time_step
        self.start_time_step = time.time()
        rec = {'exp_id': self.exp_id,
            #    'cfg': preprocess_config(cfg),
            #    'saved_filters': saved_filters,
               'kind': 'validation',
               'step': step,
               'duration': 1000 * elapsed_time_step}
        rec.update(results)
        if step > 0:
            # write loss to file
            if self.save_valid and step % self.save_valid_freq == 0:
                pass
                # self.coll.insert(rec)

        message = ('Step {} validation -- ' +
                   '{}: {:.3f}, ' * len(results) +
                   '{:.0f} ms')
        args = []
        for k, v in results.items():
            args.extend([k,v])
        args = [rec['step']] + args + [rec['duration']]
        print(message.format(*args))

    def predict(self, step, results):
        if not hasattr(results['output'], '__iter__'):
            outputs = [results['outputs']]
        else:
            outputs = results['outputs']

        preds = [tf.argmax(output, 1) for output in outputs]

        return preds

    def test(self, step, results):
        raise NotImplementedError

def run_loop(sess, queues, saver, train_targets, valid_targets=None,
        start_step=0, end_step=None):
    """
    Args:
        - queues (~ data)
        - saver
        - targets
    """
    # initialize and/or restore variables for graph
    init = tf.initialize_all_variables()
    sess.run(init)
    print('variables initialized')

    tf.train.start_queue_runners(sess=sess)
    # start our custom queue runner's threads
    if not hasattr(queues, '__iter__'):
        queues = [queues]
    for queue in queues:
        queue.start_threads(sess)

    # start_time_step = time.time()  # start timer
    print('start training')
    for step in xrange(start_step, end_step):
        # get run output as dictionary {'2': loss2, 'lr': lr, etc..}
        results = sess.run(train_targets)
        # print output, save variables to checkpoint and save loss etc
        saver.save(step, results)
    sess.close()

def run(model_func,
        model_func_kwargs,
        data_func,
        data_func_kwargs,
        loss_func,
        loss_func_kwargs,
        lr_func,
        lr_func_kwargs,
        opt_func,
        opt_func_kwargs,
        saver_kwargs,
        train_targets=None,
        valid_targets=None,
        seed=None,
        start_step=,
        end_step=math.inf,
        log_device_placement=True
        ):
    with tf.Graph().as_default():  # to have multiple graphs [ex: eval, train]
        rng = np.random.RandomState(seed=seed)
        tf.set_random_seed(seed)

        tf.get_variable('global_step', [],
                        initializer=tf.constant_initializer(0),
                        trainable=False)

        train_data_node, train_data_provider = data_func(train=True,
                                                     **data_func_kwargs)
        valid_data_node, valid_data_provider = data_func(train=False,
                                                     **data_func_kwargs)

        outputs = model_func(train_data_node['data'], **model_func_kwargs)
        loss = loss_func(outputs, train_data_node['labels'],
                         **loss_func_kwargs)
        lr = lr_func(**lr_func_kwargs)
        optimizer = opt_func(loss, lr, **opt_func_kwargs)

        if train_targets is None:
            pass
        elif isinstance(train_targets, dict):
            train_targets.update({'loss': loss, 'lr': lr, 'opt': optimizer})
        else:
            raise ValueError('Train targets must be None or dict, got {}'.format(type(train_targets)))

        # create session
        sess = tf.Session(config=tf.ConfigProto(
                                allow_soft_placement=True,
                                log_device_placement=log_device_placement))

        saver = base.Saver(sess, **saver_kwargs)
        run_loop(sess,
            [train_data_provider, valid_data_provider],
            saver,
            train_targets=train_targets,
            valid_targets=valid_targets,
            start_step=start_step,
            end_step=end_step)

<commit_msg>generic run (draft2)<commit_after>from __future__ import absolute_import, division, print_function

import os, sys, time, math

import numpy as np
import pymongo
import tensorflow as tf
from tensorflow.contrib.learn import NanLossDuringTrainingError
import gridfs

class Saver(tf.train.Saver):

    def __init__(self,
                 sess,
                 host='localhost',
                 port=31001,
                 dbname=None,
                 collname='test',
                 exp_id=None,
                 restore_vars=False,
                 restore_var_file='',
                 save_filters_to_db=False,
                 start_step=None,
                 save_vars=True,
                 save_vars_freq=3000,
                 save_path='',
                 save_loss=True,
                 save_loss_freq=5,
                 tensorboard=False,
                 tensorboard_dir='',
                 *args, **kwargs):
        """
        Output printer, logger and saver to a database

        NOTE: Not working yet
        """
        super(Saver, self).__init__(*args, **kwargs)
        if dbname is not None:
            self.conn = pymongo.MongoClient(host=host, port=port)
            self.coll = self.conn[dbname][collname + '.files']
            self.collfs = gridfs.GridFS(self.conn[dbname], collname)
            self.collfs_recent = gridfs.GridFS(self.conn[dbname + '__RECENT'],collname)
            self.save_filters_to_db = save_filters_to_db
        else:
            # TODO: save locally
            raise ValueError('Please specify database name for storing data')
        self.sess = sess
        self.exp_id = exp_id
        self.save_vars = save_vars
        self.save_vars_freq = save_vars_freq
        self.save_path = save_path
        self.save_loss = save_loss
        self.save_loss_freq = save_loss_freq
        if restore_vars:
            self.restore_model(restore_var_file)
            print('Variables restored')

        if tensorboard:  # save graph to tensorboard
            tf.train.SummaryWriter(tensorboard_dir, tf.get_default_graph())

        self.start_time_step = time.time()  # start timer

    def restore_model(self):
        """
        Fetches record, saves locally, then uses tf's saver.restore
        """
        # fetch record from database and get the filename info from record
        rec = self.load_from_db(self.exp_id)
        # should be of form *-1000 (step)
        filename_suffix = rec.filename.split('/')[-1]
        loaded_filename = os.path.join(self.save_path, 'retrieved_' + filename_suffix)

        # create new file to write from gridfs
        load_dest = open(loaded_filename, "w+")
        load_dest.close()
        load_dest = open(loaded_filename, 'rwb+')
        fsbucket = gridfs.GridFSBucket(rec._GridOut__files.database,
                            bucket_name=rec._GridOut__files.name.split('.')[0])
        # save to local disk
        fsbucket.download_to_stream(rec._id, load_dest)

        # tf restore
        self.restore(self.sess, loaded_filename)
        print('Model variables restored.')

    def load_from_db(self, query):
        """
        Loads checkpoint from the database

        Checks the recent and regular checkpoint fs to find the latest one
        matching the query. Returns the GridOut obj corresponding to the
        record.

        Args:
            query: dict of Mongo queries
        """
        count = self.collfs.find(query).count()
        if count > 0:  # get latest that matches query
            ckpt_record = self.collfs._GridFS__files.find(query,
                            sort=[('uploadDate', -1)])[0]
            loading_from = 'long-term storage'

        count_recent = self.collfs_recents.find(query).count()
        if count_recent > 0:  # get latest that matches query
            ckpt_record_rec = self.collfs_recent._GridFS__files.find(query,
                                sort=[('uploadDate', -1)])[0]
            # use the record with latest timestamp
            if ckpt_record is None or ckpt_record_rec['uploadDate'] > ckpt_record['uploadDate']:
                loading_from = 'recent storage'
                ckpt_record = ckpt_record_rec

        if count + count_recent == 0:  # no matches for query
            raise Exception('No matching checkpoint for query "{}"'.format(repr(query)))

        print('Loading checkpoint from ', loading_from)
        return ckpt_record

    def save(self, step, results):
        elapsed_time_step = time.time() - self.start_time_step
        self.start_time_step = time.time()

        if math.isnan(results['loss']):
            raise NanLossDuringTrainingError

        rec = {'exp_id': self.exp_id,
            #    'cfg': preprocess_config(cfg),
            #    'saved_filters': saved_filters,
                'kind': 'train',
                'step': step,
                'loss': float(results['loss']),
                'learning_rate': float(results['lr']),
                'duration': int(1000 * elapsed_time_step)}

        if step > 0:
            # write loss to db
            if self.save_loss and step % self.save_loss_freq == 0:
                self.coll.insert_one(rec)

        if self.save_vars and step % self.save_vars_freq == 0 and step > 0:
            saved_path = super(Saver, self).save(self.sess,
                                    save_path=self.save_path,
                                    global_step=step,
                                    write_meta_graph=False)
            # save the saved file to 'recent' checkpoint fs
            if self.save_filters_to_db:
                self.collfs_recent.put(open(saved_path, 'rb'),
                                       filename=saved_path,
                                       saved_filters=True,
                                       **rec)
                # TODO: when do we move from recent to non recent...
                print('Saved variable checkpoint to recent fs.')

        print('Step {} -- loss: {:.6f}, lr: {:.6f}, time: {:.0f}'
              'ms'.format(rec['step'], rec['loss'], rec['learning_rate'], rec['duration']))
        sys.stdout.flush()  # flush the stdout buffer

    def valid(self, step, results):
        elapsed_time_step = time.time() - self.start_time_step
        self.start_time_step = time.time()
        rec = {'exp_id': self.exp_id,
            #    'cfg': preprocess_config(cfg),
            #    'saved_filters': saved_filters,
               'kind': 'validation',
               'step': step,
               'duration': 1000 * elapsed_time_step}
        rec.update(results)
        if step > 0:
            # write loss to file
            if self.save_valid and step % self.save_valid_freq == 0:
                pass
                # self.coll.insert(rec)

        message = ('Step {} validation -- ' +
                   '{}: {:.3f}, ' * len(results) +
                   '{:.0f} ms')
        args = []
        for k, v in results.items():
            args.extend([k,v])
        args = [rec['step']] + args + [rec['duration']]
        print(message.format(*args))

    def predict(self, step, results):
        if not hasattr(results['output'], '__iter__'):
            outputs = [results['outputs']]
        else:
            outputs = results['outputs']

        preds = [tf.argmax(output, 1) for output in outputs]

        return preds

    def test(self, step, results):
        raise NotImplementedError

def run_loop(sess, queues, saver, train_targets, valid_targets=None,
        start_step=0, end_step=None):
    """
    Args:
        - queues (~ data)
        - saver
        - targets
    """
    # initialize and/or restore variables for graph
    init = tf.initialize_all_variables()
    sess.run(init)
    print('variables initialized')

    tf.train.start_queue_runners(sess=sess)
    # start our custom queue runner's threads
    if not hasattr(queues, '__iter__'):
        queues = [queues]
    for queue in queues:
        queue.start_threads(sess)

    # start_time_step = time.time()  # start timer
    print('start training')
    for step in xrange(start_step, end_step):
        # get run output as dictionary {'2': loss2, 'lr': lr, etc..}
        results = sess.run(train_targets)
        # print output, save variables to checkpoint and save loss etc
        saver.save(step, results)
    sess.close()

def run(model_func,
        model_func_kwargs,
        data_func,
        data_func_kwargs,
        loss_func,
        loss_func_kwargs,
        lr_func,
        lr_func_kwargs,
        opt_func,
        opt_func_kwargs,
        saver_kwargs,
        train_targets=None,
        valid_targets=None,
        seed=None,
        start_step=0,
        end_step=float('inf'),
        log_device_placement=True
        ):
    with tf.Graph().as_default():  # to have multiple graphs [ex: eval, train]
        rng = np.random.RandomState(seed=seed)
        tf.set_random_seed(seed)

        tf.get_variable('global_step', [],
                        initializer=tf.constant_initializer(0),
                        trainable=False)

        train_data_node, train_labels_node, train_data_provider = data_func(train=True, **data_func_kwargs)
        valid_data_node, valid_labels_node, valid_data_provider = data_func(train=False, **data_func_kwargs)

        train_outputs = model_func(train_data_node, train=True, **model_func_kwargs)
        loss = loss_func(train_outputs, train_labels_node, **loss_func_kwargs)
        lr = lr_func(**lr_func_kwargs)
        optimizer = opt_func(loss, lr, **opt_func_kwargs)

        ttarg = {'loss': loss, 'lr': lr, 'opt': optimizer}
        if train_targets is None:
            train_targets = ttarg
        elif isinstance(train_targets, dict):
            train_targets.update(ttarg)
        else:
            raise ValueError('Train targets must be None or dict, got {}'.format(type(train_targets)))

        valid_outputs = model_func(valid_data_node, train=False, **model_func_kwargs)
        top_1_ops = [tf.nn.in_top_k(output, labels, 1)
                    for output, labels in zip(valid_outputs, valid_labels_node)]
        top_5_ops = [tf.nn.in_top_k(output, labels, 5)
                    for output, labels in zip(valid_outputs, valid_labels_node)]
        vtarg = {'top1': top_1_ops, 'top5': top_5_ops}
        if valid_targets is None:
            valid_targets = vtarg
        elif isinstance(valid_targets, dict):
            valid_targets.update(vtarg)
        else:
            raise ValueError('Validation targets must be None or dict, got {}'.format(type(valid_targets)))

        # create session
        sess = tf.Session(config=tf.ConfigProto(
                                allow_soft_placement=True,
                                log_device_placement=log_device_placement))

        saver = Saver(sess, **saver_kwargs)
        run_loop(sess,
            [train_data_provider, valid_data_provider],
            saver,
            train_targets=train_targets,
            valid_targets=valid_targets,
            start_step=start_step,
            end_step=end_step)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import numpy as np
from geometry import gram_schimdt, reflection_matrix, get_mirrors
from todd_coxeter import CosetTable

polytopes = {
    "tetrahedron": (3, 3, 2),
    "cube": (4, 3, 2),
    "octahedron": (3, 4, 2),
    "dodecahedron": (5, 3, 2),
    "icosahedron": (3, 5, 2),
    "5-cell": (3, 3, 3),
    "8-cell": (4, 3, 3),
    "16-cell": (3, 3, 4),
    "24-cell": (3, 4, 3),
    "120-cell": (5, 3, 3),
    "600-cell": (3, 3, 5)
    }

class Polytope4d(object):

    def __init__(self, p, q, r, truncated=True):
        self.pqr = (p, q, r)
        self.build_group()
        self.build_geometry()
        self.truncated = truncated

    def build_group(self):
        p, q, r = self.pqr
        gens = list(range(8))
        rels = [[0, 2]*p, [2, 4]*q, [4, 6]*r,
                [0, 4]*2, [2, 6]*2, [0, 6]*2,
                [0, 0], [2, 2], [4, 4], [6, 6],
                [0, 1], [2, 3], [4, 5], [6, 7]]
        self.vtable = CosetTable(gens, rels, [[2], [4], [6]]).run()
        self.vwords = self.vtable.get_words()
        self.ewords = CosetTable(gens, rels, [[0], [4], [6]]).get_words()
        self.fwords = CosetTable(gens, rels, [[0], [2], [6]]).get_words()

    def build_geometry(self):
        """
        1. get the 4 reflection matrices.
        2. get the coordinates of all vertices.
        """
        # the 4 reflection matrices
        M = get_mirrors(*self.pqr)
        self.reflectors = [reflection_matrix(v) for v in M]
        # an initial vertex
        v0 = gram_schimdt(M[::-1])[-1]
        self.v0 = v0 / np.linalg.norm(v0)
        # all vertex coordinates
        self.vertex_coords = [self.transform(self.v0, word) for word in self.vwords]
        # an initial edge
        e0 = (0, self.map_vertex(0, (0,)))
        # get all edges
        self.edge_list = [tuple(self.map_vertex(v, word) for v in e0) for word in self.ewords]
        # an initial face
        f0 = [self.map_vertex(0, (0, 2)*i) for i in range(self.pqr[0])]
        # get all faces
        self.face_list = [tuple(self.map_vertex(v, word) for v in f0) for word in self.fwords]

    def transform(self, v, word):
        for w in word:
            v = np.dot(v, self.reflectors[w//2])
        return v

    def map_vertex(self, v, word):
        for w in word:
            v = self.vtable[v][w]
        return v

    def write_to_pov(self):
        with open("polytope_data.inc".format(*self.pqr), "w") as f:
            for v in self.vertex_coords:
                f.write("Vertex(<{}, {}, {}, {}>)\n".format(*v))

            for i, j in self.edge_list:
                s = self.vertex_coords[i]
                e = self.vertex_coords[j]
                f.write("Arc(<{}, {}, {}, {}>, <{}, {}, {}, {}>)\n".format(*np.concatenate((s, e))))

if __name__ == "__main__":
    p, q, r = polytopes["8-cell"]
    P = Polytope4d(p, q, r)
    P.write_to_pov()
<commit_msg>change apeture point for stereo projection to make it work for 5-cell<commit_after># -*- coding: utf-8 -*-
import numpy as np
from geometry import gram_schimdt, reflection_matrix, get_mirrors
from todd_coxeter import CosetTable

polytopes = {
    "tetrahedron": (3, 3, 2),
    "cube": (4, 3, 2),
    "octahedron": (3, 4, 2),
    "dodecahedron": (5, 3, 2),
    "icosahedron": (3, 5, 2),
    "5-cell": (3, 3, 3),
    "8-cell": (4, 3, 3),
    "16-cell": (3, 3, 4),
    "24-cell": (3, 4, 3),
    "120-cell": (5, 3, 3),
    "600-cell": (3, 3, 5)
    }

class Polytope4d(object):

    def __init__(self, p, q, r):
        self.pqr = (p, q, r)
        self.build_group()
        self.build_geometry()

    def build_group(self):
        p, q, r = self.pqr
        gens = list(range(8))
        rels = [[0, 2]*p, [2, 4]*q, [4, 6]*r,
                [0, 4]*2, [2, 6]*2, [0, 6]*2,
                [0, 0], [2, 2], [4, 4], [6, 6],
                [0, 1], [2, 3], [4, 5], [6, 7]]
        self.vtable = CosetTable(gens, rels, [[2], [4], [6]]).run()
        self.vwords = self.vtable.get_words()
        self.ewords = CosetTable(gens, rels, [[0], [4], [6]]).get_words()
        self.fwords = CosetTable(gens, rels, [[0], [2], [6]]).get_words()

    def build_geometry(self):
        """
        1. get the 4 reflection matrices.
        2. get the coordinates of all vertices.
        """
        # the 4 reflection matrices
        M = get_mirrors(*self.pqr)
        self.reflectors = [reflection_matrix(v) for v in M]
        # an initial vertex
        v0 = gram_schimdt(M[::-1])[-1]
        self.v0 = v0 / np.linalg.norm(v0)
        # all vertex coordinates
        self.vertex_coords = [self.transform(self.v0, word) for word in self.vwords]
        # an initial edge
        e0 = (0, self.map_vertex(0, (0,)))
        # get all edges
        self.edge_list = [tuple(self.map_vertex(v, word) for v in e0) for word in self.ewords]
        # an initial face
        f0 = [self.map_vertex(0, (0, 2)*i) for i in range(self.pqr[0])]
        # get all faces
        self.face_list = [tuple(self.map_vertex(v, word) for v in f0) for word in self.fwords]

    def transform(self, v, word):
        for w in word:
            v = np.dot(v, self.reflectors[w//2])
        return v

    def map_vertex(self, v, word):
        for w in word:
            v = self.vtable[v][w]
        return v

    def write_to_pov(self):
        with open("polytope_data.inc".format(*self.pqr), "w") as f:
            for v in self.vertex_coords:
                f.write("Vertex(<{}, {}, {}, {}>)\n".format(*v))

            for i, j in self.edge_list:
                s = self.vertex_coords[i]
                e = self.vertex_coords[j]
                f.write("Arc(<{}, {}, {}, {}>, <{}, {}, {}, {}>)\n".format(*np.concatenate((s, e))))

if __name__ == "__main__":
    p, q, r = polytopes["8-cell"]
    P = Polytope4d(p, q, r)
    P.write_to_pov()
<|endoftext|>


<commit_before><commit_msg>Added strip to command line<commit_after><|endoftext|>


<commit_before># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code, optparse, os, socket, sys
from datetime import datetime
from attrdict import attrdict

try:
    import info
except ImportError:
    info = None

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
'''

# there's only one option parsing done, so make it global and add some
# helper functions to make it work well.
parser = optparse.OptionParser(usage=usage, version=version,
                               description=brief_copyright,
                               formatter=optparse.TitledHelpFormatter())
parser.disable_interspersed_args()

# current option group
group = None

def set_group(*args, **kwargs):
    '''set the current option group'''
    global group
    if not args and not kwargs:
        group = None
    else:
        group = parser.add_option_group(*args, **kwargs)

class splitter(object):
    def __init__(self, split):
        self.split = split
    def __call__(self, option, opt_str, value, parser):
        getattr(parser.values, option.dest).extend(value.split(self.split))

def add_option(*args, **kwargs):
    '''add an option to the current option group, or global none set'''

    # if action=split, but allows the option arguments
    # themselves to be lists separated by the split variable'''

    if kwargs.get('action', None) == 'append' and 'split' in kwargs:
        split = kwargs.pop('split')
        kwargs['default'] = []
        kwargs['type'] = 'string'
        kwargs['action'] = 'callback'
        kwargs['callback'] = splitter(split)

    if group:
        return group.add_option(*args, **kwargs)

    return parser.add_option(*args, **kwargs)

def bool_option(name, default, help):
    '''add a boolean option called --name and --no-name.
    Display help depending on which is the default'''

    tname = '--%s' % name
    fname = '--no-%s' % name
    dest = name.replace('-', '_')
    if default:
        thelp = optparse.SUPPRESS_HELP
        fhelp = help
    else:
        thelp = help
        fhelp = optparse.SUPPRESS_HELP

    add_option(tname, action="store_true", default=default, help=thelp)
    add_option(fname, action="store_false", dest=dest, help=fhelp)

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")

# Tracing options
set_group("Trace Options")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing")
add_option("--trace-start", metavar="TIME", default='0s',
    help="Start tracing at TIME (must have units)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-circlebuf", metavar="SIZE", type="int", default=0,
    help="If SIZE is non-zero, turn on the circular buffer with SIZE lines")
add_option("--no-trace-circlebuf", action="store_const", const=0,
    dest='trace_circlebuf', help=optparse.SUPPRESS_HELP)
bool_option("trace-dumponexit", default=False,
    help="Dump trace buffer on exit")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Execution Trace options
set_group("Execution Trace Options")
bool_option("speculative", default=True,
    help="Don't capture speculative instructions")
bool_option("print-cycle", default=True,
    help="Don't print cycle numbers in trace output")
bool_option("print-symbol", default=True,
    help="Disable PC symbols in trace output")
bool_option("print-opclass", default=True,
    help="Don't print op class type in trace output")
bool_option("print-thread", default=True,
    help="Don't print thread number in trace output")
bool_option("print-effaddr", default=True,
    help="Don't print effective address in trace output")
bool_option("print-data", default=True,
    help="Don't print result data in trace output")
bool_option("print-iregs", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-fetch-seq", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-cpseq", default=False,
    help="Print correct path sequence numbers in trace output")

options = attrdict()
arguments = []

def usage(exitcode=None):
    parser.print_help()
    if exitcode is not None:
        sys.exit(exitcode)

def parse_args():
    _opts,args = parser.parse_args()
    opts = attrdict(_opts.__dict__)

    # setting verbose and quiet at the same time doesn't make sense
    if opts.verbose > 0 and opts.quiet > 0:
        usage(2)

    # store the verbosity in a single variable.  0 is default,
    # negative numbers represent quiet and positive values indicate verbose
    opts.verbose -= opts.quiet

    del opts.quiet

    options.update(opts)
    arguments.extend(args)
    return opts,args

def main():
    import cc_main

    parse_args()

    done = False
    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if done:
        sys.exit(0)

    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % cc_main.cvar.compileDate;
        print "M5 started %s" % datetime.now().ctime()
        print "M5 executing on %s" % socket.gethostname()

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]
        usage(2)

    # tell C++ about output directory
    cc_main.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    objects.Statistics.text_file = options.stats_file

    # set debugging options
    objects.Debug.break_cycles = options.debug_break

    # set tracing options
    objects.Trace.flags = options.trace_flags
    objects.Trace.start = options.trace_start
    objects.Trace.file = options.trace_file
    objects.Trace.bufsize = options.trace_circlebuf
    objects.Trace.dump_on_exit = options.trace_dumponexit
    objects.Trace.ignore = options.trace_ignore

    # set execution trace options
    objects.ExecutionTrace.speculative = options.speculative
    objects.ExecutionTrace.print_cycle = options.print_cycle
    objects.ExecutionTrace.pc_symbol = options.print_symbol
    objects.ExecutionTrace.print_opclass = options.print_opclass
    objects.ExecutionTrace.print_thread = options.print_thread
    objects.ExecutionTrace.print_effaddr = options.print_effaddr
    objects.ExecutionTrace.print_data = options.print_data
    objects.ExecutionTrace.print_iregs = options.print_iregs
    objects.ExecutionTrace.print_fetchseq = options.print_fetch_seq
    objects.ExecutionTrace.print_cpseq = options.print_cpseq

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    scope = { '__file__' : sys.argv[0] }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec("import readline", scope)

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        from pdb import Pdb
        debugger = Pdb()
        debugger.run('execfile("%s")' % sys.argv[0], scope)
    else:
        execfile(sys.argv[0], scope)

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<commit_msg>Output the command line.<commit_after># Copyright (c) 2005 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert

import code, optparse, os, socket, sys
from datetime import datetime
from attrdict import attrdict

try:
    import info
except ImportError:
    info = None

__all__ = [ 'options', 'arguments', 'main' ]

usage="%prog [m5 options] script.py [script options]"
version="%prog 2.0"
brief_copyright='''
Copyright (c) 2001-2006
The Regents of The University of Michigan
All Rights Reserved
'''

# there's only one option parsing done, so make it global and add some
# helper functions to make it work well.
parser = optparse.OptionParser(usage=usage, version=version,
                               description=brief_copyright,
                               formatter=optparse.TitledHelpFormatter())
parser.disable_interspersed_args()

# current option group
group = None

def set_group(*args, **kwargs):
    '''set the current option group'''
    global group
    if not args and not kwargs:
        group = None
    else:
        group = parser.add_option_group(*args, **kwargs)

class splitter(object):
    def __init__(self, split):
        self.split = split
    def __call__(self, option, opt_str, value, parser):
        getattr(parser.values, option.dest).extend(value.split(self.split))

def add_option(*args, **kwargs):
    '''add an option to the current option group, or global none set'''

    # if action=split, but allows the option arguments
    # themselves to be lists separated by the split variable'''

    if kwargs.get('action', None) == 'append' and 'split' in kwargs:
        split = kwargs.pop('split')
        kwargs['default'] = []
        kwargs['type'] = 'string'
        kwargs['action'] = 'callback'
        kwargs['callback'] = splitter(split)

    if group:
        return group.add_option(*args, **kwargs)

    return parser.add_option(*args, **kwargs)

def bool_option(name, default, help):
    '''add a boolean option called --name and --no-name.
    Display help depending on which is the default'''

    tname = '--%s' % name
    fname = '--no-%s' % name
    dest = name.replace('-', '_')
    if default:
        thelp = optparse.SUPPRESS_HELP
        fhelp = help
    else:
        thelp = help
        fhelp = optparse.SUPPRESS_HELP

    add_option(tname, action="store_true", default=default, help=thelp)
    add_option(fname, action="store_false", dest=dest, help=fhelp)

# Help options
add_option('-A', "--authors", action="store_true", default=False,
    help="Show author information")
add_option('-C', "--copyright", action="store_true", default=False,
    help="Show full copyright information")
add_option('-R', "--readme", action="store_true", default=False,
    help="Show the readme")
add_option('-N', "--release-notes", action="store_true", default=False,
    help="Show the release notes")

# Options for configuring the base simulator
add_option('-d', "--outdir", metavar="DIR", default=".",
    help="Set the output directory to DIR [Default: %default]")
add_option('-i', "--interactive", action="store_true", default=False,
    help="Invoke the interactive interpreter after running the script")
add_option("--pdb", action="store_true", default=False,
    help="Invoke the python debugger before running the script")
add_option('-p', "--path", metavar="PATH[:PATH]", action='append', split=':',
    help="Prepend PATH to the system path when invoking the script")
add_option('-q', "--quiet", action="count", default=0,
    help="Reduce verbosity")
add_option('-v', "--verbose", action="count", default=0,
    help="Increase verbosity")

# Statistics options
set_group("Statistics Options")
add_option("--stats-file", metavar="FILE", default="m5stats.txt",
    help="Sets the output file for statistics [Default: %default]")

# Debugging options
set_group("Debugging Options")
add_option("--debug-break", metavar="TIME[,TIME]", action='append', split=',',
    help="Cycle to create a breakpoint")

# Tracing options
set_group("Trace Options")
add_option("--trace-flags", metavar="FLAG[,FLAG]", action='append', split=',',
    help="Sets the flags for tracing")
add_option("--trace-start", metavar="TIME", default='0s',
    help="Start tracing at TIME (must have units)")
add_option("--trace-file", metavar="FILE", default="cout",
    help="Sets the output file for tracing [Default: %default]")
add_option("--trace-circlebuf", metavar="SIZE", type="int", default=0,
    help="If SIZE is non-zero, turn on the circular buffer with SIZE lines")
add_option("--no-trace-circlebuf", action="store_const", const=0,
    dest='trace_circlebuf', help=optparse.SUPPRESS_HELP)
bool_option("trace-dumponexit", default=False,
    help="Dump trace buffer on exit")
add_option("--trace-ignore", metavar="EXPR", action='append', split=':',
    help="Ignore EXPR sim objects")

# Execution Trace options
set_group("Execution Trace Options")
bool_option("speculative", default=True,
    help="Don't capture speculative instructions")
bool_option("print-cycle", default=True,
    help="Don't print cycle numbers in trace output")
bool_option("print-symbol", default=True,
    help="Disable PC symbols in trace output")
bool_option("print-opclass", default=True,
    help="Don't print op class type in trace output")
bool_option("print-thread", default=True,
    help="Don't print thread number in trace output")
bool_option("print-effaddr", default=True,
    help="Don't print effective address in trace output")
bool_option("print-data", default=True,
    help="Don't print result data in trace output")
bool_option("print-iregs", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-fetch-seq", default=False,
    help="Print fetch sequence numbers in trace output")
bool_option("print-cpseq", default=False,
    help="Print correct path sequence numbers in trace output")

options = attrdict()
arguments = []

def usage(exitcode=None):
    parser.print_help()
    if exitcode is not None:
        sys.exit(exitcode)

def parse_args():
    _opts,args = parser.parse_args()
    opts = attrdict(_opts.__dict__)

    # setting verbose and quiet at the same time doesn't make sense
    if opts.verbose > 0 and opts.quiet > 0:
        usage(2)

    # store the verbosity in a single variable.  0 is default,
    # negative numbers represent quiet and positive values indicate verbose
    opts.verbose -= opts.quiet

    del opts.quiet

    options.update(opts)
    arguments.extend(args)
    return opts,args

def main():
    import cc_main

    parse_args()

    done = False
    if options.copyright:
        done = True
        print info.LICENSE
        print

    if options.authors:
        done = True
        print 'Author information:'
        print
        print info.AUTHORS
        print

    if options.readme:
        done = True
        print 'Readme:'
        print
        print info.README
        print

    if options.release_notes:
        done = True
        print 'Release Notes:'
        print
        print info.RELEASE_NOTES
        print

    if done:
        sys.exit(0)

    if options.verbose >= 0:
        print "M5 Simulator System"
        print brief_copyright
        print
        print "M5 compiled %s" % cc_main.cvar.compileDate;
        print "M5 started %s" % datetime.now().ctime()
        print "M5 executing on %s" % socket.gethostname()
        print "command line:",
        for argv in sys.argv:
            print argv,
        print

    # check to make sure we can find the listed script
    if not arguments or not os.path.isfile(arguments[0]):
        if arguments and not os.path.isfile(arguments[0]):
            print "Script %s not found" % arguments[0]
        usage(2)

    # tell C++ about output directory
    cc_main.setOutputDir(options.outdir)

    # update the system path with elements from the -p option
    sys.path[0:0] = options.path

    import objects

    # set stats options
    objects.Statistics.text_file = options.stats_file

    # set debugging options
    objects.Debug.break_cycles = options.debug_break

    # set tracing options
    objects.Trace.flags = options.trace_flags
    objects.Trace.start = options.trace_start
    objects.Trace.file = options.trace_file
    objects.Trace.bufsize = options.trace_circlebuf
    objects.Trace.dump_on_exit = options.trace_dumponexit
    objects.Trace.ignore = options.trace_ignore

    # set execution trace options
    objects.ExecutionTrace.speculative = options.speculative
    objects.ExecutionTrace.print_cycle = options.print_cycle
    objects.ExecutionTrace.pc_symbol = options.print_symbol
    objects.ExecutionTrace.print_opclass = options.print_opclass
    objects.ExecutionTrace.print_thread = options.print_thread
    objects.ExecutionTrace.print_effaddr = options.print_effaddr
    objects.ExecutionTrace.print_data = options.print_data
    objects.ExecutionTrace.print_iregs = options.print_iregs
    objects.ExecutionTrace.print_fetchseq = options.print_fetch_seq
    objects.ExecutionTrace.print_cpseq = options.print_cpseq

    sys.argv = arguments
    sys.path = [ os.path.dirname(sys.argv[0]) ] + sys.path

    scope = { '__file__' : sys.argv[0] }

    # we want readline if we're doing anything interactive
    if options.interactive or options.pdb:
        exec("import readline", scope)

    # if pdb was requested, execfile the thing under pdb, otherwise,
    # just do the execfile normally
    if options.pdb:
        from pdb import Pdb
        debugger = Pdb()
        debugger.run('execfile("%s")' % sys.argv[0], scope)
    else:
        execfile(sys.argv[0], scope)

    # once the script is done
    if options.interactive:
        interact = code.InteractiveConsole(scope)
        interact.interact("M5 Interactive Console")

if __name__ == '__main__':
    from pprint import pprint

    parse_args()

    print 'opts:'
    pprint(options, indent=4)
    print

    print 'args:'
    pprint(arguments, indent=4)
<|endoftext|>


<commit_before>#! /usr/bin/env python2.7
"""Script to """

from __future__ import print_function
import os
import sys
import argparse
from bdb import BdbQuit

import script_paths

__version__ = '0.1.0'

class ScriptError(NotImplementedError):
    pass

def run_args(args, methods):
    """Run any methods eponymous with args"""
    if not args:
        return False
    valuable_args = {k for k, v in args.__dict__.items() if v}
    arg_methods = {methods[a] for a in valuable_args if a in methods}
    for method in arg_methods:
        method(args)

def version(args):
    print('%s %s' % (args, __version__))
    raise SystemExit

def Use_debugger(_args):
    try:
        import pudb as pdb
    except ImportError:
        import pdb
    pdb.set_trace()

def parse_args(methods):
    """Parse out command line arguments"""
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument('keys', metavar='keys', type=str, nargs='+',
                        help='keys to find')
    parser.add_argument('-i', '--inverse', action='store_true',
                        help='show inverse')
    parser.add_argument('-m', '--mains', action='store_true',
                        help='show python scripts with main methods')
    parser.add_argument('-p', '--python', action='store_true',
                        help='show python scripts only')
    parser.add_argument('-v', '--version', action='store_true',
                        help='Show version')
    parser.add_argument('-U', '--Use_debugger', action='store_true',
                        help='Run the script with pdb (or pudb if available)')
    args = parser.parse_args()
    run_args(args, methods)
    return args

def lines(path_to_item):
    if path_to_item.isfile():
        return path_to_item.lines()
    return ['']

def script(args):
    paths_to_items = script_paths.get(args.keys)
    result = False
    for path_to_item in paths_to_items:
        if args.python:
            if path_to_item.ext != '.py':
                if 'python' not in lines(path_to_item)[0]:
                    continue
        elif args.mains:
            for line in lines(path_to_item):
                if line.startswith('def main('):
                    break
            else:
                continue
        print(path_to_item)
        result = True
    return result

def main():
    """Run the script"""
    try:
        args = parse_args(globals())
        return os.EX_OK if script(args) else not os.EX_OK
    except BdbQuit:
        pass
    except SystemExit as e:
        return e.code
    except Exception, e:  # pylint: disable=broad-except
        if __version__[0] < '1':
            raise
        print(e, file=sys.stderr)
        return not os.EX_OK
    return os.EX_OK

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Remove redundant option<commit_after>#! /usr/bin/env python2.7
"""Script to """

from __future__ import print_function
import os
import sys
import argparse
from bdb import BdbQuit

import script_paths

__version__ = '0.1.0'

class ScriptError(NotImplementedError):
    pass

def run_args(args, methods):
    """Run any methods eponymous with args"""
    if not args:
        return False
    valuable_args = {k for k, v in args.__dict__.items() if v}
    arg_methods = {methods[a] for a in valuable_args if a in methods}
    for method in arg_methods:
        method(args)

def version(args):
    print('%s %s' % (args, __version__))
    raise SystemExit

def Use_debugger(_args):
    try:
        import pudb as pdb
    except ImportError:
        import pdb
    pdb.set_trace()

def parse_args(methods):
    """Parse out command line arguments"""
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument('keys', metavar='keys', type=str, nargs='*',
                        help='keys to find')
    parser.add_argument('-m', '--mains', action='store_true',
                        help='show python scripts with main methods')
    parser.add_argument('-p', '--python', action='store_true',
                        help='show python scripts only')
    parser.add_argument('-v', '--version', action='store_true',
                        help='Show version')
    parser.add_argument('-U', '--Use_debugger', action='store_true',
                        help='Run the script with pdb (or pudb if available)')
    args = parser.parse_args()
    run_args(args, methods)
    return args

def lines(path_to_item):
    if path_to_item.isfile():
        return path_to_item.lines()
    return ['']

def script(args):
    paths_to_items = script_paths.get(args.keys)
    result = False
    for path_to_item in paths_to_items:
        if args.python:
            if path_to_item.ext != '.py':
                if 'python' not in lines(path_to_item)[0]:
                    continue
        elif args.mains:
            for line in lines(path_to_item):
                if line.startswith('def main('):
                    break
            else:
                continue
        print(path_to_item)
        result = True
    return result

def main():
    """Run the script"""
    try:
        args = parse_args(globals())
        return os.EX_OK if script(args) else not os.EX_OK
    except BdbQuit:
        pass
    except SystemExit as e:
        return e.code
    except Exception, e:  # pylint: disable=broad-except
        if __version__[0] < '1':
            raise
        print(e, file=sys.stderr)
        return not os.EX_OK
    return os.EX_OK

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import numpy as np

def norm_dir_vect(A, B):
    """
    Normalized direction vector from point A to point B

    Parameters
    ----------
    A, B : array_like
           Points in space

    Returns
    -------
    a : array_like
    """
    if (A == B).all():
        msg = "Can't calculate vector, same points"
        raise ValueError(msg)

    a = (B - A) / np.linalg.norm(B - A)
    return a

def vect_perpendicular(a):
    """
    Normal vector to vector a, so cross_product(a, b) < 0
    in a 2D euclidean space.

    Parameters
    ----------
    a : array_like
        Vector in a 2D-euclidean space

    Returns
    -------
    b : array_like
    """
    if (a == np.zeros_like(a)).all():
        msg = "Can't calculate, null vector"
        raise ValueError(msg)

    b = np.empty_like(a)
    b[0] = a[1]
    b[1] = - a[0]
    return b

def dist_point2line(P, A, B):
    """
    Distance from a point P to a straight line
    in a 2D euclidean space defined by A and B

    Parameters
    ----------
    P, A, B : array_like
              P - point of reference
              A,B - points that define the line

    Returns
    -------
    d : float
    """
    a = (B - A) / np.linalg.norm(B - A)
    b = vect_perpendicular(a)

    if np.cross((B - P), a) == 0:
        msg = 'Point P belongs to the line AB'
        raise ValueError(msg)

    d = abs(np.vdot((B - P), b))
    return d

def area_4points(A, B, C, D):
    """

  y	^   C +-+ D     Area of the polygon defined by 4 points,
    |    /   \		previously ordered in a clockwise
    | B +-----+ A   (or counterclockwise) manner in a 2D
    +----->			euclidean space.
           x

    Parameters
    ----------
    A, B, C, D : array_like
                 points of the polygon

    Returns
    -------
    S : float
    """
    AB = B - A
    AC = C - A
    AD = D - A

    if np.sign(np.cross(AB, AC)) != np.sign(np.cross(AC, AD)):
        msg = 'Points not in a clockwise/counterclockwise fashion'
        raise ValueError(msg)

    S = np.cross(A, B) + np.cross(B, C) + np.cross(C, D) + np.cross(D, A)

    S *= 1/2
    return abs(S)
<commit_msg>null distance point2line fixed<commit_after>import numpy as np

def norm_dir_vect(A, B):
    """
    Normalized direction vector from point A to point B

    Parameters
    ----------
    A, B : array_like
           Points in space

    Returns
    -------
    a : array_like
    """
    if (A == B).all():
        msg = "Can't calculate vector, same points"
        raise ValueError(msg)

    a = (B - A) / np.linalg.norm(B - A)
    return a

def vect_perpendicular(a):
    """
    Normal vector to vector a, so cross_product(a, b) < 0
    in a 2D euclidean space.

    Parameters
    ----------
    a : array_like
        Vector in a 2D-euclidean space

    Returns
    -------
    b : array_like
    """
    if (a == np.zeros_like(a)).all():
        msg = "Can't calculate, null vector"
        raise ValueError(msg)

    b = np.empty_like(a)
    b[0] = a[1]
    b[1] = - a[0]
    return b

def dist_point2line(P, A, B):
    """
    Distance from a point P to a straight line
    in a 2D euclidean space defined by A and B

    Parameters
    ----------
    P, A, B : array_like
              P - point of reference
              A,B - points that define the line

    Returns
    -------
    d : float
    """
    a = (B - A) / np.linalg.norm(B - A)
    b = vect_perpendicular(a)

    if np.cross((B - P), a) == 0:
        d = 0
    else:
        d = abs(np.vdot((B - P), b))
    return d

def area_4points(A, B, C, D):
    """

  y	^   C +-+ D     Area of the polygon defined by 4 points,
    |    /   \		previously ordered in a clockwise
    | B +-----+ A   (or counterclockwise) manner in a 2D
    +----->			euclidean space.
           x

    Parameters
    ----------
    A, B, C, D : array_like
                 points of the polygon

    Returns
    -------
    S : float
    """
    AB = B - A
    AC = C - A
    AD = D - A

    if np.sign(np.cross(AB, AC)) != np.sign(np.cross(AC, AD)):
        msg = 'Points not in a clockwise/counterclockwise fashion'
        raise ValueError(msg)

    S = np.cross(A, B) + np.cross(B, C) + np.cross(C, D) + np.cross(D, A)

    S *= 1/2
    return abs(S)
<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

if __name__ == '__main__':
    sys.stderr.write("Use 'runner' or 'rebot' for executing.\n")
    sys.exit(252)  # 252 == DATA_ERROR

# Global workaround for os.listdir bug http://bugs.jython.org/issue1593
# This bug has been fixed in Jython 2.5.2 RC 2
if sys.platform.startswith('java') and sys.version_info[:3] < (2,5,2):
    from java.lang import String
    def listdir(path):
        items = os._listdir(path)
        if isinstance(path, unicode):
            items = [unicode(String(i).toString()) for i in items]
        return items
    os._listdir = os.listdir
    os.listdir = listdir

# Global workaround for os.stat bug http://bugs.jython.org/issue1658
# Jython 2.5.2 RC 2 still contains this bug, but additionally the workaround used
# here does not work on that version either.
if sys.platform.startswith('java') and os.sep == '\\' and sys.version_info < (2,5,2):
    os._posix = os.JavaPOSIX(os.PythonPOSIXHandler())
    os._native_posix = False

if 'pythonpathsetter' not in sys.modules:
    import pythonpathsetter
from output import Output, LOGGER, pyloggingconf
from conf import RobotSettings, RebotSettings
from running import TestSuite, STOP_SIGNAL_MONITOR
from serializing import RobotTestOutput, RebotTestOutput, SplitIndexTestOutput
from errors import DataError, Information, INFO_PRINTED, DATA_ERROR, STOPPED_BY_USER
from variables import init_global_variables
from version import get_version, get_full_version
import utils

__version__ = get_version()

def run_from_cli(args, usage):
    LOGGER.info(get_full_version('Robot Framework'))
    return _run_or_rebot_from_cli(run, args, usage, pythonpath='pythonpath')

def rebot_from_cli(args, usage):
    LOGGER.info(get_full_version('Rebot'))
    return _run_or_rebot_from_cli(run_rebot, args, usage)

def _run_or_rebot_from_cli(method, cliargs, usage, **argparser_config):
    LOGGER.register_file_logger()
    try:
        options, datasources = _parse_arguments(cliargs, usage,
                                                **argparser_config)
    except Information, msg:
        print utils.encode_output(unicode(msg))
        return INFO_PRINTED
    except DataError, err:
        _report_error(unicode(err), help=True)
        return DATA_ERROR
    LOGGER.info('Data sources: %s' % utils.seq2str(datasources))
    return _execute(method, datasources, options)

def _parse_arguments(cliargs, usage, **argparser_config):
    ap = utils.ArgumentParser(usage, get_full_version())
    return ap.parse_args(cliargs, argfile='argumentfile', unescape='escape',
                         help='help', version='version', check_args=True,
                         **argparser_config)

def _execute(method, datasources, options):
    try:
        suite = method(*datasources, **options)
    except DataError, err:
        _report_error(unicode(err), help=True)
        return DATA_ERROR
    except (KeyboardInterrupt, SystemExit):
        _report_error('Execution stopped by user.')
        return STOPPED_BY_USER
    except:
        error, details = utils.get_error_details()
        return _report_error('Unexpected error: %s' % error, details)
    else:
        return suite.return_code

def run(*datasources, **options):
    """Executes given Robot data sources with given options.

    Data sources are paths to files and directories, similarly as when running
    pybot/jybot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run('/path/to/tests.html')
    run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')

    Equivalent command line usage:
    pybot /path/to/tests.html
    pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
    """
    STOP_SIGNAL_MONITOR.start()
    settings = RobotSettings(options)
    LOGGER.register_console_logger(settings['MonitorWidth'],
                                   settings['MonitorColors'])
    output = Output(settings)
    init_global_variables(settings)
    suite = TestSuite(datasources, settings)
    suite.run(output)
    LOGGER.info("Tests execution ended. Statistics:\n%s"
                % suite.get_stat_message())
    testoutput = RobotTestOutput(suite, settings)
    output.close(suite)
    if settings.is_rebot_needed():
        datasources, settings = settings.get_rebot_datasources_and_settings()
        if settings['SplitOutputs'] > 0:
            testoutput = SplitIndexTestOutput(suite, datasources[0], settings)
        else:
            testoutput = RebotTestOutput(datasources, settings)
        testoutput.serialize(settings)
    LOGGER.close()
    return suite

def run_rebot(*datasources, **options):
    """Creates reports/logs from given Robot output files with given options.

    Given input files are paths to Robot output files similarly as when running
    rebot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run_rebot('/path/to/output.xml')
    run_rebot('/path/out1.xml', '/path/out2.xml', report='myrep.html', log='NONE')

    Equivalent command line usage:
    rebot /path/to/output.xml
    rebot --report myrep.html --log NONE /path/out1.xml /path/out2.xml
    """
    settings = RebotSettings(options)
    LOGGER.register_console_logger(colors=settings['MonitorColors'])
    LOGGER.disable_message_cache()
    testoutput = RebotTestOutput(datasources, settings)
    testoutput.serialize(settings, generator='Rebot')
    LOGGER.close()
    return testoutput.suite

def _report_error(message, details=None, help=False):
    if help:
        message += '\n\nTry --help for usage information.'
    LOGGER.error(message)
    if details:
        LOGGER.info(details)
<commit_msg>Correct return code when framework error happens<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

if __name__ == '__main__':
    sys.stderr.write("Use 'runner' or 'rebot' for executing.\n")
    sys.exit(252)  # 252 == DATA_ERROR

# Global workaround for os.listdir bug http://bugs.jython.org/issue1593
# This bug has been fixed in Jython 2.5.2 RC 2
if sys.platform.startswith('java') and sys.version_info[:3] < (2,5,2):
    from java.lang import String
    def listdir(path):
        items = os._listdir(path)
        if isinstance(path, unicode):
            items = [unicode(String(i).toString()) for i in items]
        return items
    os._listdir = os.listdir
    os.listdir = listdir

# Global workaround for os.stat bug http://bugs.jython.org/issue1658
# Jython 2.5.2 RC 2 still contains this bug, but additionally the workaround used
# here does not work on that version either.
if sys.platform.startswith('java') and os.sep == '\\' and sys.version_info < (2,5,2):
    os._posix = os.JavaPOSIX(os.PythonPOSIXHandler())
    os._native_posix = False

if 'pythonpathsetter' not in sys.modules:
    import pythonpathsetter
from output import Output, LOGGER, pyloggingconf
from conf import RobotSettings, RebotSettings
from running import TestSuite, STOP_SIGNAL_MONITOR
from serializing import RobotTestOutput, RebotTestOutput, SplitIndexTestOutput
from errors import (DataError, Information, INFO_PRINTED, DATA_ERROR,
                    STOPPED_BY_USER, FRAMEWORK_ERROR)
from variables import init_global_variables
from version import get_version, get_full_version
import utils

__version__ = get_version()

def run_from_cli(args, usage):
    LOGGER.info(get_full_version('Robot Framework'))
    return _run_or_rebot_from_cli(run, args, usage, pythonpath='pythonpath')

def rebot_from_cli(args, usage):
    LOGGER.info(get_full_version('Rebot'))
    return _run_or_rebot_from_cli(run_rebot, args, usage)

def _run_or_rebot_from_cli(method, cliargs, usage, **argparser_config):
    LOGGER.register_file_logger()
    try:
        options, datasources = _parse_arguments(cliargs, usage,
                                                **argparser_config)
    except Information, msg:
        print utils.encode_output(unicode(msg))
        return INFO_PRINTED
    except DataError, err:
        _report_error(unicode(err), help=True)
        return DATA_ERROR
    LOGGER.info('Data sources: %s' % utils.seq2str(datasources))
    return _execute(method, datasources, options)

def _parse_arguments(cliargs, usage, **argparser_config):
    ap = utils.ArgumentParser(usage, get_full_version())
    return ap.parse_args(cliargs, argfile='argumentfile', unescape='escape',
                         help='help', version='version', check_args=True,
                         **argparser_config)

def _execute(method, datasources, options):
    try:
        suite = method(*datasources, **options)
    except DataError, err:
        _report_error(unicode(err), help=True)
        return DATA_ERROR
    except (KeyboardInterrupt, SystemExit):
        _report_error('Execution stopped by user.')
        return STOPPED_BY_USER
    except:
        error, details = utils.get_error_details()
        _report_error('Unexpected error: %s' % error, details)
        return FRAMEWORK_ERROR
    else:
        return suite.return_code

def run(*datasources, **options):
    """Executes given Robot data sources with given options.

    Data sources are paths to files and directories, similarly as when running
    pybot/jybot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run('/path/to/tests.html')
    run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')

    Equivalent command line usage:
    pybot /path/to/tests.html
    pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
    """
    STOP_SIGNAL_MONITOR.start()
    settings = RobotSettings(options)
    LOGGER.register_console_logger(settings['MonitorWidth'],
                                   settings['MonitorColors'])
    output = Output(settings)
    init_global_variables(settings)
    suite = TestSuite(datasources, settings)
    suite.run(output)
    LOGGER.info("Tests execution ended. Statistics:\n%s"
                % suite.get_stat_message())
    testoutput = RobotTestOutput(suite, settings)
    output.close(suite)
    if settings.is_rebot_needed():
        datasources, settings = settings.get_rebot_datasources_and_settings()
        if settings['SplitOutputs'] > 0:
            testoutput = SplitIndexTestOutput(suite, datasources[0], settings)
        else:
            testoutput = RebotTestOutput(datasources, settings)
        testoutput.serialize(settings)
    LOGGER.close()
    return suite

def run_rebot(*datasources, **options):
    """Creates reports/logs from given Robot output files with given options.

    Given input files are paths to Robot output files similarly as when running
    rebot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run_rebot('/path/to/output.xml')
    run_rebot('/path/out1.xml', '/path/out2.xml', report='myrep.html', log='NONE')

    Equivalent command line usage:
    rebot /path/to/output.xml
    rebot --report myrep.html --log NONE /path/out1.xml /path/out2.xml
    """
    settings = RebotSettings(options)
    LOGGER.register_console_logger(colors=settings['MonitorColors'])
    LOGGER.disable_message_cache()
    testoutput = RebotTestOutput(datasources, settings)
    testoutput.serialize(settings, generator='Rebot')
    LOGGER.close()
    return testoutput.suite

def _report_error(message, details=None, help=False):
    if help:
        message += '\n\nTry --help for usage information.'
    LOGGER.error(message)
    if details:
        LOGGER.info(details)
<|endoftext|>