<commit_before># coding=utf-8
import logging
import threading

from contextlib import contextmanager
from pika.adapters import TwistedProtocolConnection
from pika.adapters.twisted_connection import TwistedChannel
from pika.connection import Parameters
from twisted.internet import reactor, defer
from twisted.internet.protocol import ClientCreator

__all__ = ["VERSION", "PooledConn"]

VERSION = "0.1.3"

logger = logging.getLogger(__name__)

__lock = threading.Lock()

@contextmanager
def _lock():
    __lock.acquire()
    try:
        yield
    finally:
        __lock.release()

class PooledConn(object):
    """
    """

    _my_params = {}
    _my_pools = ({}, {})
    _my_channels = {}
    _timeout = {}
    _max_size = {}
    _waiting = {}

    max_size = 100

    # retry once by default
    retry = True

    timeout_conn = 1

    loop = reactor

    def __new__(cls, params, timeout_conn=None, max_size=None):
        """
        :param cls:
        :param params: connection params
        :param timeout_conn: connection timeout
        :param max_size: max pool size for each connection
        :return: PoolConn instance
        """
        if isinstance(params, Parameters):
            _id = repr(params)  # take repr of params for id to identify the pool
            instance = super(PooledConn, cls).__new__(cls)
            if _id in cls._my_params:
                # always update the params instance
                cls._my_params[_id] = params
            else:
                cls._my_params[_id] = params
                cls._my_pools[0][_id] = {}
                cls._my_pools[1][_id] = {}
                cls._my_channels[_id] = {}
                cls._max_size[_id] = max_size if max_size else cls.max_size
                cls._timeout[_id] = timeout_conn if timeout_conn > 0 else cls.timeout_conn
                cls._waiting[_id] = []
                # only works when first created
            instance.__params = cls._my_params[_id]
            instance.__max_size = cls._max_size[_id]
            instance.timeout_conn = cls._timeout[_id]
            instance.__idle_pool, instance.__using_pool = (cls._my_pools[i][_id] for i in (0, 1))
            instance.__channel_pool = cls._my_channels[_id]
            instance.waiting = cls._waiting[_id]
            return instance
        else:
            raise TypeError('only accept pika Parameters type')

    def __init__(self, *args, **kwargs):
        """
        :param args: to keep with the __new__
        :param kwargs: to keep with the __new__
        :return:
        """
        # self.__params
        # self.__max_size
        # self.timeout_conn
        # self.__idle_pool
        # self.__using_pool
        # self.__channel_pool

    def __connect(self, retrying=False):
        params = self.__params
        cc = ClientCreator(self.loop, TwistedProtocolConnection, params)
        _d = cc.connectTCP(params.host, params.port, timeout=self.timeout_conn)
        _d.addCallback(lambda p: p.ready)
        _d.addCallbacks(self._in_pool,
                        lambda err: err if retrying or not self.retry else self.__connect(True))  # retry once when err
        return _d

    def _in_pool(self, conn):
        assert isinstance(conn, TwistedProtocolConnection), 'conn must be TwistedProtocolConnection'
        logger.debug('in pool : %s' % conn)

        _id = id(conn)

        if self.size < self.__max_size:
            # add hook to clear the bad connection object in the pool
            conn.ready = defer.Deferred()
            conn.ready.addErrback(self._clear, self.__idle_pool, self.__using_pool, self.__channel_pool, _id)
            # add new conn in using pool
            self.__using_pool[_id] = conn
        else:
            raise RuntimeError('_in_pool, unexpected reach')

        return conn

    @staticmethod
    def _clear(reason, idle_pool, using_pool, channel_pool, conn_id):
        """
        clear the bad connection
        :param reason:
        :param idle_pool:
        :param using_pool:
        :param channel_pool:
        :param conn_id:
        :return:
        """
        with _lock():
            try:
                idle_pool.pop(conn_id)
                logger.info('a connection lost when not using')
            except KeyError:
                if using_pool.pop(conn_id, None):
                    logger.warn('connection lost when using, should be handled later')
                    return reason
            finally:
                channel_pool.pop(conn_id, None)

    def _get_channel(self, conn):
        _id = id(conn)
        d = None
        p = self.__channel_pool
        if _id in p:
            channel = p[_id]
            if channel.is_open:
                d = defer.Deferred()
                d.callback(p[_id])
        if d is None:
            d = conn.channel()
            d.addCallback(lambda ch: p.update({_id: ch}) or setattr(ch, 'pool_id_', _id) or ch)

        def _h_err(ch, _conn):
            _conn.ready.addErrback(lambda _, _c: p.pop(id(_c), None), _conn)
            return ch

        d.addCallback(_h_err, conn)
        return d

    def acquire(self, channel=False):
        d = defer.Deferred()
        if channel:
            d.addCallback(self._get_channel)
        with _lock():
            if self.__idle_pool:
                _id, conn = self.__idle_pool.popitem()
                self.__using_pool[_id] = conn
                self.loop.callLater(0, d.callback, conn)
                return d
        if self.size >= self.__max_size:
            self.waiting.append(d)
        else:
            self.__connect().chainDeferred(d)
        return d

    def release(self, c):
        if isinstance(c, TwistedProtocolConnection):
            _id = id(c)
        elif isinstance(c, TwistedChannel):
            _id = c.pool_id_
            c = self.__using_pool.get(_id, None)
        else:
            return c
        if _id in self.__using_pool:
            # clear each hook add to ready when using
            c.ready.callbacks = c.ready.callbacks[:1]
            if self.waiting:
                self.waiting.pop(0).callback(c)
            else:
                with _lock():
                    # put the conn back to idle
                    self.__idle_pool[_id] = self.__using_pool.pop(_id)

    @property
    def size(self):
        with _lock():
            return len(self.__idle_pool) + len(self.__using_pool)

    def clear(self):
        with _lock():
            for c in self.__idle_pool.itervalues():
                c.close()
<commit_msg>fix a potential bug getting a closed connection<commit_after># coding=utf-8
import logging
import threading

from contextlib import contextmanager
from pika.adapters import TwistedProtocolConnection
from pika.adapters.twisted_connection import TwistedChannel
from pika.connection import Parameters
from twisted.internet import reactor, defer
from twisted.internet.protocol import ClientCreator

__all__ = ["VERSION", "PooledConn"]

VERSION = "0.1.4"

logger = logging.getLogger(__name__)

__lock = threading.Lock()

@contextmanager
def _lock():
    __lock.acquire()
    try:
        yield
    finally:
        __lock.release()

class PooledConn(object):
    """
    """

    _my_params = {}
    _my_pools = ({}, {})
    _my_channels = {}
    _timeout = {}
    _max_size = {}
    _waiting = {}

    max_size = 100

    # retry once by default
    retry = True

    timeout_conn = 1

    loop = reactor

    def __new__(cls, params, timeout_conn=None, max_size=None):
        """
        :param cls:
        :param params: connection params
        :param timeout_conn: connection timeout
        :param max_size: max pool size for each connection
        :return: PoolConn instance
        """
        if isinstance(params, Parameters):
            _id = repr(params)  # take repr of params for id to identify the pool
            instance = super(PooledConn, cls).__new__(cls)
            if _id in cls._my_params:
                # always update the params instance
                cls._my_params[_id] = params
            else:
                cls._my_params[_id] = params
                cls._my_pools[0][_id] = {}
                cls._my_pools[1][_id] = {}
                cls._my_channels[_id] = {}
                cls._max_size[_id] = max_size if max_size else cls.max_size
                cls._timeout[_id] = timeout_conn if timeout_conn > 0 else cls.timeout_conn
                cls._waiting[_id] = []
                # only works when first created
            instance.__params = cls._my_params[_id]
            instance.__max_size = cls._max_size[_id]
            instance.timeout_conn = cls._timeout[_id]
            instance.__idle_pool, instance.__using_pool = (cls._my_pools[i][_id] for i in (0, 1))
            instance.__channel_pool = cls._my_channels[_id]
            instance.waiting = cls._waiting[_id]
            return instance
        else:
            raise TypeError('only accept pika Parameters type')

    def __init__(self, *args, **kwargs):
        """
        :param args: to keep with the __new__
        :param kwargs: to keep with the __new__
        :return:
        """
        # self.__params
        # self.__max_size
        # self.timeout_conn
        # self.__idle_pool
        # self.__using_pool
        # self.__channel_pool

    def __connect(self, retrying=False):
        params = self.__params
        cc = ClientCreator(self.loop, TwistedProtocolConnection, params)
        _d = cc.connectTCP(params.host, params.port, timeout=self.timeout_conn)
        _d.addCallback(lambda p: p.ready)
        _d.addCallbacks(self._in_pool,
                        lambda err: err if retrying or not self.retry else self.__connect(True))  # retry once when err
        return _d

    def _in_pool(self, conn):
        assert isinstance(conn, TwistedProtocolConnection), 'conn must be TwistedProtocolConnection'
        logger.debug('in pool : %s' % conn)

        _id = id(conn)

        if self.size < self.__max_size:
            # add hook to clear the bad connection object in the pool
            conn.ready = defer.Deferred()
            conn.ready.addErrback(self._clear, self.__idle_pool, self.__using_pool, self.__channel_pool, _id)
            # add new conn in using pool
            self.__using_pool[_id] = conn
        else:
            raise RuntimeError('_in_pool, unexpected reach')

        return conn

    @staticmethod
    def _clear(reason, idle_pool, using_pool, channel_pool, conn_id):
        """
        clear the bad connection
        :param reason:
        :param idle_pool:
        :param using_pool:
        :param channel_pool:
        :param conn_id:
        :return:
        """
        with _lock():
            try:
                idle_pool.pop(conn_id)
                logger.info('a connection lost when not using')
            except KeyError:
                if using_pool.pop(conn_id, None):
                    logger.warn('connection lost when using, should be handled later')
                    return reason
            finally:
                channel_pool.pop(conn_id, None)

    def _get_channel(self, conn):
        _id = id(conn)
        d = None
        p = self.__channel_pool
        if _id in p:
            channel = p[_id]
            if channel.is_open:
                d = defer.Deferred()
                d.callback(p[_id])
        if d is None:
            d = conn.channel()
            d.addCallback(lambda ch: p.update({_id: ch}) or setattr(ch, 'pool_id_', _id) or ch)

        def _h_err(ch, _conn):
            _conn.ready.addErrback(lambda _, _c: p.pop(id(_c), None), _conn)
            return ch

        d.addCallback(_h_err, conn)
        return d

    def acquire(self, channel=False):
        d = defer.Deferred()
        if channel:
            d.addCallback(self._get_channel)
        with _lock():
            if self.__idle_pool:
                _id, conn = self.__idle_pool.popitem()
                self.__using_pool[_id] = conn
                d.callback(conn)
                return d
        if self.size >= self.__max_size:
            self.waiting.append(d)
        else:
            self.__connect().chainDeferred(d)
        return d

    def release(self, c):
        if isinstance(c, TwistedProtocolConnection):
            _id = id(c)
        elif isinstance(c, TwistedChannel):
            _id = c.pool_id_
            c = self.__using_pool.get(_id, None)
        else:
            return c
        if _id in self.__using_pool:
            # clear each hook add to ready when using
            c.ready.callbacks = c.ready.callbacks[:1]
            if self.waiting:
                self.waiting.pop(0).callback(c)
            else:
                with _lock():
                    # put the conn back to idle
                    self.__idle_pool[_id] = self.__using_pool.pop(_id)

    @property
    def size(self):
        with _lock():
            return len(self.__idle_pool) + len(self.__using_pool)

    def clear(self):
        with _lock():
            for c in self.__idle_pool.itervalues():
                c.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import urllib2
import ssl
import functools

# Define the sites we want to poll and the timeout.
SITES = (
    'https://redmine.codegrove.org',
    'http://koodilehto.fi',
    'http://vakiopaine.net',
)
TIMEOUT = 5

try:
    import gntp.notifier

    notification = gntp.notifier.mini
except ImportError:
    try:
        import pygtk
        pygtk.require('2.0')
        import pynotify

        notification = functools.partial(
            pynotify.Notification,
            'Koodilehto Service Error'
        )
    except ImportError:
        def out(data):
            print data

        notification = out

def poll(sites, timeout, ok, error):
    """Checks if the given URLs are online."""
    for site in sites:
        ok('Polling ' + site)

        try:
            response = urllib2.urlopen(site, timeout=timeout)
            response.read()
        except urllib2.URLError as e:
            error(site + ' ' + e.code)
        except ssl.SSLError as e:
            error(site + ' ' + e.message)
        else:
            print 'OK'

def empty(data):
    pass

if __name__ == '__main__':
    poll(SITES, timeout=TIMEOUT, ok=empty, error=notification)
<commit_msg>Add a catch-all case<commit_after>#!/usr/bin/env python
import urllib2
import ssl
import functools

# Define the sites we want to poll and the timeout.
SITES = (
    'https://redmine.codegrove.org',
    'http://koodilehto.fi',
    'http://vakiopaine.net',
    'http://thisshouldtimeout.net',
)
TIMEOUT = 5

try:
    import gntp.notifier

    notification = gntp.notifier.mini
except ImportError:
    try:
        import pygtk
        pygtk.require('2.0')
        import pynotify

        notification = functools.partial(
            pynotify.Notification,
            'Koodilehto Service Error'
        )
    except ImportError:
        def out(data):
            print data

        notification = out

def poll(sites, timeout, ok, error):
    """Checks if the given URLs are online."""
    for site in sites:
        ok('Polling ' + site)

        try:
            response = urllib2.urlopen(site, timeout=timeout)
            response.read()
        except urllib2.URLError as e:
            error(site + ' ' + str(e.code))
        except ssl.SSLError as e:
            error(site + ' ' + e.message)
        except Exception as e:
            error(site + ' ' + e.message)
        else:
            ok('OK')

def empty(data):
    pass

if __name__ == '__main__':
    poll(SITES, timeout=TIMEOUT, ok=empty, error=notification)
<|endoftext|>


<commit_before>import pandas as pd
import logging
import traceback
import os.path

from datetime import date, timedelta

pd.options.mode.chained_assignment = None  # Turns off Pandas copy warning

logger = logging.getLogger(__name__)

class PollIO(object):

    def __init__(self, state, poll_data_url, file_path, file_name):
        self.state = state
        self.poll_data_url = poll_data_url
        self.file_path = file_path
        self.file_name = file_name

        self.latest_poll_data = self.get_latest_poll_data()
        self.saved_poll_data = self.load_saved_poll_data()
        self.new_poll_data = self.return_new_polls(self.saved_poll_data, self.latest_poll_data)

    def get_latest_poll_data(self, csv_url=None):
        """Loads the latest polls from Huffington Post Pollster

        Parameters
        ----------
        csv_url: string, optional
            The url to the longform csv to be loaded
            Default: self.poll_data_url

        Returns
        -------
        pandas dataframe of the loaded data
        """
        try:
            logger.debug('Downloading latest %s polls from web address', self.state)
            if csv_url is None:
                csv_url = self.poll_data_url
            df = pd.read_csv(csv_url)
            # Extra formatting for state polls
            df['State'] = self.state
            if 'Other' not in df.columns:
                df['Other'] = 0.0
            if 'Johnson' in df.columns:
                df['Other'] = df['Johnson'] + df['Other']
            # End of extra formatting
            logger.debug('Downloaded data contains %d polls', len(df.index))
            self.latest_poll_data = df
            return self.latest_poll_data
        except Exception as error:
            logger.error(traceback.format_exc())
            return None

    def save_poll_data(self, dataframe=None, filename=None, filepath=None):
        """Saves a dataframe to the designated location

        Parameters
        ----------
        dataframe: pandas dataframe
            The dataframe to be saved
        filename: string, optional
            Name of the file stored to disk
            Default: self.file_name
        filepath: string, optional
            Path to folder to save data to
            Default: self.file_path
        """
        logger.debug('Saving polls to disk')
        if filename is None:
            filename = self.file_name
        if filepath is None:
            filepath = self.file_path
        if dataframe is None:
            dataframe = self.latest_poll_data
        dataframe.to_csv(filepath + filename, index=False)

    def load_saved_poll_data(self):
        """Loads a saved dataframe csv from disk

        Returns
        -------
        pandas dataframe of the loaded data
        """

        if os.path.isfile(self.file_path + self.file_name):
            logging.debug('Loading saved polls from disk')
            df = pd.read_csv(self.file_path + self.file_name)
            self.saved_poll_data = df
            return self.saved_poll_data
        else:
            logging.error('No saved datafile found, downloading latest data')
            df = self.get_latest_poll_data()
            self.save_poll_data(df)
            self.saved_poll_data = df
            return self.saved_poll_data

    def return_new_polls(self, saved_poll_df, latest_poll_df):
        """Compares latest polls to those previously saved

        Parameters
        ----------
        saved_poll_df: pandas dataframe
            Old polls, previously scraped
        latest_poll_df: pandas dataframe
            Newest polls, which may or may not contain records not in the other

        Returns
        -------
        pandas dataframe containing the records only contained in the latest polls
        """
        # Applies a hash function to create a unqiue identifier for each poll/row in both dataframes
        latest_hash = latest_poll_df.apply(lambda x: hash(tuple(x)), axis=1)
        saved_hash = saved_poll_df.apply(lambda x: hash(tuple(x)), axis=1)

        # Returns rows from the latest poll dataframe whose hash value isn't in the last saved file
        new_polls_df = latest_poll_df[latest_hash.isin(saved_hash).apply(lambda x: not x)]

        if len(new_polls_df.index) > 0:
            logging.info('Hash comparison indicates %d new polls.', len(new_polls_df.index))
            new_polls_df['Start Date'] = pd.to_datetime(new_polls_df['Start Date'])
            new_polls_df['End Date'] = pd.to_datetime(new_polls_df['End Date'])
            new_polls_df = new_polls_df.fillna(0)
            # This prevents the posting of any poll with an end date older than a week ago
            lastweek = date.today() - timedelta(7)
            new_polls_df = new_polls_df[new_polls_df['End Date'] >= lastweek]
            logging.info('%d polls found within the last 7 days', len(new_polls_df.index))

            if len(new_polls_df.index) == 0:
                self.new_poll_data = None
                return None

            self.new_poll_data = new_polls_df
            return self.new_poll_data
        else:
            self.new_poll_data = None
            return None
<commit_msg>Tweak: additional logging statement when new polls are found<commit_after>import pandas as pd
import logging
import traceback
import os.path

from datetime import date, timedelta

pd.options.mode.chained_assignment = None  # Turns off Pandas copy warning

logger = logging.getLogger(__name__)

class PollIO(object):

    def __init__(self, state, poll_data_url, file_path, file_name):
        self.state = state
        self.poll_data_url = poll_data_url
        self.file_path = file_path
        self.file_name = file_name

        self.latest_poll_data = self.get_latest_poll_data()
        self.saved_poll_data = self.load_saved_poll_data()
        self.new_poll_data = self.return_new_polls(self.saved_poll_data, self.latest_poll_data)

    def get_latest_poll_data(self, csv_url=None):
        """Loads the latest polls from Huffington Post Pollster

        Parameters
        ----------
        csv_url: string, optional
            The url to the longform csv to be loaded
            Default: self.poll_data_url

        Returns
        -------
        pandas dataframe of the loaded data
        """
        try:
            logger.debug('Downloading latest %s polls from web address', self.state)
            if csv_url is None:
                csv_url = self.poll_data_url
            df = pd.read_csv(csv_url)
            # Extra formatting for state polls
            df['State'] = self.state
            if 'Other' not in df.columns:
                df['Other'] = 0.0
            if 'Johnson' in df.columns:
                df['Other'] = df['Johnson'] + df['Other']
            # End of extra formatting
            logger.debug('Downloaded data contains %d polls', len(df.index))
            self.latest_poll_data = df
            return self.latest_poll_data
        except Exception as error:
            logger.error(traceback.format_exc())
            return None

    def save_poll_data(self, dataframe=None, filename=None, filepath=None):
        """Saves a dataframe to the designated location

        Parameters
        ----------
        dataframe: pandas dataframe
            The dataframe to be saved
        filename: string, optional
            Name of the file stored to disk
            Default: self.file_name
        filepath: string, optional
            Path to folder to save data to
            Default: self.file_path
        """
        logger.debug('Saving polls to disk')
        if filename is None:
            filename = self.file_name
        if filepath is None:
            filepath = self.file_path
        if dataframe is None:
            dataframe = self.latest_poll_data
        dataframe.to_csv(filepath + filename, index=False)

    def load_saved_poll_data(self):
        """Loads a saved dataframe csv from disk

        Returns
        -------
        pandas dataframe of the loaded data
        """

        if os.path.isfile(self.file_path + self.file_name):
            logging.debug('Loading saved polls from disk')
            df = pd.read_csv(self.file_path + self.file_name)
            self.saved_poll_data = df
            return self.saved_poll_data
        else:
            logging.error('No saved datafile found, downloading latest data')
            df = self.get_latest_poll_data()
            self.save_poll_data(df)
            self.saved_poll_data = df
            return self.saved_poll_data

    def return_new_polls(self, saved_poll_df, latest_poll_df):
        """Compares latest polls to those previously saved

        Parameters
        ----------
        saved_poll_df: pandas dataframe
            Old polls, previously scraped
        latest_poll_df: pandas dataframe
            Newest polls, which may or may not contain records not in the other

        Returns
        -------
        pandas dataframe containing the records only contained in the latest polls
        """
        # Applies a hash function to create a unqiue identifier for each poll/row in both dataframes
        latest_hash = latest_poll_df.apply(lambda x: hash(tuple(x)), axis=1)
        saved_hash = saved_poll_df.apply(lambda x: hash(tuple(x)), axis=1)

        # Returns rows from the latest poll dataframe whose hash value isn't in the last saved file
        new_polls_df = latest_poll_df[latest_hash.isin(saved_hash).apply(lambda x: not x)]

        if len(new_polls_df.index) > 0:
            logging.info('Possible new polls found for %s', self.state)
            logging.info('Hash comparison indicates %d new polls.', len(new_polls_df.index))
            new_polls_df['Start Date'] = pd.to_datetime(new_polls_df['Start Date'])
            new_polls_df['End Date'] = pd.to_datetime(new_polls_df['End Date'])
            new_polls_df = new_polls_df.fillna(0)
            # This prevents the posting of any poll with an end date older than a week ago
            lastweek = date.today() - timedelta(7)
            new_polls_df = new_polls_df[new_polls_df['End Date'] >= lastweek]
            logging.info('%d polls found within the last 7 days', len(new_polls_df.index))

            if len(new_polls_df.index) == 0:
                self.new_poll_data = None
                return None

            self.new_poll_data = new_polls_df
            return self.new_poll_data
        else:
            self.new_poll_data = None
            return None
<|endoftext|>


<commit_before>#! /usr/bin/python3
try:
    import tables
except ImportError:
    print('Error: PyTables must be installed')
    sys.exit(1)

with tables.open_file('save.hdf','r') as save_file:
    # To print information about the save file, just type the name of
    # its variable
    save_file

    # The data is saved in the save file with the Table format. Each
    # Row in the Table represents one time step. Each Row further
    # consists of a number of Columns where the data is stored. The
    # Columns can be of arbitrary shape - thus, the entire 2-D
    # sensitivity array is saved in one Column on each time step (i.e.
    # in each row).
    #
    # The format of the save file is hierarchical. The Table with each
    # time step is stored in a Group, which is stored in the Root. It
    # can be thought of as nested directories, with the Root as the top
    # directory, then the Group, then the Table, like so:
    #   Root
    #       |-Group
    #             |-Table
    #
    # To access the information in the Table, it should be stored in a
    # variable for quick access. The name of the Group in the save
    # files from CanSen is ``reactor``.
    table = save_file.root.reactor

    # The Table can now be used like any other class instance. In
    # particular, the Table class defines a number of useful functions
    # and attributes, such as ``nrows``, which prints the number of
    # rows in the Table.
    table.nrows

    # PyTables provides a method to iterate over the rows in a table
    # automatically, called ``iterrows``. Here we introduce one way to
    # access information in a particular Column in the Table, by using
    # natural name indexing. In this case, we print the value of the
    # time at each time step.
    for row in table.iterrows():
        print(row['time'])

    # Note that numerical indexing is also supported. The following is
    # equivalent to the above:
    for row in table.iterrows():
        print(row[0])

    # The information stored by CanSen is written into columns named:
    #   0. time
    #   1. temperature
    #   2. pressure
    #   3. volume
    #   4. massfractions
    #   5. sensitivity
    # Columns 0-3 have a single value in each row. Column 4
    # (massfractions) contains a vector with length of the number of
    # species in the mechanism. Column 5 is optional and included only
    # if the user requested sensitivity analysis during the simulation.
    # The dimensions of Column 5 are (n_vars, n_sensitivity_params).
    #
    # In addition to the method of iterating through Rows, entire
    # Columns can be accessed and stored in variables. First, all of
    # the Columns can be stored in a variable.
    all_cols = table.cols

    # In this method, different Columns are accessed by their numerical
    # index. The first index to ``all_cols`` gives the row and the
    # second index gives the column number. Remembering that Python is
    # zero-based, to access the mass fractions on the 4th time step, do
    mass_fracs_4 = all_cols[3][4]

    # Individual columns can be stored in variables as well.<commit_msg>Finish postprocessor example. Work on #10<commit_after>#! /usr/bin/python3
try:
    import tables
except ImportError:
    print('Error: PyTables must be installed')
    sys.exit(1)

try:
    import cantera as ct
except ImportError:
    print('Error: Cantera must be installed')
    sys.exit(1)

# Use a ``with`` statmement to ensure that the file is closed when the
# code completes, even if there is an error.
with tables.open_file('save.hdf','r') as save_file:
    # To print information about the save file, just type the name of
    # its variable
    save_file

    # The data is saved in the save file with the Table format. Each
    # Row in the Table represents one time step. Each Row further
    # consists of a number of Columns where the data is stored. The
    # Columns can be of arbitrary shape - thus, the entire 2-D
    # sensitivity array is saved in one Column on each time step (i.e.
    # in each row).
    #
    # The format of the save file is hierarchical. The Table with each
    # time step is stored in a Group, which is stored in the Root. It
    # can be thought of as nested directories, with the Root as the top
    # directory, then the Group, then the Table, like so:
    #   Root
    #       |-Group
    #             |-Table
    #
    # To access the information in the Table, it should be stored in a
    # variable for quick access. The name of the Group in the save
    # files from CanSen is ``reactor``.
    table = save_file.root.reactor

    # The Table can now be used like any other class instance. In
    # particular, the Table class defines a number of useful functions
    # and attributes, such as ``nrows``, which prints the number of
    # rows in the Table.
    table.nrows

    # PyTables provides a method to iterate over the rows in a table
    # automatically, called ``iterrows``. Here we introduce one way to
    # access information in a particular Column in the Table, by using
    # natural name indexing. In this case, we print the value of the
    # time at each time step.
    for row in table.iterrows():
        print(row['time'])

    # Note that numerical indexing is also supported. The following is
    # equivalent to the above:
    for row in table.iterrows():
        print(row[0])

    # The information stored by CanSen is written into columns named:
    #   0. time
    #   1. temperature
    #   2. pressure
    #   3. volume
    #   4. massfractions
    #   5. sensitivity
    # Columns 0-3 have a single value in each row. Column 4
    # (massfractions) contains a vector with length of the number of
    # species in the mechanism. Column 5 is optional and included only
    # if the user requested sensitivity analysis during the simulation.
    # The dimensions of Column 5 are (n_vars, n_sensitivity_params).
    #
    # In addition to the method of iterating through Rows, entire
    # Columns can be accessed and stored in variables. First, all of
    # the Columns can be stored in a variable.
    all_cols = table.cols

    # In this method, different Columns are accessed by their numerical
    # index. The first index to ``all_cols`` gives the row and the
    # second index gives the column number. Remembering that Python is
    # zero-based, to access the mass fractions on the 4th time step, do
    mass_fracs_4 = all_cols[3][4]

    # Individual Columns can be stored in variables as well. This is
    # done by the natural naming scheme.
    all_mass_fracs = table.cols.massfractions

    # This stores an instance of the Column class in ``all_mass_fracs``.
    # It may be more useful to store the data in a particular column in
    # a variable. To do that, get a slice of the column by using the
    # index and the colon operator. For instance, to store all of the
    # mass fraction data in a variable
    all_mass_fracs = table.cols.massfractions[:]

    # Or, to store the fifth through tenth time steps
    mass_fracs_5_10 = table.cols.massfractions[4:9]

    # Or, to store every other time step from the sixth through the
    # 20th
    mass_fracs_alt = table.cols.massfractions[5:19:2]

    # Once the data has been extracted from the save file, we need to
    # actually be able to do something with it. Fortunately, Cantera
    # offers a simple way to do this, simply by initializing a
    # Solution to the desired conditions.
    gas = ct.Solution('mech.xml')
    for row in table.iterrows():
        gas.TPY = row['temperature'], row['pressure'], row['massfractions']
        print(gas.creation_rates)

    # This will print the creation rates of each species at each time
    # step. Any method or parameter supported by the Solution class
    # can be used to retrieve data at any given time step.
    #
    # Further information about the PyTables package can be found at
    # http://pytables.github.io/usersguide/index.html and information
    # about Cantera can be found at
    # http://cantera.github.io/docs/sphinx/html/index.html<|endoftext|>


<commit_before># Create your views here.
from django.shortcuts import render, get_object_or_404
from django.contrib.syndication.views import Feed
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import Http404
from models import Post, Tag

def by_slug(request, slug=''):
    post = get_object_or_404(Post.objects.prefetch_related('tags'), slug=slug)
    editable = request.user.is_authenticated() and request.user.is_staff
    if not (post.published or editable):
        raise Http404
    if request.method == 'POST' and editable:
        form = PostForm()
    return render(request, 'post.html', { 'post': post,
                                          'editable':editable,
                                          'title':post.title,
                                          'mathjax':True })

def tag(request, slug='', page=0):
    postList = Post.objects.prefetch_related('tags').filter(published=True).order_by('-datePosted')
    if page is None:
        page = 1
    if slug is None:
        title = 'archive'
    else:
        tag = get_object_or_404(Tag, slug=slug)
        title = 'posts tagged ' + tag.name
        postList = postList.filter(tags__slug=slug)
    paginator = Paginator(postList, 10)
    posts = paginator.page(page)
    if len(posts) == 0:
        raise Http404
    return render(request, 'tag.html', { 'posts':posts,
                                         'title':title })

def archive(request, page=0):
    return tag(request, slug=None, page=page)

class RssFeed(Feed):
    title = "benkuhn.net"
    link = "/archive/1/"
    feed_url = "/rss/"
    author_name = "Ben Kuhn"
    description = "New posts on benkuhn.net."
    description_template = "rsspost.html"
    ttl = 5

    def items(self):
        return Post.objects.filter(published=True).prefetch_related('tags').order_by('-datePosted')[:10]

    def item_title(self, post):
        return post.title

    def item_description(self, post):
        return post.excerpt

    def item_categories(self, post):
        return [tag.name for tag in post.tags.all()]

rss = RssFeed()

def queue(request):
    if not request.user.is_authenticated() and request.user.is_staff:
        raise Http404
    posts = Post.objects.filter(published=False)
    return render(request, 'queue.html', { 'posts':posts, 'title':'queue' })
<commit_msg>separate GUIDs from permalinks so titles can be scrubbed in case of accidentally publishing a post<commit_after># Create your views here.
from django.shortcuts import render, get_object_or_404
from django.contrib.syndication.views import Feed
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import Http404
from models import Post, Tag
from django.db.models import Q

def by_slug(request, slug=''):
    post = get_object_or_404(Post.objects.prefetch_related('tags'), slug=slug)
    editable = request.user.is_authenticated() and request.user.is_staff
    if not (post.published or editable):
        raise Http404
    if request.method == 'POST' and editable:
        form = PostForm()
    return render(request, 'post.html', { 'post': post,
                                          'editable':editable,
                                          'title':post.title,
                                          'mathjax':True })

def tag(request, slug='', page=0):
    postList = Post.objects.prefetch_related('tags').filter(published=True).order_by('-datePosted')
    if page is None:
        page = 1
    if slug is None:
        title = 'archive'
    else:
        tag = get_object_or_404(Tag, slug=slug)
        title = 'posts tagged ' + tag.name
        postList = postList.filter(tags__slug=slug)
    paginator = Paginator(postList, 10)
    posts = paginator.page(page)
    if len(posts) == 0:
        raise Http404
    return render(request, 'tag.html', { 'posts':posts,
                                         'title':title })

def archive(request, page=0):
    return tag(request, slug=None, page=page)

class RssFeed(Feed):
    title = "benkuhn.net"
    link = "/archive/1/"
    feed_url = "/rss/"
    author_name = "Ben Kuhn"
    description = "New posts on benkuhn.net."
    description_template = "rsspost.html"
    ttl = 5

    def items(self):
        return Post.objects.filter(published=True).prefetch_related('tags').order_by('-datePosted')[:10]

    def item_title(self, post):
        return post.title

    def item_description(self, post):
        return post.excerpt

    item_guid_is_permalink = False
    def item_guid(self, post):
        return md5.new(post.id).hexdigest()

    def item_categories(self, post):
        return [tag.name for tag in post.tags.all()]

rss = RssFeed()

def queue(request):
    if not request.user.is_authenticated() and request.user.is_staff:
        raise Http404
    posts = Post.objects.filter(published=False)
    return render(request, 'queue.html', { 'posts':posts, 'title':'queue' })
<|endoftext|>


<commit_before># ##### BEGIN MIT LICENSE BLOCK #####
#
# Copyright (c) 2015 - 2017 Pixar
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
#
# ##### END MIT LICENSE BLOCK #####

import bpy
import sys
import os
from bpy.types import AddonPreferences
from bpy.props import CollectionProperty, BoolProperty, StringProperty, FloatProperty
from bpy.props import IntProperty, PointerProperty, EnumProperty, FloatVectorProperty

from .rman_utils import filepath_utils
from . import rfb_logger
from . import rfb_icons

from .presets.properties import RendermanPresetCategory

class RendermanPreferencePath(bpy.types.PropertyGroup):
    name: StringProperty(name="", subtype='DIR_PATH')

class RendermanEnvVarSettings(bpy.types.PropertyGroup):
    if sys.platform == ("win32"):
        outpath = os.path.join(
            "C:", "Users", os.getlogin(), "Documents", "PRMan")
        out: StringProperty(
            name="OUT (Output Root)",
            description="Default RIB export path root",
            subtype='DIR_PATH',
            default='C:/tmp/renderman_for_blender/{blend}')

    else:
        outpath = os.path.join(os.environ.get('HOME'), "Documents", "PRMan")
        out: StringProperty(
            name="OUT (Output Root)",
            description="Default RIB export path root",
            subtype='DIR_PATH',
            default='/tmp/renderman_for_blender/{blend}')

class RendermanPreferences(AddonPreferences):
    bl_idname = __package__

    # find the renderman options installed
    def find_installed_rendermans(self, context):
        options = [('NEWEST', 'Newest Version Installed',
                    'Automatically updates when new version installed. NB: If an RMANTREE environment variable is set, this will always take precedence.')]
        for vers, path in filepath_utils.get_installed_rendermans():
            options.append((path, vers, path))
        return options

    rmantree_choice: EnumProperty(
        name='RenderMan Version to use',
        description='Leaving as "Newest" will automatically update when you install a new RenderMan version',
        # default='NEWEST',
        items=find_installed_rendermans
    )

    rmantree_method: EnumProperty(
        name='RenderMan Location',
        description='''How RenderMan should be detected.  Most users should leave to "Detect".
                    Users should restart Blender after making a change.
                    ''',
        items=[('ENV', 'Get From RMANTREE Environment Variable',
                'This will use the RMANTREE set in the enviornment variables'),
                ('DETECT', 'Choose From Installed',
                '''This will scan for installed RenderMan locations to choose from.'''),
                ('MANUAL', 'Set Manually', 'Manually set the RenderMan installation (for expert users)')],
        default='ENV')

    path_rmantree: StringProperty(
        name="RMANTREE Path",
        description="Path to RenderMan Pro Server installation folder",
        subtype='DIR_PATH',
        default='')

    draw_ipr_text: BoolProperty(
        name="Draw IPR Text",
        description="Draw notice on View3D when IPR is active",
        default=True)

    draw_panel_icon: BoolProperty(
        name="Draw Panel Icon",
        description="Draw an icon on RenderMan Panels",
        default=True)

    path_fallback_textures_path: StringProperty(
        name="Fallback Texture Path",
        description="Fallback path for textures, when the current directory is not writable",
        subtype='FILE_PATH',
        default=os.path.join('{OUT}', 'textures'))

    env_vars: PointerProperty(
        type=RendermanEnvVarSettings,
        name="Environment Variable Settings")

    auto_check_update: bpy.props.BoolProperty(
        name = "Auto-check for Update",
        description = "If enabled, auto-check for updates using an interval",
        default = True,
        )

    def update_rman_logging_level(self, context):
        level = rfb_logger.__LOG_LEVELS__[self.rman_logging_level]
        rfb_logger.set_logger_level(level)

    rman_logging_level: EnumProperty(
        name='Logging Level',
        description='''Log level verbosity. Advanced: Setting the RFB_LOG_LEVEL environment variable will override this preference. Requires a restart.
                    ''',
        items=[('CRITICAL', 'Critical', ''),
                ('ERROR', 'Error', ''),
                ('WARNING', 'Warning', ''),
                ('INFO', 'Info', ''),
                ('VERBOSE', 'Verbose', ''),
                ('DEBUG', 'Debug', ''),
        ],
        default='WARNING',
        update=update_rman_logging_level)

    rman_logging_file: StringProperty(
        name='Logging File',
        description='''A file to write logging to. This will always write at DEBUG level. Setting the RFB_LOG_FILE environment variable will override this preference. Requires a restart.''',
        default = '',
        subtype='FILE_PATH'
    )

    rman_do_preview_renders: BoolProperty(
        name="Render Previews",
        description="Enable rendering of material previews. This is considered a WIP.",
        default=False)

    rman_viewport_crop_color: FloatVectorProperty(
        name="CropWindow Color",
        description="Color of the cropwindow border in the viewport when in IPR.",
        default=(0.0, 0.498, 1.0, 1.0),
        size=4,
        subtype="COLOR")

    rman_viewport_refresh_rate: FloatProperty(
        name="Viewport Refresh Rate",
        description="The number of seconds to wait before the viewport refreshes during IPR.",
        default=0.01,
        precision=4,
        min=0.0001,
        max=0.1
    )

    rman_editor: StringProperty(
        name="Editor",
        subtype='FILE_PATH',
        description="Text editor excutable you want to use to view RIB.",
        default=""
    )

    rman_do_cycles_convert: BoolProperty(
        name="Convert Cycles Nodes",
        default=False,
        description="Add convert Cycles Networks buttons to the material properties panel. N.B.: This isn't guaranteed to fully convert Cycles networks successfully. Also, because of differences in OSL implementations, converted networks may cause stability problems when rendering."

    )

    rman_render_nurbs_as_mesh: BoolProperty(
        name="NURBS as Mesh",
        default=False,
        description="Render all NURBS surfaces as meshes."
    )

    rman_emit_default_params: BoolProperty(
        name="Emit Default Params",
        default=False,
        description="Controls whether or not parameters that are not changed from their defaults should be emitted to RenderMan. Turning this on is only useful for debugging purposes."
    )

    presets_current_category: PointerProperty(
        type=RendermanPresetCategory,
    )
    presets_root_category: PointerProperty(
        type=RendermanPresetCategory,
    )
    presets_current_category_path: StringProperty(default='')
    presets_show_large_icons: BoolProperty(
        name="Show Large Icons",
        description="Turns this off if you do not want to see the large version of the preset's icon",
        default=True
    )
    presets_show_subcategories: BoolProperty(
        name="Show Subcategories",
        description="By default, we only show presets in the current category. Turn this on if you want to also show subcategories (can be slow if there are large number of presets).",
        default=False
    )

    def draw(self, context):
        self.layout.use_property_split = True
        self.layout.use_property_decorate = False
        layout = self.layout

        rman_r_icon = rfb_icons.get_icon("rman_blender")

        row = layout.row()
        row.use_property_split = False
        col = row.column()
        col.prop(self, 'rmantree_method')
        if self.rmantree_method == 'DETECT':
            col.prop(self, 'rmantree_choice')
            if self.rmantree_choice == 'NEWEST':
                col.label(text="RMANTREE: %s " % filepath_utils.guess_rmantree())
        elif self.rmantree_method == 'ENV':
            col.label(text="RMANTREE: %s" % filepath_utils.rmantree_from_env())
        else:
            col.prop(self, "path_rmantree")
        if filepath_utils.guess_rmantree() is None:
            row = layout.row()
            row.alert = True
            row.label(text='Error in RMANTREE. Reload addon to reset.', icon='ERROR')

        # Behavior Prefs
        row = layout.row()
        row.label(text='Behavior', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_do_preview_renders')
        col.prop(self, 'rman_viewport_crop_color')
        col.prop(self, 'rman_render_nurbs_as_mesh')
        col.prop(self, 'rman_do_cycles_convert')
        col.prop(self, 'rman_emit_default_params')

        # Workspace
        env = self.env_vars
        row = layout.row()
        row.label(text='Workspace', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(env, "out")
        col.prop(self, 'path_fallback_textures_path')

        # UI Prefs
        row = layout.row()
        row.label(text='UI', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'draw_ipr_text')
        col.prop(self, 'draw_panel_icon')
        col.prop(self, 'rman_editor')

        # Preset Browser
        row = layout.row()
        row.label(text='Preset Browser', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        if self.presets_root_category:
            col.label(text='Path: %s' % self.presets_root_category.path)
        col.prop(self, 'presets_show_large_icons')
        col.prop(self, 'presets_show_subcategories')

        # Logging
        row = layout.row()
        row.label(text='Logging', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_logging_level')
        col.prop(self, 'rman_logging_file')

        row = layout.row()
        row.label(text='Advanced', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_viewport_refresh_rate')

def register():
    bpy.utils.register_class(RendermanPreferencePath)
    bpy.utils.register_class(RendermanEnvVarSettings)
    bpy.utils.register_class(RendermanPreferences)

def unregister():
    bpy.utils.unregister_class(RendermanPreferences)
    bpy.utils.unregister_class(RendermanEnvVarSettings)
    bpy.utils.unregister_class(RendermanPreferencePath)
<commit_msg>Add an extra bit of precision for the viewport refresh rate pref.<commit_after># ##### BEGIN MIT LICENSE BLOCK #####
#
# Copyright (c) 2015 - 2017 Pixar
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
#
# ##### END MIT LICENSE BLOCK #####

import bpy
import sys
import os
from bpy.types import AddonPreferences
from bpy.props import CollectionProperty, BoolProperty, StringProperty, FloatProperty
from bpy.props import IntProperty, PointerProperty, EnumProperty, FloatVectorProperty

from .rman_utils import filepath_utils
from . import rfb_logger
from . import rfb_icons

from .presets.properties import RendermanPresetCategory

class RendermanPreferencePath(bpy.types.PropertyGroup):
    name: StringProperty(name="", subtype='DIR_PATH')

class RendermanEnvVarSettings(bpy.types.PropertyGroup):
    if sys.platform == ("win32"):
        outpath = os.path.join(
            "C:", "Users", os.getlogin(), "Documents", "PRMan")
        out: StringProperty(
            name="OUT (Output Root)",
            description="Default RIB export path root",
            subtype='DIR_PATH',
            default='C:/tmp/renderman_for_blender/{blend}')

    else:
        outpath = os.path.join(os.environ.get('HOME'), "Documents", "PRMan")
        out: StringProperty(
            name="OUT (Output Root)",
            description="Default RIB export path root",
            subtype='DIR_PATH',
            default='/tmp/renderman_for_blender/{blend}')

class RendermanPreferences(AddonPreferences):
    bl_idname = __package__

    # find the renderman options installed
    def find_installed_rendermans(self, context):
        options = [('NEWEST', 'Newest Version Installed',
                    'Automatically updates when new version installed. NB: If an RMANTREE environment variable is set, this will always take precedence.')]
        for vers, path in filepath_utils.get_installed_rendermans():
            options.append((path, vers, path))
        return options

    rmantree_choice: EnumProperty(
        name='RenderMan Version to use',
        description='Leaving as "Newest" will automatically update when you install a new RenderMan version',
        # default='NEWEST',
        items=find_installed_rendermans
    )

    rmantree_method: EnumProperty(
        name='RenderMan Location',
        description='''How RenderMan should be detected.  Most users should leave to "Detect".
                    Users should restart Blender after making a change.
                    ''',
        items=[('ENV', 'Get From RMANTREE Environment Variable',
                'This will use the RMANTREE set in the enviornment variables'),
                ('DETECT', 'Choose From Installed',
                '''This will scan for installed RenderMan locations to choose from.'''),
                ('MANUAL', 'Set Manually', 'Manually set the RenderMan installation (for expert users)')],
        default='ENV')

    path_rmantree: StringProperty(
        name="RMANTREE Path",
        description="Path to RenderMan Pro Server installation folder",
        subtype='DIR_PATH',
        default='')

    draw_ipr_text: BoolProperty(
        name="Draw IPR Text",
        description="Draw notice on View3D when IPR is active",
        default=True)

    draw_panel_icon: BoolProperty(
        name="Draw Panel Icon",
        description="Draw an icon on RenderMan Panels",
        default=True)

    path_fallback_textures_path: StringProperty(
        name="Fallback Texture Path",
        description="Fallback path for textures, when the current directory is not writable",
        subtype='FILE_PATH',
        default=os.path.join('{OUT}', 'textures'))

    env_vars: PointerProperty(
        type=RendermanEnvVarSettings,
        name="Environment Variable Settings")

    auto_check_update: bpy.props.BoolProperty(
        name = "Auto-check for Update",
        description = "If enabled, auto-check for updates using an interval",
        default = True,
        )

    def update_rman_logging_level(self, context):
        level = rfb_logger.__LOG_LEVELS__[self.rman_logging_level]
        rfb_logger.set_logger_level(level)

    rman_logging_level: EnumProperty(
        name='Logging Level',
        description='''Log level verbosity. Advanced: Setting the RFB_LOG_LEVEL environment variable will override this preference. Requires a restart.
                    ''',
        items=[('CRITICAL', 'Critical', ''),
                ('ERROR', 'Error', ''),
                ('WARNING', 'Warning', ''),
                ('INFO', 'Info', ''),
                ('VERBOSE', 'Verbose', ''),
                ('DEBUG', 'Debug', ''),
        ],
        default='WARNING',
        update=update_rman_logging_level)

    rman_logging_file: StringProperty(
        name='Logging File',
        description='''A file to write logging to. This will always write at DEBUG level. Setting the RFB_LOG_FILE environment variable will override this preference. Requires a restart.''',
        default = '',
        subtype='FILE_PATH'
    )

    rman_do_preview_renders: BoolProperty(
        name="Render Previews",
        description="Enable rendering of material previews. This is considered a WIP.",
        default=False)

    rman_viewport_crop_color: FloatVectorProperty(
        name="CropWindow Color",
        description="Color of the cropwindow border in the viewport when in IPR.",
        default=(0.0, 0.498, 1.0, 1.0),
        size=4,
        subtype="COLOR")

    rman_viewport_refresh_rate: FloatProperty(
        name="Viewport Refresh Rate",
        description="The number of seconds to wait before the viewport refreshes during IPR.",
        default=0.01,
        precision=5,
        min=0.00001,
        max=0.1
    )

    rman_editor: StringProperty(
        name="Editor",
        subtype='FILE_PATH',
        description="Text editor excutable you want to use to view RIB.",
        default=""
    )

    rman_do_cycles_convert: BoolProperty(
        name="Convert Cycles Nodes",
        default=False,
        description="Add convert Cycles Networks buttons to the material properties panel. N.B.: This isn't guaranteed to fully convert Cycles networks successfully. Also, because of differences in OSL implementations, converted networks may cause stability problems when rendering."

    )

    rman_render_nurbs_as_mesh: BoolProperty(
        name="NURBS as Mesh",
        default=False,
        description="Render all NURBS surfaces as meshes."
    )

    rman_emit_default_params: BoolProperty(
        name="Emit Default Params",
        default=False,
        description="Controls whether or not parameters that are not changed from their defaults should be emitted to RenderMan. Turning this on is only useful for debugging purposes."
    )

    presets_current_category: PointerProperty(
        type=RendermanPresetCategory,
    )
    presets_root_category: PointerProperty(
        type=RendermanPresetCategory,
    )
    presets_current_category_path: StringProperty(default='')
    presets_show_large_icons: BoolProperty(
        name="Show Large Icons",
        description="Turns this off if you do not want to see the large version of the preset's icon",
        default=True
    )
    presets_show_subcategories: BoolProperty(
        name="Show Subcategories",
        description="By default, we only show presets in the current category. Turn this on if you want to also show subcategories (can be slow if there are large number of presets).",
        default=False
    )

    def draw(self, context):
        self.layout.use_property_split = True
        self.layout.use_property_decorate = False
        layout = self.layout

        rman_r_icon = rfb_icons.get_icon("rman_blender")

        row = layout.row()
        row.use_property_split = False
        col = row.column()
        col.prop(self, 'rmantree_method')
        if self.rmantree_method == 'DETECT':
            col.prop(self, 'rmantree_choice')
            if self.rmantree_choice == 'NEWEST':
                col.label(text="RMANTREE: %s " % filepath_utils.guess_rmantree())
        elif self.rmantree_method == 'ENV':
            col.label(text="RMANTREE: %s" % filepath_utils.rmantree_from_env())
        else:
            col.prop(self, "path_rmantree")
        if filepath_utils.guess_rmantree() is None:
            row = layout.row()
            row.alert = True
            row.label(text='Error in RMANTREE. Reload addon to reset.', icon='ERROR')

        # Behavior Prefs
        row = layout.row()
        row.label(text='Behavior', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_do_preview_renders')
        col.prop(self, 'rman_viewport_crop_color')
        col.prop(self, 'rman_render_nurbs_as_mesh')
        col.prop(self, 'rman_do_cycles_convert')
        col.prop(self, 'rman_emit_default_params')

        # Workspace
        env = self.env_vars
        row = layout.row()
        row.label(text='Workspace', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(env, "out")
        col.prop(self, 'path_fallback_textures_path')

        # UI Prefs
        row = layout.row()
        row.label(text='UI', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'draw_ipr_text')
        col.prop(self, 'draw_panel_icon')
        col.prop(self, 'rman_editor')

        # Preset Browser
        row = layout.row()
        row.label(text='Preset Browser', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        if self.presets_root_category:
            col.label(text='Path: %s' % self.presets_root_category.path)
        col.prop(self, 'presets_show_large_icons')
        col.prop(self, 'presets_show_subcategories')

        # Logging
        row = layout.row()
        row.label(text='Logging', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_logging_level')
        col.prop(self, 'rman_logging_file')

        row = layout.row()
        row.label(text='Advanced', icon_value=rman_r_icon.icon_id)
        row = layout.row()
        col = row.column()
        col.prop(self, 'rman_viewport_refresh_rate')

def register():
    bpy.utils.register_class(RendermanPreferencePath)
    bpy.utils.register_class(RendermanEnvVarSettings)
    bpy.utils.register_class(RendermanPreferences)

def unregister():
    bpy.utils.unregister_class(RendermanPreferences)
    bpy.utils.unregister_class(RendermanEnvVarSettings)
    bpy.utils.unregister_class(RendermanPreferencePath)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
MIT License

Copyright (c) 2016 Jon Sellars

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

Please see landsat-util for their license.

https://github.com/developmentseed/landsat-util

Options parser code
***************************************
# MBUtil: a tool for MBTiles files
# Supports importing, exporting, and more
#
# (c) Development Seed 2012
# (c) 2016 ePi Rational, Inc.
# Licensed under BSD
***************************************

'''

import os,sys
from optparse import OptionParser
from xml.dom.minidom import parse
def init_vm():
	cmd = '''
sudo apt-get update
sudo apt-get -y install imagemagick geotiff-bin enblend exiftool s3cmd gdal-bin python-pip python-gdal python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev parallel
sudo pip install Cython
sudo pip install landsat-util
'''
	fout = open('init.sh','w')
	fout.write(cmd)
	fout.close()
	os.system('bash init.sh')

def parseInfo(anXML):
	histInfo = parse(anXML)
	pam = histInfo.getElementsByTagName("PAMRasterBand")
	histMean = 9999

	for pd in pam[0:2]:
		md = pd.getElementsByTagName("Metadata")
		MDIS = md[0].getElementsByTagName("MDI")
		thisMean = int(float(MDIS[1].firstChild.data))
		if thisMean < histMean:
			histMean = thisMean
			histStd = int(float(MDIS[3].firstChild.data))
	histMean = float(histMean)
	return int((histMean/255)*100)

def process(sceneID,procDir):
	args = {}
	args['sceneID'] = sceneID
	args['procDir'] = procDir
	# Download scene
	# No sharpen
	#downloadCmd = 'landsat download %(sceneID)s -b 654 -p ' % args
	# Pansharpen
	downloadCmd = 'landsat download %(sceneID)s -b 654 --pansharpen -p ' % args
	#downloadCmd = 'landsat download %(sceneID)s -b 432 -p ' % args
	os.system(downloadCmd)

	if not os.path.isdir('temp'):
		os.system('mkdir temp')
	if not os.path.isdir(procDir+'/mosaic'):
		os.system('mkdir %s/mosaic'%procDir)

	warpCmd = 'gdalwarp -srcnodata "0 0 0" -tr 15 15 -dstalpha  -co TFW=YES -r cubic ~/landsat/processed/%(sceneID)s/%(sceneID)s_bands_654_pan.TIF %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	os.system(warpCmd)
	#mvCmd = 'mv ~/landsat/processed/%(sceneID)s/%(sceneID)s_bands_654_pan.TIF %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	#os.system(mvCmd)
	tfwCmd = 'listgeo -tfw %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	os.system(tfwCmd)
	infoCmd = 'gdalinfo -stats %(procDir)s/mosaic/%(sceneID)s.tif' % args
	os.system(infoCmd)
	levelVal = parseInfo('%(procDir)s/mosaic/%(sceneID)s.tif.aux.xml' % args)
	args['levelVal'] = levelVal
	imCmd = 'mogrify -level %(levelVal)s%%,100%% %(procDir)s/mosaic/%(sceneID)s.tif' % args
	print imCmd
	os.system(imCmd)
	cleanUp = 'rm -r ~/landsat/downloads/%(sceneID)s* && rm -r ~/landsat/processed/%(sceneID)s* ' % args
	print cleanUp
	#os.system(cleanUp)

if __name__ == '__main__':
	parser = OptionParser(usage="""usage: %prog [options] setup/scene

	Examples:

	Prepare VM for processing:
	$ prep-mosaic.py setup

	Setup takes a while and you should see:
	Successfully installed landsat-util...
	Upon completion.

	Process a scene:
	$ prep-mosaic.py LC80110312013227LGN00

	Process a scene to specific existing directory:
	$ prep-mosaic.py --working-dir /path/to/existing/directory LC80110312013227LGN00""")

	parser.add_option('--working-dir', dest='workdir',
	    help='''Defaults to /home/ubuntu''',
	    default='/home/ubuntu')

	(options, cli) = parser.parse_args()
	options = options.__dict__
	if not os.path.isdir(options['workdir']):
		print '\nERROR: Working directory must exist.\n'
		parser.print_help()
		sys.exit(1)

	if len(cli) != 1:
	    parser.print_help()
	    sys.exit(1)

	if cli[0] == 'setup':
		init_vm()

	else:
		process(cli[0],options['workdir']) <commit_msg>added todo to fix misplaced alpha pixels from pan sharpen<commit_after>#!/usr/bin/env python
'''
MIT License

Copyright (c) 2016 Jon Sellars

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

Please see landsat-util for their license.

https://github.com/developmentseed/landsat-util

Options parser code
***************************************
# MBUtil: a tool for MBTiles files
# Supports importing, exporting, and more
#
# (c) Development Seed 2012
# (c) 2016 ePi Rational, Inc.
# Licensed under BSD
***************************************

'''

import os,sys
from optparse import OptionParser
from xml.dom.minidom import parse
def init_vm():
	cmd = '''
sudo apt-get update
sudo apt-get -y install imagemagick geotiff-bin enblend exiftool s3cmd gdal-bin python-pip python-gdal python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev parallel
sudo pip install Cython
sudo pip install landsat-util
'''
	fout = open('init.sh','w')
	fout.write(cmd)
	fout.close()
	os.system('bash init.sh')

def parseInfo(anXML):
	histInfo = parse(anXML)
	pam = histInfo.getElementsByTagName("PAMRasterBand")
	histMean = 9999

	for pd in pam[0:2]:
		md = pd.getElementsByTagName("Metadata")
		MDIS = md[0].getElementsByTagName("MDI")
		thisMean = int(float(MDIS[1].firstChild.data))
		if thisMean < histMean:
			histMean = thisMean
			histStd = int(float(MDIS[3].firstChild.data))
	histMean = float(histMean)
	return int((histMean/255)*100)

def process(sceneID,procDir):
	args = {}
	args['sceneID'] = sceneID
	args['procDir'] = procDir
	# Download scene
	# No sharpen
	#downloadCmd = 'landsat download %(sceneID)s -b 654 -p ' % args
	# Pansharpen
	downloadCmd = 'landsat download %(sceneID)s -b 654 --pansharpen -p ' % args
	#downloadCmd = 'landsat download %(sceneID)s -b 432 -p ' % args
	os.system(downloadCmd)

	if not os.path.isdir('temp'):
		os.system('mkdir temp')
	if not os.path.isdir(procDir+'/mosaic'):
		os.system('mkdir %s/mosaic'%procDir)
	#### TODO ####
	# this will need to be nearblack to get rid of misplaced alpha pixels from pansharpen
	# nearblack -of GTiff -o near.tif -setalpha LC80210392015287LGN00_bands_654_pan.TIF
	warpCmd = 'gdalwarp -srcnodata "0 0 0" -tr 15 15 -dstalpha  -co TFW=YES -r cubic ~/landsat/processed/%(sceneID)s/%(sceneID)s_bands_654_pan.TIF %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	os.system(warpCmd)
	#mvCmd = 'mv ~/landsat/processed/%(sceneID)s/%(sceneID)s_bands_654_pan.TIF %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	#os.system(mvCmd)
	tfwCmd = 'listgeo -tfw %(procDir)s/mosaic/%(sceneID)s.tif ' % args
	os.system(tfwCmd)
	infoCmd = 'gdalinfo -stats %(procDir)s/mosaic/%(sceneID)s.tif' % args
	os.system(infoCmd)
	levelVal = parseInfo('%(procDir)s/mosaic/%(sceneID)s.tif.aux.xml' % args)
	args['levelVal'] = levelVal
	imCmd = 'mogrify -level %(levelVal)s%%,100%% %(procDir)s/mosaic/%(sceneID)s.tif' % args
	print imCmd
	os.system(imCmd)
	cleanUp = 'rm -r ~/landsat/downloads/%(sceneID)s* && rm -r ~/landsat/processed/%(sceneID)s* ' % args
	print cleanUp
	#os.system(cleanUp)

if __name__ == '__main__':
	parser = OptionParser(usage="""usage: %prog [options] setup/scene

	Examples:

	Prepare VM for processing:
	$ prep-mosaic.py setup

	Setup takes a while and you should see:
	Successfully installed landsat-util...
	Upon completion.

	Process a scene:
	$ prep-mosaic.py LC80110312013227LGN00

	Process a scene to specific existing directory:
	$ prep-mosaic.py --working-dir /path/to/existing/directory LC80110312013227LGN00""")

	parser.add_option('--working-dir', dest='workdir',
	    help='''Defaults to /home/ubuntu''',
	    default='/home/ubuntu')

	(options, cli) = parser.parse_args()
	options = options.__dict__
	if not os.path.isdir(options['workdir']):
		print '\nERROR: Working directory must exist.\n'
		parser.print_help()
		sys.exit(1)

	if len(cli) != 1:
	    parser.print_help()
	    sys.exit(1)

	if cli[0] == 'setup':
		init_vm()

	else:
		process(cli[0],options['workdir']) <|endoftext|>


<commit_before>"""
This script first erases all the files in a target directory, and then
copies the necessary files to run Repy into it. Afterwards, the .mix
files in the target directory are ran through the preprocessor.
The target directory that is passed to the script must exist. It is
emptied before files are copied over.

It is assumed that you have checked out all the required repos of
SeattleTestbed into the parent directory of this script.

NOTE WELL: The repositories are used as-is. No attempt is made to switch
    to a specific branch, pull from remotes, etc.
    (In a future version of this script, the currently active branch
    for each repo will be displayed as a visual reminder of this fact.)

<Usage>
  preparetest.py  [-t] [-v] [-c] [-r] <target_directory>

    -t or --testfiles copies in all the files required to run the unit tests
    -v or --verbose displays significantly more output on failure to process
                    a mix file
    -c or --checkapi copies the checkapi source files
    -r or --randomports replaces the default ports of 12345, 12346, and 12347
                        with three random ports between 52000 and 53000.

<Example>
  Put the Repy runtime and unit test files into a temporary dir,
  and run the unit tests for module "repyv2api" there.
    user@vm:seattle$ cd dist
    user@vm:dist$ mkdir /tmp/test
    user@vm:dist$ python preparetest.py -t /tmp/test
    user@vm:dist$ cd /tmp/test
    user@vm:test$ python utf.py -m repyv2api

"""

import os
import sys
import glob
import random
import shutil
import optparse
import subprocess

# import testportfiller from path ../repy_v1/tests
sys.path.insert(0, os.path.join(os.path.dirname(os.getcwd()), "repy_v1", "tests"))
import testportfiller
# Remove testportfiller's path again
sys.path = sys.path[1:]

def copy_to_target(file_expr, target):
  """
  This function copies files (in the current directory) that match the
  expression file_expr to the target folder.
  The source files are from the current directory.
  The target directory must exist.
  file_expr may contain wildcards (shell globs).
  """
  files_to_copy = glob.glob(file_expr)
  if files_to_copy == []:
    print "WARNING: File expression '" + file_expr + "' does not match any files. Maybe the directory is empty, or the file / directory doesn't exist?"

  for file_path in files_to_copy:
    if os.path.isfile(file_path):
      shutil.copyfile(file_path, target + "/" +os.path.basename(file_path))

def copy_tree_to_target(source, target, ignore=None):
  """
  Copies a directory to the target destination.

  If you pass a string for ignore, then subdirectories that contain the ignore
  string will not be copied over (as well as the files they contain).
  """

  full_source_path = os.path.abspath(source)
  full_target_path = os.path.abspath(target)

  for root, directories, filenames in os.walk(source):
    # Relative path is needed to build the absolute target path.

    # If we leave a leading directory separator in the relative folder
    # path, then attempts to join it will cause the relative folder path
    # to be treated as an absolute path.
    relative_folder_path = os.path.abspath(root)[len(full_source_path):].lstrip(os.sep)

    # If the ignore string is in the relative path, skip this directory.
    if ignore and ignore in relative_folder_path:
      continue

    # Attempts to copy over a file when the containing directories above it do not
    # exist will trigger an exception.
    full_target_subdir_path = os.path.join(full_target_path, relative_folder_path)
    if not os.path.isdir(full_target_subdir_path):
      os.makedirs(full_target_subdir_path)

    for name in filenames:
      relative_path = os.path.join(relative_folder_path, name)
      shutil.copyfile(
        os.path.join(full_source_path, relative_path),
        os.path.join(full_target_path, relative_path))

def process_mix(script_path, verbose):
  """
  Run the .mix files in current directory through the preprocessor.
  script_path specifies the name of the preprocessor script.
  The preprocessor script must be in the working directory.
  """
  mix_files = glob.glob("*.mix")
  error_list = []

  for file_path in mix_files:
    # Generate a .py file for the .mix file specified by file_path
    processed_file_path = (os.path.basename(file_path)).replace(".mix",".py")
    (theout, theerr) =  exec_command(sys.executable + " " + script_path + " " + file_path + " " + processed_file_path)

    # If there was any problem processing the files, then notify the user.
    if theerr:
      print "Unable to process the file: " + file_path
      error_list.append((file_path, theerr))

  # If the verbose option is on then print the error.
  if verbose and len(error_list) > 0:
    print "\n" + '#'*50 + "\nPrinting all the exceptions (verbose option)\n" + '#'*50
    for file_name, error in error_list:
      print "\n" + file_name + ":"
      print error
      print '-'*80

def exec_command(command):
  """
  Execute command on a shell, return a tuple containing the resulting
  standard output and standard error (as strings).
  """
  # Windows does not like close_fds and we shouldn't need it so...
  process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
      stderr=subprocess.PIPE)

  # get the output and close
  theout = process.stdout.read()
  process.stdout.close()

  # get the errput and close
  theerr = process.stderr.read()
  process.stderr.close()

  # FreeBSD prints on stdout, when it gets a signal...
  # I want to look at the last line. It ends in \n, so I use index -2
  if len(theout.split('\n')) > 1 and theout.split('\n')[-2].strip() == 'Terminated':
    # remove the last line
    theout = '\n'.join(theout.split('\n')[:-2])

    # However we threw away an extra '\n'. If anything remains, let's replace it
    if theout != '':
      theout = theout + '\n'

  # OS's besides FreeBSD uses stderr
  if theerr.strip() == 'Terminated':
    theerr = ''

  # Windows isn't fond of this either...
  # clean up after the child
  #os.waitpid(p.pid,0)

  return (theout, theerr)

def replace_string(old_string, new_string, file_name_pattern="*"):
  """
  <Purpose>
    Go through all the files in the current folder and replace
    every match of the old string in the file with the new
    string.

  <Arguments>
    old_string - The string we want to replace.

    new_string - The new string we want to replace the old string
      with.

    file_name_pattern - The pattern of the file name if you want
      to reduce the number of files we look at. By default the
      function looks at all files.

  <Exceptions>
    None.

  <Side Effects>
    Many files may get modified.

  <Return>
    None
  """

  for testfile in glob.glob(file_name_pattern):
    # Read in the initial file.
    inFile = file(testfile, 'r')
    filestring = inFile.read()
    inFile.close()

    # Replace any form of the matched old string with
    # the new string.
    filestring = filestring.replace(old_string, new_string)

    # Write the file back.
    outFile = file(testfile, 'w')
    outFile.write(filestring)
    outFile.close()

def help_exit(errMsg, parser):
  """
   Prints the given error message and the help string, then exits
  """
  print errMsg
  parser.print_help()
  sys.exit(1)

def main():
  # Parse the options provided.
  helpstring = "python preparetest.py [-t] [-v] [-c] [-r] <target>"
  parser = optparse.OptionParser(usage=helpstring)

  parser.add_option("-t", "--testfiles", action="store_true",
      dest="include_tests", default=False,
      help="Include files required to run the unit tests ")
  parser.add_option("-v", "--verbose", action="store_true",
      dest="verbose", default=False,
      help="Show more output on failure to process a .mix file")
  parser.add_option("-c", "--checkapi", action="store_true",
      dest="copy_checkapi", default=False,
      help="Include checkAPI files")
  parser.add_option("-r", "--randomports", action="store_true",
      dest="randomports", default=False,
      help="Replace the default ports with random ports between 52000 and 53000. ")

  (options, args) = parser.parse_args()

  # Extract the target directory.
  if len(args) == 0:
    help_exit("Please pass the target directory as a parameter.", parser)
  else:
    target_dir = args[0]

  # Make sure they gave us a valid directory
  if not os.path.isdir(target_dir):
    help_exit("Supplied target is not a directory", parser)

  # Set variables according to the provided options.
  repytest = options.include_tests
  RANDOMPORTS = options.randomports
  verbose = options.verbose
  copy_checkapi = options.copy_checkapi

  # This script's parent directory is the root dir of all repositories
  repos_root_dir = os.path.dirname(os.getcwd())

  # Set working directory to the target
  os.chdir(target_dir)
  files_to_remove = glob.glob("*")

  # Empty the destination
  for entry in files_to_remove:
    if os.path.isdir(entry):
      shutil.rmtree(entry)
    else:
      os.remove(entry)

  # Create directories for each Repy version under the target
  repy_dir = {"v1" : os.path.join(target_dir, "repyV1"),
      "v2" : os.path.join(target_dir, "repyV2") }

  for dir_name in repy_dir.values():
    if not os.path.exists(dir_name):
      os.makedirs(dir_name)

  # Return to the repo root
  os.chdir(repos_root_dir)

  # Copy the necessary files to the respective target folders:
  # Affix framework and components
  copy_to_target("affix-repo/affix/*", target_dir)
  copy_to_target("affix-repo/affix/affix_components/*", target_dir)
  copy_to_target("affix-repo/affix/affix_components/tcp_relay/*", target_dir)

  # Nodemanager and RepyV2 runtime
  copy_to_target("repy_v2/*", target_dir)
  copy_to_target("nodemanager/*", target_dir)
  copy_to_target("portability/*", target_dir)
  copy_to_target("seattlelib_v2/*", target_dir)

  # RepyV2 runtime for vessels
  copy_to_target("portability/*", repy_dir["v2"])
  copy_to_target("repy_v2/*", repy_dir["v2"])
  copy_to_target("seattlelib_v2/dylink.r2py", repy_dir["v2"])
  copy_to_target("seattlelib_v2/textops.py", repy_dir["v2"])
  copy_to_target("nodemanager/servicelogger.py", repy_dir["v2"])

  # RepyV1 runtime for vessels
  copy_to_target("repy_v1/*", repy_dir["v1"])
  copy_to_target("seattlelib_v1/textops.py", repy_dir["v1"])
  copy_to_target("repy_v2/persist.py", repy_dir["v1"])
  copy_to_target("nodemanager/servicelogger.py", repy_dir["v1"])

  # Seash
  copy_to_target("seash/*", target_dir)
  copy_tree_to_target("seash/pyreadline/", os.path.join(target_dir, 'pyreadline/'), ignore=".git")
  copy_tree_to_target("seash/modules/", os.path.join(target_dir, 'modules/'), ignore=".git")

  # Clearinghouse XML-RPC interface
  copy_to_target("clearinghouse/xmlrpc_clients/*", target_dir)

  # Software updater
  copy_to_target("softwareupdater/*", target_dir)
  copy_to_target("dist/update_crontab_entry.py", target_dir)

  # The license must be included in anything we distribute.
  copy_to_target("dist/LICENSE.TXT", target_dir)

  if repytest:
    # Only copy the tests if they were requested.
    copy_to_target("repy_v2/tests/restrictions.*", target_dir)
    copy_to_target("utf/*.py", target_dir)
    copy_to_target("utf/tests/*.py", target_dir)
    copy_to_target("repy_v2/testsV2/*", target_dir) # XXX Scheduled for merge with repy_v2/tests
    copy_to_target("nodemanager/tests/*", target_dir)
    copy_to_target("portability/tests/*", target_dir)
    copy_to_target("seash/tests/*", target_dir)
    copy_tree_to_target("seash/tests/modules/", os.path.join(target_dir, 'modules/'), ignore=".git")
    copy_to_target("seattlelib_v2/tests/*", target_dir)

    # The web server is used in the software updater tests
    #copy_to_target("assignments/webserver/*", target_dir)
    #copy_to_target("softwareupdater/test/*", target_dir)

  # Set working directory to the target
  os.chdir(target_dir)

  # Set up dynamic port information
  if RANDOMPORTS:
    print "\n[ Randomports option was chosen ]\n"+'-'*50
    ports_as_ints = random.sample(range(52000, 53000), 5)
    ports_as_strings = []
    for port in ports_as_ints:
      ports_as_strings.append(str(port))

    print "Randomly chosen ports: ", ports_as_strings
    testportfiller.replace_ports(ports_as_strings, ports_as_strings)

    # Replace the string <nodemanager_port> with a random port
    random_nodemanager_port = random.randint(53000, 54000)
    print "Chosen random nodemanager port: " + str(random_nodemanager_port)
    print '-'*50 + "\n"
    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*nm*")
    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*securitylayers*")

  else:
    # Otherwise use the default ports...
    testportfiller.replace_ports(['12345','12346','12347', '12348', '12349'], ['12345','12346','12347', '12348', '12349'])

    # Use default port 1224 for the nodemanager port if --random flag is not provided.
    replace_string("<nodemanager_port>", '1224', "*nm*")
    replace_string("<nodemanager_port>", '1224', "*securitylayers*")

  os.chdir("repyV1")
  process_mix("repypp.py", verbose)

  # Change back to root project directory
  os.chdir(repos_root_dir)

if __name__ == '__main__':
  main()
<commit_msg>Pointing preparetest.py to the new Affix directories.<commit_after>"""
This script first erases all the files in a target directory, and then
copies the necessary files to run Repy into it. Afterwards, the .mix
files in the target directory are ran through the preprocessor.
The target directory that is passed to the script must exist. It is
emptied before files are copied over.

It is assumed that you have checked out all the required repos of
SeattleTestbed into the parent directory of this script.

NOTE WELL: The repositories are used as-is. No attempt is made to switch
    to a specific branch, pull from remotes, etc.
    (In a future version of this script, the currently active branch
    for each repo will be displayed as a visual reminder of this fact.)

<Usage>
  preparetest.py  [-t] [-v] [-c] [-r] <target_directory>

    -t or --testfiles copies in all the files required to run the unit tests
    -v or --verbose displays significantly more output on failure to process
                    a mix file
    -c or --checkapi copies the checkapi source files
    -r or --randomports replaces the default ports of 12345, 12346, and 12347
                        with three random ports between 52000 and 53000.

<Example>
  Put the Repy runtime and unit test files into a temporary dir,
  and run the unit tests for module "repyv2api" there.
    user@vm:seattle$ cd dist
    user@vm:dist$ mkdir /tmp/test
    user@vm:dist$ python preparetest.py -t /tmp/test
    user@vm:dist$ cd /tmp/test
    user@vm:test$ python utf.py -m repyv2api

"""

import os
import sys
import glob
import random
import shutil
import optparse
import subprocess

# import testportfiller from path ../repy_v1/tests
sys.path.insert(0, os.path.join(os.path.dirname(os.getcwd()), "repy_v1", "tests"))
import testportfiller
# Remove testportfiller's path again
sys.path = sys.path[1:]

def copy_to_target(file_expr, target):
  """
  This function copies files (in the current directory) that match the
  expression file_expr to the target folder.
  The source files are from the current directory.
  The target directory must exist.
  file_expr may contain wildcards (shell globs).
  """
  files_to_copy = glob.glob(file_expr)
  if files_to_copy == []:
    print "WARNING: File expression '" + file_expr + "' does not match any files. Maybe the directory is empty, or the file / directory doesn't exist?"

  for file_path in files_to_copy:
    if os.path.isfile(file_path):
      shutil.copyfile(file_path, target + "/" +os.path.basename(file_path))

def copy_tree_to_target(source, target, ignore=None):
  """
  Copies a directory to the target destination.

  If you pass a string for ignore, then subdirectories that contain the ignore
  string will not be copied over (as well as the files they contain).
  """

  full_source_path = os.path.abspath(source)
  full_target_path = os.path.abspath(target)

  for root, directories, filenames in os.walk(source):
    # Relative path is needed to build the absolute target path.

    # If we leave a leading directory separator in the relative folder
    # path, then attempts to join it will cause the relative folder path
    # to be treated as an absolute path.
    relative_folder_path = os.path.abspath(root)[len(full_source_path):].lstrip(os.sep)

    # If the ignore string is in the relative path, skip this directory.
    if ignore and ignore in relative_folder_path:
      continue

    # Attempts to copy over a file when the containing directories above it do not
    # exist will trigger an exception.
    full_target_subdir_path = os.path.join(full_target_path, relative_folder_path)
    if not os.path.isdir(full_target_subdir_path):
      os.makedirs(full_target_subdir_path)

    for name in filenames:
      relative_path = os.path.join(relative_folder_path, name)
      shutil.copyfile(
        os.path.join(full_source_path, relative_path),
        os.path.join(full_target_path, relative_path))

def process_mix(script_path, verbose):
  """
  Run the .mix files in current directory through the preprocessor.
  script_path specifies the name of the preprocessor script.
  The preprocessor script must be in the working directory.
  """
  mix_files = glob.glob("*.mix")
  error_list = []

  for file_path in mix_files:
    # Generate a .py file for the .mix file specified by file_path
    processed_file_path = (os.path.basename(file_path)).replace(".mix",".py")
    (theout, theerr) =  exec_command(sys.executable + " " + script_path + " " + file_path + " " + processed_file_path)

    # If there was any problem processing the files, then notify the user.
    if theerr:
      print "Unable to process the file: " + file_path
      error_list.append((file_path, theerr))

  # If the verbose option is on then print the error.
  if verbose and len(error_list) > 0:
    print "\n" + '#'*50 + "\nPrinting all the exceptions (verbose option)\n" + '#'*50
    for file_name, error in error_list:
      print "\n" + file_name + ":"
      print error
      print '-'*80

def exec_command(command):
  """
  Execute command on a shell, return a tuple containing the resulting
  standard output and standard error (as strings).
  """
  # Windows does not like close_fds and we shouldn't need it so...
  process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
      stderr=subprocess.PIPE)

  # get the output and close
  theout = process.stdout.read()
  process.stdout.close()

  # get the errput and close
  theerr = process.stderr.read()
  process.stderr.close()

  # FreeBSD prints on stdout, when it gets a signal...
  # I want to look at the last line. It ends in \n, so I use index -2
  if len(theout.split('\n')) > 1 and theout.split('\n')[-2].strip() == 'Terminated':
    # remove the last line
    theout = '\n'.join(theout.split('\n')[:-2])

    # However we threw away an extra '\n'. If anything remains, let's replace it
    if theout != '':
      theout = theout + '\n'

  # OS's besides FreeBSD uses stderr
  if theerr.strip() == 'Terminated':
    theerr = ''

  # Windows isn't fond of this either...
  # clean up after the child
  #os.waitpid(p.pid,0)

  return (theout, theerr)

def replace_string(old_string, new_string, file_name_pattern="*"):
  """
  <Purpose>
    Go through all the files in the current folder and replace
    every match of the old string in the file with the new
    string.

  <Arguments>
    old_string - The string we want to replace.

    new_string - The new string we want to replace the old string
      with.

    file_name_pattern - The pattern of the file name if you want
      to reduce the number of files we look at. By default the
      function looks at all files.

  <Exceptions>
    None.

  <Side Effects>
    Many files may get modified.

  <Return>
    None
  """

  for testfile in glob.glob(file_name_pattern):
    # Read in the initial file.
    inFile = file(testfile, 'r')
    filestring = inFile.read()
    inFile.close()

    # Replace any form of the matched old string with
    # the new string.
    filestring = filestring.replace(old_string, new_string)

    # Write the file back.
    outFile = file(testfile, 'w')
    outFile.write(filestring)
    outFile.close()

def help_exit(errMsg, parser):
  """
   Prints the given error message and the help string, then exits
  """
  print errMsg
  parser.print_help()
  sys.exit(1)

def main():
  # Parse the options provided.
  helpstring = "python preparetest.py [-t] [-v] [-c] [-r] <target>"
  parser = optparse.OptionParser(usage=helpstring)

  parser.add_option("-t", "--testfiles", action="store_true",
      dest="include_tests", default=False,
      help="Include files required to run the unit tests ")
  parser.add_option("-v", "--verbose", action="store_true",
      dest="verbose", default=False,
      help="Show more output on failure to process a .mix file")
  parser.add_option("-c", "--checkapi", action="store_true",
      dest="copy_checkapi", default=False,
      help="Include checkAPI files")
  parser.add_option("-r", "--randomports", action="store_true",
      dest="randomports", default=False,
      help="Replace the default ports with random ports between 52000 and 53000. ")

  (options, args) = parser.parse_args()

  # Extract the target directory.
  if len(args) == 0:
    help_exit("Please pass the target directory as a parameter.", parser)
  else:
    target_dir = args[0]

  # Make sure they gave us a valid directory
  if not os.path.isdir(target_dir):
    help_exit("Supplied target is not a directory", parser)

  # Set variables according to the provided options.
  repytest = options.include_tests
  RANDOMPORTS = options.randomports
  verbose = options.verbose
  copy_checkapi = options.copy_checkapi

  # This script's parent directory is the root dir of all repositories
  repos_root_dir = os.path.dirname(os.getcwd())

  # Set working directory to the target
  os.chdir(target_dir)
  files_to_remove = glob.glob("*")

  # Empty the destination
  for entry in files_to_remove:
    if os.path.isdir(entry):
      shutil.rmtree(entry)
    else:
      os.remove(entry)

  # Create directories for each Repy version under the target
  repy_dir = {"v1" : os.path.join(target_dir, "repyV1"),
      "v2" : os.path.join(target_dir, "repyV2") }

  for dir_name in repy_dir.values():
    if not os.path.exists(dir_name):
      os.makedirs(dir_name)

  # Return to the repo root
  os.chdir(repos_root_dir)

  # Copy the necessary files to the respective target folders:
  # Affix framework and components
  copy_to_target("affix-repo/affix/*", target_dir)
  copy_to_target("affix-repo/affix/affix_components/*", target_dir)
  copy_to_target("affix-repo/affix/affix_components/tcp_relay/*", target_dir)

  # Nodemanager and RepyV2 runtime
  copy_to_target("repy_v2/*", target_dir)
  copy_to_target("nodemanager/*", target_dir)
  copy_to_target("portability/*", target_dir)
  copy_to_target("seattlelib_v2/*", target_dir)

  # RepyV2 runtime for vessels
  copy_to_target("portability/*", repy_dir["v2"])
  copy_to_target("repy_v2/*", repy_dir["v2"])
  copy_to_target("seattlelib_v2/dylink.r2py", repy_dir["v2"])
  copy_to_target("seattlelib_v2/textops.py", repy_dir["v2"])
  copy_to_target("nodemanager/servicelogger.py", repy_dir["v2"])

  # RepyV1 runtime for vessels
  copy_to_target("repy_v1/*", repy_dir["v1"])

  # Seash
  copy_to_target("seash/*", target_dir)
  copy_tree_to_target("seash/pyreadline/", os.path.join(target_dir, 'pyreadline/'), ignore=".git")
  copy_tree_to_target("seash/modules/", os.path.join(target_dir, 'modules/'), ignore=".git")

  # Clearinghouse XML-RPC interface
  copy_to_target("clearinghouse/xmlrpc_clients/*", target_dir)

  # Software updater
  copy_to_target("softwareupdater/*", target_dir)
  copy_to_target("dist/update_crontab_entry.py", target_dir)

  # The license must be included in anything we distribute.
  copy_to_target("dist/LICENSE.TXT", target_dir)

  if repytest:
    # Only copy the tests if they were requested.
    copy_to_target("repy_v2/tests/restrictions.*", target_dir)
    copy_to_target("utf/*.py", target_dir)
    copy_to_target("utf/tests/*.py", target_dir)
    copy_to_target("repy_v2/testsV2/*", target_dir) # XXX Scheduled for merge with repy_v2/tests
    copy_to_target("nodemanager/tests/*", target_dir)
    copy_to_target("portability/tests/*", target_dir)
    copy_to_target("seash/tests/*", target_dir)
    copy_tree_to_target("seash/tests/modules/", os.path.join(target_dir, 'modules/'), ignore=".git")
    copy_to_target("seattlelib_v2/tests/*", target_dir)

    # The web server is used in the software updater tests
    #copy_to_target("assignments/webserver/*", target_dir)
    #copy_to_target("softwareupdater/test/*", target_dir)

  # Set working directory to the target
  os.chdir(target_dir)

  # Set up dynamic port information
  if RANDOMPORTS:
    print "\n[ Randomports option was chosen ]\n"+'-'*50
    ports_as_ints = random.sample(range(52000, 53000), 5)
    ports_as_strings = []
    for port in ports_as_ints:
      ports_as_strings.append(str(port))

    print "Randomly chosen ports: ", ports_as_strings
    testportfiller.replace_ports(ports_as_strings, ports_as_strings)

    # Replace the string <nodemanager_port> with a random port
    random_nodemanager_port = random.randint(53000, 54000)
    print "Chosen random nodemanager port: " + str(random_nodemanager_port)
    print '-'*50 + "\n"
    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*nm*")
    replace_string("<nodemanager_port>", str(random_nodemanager_port), "*securitylayers*")

  else:
    # Otherwise use the default ports...
    testportfiller.replace_ports(['12345','12346','12347', '12348', '12349'], ['12345','12346','12347', '12348', '12349'])

    # Use default port 1224 for the nodemanager port if --random flag is not provided.
    replace_string("<nodemanager_port>", '1224', "*nm*")
    replace_string("<nodemanager_port>", '1224', "*securitylayers*")

  os.chdir("repyV1")
  process_mix("repypp.py", verbose)

  # Change back to root project directory
  os.chdir(repos_root_dir)

if __name__ == '__main__':
  main()
<|endoftext|>