<commit_before>'''
Routines to set up a minion
'''
# Import python libs
import os
import distutils.sysconfig
import glob
import re
import time
import logging
import tempfile
import traceback
import shutil
import threading
import multiprocessing

# Import zeromq libs
import zmq
# Import salt libs
import salt.crypt
from salt.crypt import AuthenticationError
import salt.utils
import salt.modules
import salt.returners
import salt.loader

log = logging.getLogger(__name__)

# To set up a minion:
# 1, Read in the configuration
# 2. Generate the function mapping dict
# 3. Authenticate with the master
# 4. Store the aes key
# 5. connect to the publisher
# 6. handle publications

class MinionError(Exception): pass

class Minion(object):
    '''
    This class instantiates a minion, runs connections for a minion, and loads
    all of the functions into the minion
    '''
    def __init__(self, opts):
        '''
        Pass in the options dict
        '''
        self.opts = opts
        self.mod_opts = self.__prep_mod_opts()
        self.functions, self.returners = self.__load_modules()
        self.matcher = Matcher(self.opts, self.functions)
        self.authenticate()

    def __prep_mod_opts(self):
        '''
        Returns a deep copy of the opts with key bits stripped out
        '''
        mod_opts = {}
        for key, val in self.opts.items():
            if key == 'logger':
                continue
            mod_opts[key] = val
        return mod_opts

    def __load_modules(self):
        '''
        Return the functions and the returners loaded up from the loader module
        '''
        functions = salt.loader.minion_mods(self.opts)
        returners = salt.loader.returners(self.opts)
        return functions, returners

    def _handle_payload(self, payload):
        '''
        Takes a payload from the master publisher and does whatever the
        master wants done.
        '''
        {'aes': self._handle_aes,
         'pub': self._handle_pub,
         'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_aes(self, load):
        '''
        Takes the aes encrypted load, decrypts is and runs the encapsulated
        instructions
        '''
        data = None
        try:
            data = self.crypticle.loads(load)
        except AuthenticationError:
            self.authenticate()
            data = self.crypticle.loads(load)
        # Verify that the publication is valid
        if not data.has_key('tgt')\
                or not data.has_key('jid')\
                or not data.has_key('fun')\
                or not data.has_key('arg'):
            return
        # Verify that the publication applies to this minion
        if data.has_key('tgt_type'):
            if not getattr(self.matcher, data['tgt_type'] + '_match')(data['tgt']):
                return
        else:
            if not self.matcher.glob_match(data['tgt']):
                return
        self._handle_decoded_payload(data)

    def _handle_pub(self, load):
        '''
        Handle public key payloads
        '''
        pass

    def _handle_clear(self, load):
        '''
        Handle un-encrypted transmissions
        '''
        pass

    def _handle_decoded_payload(self, data):
        '''
        Override this method if you wish to handle the decoded data differently.
        '''
        if self.opts['multiprocessing']:
            if type(data['fun']) == type(list()):
                multiprocessing.Process(
                    target=lambda: self._thread_multi_return(data)
                ).start()
            else:
                multiprocessing.Process(
                    target=lambda: self._thread_return(data)
                ).start()
        else:
            if type(data['fun']) == type(list()):
                threading.Thread(
                    target=lambda: self._thread_multi_return(data)
                ).start()
            else:
                threading.Thread(
                    target=lambda: self._thread_return(data)
                ).start()

    def _thread_return(self, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        ret = {}
        for ind in range(0, len(data['arg'])):
            try:
                data['arg'][ind] = eval(data['arg'][ind])
            except:
                pass

        try:
            ret['return'] = self.functions[data['fun']](*data['arg'])
        except Exception as exc:
            trb = traceback.format_exc()
            log.warning('The minion function caused an exception: %s', exc)
            ret['return'] = trb
        ret['jid'] = data['jid']
        if data['ret']:
            ret['id'] = self.opts['id']
            try:
                self.returners[data['ret']](ret)
            except Exception as exc:
                log.error('The return failed for job %s %s', data['jid'], exc)
        else:
            self._return_pub(ret)

    def _thread_multi_return(self, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        ret = {'return': {}}
        for ind in range(0, len(data['fun'])):
            for index in range(0, len(data['arg'][ind])):
                try:
                    data['arg'][ind][index] = eval(data['arg'][ind][index])
                except:
                    pass

            try:
                ret['return'][data['fun'][ind]]\
                    = self.functions[data['fun'][ind]](*data['arg'][ind])
            except Exception as exc:
                trb = traceback.format_exc()
                log.warning('The minion function caused an exception: %s', exc)
                ret['return'][data['fun'][ind]] = trb
            ret['jid'] = data['jid']
        if data['ret']:
            ret['id'] = self.opts['id']
            try:
                self.returners[data['ret']](ret)
            except Exception as exc:
                log.error('The return failed for job %s %s', data['jid'], exc)
        else:
            self._return_pub(ret)

    def _return_pub(self, ret):
        '''
        Return the data from the executed command to the master server
        '''
        log.info('Returning information for job: %(jid)s', ret)
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect(self.opts['master_uri'])
        payload = {'enc': 'aes'}
        load = {'return': ret['return'],
                'cmd': '_return',
                'jid': ret['jid'],
                'id': self.opts['id']}
        payload['load'] = self.crypticle.dumps(load)
        socket.send_pyobj(payload)
        return socket.recv()

    def reload_functions(self):
        '''
        Reload the functions dict for this minion, reading in any new functions
        '''
        self.functions = self.__load_functions()
        log.debug('Refreshed functions, loaded functions: %s', self.functions)
        return True

    def authenticate(self):
        '''
        Authenticate with the master, this method breaks the functional
        paradigm, it will update the master information from a fresh sign in,
        signing in can occur as often as needed to keep up with the revolving
        master aes key.
        '''
        log.debug('Attempting to authenticate with the Salt Master')
        auth = salt.crypt.Auth(self.opts)
        while True:
            creds = auth.sign_in()
            if creds != 'retry':
                log.info('Authentication with master successful!')
                break
            log.info('Waiting for minion key to be accepted by the master.')
            time.sleep(10)
        self.aes = creds['aes']
        self.publish_port = creds['publish_port']
        self.crypticle = salt.crypt.Crypticle(self.aes)

    def tune_in(self):
        '''
        Lock onto the publisher. This is the main event loop for the minion
        '''
        master_pub = 'tcp://' + self.opts['master'] + ':'\
                   + str(self.publish_port)
        context = zmq.Context()
        socket = context.socket(zmq.SUB)
        socket.connect(master_pub)
        socket.setsockopt(zmq.SUBSCRIBE, '')
        while True:
            payload = socket.recv_pyobj()
            self._handle_payload(payload)

class Matcher(object):
    '''
    Use to return the value for matching calls from the master
    '''
    def __init__(self, opts, functions=None):
        self.opts = opts
        if not functions:
            functions = salt.loader.minion_mods(self.opts)
        else:
            self.functions = functions

    def confirm_top(self, data):
        '''
        Takes the data passed to a top file environment and determines if the
        data matches this minion
        '''
        matcher = 'glob'
        for item in data:
            if type(item) == type(dict()):
                if item.has_key('match'):
                    matcher = item['match']
        if hasattr(self, matcher + '_match'):
            return getattr(self, matcher)
        else:
            log.error('Attempting to match with unknown matcher: %s', matcher)
            return False

    def glob_match(self, tgt):
        '''
        Returns true if the passed glob matches the id
        '''
        tmp_dir = tempfile.mkdtemp()
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        open(self.opts['id'], 'w+').write('salt')
        ret = bool(glob.glob(tgt))
        os.chdir(cwd)
        shutil.rmtree(tmp_dir)
        return ret

    def pcre_match(self, tgt):
        '''
        Returns true if the passed pcre regex matches
        '''
        return bool(re.match(tgt, self.opts['id']))

    def list_match(self, tgt):
        '''
        Determines if this host is on the list
        '''
        return bool(tgt.count(self.opts['id']))

    def grain_match(self, tgt):
        '''
        Reads in the grains regular expression match
        '''
        comps = tgt.split(':')
        return bool(re.match(comps[1], self.opts['grains'][comps[0]]))

    def exsel_match(self, tgt):
        '''
        Runs a function and return the exit code
        '''
        if not self.functions.has_key(tgt):
            return False
        return(self.functions[tgt]())

class FileClient(object):
    '''
    Interact with the salt master file server.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.auth = salt.crypt.SAuth(opts)
        self.socket = self.__get_socket()

    def __get_socket(self):
        '''
        Return the ZeroMQ socket to use
        '''
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect(self.opts['master_uri'])
        return socket

    def _check_proto(self, path):
        '''
        Make sure that this path is intended for the salt master and trim it
        '''
        if not path.startswith('salt://'):
            raise MinionError('Unsupported path')
        return path[:7]

    def get_file(self, path, dest, makedirs=False, env='base'):
        '''
        Get a single file from the salt-master
        '''
        path = self._check_proto(path)
        payload = {'enc': 'aes'}
        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            if makedirs:
                os.makedirs(destdir)
            else:
                return False
        fn_ = open(dest, 'w+')
        load = {'path': path,
                'env': env,
                'cmd': '_serve_file'}
        while True:
            load['loc'] = fn_.tell()
            payload['load'] = self.crypticle.dumps(load)
            self.socket.send_pyobj(payload)
            data = self.auth.crypticle.loads(self.socket.recv_pyobj())
            if data == False:
                return False
            if not data:
                break
            fn_.write(data)
        return dest

    def cache_file(self, path, env='base'):
        '''
        Pull a file down from the file server and store it in the minion file
        cache
        '''
        dest = os.path.join(self.opts['cachedir'], 'files', path)
        return self.get_file(path, dest, True, env)

    def cache_files(self, paths, env='base'):
        '''
        Download a list of files stored on the master and put them in the minion
        file cache
        '''
        ret = []
        for path in paths:
            ret.append(self.cache_file(path, env))
        return ret

    def hash_file(self, path, env='base'):
        '''
        Return the hash of a file, to get the hash of a file on the
        salt master file server prepend the path with salt://<file on server>
        otherwise, prepend the file with / for a local file.
        '''
        path = self._check_proto(path)
        payload = {'enc': 'aes'}
        load = {'path': path,
                'env': env,
                'cmd': '_file_hash'}
        payload['load'] = self.auth.crypticle.dumps(load)
        self.socket.send_pyobj(payload)
        return self.auth.crypticle.loads(socket.recv_pyobj())

    def get_state(self, sls, env):
        '''
        Get a state file from the master and store it in the local minion cache
        return the location of the file
        '''
        if sls.count('.'):
            root = sls.replace('.', '/')
            for path in [root + '.sls', os.path.join(root, 'init.sls')]:
                dest = self.cache_file(path, env)
                if dest:
                    return dest
        return False

    def master_opts(self):
        '''
        Return the master opts data
        '''
        payload = {'enc': 'aes'}
        load = {'cmd': '_master_opts'}
        payload['load'] = self.auth.crypticle.dumps(load)
        self.socket.send_pyobj(payload)
        return self.auth.crypticle.loads(socket.recv_pyobj())

<commit_msg>Add self to the socket in the Matcher<commit_after>'''
Routines to set up a minion
'''
# Import python libs
import os
import distutils.sysconfig
import glob
import re
import time
import logging
import tempfile
import traceback
import shutil
import threading
import multiprocessing

# Import zeromq libs
import zmq
# Import salt libs
import salt.crypt
from salt.crypt import AuthenticationError
import salt.utils
import salt.modules
import salt.returners
import salt.loader

log = logging.getLogger(__name__)

# To set up a minion:
# 1, Read in the configuration
# 2. Generate the function mapping dict
# 3. Authenticate with the master
# 4. Store the aes key
# 5. connect to the publisher
# 6. handle publications

class MinionError(Exception): pass

class Minion(object):
    '''
    This class instantiates a minion, runs connections for a minion, and loads
    all of the functions into the minion
    '''
    def __init__(self, opts):
        '''
        Pass in the options dict
        '''
        self.opts = opts
        self.mod_opts = self.__prep_mod_opts()
        self.functions, self.returners = self.__load_modules()
        self.matcher = Matcher(self.opts, self.functions)
        self.authenticate()

    def __prep_mod_opts(self):
        '''
        Returns a deep copy of the opts with key bits stripped out
        '''
        mod_opts = {}
        for key, val in self.opts.items():
            if key == 'logger':
                continue
            mod_opts[key] = val
        return mod_opts

    def __load_modules(self):
        '''
        Return the functions and the returners loaded up from the loader module
        '''
        functions = salt.loader.minion_mods(self.opts)
        returners = salt.loader.returners(self.opts)
        return functions, returners

    def _handle_payload(self, payload):
        '''
        Takes a payload from the master publisher and does whatever the
        master wants done.
        '''
        {'aes': self._handle_aes,
         'pub': self._handle_pub,
         'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_aes(self, load):
        '''
        Takes the aes encrypted load, decrypts is and runs the encapsulated
        instructions
        '''
        data = None
        try:
            data = self.crypticle.loads(load)
        except AuthenticationError:
            self.authenticate()
            data = self.crypticle.loads(load)
        # Verify that the publication is valid
        if not data.has_key('tgt')\
                or not data.has_key('jid')\
                or not data.has_key('fun')\
                or not data.has_key('arg'):
            return
        # Verify that the publication applies to this minion
        if data.has_key('tgt_type'):
            if not getattr(self.matcher, data['tgt_type'] + '_match')(data['tgt']):
                return
        else:
            if not self.matcher.glob_match(data['tgt']):
                return
        self._handle_decoded_payload(data)

    def _handle_pub(self, load):
        '''
        Handle public key payloads
        '''
        pass

    def _handle_clear(self, load):
        '''
        Handle un-encrypted transmissions
        '''
        pass

    def _handle_decoded_payload(self, data):
        '''
        Override this method if you wish to handle the decoded data differently.
        '''
        if self.opts['multiprocessing']:
            if type(data['fun']) == type(list()):
                multiprocessing.Process(
                    target=lambda: self._thread_multi_return(data)
                ).start()
            else:
                multiprocessing.Process(
                    target=lambda: self._thread_return(data)
                ).start()
        else:
            if type(data['fun']) == type(list()):
                threading.Thread(
                    target=lambda: self._thread_multi_return(data)
                ).start()
            else:
                threading.Thread(
                    target=lambda: self._thread_return(data)
                ).start()

    def _thread_return(self, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        ret = {}
        for ind in range(0, len(data['arg'])):
            try:
                data['arg'][ind] = eval(data['arg'][ind])
            except:
                pass

        try:
            ret['return'] = self.functions[data['fun']](*data['arg'])
        except Exception as exc:
            trb = traceback.format_exc()
            log.warning('The minion function caused an exception: %s', exc)
            ret['return'] = trb
        ret['jid'] = data['jid']
        if data['ret']:
            ret['id'] = self.opts['id']
            try:
                self.returners[data['ret']](ret)
            except Exception as exc:
                log.error('The return failed for job %s %s', data['jid'], exc)
        else:
            self._return_pub(ret)

    def _thread_multi_return(self, data):
        '''
        This method should be used as a threading target, start the actual
        minion side execution.
        '''
        ret = {'return': {}}
        for ind in range(0, len(data['fun'])):
            for index in range(0, len(data['arg'][ind])):
                try:
                    data['arg'][ind][index] = eval(data['arg'][ind][index])
                except:
                    pass

            try:
                ret['return'][data['fun'][ind]]\
                    = self.functions[data['fun'][ind]](*data['arg'][ind])
            except Exception as exc:
                trb = traceback.format_exc()
                log.warning('The minion function caused an exception: %s', exc)
                ret['return'][data['fun'][ind]] = trb
            ret['jid'] = data['jid']
        if data['ret']:
            ret['id'] = self.opts['id']
            try:
                self.returners[data['ret']](ret)
            except Exception as exc:
                log.error('The return failed for job %s %s', data['jid'], exc)
        else:
            self._return_pub(ret)

    def _return_pub(self, ret):
        '''
        Return the data from the executed command to the master server
        '''
        log.info('Returning information for job: %(jid)s', ret)
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect(self.opts['master_uri'])
        payload = {'enc': 'aes'}
        load = {'return': ret['return'],
                'cmd': '_return',
                'jid': ret['jid'],
                'id': self.opts['id']}
        payload['load'] = self.crypticle.dumps(load)
        socket.send_pyobj(payload)
        return socket.recv()

    def reload_functions(self):
        '''
        Reload the functions dict for this minion, reading in any new functions
        '''
        self.functions = self.__load_functions()
        log.debug('Refreshed functions, loaded functions: %s', self.functions)
        return True

    def authenticate(self):
        '''
        Authenticate with the master, this method breaks the functional
        paradigm, it will update the master information from a fresh sign in,
        signing in can occur as often as needed to keep up with the revolving
        master aes key.
        '''
        log.debug('Attempting to authenticate with the Salt Master')
        auth = salt.crypt.Auth(self.opts)
        while True:
            creds = auth.sign_in()
            if creds != 'retry':
                log.info('Authentication with master successful!')
                break
            log.info('Waiting for minion key to be accepted by the master.')
            time.sleep(10)
        self.aes = creds['aes']
        self.publish_port = creds['publish_port']
        self.crypticle = salt.crypt.Crypticle(self.aes)

    def tune_in(self):
        '''
        Lock onto the publisher. This is the main event loop for the minion
        '''
        master_pub = 'tcp://' + self.opts['master'] + ':'\
                   + str(self.publish_port)
        context = zmq.Context()
        socket = context.socket(zmq.SUB)
        socket.connect(master_pub)
        socket.setsockopt(zmq.SUBSCRIBE, '')
        while True:
            payload = socket.recv_pyobj()
            self._handle_payload(payload)

class Matcher(object):
    '''
    Use to return the value for matching calls from the master
    '''
    def __init__(self, opts, functions=None):
        self.opts = opts
        if not functions:
            functions = salt.loader.minion_mods(self.opts)
        else:
            self.functions = functions

    def confirm_top(self, data):
        '''
        Takes the data passed to a top file environment and determines if the
        data matches this minion
        '''
        matcher = 'glob'
        for item in data:
            if type(item) == type(dict()):
                if item.has_key('match'):
                    matcher = item['match']
        if hasattr(self, matcher + '_match'):
            return getattr(self, matcher)
        else:
            log.error('Attempting to match with unknown matcher: %s', matcher)
            return False

    def glob_match(self, tgt):
        '''
        Returns true if the passed glob matches the id
        '''
        tmp_dir = tempfile.mkdtemp()
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        open(self.opts['id'], 'w+').write('salt')
        ret = bool(glob.glob(tgt))
        os.chdir(cwd)
        shutil.rmtree(tmp_dir)
        return ret

    def pcre_match(self, tgt):
        '''
        Returns true if the passed pcre regex matches
        '''
        return bool(re.match(tgt, self.opts['id']))

    def list_match(self, tgt):
        '''
        Determines if this host is on the list
        '''
        return bool(tgt.count(self.opts['id']))

    def grain_match(self, tgt):
        '''
        Reads in the grains regular expression match
        '''
        comps = tgt.split(':')
        return bool(re.match(comps[1], self.opts['grains'][comps[0]]))

    def exsel_match(self, tgt):
        '''
        Runs a function and return the exit code
        '''
        if not self.functions.has_key(tgt):
            return False
        return(self.functions[tgt]())

class FileClient(object):
    '''
    Interact with the salt master file server.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.auth = salt.crypt.SAuth(opts)
        self.socket = self.__get_socket()

    def __get_socket(self):
        '''
        Return the ZeroMQ socket to use
        '''
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect(self.opts['master_uri'])
        return socket

    def _check_proto(self, path):
        '''
        Make sure that this path is intended for the salt master and trim it
        '''
        if not path.startswith('salt://'):
            raise MinionError('Unsupported path')
        return path[:7]

    def get_file(self, path, dest, makedirs=False, env='base'):
        '''
        Get a single file from the salt-master
        '''
        path = self._check_proto(path)
        payload = {'enc': 'aes'}
        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            if makedirs:
                os.makedirs(destdir)
            else:
                return False
        fn_ = open(dest, 'w+')
        load = {'path': path,
                'env': env,
                'cmd': '_serve_file'}
        while True:
            load['loc'] = fn_.tell()
            payload['load'] = self.crypticle.dumps(load)
            self.socket.send_pyobj(payload)
            data = self.auth.crypticle.loads(self.socket.recv_pyobj())
            if data == False:
                return False
            if not data:
                break
            fn_.write(data)
        return dest

    def cache_file(self, path, env='base'):
        '''
        Pull a file down from the file server and store it in the minion file
        cache
        '''
        dest = os.path.join(self.opts['cachedir'], 'files', path)
        return self.get_file(path, dest, True, env)

    def cache_files(self, paths, env='base'):
        '''
        Download a list of files stored on the master and put them in the minion
        file cache
        '''
        ret = []
        for path in paths:
            ret.append(self.cache_file(path, env))
        return ret

    def hash_file(self, path, env='base'):
        '''
        Return the hash of a file, to get the hash of a file on the
        salt master file server prepend the path with salt://<file on server>
        otherwise, prepend the file with / for a local file.
        '''
        path = self._check_proto(path)
        payload = {'enc': 'aes'}
        load = {'path': path,
                'env': env,
                'cmd': '_file_hash'}
        payload['load'] = self.auth.crypticle.dumps(load)
        self.socket.send_pyobj(payload)
        return self.auth.crypticle.loads(socket.recv_pyobj())

    def get_state(self, sls, env):
        '''
        Get a state file from the master and store it in the local minion cache
        return the location of the file
        '''
        if sls.count('.'):
            root = sls.replace('.', '/')
            for path in [root + '.sls', os.path.join(root, 'init.sls')]:
                dest = self.cache_file(path, env)
                if dest:
                    return dest
        return False

    def master_opts(self):
        '''
        Return the master opts data
        '''
        payload = {'enc': 'aes'}
        load = {'cmd': '_master_opts'}
        payload['load'] = self.auth.crypticle.dumps(load)
        self.socket.send_pyobj(payload)
        return self.auth.crypticle.loads(self.socket.recv_pyobj())

<|endoftext|>


<commit_before>import click
from sample.classifier import Classifier

@click.group(invoke_without_command=True)
@click.argument('expr')
@click.argument('src', type=click.Path(exists=True))
@click.argument('dst', type=click.Path())
@click.option('--autoclean', '-c', is_flag=True)
def cli(expr, src, dst, autoclean):
    worker = Classifier(expr, src, dst, autoclean=autoclean)
    worker.classify()
<commit_msg>Minor change in main.py<commit_after>import click
from sample.classifier import Classifier

@click.command()
@click.argument('expr')
@click.argument('src', type=click.Path(exists=True))
@click.argument('dst', type=click.Path())
@click.option('--autoclean', '-c', is_flag=True)
def cli(expr, src, dst, autoclean):
    worker = Classifier(expr, src, dst, autoclean=autoclean)
    worker.classify()
<|endoftext|>


<commit_before>import numpy as np

sample_t = np.dtype([('T', np.float64),
                     ('T_c', np.float64),
                     ('accx', np.float64),
                     ('accy', np.float64),
                     ('accz', np.float64),
                     ('Temperature', np.float64),
                     ('gyrox', np.float64),
                     ('gyroy', np.float64),
                     ('gyroz', np.float64),
                     ('RearWheelAngle', np.float64),
                     ('SteerAngle', np.float64),
                     ('FrontWheelAngle', np.float64),
                     ('RearWheelRate_sp', np.float64),
                     ('YawRate_sp', np.float64),
                     ('I_rw', np.float64),
                     ('I_steer', np.float64),
                     ('SystemState', np.uint32)])

SpeedControl = np.uint32(0x0001)
YawRateControl = np.uint32(0x0002)
HubMotorFault = np.uint32(0x0004)
SteerMotorFault = np.uint32(0x0008)
RearWheelEncoderDir = np.uint32(0x0010)
SteerEncoderDir = np.uint32(0x0020)
FrontWheelEncoderDir = np.uint32(0x0040)
RearWheelMotorCurrentDir = np.uint32(0x0080)
SteerMotorCurrentDir = np.uint32(0x0100)
<commit_msg>Added raw data to python sample types.<commit_after>import numpy as np

sample_t = np.dtype([('T', np.float64),
                     ('T_c', np.float64),
                     ('accx', np.float64),
                     ('accy', np.float64),
                     ('accz', np.float64),
                     ('Temperature', np.float64),
                     ('gyrox', np.float64),
                     ('gyroy', np.float64),
                     ('gyroz', np.float64),
                     ('RearWheelAngle', np.float64),
                     ('SteerAngle', np.float64),
                     ('FrontWheelAngle', np.float64),
                     ('RearWheelRate_sp', np.float64),
                     ('YawRate_sp', np.float64),
                     ('I_rw', np.float64),
                     ('I_steer', np.float64),
                     ('SystemState', np.uint32)])

sample_raw_t = np.dtype([('SystemTime', np.uint32),
                         ('ComputationTime', np.uint32),
                         ('RearWheelAngle', np.uint32),
                         ('SteerAngle', np.int32),
                         ('FrontWheelAngle', np.uint32),
                         ('SystemState', np.uint32),
                         ('RearWheelRate_sp', np.float32),
                         ('YawRate_sp', np.float32),
                         ('CCR_rw', np.uint16),
                         ('CCR_steer', np.uint16),
                         ('accx', np.int16),
                         ('accy', np.int16),
                         ('accz', np.int16),
                         ('temp', np.int16),
                         ('gyrox', np.int16),
                         ('gyroy', np.int16),
                         ('gyroz', np.int16)])

SpeedControl = np.uint32(0x0001)
YawRateControl = np.uint32(0x0002)
HubMotorFault = np.uint32(0x0004)
SteerMotorFault = np.uint32(0x0008)
RearWheelEncoderDir = np.uint32(0x0010)
SteerEncoderDir = np.uint32(0x0020)
FrontWheelEncoderDir = np.uint32(0x0040)
RearWheelMotorCurrentDir = np.uint32(0x0080)
SteerMotorCurrentDir = np.uint32(0x0100)
<|endoftext|>


<commit_before>from scapy.all import *

packetCount = 0
message = False
messageend = False
characterlist = []

def customAction(packet):
	global packetCount
	global message
	global messageend
	global characterlist
	#print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		#print message
		messageend = False
		#print messageend
		packetCount += 1
	if(suffixnum == 23130 and packetCount > 1):
		messageend = True
		#print messageend
	if(messageend == True and suffixnum == 23130):
		message = False
		packetCount = 0
		print characterlist
		#print message
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message == True):
		characterlist.append(character)
		packetCount += 1

sniff(filter="port 137", prn=customAction)
<commit_msg>update<commit_after>from scapy.all import *

packetCount = 0
message = False
messageend = False
characterlist = []

def customAction(packet):
	global packetCount
	global message
	global messageend
	global characterlist
	#print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		#print message
		messageend = False
		#print messageend
		packetCount += 1
	if(suffixnum == 23130 and packetCount > 1):
		messageend = True
		#print messageend
	if(messageend == True and suffixnum == 23130):
		message = False
		packetCount = 0
		print characterlist
		#print message
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message == True and packetCount == 1):
		characterlist.append(character)
		packetCount += 1

sniff(filter="port 137", prn=customAction)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from feed_manager import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
import time

from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import MIN_IMAGE_SIZE
from config import THUMBNAIL_LANDSCAPE_SIZE
from config import THUMBNAIL_PORTRAIT_SIZE
from config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<commit_msg>reorged imports sequence in rss.py and renamed ima... to data...<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import calendar
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from data_processor import image_helper
from data_processor import thumbnail
from feed_manager import database as db_feeds
import random
from scraper import database as db_news
from scraper import memory
from scraper import rss_parser
import time

# CONSTANTS
from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import MIN_IMAGE_SIZE
from config import THUMBNAIL_LANDSCAPE_SIZE
from config import THUMBNAIL_PORTRAIT_SIZE
from config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
rss is the interface to rss parsing, processing and storing.
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import calendar
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from data_processor import image_helper
from feed_manager import database as db_feeds
import random
from scraper import database as db_news
from scraper import memory
from scraper import rss_parser
import time

# CONSTANTS
from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import THUMBNAIL_LANDSCAPE_SIZE_HIGH
from config import THUMBNAIL_LANDSCAPE_SIZE_NORMAL
from config import THUMBNAIL_LANDSCAPE_SIZE_LOW
from config import THUMBNAIL_PORTRAIT_SIZE_HIGH
from config import THUMBNAIL_PORTRAIT_SIZE_NORMAL
from config import THUMBNAIL_PORTRAIT_SIZE_LOW
from config import THUMBNAIL_STYLE

def _generate_images(image=None, entry=None, rand=None):
    """
    generate hot news, category and thumbnail images, and maybe more sizes
    """
    if not image or not entry:
        raise Exception(
            '[rss._generate_images] ERROR: Cannot generate images from void content!')
    if not rand:
        # get a new rand
        rand = random.randint(0, 100000000)

    image_relative_path = '%s_%s_%s_%i' % (
        entry['language'], entry['feed_id'], entry['updated'], rand)

    # hot news image
    hot_web, hot_local = image_helper.scale_image(
        image=image, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
    entry['hot_news_image'] = hot_web if hot_web else None
    entry['hot_news_image_local'] = hot_local if hot_local else None

    # category image
    category_web, category_local = image_helper.scale_image(
        image=image, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
    entry['category_image'] = category_web if category_web else None
    entry['category_image_local'] = category_local if category_local else None

    # thumbnail image
    # landscape
    if float(image['width']) / float(image['height']) >= THUMBNAIL_STYLE:
        # high resolution
        thumbnail_web, thumbnail_local = image_helper.scale_image(
            image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_HIGH, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
        if not thumbnail_web:
            # normal resolution
            thumbnail_web, thumbnail_local = image_helper.scale_image(
                image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_NORMAL, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
            if not thumbnail_web:
                # low resolution
                thumbnail_web, thumbnail_local = image_helper.scale_image(
                    image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_LOW, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
    else:  # portrait
        # high resolution
        thumbnail_web, thumbnail_local = image_helper.scale_image(
            image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_HIGH, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
        if not thumbnail_web:
            # normal resolution
            thumbnail_web, thumbnail_local = image_helper.scale_image(
                image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_NORMAL, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
            if not thumbnail_web:
                # low resolution
                thumbnail_web, thumbnail_local = image_helper.scale_image(
                    image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_LOW, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
    entry['thumbnail_image'] = thumbnail_web if thumbnail_web else None
    entry['thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

    return entry

# TODO: replace primitive exception recording with logging
def _get_tts(entry=None, rand=None):
    """
    get tts from the provider
    """
    if not entry:
        raise Exception(
            '[rss._get_tts] ERROR: Cannot generate tts from void content!')
    if not rand:
        # get a new rand
        rand = random.randint(0, 100000000)

    try:
        tts_relative_path = '%s_%s_%s_%i.mp3' % (
            entry['language'], entry['feed_id'], entry['updated'], rand)
        read_content = '%s. %s' % (entry['title'], entry[
                                   'summary'] if entry.has_key('summary') and entry['summary'] else "")
        entry['mp3'], entry['mp3_local'] = tts_provider.google(
            entry['language'], read_content, tts_relative_path)
    except Exception as k:
        print '[rss._get_tts]', str(k), ':%s' % entry['link']
        entry['error'].append(str(k) + '\n')
        entry['mp3'] = None
        entry['mp3_local'] = None
    return entry

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception(
            "[rss._value_added_process] ERROR: language not found or not supported!")

    for entry in entries:
        try:
            print entry['title']
            print entry['link']
            # [MUST-HAVE] transcoding
            # get a random int from 100 million possibilities
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)

            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None

            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail,
            # category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        entry = _generate_images(biggest, entry, rand)
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')
            # for older version users
            entry['image'] = entry['thumbnail_image']['url'] if entry.has_key(
                'thumbnail_image') and entry['thumbnail_image'] else None

            # [OPTIONAL] google tts not for indonesian
            if entry['language'] != 'ind':
                entry = _get_tts(entry, rand)

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            # [OPTIONAL] if logging is used, this could be removed
            entry['error'] = entry['error'] if entry['error'] else None

            # [MUST-HAVE] update new entry to db_news
            # each entry is added with _id
            entry = db_news.update(entry)

            # [MUST-HAVE] store in memory
            memory.update(entry)

            print
            print
        except Exception as k:
            print '[rss._value_added_process:191L]', str(k)
            print
            print

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "[rss.update] ERROR: Method signature not well formed!")

    # try to find the feed in database
    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    if feed:
        # read latest feed info from database
        feed_id = str(feed['_id'])
        feed_link = feed['feed_link']
        language = feed['language']
        categories = feed['categories']
        transcoder_type = feed['transcoder']
        feed_title = feed['feed_title'] if 'feed_title' in feed else None
        etag = feed['etag'] if 'etag' in feed else None
        modified = feed['modified'] if 'modified' in feed else None

        try:
            # parse rss reading from remote rss servers
            entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
                feed_link, feed_id, feed_title, language, categories, etag, modified)

            # filter out existing entries in db_news
            # there are some possible exceptions -- yet let it be
            entries = db_news.dedup(entries, language)

            # and do tts, big_images, image as well as transcode.
            _value_added_process(entries, language, transcoder_type)

            # feed_title, etag and modified to db_feeds
            # only feed_id is necessary, others are optional **kwargs
            db_feeds.update(feed_id=feed_id, status=status_new,
                            feed_title=feed_title_new, etag=etag_new, modified=modified_new)
        except Exception as k:
            print '[rss.update]', str(k)
    else:
        raise Exception(
            '[rss.update] ERROR: Register feed in database before updating!')
<commit_msg>changed hot_news_image to hotnews_image<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
rss is the interface to rss parsing, processing and storing.
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import calendar
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from data_processor import image_helper
from feed_manager import database as db_feeds
import random
from scraper import database as db_news
from scraper import memory
from scraper import rss_parser
import time

# CONSTANTS
from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import THUMBNAIL_LANDSCAPE_SIZE_HIGH
from config import THUMBNAIL_LANDSCAPE_SIZE_NORMAL
from config import THUMBNAIL_LANDSCAPE_SIZE_LOW
from config import THUMBNAIL_PORTRAIT_SIZE_HIGH
from config import THUMBNAIL_PORTRAIT_SIZE_NORMAL
from config import THUMBNAIL_PORTRAIT_SIZE_LOW
from config import THUMBNAIL_STYLE

def _generate_images(image=None, entry=None, rand=None):
    """
    generate hot news, category and thumbnail images, and maybe more sizes
    """
    if not image or not entry:
        raise Exception(
            '[rss._generate_images] ERROR: Cannot generate images from void content!')
    if not rand:
        # get a new rand
        rand = random.randint(0, 100000000)

    image_relative_path = '%s_%s_%s_%i' % (
        entry['language'], entry['feed_id'], entry['updated'], rand)

    # hot news image
    hot_web, hot_local = image_helper.scale_image(
        image=image, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
    entry['hotnews_image'] = hot_web if hot_web else None
    entry['hotnews_image_local'] = hot_local if hot_local else None

    # category image
    category_web, category_local = image_helper.scale_image(
        image=image, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
    entry['category_image'] = category_web if category_web else None
    entry['category_image_local'] = category_local if category_local else None

    # thumbnail image
    # landscape
    if float(image['width']) / float(image['height']) >= THUMBNAIL_STYLE:
        # high resolution
        thumbnail_web, thumbnail_local = image_helper.scale_image(
            image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_HIGH, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
        if not thumbnail_web:
            # normal resolution
            thumbnail_web, thumbnail_local = image_helper.scale_image(
                image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_NORMAL, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
            if not thumbnail_web:
                # low resolution
                thumbnail_web, thumbnail_local = image_helper.scale_image(
                    image=image, size_expected=THUMBNAIL_LANDSCAPE_SIZE_LOW, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
    else:  # portrait
        # high resolution
        thumbnail_web, thumbnail_local = image_helper.scale_image(
            image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_HIGH, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
        if not thumbnail_web:
            # normal resolution
            thumbnail_web, thumbnail_local = image_helper.scale_image(
                image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_NORMAL, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
            if not thumbnail_web:
                # low resolution
                thumbnail_web, thumbnail_local = image_helper.scale_image(
                    image=image, size_expected=THUMBNAIL_PORTRAIT_SIZE_LOW, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
    entry['thumbnail_image'] = thumbnail_web if thumbnail_web else None
    entry['thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

    return entry

# TODO: replace primitive exception recording with logging
def _get_tts(entry=None, rand=None):
    """
    get tts from the provider
    """
    if not entry:
        raise Exception(
            '[rss._get_tts] ERROR: Cannot generate tts from void content!')
    if not rand:
        # get a new rand
        rand = random.randint(0, 100000000)

    try:
        tts_relative_path = '%s_%s_%s_%i.mp3' % (
            entry['language'], entry['feed_id'], entry['updated'], rand)
        read_content = '%s. %s' % (entry['title'], entry[
                                   'summary'] if entry.has_key('summary') and entry['summary'] else "")
        entry['mp3'], entry['mp3_local'] = tts_provider.google(
            entry['language'], read_content, tts_relative_path)
    except Exception as k:
        print '[rss._get_tts]', str(k), ':%s' % entry['link']
        entry['error'].append(str(k) + '\n')
        entry['mp3'] = None
        entry['mp3_local'] = None
    return entry

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception(
            "[rss._value_added_process] ERROR: language not found or not supported!")

    for entry in entries:
        try:
            print entry['title']
            print entry['link']
            # [MUST-HAVE] transcoding
            # get a random int from 100 million possibilities
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)

            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None

            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail,
            # category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        entry = _generate_images(biggest, entry, rand)
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')
            # for older version users
            entry['image'] = entry['thumbnail_image']['url'] if entry.has_key(
                'thumbnail_image') and entry['thumbnail_image'] else None

            # [OPTIONAL] google tts not for indonesian
            if entry['language'] != 'ind':
                entry = _get_tts(entry, rand)

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            # [OPTIONAL] if logging is used, this could be removed
            entry['error'] = entry['error'] if entry['error'] else None

            # [MUST-HAVE] update new entry to db_news
            # each entry is added with _id
            entry = db_news.update(entry)

            # [MUST-HAVE] store in memory
            memory.update(entry)

            print
            print
        except Exception as k:
            print '[rss._value_added_process:191L]', str(k)
            print
            print

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "[rss.update] ERROR: Method signature not well formed!")

    # try to find the feed in database
    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    if feed:
        # read latest feed info from database
        feed_id = str(feed['_id'])
        feed_link = feed['feed_link']
        language = feed['language']
        categories = feed['categories']
        transcoder_type = feed['transcoder']
        feed_title = feed['feed_title'] if 'feed_title' in feed else None
        etag = feed['etag'] if 'etag' in feed else None
        modified = feed['modified'] if 'modified' in feed else None

        try:
            # parse rss reading from remote rss servers
            entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
                feed_link, feed_id, feed_title, language, categories, etag, modified)

            # filter out existing entries in db_news
            # there are some possible exceptions -- yet let it be
            entries = db_news.dedup(entries, language)

            # and do tts, big_images, image as well as transcode.
            _value_added_process(entries, language, transcoder_type)

            # feed_title, etag and modified to db_feeds
            # only feed_id is necessary, others are optional **kwargs
            db_feeds.update(feed_id=feed_id, status=status_new,
                            feed_title=feed_title_new, etag=etag_new, modified=modified_new)
        except Exception as k:
            print '[rss.update]', str(k)
    else:
        raise Exception(
            '[rss.update] ERROR: Register feed in database before updating!')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import subprocess
import sys

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

def main():
  os.chdir(SOURCE_ROOT)

  if sys.platform == 'darwin':
    atom_shell = os.path.join(SOURCE_ROOT, 'out', 'D', 'Atom.app',
                              'Contents', 'MacOS', 'Atom')
  elif sys.platform == 'win32':
    atom_shell = os.path.join(SOURCE_ROOT, 'out', 'D', 'atom.exe')
  else:
    atom_shell = os.path.join(SOURCE_ROOT, 'out', 'D', 'atom')

  subprocess.check_call([atom_shell, 'spec'] + sys.argv[1:])

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Allow testing release build<commit_after>#!/usr/bin/env python

import os
import subprocess
import sys

SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

def main():
  os.chdir(SOURCE_ROOT)

  config = 'D'
  if len(sys.argv) == 2 and sys.argv[1] == '-R':
    config = 'R'

  if sys.platform == 'darwin':
    atom_shell = os.path.join(SOURCE_ROOT, 'out', config, 'Atom.app',
                              'Contents', 'MacOS', 'Atom')
  elif sys.platform == 'win32':
    atom_shell = os.path.join(SOURCE_ROOT, 'out', config, 'atom.exe')
  else:
    atom_shell = os.path.join(SOURCE_ROOT, 'out', config, 'atom')

  subprocess.check_call([atom_shell, 'spec'] + sys.argv[1:])

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before># Copyright (c) 2014 Quanta Research Cambridge, Inc
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

import cppgen, bsvgen

class Method(cppgen.MethodMixin,bsvgen.MethodMixin):
    def __init__(self, name, return_type, params):
        self.type = 'Method'
        self.name = name
        self.return_type = return_type
        self.params = params
    def __repr__(self):
        sparams = [p.__repr__() for p in self.params]
        return '<method: %s %s %s>' % (self.name, self.return_type, sparams)
    def instantiate(self, paramBindings):
        #print 'instantiate method', self.name, self.params
        return Method(self.name,
                      self.return_type.instantiate(paramBindings),
                      [ p.instantiate(paramBindings) for p in self.params])

class Function(cppgen.NoCMixin):
    def __init__(self, name, return_type, params):
        self.type = 'Function'
        self.name = name
        self.return_type = return_type
        self.params = params
    def __repr__(self):
        sparams = map(str, self.params)
        return '<function: %s %s %s>' % (self.name, self.return_type, sparams)

class Variable:
    def __init__(self, name, t):
        self.type = 'Variable'
        self.name = name
        self.type = t
    def __repr__(self):
        return '<variable: %s : %s>' % (self.name, self.type)

class Interface(cppgen.InterfaceMixin,bsvgen.InterfaceMixin):
    def __init__(self, name, params, decls, subinterfacename, packagename):
        self.type = 'Interface'
        self.name = name
        self.params = params
        self.decls = decls
        self.subinterfacename = subinterfacename
        self.typeClassInstances = []
        self.hasSource = True
        self.package = packagename
    def interfaceType(self):
        return Type(self.name,self.params)
    def __repr__(self):
        return '{interface: %s (%s) : %s}' % (self.name, self.params, self.typeClassInstances)
    def instantiate(self, paramBindings):
        newInterface = Interface(self.name, [],
                                 [d.instantiate(paramBindings) for d in self.decls],
                                 self.subinterfacename,
                                 self.package)
        newInterface.hasSource = self.hasSource
        newInterface.typeClassInstances = self.typeClassInstances
        return newInterface

class Typeclass:
    def __init__(self, name):
        self.name = name
        self.type = 'TypeClass'
    def __repr__(self):
        return '{typeclass %s}' % (self.name)

class TypeclassInstance:
    def __init__(self, name, params, provisos, decl):
        self.name = name
        self.params = params
        self.provisos = provisos
        self.decl = decl
        self.type = 'TypeclassInstance'
    def __repr__(self):
        return '{typeclassinstance %s %s}' % (self.name, self.params)

class Module(cppgen.NoCMixin):
    def __init__(self, moduleContext, name, params, interface, provisos, decls):
        self.type = 'Module'
        self.name = name
        self.moduleContext = moduleContext
        self.interface = interface
        self.params = params
        self.provisos = provisos
        self.decls = decls
    def __repr__(self):
        return '{module: %s %s}' % (self.name, self.decls)
    def collectTypes(self):
        result = []
        for d in self.decls:
            if d:
                result.extend(d.collectTypes())
        return result

class EnumElement(cppgen.EnumElementMixin):
    def __init__(self, name, qualifiers, value):
        self.qualifiers = qualifiers
        self.value = value
    def __repr__(self):
        return '{enumelt: %s}' % (self.name)

class Enum(cppgen.EnumMixin,bsvgen.EnumMixin):
    def __init__(self, elements):
        self.type = 'Enum'
        self.elements = elements
    def __repr__(self):
        return '{enum: %s}' % (self.elements)

class StructMember(cppgen.StructMemberMixin):
    def __init__(self, t, name):
        self.type = t
        self.name = name
    def __repr__(self):
        return '{field: %s %s}' % (self.type, self.name)

class Struct(cppgen.StructMixin):
    def __init__(self, elements):
        self.type = 'Struct'
        self.elements = elements
    def __repr__(self):
        return '{struct: %s}' % (self.elements)

class TypeDef(cppgen.TypeDefMixin):
    def __init__(self, tdtype, name):
        self.name = name
        self.tdtype = tdtype
        tdtype.name = name
        self.type = 'TypeDef'
    def __repr__(self):
        return '{typedef: %s %s}' % (self.tdtype, self.name)

class Param(cppgen.ParamMixin,bsvgen.ParamMixin):
    def __init__(self, name, t):
        self.name = name
        self.type = t
    def __repr__(self):
        return '{param %s: %s}' % (self.name, self.type)
    def instantiate(self, paramBindings):
        return Param(self.name,
                     self.type.instantiate(paramBindings))

class Type(cppgen.TypeMixin,bsvgen.TypeMixin):
    def __init__(self, name, params):
        self.type = 'Type'
        self.name = name
        if params:
            self.params = params
        else:
            self.params = []
    def __repr__(self):
        sparams = map(str, self.params)
        return '{type: %s %s}' % (self.name, sparams)
    def instantiate(self, paramBindings):
        #print 'instantiate', self.name
        if paramBindings.has_key(self.name):
            return paramBindings[self.name]
        else:
            return Type(self.name, [p.instantiate(paramBindings) for p in self.params])
    def numeric(self):
        return int(self.name)
<commit_msg>do not overwrite the name of a typedef<commit_after># Copyright (c) 2014 Quanta Research Cambridge, Inc
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

import cppgen, bsvgen

class Method(cppgen.MethodMixin,bsvgen.MethodMixin):
    def __init__(self, name, return_type, params):
        self.type = 'Method'
        self.name = name
        self.return_type = return_type
        self.params = params
    def __repr__(self):
        sparams = [p.__repr__() for p in self.params]
        return '<method: %s %s %s>' % (self.name, self.return_type, sparams)
    def instantiate(self, paramBindings):
        #print 'instantiate method', self.name, self.params
        return Method(self.name,
                      self.return_type.instantiate(paramBindings),
                      [ p.instantiate(paramBindings) for p in self.params])

class Function(cppgen.NoCMixin):
    def __init__(self, name, return_type, params):
        self.type = 'Function'
        self.name = name
        self.return_type = return_type
        self.params = params
    def __repr__(self):
        sparams = map(str, self.params)
        return '<function: %s %s %s>' % (self.name, self.return_type, sparams)

class Variable:
    def __init__(self, name, t):
        self.type = 'Variable'
        self.name = name
        self.type = t
    def __repr__(self):
        return '<variable: %s : %s>' % (self.name, self.type)

class Interface(cppgen.InterfaceMixin,bsvgen.InterfaceMixin):
    def __init__(self, name, params, decls, subinterfacename, packagename):
        self.type = 'Interface'
        self.name = name
        self.params = params
        self.decls = decls
        self.subinterfacename = subinterfacename
        self.typeClassInstances = []
        self.hasSource = True
        self.package = packagename
    def interfaceType(self):
        return Type(self.name,self.params)
    def __repr__(self):
        return '{interface: %s (%s) : %s}' % (self.name, self.params, self.typeClassInstances)
    def instantiate(self, paramBindings):
        newInterface = Interface(self.name, [],
                                 [d.instantiate(paramBindings) for d in self.decls],
                                 self.subinterfacename,
                                 self.package)
        newInterface.hasSource = self.hasSource
        newInterface.typeClassInstances = self.typeClassInstances
        return newInterface

class Typeclass:
    def __init__(self, name):
        self.name = name
        self.type = 'TypeClass'
    def __repr__(self):
        return '{typeclass %s}' % (self.name)

class TypeclassInstance:
    def __init__(self, name, params, provisos, decl):
        self.name = name
        self.params = params
        self.provisos = provisos
        self.decl = decl
        self.type = 'TypeclassInstance'
    def __repr__(self):
        return '{typeclassinstance %s %s}' % (self.name, self.params)

class Module(cppgen.NoCMixin):
    def __init__(self, moduleContext, name, params, interface, provisos, decls):
        self.type = 'Module'
        self.name = name
        self.moduleContext = moduleContext
        self.interface = interface
        self.params = params
        self.provisos = provisos
        self.decls = decls
    def __repr__(self):
        return '{module: %s %s}' % (self.name, self.decls)
    def collectTypes(self):
        result = []
        for d in self.decls:
            if d:
                result.extend(d.collectTypes())
        return result

class EnumElement(cppgen.EnumElementMixin):
    def __init__(self, name, qualifiers, value):
        self.qualifiers = qualifiers
        self.value = value
    def __repr__(self):
        return '{enumelt: %s}' % (self.name)

class Enum(cppgen.EnumMixin,bsvgen.EnumMixin):
    def __init__(self, elements):
        self.type = 'Enum'
        self.elements = elements
    def __repr__(self):
        return '{enum: %s}' % (self.elements)

class StructMember(cppgen.StructMemberMixin):
    def __init__(self, t, name):
        self.type = t
        self.name = name
    def __repr__(self):
        return '{field: %s %s}' % (self.type, self.name)

class Struct(cppgen.StructMixin):
    def __init__(self, elements):
        self.type = 'Struct'
        self.elements = elements
    def __repr__(self):
        return '{struct: %s}' % (self.elements)

class TypeDef(cppgen.TypeDefMixin):
    def __init__(self, tdtype, name):
        self.name = name
        self.tdtype = tdtype
        if tdtype.type != 'Type':
            tdtype.name = name
        self.type = 'TypeDef'
    def __repr__(self):
        return '{typedef: %s %s}' % (self.tdtype, self.name)

class Param(cppgen.ParamMixin,bsvgen.ParamMixin):
    def __init__(self, name, t):
        self.name = name
        self.type = t
    def __repr__(self):
        return '{param %s: %s}' % (self.name, self.type)
    def instantiate(self, paramBindings):
        return Param(self.name,
                     self.type.instantiate(paramBindings))

class Type(cppgen.TypeMixin,bsvgen.TypeMixin):
    def __init__(self, name, params):
        self.type = 'Type'
        self.name = name
        if params:
            self.params = params
        else:
            self.params = []
    def __repr__(self):
        sparams = map(str, self.params)
        return '{type: %s %s}' % (self.name, sparams)
    def instantiate(self, paramBindings):
        #print 'instantiate', self.name
        if paramBindings.has_key(self.name):
            return paramBindings[self.name]
        else:
            return Type(self.name, [p.instantiate(paramBindings) for p in self.params])
    def numeric(self):
        return int(self.name)
<|endoftext|>