<commit_before>from copy import deepcopy
import logging

# 3rd-party
from lxml import etree
from lxml.builder import E

# local
from jnpr.junos.factory.table import Table
from jnpr.junos.jxml import remove_namespaces, remove_namespaces_and_spaces
from jnpr.junos.decorators import checkSAXParserDecorator

logger = logging.getLogger("jnpr.junos.factory.optable")

class OpTable(Table):

    # -------------------------------------------------------------------------
    # PUBLIC METHODS
    # -------------------------------------------------------------------------

    @checkSAXParserDecorator
    def get(self, *vargs, **kvargs):
        """
        Retrieve the XML table data from the Device instance and
        returns back the Table instance - for call-chaining purposes.

        If the Table was created with a :path: rather than a Device,
        then this method will load the XML from that file.  In this
        case, the \*vargs, and \**kvargs are not used.

        ALIAS: __call__

        :vargs:
          [0] is the table :arg_key: value.  This is used so that
          the caller can retrieve just one item from the table without
          having to know the Junos RPC argument.

        :kvargs:
          these are the name/value pairs relating to the specific Junos
          XML command attached to the table.  For example, if the RPC
          is 'get-route-information', there are parameters such as
          'table' and 'destination'.  Any valid RPC argument can be
          passed to :kvargs: to further filter the results of the :get():
          operation.  neato!

        NOTES:
          If you need to create a 'stub' for unit-testing
          purposes, you want to create a subclass of your table and
          overload this methods.
        """
        self._clearkeys()

        if self._path is not None:
            # for loading from local file-path
            self.xml = remove_namespaces(etree.parse(self._path).getroot())
            return self

        if self._lxml is not None:
            return self

        argkey = vargs[0] if len(vargs) else None

        rpc_args = {}

        if self._use_filter:
            try:
                filter_xml = generate_sax_parser_input(self)
                rpc_args['filter_xml'] = filter_xml
            except ValueError as ex:
                logger.warning("Not able to create SAX parser input due to "
                               "'%s', hence using DOM" % ex)

        self.D.transform = lambda: remove_namespaces_and_spaces
        rpc_args.update(self.GET_ARGS)    # copy default args
        # saltstack get_table pass args as named keyword
        if 'args' in kvargs and isinstance(kvargs['args'], dict):
            rpc_args.update(kvargs.pop('args'))
        rpc_args.update(kvargs)           # copy caller provided args

        if hasattr(self, 'GET_KEY') and argkey is not None:
            rpc_args.update({self.GET_KEY: argkey})

        # execute the Junos RPC to retrieve the table
        self.xml = getattr(self.RPC, self.GET_RPC)(**rpc_args)

        # returning self for call-chaining purposes, yo!
        return self

def generate_sax_parser_input(obj):
    """
    Used to generate xml object from Table/view to be used in SAX parsing
    Args:
        obj: self object which contains table/view details

    Returns: lxml etree object to be used as sax parser input

    """
    parser_ingest = E(obj.ITEM_XPATH, E(obj.ITEM_NAME_XPATH))
    local_field_dict = deepcopy(obj.VIEW.FIELDS)
    # first make element out of group fields
    if obj.VIEW.GROUPS:
        for group, group_xpath in obj.VIEW.GROUPS.items():
            # need to pop out group items so that it wont be reused with fields
            group_field_dict = ({k: local_field_dict.pop(k)
                                 for k, v in obj.VIEW.FIELDS.items()
                                 if v.get('group') == group})
            group_ele = E(group_xpath)
            for key, val in group_field_dict.items():
                group_ele.append(E(val.get('xpath')))
            parser_ingest.append(group_ele)
    for i, item in enumerate(local_field_dict.items()):
        # i is the index and item will be taple of field key and value
        field_dict = item[1]
        if 'table' in field_dict:
            # handle nested table/view
            child_table = field_dict.get('table')
            parser_ingest.insert(i + 1, generate_sax_parser_input(child_table))
        else:
            parser_ingest.insert(i + 1, E(field_dict.get('xpath')))
    return parser_ingest
<commit_msg>handle multi level xpath in table/view<commit_after>from copy import deepcopy
import logging

# 3rd-party
from lxml import etree
from lxml.builder import E

# local
from jnpr.junos.factory.table import Table
from jnpr.junos.jxml import remove_namespaces, remove_namespaces_and_spaces
from jnpr.junos.decorators import checkSAXParserDecorator

logger = logging.getLogger("jnpr.junos.factory.optable")

class OpTable(Table):

    # -------------------------------------------------------------------------
    # PUBLIC METHODS
    # -------------------------------------------------------------------------

    @checkSAXParserDecorator
    def get(self, *vargs, **kvargs):
        """
        Retrieve the XML table data from the Device instance and
        returns back the Table instance - for call-chaining purposes.

        If the Table was created with a :path: rather than a Device,
        then this method will load the XML from that file.  In this
        case, the \*vargs, and \**kvargs are not used.

        ALIAS: __call__

        :vargs:
          [0] is the table :arg_key: value.  This is used so that
          the caller can retrieve just one item from the table without
          having to know the Junos RPC argument.

        :kvargs:
          these are the name/value pairs relating to the specific Junos
          XML command attached to the table.  For example, if the RPC
          is 'get-route-information', there are parameters such as
          'table' and 'destination'.  Any valid RPC argument can be
          passed to :kvargs: to further filter the results of the :get():
          operation.  neato!

        NOTES:
          If you need to create a 'stub' for unit-testing
          purposes, you want to create a subclass of your table and
          overload this methods.
        """
        self._clearkeys()

        if self._path is not None:
            # for loading from local file-path
            self.xml = remove_namespaces(etree.parse(self._path).getroot())
            return self

        if self._lxml is not None:
            return self

        argkey = vargs[0] if len(vargs) else None

        rpc_args = {}

        if self._use_filter:
            try:
                filter_xml = generate_sax_parser_input(self)
                rpc_args['filter_xml'] = filter_xml
            except ValueError as ex:
                logger.warning("Not able to create SAX parser input due to "
                               "'%s', hence using DOM" % ex)

        self.D.transform = lambda: remove_namespaces_and_spaces
        rpc_args.update(self.GET_ARGS)    # copy default args
        # saltstack get_table pass args as named keyword
        if 'args' in kvargs and isinstance(kvargs['args'], dict):
            rpc_args.update(kvargs.pop('args'))
        rpc_args.update(kvargs)           # copy caller provided args

        if hasattr(self, 'GET_KEY') and argkey is not None:
            rpc_args.update({self.GET_KEY: argkey})

        # execute the Junos RPC to retrieve the table
        self.xml = getattr(self.RPC, self.GET_RPC)(**rpc_args)

        # returning self for call-chaining purposes, yo!
        return self

def generate_sax_parser_input(obj):
    """
    Used to generate xml object from Table/view to be used in SAX parsing
    Args:
        obj: self object which contains table/view details

    Returns: lxml etree object to be used as sax parser input

    """
    parser_ingest = E(obj.ITEM_XPATH, E(obj.ITEM_NAME_XPATH))
    local_field_dict = deepcopy(obj.VIEW.FIELDS)
    # first make element out of group fields
    if obj.VIEW.GROUPS:
        for group, group_xpath in obj.VIEW.GROUPS.items():
            # need to pop out group items so that it wont be reused with fields
            group_field_dict = ({k: local_field_dict.pop(k)
                                 for k, v in obj.VIEW.FIELDS.items()
                                 if v.get('group') == group})
            group_ele = E(group_xpath)
            for key, val in group_field_dict.items():
                group_ele.append(E(val.get('xpath')))
            parser_ingest.append(group_ele)
    for i, item in enumerate(local_field_dict.items()):
        # i is the index and item will be taple of field key and value
        field_dict = item[1]
        if 'table' in field_dict:
            # handle nested table/view
            child_table = field_dict.get('table')
            parser_ingest.insert(i + 1, generate_sax_parser_input(child_table))
        else:
            xpath = field_dict.get('xpath')
            # xpath can be multi level, for ex traffic-statistics/input-pps
            if '/' in xpath:
                tags = xpath.split('/')
                obj = E(tags[0])
                for tag in tags[1:]:
                    obj.append(E(tag))
                parser_ingest.insert(i + 1, obj)
            else:
                parser_ingest.insert(i + 1, E(xpath))
    return parser_ingest
<|endoftext|>


<commit_before>#
#

# Copyright (C) 2014 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""KVM hypervisor tap device helpers

"""

import os
import logging
import struct
import fcntl

from ganeti import errors

# TUN/TAP driver constants, taken from <linux/if_tun.h>
# They are architecture-independent and already hardcoded in qemu-kvm source,
# so we can safely include them here.
TUNSETIFF = 0x400454ca
TUNGETIFF = 0x800454d2
TUNGETFEATURES = 0x800454cf
IFF_TAP = 0x0002
IFF_NO_PI = 0x1000
IFF_ONE_QUEUE = 0x2000
IFF_VNET_HDR = 0x4000
IFF_MULTI_QUEUE = 0x0100

def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
  """Retrieves supported TUN features from file descriptor.

  @see: L{_ProbeTapVnetHdr}

  """
  req = struct.pack("I", 0)
  try:
    buf = _ioctl(fd, TUNGETFEATURES, req)
  except EnvironmentError, err:
    logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
    return None
  else:
    (flags, ) = struct.unpack("I", buf)
    return flags

def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
  """Check whether to enable the IFF_VNET_HDR flag.

  To do this, _all_ of the following conditions must be met:
   1. TUNGETFEATURES ioctl() *must* be implemented
   2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
   3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
      drivers/net/tun.c there is no way to test this until after the tap device
      has been created using TUNSETIFF, and there is no way to change the
      IFF_VNET_HDR flag after creating the interface, catch-22! However both
      TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
      thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.

   @type fd: int
   @param fd: the file descriptor of /dev/net/tun

  """
  flags = _features_fn(fd)

  if flags is None:
    # Not supported
    return False

  result = bool(flags & IFF_VNET_HDR)

  if not result:
    logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")

  return result

def _ProbeTapMqVirtioNet(fd, _features_fn=_GetTunFeatures):
  """Check whether to enable the IFF_MULTI_QUEUE flag.

  This flag was introduced in Linux kernel 3.8.

   @type fd: int
   @param fd: the file descriptor of /dev/net/tun

  """
  flags = _features_fn(fd)

  if flags is None:
    # Not supported
    return False

  result = bool(flags & IFF_MULTI_QUEUE)

  if not result:
    logging.warning("Kernel does not support IFF_MULTI_QUEUE, not enabling")

  return result

def OpenTap(vnet_hdr=True, virtio_net_queues=1, name=""):
  """Open a new tap device and return its file descriptor.

  This is intended to be used by a qemu-type hypervisor together with the -net
  tap,fd=<fd> or -net tap,fds=x:y:...:z command line parameter.

  @type vnet_hdr: boolean
  @param vnet_hdr: Enable the VNET Header

  @type virtio_net_queues: int
  @param virtio_net_queues: Set number of tap queues but not more than 8,
                            queues only work with virtio-net device;
                            disabled by default (one queue).

  @type name: string
  @param name: name for the TAP interface being created; if an empty
               string is passed, the OS will generate a unique name

  @return: (ifname, [tapfds])
  @rtype: tuple

  """
  tapfds = []

  for _ in range(virtio_net_queues):
    try:
      tapfd = os.open("/dev/net/tun", os.O_RDWR)
    except EnvironmentError:
      raise errors.HypervisorError("Failed to open /dev/net/tun")

    flags = IFF_TAP | IFF_NO_PI

    if vnet_hdr and _ProbeTapVnetHdr(tapfd):
      flags |= IFF_VNET_HDR

    # Check if it's ok to enable IFF_MULTI_QUEUE
    if virtio_net_queues > 1 and _ProbeTapMqVirtioNet(tapfd):
      flags |= IFF_MULTI_QUEUE
    else:
      flags |= IFF_ONE_QUEUE

    # The struct ifreq ioctl request (see netdevice(7))
    ifr = struct.pack("16sh", name, flags)

    try:
      res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
    except EnvironmentError, err:
      raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
                                   err)

    tapfds.append(tapfd)

  # Get the interface name from the ioctl
  ifname = struct.unpack("16sh", res)[0].strip("\x00")

  return (ifname, tapfds)
<commit_msg>netdev: Refactor OpenTap for future use<commit_after>#
#

# Copyright (C) 2014 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""KVM hypervisor tap device helpers

"""

import os
import logging
import struct
import fcntl

from ganeti import errors

# TUN/TAP driver constants, taken from <linux/if_tun.h>
# They are architecture-independent and already hardcoded in qemu-kvm source,
# so we can safely include them here.
TUNSETIFF = 0x400454ca
TUNGETIFF = 0x800454d2
TUNGETFEATURES = 0x800454cf
IFF_TAP = 0x0002
IFF_NO_PI = 0x1000
IFF_ONE_QUEUE = 0x2000
IFF_VNET_HDR = 0x4000
IFF_MULTI_QUEUE = 0x0100

def _GetTunFeatures(fd, _ioctl=fcntl.ioctl):
  """Retrieves supported TUN features from file descriptor.

  @see: L{_ProbeTapVnetHdr}

  """
  req = struct.pack("I", 0)
  try:
    buf = _ioctl(fd, TUNGETFEATURES, req)
  except EnvironmentError, err:
    logging.warning("ioctl(TUNGETFEATURES) failed: %s", err)
    return None
  else:
    (flags, ) = struct.unpack("I", buf)
    return flags

def _ProbeTapVnetHdr(fd, _features_fn=_GetTunFeatures):
  """Check whether to enable the IFF_VNET_HDR flag.

  To do this, _all_ of the following conditions must be met:
   1. TUNGETFEATURES ioctl() *must* be implemented
   2. TUNGETFEATURES ioctl() result *must* contain the IFF_VNET_HDR flag
   3. TUNGETIFF ioctl() *must* be implemented; reading the kernel code in
      drivers/net/tun.c there is no way to test this until after the tap device
      has been created using TUNSETIFF, and there is no way to change the
      IFF_VNET_HDR flag after creating the interface, catch-22! However both
      TUNGETIFF and TUNGETFEATURES were introduced in kernel version 2.6.27,
      thus we can expect TUNGETIFF to be present if TUNGETFEATURES is.

   @type fd: int
   @param fd: the file descriptor of /dev/net/tun

  """
  flags = _features_fn(fd)

  if flags is None:
    # Not supported
    return False

  result = bool(flags & IFF_VNET_HDR)

  if not result:
    logging.warning("Kernel does not support IFF_VNET_HDR, not enabling")

  return result

def _ProbeTapMqVirtioNet(fd, _features_fn=_GetTunFeatures):
  """Check whether to enable the IFF_MULTI_QUEUE flag.

  This flag was introduced in Linux kernel 3.8.

   @type fd: int
   @param fd: the file descriptor of /dev/net/tun

  """
  flags = _features_fn(fd)

  if flags is None:
    # Not supported
    return False

  result = bool(flags & IFF_MULTI_QUEUE)

  if not result:
    logging.warning("Kernel does not support IFF_MULTI_QUEUE, not enabling")

  return result

def OpenTap(name="", features=None):
  """Open a new tap device and return its file descriptor.

  This is intended to be used by a qemu-type hypervisor together with the -net
  tap,fd=<fd> or -net tap,fds=x:y:...:z command line parameter.

  @type name: string
  @param name: name for the TAP interface being created; if an empty
               string is passed, the OS will generate a unique name

  @type features: dict
  @param features: A dict denoting whether vhost, vnet_hdr, mq
    netdev features are enabled or not.

  @return: (ifname, [tapfds], [vhostfds])
  @rtype: tuple

  """
  tapfds = []
  vhostfds = []
  if features is None:
    features = {}
  vhost = features.get("vhost", False)
  vnet_hdr = features.get("vnet_hdr", True)
  _, virtio_net_queues = features.get("mq", (False, 1))

  for _ in range(virtio_net_queues):
    try:
      tapfd = os.open("/dev/net/tun", os.O_RDWR)
    except EnvironmentError:
      raise errors.HypervisorError("Failed to open /dev/net/tun")

    flags = IFF_TAP | IFF_NO_PI

    if vnet_hdr and _ProbeTapVnetHdr(tapfd):
      flags |= IFF_VNET_HDR

    # Check if it's ok to enable IFF_MULTI_QUEUE
    if virtio_net_queues > 1 and _ProbeTapMqVirtioNet(tapfd):
      flags |= IFF_MULTI_QUEUE
    else:
      flags |= IFF_ONE_QUEUE

    # The struct ifreq ioctl request (see netdevice(7))
    ifr = struct.pack("16sh", name, flags)

    try:
      res = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
    except EnvironmentError, err:
      raise errors.HypervisorError("Failed to allocate a new TAP device: %s" %
                                   err)

    if vhost:
      # This is done regularly by the qemu process if vhost=on was passed with
      # --netdev option. Still, in case of hotplug and if the process does not
      # run with root privileges, we have to get the fds and pass them via
      # SCM_RIGHTS prior to qemu using them.
      try:
        vhostfd = os.open("/dev/vhost-net", os.O_RDWR)
        vhostfds.append(vhostfd)
      except EnvironmentError:
        raise errors.HypervisorError("Failed to open /dev/vhost-net")

    tapfds.append(tapfd)

  # Get the interface name from the ioctl
  ifname = struct.unpack("16sh", res)[0].strip("\x00")

  return (ifname, tapfds, vhostfds)
<|endoftext|>


<commit_before>#!/usr/bin/python
import argparse
import logging
import json
import sjparameters
import configuration
import streamjit
import sys
import subprocess
import sqlite3
import traceback

import deps #fix sys.path
import opentuner
from opentuner.search.manipulator import (ConfigurationManipulator, IntegerParameter, FloatParameter)
from opentuner.measurement import MeasurementInterface
from opentuner.measurement.inputmanager import FixedInputManager
from opentuner.tuningrunmain import TuningRunMain
from opentuner.search.objective import MinimizeTime
from jvmparameters import *

class StreamJitMI(MeasurementInterface):
	''' Measurement Interface for tunning a StreamJit application'''
	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
		self.trycount = 0
		self.jvmOptions = jvmOptions
		self.program = args.program
		try:
			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
			c = self.tunedataDB.cursor()
			c.execute("drop table if exists results")
			c.execute('''CREATE TABLE results ( Round int, JVMOption text, SJConfig text, Exectime real)''')
			self.tunedataDB.commit()
		except Exception, e:
			print "Exception occured : %s"%e
			traceback.print_exc()
			data = raw_input ( "Press Keyboard to exit..." )

	def run(self, desired_result, input, limit):
		print 'RUN.....................'
		self.trycount = self.trycount + 1
		cfg = dict.copy(desired_result.configuration.data)
		#self.niceprint(cfg)
		commandStr = ''
		args = ["java"]
		for key in self.jvmOptions.keys():
			#print "\t", key
  			val = cfg[key]
			#del cfg[key]
			self.jvmOptions.get(key).setValue(val)
			cmd = self.jvmOptions.get(key).getCommand()
			commandStr += cmd
			args.append(cmd)

		cur = self.tunedataDB.cursor()
		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, commandStr, cfg, -1)
		cur.execute(query)
		self.tunedataDB.commit()

		print commandStr

		args.append("-jar")
		args.append("RunApp.jar")
		args.append("%s"%self.program)
		args.append("%d"%self.trycount)

		#p = subprocess.Popen(["java",'%s'%commandStr, "-jar","RunApp.jar", "%s"%self.program, "%d"%self.trycount])
		p = subprocess.Popen(args, stderr=subprocess.PIPE)
		p.wait()
		out, err = p.communicate()
		print err
		if err.find("Exception") > 0:
			return opentuner.resultsdb.models.Result(state='ERROR', time=float('inf'))

		cur.execute('SELECT exectime FROM results WHERE round=%d'%self.trycount)
		row = cur.fetchone()
		time = row[0]

		print time
		exetime = float(time)
		if exetime < 0:
			print "Error in execution"
			return opentuner.resultsdb.models.Result(state='ERROR', time=float('inf'))
		else:
			print "Execution time is %f"%exetime
			return opentuner.resultsdb.models.Result(time=exetime)

	def niceprint(self, cfg):
		print "\n--------------------------------------------------"
		print self.trycount
		for key in cfg.keys():
			print "%s - %s"%(key, cfg[key])

	def program_name(self):
		return self.args.program

	def program_version(self):
		return "1.0"

	def save_final_config(self, configuration):
		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
		self.trycount = self.trycount + 1
		cfg = dict.copy(configuration.data)
		commandStr = ''
		args = ["java"]
		for key in self.jvmOptions.keys():
  			val = cfg[key]
			self.jvmOptions.get(key).setValue(val)
			cmd = self.jvmOptions.get(key).getCommand()
			commandStr += cmd
			args.append(cmd)

		cur = self.tunedataDB.cursor()
		query = 'INSERT INTO results VALUES (%d,"%s","%s", 0)'%(self.trycount, commandStr, cfg)
		cur.execute(query)
		self.tunedataDB.commit()

		print commandStr

		args.append("-jar")
		args.append("RunApp.jar")
		args.append("%s"%self.program)
		args.append("%d"%self.trycount)

		#p = subprocess.Popen(["java",'%s'%commandStr, "-jar","RunApp.jar", "%s"%self.program, "%d"%self.trycount])
		p = subprocess.Popen(args)
		p.wait()

		cur.execute('SELECT exectime FROM results WHERE round=%d'%self.trycount)
		row = cur.fetchone()
		time = row[0]

		print time

		conn = sqlite3.connect('streamjit.db')
		cursor = conn.cursor()
		query = 'INSERT INTO FinalResult VALUES ("%s","%s","%s",%d, %f)'%(self.program, commandStr, cfg, self.trycount, time)
		cursor.execute(query)
		conn.commit()

def main(args, cfg, jvmOptions):
	logging.basicConfig(level=logging.INFO)
	manipulator = ConfigurationManipulator()

	params = dict(cfg.items() + jvmOptions.items())
	#print "\nFeature variables...."
	for key in params.keys():
		#print "\t", key
  		manipulator.add_parameter(params.get(key))

	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
                    MinimizeTime())

	m = TuningRunMain(mi, args)
	m.main()

def start(program):
	log = logging.getLogger(__name__)
	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
	parser.add_argument('--program', help='Name of the StreamJit application')

	argv = ['--program', program,  '--test-limit', '10']
	args = parser.parse_args(argv)

	if not args.database:
    		args.database = 'sqlite:///' + program + '.db'

	try:
		conn = sqlite3.connect('streamjit.db')
		c = conn.cursor()
		query = 'SELECT configuration FROM apps WHERE name="%s"'%program
		c.execute(query)
		row = c.fetchone()
		if not row:
			data = raw_input ( "No entry found with name = %s \nPlease press anykey to exit"%program )
			sys.exit(1)
		cfgString = row[0]
		cfg = configuration.getConfiguration(cfgString)
		cfgparams = cfg.getAllParameters()
	except Exception, e:
		print 'Exception occured'
		traceback.print_exc()
		data = raw_input ( "Press Keyboard to exit..." )

	maxHeap = jvmPowerOfTwoParameter("maxHeap", 512, 2048, "-Xmx%dm")
	gc = jvmSwitchParameter("GC",['UseSerialGC','UseParallelGC','UseParallelOldGC','UseConcMarkSweepGC'],2, "-XX:+%s")

	jvmOptions = {"GC":gc, "maxHeap":maxHeap}

	main(args, cfgparams, jvmOptions)

if __name__ == '__main__':
	start('ChannelVocoder 4, 64')
	start('FMRadio 11, 64')
	start('BitonicSort (N = 4, asc)')

<commit_msg>Exception log database is added<commit_after>#!/usr/bin/python
import argparse
import logging
import json
import sjparameters
import configuration
import streamjit
import sys
import subprocess
import sqlite3
import traceback

import deps #fix sys.path
import opentuner
from opentuner.search.manipulator import (ConfigurationManipulator, IntegerParameter, FloatParameter)
from opentuner.measurement import MeasurementInterface
from opentuner.measurement.inputmanager import FixedInputManager
from opentuner.tuningrunmain import TuningRunMain
from opentuner.search.objective import MinimizeTime
from jvmparameters import *

class StreamJitMI(MeasurementInterface):
	''' Measurement Interface for tunning a StreamJit application'''
	def __init__(self, args, jvmOptions, manipulator, inputmanager, objective):
		super(StreamJitMI, self).__init__(args = args, program_name = args.program, manipulator = manipulator, input_manager = inputmanager, objective = objective)
		self.trycount = 0
		self.jvmOptions = jvmOptions
		self.program = args.program
		try:
			self.tunedataDB = sqlite3.connect('sj' + args.program + '.db')
			c = self.tunedataDB.cursor()
			c.execute("drop table if exists results")
			c.execute('''CREATE TABLE results ( Round int, JVMOption text, SJConfig text, Exectime real)''')
			self.tunedataDB.commit()

			self.exceptionLogDB = sqlite3.connect('ExptnLog_' + args.program + '.db')
			cur = self.exceptionLogDB.cursor()
			cur.execute('''CREATE TABLE if not exists exceptions (ExpMsg text, JVMOption text, SJConfig text)''')
			self.exceptionLogDB.commit()
		except Exception, e:
			print "Exception occured : %s"%e
			traceback.print_exc()
			data = raw_input ( "Press Keyboard to exit..." )

	def run(self, desired_result, input, limit):
		print 'RUN.....................'
		self.trycount = self.trycount + 1
		cfg = dict.copy(desired_result.configuration.data)
		#self.niceprint(cfg)
		commandStr = ''
		args = ["java"]
		for key in self.jvmOptions.keys():
			#print "\t", key
  			val = cfg[key]
			#del cfg[key]
			self.jvmOptions.get(key).setValue(val)
			cmd = self.jvmOptions.get(key).getCommand()
			commandStr += cmd
			args.append(cmd)

		cur = self.tunedataDB.cursor()
		query = 'INSERT INTO results VALUES (%d,"%s","%s", "%f")'%(self.trycount, commandStr, cfg, -1)
		cur.execute(query)
		self.tunedataDB.commit()

		print commandStr

		args.append("-jar")
		args.append("RunApp.jar")
		args.append("%s"%self.program)
		args.append("%d"%self.trycount)

		#p = subprocess.Popen(["java",'%s'%commandStr, "-jar","RunApp.jar", "%s"%self.program, "%d"%self.trycount])
		p = subprocess.Popen(args, stderr=subprocess.PIPE)
		p.wait()
		out, err = p.communicate()
		print err
		if err.find("Exception") > 0:
			cur = self.exceptionLogDB.cursor()
			str1 = str(commandStr)
			str2 = str(cfg)
			cur.execute('INSERT INTO exceptions VALUES (?,?,?)', (err, str1, str2))
			self.exceptionLogDB.commit()
			return opentuner.resultsdb.models.Result(state='ERROR', time=float('inf'))

		cur.execute('SELECT exectime FROM results WHERE round=%d'%self.trycount)
		row = cur.fetchone()
		time = row[0]

		print time
		exetime = float(time)
		if exetime < 0:
			print "Error in execution"
			return opentuner.resultsdb.models.Result(state='ERROR', time=float('inf'))
		else:
			print "Execution time is %f"%exetime
			return opentuner.resultsdb.models.Result(time=exetime)

	def niceprint(self, cfg):
		print "\n--------------------------------------------------"
		print self.trycount
		for key in cfg.keys():
			print "%s - %s"%(key, cfg[key])

	def program_name(self):
		return self.args.program

	def program_version(self):
		return "1.0"

	def save_final_config(self, configuration):
		'''called at the end of autotuning with the best resultsdb.models.Configuration'''
		self.trycount = self.trycount + 1
		cfg = dict.copy(configuration.data)
		commandStr = ''
		args = ["java"]
		for key in self.jvmOptions.keys():
  			val = cfg[key]
			self.jvmOptions.get(key).setValue(val)
			cmd = self.jvmOptions.get(key).getCommand()
			commandStr += cmd
			args.append(cmd)

		cur = self.tunedataDB.cursor()
		query = 'INSERT INTO results VALUES (%d,"%s","%s", 0)'%(self.trycount, commandStr, cfg)
		cur.execute(query)
		self.tunedataDB.commit()

		print commandStr

		args.append("-jar")
		args.append("RunApp.jar")
		args.append("%s"%self.program)
		args.append("%d"%self.trycount)

		#p = subprocess.Popen(["java",'%s'%commandStr, "-jar","RunApp.jar", "%s"%self.program, "%d"%self.trycount])
		p = subprocess.Popen(args)
		p.wait()

		cur.execute('SELECT exectime FROM results WHERE round=%d'%self.trycount)
		row = cur.fetchone()
		time = row[0]

		print time

		conn = sqlite3.connect('streamjit.db')
		cur = conn.cursor()
		query = 'INSERT INTO FinalResult VALUES ("%s","%s","%s",%d, %f)'%(self.program, commandStr, cfg, self.trycount, time)
		cur.execute(query)
		conn.commit()

def main(args, cfg, jvmOptions):
	logging.basicConfig(level=logging.INFO)
	manipulator = ConfigurationManipulator()

	params = dict(cfg.items() + jvmOptions.items())
	#print "\nFeature variables...."
	for key in params.keys():
		#print "\t", key
  		manipulator.add_parameter(params.get(key))

	mi = StreamJitMI(args,jvmOptions, manipulator, FixedInputManager(),
                    MinimizeTime())

	m = TuningRunMain(mi, args)
	m.main()

def start(program):
	log = logging.getLogger(__name__)
	parser = argparse.ArgumentParser(parents=opentuner.argparsers())
	parser.add_argument('--program', help='Name of the StreamJit application')

	argv = ['--program', program,  '--test-limit', '10']
	args = parser.parse_args(argv)

	if not args.database:
    		args.database = 'sqlite:///' + program + '.db'

	try:
		conn = sqlite3.connect('streamjit.db')
		c = conn.cursor()
		query = 'SELECT configuration FROM apps WHERE name="%s"'%program
		c.execute(query)
		row = c.fetchone()
		if not row:
			data = raw_input ( "No entry found with name = %s \nPlease press anykey to exit"%program )
			sys.exit(1)
		cfgString = row[0]
		cfg = configuration.getConfiguration(cfgString)
		cfgparams = cfg.getAllParameters()
	except Exception, e:
		print 'Exception occured'
		traceback.print_exc()
		data = raw_input ( "Press Keyboard to exit..." )

	maxHeap = jvmPowerOfTwoParameter("maxHeap", 512, 2048, "-Xmx%dm")
	gc = jvmSwitchParameter("GC",['UseSerialGC','UseParallelGC','UseParallelOldGC','UseConcMarkSweepGC'],2, "-XX:+%s")

	jvmOptions = {"GC":gc, "maxHeap":maxHeap}

	main(args, cfgparams, jvmOptions)

if __name__ == '__main__':
	start('ChannelVocoder 4, 64')
	start('FMRadio 11, 64')
	start('BitonicSort (N = 4, asc)')

<|endoftext|>


<commit_before>#!/usr/bin/python3
'Make features.fea file'
# TODO: add conditional compilation, compare to fea, compile to ttf
__url__ = 'http://github.com/silnrsi/pysilfont'
__copyright__ = 'Copyright (c) 2017 SIL International  (http://www.sil.org)'
__license__ = 'Released under the MIT License (http://opensource.org/licenses/MIT)'
__author__ = 'Martin Hosken, Alan Ward'

import silfont.ufo as ufo
from collections import OrderedDict
from silfont.feaplus import feaplus_parser
from xml.etree import ElementTree as et
import fontTools.feaLib.ast as ast
import StringIO
import os

from silfont.core import execute

class Glyph(object) :
    def __init__(self, name) :
        self.name = name
        self.anchors = {}
        self.is_mark = False

    def add_anchor(self, info) :
        self.anchors[info['name']] = (int(info['x']), int(info['y']))

    def decide_if_mark(self) :
        for a in self.anchors.keys() :
            if a.startswith("_") :
                self.is_mark = True
                break

class Font(object) :
    def __init__(self):
        self.glyphs = OrderedDict()
        self.classes = {}
        self.all_aps = {}

    def readaps(self, filename) :
        if filename.endswith('.ufo') :
            f = ufo.Ufont(filename)
            for g in f.deflayer :
                glyph = Glyph(g)
                self.glyphs[g] = glyph
                ufo_g = f.deflayer[g]
                if 'anchor' in ufo_g._contents :
                    for a in ufo_g._contents['anchor'] :
                        glyph.add_anchor(a.element.attrib)
                        self.all_aps.setdefault(a.element.attrib['name'], []).append(glyph)
        elif filename.endswith('.xml') :
            currGlyph = None
            currPoint = None
            for event, elem in et.iterparse(filename, events=('start', 'end')):
                if event == 'start':
                    if elem.tag == 'glyph':
                        name = elem.get('PSName', '')
                        if name:
                            currGlyph = Glyph(name)
                            self.glyphs[name] = currGlyph
                        currPoint = None
                    elif elem.tag == 'point':
                        currPoint = {'name' : elem.get('type', '')}
                    elif elem.tag == 'location' and currPoint is not None:
                        currPoint['x'] = elem.get('x', 0)
                        currPoint['y'] = elem.get('y', 0)
                elif event == 'end':
                    if elem.tag == 'point':
                        if currGlyph:
                            currGlyph.add_anchor(currPoint)
                            self.all_aps.setdefault(currPoint['name'], []).append(currGlyph)
                        currPoint = None
                    elif elem.tag == 'glyph':
                        currGlyph = None

    def read_classes(self, fname, classproperties=False):
        doc = et.parse(fname)
        for c in doc.findall('.//class'):
            cl = []
            self.classes[c.get('name')] = cl  # assumes class does not already exist
            for e in c.get('exts', '').split() + [""]:
                for g in c.text.split():
                    if g+e in self.glyphs or (e == '' and g.startswith('@')):
                        cl.append(g+e)
        if not classproperties:
            return
        for c in doc.findall('.//property'):
            for e in c.get('exts', '').split() + [""]:
                for g in c.text.split():
                    if g+e in self.glyphs:
                        cname = c.get('name') + "_" + c.get('value')
                        self.classes.set_default(cname, []).append(g+e)

    def make_classes(self) :
        for name, g in self.glyphs.items() :
            # pull off suffix and make classes
            # TODO: handle ligatures
            base = name
            pos = base.rfind('.')
            while pos > 0 :
                old_base = base
                ext = base[pos+1:]
                base = base[:pos]
                ext_class_nm = "c_" + ext
                if base in self.glyphs :
                    glyph_lst = self.classes.setdefault(ext_class_nm, [])
                    if not old_base in glyph_lst:
                        glyph_lst.append(old_base)
                        self.classes.setdefault("cno_" + ext, []).append(base)
                pos = base.rfind('.')
            if g.is_mark :
                self.classes.setdefault('GDEF_marks', []).append(name)
            else :
                self.classes.setdefault('GDEF_bases', []).append(name)

    def make_marks(self) :
        for name, g in self.glyphs.items() :
            g.decide_if_mark()

    def append_classes(self, parser) :
        # normal glyph classes
        def sortkey(x):
            i = x[0].find('_')
            return (x[0][i:], x[0][:i], x[1]) if i > 0 else (x[0], "", x[1])

        for name, c in sorted(self.classes.items(), key=sortkey):
            gc = parser.ast.GlyphClass(0, None)
            for g in c :
                gc.append(g)
            gcd = parser.ast.GlyphClassDefinition(0, name, gc)
            parser.add_statement(gcd)
            parser.define_glyphclass(name, gcd)

    def append_positions(self, parser):
        # create base and mark classes, add to fea file dicts and parser symbol table
        bclassdef_lst = []
        mclassdef_lst = []
        for ap_nm, glyphs_w_ap in self.all_aps.items() :
            # e.g. all glyphs with U AP
            if not ap_nm.startswith("_"):
                gc = parser.set_baseclass(ap_nm)
            else:
                gc = parser.set_markclass(ap_nm)

            # create lists of glyphs that use the same point (name and coordinates)
            # that can share a class definition
            anchor_cache = {}
            for g in glyphs_w_ap :
                p = g.anchors[ap_nm]
                anchor_cache.setdefault(p, []).append(g.name)

            # create and add class definitions to base and mark classes
            for p, glyphs_w_pt in anchor_cache.items() :
                anchor = parser.ast.Anchor(0, None, p[0], p[1], None, None, None)
                if len(glyphs_w_pt) > 1 :
                    val = parser.ast.GlyphClass(0, glyphs_w_pt)
                else :
                    val = parser.ast.GlyphName(0, glyphs_w_pt[0])
                if not ap_nm.startswith("_"):
                    classdef = parser.ast.BaseClassDefinition(0, gc, anchor, val)
                    bclassdef_lst.append(classdef)
                else:
                    classdef = parser.ast.MarkClassDefinition(0, gc, anchor, val)
                    mclassdef_lst.append(classdef)
                gc.addDefinition(classdef)

        # insert base classes before mark classes in fea file
        for classdef in bclassdef_lst:
            parser.add_statement(classdef)
        for classdef in mclassdef_lst:
            parser.add_statement(classdef)

#TODO: provide more argument info
argspec = [
    ('infile', {'help': 'Input UFO or file'}, {}),
    ('-i', '--input', {'help': 'Fea file to merge'}, {}),
    ('-o', '--output', {'help': 'Output fea file'}, {}),
    ('-c', '--classfile', {'help': 'Classes file'}, {}),
    ('--classprops', {'help': 'Include property elements from classes file', 'action': 'store_true'}, {})
]

def doit(args) :
    font = Font()
    if args.infile :
        font.readaps(args.infile)

    font.make_marks()
    font.make_classes()
    if args.classfile:
        font.read_classes(args.classfile, classproperties = args.classprops)

    p = feaplus_parser(None, [])
    doc_ufo = p.parse() # returns an empty ast.FeatureFile

    # Add goodies from the font
    font.append_classes(p)
    font.append_positions(p)

    # parse the input fea file
    if args.input :
        doc_fea = p.parse(args.input)
    else:
        doc_fea = doc_ufo

    # output as doc.asFea()
    if args.output :
        with open(args.output, "w") as of :
            of.write(doc_fea.asFea())

def cmd(): execute(None, doit, argspec)
if __name__ == '__main__': cmd()
<commit_msg>Correct class sorting for user-supplied class names containing underscore.<commit_after>#!/usr/bin/python3
'Make features.fea file'
# TODO: add conditional compilation, compare to fea, compile to ttf
__url__ = 'http://github.com/silnrsi/pysilfont'
__copyright__ = 'Copyright (c) 2017 SIL International  (http://www.sil.org)'
__license__ = 'Released under the MIT License (http://opensource.org/licenses/MIT)'
__author__ = 'Martin Hosken, Alan Ward'

import silfont.ufo as ufo
from collections import OrderedDict
from silfont.feaplus import feaplus_parser
from xml.etree import ElementTree as et
import fontTools.feaLib.ast as ast
import StringIO
import os

from silfont.core import execute

class Glyph(object) :
    def __init__(self, name) :
        self.name = name
        self.anchors = {}
        self.is_mark = False

    def add_anchor(self, info) :
        self.anchors[info['name']] = (int(info['x']), int(info['y']))

    def decide_if_mark(self) :
        for a in self.anchors.keys() :
            if a.startswith("_") :
                self.is_mark = True
                break

class Font(object) :
    def __init__(self):
        self.glyphs = OrderedDict()
        self.classes = {}
        self.all_aps = {}

    def readaps(self, filename) :
        if filename.endswith('.ufo') :
            f = ufo.Ufont(filename)
            for g in f.deflayer :
                glyph = Glyph(g)
                self.glyphs[g] = glyph
                ufo_g = f.deflayer[g]
                if 'anchor' in ufo_g._contents :
                    for a in ufo_g._contents['anchor'] :
                        glyph.add_anchor(a.element.attrib)
                        self.all_aps.setdefault(a.element.attrib['name'], []).append(glyph)
        elif filename.endswith('.xml') :
            currGlyph = None
            currPoint = None
            for event, elem in et.iterparse(filename, events=('start', 'end')):
                if event == 'start':
                    if elem.tag == 'glyph':
                        name = elem.get('PSName', '')
                        if name:
                            currGlyph = Glyph(name)
                            self.glyphs[name] = currGlyph
                        currPoint = None
                    elif elem.tag == 'point':
                        currPoint = {'name' : elem.get('type', '')}
                    elif elem.tag == 'location' and currPoint is not None:
                        currPoint['x'] = elem.get('x', 0)
                        currPoint['y'] = elem.get('y', 0)
                elif event == 'end':
                    if elem.tag == 'point':
                        if currGlyph:
                            currGlyph.add_anchor(currPoint)
                            self.all_aps.setdefault(currPoint['name'], []).append(currGlyph)
                        currPoint = None
                    elif elem.tag == 'glyph':
                        currGlyph = None

    def read_classes(self, fname, classproperties=False):
        doc = et.parse(fname)
        for c in doc.findall('.//class'):
            cl = []
            self.classes[c.get('name')] = cl  # assumes class does not already exist
            for e in c.get('exts', '').split() + [""]:
                for g in c.text.split():
                    if g+e in self.glyphs or (e == '' and g.startswith('@')):
                        cl.append(g+e)
        if not classproperties:
            return
        for c in doc.findall('.//property'):
            for e in c.get('exts', '').split() + [""]:
                for g in c.text.split():
                    if g+e in self.glyphs:
                        cname = c.get('name') + "_" + c.get('value')
                        self.classes.set_default(cname, []).append(g+e)

    def make_classes(self) :
        for name, g in self.glyphs.items() :
            # pull off suffix and make classes
            # TODO: handle ligatures
            base = name
            pos = base.rfind('.')
            while pos > 0 :
                old_base = base
                ext = base[pos+1:]
                base = base[:pos]
                ext_class_nm = "c_" + ext
                if base in self.glyphs :
                    glyph_lst = self.classes.setdefault(ext_class_nm, [])
                    if not old_base in glyph_lst:
                        glyph_lst.append(old_base)
                        self.classes.setdefault("cno_" + ext, []).append(base)
                pos = base.rfind('.')
            if g.is_mark :
                self.classes.setdefault('GDEF_marks', []).append(name)
            else :
                self.classes.setdefault('GDEF_bases', []).append(name)

    def make_marks(self) :
        for name, g in self.glyphs.items() :
            g.decide_if_mark()

    def append_classes(self, parser) :
        # normal glyph classes
        def sortkey(x):
            key1 = 'c_' + x[0][4:] if x[0].startswith('cno_') else x[0]
            return (key1, x[0], x[1])

        for name, c in sorted(self.classes.items(), key=sortkey):
            gc = parser.ast.GlyphClass(0, None)
            for g in c :
                gc.append(g)
            gcd = parser.ast.GlyphClassDefinition(0, name, gc)
            parser.add_statement(gcd)
            parser.define_glyphclass(name, gcd)

    def append_positions(self, parser):
        # create base and mark classes, add to fea file dicts and parser symbol table
        bclassdef_lst = []
        mclassdef_lst = []
        for ap_nm, glyphs_w_ap in self.all_aps.items() :
            # e.g. all glyphs with U AP
            if not ap_nm.startswith("_"):
                gc = parser.set_baseclass(ap_nm)
            else:
                gc = parser.set_markclass(ap_nm)

            # create lists of glyphs that use the same point (name and coordinates)
            # that can share a class definition
            anchor_cache = {}
            for g in glyphs_w_ap :
                p = g.anchors[ap_nm]
                anchor_cache.setdefault(p, []).append(g.name)

            # create and add class definitions to base and mark classes
            for p, glyphs_w_pt in anchor_cache.items() :
                anchor = parser.ast.Anchor(0, None, p[0], p[1], None, None, None)
                if len(glyphs_w_pt) > 1 :
                    val = parser.ast.GlyphClass(0, glyphs_w_pt)
                else :
                    val = parser.ast.GlyphName(0, glyphs_w_pt[0])
                if not ap_nm.startswith("_"):
                    classdef = parser.ast.BaseClassDefinition(0, gc, anchor, val)
                    bclassdef_lst.append(classdef)
                else:
                    classdef = parser.ast.MarkClassDefinition(0, gc, anchor, val)
                    mclassdef_lst.append(classdef)
                gc.addDefinition(classdef)

        # insert base classes before mark classes in fea file
        for classdef in bclassdef_lst:
            parser.add_statement(classdef)
        for classdef in mclassdef_lst:
            parser.add_statement(classdef)

#TODO: provide more argument info
argspec = [
    ('infile', {'help': 'Input UFO or file'}, {}),
    ('-i', '--input', {'help': 'Fea file to merge'}, {}),
    ('-o', '--output', {'help': 'Output fea file'}, {}),
    ('-c', '--classfile', {'help': 'Classes file'}, {}),
    ('--classprops', {'help': 'Include property elements from classes file', 'action': 'store_true'}, {})
]

def doit(args) :
    font = Font()
    if args.infile :
        font.readaps(args.infile)

    font.make_marks()
    font.make_classes()
    if args.classfile:
        font.read_classes(args.classfile, classproperties = args.classprops)

    p = feaplus_parser(None, [])
    doc_ufo = p.parse() # returns an empty ast.FeatureFile

    # Add goodies from the font
    font.append_classes(p)
    font.append_positions(p)

    # parse the input fea file
    if args.input :
        doc_fea = p.parse(args.input)
    else:
        doc_fea = doc_ufo

    # output as doc.asFea()
    if args.output :
        with open(args.output, "w") as of :
            of.write(doc_fea.asFea())

def cmd(): execute(None, doit, argspec)
if __name__ == '__main__': cmd()
<|endoftext|>


<commit_before>#!/usr/bin/env python
'Generate a ttf file without OpenType tables from a UFO'
__url__ = 'http://github.com/silnrsi/pysilfont'
__copyright__ = 'Copyright (c) 2017 SIL International  (http://www.sil.org)'
__license__ = 'Released under the MIT License (http://opensource.org/licenses/MIT)'
__author__ = 'Alan Ward'

# Compared to fontmake it does not decompose glyphs or remove overlaps
# and curve conversion seems to happen in a different way.

from silfont.core import execute
import defcon, ufo2ft.outlineCompiler, ufo2ft.preProcessor

argspec = [
    ('iufo', {'help': 'Input UFO folder'}, {}),
    ('ottf', {'help': 'Output ttf file name'}, {}),
    ('--removeOverlap', {'help': 'Remove overlap between glyph components', 'action': 'store_true'}, {})]

PUBLIC_PREFIX = 'public.'

def doit(args):
    ufo = defcon.Font(args.iufo)

#    args.logger.log('Converting UFO to ttf and compiling fea')
#    font = ufo2ft.compileTTF(ufo,
#        glyphOrder = ufo.lib.get(PUBLIC_PREFIX + 'glyphOrder'),
#        useProductionNames = False)

    args.logger.log('Converting UFO to ttf without OT', 'P')

    # default arg value for TTFPreProcessor class: removeOverlaps = False, convertCubics = True
    removeOverlaps = False
    if args.removeOverlap:
        removeOverlaps = True
    preProcessor = ufo2ft.preProcessor.TTFPreProcessor(ufo, removeOverlaps = removeOverlaps, convertCubics=True)
    glyphSet = preProcessor.process()
    outlineCompiler = ufo2ft.outlineCompiler.OutlineTTFCompiler(ufo,
        glyphSet=glyphSet,
        glyphOrder=ufo.lib.get(PUBLIC_PREFIX + 'glyphOrder'))
    font = outlineCompiler.compile()

    args.logger.log('Saving ttf file', 'P')
    font.save(args.ottf)

    args.logger.log('Done', 'P')

def cmd(): execute(None, doit, argspec)
if __name__ == '__main__': cmd()
<commit_msg>Streamlined code and changed usage message in psfufo2ttf<commit_after>#!/usr/bin/env python
'Generate a ttf file without OpenType tables from a UFO'
__url__ = 'http://github.com/silnrsi/pysilfont'
__copyright__ = 'Copyright (c) 2017 SIL International  (http://www.sil.org)'
__license__ = 'Released under the MIT License (http://opensource.org/licenses/MIT)'
__author__ = 'Alan Ward'

# Compared to fontmake it does not decompose glyphs or remove overlaps
# and curve conversion seems to happen in a different way.

from silfont.core import execute
import defcon, ufo2ft.outlineCompiler, ufo2ft.preProcessor

argspec = [
    ('iufo', {'help': 'Input UFO folder'}, {}),
    ('ottf', {'help': 'Output ttf file name'}, {}),
    ('--removeOverlap', {'help': 'Merge overlapping contours', 'action': 'store_true'}, {})]

PUBLIC_PREFIX = 'public.'

def doit(args):
    ufo = defcon.Font(args.iufo)

#    args.logger.log('Converting UFO to ttf and compiling fea')
#    font = ufo2ft.compileTTF(ufo,
#        glyphOrder = ufo.lib.get(PUBLIC_PREFIX + 'glyphOrder'),
#        useProductionNames = False)

    args.logger.log('Converting UFO to ttf without OT', 'P')

    # default arg value for TTFPreProcessor class: removeOverlaps = False, convertCubics = True
    preProcessor = ufo2ft.preProcessor.TTFPreProcessor(ufo, removeOverlaps = args.removeOverlap, convertCubics=True)
    glyphSet = preProcessor.process()
    outlineCompiler = ufo2ft.outlineCompiler.OutlineTTFCompiler(ufo,
        glyphSet=glyphSet,
        glyphOrder=ufo.lib.get(PUBLIC_PREFIX + 'glyphOrder'))
    font = outlineCompiler.compile()

    args.logger.log('Saving ttf file', 'P')
    font.save(args.ottf)

    args.logger.log('Done', 'P')

def cmd(): execute(None, doit, argspec)
if __name__ == '__main__': cmd()
<|endoftext|>


<commit_before># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import spack.compiler

class Fj(spack.compiler.Compiler):
    # Subclasses use possible names of C compiler
    cc_names = ['fcc']

    # Subclasses use possible names of C++ compiler
    cxx_names = ['FCC']

    # Subclasses use possible names of Fortran 77 compiler
    f77_names = ['frt']

    # Subclasses use possible names of Fortran 90 compiler
    fc_names = ['frt']

    # Named wrapper links within build_env_path
    link_paths = {'cc': 'fj/fcc',
                  'cxx': 'fj/case-insensitive/FCC',
                  'f77': 'fj/frt',
                  'fc': 'fj/frt'}

    version_argument = '--version'
    version_regex = r'\((?:FCC|FRT)\) ([\d.]+)'

    @property
    def openmp_flag(self):
        return "-Kopenmp"

    @property
    def cxx98_flag(self):
        return "-std=c++98"

    @property
    def cxx11_flag(self):
        return "-std=c++11"

    @property
    def cxx14_flag(self):
        return "-std=c++14"

    @property
    def c99_flag(self):
        return "-std=c99"

    @property
    def c11_flag(self):
        return "-std=c11"

    @property
    def pic_flag(self):
        return "-KPIC"
<commit_msg>Fujitsu compilers: added 'verbose_flag' method (#12922)<commit_after># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import spack.compiler

class Fj(spack.compiler.Compiler):
    # Subclasses use possible names of C compiler
    cc_names = ['fcc']

    # Subclasses use possible names of C++ compiler
    cxx_names = ['FCC']

    # Subclasses use possible names of Fortran 77 compiler
    f77_names = ['frt']

    # Subclasses use possible names of Fortran 90 compiler
    fc_names = ['frt']

    # Named wrapper links within build_env_path
    link_paths = {'cc': 'fj/fcc',
                  'cxx': 'fj/case-insensitive/FCC',
                  'f77': 'fj/frt',
                  'fc': 'fj/frt'}

    version_argument = '--version'
    version_regex = r'\((?:FCC|FRT)\) ([\d.]+)'

    @classmethod
    def verbose_flag(cls):
        return "-v"

    @property
    def openmp_flag(self):
        return "-Kopenmp"

    @property
    def cxx98_flag(self):
        return "-std=c++98"

    @property
    def cxx11_flag(self):
        return "-std=c++11"

    @property
    def cxx14_flag(self):
        return "-std=c++14"

    @property
    def c99_flag(self):
        return "-std=c99"

    @property
    def c11_flag(self):
        return "-std=c11"

    @property
    def pic_flag(self):
        return "-KPIC"
<|endoftext|>


<commit_before># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import, unicode_literals
import re
import json
from datetime import datetime, timedelta
from urllib.parse import urlparse

from svtplay_dl.service import Service, OpenGraphThumbMixin
from svtplay_dl.fetcher.hls import hlsparse
from svtplay_dl.error import ServiceError

class Tv4play(Service, OpenGraphThumbMixin):
    supported_domains = ['tv4play.se']

    def get(self):
        parse = urlparse(self.url)
        if parse.path[:8] == "/kanaler":
            end_time_stamp = (datetime.utcnow() - timedelta(minutes=1, seconds=20)).replace(microsecond=0)
            start_time_stamp = end_time_stamp - timedelta(minutes=1)

            url = "https://bbr-l2v.akamaized.net/live/{0}/master.m3u8?in={1}&out={2}?".format(parse.path[9:],
                                                                                start_time_stamp.isoformat(),
                                                                                end_time_stamp.isoformat())

            self.config.set("live", True)
            streams = hlsparse(self.config, self.http.request("get", url), url, output=self.output, hls_time_stamp=True)
            for n in list(streams.keys()):
                yield streams[n]
            return

        match = self._getjson()
        if not match:
            yield ServiceError("Can't find json data")
            return

        jansson = json.loads(match.group(1))
        vid = None
        for i in jansson:
            janson2 = json.loads(i["data"])
            json.dumps(janson2)
            if "videoAsset" in janson2["data"]:
                vid = janson2["data"]["videoAsset"]["id"]
                if janson2["data"]["videoAsset"]["is_drm_protected"]:
                    yield ServiceError("We can't download DRM protected content from this site.")
                    return
                if janson2["data"]["videoAsset"]["is_live"]:
                    self.config.set("live", True)
                if janson2["data"]["videoAsset"]["season"] > 0:
                    self.output["season"] = janson2["data"]["videoAsset"]["season"]
                if janson2["data"]["videoAsset"]["episode"] > 0:
                    self.output["episode"] = janson2["data"]["videoAsset"]["episode"]
                self.output["title"] = janson2["data"]["videoAsset"]["program"]["name"]
                self.output["episodename"] = janson2["data"]["videoAsset"]["title"]
                vid = str(vid)
                self.output["id"] = str(vid)
            if "program" in janson2["data"] and vid is None:
                if "contentfulPanels" in janson2["data"]["program"]:
                    match = re.search(r"[\/-](\d+)$", self.url)
                    if match and "panels" in janson2["data"]["program"]:
                        for n in janson2["data"]["program"]["panels"]:
                            for z in n["videoList"]["videoAssets"]:
                                if z["id"] == int(match.group(1)):
                                    vid = z["id"]
                                    self.output["id"] = str(vid)
                                    self.output["episodename"] = z["title"]
                                    self.output["title"] = z["program"]["name"]

        if vid is None:
            yield ServiceError("Cant find video id for the video")
            return

        url = "https://playback-api.b17g.net/media/{}?service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine".format(vid)
        res = self.http.request("get", url, cookies=self.cookies)
        if res.json()["playbackItem"]["type"] == "hls":
            streams = hlsparse(self.config, self.http.request("get", res.json()["playbackItem"]["manifestUrl"]),
                               res.json()["playbackItem"]["manifestUrl"], output=self.output, httpobject=self.http)
            for n in list(streams.keys()):
                yield streams[n]

    def _getjson(self):
        match = re.search(".prefetched = (\[.*\]);", self.get_urldata())
        return match

    def find_all_episodes(self, config):
        episodes = []
        items = []
        show = None
        match = self._getjson()
        jansson = json.loads(match.group(1))
        for i in jansson:
            janson2 = json.loads(i["data"])
            if "program" in janson2["data"]:
                if "programPanels" in janson2["data"]["program"]:
                    for n in janson2["data"]["program"]["programPanels"]["panels"]:
                        if n.get("assetType", None) == "EPISODE":
                            for z in n["videoList"]["videoAssets"]:
                                show = z["program_nid"]
                                items.append(z["id"])
                        if n.get("assetType", None) == "CLIP" and config.get("include_clips"):
                            for z in n["videoList"]["videoAssets"]:
                                show = z["program_nid"]
                                items.append(z["id"])

        items = sorted(items)
        for item in items:
            episodes.append("https://www.tv4play.se/program/{}/{}".format(show, item))

        if config.get("all_last") > 0:
            return episodes[-config.get("all_last"):]
        return episodes

class Tv4(Service, OpenGraphThumbMixin):
    supported_domains = ['tv4.se']

    def get(self):
        match = re.search(r"[\/-](\d+)$", self.url)
        if not match:
            yield ServiceError("Cant find video id")
            return
        self.output["id"] = match.group(1)

        match = re.search("data-program-format='([^']+)'", self.get_urldata())
        if not match:
            yield ServiceError("Cant find program name")
            return
        self.output["title"] = match.group(1)

        match = re.search('img alt="([^"]+)" class="video-image responsive"', self.get_urldata())
        if not match:
            yield ServiceError("Cant find title of the video")
            return
        self.output["episodename"] = match.group(1)

        url = "https://playback-api.b17g.net/media/{}?service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine".format(self.output["id"])
        res = self.http.request("get", url, cookies=self.cookies)
        if res.json()["playbackItem"]["type"] == "hls":
            streams = hlsparse(self.config, self.http.request("get", res.json()["playbackItem"]["manifestUrl"]),
                               res.json()["playbackItem"]["manifestUrl"], output=self.output)
            for n in list(streams.keys()):
                yield streams[n]
<commit_msg>tv4play: fixed a crash if someone tried to download a geoblock file<commit_after># ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
from __future__ import absolute_import, unicode_literals
import re
import json
from datetime import datetime, timedelta
from urllib.parse import urlparse

from svtplay_dl.service import Service, OpenGraphThumbMixin
from svtplay_dl.fetcher.hls import hlsparse
from svtplay_dl.error import ServiceError

class Tv4play(Service, OpenGraphThumbMixin):
    supported_domains = ['tv4play.se']

    def get(self):
        parse = urlparse(self.url)
        if parse.path[:8] == "/kanaler":
            end_time_stamp = (datetime.utcnow() - timedelta(minutes=1, seconds=20)).replace(microsecond=0)
            start_time_stamp = end_time_stamp - timedelta(minutes=1)

            url = "https://bbr-l2v.akamaized.net/live/{0}/master.m3u8?in={1}&out={2}?".format(parse.path[9:],
                                                                                start_time_stamp.isoformat(),
                                                                                end_time_stamp.isoformat())

            self.config.set("live", True)
            streams = hlsparse(self.config, self.http.request("get", url), url, output=self.output, hls_time_stamp=True)
            for n in list(streams.keys()):
                yield streams[n]
            return

        match = self._getjson()
        if not match:
            yield ServiceError("Can't find json data")
            return

        jansson = json.loads(match.group(1))
        vid = None
        for i in jansson:
            janson2 = json.loads(i["data"])
            json.dumps(janson2)
            if "videoAsset" in janson2["data"]:
                vid = janson2["data"]["videoAsset"]["id"]
                if janson2["data"]["videoAsset"]["is_drm_protected"]:
                    yield ServiceError("We can't download DRM protected content from this site.")
                    return
                if janson2["data"]["videoAsset"]["is_live"]:
                    self.config.set("live", True)
                if janson2["data"]["videoAsset"]["season"] > 0:
                    self.output["season"] = janson2["data"]["videoAsset"]["season"]
                if janson2["data"]["videoAsset"]["episode"] > 0:
                    self.output["episode"] = janson2["data"]["videoAsset"]["episode"]
                self.output["title"] = janson2["data"]["videoAsset"]["program"]["name"]
                self.output["episodename"] = janson2["data"]["videoAsset"]["title"]
                vid = str(vid)
                self.output["id"] = str(vid)
            if "program" in janson2["data"] and vid is None:
                if "contentfulPanels" in janson2["data"]["program"]:
                    match = re.search(r"[\/-](\d+)$", self.url)
                    if match and "panels" in janson2["data"]["program"]:
                        for n in janson2["data"]["program"]["panels"]:
                            for z in n["videoList"]["videoAssets"]:
                                if z["id"] == int(match.group(1)):
                                    vid = z["id"]
                                    self.output["id"] = str(vid)
                                    self.output["episodename"] = z["title"]
                                    self.output["title"] = z["program"]["name"]

        if vid is None:
            yield ServiceError("Cant find video id for the video")
            return

        url = "https://playback-api.b17g.net/media/{}?service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine".format(vid)
        res = self.http.request("get", url, cookies=self.cookies)
        if res.status_code > 200:
            yield ServiceError("Can't play this because the video is geoblocked.")
            return
        if res.json()["playbackItem"]["type"] == "hls":
            streams = hlsparse(self.config, self.http.request("get", res.json()["playbackItem"]["manifestUrl"]),
                               res.json()["playbackItem"]["manifestUrl"], output=self.output, httpobject=self.http)
            for n in list(streams.keys()):
                yield streams[n]

    def _getjson(self):
        match = re.search(".prefetched = (\[.*\]);", self.get_urldata())
        return match

    def find_all_episodes(self, config):
        episodes = []
        items = []
        show = None
        match = self._getjson()
        jansson = json.loads(match.group(1))
        for i in jansson:
            janson2 = json.loads(i["data"])
            if "program" in janson2["data"]:
                if "programPanels" in janson2["data"]["program"]:
                    for n in janson2["data"]["program"]["programPanels"]["panels"]:
                        if n.get("assetType", None) == "EPISODE":
                            for z in n["videoList"]["videoAssets"]:
                                show = z["program_nid"]
                                items.append(z["id"])
                        if n.get("assetType", None) == "CLIP" and config.get("include_clips"):
                            for z in n["videoList"]["videoAssets"]:
                                show = z["program_nid"]
                                items.append(z["id"])

        items = sorted(items)
        for item in items:
            episodes.append("https://www.tv4play.se/program/{}/{}".format(show, item))

        if config.get("all_last") > 0:
            return episodes[-config.get("all_last"):]
        return episodes

class Tv4(Service, OpenGraphThumbMixin):
    supported_domains = ['tv4.se']

    def get(self):
        match = re.search(r"[\/-](\d+)$", self.url)
        if not match:
            yield ServiceError("Cant find video id")
            return
        self.output["id"] = match.group(1)

        match = re.search("data-program-format='([^']+)'", self.get_urldata())
        if not match:
            yield ServiceError("Cant find program name")
            return
        self.output["title"] = match.group(1)

        match = re.search('img alt="([^"]+)" class="video-image responsive"', self.get_urldata())
        if not match:
            yield ServiceError("Cant find title of the video")
            return
        self.output["episodename"] = match.group(1)

        url = "https://playback-api.b17g.net/media/{}?service=tv4&device=browser&protocol=hls%2Cdash&drm=widevine".format(self.output["id"])
        res = self.http.request("get", url, cookies=self.cookies)
        if res.status_code > 200:
            yield ServiceError("Can't play this because the video is geoblocked.")
            return
        if res.json()["playbackItem"]["type"] == "hls":
            streams = hlsparse(self.config, self.http.request("get", res.json()["playbackItem"]["manifestUrl"]),
                               res.json()["playbackItem"]["manifestUrl"], output=self.output)
            for n in list(streams.keys()):
                yield streams[n]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_lib_scapy communication library implementation.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division
from __future__ import with_statement
#import urllib2

# Add your library functions here.

class CLI:

    def __init__(self, enode):
        self.enode = enode

    def __enter__(self):
        self.enode.get_shell('bash').send_command('scapy', matches='>>> ')
        return self

    def __exit__(self, type, value, traceback):
        self.enode.get_shell('bash').send_command('exit()')

    '''
    def load_protocols(self, file_name):
        req = urllib2.Request(url=file_name)
        f = urllib2.urlopen(req)
        protocols_src = f.read()
        command = 'exec({},globals())'.format(protocols_src)
        self.send_command(command)
    '''

    def send_command(self, command):
        self.enode.get_shell('bash').send_command(command, matches='>>> ')
        response = self.enode.get_shell('bash').get_response()
        return response

__all__ = [
    'CLI'
]
<commit_msg>Addded load protocols<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_lib_scapy communication library implementation.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division
from __future__ import with_statement
from __future__ import urllib2

class CLI:

    def __init__(self, enode):
        self.enode = enode

    def __enter__(self):
        self.enode.get_shell('bash').send_command('scapy', matches='>>> ')
        return self

    def __exit__(self, type, value, traceback):
        self.enode.get_shell('bash').send_command('exit()')

    '''
    def load_protocols(self, file_name):
        req = urllib2.Request(url=file_name)
        f = urllib2.urlopen(req)
        protocols_src = f.read()
        command = 'exec({},globals())'.format(protocols_src)
        self.send_command(command)
    '''

    def send_command(self, command):
        self.enode.get_shell('bash').send_command(command, matches='>>> ')
        response = self.enode.get_shell('bash').get_response()
        return response

__all__ = [
    'CLI'
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_lib_scapy communication library implementation.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division
from __future__ import with_statement

# Add your library functions here.

class CLI(object):

    def __init__(self, enode):
        self.enode = enode

    def __enter__(self):
        self.initial_prompt = self.enode._shells['bash']._prompt
        self.enode._shells['bash']._prompt = '>>> '
        self.enode._shells['bash'].send_command('scapy',matches='>>> ')
        return self

    def __exit__(self, type, value, traceback):
        self.enode._shells['bash']._prompt = self.initial_prompt
        self.enode._shells['bash'].send_command('exit()')

    def send_command(self, command):
        self.enode._shells['bash'].send_command(command)
        response = self.enode._shells['bash'].get_response()
        return response

__all__ = [
    'CLI'
]
<commit_msg>Context<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""
topology_lib_scapy communication library implementation.
"""

from __future__ import unicode_literals, absolute_import
from __future__ import print_function, division
from __future__ import with_statement

# Add your library functions here.

class ContextManager(object):

class CLI(ContextManager):

    def __init__(self, enode):
        self.enode = enode

    def __enter__(self):
        self.initial_prompt = self.enode._shells['bash']._prompt
        self.enode._shells['bash']._prompt = '>>> '
        self.enode._shells['bash'].send_command('scapy',matches='>>> ')
        return self

    def __exit__(self, type, value, traceback):
        self.enode._shells['bash']._prompt = self.initial_prompt
        self.enode._shells['bash'].send_command('exit()')

    def send_command(self, command):
        self.enode._shells['bash'].send_command(command)
        response = self.enode._shells['bash'].get_response()
        return response

__all__ = [
    'CLI'
]
<|endoftext|>


<commit_before># Licensed to libcloud.org under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# libcloud.org licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2009 RedRata Ltd

from libcloud.types import Provider, NodeState
from libcloud.base import ConnectionKey, Response, NodeDriver, NodeSize, Node
from libcloud.base import NodeImage
from copy import copy

# JSON is included in the standard library starting with Python 2.6.  For 2.5
# and 2.4, there's a simplejson egg at: http://pypi.python.org/pypi/simplejson
try: import json
except: import simplejson as json

# Defaults
API_CONTEXT = '/r'
API_HOST = 'api.rimuhosting.com'
API_PORT = (80,443)
API_SECURE = True

class RimuHostingException(BaseException):
    def __init__(self, error):
        self.error = error

    def __str__(self):
        return self.error

    def __repr__(self):
        return "<RimuHostingException '%s'>" % (self.error)

class RimuHostingResponse(Response):
    def __init__(self, response):
        self.body = response.read()
        self.status = response.status
        self.headers = dict(response.getheaders())
        self.error = response.reason

        self.object = self.parse_body()

    def parse_body(self):
        try:
            js = json.loads(self.body)
            if js[js.keys()[0]]['response_type'] == "ERROR":
                raise RimuHostingException(js[js.keys()[0]]['human_readable_message'])
            return js[js.keys()[0]]
        except ValueError:
            raise RimuHostingException('Could not parse body: %s' % (self.body))
        except KeyError:
            raise RimuHostingException('Could not parse body: %s' % (self.body))

class RimuHostingConnection(ConnectionKey):

    api_context = API_CONTEXT
    host = API_HOST
    port = API_PORT
    responseCls = RimuHostingResponse

    def __init__(self, key, secure=True):
        # override __init__ so that we can set secure of False for testing
        ConnectionKey.__init__(self,key,secure)

    def add_default_headers(self, headers):
        # We want JSON back from the server. Could be application/xml (but JSON
        # is better).
        headers['Accept'] = 'application/json'
        # Must encode all data as json, or override this header.
        headers['Content-Type'] = 'application/json'

        headers['Authorization'] = 'rimuhosting apikey=%s' % (self.key)
        return headers;

    def request(self, action, params={}, data='', headers={}, method='GET'):
        # Override this method to prepend the api_context
        return ConnectionKey.request(self, self.api_context + action, params, data, headers, method)

class RimuHostingNodeDriver(NodeDriver):
    type = Provider.RIMUHOSTING
    name = 'RimuHosting'
    connectionCls = RimuHostingConnection

    def __init__(self, key, host=API_HOST, port=API_PORT, api_context=API_CONTEXT, secure=API_SECURE):
        # Pass in some extra vars so that
        self.key = key
        self.secure = secure
        self.connection = self.connectionCls(key ,secure)
        self.connection.host = host
        self.connection.api_context = api_context
        self.connection.port = port
        self.connection.driver = self
        self.connection.connect()

    def _order_uri(self, node,resource):
        # Returns the order uri with its resourse appended.
        return "/orders/%s/%s" % (node.slug,resource)

    # TODO: Get the node state.
    def _to_node(self, order):
        n = Node(id=order['order_oid'],
                name=order['domain_name'],
                state=NodeState.RUNNING,
                public_ip=[order['allocated_ips']['primary_ip']]+order['allocated_ips']['secondary_ips'],
                private_ip=None,
                driver=self.connection.driver)
        n.slug = order['slug']
        return n

    def _to_size(self,plan):
        return NodeSize(id=plan['pricing_plan_code'],
            name=plan['pricing_plan_description'],
            ram=plan['minimum_memory_mb'],
            disk=plan['minimum_disk_gb'],
            bandwidth=plan['minimum_data_transfer_allowance_gb'],
            price=plan['monthly_recurring_fee_usd'],
            driver=self.connection.driver)

    def _to_image(self,image):
        return NodeImage(id=image['distro_code'],
            name=image['distro_description'],
            driver=self.connection.driver)

    def list_sizes(self):
        # Returns a list of sizes (aka plans)
        # Get plans. Note this is really just for libcloud. We are happy with any size.
        res = self.connection.request('/pricing-plans;server-type=VPS').object
        return map(lambda x : self._to_size(x), res['pricing_plan_infos'])

    def list_nodes(self):
        # Returns a list of Nodes
        # Will only include active ones.
        res = self.connection.request('/orders;include_inactive=N').object
        return map(lambda x : self._to_node(x), res['about_orders'])

    def list_images(self):
        # Get all base images.
        # TODO: add other image sources. (Such as a backup of a VPS)
        res = self.connection.request('/distributions').object
        return map(lambda x : self._to_image(x), res['distro_infos'])

    def reboot_node(self, node):
        # Reboot
        # PUT the state of RESTARTING to restart a VPS.
        # All data is encoded as JSON
        data = {'reboot_request':{'running_state':'RESTARTING'}}
        uri = self._order_uri(node,'vps/running-state')
        self.connection.request(uri,data=json.dumps(data),method='PUT')
        return True

    def destroy_node(self, node):
        # Shutdown a VPS.
        uri = self._order_uri(node,'vps')
        self.connection.request(uri,method='DELETE')
        return False

    def create_node(self, name, image, size, **kwargs):
        # Creates a RimuHosting instance
        #
        #   name    Must be a FQDN. e.g example.com.
        #   image   NodeImage from list_images
        #   size    NodeSize from list_sizes
        #
        # Keyword arguements supported:
        #
        #   billing_oid             If not set, a billing method is automatically picked.
        #   host_server_oid         The host server to set the VPS up on.
        #   vps_order_oid_to_clone  Clone another VPS to use as the image for the new VPS.
        #
        #   num_ips = 1         Number of IPs to allocate. Defaults to 1.
        #   extra_ip_reason     Reason for needing the extra IPS.
        #
        #   memory_mb           Memory to allocate to the VPS.
        #   disk_space_mb=4096  Diskspace to allocate to the VPS. Default is 4GB.
        #   disk_space_2_mb     Secondary disk size allocation. Disabled by default.
        #
        #   pricing_plan_code       Plan from list_sizes
        #
        #   control_panel       Control panel to install on the VPS.
        #   password            Password to set on the VPS.
        #
        #
        # Note we don't do much error checking in this because we the API to error out if there is a problem.
        data = {
            'instantiation_options':{'domain_name': name, 'distro': image.id},
            'pricing_plan_code': size.id,
        }

        if kwargs.has_key('control_panel'):
            data['instantiation_options']['control_panel'] = kwargs['control_panel']

        if kwargs.has_key('password'):
            data['instantiation_options']['password'] = kwargs['password']

        if kwargs.has_key('billing_oid'):
            #TODO check for valid oid.
            data['billing_oid'] = kwargs['billing_oid']

        if kwargs.has_key('host_server_oid'):
            data['host_server_oid'] = kwargs['host_server_oid']

        if kwargs.has_key('vps_order_oid_to_clone'):
            data['vps_order_oid_to_clone'] = kwargs['vps_order_oid_to_clone']

        if kwargs.has_key('num_ips') and int(kwargs['num_ips']) > 1:
            if not kwargs.has_key('extra_ip_reason'):
                raise RimuHostingException('Need an reason for having an extra IP')
            else:
                if not data.has_key('ip_request'):
                    data['ip_request'] = {}
                data['ip_request']['num_ips'] = int(kwargs['num_ips'])
                data['ip_request']['extra_ip_reason'] = kwargs['extra_ip_reason']

        if kwargs.has_key('memory_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['memory_mb'] = kwargs['memory_mb']

        if kwargs.has_key('disk_space_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['disk_space_mb'] = kwargs['disk_space_mb']

        if kwargs.has_key('disk_space_2_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['disk_space_2_mb'] = kwargs['disk_space_2_mb']

        res = self.connection.request('/orders/new-vps', method='POST', data=json.dumps({"new-vps":data})).object
        return self._to_node(res['about_order'])

<commit_msg>make destroy return True if it worked<commit_after># Licensed to libcloud.org under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# libcloud.org licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2009 RedRata Ltd

from libcloud.types import Provider, NodeState
from libcloud.base import ConnectionKey, Response, NodeDriver, NodeSize, Node
from libcloud.base import NodeImage
from copy import copy

# JSON is included in the standard library starting with Python 2.6.  For 2.5
# and 2.4, there's a simplejson egg at: http://pypi.python.org/pypi/simplejson
try: import json
except: import simplejson as json

# Defaults
API_CONTEXT = '/r'
API_HOST = 'api.rimuhosting.com'
API_PORT = (80,443)
API_SECURE = True

class RimuHostingException(BaseException):
    def __init__(self, error):
        self.error = error

    def __str__(self):
        return self.error

    def __repr__(self):
        return "<RimuHostingException '%s'>" % (self.error)

class RimuHostingResponse(Response):
    def __init__(self, response):
        self.body = response.read()
        self.status = response.status
        self.headers = dict(response.getheaders())
        self.error = response.reason

        self.object = self.parse_body()

    def parse_body(self):
        try:
            js = json.loads(self.body)
            if js[js.keys()[0]]['response_type'] == "ERROR":
                raise RimuHostingException(js[js.keys()[0]]['human_readable_message'])
            return js[js.keys()[0]]
        except ValueError:
            raise RimuHostingException('Could not parse body: %s' % (self.body))
        except KeyError:
            raise RimuHostingException('Could not parse body: %s' % (self.body))

class RimuHostingConnection(ConnectionKey):

    api_context = API_CONTEXT
    host = API_HOST
    port = API_PORT
    responseCls = RimuHostingResponse

    def __init__(self, key, secure=True):
        # override __init__ so that we can set secure of False for testing
        ConnectionKey.__init__(self,key,secure)

    def add_default_headers(self, headers):
        # We want JSON back from the server. Could be application/xml (but JSON
        # is better).
        headers['Accept'] = 'application/json'
        # Must encode all data as json, or override this header.
        headers['Content-Type'] = 'application/json'

        headers['Authorization'] = 'rimuhosting apikey=%s' % (self.key)
        return headers;

    def request(self, action, params={}, data='', headers={}, method='GET'):
        # Override this method to prepend the api_context
        return ConnectionKey.request(self, self.api_context + action, params, data, headers, method)

class RimuHostingNodeDriver(NodeDriver):
    type = Provider.RIMUHOSTING
    name = 'RimuHosting'
    connectionCls = RimuHostingConnection

    def __init__(self, key, host=API_HOST, port=API_PORT, api_context=API_CONTEXT, secure=API_SECURE):
        # Pass in some extra vars so that
        self.key = key
        self.secure = secure
        self.connection = self.connectionCls(key ,secure)
        self.connection.host = host
        self.connection.api_context = api_context
        self.connection.port = port
        self.connection.driver = self
        self.connection.connect()

    def _order_uri(self, node,resource):
        # Returns the order uri with its resourse appended.
        return "/orders/%s/%s" % (node.slug,resource)

    # TODO: Get the node state.
    def _to_node(self, order):
        n = Node(id=order['order_oid'],
                name=order['domain_name'],
                state=NodeState.RUNNING,
                public_ip=[order['allocated_ips']['primary_ip']]+order['allocated_ips']['secondary_ips'],
                private_ip=None,
                driver=self.connection.driver)
        n.slug = order['slug']
        return n

    def _to_size(self,plan):
        return NodeSize(id=plan['pricing_plan_code'],
            name=plan['pricing_plan_description'],
            ram=plan['minimum_memory_mb'],
            disk=plan['minimum_disk_gb'],
            bandwidth=plan['minimum_data_transfer_allowance_gb'],
            price=plan['monthly_recurring_fee_usd'],
            driver=self.connection.driver)

    def _to_image(self,image):
        return NodeImage(id=image['distro_code'],
            name=image['distro_description'],
            driver=self.connection.driver)

    def list_sizes(self):
        # Returns a list of sizes (aka plans)
        # Get plans. Note this is really just for libcloud. We are happy with any size.
        res = self.connection.request('/pricing-plans;server-type=VPS').object
        return map(lambda x : self._to_size(x), res['pricing_plan_infos'])

    def list_nodes(self):
        # Returns a list of Nodes
        # Will only include active ones.
        res = self.connection.request('/orders;include_inactive=N').object
        return map(lambda x : self._to_node(x), res['about_orders'])

    def list_images(self):
        # Get all base images.
        # TODO: add other image sources. (Such as a backup of a VPS)
        res = self.connection.request('/distributions').object
        return map(lambda x : self._to_image(x), res['distro_infos'])

    def reboot_node(self, node):
        # Reboot
        # PUT the state of RESTARTING to restart a VPS.
        # All data is encoded as JSON
        data = {'reboot_request':{'running_state':'RESTARTING'}}
        uri = self._order_uri(node,'vps/running-state')
        self.connection.request(uri,data=json.dumps(data),method='PUT')
        return True

    def destroy_node(self, node):
        # Shutdown a VPS.
        uri = self._order_uri(node,'vps')
        self.connection.request(uri,method='DELETE')
        return True

    def create_node(self, name, image, size, **kwargs):
        # Creates a RimuHosting instance
        #
        #   name    Must be a FQDN. e.g example.com.
        #   image   NodeImage from list_images
        #   size    NodeSize from list_sizes
        #
        # Keyword arguements supported:
        #
        #   billing_oid             If not set, a billing method is automatically picked.
        #   host_server_oid         The host server to set the VPS up on.
        #   vps_order_oid_to_clone  Clone another VPS to use as the image for the new VPS.
        #
        #   num_ips = 1         Number of IPs to allocate. Defaults to 1.
        #   extra_ip_reason     Reason for needing the extra IPS.
        #
        #   memory_mb           Memory to allocate to the VPS.
        #   disk_space_mb=4096  Diskspace to allocate to the VPS. Default is 4GB.
        #   disk_space_2_mb     Secondary disk size allocation. Disabled by default.
        #
        #   pricing_plan_code       Plan from list_sizes
        #
        #   control_panel       Control panel to install on the VPS.
        #   password            Password to set on the VPS.
        #
        #
        # Note we don't do much error checking in this because we the API to error out if there is a problem.
        data = {
            'instantiation_options':{'domain_name': name, 'distro': image.id},
            'pricing_plan_code': size.id,
        }

        if kwargs.has_key('control_panel'):
            data['instantiation_options']['control_panel'] = kwargs['control_panel']

        if kwargs.has_key('password'):
            data['instantiation_options']['password'] = kwargs['password']

        if kwargs.has_key('billing_oid'):
            #TODO check for valid oid.
            data['billing_oid'] = kwargs['billing_oid']

        if kwargs.has_key('host_server_oid'):
            data['host_server_oid'] = kwargs['host_server_oid']

        if kwargs.has_key('vps_order_oid_to_clone'):
            data['vps_order_oid_to_clone'] = kwargs['vps_order_oid_to_clone']

        if kwargs.has_key('num_ips') and int(kwargs['num_ips']) > 1:
            if not kwargs.has_key('extra_ip_reason'):
                raise RimuHostingException('Need an reason for having an extra IP')
            else:
                if not data.has_key('ip_request'):
                    data['ip_request'] = {}
                data['ip_request']['num_ips'] = int(kwargs['num_ips'])
                data['ip_request']['extra_ip_reason'] = kwargs['extra_ip_reason']

        if kwargs.has_key('memory_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['memory_mb'] = kwargs['memory_mb']

        if kwargs.has_key('disk_space_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['disk_space_mb'] = kwargs['disk_space_mb']

        if kwargs.has_key('disk_space_2_mb'):
            if not data.has_key('vps_parameters'):
                data['vps_parameters'] = {}
            data['vps_parameters']['disk_space_2_mb'] = kwargs['disk_space_2_mb']

        res = self.connection.request('/orders/new-vps', method='POST', data=json.dumps({"new-vps":data})).object
        return self._to_node(res['about_order'])

<|endoftext|>