<commit_before>#!/usr/bin/env python

from __future__ import print_function
import libvirt
import sys
import os
import glob
import subprocess
import argparse

conn = libvirt.open("qemu:///system")
if not conn:
    print("Failed to open connection to the hypervisor")
    sys.exit(1)

parser = argparse.ArgumentParser(description="Cleanup libvirt resources.")
parser.add_argument("cloud", type=str, help="Name of the cloud")
parser.add_argument("nodenumber", type=int, help="Number of nodes")
parser.add_argument("cloudbr", type=str, help="Name of the virtual bridge")
parser.add_argument("public_vlan", type=str, help="Name of the public vlan")

args = parser.parse_args()

def remove_files(files):
    for f in glob.glob(files):
        print("removing {0}".format(f))
        os.remove(f)

def main():
    devnull = open(os.devnull, "w")
    allnodenames = ["admin"]
    allnodenames.extend("node%s"%i for i in range(1, args.nodenumber+20))

    for nodename in allnodenames:
        try:
            vm = "{0}-{1}".format(args.cloud, nodename)
            dom = conn.lookupByName(vm)
            if dom.isActive():
                print("destroying {0}".format(dom.name()))
                dom.destroy()
            if dom.ID():
                print("undefining {0}".format(dom.name()))
                dom.undefine()

            machine = "{0}-{1}".format("qemu", vm)
            machinectl = os.access("/usr/bin/machinectl", os.X_OK)
            machine_status = subprocess.call(["machinectl", "status", machine], stdout=devnull, stderr=subprocess.STDOUT)
            if (machinectl == 0 and machine_status == 0):
                subprocess.call(["machinectl", "terminate", machine]) # workaround bnc#916518
        except libvirt.libvirtError:
            print("...skipping undefined domains")

    try:
        net_name = "{0}-admin".format(args.cloud)
        network = conn.networkLookupByName(net_name)
        print("destroying {0}".format(net_name))
        network.destroy()
        print("undefining {0}".format(net_name))
        network.undefine()
    except libvirt.libvirtError:
        print("...skipping undefined network")

    remove_files("/var/run/libvirt/qemu/{0}-*.xml".format(args.cloud))
    remove_files("/var/lib/libvirt/network/{0}-*.xml".format(args.cloud))
    remove_files("/etc/sysconfig/network/ifcfg-{0}.{1}".format(args.cloudbr, args.public_vlan))

if __name__ == "__main__":
    main()
<commit_msg>cleanup.py: fix machinectl<commit_after>#!/usr/bin/env python

from __future__ import print_function
import libvirt
import sys
import os
import glob
import subprocess
import argparse

conn = libvirt.open("qemu:///system")
if not conn:
    print("Failed to open connection to the hypervisor")
    sys.exit(1)

parser = argparse.ArgumentParser(description="Cleanup libvirt resources.")
parser.add_argument("cloud", type=str, help="Name of the cloud")
parser.add_argument("nodenumber", type=int, help="Number of nodes")
parser.add_argument("cloudbr", type=str, help="Name of the virtual bridge")
parser.add_argument("public_vlan", type=str, help="Name of the public vlan")

args = parser.parse_args()

def remove_files(files):
    for f in glob.glob(files):
        print("removing {0}".format(f))
        os.remove(f)

def main():
    devnull = open(os.devnull, "w")
    allnodenames = ["admin"]
    allnodenames.extend("node%s"%i for i in range(1, args.nodenumber+20))

    for nodename in allnodenames:
        try:
            vm = "{0}-{1}".format(args.cloud, nodename)
            dom = conn.lookupByName(vm)
            if dom.isActive():
                print("destroying {0}".format(dom.name()))
                dom.destroy()
            if dom.ID():
                print("undefining {0}".format(dom.name()))
                dom.undefine()

            machine = "{0}-{1}".format("qemu", vm)
            machinectl = os.access("/usr/bin/machinectl", os.X_OK)
            if machinectl:
                machine_status = subprocess.call(["machinectl", "status", machine], stdout=devnull, stderr=subprocess.STDOUT)
                if machine_status == 0:
                    subprocess.call(["machinectl", "terminate", machine]) # workaround bnc#916518
        except libvirt.libvirtError:
            print("...skipping undefined domains")

    try:
        net_name = "{0}-admin".format(args.cloud)
        network = conn.networkLookupByName(net_name)
        print("destroying {0}".format(net_name))
        network.destroy()
        print("undefining {0}".format(net_name))
        network.undefine()
    except libvirt.libvirtError:
        print("...skipping undefined network")

    remove_files("/var/run/libvirt/qemu/{0}-*.xml".format(args.cloud))
    remove_files("/var/lib/libvirt/network/{0}-*.xml".format(args.cloud))
    remove_files("/etc/sysconfig/network/ifcfg-{0}.{1}".format(args.cloudbr, args.public_vlan))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import requests

import dcos

import shakedown

def dcos_url():
    """Return the DCOS URL as configured in the DCOS library. This is
    equivalent to the value of '--dcos_url' passed into Shakedown on the
    command line.
    :return: DCOS cluster URL as a string
    """
    return dcos.util.get_config().get('core.dcos_url')

def dcos_service_url(service):
    """Return the URL of a service running on DCOS, based on the value of
    shakedown.dcos.dcos_url() and the service name.
    :param service: the name of a registered DCOS service, as a string
    :return: the full DCOS service URL, as a string
    """
    return '/'.join([dcos_url(), 'service', service])

def master_ip():
    """Returns the public IP address of the DCOS master.
    return: DCOS IP address as a string
    """
    return requests.get(dcos_url() + '/metadata').json()["PUBLIC_IPV4"].strip()
<commit_msg>Retrieve master IP via `dcos.mesos.DCOSClient()`<commit_after>import os
import dcos

import shakedown

def dcos_url():
    """Return the DCOS URL as configured in the DCOS library. This is
    equivalent to the value of '--dcos_url' passed into Shakedown on the
    command line.
    :return: DCOS cluster URL as a string
    """
    return dcos.util.get_config().get('core.dcos_url')

def dcos_service_url(service):
    """Return the URL of a service running on DCOS, based on the value of
    shakedown.dcos.dcos_url() and the service name.
    :param service: the name of a registered DCOS service, as a string
    :return: the full DCOS service URL, as a string
    """
    return '/'.join([dcos_url(), 'service', service])

def master_ip():
    """Returns the public IP address of the DCOS master.
    return: DCOS IP address as a string
    """
    return dcos.mesos.DCOSClient().metadata().get('PUBLIC_IPV4')
<|endoftext|>


<commit_before>import argparse
import ConfigParser
import sys
import os
import pkg_resources
import time
from datetime import datetime, timedelta
from termcolor import colored
from fake_section_head import FakeSectionHead
from live_logs import download_live_logs
from s3_logs import download_s3_logs
from tail import start_tail
from grep import grep_files
from cat import cat_files

VERSION = pkg_resources.require("singularity-logfetch")[0].version
CONF_READ_ERR_FORMAT = 'Could not load config from {0} due to {1}'
DEFAULT_CONF_DIR = os.path.expanduser('~/.logfetch')
DEFAULT_CONF_FILE = 'default'
DEFAULT_PARALLEL_FETCHES = 10
DEFAULT_CHUNK_SIZE = 8192
DEFAULT_DEST = os.path.expanduser('~/.logfetch_cache')
DEFAULT_TASK_COUNT = 10
DEFAULT_DAYS = 7
DEFAULT_S3_PATTERN = '%requestId/%Y/%m/%taskId_%index-%s-%filename'

def exit(reason, color='red'):
  sys.stderr.write(colored(reason, color) + '\n')
  sys.exit(1)

def tail_logs(args):
  try:
    start_tail(args)
  except KeyboardInterrupt:
    exit('Stopping logtail...', 'magenta')

def fetch_logs(args):
  try:
    check_dest(args)
    all_logs = []
    all_logs += download_s3_logs(args)
    all_logs += download_live_logs(args)
    grep_files(args, all_logs)
  except KeyboardInterrupt:
    exit('Stopping logfetch...', 'magenta')

def cat_logs(args):
  try:
    check_dest(args)
    all_logs = []
    all_logs += download_s3_logs(args)
    all_logs += download_live_logs(args)
    cat_files(args, all_logs)
  except KeyboardInterrupt:
    exit('Stopping logcat...', 'magenta')

def check_dest(args):
  if not os.path.exists(args.dest):
    os.makedirs(args.dest)

def check_args(args):
  if args.deployId and not args.requestId:
    exit("Must specify request-id (-r) when specifying deploy-id")
  elif not args.requestId and not args.deployId and not args.taskId:
    exit('Must specify one of\n -t task-id\n -r request-id and -d deploy-id\n -r request-id')

def convert_to_date(args, argument):
    try:
      if isinstance(argument, datetime):
        return argument
      else:
        val = datetime.utcnow() - timedelta(days=int(argument))
    except:
      try:
        if args.zone:
            timestring = '{0} {1}'.format(argument, datetime.now().strftime("%H:%M:%S")) if len(argument) < 11 else argument
            val = datetime.utcfromtimestamp(time.mktime(datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S").timetuple()))
        else:
            timestring = '{0} {1}'.format(argument, datetime.utcnow().strftime("%H:%M:%S")) if len(argument) < 11 else argument
            val = datetime.strptime('{0} UTC'.format(timestring), "%Y-%m-%d %H:%M:%S %Z")
      except:
          exit('Start/End days value must be either a number of days or a date in format "%Y-%m-%d %H:%M:%S" or "%Y-%m-%d"')
    return val

def fetch():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest='conf_folder', help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest='conf_file', help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {
    "num_parallel_fetches" : DEFAULT_PARALLEL_FETCHES,
    "chunk_size" : DEFAULT_CHUNK_SIZE,
    "dest" : DEFAULT_DEST,
    "task_count" : DEFAULT_TASK_COUNT,
    "start_days" : datetime.utcnow() - timedelta(days=DEFAULT_DAYS),
    "file_pattern" : DEFAULT_S3_PATTERN,
    "end_days" : datetime.utcnow()
  }

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logfetch")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-tc","--task-count", dest="task_count", help="Number of recent tasks per request to fetch logs from")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of task to fetch logs for (can be a glob)")
  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d'")
  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no newer than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d' (defaults to None/now)")
  parser.add_argument("-l", "--log-type", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
  parser.add_argument("-p", "--file-pattern", dest="file_pattern", help="S3 uploader file pattern")
  parser.add_argument("-nn", "--no-name-fetch-off", dest="no_name_fetch_off", help="If a logtype matcher is specified, but the s3 log pattern does not include file name, don't download any s3 files", action="store_true")
  parser.add_argument("-g", "--grep", dest="grep", help="Regex to grep for (normal grep syntax) or a full grep command")
  parser.add_argument("-z", "--local-zone", dest="zone", help="If specified, input times in the local time zone and convert to UTC, if not specified inputs are assumed to be UTC", action="store_true")
  parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  check_args(args)
  args.start_days = convert_to_date(args, args.start_days)
  args.end_days = convert_to_date(args, args.end_days)

  args.dest = os.path.expanduser(args.dest)

  fetch_logs(args)

def cat():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {
    "num_parallel_fetches" : DEFAULT_PARALLEL_FETCHES,
    "chunk_size" : DEFAULT_CHUNK_SIZE,
    "dest" : DEFAULT_DEST,
    "task_count" : DEFAULT_TASK_COUNT,
    "start_days" : datetime.now() - timedelta(days=DEFAULT_DAYS),
    "file_pattern" : DEFAULT_S3_PATTERN,
    "end_days" : datetime.now()
  }

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity and cat to stdout. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logcat")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-tc","--task-count", dest="taskCount", help="Number of recent tasks per request to fetch logs from")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d'")
  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no newer than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d' (defaults to None/now)")
  parser.add_argument("-l", "--logtype", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
  parser.add_argument("-p", "--file-pattern", dest="file_pattern", help="S3 uploader file pattern")
  parser.add_argument("-nn", "--no-name-fetch-off", dest="no_name_fetch_off", help="If a logtype matcher is specified, but the s3 log pattern does not include file name, don't download any s3 files", action="store_true")
  parser.add_argument("-z", "--local-zone", dest="zone", help="If specified, input times in the local time zone and convert to UTC, if not specified inputs are assumed to be UTC", action="store_true")
  parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  check_args(args)
  args.start_days = convert_to_date(args, args.start_days)
  args.end_days = convert_to_date(args, args.end_days)

  args.dest = os.path.expanduser(args.dest)

  cat_logs(args)

def tail():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {'verbose': False}

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Tail log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logtail")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-g", "--grep", dest="grep", help="String to grep for")
  parser.add_argument("-l", "--logfile", dest="logfile", help="Logfile path/name to tail (ie 'logs/access.log')")
  parser.add_argument("-V", "--verbose", dest="verbose", help="more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  if not args.logfile:
    exit("Must specify logfile to tail (-l)")
  check_args(args)

  args.dest = os.path.expanduser(args.dest)

  tail_logs(args)
<commit_msg>utcnow default<commit_after>import argparse
import ConfigParser
import sys
import os
import pkg_resources
import time
from datetime import datetime, timedelta
from termcolor import colored
from fake_section_head import FakeSectionHead
from live_logs import download_live_logs
from s3_logs import download_s3_logs
from tail import start_tail
from grep import grep_files
from cat import cat_files

VERSION = pkg_resources.require("singularity-logfetch")[0].version
CONF_READ_ERR_FORMAT = 'Could not load config from {0} due to {1}'
DEFAULT_CONF_DIR = os.path.expanduser('~/.logfetch')
DEFAULT_CONF_FILE = 'default'
DEFAULT_PARALLEL_FETCHES = 10
DEFAULT_CHUNK_SIZE = 8192
DEFAULT_DEST = os.path.expanduser('~/.logfetch_cache')
DEFAULT_TASK_COUNT = 10
DEFAULT_DAYS = 7
DEFAULT_S3_PATTERN = '%requestId/%Y/%m/%taskId_%index-%s-%filename'

def exit(reason, color='red'):
  sys.stderr.write(colored(reason, color) + '\n')
  sys.exit(1)

def tail_logs(args):
  try:
    start_tail(args)
  except KeyboardInterrupt:
    exit('Stopping logtail...', 'magenta')

def fetch_logs(args):
  try:
    check_dest(args)
    all_logs = []
    all_logs += download_s3_logs(args)
    all_logs += download_live_logs(args)
    grep_files(args, all_logs)
  except KeyboardInterrupt:
    exit('Stopping logfetch...', 'magenta')

def cat_logs(args):
  try:
    check_dest(args)
    all_logs = []
    all_logs += download_s3_logs(args)
    all_logs += download_live_logs(args)
    cat_files(args, all_logs)
  except KeyboardInterrupt:
    exit('Stopping logcat...', 'magenta')

def check_dest(args):
  if not os.path.exists(args.dest):
    os.makedirs(args.dest)

def check_args(args):
  if args.deployId and not args.requestId:
    exit("Must specify request-id (-r) when specifying deploy-id")
  elif not args.requestId and not args.deployId and not args.taskId:
    exit('Must specify one of\n -t task-id\n -r request-id and -d deploy-id\n -r request-id')

def convert_to_date(args, argument):
    try:
      if isinstance(argument, datetime):
        return argument
      else:
        val = datetime.utcnow() - timedelta(days=int(argument))
    except:
      try:
        if args.zone:
            timestring = '{0} {1}'.format(argument, datetime.now().strftime("%H:%M:%S")) if len(argument) < 11 else argument
            val = datetime.utcfromtimestamp(time.mktime(datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S").timetuple()))
        else:
            timestring = '{0} {1}'.format(argument, datetime.utcnow().strftime("%H:%M:%S")) if len(argument) < 11 else argument
            val = datetime.strptime('{0} UTC'.format(timestring), "%Y-%m-%d %H:%M:%S %Z")
      except:
          exit('Start/End days value must be either a number of days or a date in format "%Y-%m-%d %H:%M:%S" or "%Y-%m-%d"')
    return val

def fetch():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest='conf_folder', help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest='conf_file', help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {
    "num_parallel_fetches" : DEFAULT_PARALLEL_FETCHES,
    "chunk_size" : DEFAULT_CHUNK_SIZE,
    "dest" : DEFAULT_DEST,
    "task_count" : DEFAULT_TASK_COUNT,
    "start_days" : datetime.utcnow() - timedelta(days=DEFAULT_DAYS),
    "file_pattern" : DEFAULT_S3_PATTERN,
    "end_days" : datetime.utcnow()
  }

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logfetch")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-tc","--task-count", dest="task_count", help="Number of recent tasks per request to fetch logs from")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of task to fetch logs for (can be a glob)")
  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d'")
  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no newer than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d' (defaults to None/now)")
  parser.add_argument("-l", "--log-type", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
  parser.add_argument("-p", "--file-pattern", dest="file_pattern", help="S3 uploader file pattern")
  parser.add_argument("-nn", "--no-name-fetch-off", dest="no_name_fetch_off", help="If a logtype matcher is specified, but the s3 log pattern does not include file name, don't download any s3 files", action="store_true")
  parser.add_argument("-g", "--grep", dest="grep", help="Regex to grep for (normal grep syntax) or a full grep command")
  parser.add_argument("-z", "--local-zone", dest="zone", help="If specified, input times in the local time zone and convert to UTC, if not specified inputs are assumed to be UTC", action="store_true")
  parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  check_args(args)
  args.start_days = convert_to_date(args, args.start_days)
  args.end_days = convert_to_date(args, args.end_days)

  args.dest = os.path.expanduser(args.dest)

  fetch_logs(args)

def cat():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {
    "num_parallel_fetches" : DEFAULT_PARALLEL_FETCHES,
    "chunk_size" : DEFAULT_CHUNK_SIZE,
    "dest" : DEFAULT_DEST,
    "task_count" : DEFAULT_TASK_COUNT,
    "start_days" : datetime.utcnow() - timedelta(days=DEFAULT_DAYS),
    "file_pattern" : DEFAULT_S3_PATTERN,
    "end_days" : datetime.utcnow()
  }

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Fetch log files from Singularity and cat to stdout. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logcat")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-tc","--task-count", dest="taskCount", help="Number of recent tasks per request to fetch logs from")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
  parser.add_argument("-o", "--dest", dest="dest", help="Destination directory")
  parser.add_argument("-n", "--num-parallel-fetches", dest="num_parallel_fetches", help="Number of fetches to make at once", type=int)
  parser.add_argument("-cs", "--chunk-size", dest="chunk_size", help="Chunk size for writing from response to filesystem", type=int)
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-s", "--start-days", dest="start_days", help="Search for logs no older than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d'")
  parser.add_argument("-e", "--end-days", dest="end_days", help="Search for logs no newer than this, can be an integer number of days or date in format '%Y-%m-%d %H:%M:%S' or '%Y-%m-%d' (defaults to None/now)")
  parser.add_argument("-l", "--logtype", dest="logtype", help="Logfile type to downlaod (ie 'access.log'), can be a glob (ie *.log)")
  parser.add_argument("-p", "--file-pattern", dest="file_pattern", help="S3 uploader file pattern")
  parser.add_argument("-nn", "--no-name-fetch-off", dest="no_name_fetch_off", help="If a logtype matcher is specified, but the s3 log pattern does not include file name, don't download any s3 files", action="store_true")
  parser.add_argument("-z", "--local-zone", dest="zone", help="If specified, input times in the local time zone and convert to UTC, if not specified inputs are assumed to be UTC", action="store_true")
  parser.add_argument("-V", "--verbose", dest="verbose", help="Print more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  check_args(args)
  args.start_days = convert_to_date(args, args.start_days)
  args.end_days = convert_to_date(args, args.end_days)

  args.dest = os.path.expanduser(args.dest)

  cat_logs(args)

def tail():
  conf_parser = argparse.ArgumentParser(version=VERSION, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, add_help=False)
  conf_parser.add_argument("-f", "--conf-folder", dest="conf_folder", help="specify a folder for config files to live")
  conf_parser.add_argument("-c", "--conf-file", dest="conf_file", help="Specify config file within the conf folder", metavar="FILE")
  args, remaining_argv = conf_parser.parse_known_args()
  conf_dir = args.conf_folder if args.conf_folder else DEFAULT_CONF_DIR
  conf_file = os.path.expanduser(conf_dir + '/' + args.conf_file) if args.conf_file else os.path.expanduser(conf_dir + '/' + DEFAULT_CONF_FILE)
  config = ConfigParser.SafeConfigParser()

  defaults = {'verbose': False}

  try:
    config.readfp(FakeSectionHead(open(os.path.expanduser(conf_file))))
    defaults.update(dict(config.items("Defaults")))
  except Exception, err:
    sys.stderr.write(CONF_READ_ERR_FORMAT.format(conf_file, err) + '\n')

  parser = argparse.ArgumentParser(parents=[conf_parser], description="Tail log files from Singularity. One can specify either a TaskId, RequestId and DeployId, or RequestId",
          prog="logtail")

  parser.set_defaults(**defaults)
  parser.add_argument("-t", "--task-id", dest="taskId", help="TaskId of task to fetch logs for")
  parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)")
  parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)")
  parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)")
  parser.add_argument("-g", "--grep", dest="grep", help="String to grep for")
  parser.add_argument("-l", "--logfile", dest="logfile", help="Logfile path/name to tail (ie 'logs/access.log')")
  parser.add_argument("-V", "--verbose", dest="verbose", help="more verbose output", action='store_true')

  args = parser.parse_args(remaining_argv)

  if not args.logfile:
    exit("Must specify logfile to tail (-l)")
  check_args(args)

  args.dest = os.path.expanduser(args.dest)

  tail_logs(args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016 Onestein (<http://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo import api, models

class AccountChartTemplate(models.Model):
    _inherit = "account.chart.template"

    @api.multi
    def try_loading_for_current_company(self):
        self.ensure_one()
        res = super(AccountChartTemplate,
                    self).try_loading_for_current_company()
        company_list = self.env['res.company'].search([])
        for company in company_list:
            # If we don't have any chart of account on this company,
            # install this chart of account
            if not company.chart_template_id:
                wizard = self.env['wizard.multi.charts.accounts'].create({
                    'company_id': company.id,
                    'chart_template_id': self.id,
                    'code_digits': self.code_digits,
                    'transfer_account_id': self.transfer_account_id.id,
                    'currency_id': self.currency_id.id,
                    'bank_account_code_prefix': self.bank_account_code_prefix,
                    'cash_account_code_prefix': self.cash_account_code_prefix,
                })
                wizard.onchange_chart_template_id()
                wizard.execute()
        return res
<commit_msg>Updated account_chart_template_multicompany<commit_after># -*- coding: utf-8 -*-
# Copyright 2016 Onestein (<http://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo import api, models

class AccountChartTemplate(models.Model):
    _inherit = "account.chart.template"

    @api.multi
    def try_loading_for_current_company(self):
        self.ensure_one()
        res = super(AccountChartTemplate,
                    self).try_loading_for_current_company()
        company_list = self.env['res.company'].search([
            ('id', '!=', self.env.user.company_id.id)
        ])
        for company in company_list:
            # If we don't have any chart of account on this company,
            # install this chart of account
            if not company.chart_template_id:
                wizard = self.env['wizard.multi.charts.accounts'].create({
                    'company_id': company.id,
                    'chart_template_id': self.id,
                    'code_digits': self.code_digits,
                    'transfer_account_id': self.transfer_account_id.id,
                    'currency_id': self.currency_id.id,
                    'bank_account_code_prefix': self.bank_account_code_prefix,
                    'cash_account_code_prefix': self.cash_account_code_prefix,
                })
                wizard.onchange_chart_template_id()
                wizard.execute()
        return res
<|endoftext|>


<commit_before>#!/usr/bin/env python

import math
import numpy
import operator
import vtk

class RefineSurface:

    def __init__(self, pdata):
        """
        Constructor
        @param pdata vtkPolyData instance
        """
        self.polydata = vtk.vtkPolyData()
        self.points = vtk.vtkPoints()

        # populate with input data
        points = pdata.GetPoints()
        numPoints = points.GetNumberOfPoints()
        for i in range(numPoints):
            self.points.InsertNextPoint(points.GetPoint(i))
        self.polydata.SetPoints(self.points)

        polys = pdata.GetPolys()
        numPolys = polys.GetNumberOfCells()
        self.polydata.Allocate(numPolys, 1)
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        for i in range(numPolys):
            polys.GetNextCell(ptIds)
            self.polydata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    def getVtkPolyData(self):
        """
        Get the vtkPolyData instance associated with the grid
        @return vtkPolyData instance
        """
        return self.polydata

    def computeNormal(self, verts):
        """
        Compute the normal to the polygon by taking the first three vertices
        that are not degenerate
        @param vertices as numpy arrays
        @return normal vector or (0., 0., 0.), tangential lengths, and area
        """
        pa = verts[0]
        pb = verts[1] - pa
        for i in range(2, len(verts)):
            pc = verts[i] - pa
            area = numpy.cross(pb, pc)
            areaVal = numpy.sqrt(numpy.dot(area, area))
            if areaVal > 0.0:
                return area/areaVal, pb, pc, areaVal
        return numpy.array([0.,0.,0.]), pb, pc, 0.0

    def refine(self, max_edge_length):
        """
        Refine each cell by adding points on edges longer than max_edge_length
        @param max_edge_length maximum edge length
        @note operation is in place
        """

        # iterate over the polys
        polys = self.polydata.GetPolys()
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        cells = []
        edge2PtIds = {}
        for iPoly in range(polys.GetNumberOfCells()):

            polys.GetNextCell(ptIds)
            numPts = ptIds.GetNumberOfIds()
            polyPtIds = []

            if numPts < 3:
                # need at least three points, next polygon
                continue

            # compute the two tangential unit vectors
            uVec, vVec, normal = self.computeUVNormal(ptIds)
            if normal.dot(normal) == 0:
                # zero area polygon, nothing to do
                continue

            # iterate over edges
            numNodes = ptIds.GetNumberOfIds()
            for iNode in range(numNodes):
                i0 = ptIds.GetId(iNode)
                i1 = ptIds.GetId((iNode + 1) % numNodes)
                edge = (i0, i1)
                edgeCompl = (i1, i0)

                edgePtIds = edge2PtIds.get(edgeCompl, [])
                if len(edgePtIds) > 0:
                    # edge has already been split, take edge and reverse order
                    edgePtIds.reverse()
                    edgePtIds = [i0,] + edgePtIds[:-1]
                    polyPtIds += edgePtIds
                    continue

                p0 = numpy.array(self.points.GetPoint(i0))
                p1 = numpy.array(self.points.GetPoint(i1))
                dEdge = p1 - p0
                edgeLength = numpy.sqrt(dEdge.dot(dEdge))
                numSegs = int(max(1, math.ceil(edgeLength/max_edge_length)))

                # add points along edge
                d10 = (p1 - p0) / float(numSegs)
                edgePtIds.append(i0)
                for iSeg in range(1, numSegs):
                    pt = p0 + iSeg*d10
                    # index of point to add
                    ptId = self.points.GetNumberOfPoints()
                    # insert point
                    self.points.InsertNextPoint(pt)
                    edgePtIds.append(ptId)

                edge2PtIds[edge] = edgePtIds
                polyPtIds += edgePtIds

            # triangulate the cell
            polyCells = self.triangulatePolygon(uVec, vVec, polyPtIds)
            cells += polyCells

        # build the output vtkPolyData object
        pdata = vtk.vtkPolyData()
        ptIds = vtk.vtkIdList()
        pdata.SetPoints(self.points)
        numPolys = len(cells)
        pdata.Allocate(numPolys, 1)
        for cell in cells:
            numPts = len(cell)
            ptIds.SetNumberOfIds(numPts)
            for j in range(numPts):
                ptIds.SetId(j, cell[j])
            pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

        # Reset the polydata struct
        self.polydata = pdata

    def computeUVNormal(self, ptIds):
        """
        Compute the two tangential unit vectors and the normal vector
        @param ptIds point indices
        @return u vector, v vector, normal
        """
        uVec = numpy.zeros((3,), numpy.float64)
        vVec = numpy.zeros((3,), numpy.float64)
        numPts = ptIds.GetNumberOfIds()
        if numPts < 3:
            return uVec, vVec, uVec
        p0 = numpy.array(self.points.GetPoint(ptIds.GetId(0)))
        for i in range(1, numPts - 1):
            dp1 = numpy.array(self.points.GetPoint(ptIds.GetId(i))) - p0
            dp2 = numpy.array(self.points.GetPoint(ptIds.GetId(i + 1))) - p0
            perp = numpy.cross(dp1, dp2)
            pDotp = perp.dot(perp)
            if pDotp > 0:
                normal = perp / numpy.sqrt(pDotp)
                uVec = dp1 / numpy.sqrt(dp1.dot(dp1))
                vVec = numpy.cross(normal, uVec)
                return uVec, vVec, normal
        return uVec, vVec, numpy.zeros((3,), numpy.float64)

    def triangulatePolygon(self, uVec, vVec, polyPtIds):
        """
        Triangulate polygon using the uVec x vVec projection
        @param uVec unit vector tangential to the polygon
        @param vVec second unit vector tangential to the polygon
        @param polyPtIds list of polygon's point indices
        @return list of cells (list of point indices)
        """

        numPts = len(polyPtIds)
        if numPts < 3:
            return []
        elif numPts == 3:
            # no need to do any triangulation, just return the cell
            return [polyPtIds]

        pts = vtk.vtkPoints()
        pts.SetNumberOfPoints(numPts)

        # project each point onto the plane
        pt = numpy.zeros((3,), numpy.float64)
        for i in range(numPts):
            p = numpy.array(self.points.GetPoint(polyPtIds[i]))
            pt[0:2] = p.dot(uVec), p.dot(vVec)
            pts.SetPoint(i, pt)

        pdata = vtk.vtkPolyData()
        pdata.SetPoints(pts)

        delaunay = vtk.vtkDelaunay2D()
        delaunay.SetTolerance(1.e-5)
        delaunay.SetAlpha(0.0)
        if vtk.VTK_MAJOR_VERSION >= 6:
            delaunay.SetInputData(pdata)
        else:
            delaunay.SetInput(pdata)

        delaunay.Update()

        ugrid = delaunay.GetOutput()
        cells = []
        numCells = ugrid.GetNumberOfCells()
        if numCells == 0:
            # Delaunay triangulation failed
            # not sure why this is happening...
            return [polyPtIds]
        else:
            for i in range(numCells):
                ptIds = ugrid.GetCell(i).GetPointIds()
                cell = []
                for j in range(ptIds.GetNumberOfIds()):
                    localId = ptIds.GetId(j)
                    ptId = polyPtIds[localId]
                    cell.append(ptId)
                cells.append(cell)
        return cells

##############################################################################

def printVtkPolyData(pdata):

    points = pdata.GetPoints()
    polys = pdata.GetPolys()
    ptIds = vtk.vtkIdList()
    numPolys = polys.GetNumberOfCells()
    print 'Number of polygons: {}'.format(numPolys)
    polys.InitTraversal()
    for i in range(numPolys):
        cell = polys.GetNextCell(ptIds)
        numPts = ptIds.GetNumberOfIds()
        print '\tCell {} has {} points: '.format(i, numPts)
        for j in range(numPts):
            ptId = ptIds.GetId(j)
            pt = points.GetPoint(ptId)
            print '\t\t{} -> {}'.format(ptId, pt)

def testNoRefinement():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((1., 0., 0.))
    points.InsertNextPoint((1., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.5)
    pdata2 = rs.getVtkPolyData()
    assert(pdata2.GetNumberOfPolys() == 1)

def testAddingThreePointsThenMore():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.1)
    #printVtkPolyData(rs.getVtkPolyData())
    assert(rs.getVtkPolyData().GetNumberOfPolys() == 4)
    #rs.refine(max_edge_length=0.1)
    #assert(rs.getVtkPolyData().GetNumberOfPolys() == 104)

def testStartingWithTwoCells():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))
    points.InsertNextPoint((0., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    pdata.Allocate(2, 1)
    ptIds = vtk.vtkIdList()
    ptIds.SetNumberOfIds(3)
    ptIds.SetId(0, 0)
    ptIds.SetId(1, 1)
    ptIds.SetId(2, 2)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)
    ptIds.SetId(0, 2)
    ptIds.SetId(1, 3)
    ptIds.SetId(2, 0)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.1)
    pdata2 = rs.getVtkPolyData()
    #printVtkPolyData(pdata2)
    assert(rs.getVtkPolyData().GetNumberOfPolys() == 8)
    #rs.refine(max_edge_length=0.1)
    #assert(rs.getVtkPolyData().GetNumberOfPolys() == 208)

if __name__ == '__main__':
    testNoRefinement()
    testAddingThreePointsThenMore()
    testStartingWithTwoCells()
<commit_msg> working on the triangulate method<commit_after>#!/usr/bin/env python

import math
import numpy
import operator
import vtk
#import triangle

class RefineSurface:

    def __init__(self, pdata):
        """
        Constructor
        @param pdata vtkPolyData instance
        """
        self.polydata = vtk.vtkPolyData()
        self.points = vtk.vtkPoints()

        # populate with input data
        points = pdata.GetPoints()
        numPoints = points.GetNumberOfPoints()
        for i in range(numPoints):
            self.points.InsertNextPoint(points.GetPoint(i))
        self.polydata.SetPoints(self.points)

        polys = pdata.GetPolys()
        numPolys = polys.GetNumberOfCells()
        self.polydata.Allocate(numPolys, 1)
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        for i in range(numPolys):
            polys.GetNextCell(ptIds)
            self.polydata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    def getVtkPolyData(self):
        """
        Get the vtkPolyData instance associated with the grid
        @return vtkPolyData instance
        """
        return self.polydata

    def computeNormal(self, verts):
        """
        Compute the normal to the polygon by taking the first three vertices
        that are not degenerate
        @param vertices as numpy arrays
        @return normal vector or (0., 0., 0.), tangential lengths, and area
        """
        pa = verts[0]
        pb = verts[1] - pa
        for i in range(2, len(verts)):
            pc = verts[i] - pa
            area = numpy.cross(pb, pc)
            areaVal = numpy.sqrt(numpy.dot(area, area))
            if areaVal > 0.0:
                return area/areaVal, pb, pc, areaVal
        return numpy.array([0.,0.,0.]), pb, pc, 0.0

    def refine(self, max_edge_length):
        """
        Refine each cell by adding points on edges longer than max_edge_length
        @param max_edge_length maximum edge length
        @note operation is in place
        """

        # iterate over the polys
        polys = self.polydata.GetPolys()
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        cells = []
        edge2PtIds = {}
        for iPoly in range(polys.GetNumberOfCells()):

            polys.GetNextCell(ptIds)
            numPts = ptIds.GetNumberOfIds()
            polyPtIds = []

            if numPts < 3:
                # need at least three points, next polygon
                continue

            # compute the two tangential unit vectors
            uVec, vVec, normal = self.computeUVNormal(ptIds)
            if normal.dot(normal) == 0:
                # zero area polygon, nothing to do
                continue

            # iterate over edges
            numNodes = ptIds.GetNumberOfIds()
            for iNode in range(numNodes):
                i0 = ptIds.GetId(iNode)
                i1 = ptIds.GetId((iNode + 1) % numNodes)
                edge = (i0, i1)
                edgeCompl = (i1, i0)

                edgePtIds = edge2PtIds.get(edgeCompl, [])
                if len(edgePtIds) > 0:
                    # edge has already been split, take edge and reverse order
                    edgePtIds.reverse()
                    edgePtIds = [i0,] + edgePtIds[:-1]
                    polyPtIds += edgePtIds
                    continue

                p0 = numpy.array(self.points.GetPoint(i0))
                p1 = numpy.array(self.points.GetPoint(i1))
                dEdge = p1 - p0
                edgeLength = numpy.sqrt(dEdge.dot(dEdge))
                numSegs = int(max(1, math.ceil(edgeLength/max_edge_length)))

                # add points along edge
                d10 = (p1 - p0) / float(numSegs)
                edgePtIds.append(i0)
                for iSeg in range(1, numSegs):
                    pt = p0 + iSeg*d10
                    # index of point to add
                    ptId = self.points.GetNumberOfPoints()
                    # insert point
                    self.points.InsertNextPoint(pt)
                    edgePtIds.append(ptId)

                edge2PtIds[edge] = edgePtIds
                polyPtIds += edgePtIds

            # triangulate the cell
            polyCells = self.triangulatePolygon(uVec, vVec, polyPtIds)
            #polyCells = self.triangulatePolygon2(p0, uVec, vVec, polyPtIds,
            #                                     max_edge_length)
            cells += polyCells

        # build the output vtkPolyData object
        pdata = vtk.vtkPolyData()
        ptIds = vtk.vtkIdList()
        pdata.SetPoints(self.points)
        numPolys = len(cells)
        pdata.Allocate(numPolys, 1)
        for cell in cells:
            numPts = len(cell)
            ptIds.SetNumberOfIds(numPts)
            for j in range(numPts):
                ptIds.SetId(j, cell[j])
            pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

        # Reset the polydata struct
        self.polydata = pdata

    def computeUVNormal(self, ptIds):
        """
        Compute the two tangential unit vectors and the normal vector
        @param ptIds point indices
        @return u vector, v vector, normal
        """
        uVec = numpy.zeros((3,), numpy.float64)
        vVec = numpy.zeros((3,), numpy.float64)
        numPts = ptIds.GetNumberOfIds()
        if numPts < 3:
            return uVec, vVec, uVec
        p0 = numpy.array(self.points.GetPoint(ptIds.GetId(0)))
        for i in range(1, numPts - 1):
            dp1 = numpy.array(self.points.GetPoint(ptIds.GetId(i))) - p0
            dp2 = numpy.array(self.points.GetPoint(ptIds.GetId(i + 1))) - p0
            perp = numpy.cross(dp1, dp2)
            pDotp = perp.dot(perp)
            if pDotp > 0:
                normal = perp / numpy.sqrt(pDotp)
                uVec = dp1 / numpy.sqrt(dp1.dot(dp1))
                vVec = numpy.cross(normal, uVec)
                return uVec, vVec, normal
        return uVec, vVec, numpy.zeros((3,), numpy.float64)

    def triangulatePolygon2(self, p0, uVec, vVec, polyPtIds, max_edge_length):
        """
        Triangulate polygon using the uVec x vVec projection
        @param p0 reference vertex
        @param uVec unit vector tangential to the polygon
        @param vVec second unit vector tangential to the polygon
        @param polyPtIds list of polygon's point indices
        @param max_edge_length maximum edge length
        @return list of cells (list of point indices)
        """
        import triangle

        numPts = len(polyPtIds)
        if numPts < 3:
            return []
        elif numPts == 3:
            # no need to do any triangulation, just return the cell
            return [polyPtIds]

        pts = []
        # project each point onto the plane
        for i in range(numPts):
            p = numpy.array(self.points.GetPoint(polyPtIds[i]))
            pts.append( (p.dot(uVec), p.dot(vVec)) )

        segs = [(i, (i + 1)%numPts) for i in range(numPts)]

        tri = triangle.Triangle()
        tri.set_points(pts)
        tri.set_segments(segs)
        maxArea = 0.5 * max_edge_length**2
        tri.triangulate(area=maxArea)

        nodes = tri.get_nodes()
        polyCells = tri.get_triangles()

        # add the new vertices
        n = len(pts) # new points follow the boundary points
        for pIndex in range(n, len(nodes)):
            u, v = nodes[pIndex][0]
            p = p0 + u*uVec + v*vVec
            # insert new point
            ptId = self.points.GetNumberOfPoints()
            self.points.InsertNextPoint(p)
            polyPtIds.append(ptId)

        cells = []
        for c in polyCells:
            ia, ib, ic = c[0]
            cell = [ polyPtIds[ia], polyPtIds[ib], polyPtIds[ic] ]
            cells.append(cell)

        return cells

    def triangulatePolygon(self, uVec, vVec, polyPtIds):
        """
        Triangulate polygon using the uVec x vVec projection
        @param uVec unit vector tangential to the polygon
        @param vVec second unit vector tangential to the polygon
        @param polyPtIds list of polygon's point indices
        @return list of cells (list of point indices)
        """

        numPts = len(polyPtIds)
        if numPts < 3:
            return []
        elif numPts == 3:
            # no need to do any triangulation, just return the cell
            return [polyPtIds]

        pts = vtk.vtkPoints()
        pts.SetNumberOfPoints(numPts)

        # project each point onto the plane
        pt = numpy.zeros((3,), numpy.float64)
        for i in range(numPts):
            p = numpy.array(self.points.GetPoint(polyPtIds[i]))
            pt[0:2] = p.dot(uVec), p.dot(vVec)
            pts.SetPoint(i, pt)

        pdata = vtk.vtkPolyData()
        pdata.SetPoints(pts)

        delaunay = vtk.vtkDelaunay2D()
        delaunay.SetTolerance(1.e-5)
        delaunay.SetAlpha(0.0)
        if vtk.VTK_MAJOR_VERSION >= 6:
            delaunay.SetInputData(pdata)
        else:
            delaunay.SetInput(pdata)

        delaunay.Update()

        ugrid = delaunay.GetOutput()
        cells = []
        numCells = ugrid.GetNumberOfCells()
        if numCells == 0:
            # Delaunay triangulation failed
            # not sure why this is happening...
            return [polyPtIds]
        else:
            for i in range(numCells):
                ptIds = ugrid.GetCell(i).GetPointIds()
                cell = []
                for j in range(ptIds.GetNumberOfIds()):
                    localId = ptIds.GetId(j)
                    ptId = polyPtIds[localId]
                    cell.append(ptId)
                cells.append(cell)
        return cells

##############################################################################

def printVtkPolyData(pdata):

    points = pdata.GetPoints()
    polys = pdata.GetPolys()
    ptIds = vtk.vtkIdList()
    numPolys = polys.GetNumberOfCells()
    print 'Number of polygons: {}'.format(numPolys)
    polys.InitTraversal()
    for i in range(numPolys):
        cell = polys.GetNextCell(ptIds)
        numPts = ptIds.GetNumberOfIds()
        print '\tCell {} has {} points: '.format(i, numPts)
        for j in range(numPts):
            ptId = ptIds.GetId(j)
            pt = points.GetPoint(ptId)
            print '\t\t{} -> {}'.format(ptId, pt)

def testNoRefinement():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((1., 0., 0.))
    points.InsertNextPoint((1., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.5)
    pdata2 = rs.getVtkPolyData()
    assert(pdata2.GetNumberOfPolys() == 1)

def testAddingThreePointsThenMore():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.1)
    #printVtkPolyData(rs.getVtkPolyData())
    assert(rs.getVtkPolyData().GetNumberOfPolys() == 4)
    #rs.refine(max_edge_length=0.1)
    #assert(rs.getVtkPolyData().GetNumberOfPolys() == 104)

def testStartingWithTwoCells():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))
    points.InsertNextPoint((0., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    pdata.Allocate(2, 1)
    ptIds = vtk.vtkIdList()
    ptIds.SetNumberOfIds(3)
    ptIds.SetId(0, 0)
    ptIds.SetId(1, 1)
    ptIds.SetId(2, 2)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)
    ptIds.SetId(0, 2)
    ptIds.SetId(1, 3)
    ptIds.SetId(2, 0)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    rs.refine(max_edge_length=1.1)
    pdata2 = rs.getVtkPolyData()
    #printVtkPolyData(pdata2)
    assert(rs.getVtkPolyData().GetNumberOfPolys() == 8)
    #rs.refine(max_edge_length=0.1)
    #assert(rs.getVtkPolyData().GetNumberOfPolys() == 208)

if __name__ == '__main__':
    testNoRefinement()
    testAddingThreePointsThenMore()
    testStartingWithTwoCells()
<|endoftext|>


<commit_before># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

from viper.common.out import bold
from viper.common.abstracts import Module
from viper.core.database import Database
from viper.core.session import __sessions__

try:
    import pydeep
    HAVE_PYDEEP = True
except ImportError:
    HAVE_PYDEEP = False

class Fuzzy(Module):
    cmd = 'fuzzy'
    description = 'Search for similar files through fuzzy hashing'
    authors = ['nex']

    def __init__(self):
        super(Fuzzy, self).__init__()
        self.parser.add_argument('-v', '--verbose', action='store_true', help='Prints verbose logging')

    def run(self):
        super(Fuzzy, self).run()

        if not __sessions__.is_set():
            self.log('error', "No session opened")
            return

        if not HAVE_PYDEEP:
            self.log('error', "Missing dependency, install pydeep (`pip install pydeep`)")
            return

        if not __sessions__.current.file.ssdeep:
            self.log('error', "No ssdeep hash available for opened file")
            return

        arg_verbose = False
        if self.args and self.args.verbose:
            arg_verbose = True

        db = Database()
        samples = db.find(key='all')

        matches = []
        for sample in samples:
            if sample.sha256 == __sessions__.current.file.sha256:
                continue

            if not sample.ssdeep:
                continue

            score = pydeep.compare(__sessions__.current.file.ssdeep, sample.ssdeep)
            if score > 40:
                matches.append(['{0}%'.format(score), sample.name, sample.sha256])

            if arg_verbose:
                self.log('info', "Match {0}%: {2} [{1}]".format(score, sample.name, sample.sha256))

        self.log('info', "{0} relevant matches found".format(bold(len(matches))))

        if len(matches) > 0:
            self.log('table', dict(header=['Score', 'Name', 'SHA256'], rows=matches))
<commit_msg>Added draft for fuzzy hash clustering<commit_after># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

import collections

from viper.common.out import bold, print_info
from viper.common.abstracts import Module
from viper.core.database import Database
from viper.core.session import __sessions__

try:
    import pydeep
    HAVE_PYDEEP = True
except ImportError:
    HAVE_PYDEEP = False

class Fuzzy(Module):
    cmd = 'fuzzy'
    description = "Search for similar files through fuzzy hashing"
    authors = ['nex']

    def __init__(self):
        super(Fuzzy, self).__init__()
        self.parser.add_argument('-v', '--verbose', action='store_true',
            help="Prints verbose logging")
        self.parser.add_argument('-c', '--cluster', action='store_true',
            help="Cluster all available samples by ssdeep")

    def run(self):
        super(Fuzzy, self).run()

        if not HAVE_PYDEEP:
            self.log('error', "Missing dependency, install pydeep (`pip install pydeep`)")
            return

        arg_verbose = False
        arg_cluster = False
        if self.args:
            if self.args.verbose:
                arg_verbose = self.args.verbose
            if self.args.cluster:
                arg_cluster = self.args.cluster

        db = Database()
        samples = db.find(key='all')

        # Check if we're operating in cluster mode, otherwise we run on the
        # currently opened file.
        if arg_cluster:
            print_info("Generating clusters, this might take a while...")

            clusters = dict()
            for sample in samples:
                if not sample.ssdeep:
                    continue

                if arg_verbose:
                    self.log('info', "Testing file {0} with ssdeep {1}".format(
                        sample.md5, sample.ssdeep))

                clustered = False
                for cluster_name, cluster_members in clusters.items():
                    # Check if sample is already in the cluster.
                    if sample.md5 in cluster_members:
                        continue

                    if arg_verbose:
                        self.log('info', "Testing {0} in cluser {1}".format(
                            sample.md5, cluster_name))

                    for member in cluster_members:
                        if sample.md5 == member:
                            continue

                        member_ssdeep = db.find(key='md5', value=member)[0].ssdeep
                        if pydeep.compare(sample.ssdeep, member_ssdeep) > 40:
                            if arg_verbose:
                                self.log('info', "Found home for {0} in cluster {1}".format(
                                    sample.md5, cluster_name))

                            clusters[cluster_name].append(sample.md5)
                            clustered = True
                            break

                if not clustered:
                    cluster_id = len(clusters) + 1
                    clusters[cluster_id] = [sample.md5,]

            ordered_clusters = collections.OrderedDict(sorted(clusters.items()))

            rows = []
            for cluster_name, cluster_members in ordered_clusters.items():
                rows.append([cluster_name, '\n'.join(cluster_members)])

            self.log('table', dict(header=['Cluster', 'Members'], rows=rows))

        # We're running against the already opened file.
        else:
            if not __sessions__.is_set():
                self.log('error', "No session opened")
                return

            if not __sessions__.current.file.ssdeep:
                self.log('error', "No ssdeep hash available for opened file")
                return

            matches = []
            for sample in samples:
                if sample.sha256 == __sessions__.current.file.sha256:
                    continue

                if not sample.ssdeep:
                    continue

                score = pydeep.compare(__sessions__.current.file.ssdeep,
                    sample.ssdeep)

                if score > 40:
                    matches.append(['{0}%'.format(score), sample.name,
                        sample.sha256])

                if arg_verbose:
                    self.log('info', "Match {0}%: {2} [{1}]".format(score,
                        sample.name, sample.sha256))

            self.log('info', "{0} relevant matches found".format(bold(len(matches))))

            if len(matches) > 0:
                self.log('table',dict(header=['Score', 'Name', 'SHA256'],
                    rows=matches))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import math
import numpy
import vtk

class RefineSurface:

    def __init__(self, pdata):
        """
        Constructor
        @param pdata vtkPolyData instance
        """
        self.polydataOutput = vtk.vtkPolyData()
        self.pointsOutput = vtk.vtkPoints()

        # populate with input data
        points = pdata.GetPoints()
        numPoints = points.GetNumberOfPoints()
        for i in range(numPoints):
            self.pointsOutput.InsertNextPoint(points.GetPoint(i))
        self.polydataOutput.SetPoints(self.pointsOutput)

        polys = pdata.GetPolys()
        numPolys = polys.GetNumberOfCells()
        self.polydataOutput.Allocate(numPolys, 1)
        ptIds = vtk.vtkIdList()
        for i in range(numPolys):
            polys.GetCell(i, ptIds)
            self.polydataOutput.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    def refine(self, max_edge_length):
        """
        Refine each cell by adding points in the middle of each edge
        @param max_edge_length maximum edge length
        @return refined vtkPolyData instance
        """

        # edge to point Ids map
        edge2PointIds = {}

        cells = []

        # iterate over the polygons
        polys = self.polydataOutput.GetPolys()
        numPolys = self.polydataOutput.GetNumberOfPolys()
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        for i in range(numPolys):

            polys.GetNextCell(ptIds)

            # number of points spanning the polygon
            numPts = ptIds.GetNumberOfIds()

            # must have at least three points
            if numPts < 3:
                continue

            # compute the normal vector of the polygon plane. Note:
            # assuming the polygon is planar
            p0 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(0)))
            p1 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(1)))
            p2 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(2)))
            p1 -= p0
            p2 -= p0
            normal = numpy.cross(p1, p2)
            normal /= math.sqrt(numpy.dot(normal, normal))

            # compute the 2D basis vectors (uVc and vVec) on the plane
            uVec = p1 / math.sqrt(numpy.dot(p1, p1))
            vVec = numpy.cross(normal, uVec)

            # ordered list of point index to uv coordinates on the plane
            uvs = {}

            # iterate over edges, add middle edge points if the edge's
            # length is > max_edge_length
            for i in range(numPts):

                # start/end point indices of the edge
                ptIdBeg = ptIds.GetId(i)
                ptIdEnd = ptIds.GetId((i + 1) % numPts)
                edge = [ptIdBeg, ptIdEnd]
                edge.sort()
                ptIdBeg = edge[0]
                ptIdEnd = edge[1]

                e = tuple(edge)
                if not e in edge2PointIds:
                    # new edge
                    ptBeg = numpy.array(self.pointsOutput.GetPoint(ptIdBeg))
                    ptEnd = numpy.array(self.pointsOutput.GetPoint(ptIdEnd))
                    d = ptEnd - ptBeg
                    edgeLength = numpy.sqrt(numpy.dot(d, d))
                    # add new points to the edge
                    numSegs = int(math.ceil(edgeLength / max_edge_length))
                    pis = [ptIdBeg]
                    u, v = numpy.dot(ptBeg, uVec), numpy.dot(ptBeg, vVec)
                    uvs[ptIdBeg] = (u, v)
                    for k in range(1, numSegs):
                        pt = ptBeg + k * d / numSegs
                        self.pointsOutput.InsertNextPoint(pt)
                        ptId = self.pointsOutput.GetNumberOfPoints() - 1
                        pis.append(ptId)
                        u, v = numpy.dot(pt, uVec), numpy.dot(pt, vVec)
                        uvs[ptId] = (u, v)
                    edge2PointIds[e] = pis

            # triangulate cell and append to list
            cells += self.triangulate(uvs)

        # build the output vtkPolyData object
        ptIds = vtk.vtkIdList()
        numCells = len(cells)
        for i in range(numCells):
            cell = cells[i]
            numPts = len(cell)
            ptIds.SetNumberOfIds(numPts)
            for j in range(numPts):
                ptIds.SetId(j, cell[j])
            self.polydataOutput.InsertNextCell(vtk.VTK_POLYGON, ptIds)

        return self.polydataOutput

    def triangulate(self, uvs):
        """
        Triangulate a set of u, v points
        @param uvs point index to u, v plane coordinates dictionary
        @return cells
        """
        uvIds = uvs.keys()
        uvPoints = uvs.values()
        points = vtk.vtkPoints()
        pdata = vtk.vtkPolyData()
        delaunay = vtk.vtkDelaunay2D()
        pt = numpy.zeros((3,), numpy.float64)
        for uv in uvPoints:
            pt[0:2] = uv
            points.InsertNextPoint(pt)
        pdata.SetPoints(points)
        if vtk.VTK_MAJOR_VERSION >= 6:
            delaunay.SetInputData(pdata)
        else:
            delaunay.SetInput(pdata)
        delaunay.Update()
        ugrid = delaunay.GetOutput()
        cells = []
        numCells = ugrid.GetNumberOfCells()
        for i in range(numCells):
            ptIds = ugrid.GetCell(i).GetPointIds()
            cell = []
            for j in range(ptIds.GetNumberOfIds()):
                localId = ptIds.GetId(j)
                ptId = uvIds[localId]
                cell.append(ptId)
            cells.append(cell)
        return cells

##############################################################################

def testNoRefinement():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((1., 0., 0.))
    points.InsertNextPoint((1., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    pdata2 = rs.refine(max_edge_length=1.5)
    assert(pdata2.GetNumberOfPolys() == 1)

def testAddingThreePoints():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    pdata2 = rs.refine(max_edge_length=1.1)
    assert(pdata2.GetNumberOfPolys() == 4)

if __name__ == '__main__':
    #testNoRefinement()
    testAddingThreePoints()
<commit_msg>running through flake8<commit_after>#!/usr/bin/env python

import math
import numpy
import vtk

class RefineSurface:

    def __init__(self, pdata):
        """
        Constructor
        @param pdata vtkPolyData instance
        """
        self.polydataOutput = vtk.vtkPolyData()
        self.pointsOutput = vtk.vtkPoints()

        # populate with input data
        points = pdata.GetPoints()
        numPoints = points.GetNumberOfPoints()
        for i in range(numPoints):
            self.pointsOutput.InsertNextPoint(points.GetPoint(i))
        self.polydataOutput.SetPoints(self.pointsOutput)

        polys = pdata.GetPolys()
        numPolys = polys.GetNumberOfCells()
        self.polydataOutput.Allocate(numPolys, 1)
        ptIds = vtk.vtkIdList()
        for i in range(numPolys):
            polys.GetCell(i, ptIds)
            self.polydataOutput.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    def refine(self, max_edge_length):
        """
        Refine each cell by adding points in the middle of each edge
        @param max_edge_length maximum edge length
        @return refined vtkPolyData instance
        """

        # edge to point Ids map
        edge2PointIds = {}

        cells = []

        # iterate over the polygons
        polys = self.polydataOutput.GetPolys()
        numPolys = self.polydataOutput.GetNumberOfPolys()
        ptIds = vtk.vtkIdList()
        polys.InitTraversal()
        for i in range(numPolys):

            polys.GetNextCell(ptIds)

            # number of points spanning the polygon
            numPts = ptIds.GetNumberOfIds()

            # must have at least three points
            if numPts < 3:
                continue

            # compute the normal vector of the polygon plane. Note:
            # assuming the polygon is planar
            p0 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(0)))
            p1 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(1)))
            p2 = numpy.array(self.pointsOutput.GetPoint(ptIds.GetId(2)))
            p1 -= p0
            p2 -= p0
            normal = numpy.cross(p1, p2)
            normal /= math.sqrt(numpy.dot(normal, normal))

            # compute the 2D basis vectors (uVc and vVec) on the plane
            uVec = p1 / math.sqrt(numpy.dot(p1, p1))
            vVec = numpy.cross(normal, uVec)

            # ordered list of point index to uv coordinates on the plane
            uvs = {}

            # iterate over edges, add middle edge points if the edge's
            # length is > max_edge_length
            for i in range(numPts):

                # start/end point indices of the edge
                ptIdBeg = ptIds.GetId(i)
                ptIdEnd = ptIds.GetId((i + 1) % numPts)
                edge = [ptIdBeg, ptIdEnd]
                edge.sort()
                ptIdBeg = edge[0]
                ptIdEnd = edge[1]

                e = tuple(edge)
                if e not in edge2PointIds:
                    # new edge
                    ptBeg = numpy.array(self.pointsOutput.GetPoint(ptIdBeg))
                    ptEnd = numpy.array(self.pointsOutput.GetPoint(ptIdEnd))
                    d = ptEnd - ptBeg
                    edgeLength = numpy.sqrt(numpy.dot(d, d))
                    # add new points to the edge
                    numSegs = int(math.ceil(edgeLength / max_edge_length))
                    pis = [ptIdBeg]
                    u, v = numpy.dot(ptBeg, uVec), numpy.dot(ptBeg, vVec)
                    uvs[ptIdBeg] = (u, v)
                    for k in range(1, numSegs):
                        pt = ptBeg + k * d / numSegs
                        self.pointsOutput.InsertNextPoint(pt)
                        ptId = self.pointsOutput.GetNumberOfPoints() - 1
                        pis.append(ptId)
                        u, v = numpy.dot(pt, uVec), numpy.dot(pt, vVec)
                        uvs[ptId] = (u, v)
                    edge2PointIds[e] = pis

            # triangulate cell and append to list
            cells += self.triangulate(uvs)

        # build the output vtkPolyData object
        ptIds = vtk.vtkIdList()
        numCells = len(cells)
        for i in range(numCells):
            cell = cells[i]
            numPts = len(cell)
            ptIds.SetNumberOfIds(numPts)
            for j in range(numPts):
                ptIds.SetId(j, cell[j])
            self.polydataOutput.InsertNextCell(vtk.VTK_POLYGON, ptIds)

        return self.polydataOutput

    def triangulate(self, uvs):
        """
        Triangulate a set of u, v points
        @param uvs point index to u, v plane coordinates dictionary
        @return cells
        """
        uvIds = uvs.keys()
        uvPoints = uvs.values()
        points = vtk.vtkPoints()
        pdata = vtk.vtkPolyData()
        delaunay = vtk.vtkDelaunay2D()
        pt = numpy.zeros((3,), numpy.float64)
        for uv in uvPoints:
            pt[0:2] = uv
            points.InsertNextPoint(pt)
        pdata.SetPoints(points)
        if vtk.VTK_MAJOR_VERSION >= 6:
            delaunay.SetInputData(pdata)
        else:
            delaunay.SetInput(pdata)
        delaunay.Update()
        ugrid = delaunay.GetOutput()
        cells = []
        numCells = ugrid.GetNumberOfCells()
        for i in range(numCells):
            ptIds = ugrid.GetCell(i).GetPointIds()
            cell = []
            for j in range(ptIds.GetNumberOfIds()):
                localId = ptIds.GetId(j)
                ptId = uvIds[localId]
                cell.append(ptId)
            cells.append(cell)
        return cells

##############################################################################

def testNoRefinement():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((1., 0., 0.))
    points.InsertNextPoint((1., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    pdata2 = rs.refine(max_edge_length=1.5)
    assert(pdata2.GetNumberOfPolys() == 1)

def testAddingThreePoints():

    points = vtk.vtkPoints()
    points.InsertNextPoint((0., 0., 0.))
    points.InsertNextPoint((2., 0., 0.))
    points.InsertNextPoint((2., 1., 0.))

    pdata = vtk.vtkPolyData()
    pdata.SetPoints(points)

    ptIds = vtk.vtkIdList()
    ptIds.InsertNextId(0)
    ptIds.InsertNextId(1)
    ptIds.InsertNextId(2)
    pdata.Allocate(1, 1)
    pdata.InsertNextCell(vtk.VTK_POLYGON, ptIds)

    rs = RefineSurface(pdata)
    pdata2 = rs.refine(max_edge_length=1.1)
    assert(pdata2.GetNumberOfPolys() == 4)

if __name__ == '__main__':
    testNoRefinement()
    testAddingThreePoints()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
radio.py - ShoutCAST radio Module
Copyright 2012, Dimitri "Tyrope" Molenaars, TyRope.nl
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net/
"""

from time import sleep
from xml.dom.minidom import parseString
import web, xml.dom.minidom

def configure(config):
    chunk = ''
    if config.option('Configure radio module', True):
        config.interactive_add('radio_URL', 'URL to the ShoutCAST administration page', 'http://127.0.0.1:8000/')
        config.interactive_add('radio_sID', 'Stream ID (only required for multi-stream servers.)', '1')
        chunk = ("\nradio_URL = '%s'\nradio_sID = '%s'\n"
                 % (config.radio_URL, config.radio_sID))
    return chunk

radioURL = '' # Set once, after the first .radio request.
checkSongs = False
current_song = ''

def getAPI(willie, trigger):
    #contact the 'heavyweight' XML API
    try:
        raw = web.get(radioURL % 'stats')
    except Exception as e:
        willie.say('The radio is not responding to the stats request.')
        willie.debug('radio', 'Exception while trying to get stats: %s' % e, 'verbose')
        return 0

    #Parse the XML
    XML = parseString(raw).documentElement
    status = XML.getElementsByTagName('STREAMSTATUS')[0].firstChild.nodeValue
    if status != '0':
        status = 'Online'
        servername = '['+XML.getElementsByTagName('SERVERTITLE')[0].firstChild.nodeValue+']'
    else:
        status = 'Offline'
        servername = ''
    curlist = XML.getElementsByTagName('CURRENTLISTENERS')[0].firstChild.nodeValue
    maxlist = XML.getElementsByTagName('MAXLISTENERS')[0].firstChild.nodeValue

    #Garbage disposal
    XML.unlink()

    #print results
    willie.say('%sStatus: %s. Listeners: %s/%s.' % (servername, status, curlist, maxlist))
    return 1

def currentSong(willie, trigger):
    # This function uses the PLAINTEXT API to get the current song only.
    try:
        song = web.get(radioURL % 'currentsong')
    except Exception as e:
        willie.say('The radio is not responding to the song request.')
        willie.debug('radio', 'Exception while trying to get current song: %s' % e, 'warning')
    if song:
        willie.say('Now playing: '+song)
    else:
        Willie.say('The radio is currently offline.')

def nextSong(willie, trigger):
    # This function uses the PLAINTEXT API to get the next song only.
    try:
        song = web.get(radioURL % 'nextsong')
    except Exception as e:
        willie.say('The radio is not responding to the song request.')
        willie.debug('radio', 'Exception while trying to get next song: %s' % e, 'warning')
    if song:
        willie.say('Next up: '+song)
    else:
        willie.say('No songs are queued up.')

def radio(willie, trigger):
    """ Radio functions, valid parameters: on, off, song, now, next, soon, stats. """
    global checkSongs, current_song, radioURL
    if not radioURL:
        if not hasattr(willie.config, 'radio_URL') or not hasattr(willie.config, 'radio_sID'):
            willie.say('Radio module not configured, make sure radio_URL and radio_sID are defined')
            return
        else:
            radioURL = willie.config.radio_URL+'%s?sid='+willie.config.radio_sID
    try:
        args = trigger.group(2).lower().split(' ')
    except AttributeError:
        willie.say('Usage: .radio (on|off|song|now|next|soon|stats)')
        return
    if args[0] == 'on':
        if not trigger.isop:
            return;
        if checkSongs == True:
            willie.reply('Radio data checking is already on.')
            return
        if not getAPI(willie, trigger):
            return
        checkSongs = True
        while checkSongs:
            last = current_song
            try:
                current_song = web.get(radioURL % 'currentsong')
                nextsong = web.get(radioURL % 'nextsong')
            except Exception as e:
                willie.debug('radio', 'Exception while trying to get periodic radio data: %s' % e, 'warning')
                willie.say('The radio is not responding to the song request.')
                willie.say('Turning off radio data checking.')
                checkSongs = False
                break
            if not current_song == last:
                if not current_song:
                    csong = 'The radio is currently offline.'
                else:
                    csong = 'Now Playing: '+current_song
                if nextsong and current_song:
                    willie.say(csong+' | Coming Up: '+nextsong)
                else:
                    willie.say(csong)
            sleep(5)
    elif args[0] == 'off':
        if not trigger.isop:
            return;
        if checkSongs == False:
            willie.reply('Radio data checking is already off.')
            return
        checkSongs = False
        current_song = ''
        willie.reply('Turning off radio data checking.')
    elif args[0] == 'song' or args[0] == 'now':
        currentSong(willie, trigger)
    elif args[0] == 'next' or args[0] == 'soon':
        nextSong(willie, trigger)
    elif args[0] == 'stats':
        getAPI(willie, trigger)
radio.commands = ['radio']
radio.priority = 'medium'

if __name__ == '__main__':
    print __doc__.strip()
<commit_msg>[radio]Dirty lil' trick to cope with invalid XMLs<commit_after>#!/usr/bin/env python
"""
radio.py - ShoutCAST radio Module
Copyright 2012, Dimitri "Tyrope" Molenaars, TyRope.nl
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net/
"""

from time import sleep
from xml.dom.minidom import parseString
import web, xml.dom.minidom

def configure(config):
    chunk = ''
    if config.option('Configure radio module', True):
        config.interactive_add('radio_URL', 'URL to the ShoutCAST administration page', 'http://127.0.0.1:8000/')
        config.interactive_add('radio_sID', 'Stream ID (only required for multi-stream servers.)', '1')
        chunk = ("\nradio_URL = '%s'\nradio_sID = '%s'\n"
                 % (config.radio_URL, config.radio_sID))
    return chunk

radioURL = '' # Set once, after the first .radio request.
checkSongs = False
current_song = ''

def getAPI(willie, trigger):
    #contact the 'heavyweight' XML API
    try:
        raw = web.get(radioURL % 'stats')
    except Exception as e:
        willie.say('The radio is not responding to the stats request.')
        willie.debug('radio', 'Exception while trying to get stats: %s' % e, 'verbose')
        return 0

    #Parse the XML
    XML = parseString(raw).documentElement
    try:
        status = XML.getElementsByTagName('STREAMSTATUS')[0].firstChild.nodeValue
    except:
        status = '0'
    if status != '0':
        status = 'Online'
        servername = '['+XML.getElementsByTagName('SERVERTITLE')[0].firstChild.nodeValue+']'
    else:
        status = 'Offline'
        servername = ''
    curlist = XML.getElementsByTagName('CURRENTLISTENERS')[0].firstChild.nodeValue
    maxlist = XML.getElementsByTagName('MAXLISTENERS')[0].firstChild.nodeValue

    #Garbage disposal
    XML.unlink()

    #print results
    willie.say('%sStatus: %s. Listeners: %s/%s.' % (servername, status, curlist, maxlist))
    return 1

def currentSong(willie, trigger):
    # This function uses the PLAINTEXT API to get the current song only.
    try:
        song = web.get(radioURL % 'currentsong')
    except Exception as e:
        willie.say('The radio is not responding to the song request.')
        willie.debug('radio', 'Exception while trying to get current song: %s' % e, 'warning')
    if song:
        willie.say('Now playing: '+song)
    else:
        Willie.say('The radio is currently offline.')

def nextSong(willie, trigger):
    # This function uses the PLAINTEXT API to get the next song only.
    try:
        song = web.get(radioURL % 'nextsong')
    except Exception as e:
        willie.say('The radio is not responding to the song request.')
        willie.debug('radio', 'Exception while trying to get next song: %s' % e, 'warning')
    if song:
        willie.say('Next up: '+song)
    else:
        willie.say('No songs are queued up.')

def radio(willie, trigger):
    """ Radio functions, valid parameters: on, off, song, now, next, soon, stats. """
    global checkSongs, current_song, radioURL
    if not radioURL:
        if not hasattr(willie.config, 'radio_URL') or not hasattr(willie.config, 'radio_sID'):
            willie.say('Radio module not configured, make sure radio_URL and radio_sID are defined')
            return
        else:
            radioURL = willie.config.radio_URL+'%s?sid='+willie.config.radio_sID
    try:
        args = trigger.group(2).lower().split(' ')
    except AttributeError:
        willie.say('Usage: .radio (on|off|song|now|next|soon|stats)')
        return
    if args[0] == 'on':
        if not trigger.isop:
            return;
        if checkSongs == True:
            willie.reply('Radio data checking is already on.')
            return
        if not getAPI(willie, trigger):
            return
        checkSongs = True
        while checkSongs:
            last = current_song
            try:
                current_song = web.get(radioURL % 'currentsong')
                nextsong = web.get(radioURL % 'nextsong')
            except Exception as e:
                willie.debug('radio', 'Exception while trying to get periodic radio data: %s' % e, 'warning')
                willie.say('The radio is not responding to the song request.')
                willie.say('Turning off radio data checking.')
                checkSongs = False
                break
            if not current_song == last:
                if not current_song:
                    csong = 'The radio is currently offline.'
                else:
                    csong = 'Now Playing: '+current_song
                if nextsong and current_song:
                    willie.say(csong+' | Coming Up: '+nextsong)
                else:
                    willie.say(csong)
            sleep(5)
    elif args[0] == 'off':
        if not trigger.isop:
            return;
        if checkSongs == False:
            willie.reply('Radio data checking is already off.')
            return
        checkSongs = False
        current_song = ''
        willie.reply('Turning off radio data checking.')
    elif args[0] == 'song' or args[0] == 'now':
        currentSong(willie, trigger)
    elif args[0] == 'next' or args[0] == 'soon':
        nextSong(willie, trigger)
    elif args[0] == 'stats':
        getAPI(willie, trigger)
radio.commands = ['radio']
radio.priority = 'medium'

if __name__ == '__main__':
    print __doc__.strip()
<|endoftext|>