<commit_before># coding=utf-8
from __future__ import absolute_import

__author__ = "Gina Häußge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import os
import octoprint.plugin
import octoprint.events
from octoprint.settings import settings

class SlicingProfile(object):
	def __init__(self, slicer, name, data, display_name=None, description=None):
		self.slicer = slicer
		self.name = name
		self.data = data
		self.display_name = display_name
		self.description = description

class TemporaryProfile(object):
	def __init__(self, save_profile, profile, overrides=None):
		self.save_profile = save_profile
		self.profile = profile
		self.overrides = overrides

	def __enter__(self):
		import tempfile
		temp_profile = tempfile.NamedTemporaryFile(prefix="slicing-profile-temp-", suffix=".profile", delete=False)
		temp_profile.close()

		self.temp_path = temp_profile.name
		self.save_profile(self.temp_path, self.profile, overrides=self.overrides)
		return self.temp_path

	def __exit__(self, type, value, traceback):
		import os
		try:
			os.remove(self.temp_path)
		except:
			pass

class SlicingCancelled(BaseException):
	pass

class SlicingManager(object):
	def __init__(self, profile_path, printer_profile_manager):
		self._profile_path = profile_path
		self._printer_profile_manager = printer_profile_manager

		self._slicers = dict()
		self._slicer_names = dict()
		self._load_slicers()

		self._progress_callbacks = []
		self._last_progress_report = None

	def register_progress_callback(self, callback):
		self._progress_callbacks.append(callback)

	def unregister_progress_callback(self, callback):
		self._progress_callbacks.remove(callback)

	def _load_slicers(self):
		plugins = octoprint.plugin.plugin_manager().get_implementations(octoprint.plugin.SlicerPlugin)
		for name, plugin in plugins.items():
			if plugin.is_slicer_configured():
				self._slicers[plugin.get_slicer_properties()["type"]] = plugin

	@property
	def slicing_enabled(self):
		return len(self.configured_slicers) > 0

	@property
	def registered_slicers(self):
		return self._slicers.keys()

	@property
	def configured_slicers(self):
		return map(lambda slicer: slicer.get_slicer_properties()["type"], filter(lambda slicer: slicer.is_slicer_configured(), self._slicers.values()))

	@property
	def default_slicer(self):
		slicer_name = settings().get(["slicing", "defaultSlicer"])
		if slicer_name in self.configured_slicers:
			return slicer_name
		else:
			return None

	def get_slicer(self, slicer, require_configured=True):
		return self._slicers[slicer] if slicer in self._slicers and (not require_configured or self._slicers[slicer].is_slicer_configured()) else None

	def slice(self, slicer_name, source_path, dest_path, profile_name, callback, callback_args=None, callback_kwargs=None, overrides=None, on_progress=None, on_progress_args=None, on_progress_kwargs=None, printer_profile_id=None):
		if callback_args is None:
			callback_args = ()
		if callback_kwargs is None:
			callback_kwargs = dict()

		if not slicer_name in self.configured_slicers:
			if not slicer_name in self.registered_slicers:
				error = "No such slicer: {slicer_name}".format(**locals())
			else:
				error = "Slicer not configured: {slicer_name}".format(**locals())
			callback_kwargs.update(dict(_error=error))
			callback(*callback_args, **callback_kwargs)
			return False, error

		slicer = self.get_slicer(slicer_name)

		printer_profile = None
		if printer_profile_id is not None:
			printer_profile = self._printer_profile_manager.get(printer_profile_id)

		if printer_profile is None:
			printer_profile = self._printer_profile_manager.get_current_or_default()["id"]

		def slicer_worker(slicer, model_path, machinecode_path, profile_name, overrides, printer_profile, callback, callback_args, callback_kwargs):
			try:
				slicer_name = slicer.get_slicer_properties()["type"]
				with self.temporary_profile(slicer_name, name=profile_name, overrides=overrides) as profile_path:
					ok, result = slicer.do_slice(
						model_path,
						printer_profile,
						machinecode_path=machinecode_path,
						profile_path=profile_path,
						on_progress=on_progress,
						on_progress_args=on_progress_args,
						on_progress_kwargs=on_progress_kwargs
					)

				if not ok:
					callback_kwargs.update(dict(_error=result))
			except SlicingCancelled:
				callback_kwargs.update(dict(_cancelled=True))
			finally:
				callback(*callback_args, **callback_kwargs)

		import threading
		slicer_worker_thread = threading.Thread(target=slicer_worker,
		                                        args=(slicer, source_path, dest_path, profile_name, overrides, printer_profile, callback, callback_args, callback_kwargs))
		slicer_worker_thread.daemon = True
		slicer_worker_thread.start()
		return True, None

	def cancel_slicing(self, slicer_name, source_path, dest_path):
		if not slicer_name in self.registered_slicers:
			return
		slicer = self.get_slicer(slicer_name)
		slicer.cancel_slicing(dest_path)

	def load_profile(self, slicer, name):
		if not slicer in self.registered_slicers:
			return None

		try:
			path = self.get_profile_path(slicer, name, must_exist=True)
		except IOError:
			return None
		return self._load_profile_from_path(slicer, path)

	def save_profile(self, slicer, name, profile, overrides=None, allow_overwrite=True, display_name=None, description=None):
		if not slicer in self.registered_slicers:
			return

		if not isinstance(profile, SlicingProfile):
			if isinstance(profile, dict):
				profile = SlicingProfile(slicer, name, profile, display_name=display_name, description=description)
			else:
				raise ValueError("profile must be a SlicingProfile")
		else:
			profile.slicer = slicer
			profile.name = name
			if display_name is not None:
				profile.display_name = display_name
			if description is not None:
				profile.description = description

		path = self.get_profile_path(slicer, name)
		self._save_profile_to_path(slicer, path, profile, overrides=overrides, allow_overwrite=allow_overwrite)
		return profile

	def temporary_profile(self, slicer, name=None, overrides=None):
		if not slicer in self.registered_slicers:
			return None

		profile = self._get_default_profile(slicer)
		if name:
			try:
				profile = self.load_profile(slicer, name)
			except IOError:
				profile = self._get_default_profile(slicer)
			else:
				if profile is None:
					profile = self._get_default_profile(slicer)

		return TemporaryProfile(self.get_slicer(slicer).save_slicer_profile, profile, overrides=overrides)

	def delete_profile(self, slicer, name):
		if not slicer in self.registered_slicers:
			return None

		path = self.get_profile_path(slicer, name)
		if not os.path.exists(path) or not os.path.isfile(path):
			return
		os.remove(path)

	def all_profiles(self, slicer):
		if not slicer in self.registered_slicers:
			return None

		profiles = dict()
		slicer_profile_path = self.get_slicer_profile_path(slicer)
		for entry in os.listdir(slicer_profile_path):
			if not entry.endswith(".profile") or entry.startswith("."):
				# we are only interested in profiles and no hidden files
				continue

			path = os.path.join(slicer_profile_path, entry)
			profile_name = entry[:-len(".profile")]

			profiles[profile_name] = self._load_profile_from_path(slicer, path)
		return profiles

	def get_slicer_profile_path(self, slicer):
		if not slicer in self.registered_slicers:
			return None

		path = os.path.join(self._profile_path, slicer)
		if not os.path.exists(path):
			os.makedirs(path)
		return path

	def get_profile_path(self, slicer, name, must_exist=False):
		if not slicer in self.registered_slicers:
			return None

		if not name:
			return None

		name = self._sanitize(name)

		path = os.path.join(self.get_slicer_profile_path(slicer), "{name}.profile".format(name=name))
		if not os.path.realpath(path).startswith(self._profile_path):
			raise IOError("Path to profile {name} tried to break out of allows sub path".format(**locals()))
		if must_exist and not (os.path.exists(path) and os.path.isfile(path)):
			raise IOError("Profile {name} doesn't exist".format(**locals()))
		return path

	def _sanitize(self, name):
		if name is None:
			return None

		if "/" in name or "\\" in name:
			raise ValueError("name must not contain / or \\")

		import string
		valid_chars = "-_.() {ascii}{digits}".format(ascii=string.ascii_letters, digits=string.digits)
		sanitized_name = ''.join(c for c in name if c in valid_chars)
		sanitized_name = sanitized_name.replace(" ", "_")
		return sanitized_name

	def _load_profile_from_path(self, slicer, path):
		return self.get_slicer(slicer).get_slicer_profile(path)

	def _save_profile_to_path(self, slicer, path, profile, allow_overwrite=True, overrides=None):
		self.get_slicer(slicer).save_slicer_profile(path, profile, allow_overwrite=allow_overwrite, overrides=overrides)

	def _get_default_profile(self, slicer):
		default_profiles = settings().get(["slicing", "defaultProfiles"])
		if default_profiles and slicer in default_profiles:
			try:
				return self.load_profile(slicer, default_profiles[slicer])
			except:
				pass

		return self.get_slicer(slicer).get_slicer_default_profile()

<commit_msg>Fixed a bug causing slicing not to work if no printer profile was provided<commit_after># coding=utf-8
from __future__ import absolute_import

__author__ = "Gina Häußge <osd@foosel.net>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'
__copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License"

import os
import octoprint.plugin
import octoprint.events
from octoprint.settings import settings

class SlicingProfile(object):
	def __init__(self, slicer, name, data, display_name=None, description=None):
		self.slicer = slicer
		self.name = name
		self.data = data
		self.display_name = display_name
		self.description = description

class TemporaryProfile(object):
	def __init__(self, save_profile, profile, overrides=None):
		self.save_profile = save_profile
		self.profile = profile
		self.overrides = overrides

	def __enter__(self):
		import tempfile
		temp_profile = tempfile.NamedTemporaryFile(prefix="slicing-profile-temp-", suffix=".profile", delete=False)
		temp_profile.close()

		self.temp_path = temp_profile.name
		self.save_profile(self.temp_path, self.profile, overrides=self.overrides)
		return self.temp_path

	def __exit__(self, type, value, traceback):
		import os
		try:
			os.remove(self.temp_path)
		except:
			pass

class SlicingCancelled(BaseException):
	pass

class SlicingManager(object):
	def __init__(self, profile_path, printer_profile_manager):
		self._profile_path = profile_path
		self._printer_profile_manager = printer_profile_manager

		self._slicers = dict()
		self._slicer_names = dict()
		self._load_slicers()

		self._progress_callbacks = []
		self._last_progress_report = None

	def register_progress_callback(self, callback):
		self._progress_callbacks.append(callback)

	def unregister_progress_callback(self, callback):
		self._progress_callbacks.remove(callback)

	def _load_slicers(self):
		plugins = octoprint.plugin.plugin_manager().get_implementations(octoprint.plugin.SlicerPlugin)
		for name, plugin in plugins.items():
			if plugin.is_slicer_configured():
				self._slicers[plugin.get_slicer_properties()["type"]] = plugin

	@property
	def slicing_enabled(self):
		return len(self.configured_slicers) > 0

	@property
	def registered_slicers(self):
		return self._slicers.keys()

	@property
	def configured_slicers(self):
		return map(lambda slicer: slicer.get_slicer_properties()["type"], filter(lambda slicer: slicer.is_slicer_configured(), self._slicers.values()))

	@property
	def default_slicer(self):
		slicer_name = settings().get(["slicing", "defaultSlicer"])
		if slicer_name in self.configured_slicers:
			return slicer_name
		else:
			return None

	def get_slicer(self, slicer, require_configured=True):
		return self._slicers[slicer] if slicer in self._slicers and (not require_configured or self._slicers[slicer].is_slicer_configured()) else None

	def slice(self, slicer_name, source_path, dest_path, profile_name, callback, callback_args=None, callback_kwargs=None, overrides=None, on_progress=None, on_progress_args=None, on_progress_kwargs=None, printer_profile_id=None):
		if callback_args is None:
			callback_args = ()
		if callback_kwargs is None:
			callback_kwargs = dict()

		if not slicer_name in self.configured_slicers:
			if not slicer_name in self.registered_slicers:
				error = "No such slicer: {slicer_name}".format(**locals())
			else:
				error = "Slicer not configured: {slicer_name}".format(**locals())
			callback_kwargs.update(dict(_error=error))
			callback(*callback_args, **callback_kwargs)
			return False, error

		slicer = self.get_slicer(slicer_name)

		printer_profile = None
		if printer_profile_id is not None:
			printer_profile = self._printer_profile_manager.get(printer_profile_id)

		if printer_profile is None:
			printer_profile = self._printer_profile_manager.get_current_or_default()

		def slicer_worker(slicer, model_path, machinecode_path, profile_name, overrides, printer_profile, callback, callback_args, callback_kwargs):
			try:
				slicer_name = slicer.get_slicer_properties()["type"]
				with self.temporary_profile(slicer_name, name=profile_name, overrides=overrides) as profile_path:
					ok, result = slicer.do_slice(
						model_path,
						printer_profile,
						machinecode_path=machinecode_path,
						profile_path=profile_path,
						on_progress=on_progress,
						on_progress_args=on_progress_args,
						on_progress_kwargs=on_progress_kwargs
					)

				if not ok:
					callback_kwargs.update(dict(_error=result))
			except SlicingCancelled:
				callback_kwargs.update(dict(_cancelled=True))
			finally:
				callback(*callback_args, **callback_kwargs)

		import threading
		slicer_worker_thread = threading.Thread(target=slicer_worker,
		                                        args=(slicer, source_path, dest_path, profile_name, overrides, printer_profile, callback, callback_args, callback_kwargs))
		slicer_worker_thread.daemon = True
		slicer_worker_thread.start()
		return True, None

	def cancel_slicing(self, slicer_name, source_path, dest_path):
		if not slicer_name in self.registered_slicers:
			return
		slicer = self.get_slicer(slicer_name)
		slicer.cancel_slicing(dest_path)

	def load_profile(self, slicer, name):
		if not slicer in self.registered_slicers:
			return None

		try:
			path = self.get_profile_path(slicer, name, must_exist=True)
		except IOError:
			return None
		return self._load_profile_from_path(slicer, path)

	def save_profile(self, slicer, name, profile, overrides=None, allow_overwrite=True, display_name=None, description=None):
		if not slicer in self.registered_slicers:
			return

		if not isinstance(profile, SlicingProfile):
			if isinstance(profile, dict):
				profile = SlicingProfile(slicer, name, profile, display_name=display_name, description=description)
			else:
				raise ValueError("profile must be a SlicingProfile")
		else:
			profile.slicer = slicer
			profile.name = name
			if display_name is not None:
				profile.display_name = display_name
			if description is not None:
				profile.description = description

		path = self.get_profile_path(slicer, name)
		self._save_profile_to_path(slicer, path, profile, overrides=overrides, allow_overwrite=allow_overwrite)
		return profile

	def temporary_profile(self, slicer, name=None, overrides=None):
		if not slicer in self.registered_slicers:
			return None

		profile = self._get_default_profile(slicer)
		if name:
			try:
				profile = self.load_profile(slicer, name)
			except IOError:
				profile = self._get_default_profile(slicer)
			else:
				if profile is None:
					profile = self._get_default_profile(slicer)

		return TemporaryProfile(self.get_slicer(slicer).save_slicer_profile, profile, overrides=overrides)

	def delete_profile(self, slicer, name):
		if not slicer in self.registered_slicers:
			return None

		path = self.get_profile_path(slicer, name)
		if not os.path.exists(path) or not os.path.isfile(path):
			return
		os.remove(path)

	def all_profiles(self, slicer):
		if not slicer in self.registered_slicers:
			return None

		profiles = dict()
		slicer_profile_path = self.get_slicer_profile_path(slicer)
		for entry in os.listdir(slicer_profile_path):
			if not entry.endswith(".profile") or entry.startswith("."):
				# we are only interested in profiles and no hidden files
				continue

			path = os.path.join(slicer_profile_path, entry)
			profile_name = entry[:-len(".profile")]

			profiles[profile_name] = self._load_profile_from_path(slicer, path)
		return profiles

	def get_slicer_profile_path(self, slicer):
		if not slicer in self.registered_slicers:
			return None

		path = os.path.join(self._profile_path, slicer)
		if not os.path.exists(path):
			os.makedirs(path)
		return path

	def get_profile_path(self, slicer, name, must_exist=False):
		if not slicer in self.registered_slicers:
			return None

		if not name:
			return None

		name = self._sanitize(name)

		path = os.path.join(self.get_slicer_profile_path(slicer), "{name}.profile".format(name=name))
		if not os.path.realpath(path).startswith(self._profile_path):
			raise IOError("Path to profile {name} tried to break out of allows sub path".format(**locals()))
		if must_exist and not (os.path.exists(path) and os.path.isfile(path)):
			raise IOError("Profile {name} doesn't exist".format(**locals()))
		return path

	def _sanitize(self, name):
		if name is None:
			return None

		if "/" in name or "\\" in name:
			raise ValueError("name must not contain / or \\")

		import string
		valid_chars = "-_.() {ascii}{digits}".format(ascii=string.ascii_letters, digits=string.digits)
		sanitized_name = ''.join(c for c in name if c in valid_chars)
		sanitized_name = sanitized_name.replace(" ", "_")
		return sanitized_name

	def _load_profile_from_path(self, slicer, path):
		return self.get_slicer(slicer).get_slicer_profile(path)

	def _save_profile_to_path(self, slicer, path, profile, allow_overwrite=True, overrides=None):
		self.get_slicer(slicer).save_slicer_profile(path, profile, allow_overwrite=allow_overwrite, overrides=overrides)

	def _get_default_profile(self, slicer):
		default_profiles = settings().get(["slicing", "defaultProfiles"])
		if default_profiles and slicer in default_profiles:
			try:
				return self.load_profile(slicer, default_profiles[slicer])
			except:
				pass

		return self.get_slicer(slicer).get_slicer_default_profile()

<|endoftext|>


<commit_before>'''
File: seir.py
Authors:
 - Sjoerd Wenker, 10617558
 - Tristan van Vaalen, 10551832

SEIR model
'''

import integration as ig

if __name__ == '__main__':
    N = 1000.0
    t = 50.0 / N
    eps = 1.0 / 6.0
    mu = 1.0 / 9.0
    beta = 0.04
    b = 0.0

    st = lambda (t, s, e, i, r): N - beta * s * i - mu * s
    se = lambda (t, s, e, i, r): beta * s * i - (eps + mu) * e
    si = lambda (t, s, e, i, r): eps * e - (t + mu) * i
    sr = lambda (t, s, e, i, r): t * i - mu * r

    m = ig.RungeKutta4([st, se, si, sr], 0, [N - 1, 1, 0, 0])
    m.generate_n(10000)
    print m
    ig.plot(m)
<commit_msg>Almost working version of seir<commit_after>'''
File: seir.py
Authors:
 - Sjoerd Wenker, 10617558
 - Tristan van Vaalen, 10551832

SEIR model
'''

import integration as ig
import matplotlib.pyplot as plt

if __name__ == '__main__':
    n = 1000.0  # population
    beta = 0.96  # infection rate
    gamma = 1.0 / 9.0  # recovery rate
    sigma = 1.0 / 6.0  # conversion rate
    mu = 0.0  # death rate (0 for stable pop)
    nu = 0.0  # pass to resistant

    st = lambda (t, s, e, i, r): mu * (n - s) - beta * ((s * i) / n) - nu * s
    et = lambda (t, s, e, i, r): beta * ((s * i) / n) - (mu + sigma) * e
    it = lambda (t, s, e, i, r): sigma * e + (mu + gamma) * i
    rt = lambda (t, s, e, i, r): gamma * i - mu * r + nu * s

    m = ig.RungeKutta4([st, et, it, rt], 0, [999, 1, 0, 0], stepsize=0.01)
    m.generate_n(100)

    plt.plot(m.get_y_values()[0], label="S")
    plt.plot(m.get_y_values()[1], label="E")
    plt.plot(m.get_y_values()[2], label="I")
    plt.plot(m.get_y_values()[3], label="R")
    plt.legend(loc='best')
    plt.show()
<|endoftext|>


<commit_before><commit_msg>Adding import to package init to help module consumer call object<commit_after>from Subscriber import Subscriber
<|endoftext|>


<commit_before>#! /usr/bin/env /usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

#
# moveTables.py
# to move cassandra tables created by analytics form ContrailAnalytics
# keyspace with Thrift to ContrailAnalyticsCql keyspace with Cql
#

import uuid
import socket
import json
import numpy
import sys
import re
import copy
import random
import gendb.ttypes
from gendb.ttypes import DbDataType

import pycassa
from pycassa.columnfamily import ColumnFamily
from pycassa.types import *
from pycassa import *
import argparse
import datetime

from datetime import datetime
from cassandra.cluster import Cluster
from cassandra.query import BatchStatement
from sandesh.viz.constants import *
from sandesh.viz.constants import _VIZD_TABLE_SCHEMA, _VIZD_FLOW_TABLE_SCHEMA, _VIZD_STAT_TABLE_SCHEMA

'''
Dealing with no UUID serialization support in json
'''
from json import JSONEncoder
from uuid import UUID
JSONEncoder_olddefault = JSONEncoder.default
def JSONEncoder_newdefault(self, o):
    if isinstance(o, UUID): return str(o)
    return JSONEncoder_olddefault(self, o)
JSONEncoder.default = JSONEncoder_newdefault

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
parser.add_argument("--cassandra_server_ip", "-s",
                    default = '127.0.0.1',
                    help = "cassandra server ip")
parser.add_argument("--cassandra_server_port", "-p",
                    type = int,
                    default = 9160,
                    help = "cassandra server port")
parser.add_argument("--max_batch_entry_count", "-w",
                    type = int,
                    default = 50,
                    help = "number of rows to be fetched")
group.add_argument("--copy_cf", help = "Copy tables from ContrailAnalytics to ContrailAnalyticsCql",
                    metavar='COLUMNFAMILY', default = None)
group.add_argument("--list_cf", help = "List all the column families",
                    action="store_true")
args = parser.parse_args()

import logging
logging.basicConfig()

log = logging.getLogger()
log.setLevel('INFO')

class SimpleClient:
    session = None

    def connect(self, nodes):
        cluster = Cluster(contact_points=nodes, protocol_version=3)
        metadata = cluster.metadata
        self.session = cluster.connect()
        log.info('Connected to cluster: ' + metadata.cluster_name)
        for host in metadata.all_hosts():
            log.info('Datacenter: %s; Host: %s; Rack: %s',
                host.datacenter, host.address, host.rack)

    def close(self):
        self.session.cluster.shutdown()
        self.session.shutdown()
        log.info('Connection closed.')

    def create_schema(self):
        self.session.execute("USE \"{0}\";".format(COLLECTOR_KEYSPACE_CQL))

    def load_data(self, batch):
        self.session.execute(batch)

    def query_schema(self, table):
        query = "SELECT * FROM {0}".format(table)
        results = self.session.execute(query)
        for row in results:
            print row
        log.info('Schema queried.')

server_and_port = args.cassandra_server_ip+':'+str(args.cassandra_server_port)
pool = pycassa.ConnectionPool(COLLECTOR_KEYSPACE, server_list=[server_and_port])
sysm = pycassa.system_manager.SystemManager(server_and_port)

def list_cf(keyspace = COLLECTOR_KEYSPACE):
    dict = sysm.get_keyspace_column_families(keyspace)
    print dict.keys()

def copy_table(name):
    found_table = False
    for t in table_list:
	print t.table_name, name
	if(t.table_name == name):
	    found_table = True
	    if(t.is_static == True):
		copy_static_table(name)
	    else:
		copy_dynamic_table(name)
	    break
    if found_table == False:
    	print 'Not sure, how to copy this table: ', name
    return

def _convert_to_cql_data(columns, col_num, thrift_data, data_index):
    if(DbDataType.Unsigned32Type == columns[col_num].datatype):
        if isinstance(thrift_data, tuple):
            return numpy.int32(thrift_data[data_index])
        else:
            return numpy.int32(thrift_data)
    elif(DbDataType.InetType == columns[col_num].datatype):
        if isinstance(thrift_data, tuple):
            ip = socket.inet_ntoa(hex(thrift_data[data_index] & 0xffffffff)[2:].zfill(8).decode('hex'))
        else:
            ip = socket.inet_ntoa(hex(thrift_data & 0xffffffff)[2:].zfill(8).decode('hex'))
        return ip
    else:
        if isinstance(thrift_data, tuple):
            return thrift_data[data_index]
        else:
            return thrift_data

def copy_static_table(table):
    c_ft = pycassa.ColumnFamily(pool, table)
    table_rows = (c_ft.get_range(include_ttl=True))

    command = "BEGIN BATCH INSERT INTO {0} (key".format(table)
    columns = []
    for t in table_list:
	if(t.table_name == table):
	    columns = t.columns
	    break
    field_dict = {}
    for column in columns:
	field_dict[column.name] = column.datatype

    num_rows_in_batch = 0
    rowKey = ''
    for rowKey,columns in table_rows:
	num_rows_in_batch += 1
	if(num_rows_in_batch % args.max_batch_entry_count == 0):
	    num_rows_in_batch = 0
	    command += 'APPLY BATCH;'
    	    client.session.execute(command)
    	    command = "BEGIN BATCH INSERT INTO {0} (key".format(table)
	if isinstance(rowKey, unicode):
	    rowKey = rowKey.encode('utf-8')
	values = [rowKey]
	ttl = 0
	for key in columns.keys():
	    value, ttl = columns.get(key)
	    if(DbDataType.Unsigned32Type == field_dict['key']):
		value = numpy.int32(value)
	    elif(DbDataType.InetType == field_dict['key']):
	    	ip = socket.inet_ntoa(hex(value & 0xffffffff)[2:].zfill(8).decode('hex'))
		value = ip
	    command = '{0}, "{1}"'.format(command, key)
	    values.append(value)
	command += ') VALUES '
	if(ttl):
	    command = "{0}{1} USING TTL {2};".format(command, tuple(values), ttl)
	else:
	    command = "{0}{1};".format(command, tuple(values))
    if(rowKey != ''):
    	command += 'APPLY BATCH;'
    	client.session.execute(command)

def is_json(myjson):
  try:
    json_object = json.loads(myjson)
  except ValueError, e:
    return False
  return True

def copy_dynamic_table(table):
    c_ft = pycassa.ColumnFamily(pool, table)
    table_rows = (c_ft.get_range(include_ttl=True))

    command = "INSERT INTO {0} ("
    table_columns = []
    values = ''
    keyCount = 0
    for t in table_list:
	if(t.table_name == table):
	    table_columns = t.columns
	    for column in t.columns:
                if keyCount == 0:
		    command += column.name
		    values = ') VALUES (?'
                else:
		    command += ', ' + column.name
		    values += ', ?'
                if (column.key == True):
		    keyCount += 1
	    break
    command += values + ') USING TTL ?'
    insert_flow = client.session.prepare(command.format(table))

    batch = BatchStatement()
    num_rows_in_batch = 0
    rowKey = ''
    for rowKey,columns in table_rows:
	key_value = []
	rowKey_size = 0
	if isinstance(rowKey, tuple):
	    for key in range(0, len(list(rowKey))):
		key_value.append(_convert_to_cql_data(table_columns, rowKey_size, rowKey, rowKey_size))
	    	rowKey_size += 1
            if(t.is_index_table == True):
                key_value.append(random.randrange(0, 16))
	else:
	    key_value.append(_convert_to_cql_data(table_columns, rowKey_size, rowKey, rowKey_size))
	    rowKey_size += 1
	for key in columns.keys():
	    value, ttl = columns.get(key)
	    jsonValue = value
	    if is_json(value) != True:
	    	jsonValue = json.dumps(value, ensure_ascii=False)
	    column_value = copy.deepcopy(key_value)
	    num_rows_in_batch += 1
	    if(num_rows_in_batch % args.max_batch_entry_count == 0):
	    	num_rows_in_batch = 0
    	    	client.load_data(batch)
	    	batch = BatchStatement()
	    for col_num in range(rowKey_size, len(table_columns)-1):
		column_value.append(_convert_to_cql_data(table_columns, col_num, key, col_num-rowKey_size))
	    column_value.append(jsonValue)
	    column_value.append(ttl if ttl else 7200)

	    batch.add(insert_flow, tuple(column_value))
    if(rowKey != ''):
    	client.load_data(batch)

client = SimpleClient()
client.connect([args.cassandra_server_ip])
client.create_schema()
table_list = _VIZD_TABLE_SCHEMA + _VIZD_FLOW_TABLE_SCHEMA + _VIZD_STAT_TABLE_SCHEMA

dict = sysm.get_keyspace_column_families(COLLECTOR_KEYSPACE)
tables = dict.keys()

if args.copy_cf == None:
    startTime = datetime.utcnow()
    print 'Start time for copying: ', startTime
    for table in tables:
    	print 'Start time for table ', table, ': ', datetime.utcnow()
	copy_table(table)
    	print 'Finish time for table ', table, ': ', datetime.utcnow()
    endTime = datetime.utcnow()
    print 'Total time taken: ', endTime - startTime
else:
    copy_table(args.copy_cf)

if args.list_cf != False:
    list_cf()
<commit_msg>Minor bug fix in the script to move tables<commit_after>#! /usr/bin/env /usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

#
# moveTables.py
# to move cassandra tables created by analytics form ContrailAnalytics
# keyspace with Thrift to ContrailAnalyticsCql keyspace with Cql
#

import uuid
import socket
import json
import numpy
import sys
import re
import copy
import random
from sandesh.viz.gendb.ttypes import DbDataType

import pycassa
from pycassa.columnfamily import ColumnFamily
from pycassa.types import *
from pycassa import *
import argparse
import datetime

from datetime import datetime
from cassandra.cluster import Cluster
from cassandra.query import BatchStatement
from sandesh.viz.constants import *
from sandesh.viz.constants import _VIZD_TABLE_SCHEMA, _VIZD_FLOW_TABLE_SCHEMA, _VIZD_STAT_TABLE_SCHEMA

'''
Dealing with no UUID serialization support in json
'''
from json import JSONEncoder
from uuid import UUID
JSONEncoder_olddefault = JSONEncoder.default
def JSONEncoder_newdefault(self, o):
    if isinstance(o, UUID): return str(o)
    return JSONEncoder_olddefault(self, o)
JSONEncoder.default = JSONEncoder_newdefault

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
parser.add_argument("--cassandra_server_ip", "-s",
                    default = '127.0.0.1',
                    help = "cassandra server ip")
parser.add_argument("--cassandra_server_port", "-p",
                    type = int,
                    default = 9160,
                    help = "cassandra server port")
parser.add_argument("--max_batch_entry_count", "-w",
                    type = int,
                    default = 50,
                    help = "number of rows to be fetched")
group.add_argument("--copy_cf", help = "Copy tables from ContrailAnalytics to ContrailAnalyticsCql",
                    metavar='COLUMNFAMILY', default = None)
group.add_argument("--list_cf", help = "List all the column families",
                    action="store_true")
args = parser.parse_args()

import logging
logging.basicConfig()

log = logging.getLogger()
log.setLevel('INFO')

class SimpleClient:
    session = None

    def connect(self, nodes):
        cluster = Cluster(contact_points=nodes, protocol_version=3)
        metadata = cluster.metadata
        self.session = cluster.connect()
        log.info('Connected to cluster: ' + metadata.cluster_name)
        for host in metadata.all_hosts():
            log.info('Datacenter: %s; Host: %s; Rack: %s',
                host.datacenter, host.address, host.rack)

    def close(self):
        self.session.cluster.shutdown()
        self.session.shutdown()
        log.info('Connection closed.')

    def create_schema(self):
        self.session.execute("USE \"{0}\";".format(COLLECTOR_KEYSPACE_CQL))

    def load_data(self, batch):
        self.session.execute(batch)

    def query_schema(self, table):
        query = "SELECT * FROM {0}".format(table)
        results = self.session.execute(query)
        for row in results:
            print row
        log.info('Schema queried.')

server_and_port = args.cassandra_server_ip+':'+str(args.cassandra_server_port)
pool = pycassa.ConnectionPool(COLLECTOR_KEYSPACE, server_list=[server_and_port])
sysm = pycassa.system_manager.SystemManager(server_and_port)

def list_cf(keyspace = COLLECTOR_KEYSPACE):
    dict = sysm.get_keyspace_column_families(keyspace)
    print dict.keys()

def copy_table(name):
    found_table = False
    for t in table_list:
	print t.table_name, name
	if(t.table_name == name):
	    found_table = True
	    if(t.is_static == True):
		copy_static_table(name)
	    else:
		copy_dynamic_table(name)
	    break
    if found_table == False:
    	print 'Not sure, how to copy this table: ', name
    return

def _convert_to_cql_data(columns, col_num, thrift_data, data_index):
    if(DbDataType.Unsigned32Type == columns[col_num].datatype):
        if isinstance(thrift_data, tuple):
            return numpy.int32(thrift_data[data_index])
        else:
            return numpy.int32(thrift_data)
    elif(DbDataType.InetType == columns[col_num].datatype):
        if isinstance(thrift_data, tuple):
            ip = socket.inet_ntoa(hex(thrift_data[data_index] & 0xffffffff)[2:].zfill(8).decode('hex'))
        else:
            ip = socket.inet_ntoa(hex(thrift_data & 0xffffffff)[2:].zfill(8).decode('hex'))
        return ip
    else:
        if isinstance(thrift_data, tuple):
            return thrift_data[data_index]
        else:
            return thrift_data

def copy_static_table(table):
    c_ft = pycassa.ColumnFamily(pool, table)
    table_rows = (c_ft.get_range(include_ttl=True))

    command = "BEGIN BATCH INSERT INTO {0} (key".format(table)
    columns = []
    for t in table_list:
	if(t.table_name == table):
	    columns = t.columns
	    break
    field_dict = {}
    for column in columns:
	field_dict[column.name] = column.datatype

    num_rows_in_batch = 0
    rowKey = ''
    for rowKey,columns in table_rows:
	num_rows_in_batch += 1
	if(num_rows_in_batch % args.max_batch_entry_count == 0):
	    num_rows_in_batch = 0
	    command += 'APPLY BATCH;'
    	    client.session.execute(command)
    	    command = "BEGIN BATCH INSERT INTO {0} (key".format(table)
	if isinstance(rowKey, unicode):
	    rowKey = rowKey.encode('utf-8')
	values = [rowKey]
	ttl = 0
	for key in columns.keys():
	    value, ttl = columns.get(key)
	    if(DbDataType.Unsigned32Type == field_dict['key']):
		value = numpy.int32(value)
	    elif(DbDataType.InetType == field_dict['key']):
	    	ip = socket.inet_ntoa(hex(value & 0xffffffff)[2:].zfill(8).decode('hex'))
		value = ip
	    command = '{0}, "{1}"'.format(command, key)
	    values.append(value)
	command += ') VALUES '
	if(ttl):
	    command = "{0}{1} USING TTL {2};".format(command, tuple(values), ttl)
	else:
	    command = "{0}{1};".format(command, tuple(values))
    if(rowKey != ''):
    	command += 'APPLY BATCH;'
    	client.session.execute(command)

def is_json(myjson):
  try:
    json_object = json.loads(myjson)
  except ValueError, e:
    return False
  return True

def copy_dynamic_table(table):
    c_ft = pycassa.ColumnFamily(pool, table)
    table_rows = (c_ft.get_range(include_ttl=True))

    command = "INSERT INTO {0} ("
    table_columns = []
    values = ''
    keyCount = 0
    for t in table_list:
	if(t.table_name == table):
	    table_columns = t.columns
	    for column in t.columns:
                if keyCount == 0:
		    command += column.name
		    values = ') VALUES (?'
                else:
		    command += ', ' + column.name
		    values += ', ?'
                if (column.key == True):
		    keyCount += 1
	    break
    command += values + ') USING TTL ?'
    insert_flow = client.session.prepare(command.format(table))

    batch = BatchStatement()
    num_rows_in_batch = 0
    rowKey = ''
    for rowKey,columns in table_rows:
	key_value = []
	rowKey_size = 0
	if isinstance(rowKey, tuple):
	    for key in range(0, len(list(rowKey))):
		key_value.append(_convert_to_cql_data(table_columns, rowKey_size, rowKey, rowKey_size))
	    	rowKey_size += 1
            if(t.is_index_table == True):
                key_value.append(random.randrange(0, 16))
	else:
	    key_value.append(_convert_to_cql_data(table_columns, rowKey_size, rowKey, rowKey_size))
	    rowKey_size += 1
	for key in columns.keys():
	    value, ttl = columns.get(key)
	    jsonValue = value
	    if is_json(value) != True:
	    	jsonValue = json.dumps(value, ensure_ascii=False)
	    column_value = copy.deepcopy(key_value)
	    num_rows_in_batch += 1
	    if(num_rows_in_batch % args.max_batch_entry_count == 0):
	    	num_rows_in_batch = 0
    	    	client.load_data(batch)
	    	batch = BatchStatement()
	    for col_num in range(rowKey_size, len(table_columns)-1):
		column_value.append(_convert_to_cql_data(table_columns, col_num, key, col_num-rowKey_size))
	    column_value.append(jsonValue)
	    column_value.append(ttl if ttl else 7200)

	    batch.add(insert_flow, tuple(column_value))
    if(rowKey != ''):
    	client.load_data(batch)

client = SimpleClient()
client.connect([args.cassandra_server_ip])
client.create_schema()
table_list = _VIZD_TABLE_SCHEMA + _VIZD_FLOW_TABLE_SCHEMA + _VIZD_STAT_TABLE_SCHEMA

dict = sysm.get_keyspace_column_families(COLLECTOR_KEYSPACE)
tables = dict.keys()

if args.copy_cf == None:
    startTime = datetime.utcnow()
    print 'Start time for copying: ', startTime
    for table in tables:
    	print 'Start time for table ', table, ': ', datetime.utcnow()
	copy_table(table)
    	print 'Finish time for table ', table, ': ', datetime.utcnow()
    endTime = datetime.utcnow()
    print 'Total time taken: ', endTime - startTime
else:
    copy_table(args.copy_cf)

if args.list_cf != False:
    list_cf()
<|endoftext|>


<commit_before>#
# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from optparse import SUPPRESS_HELP
import os
import re
import subprocess
import sys

from git_command import GIT
from command import Command, MirrorSafeCommand
from error import RepoChangedException, GitError
from project import R_HEADS

class Sync(Command, MirrorSafeCommand):
  common = True
  helpSummary = "Update working tree to the latest revision"
  helpUsage = """
%prog [<project>...]
"""
  helpDescription = """
The '%prog' command synchronizes local project directories
with the remote repositories specified in the manifest.  If a local
project does not yet exist, it will clone a new local directory from
the remote repository and set up tracking branches as specified in
the manifest.  If the local project already exists, '%prog'
will update the remote branches and rebase any new local changes
on top of the new remote changes.

'%prog' will synchronize all projects listed at the command
line.  Projects can be specified either by name, or by a relative
or absolute path to the project's local directory. If no projects
are specified, '%prog' will synchronize all projects listed in
the manifest.

The -d/--detach option can be used to switch specified projects
back to the manifest revision.  This option is especially helpful
if the project is currently on a topic branch, but the manifest
revision is temporarily needed.
"""

  def _Options(self, p):
    p.add_option('-n','--network-only',
                 dest='network_only', action='store_true',
                 help="fetch only, don't update working tree")
    p.add_option('-d','--detach',
                 dest='detach_head', action='store_true',
                 help='detach projects back to manifest revision')

    p.add_option('--no-repo-verify',
                 dest='no_repo_verify', action='store_true',
                 help='do not verify repo source code')
    p.add_option('--repo-upgraded',
                 dest='repo_upgraded', action='store_true',
                 help=SUPPRESS_HELP)

  def _Fetch(self, *projects):
    fetched = set()
    for project in projects:
      if project.Sync_NetworkHalf():
        fetched.add(project.gitdir)
      else:
        print >>sys.stderr, 'error: Cannot fetch %s' % project.name
        sys.exit(1)
    return fetched

  def Execute(self, opt, args):
    if opt.network_only and opt.detach_head:
      print >>sys.stderr, 'error: cannot combine -n and -d'
      sys.exit(1)

    rp = self.manifest.repoProject
    rp.PreSync()

    mp = self.manifest.manifestProject
    mp.PreSync()

    if opt.repo_upgraded:
      for project in self.manifest.projects.values():
        if project.Exists:
          project.PostRepoUpgrade()

    all = self.GetProjects(args, missing_ok=True)
    fetched = self._Fetch(rp, mp, *all)

    if rp.HasChanges:
      print >>sys.stderr, 'info: A new version of repo is available'
      print >>sys.stderr, ''
      if opt.no_repo_verify or _VerifyTag(rp):
        if not rp.Sync_LocalHalf():
          sys.exit(1)
        print >>sys.stderr, 'info: Restarting repo with latest version'
        raise RepoChangedException(['--repo-upgraded'])
      else:
        print >>sys.stderr, 'warning: Skipped upgrade to unverified version'

    if opt.network_only:
      # bail out now; the rest touches the working tree
      return

    if mp.HasChanges:
      if not mp.Sync_LocalHalf():
        sys.exit(1)

      self.manifest._Unload()
      all = self.GetProjects(args, missing_ok=True)
      missing = []
      for project in all:
        if project.gitdir not in fetched:
          missing.append(project)
      self._Fetch(*missing)

    for project in all:
      if project.worktree:
        if not project.Sync_LocalHalf(
            detach_head=opt.detach_head):
          sys.exit(1)

def _VerifyTag(project):
  gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
  if not os.path.exists(gpg_dir):
    print >>sys.stderr,\
"""warning: GnuPG was not available during last "repo init"
warning: Cannot automatically authenticate repo."""
    return True

  remote = project.GetRemote(project.remote.name)
  ref = remote.ToLocal(project.revision)

  try:
    cur = project.bare_git.describe(ref)
  except GitError:
    cur = None

  if not cur \
     or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
    rev = project.revision
    if rev.startswith(R_HEADS):
      rev = rev[len(R_HEADS):]

    print >>sys.stderr
    print >>sys.stderr,\
      "warning: project '%s' branch '%s' is not signed" \
      % (project.name, rev)
    return False

  env = dict(os.environ)
  env['GIT_DIR'] = project.gitdir
  env['GNUPGHOME'] = gpg_dir

  cmd = [GIT, 'tag', '-v', cur]
  proc = subprocess.Popen(cmd,
                          stdout = subprocess.PIPE,
                          stderr = subprocess.PIPE,
                          env = env)
  out = proc.stdout.read()
  proc.stdout.close()

  err = proc.stderr.read()
  proc.stderr.close()

  if proc.wait() != 0:
    print >>sys.stderr
    print >>sys.stderr, out
    print >>sys.stderr, err
    print >>sys.stderr
    return False
  return True
<commit_msg>Add 'repo sync -l' to only do local operations<commit_after>#
# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from optparse import SUPPRESS_HELP
import os
import re
import subprocess
import sys

from git_command import GIT
from command import Command, MirrorSafeCommand
from error import RepoChangedException, GitError
from project import R_HEADS

class Sync(Command, MirrorSafeCommand):
  common = True
  helpSummary = "Update working tree to the latest revision"
  helpUsage = """
%prog [<project>...]
"""
  helpDescription = """
The '%prog' command synchronizes local project directories
with the remote repositories specified in the manifest.  If a local
project does not yet exist, it will clone a new local directory from
the remote repository and set up tracking branches as specified in
the manifest.  If the local project already exists, '%prog'
will update the remote branches and rebase any new local changes
on top of the new remote changes.

'%prog' will synchronize all projects listed at the command
line.  Projects can be specified either by name, or by a relative
or absolute path to the project's local directory. If no projects
are specified, '%prog' will synchronize all projects listed in
the manifest.

The -d/--detach option can be used to switch specified projects
back to the manifest revision.  This option is especially helpful
if the project is currently on a topic branch, but the manifest
revision is temporarily needed.
"""

  def _Options(self, p):
    p.add_option('-l','--local-only',
                 dest='local_only', action='store_true',
                 help="only update working tree, don't fetch")
    p.add_option('-n','--network-only',
                 dest='network_only', action='store_true',
                 help="fetch only, don't update working tree")
    p.add_option('-d','--detach',
                 dest='detach_head', action='store_true',
                 help='detach projects back to manifest revision')

    p.add_option('--no-repo-verify',
                 dest='no_repo_verify', action='store_true',
                 help='do not verify repo source code')
    p.add_option('--repo-upgraded',
                 dest='repo_upgraded', action='store_true',
                 help=SUPPRESS_HELP)

  def _Fetch(self, *projects):
    fetched = set()
    for project in projects:
      if project.Sync_NetworkHalf():
        fetched.add(project.gitdir)
      else:
        print >>sys.stderr, 'error: Cannot fetch %s' % project.name
        sys.exit(1)
    return fetched

  def Execute(self, opt, args):
    if opt.network_only and opt.detach_head:
      print >>sys.stderr, 'error: cannot combine -n and -d'
      sys.exit(1)
    if opt.network_only and opt.local_only:
      print >>sys.stderr, 'error: cannot combine -n and -l'
      sys.exit(1)

    rp = self.manifest.repoProject
    rp.PreSync()

    mp = self.manifest.manifestProject
    mp.PreSync()

    if opt.repo_upgraded:
      for project in self.manifest.projects.values():
        if project.Exists:
          project.PostRepoUpgrade()

    all = self.GetProjects(args, missing_ok=True)

    if not opt.local_only:
      fetched = self._Fetch(rp, mp, *all)

      if rp.HasChanges:
        print >>sys.stderr, 'info: A new version of repo is available'
        print >>sys.stderr, ''
        if opt.no_repo_verify or _VerifyTag(rp):
          if not rp.Sync_LocalHalf():
            sys.exit(1)
          print >>sys.stderr, 'info: Restarting repo with latest version'
          raise RepoChangedException(['--repo-upgraded'])
        else:
          print >>sys.stderr, 'warning: Skipped upgrade to unverified version'

      if opt.network_only:
        # bail out now; the rest touches the working tree
        return

      if mp.HasChanges:
        if not mp.Sync_LocalHalf():
          sys.exit(1)

        self.manifest._Unload()
        all = self.GetProjects(args, missing_ok=True)
        missing = []
        for project in all:
          if project.gitdir not in fetched:
            missing.append(project)
        self._Fetch(*missing)

    for project in all:
      if project.worktree:
        if not project.Sync_LocalHalf(
            detach_head=opt.detach_head):
          sys.exit(1)

def _VerifyTag(project):
  gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
  if not os.path.exists(gpg_dir):
    print >>sys.stderr,\
"""warning: GnuPG was not available during last "repo init"
warning: Cannot automatically authenticate repo."""
    return True

  remote = project.GetRemote(project.remote.name)
  ref = remote.ToLocal(project.revision)

  try:
    cur = project.bare_git.describe(ref)
  except GitError:
    cur = None

  if not cur \
     or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
    rev = project.revision
    if rev.startswith(R_HEADS):
      rev = rev[len(R_HEADS):]

    print >>sys.stderr
    print >>sys.stderr,\
      "warning: project '%s' branch '%s' is not signed" \
      % (project.name, rev)
    return False

  env = dict(os.environ)
  env['GIT_DIR'] = project.gitdir
  env['GNUPGHOME'] = gpg_dir

  cmd = [GIT, 'tag', '-v', cur]
  proc = subprocess.Popen(cmd,
                          stdout = subprocess.PIPE,
                          stderr = subprocess.PIPE,
                          env = env)
  out = proc.stdout.read()
  proc.stdout.close()

  err = proc.stderr.read()
  proc.stderr.close()

  if proc.wait() != 0:
    print >>sys.stderr
    print >>sys.stderr, out
    print >>sys.stderr, err
    print >>sys.stderr
    return False
  return True
<|endoftext|>


<commit_before>"""
The latest version of this package is available at:
<http://github.com/jantman/RPyMostat>

##################################################################################
Copyright 2016 Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>

    This file is part of RPyMostat, also known as RPyMostat.

    RPyMostat is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    RPyMostat is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with RPyMostat.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
##################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/RPyMostat> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
##################################################################################

AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
##################################################################################
"""

import logging

from rpymostat.db import COLL_SENSORS, get_collection
from twisted.internet.defer import inlineCallbacks, returnValue

logger = logging.getLogger(__name__)

@inlineCallbacks
def update_sensor(dbconn, host_id, sensor_id, value, sensor_type=None,
                  sensor_alias=None, extra=None):
    """
    Update data for a single sensor in the database.

    :param dbconn: MongoDB database connection
    :type dbconn: txmongo.connection.ConnectionPool
    :param host_id: host_id that the sensor is reporting from
    :type host_id: str
    :param sensor_id: unique sensor ID
    :type sensor_id: str
    :param value: sensor reading in degress Celsius
    :type value: floar
    :param sensor_type: description of the type of sensor
    :type sensor_type: str
    :param sensor_alias: human-readable alias for the sensor
    :type sensor_alias: str
    :param extra: extra information about the sensor
    :type extra: str
    :return: database record ID
    :rtype: str
    """
    _id = '%s_%s' % (host_id, sensor_id)
    data = {'_id': _id, 'host_id': host_id, 'sensor_id': sensor_id,
            'last_reading_C': value}
    if sensor_type is not None:
        data['type'] = sensor_type
    if sensor_alias is not None:
        data['alias'] = sensor_alias
    if extra is not None:
        data['extra'] = extra
    logger.debug('Updating sensor %s: %s', _id, data)
    coll = get_collection(dbconn, COLL_SENSORS)
    res = yield coll.replace_one(
        {"_id": _id},
        data,
        upsert=True
    )
    logger.debug('Update result for %s: %s (%s)', _id, res, vars(res))
    returnValue(_id)
<commit_msg>fix typo<commit_after>"""
The latest version of this package is available at:
<http://github.com/jantman/RPyMostat>

##################################################################################
Copyright 2016 Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>

    This file is part of RPyMostat, also known as RPyMostat.

    RPyMostat is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    RPyMostat is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with RPyMostat.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
##################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/RPyMostat> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
##################################################################################

AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
##################################################################################
"""

import logging

from rpymostat.db import COLL_SENSORS, get_collection
from twisted.internet.defer import inlineCallbacks, returnValue

logger = logging.getLogger(__name__)

@inlineCallbacks
def update_sensor(dbconn, host_id, sensor_id, value, sensor_type=None,
                  sensor_alias=None, extra=None):
    """
    Update data for a single sensor in the database.

    :param dbconn: MongoDB database connection
    :type dbconn: txmongo.connection.ConnectionPool
    :param host_id: host_id that the sensor is reporting from
    :type host_id: str
    :param sensor_id: unique sensor ID
    :type sensor_id: str
    :param value: sensor reading in degress Celsius
    :type value: float
    :param sensor_type: description of the type of sensor
    :type sensor_type: str
    :param sensor_alias: human-readable alias for the sensor
    :type sensor_alias: str
    :param extra: extra information about the sensor
    :type extra: str
    :return: database record ID
    :rtype: str
    """
    _id = '%s_%s' % (host_id, sensor_id)
    data = {'_id': _id, 'host_id': host_id, 'sensor_id': sensor_id,
            'last_reading_C': value}
    if sensor_type is not None:
        data['type'] = sensor_type
    if sensor_alias is not None:
        data['alias'] = sensor_alias
    if extra is not None:
        data['extra'] = extra
    logger.debug('Updating sensor %s: %s', _id, data)
    coll = get_collection(dbconn, COLL_SENSORS)
    res = yield coll.replace_one(
        {"_id": _id},
        data,
        upsert=True
    )
    logger.debug('Update result for %s: %s (%s)', _id, res, vars(res))
    returnValue(_id)
<|endoftext|>


<commit_before>#! /usr/bin/env python2.7

import RandomGames as RG

from GameIO import read, to_JSON_str
from Regret import regret
from Nash import mixed_nash, replicator_dynamics, pure_nash

from sys import stdin
from argparse import ArgumentParser
from random import sample
from copy import copy
from functools import partial

import numpy as np

def subsample(game, num_samples):
	"""
	Returns a game with a random subset of the input game's payoff samples.

	Note: this is not intended for use on games with different numbers of
	samples per profile.
	"""
	sg = copy(game)
	sg.sample_values = map(lambda p: p[:,:,sample(range(p.shape[-1]), \
			num_samples)], sg.sample_values)
	sg.reset()
	sg.max_samples = num_samples
	return sg

def pre_aggregate(game, count):
	"""
	Aggregates samples to produce a game with |samples| / count samples.

	Note: this is not intended for use on games with different numbers of
	samples per profile.
	"""
	agg = copy(game)
	sv = agg.sample_values
	sv = np.swapaxes(sv, 0, -1)
	np.random.shuffle(sv)
	sv = np.swapaxes(sv, 0, -1)
	shape = list(sv.shape)
	samples = shape[-1]
	if samples % count != 0:
		sv = sv[:,:,:,:samples - (samples % count)]
	shape.append(count)
	shape[-2] /= count
	sv = np.reshape(sv, shape)
	sv = np.average(sv, axis=-1)
	agg.sample_values = sv
	agg.reset()
	agg.min_samples = agg.max_samples = shape[-2]
	return agg

def bootstrap(game, equilibrium, statistic=regret, method="resample", \
				method_args=[], points=1000):
	"""
	Returns a bootstrap distribution for the statistic.

	To run a resample regret boostrap:
		bootstrap(game, eq)
	To run a single-sample regret bootstrap:
		bootstrap(game, eq, method='single_sample')
	To run a replicator dynamics bootstrap:
		bootstrap(game, eq, replicator_dynamics)
	"""
	boot_dstr = []
	method = getattr(game, method)
	for __ in range(points):
		method(*method_args)
		boot_dstr.append(statistic(game, equilibrium))
	game.reset()
	boot_dstr.sort()
	return boot_dstr

def bootstrap_experiment(base_game_func, noise_model, statistic=regret, \
		num_games=1000, stdevs=[.1,1.,10.,100.], sample_sizes=[5,10,20, \
		50,100,200,500], equilibrium_search=mixed_nash, bootstrap_args=[]):
	results = [{s:{} for s in stdevs} for i in range(num_games)]
	for i in range(num_games):
		base_game = base_game_func()
		RG.rescale_payoffs(base_game, 0, 100)
		for stdev in stdevs:
			sample_game = RG.add_noise(base_game, noise_model, stdev, \
									sample_sizes[-1])
			for sample_size in sample_sizes:
				subsample_game = subsample(sample_game, sample_size)
				equilibria = equilibrium_search(subsample_game)
				results[i][stdev][sample_size] = [ \
					{ \
						"profile" : eq,
						"statistic" : statistic(base_game, eq),
						"bootstrap" : bootstrap(subsample_game, eq, statistic, \
												*bootstrap_args)
					} for eq in equilibria]
	return results

def parse_args():
	parser = ArgumentParser()
	parser.add_argument("game_func", type=str, default="", choices=\
						["","congestion","local_effect","uniform_symmetric", \
						"sym_2p2s"], help="Specifies the function generating "+\
						"random base games. If empty, the script will look "+\
						"for a file with simulated game data on stdin.")
	parser.add_argument("noise_func", type=str, default="", choices=["",\
						"normal","unimodal","bimodal","u80b20","u60b40",\
						"u40b60","u20b80","gaussian_mixture"], help=\
						"Noise model to perturb sample payoffs around the "+\
						"base game payoff. May only be empty if first "+\
						"argument is also empty.")
	parser.add_argument("-game_args", type=str, nargs="*", default=[], help=\
						"Arguments to pass to game_func. Usually players "+\
						"and strategies.")
	parser.add_argument("-noise_args", type=str, nargs="*", default=[], help=\
						"Arguments to pass to noise_func. Should not include"+\
						"noise magnitude or number of samples.")
	parser.add_argument("-num_games", type=int, default=1000, help="Number "+\
						"of games to generate per stdev/subsample "+\
						"combination. Default: 1000")
	parser.add_argument("-points", type=int, default=1000, help="Number of "+\
						"bootstrap resamples per equilibrium. Default: 1000")
	parser.add_argument("-pair", type=str, default="game", choices=["game", \
						"profile", "payoff"], help="Pairing level to use for "+\
						"bootstrap resampling. Default: game")
	parser.add_argument("-agg", type=int, default=0, help="Number of samples "+\
						"to pre-aggregate. Default: 0")
	parser.add_argument("-stdevs", type=float, nargs="*", default=\
						[.2,1.,5.,25.], help="Noise magnitude parameters "+\
						"passed to the noise model. Default: .2 1. 5. 25.")
	parser.add_argument("-sample_sizes", type=int, nargs="*", default=\
						[5,10,20,100,200,500], help="Numbers of samples "+\
						"per profile at which to test the bootstrap. "+\
						"Default: 5 10 20 100 200 500")
	parser.add_argument("--single", action="store_true", help="Set to use "+\
						"the single_sample_regret function.")
	parser.add_argument("--rd", action="store_true", help="Set to compute "+\
						"bootstrap distributions of equilibrium movement by "+\
						"replicator dynamics.")
	parser.add_argument("--pure", action="store_true", help="Find an compute "+\
						"bootstrap regret distributions for pure-strategy "+\
						"(rather than role-symmetric mixed-strategy) Nash "+\
						"equilibria.")
	args = parser.parse_args()

	if args.game_func == "none":
		game = read(stdin)
		args.game_func = lambda: game
		args.noise_func = lambda g,s,c: game.subsample(c)
	else:
		assert args.noise_func != "none", "Must specify a noise model."
		game_args = []
		for a in args.game_args:
			try:
				game_args.append(int(a))
			except ValueError:
				game_args.append(float(a))
		noise_args = []
		for a in args.noise_args:
			try:
				noise_args.append(int(a))
			except ValueError:
				noise_args.append(float(a))
		args.game_func = partial(getattr(RG, args.game_func), *game_args)
		noise_func = getattr(RG, args.noise_func + "_noise")
		args.noise_func = lambda s,c: noise_func(s,c, *noise_args)

	if args.agg > 0:
		args.noise_func = lambda s,c: pre_aggregate(args.noise_func(s,c), \
											args.agg)
	assert not (args.rd and args.pure), "Must use mixed_nash for rd bootstrap"
	args.bootstrap_args = ["resample" if not args.single else "singleSample", \
						[args.pair] if not args.single else [], args.points]
	return args

def main():
	args = parse_args()
	results = bootstrap_experiment(args.game_func, args.noise_func, \
					replicator_dynamics if args.rd else regret, \
					args.num_games, args.stdevs, args.sample_sizes, \
					pure_nash if args.pure else mixed_nash, args.bootstrap_args)
	print to_JSON_str(results, indent=None)

if __name__ == "__main__":
	main()

<commit_msg>Updated default args to Bootstrap.py.<commit_after>#! /usr/bin/env python2.7

import RandomGames as RG

from GameIO import read, to_JSON_str
from Regret import regret
from Nash import mixed_nash, replicator_dynamics, pure_nash

from sys import stdin
from argparse import ArgumentParser
from random import sample
from copy import copy
from functools import partial

import numpy as np

def subsample(game, num_samples):
	"""
	Returns a game with a random subset of the input game's payoff samples.

	Note: this is not intended for use on games with different numbers of
	samples per profile.
	"""
	sg = copy(game)
	sg.sample_values = map(lambda p: p[:,:,sample(range(p.shape[-1]), \
			num_samples)], sg.sample_values)
	sg.reset()
	sg.max_samples = num_samples
	return sg

def pre_aggregate(game, count):
	"""
	Aggregates samples to produce a game with |samples| / count samples.

	Note: this is not intended for use on games with different numbers of
	samples per profile.
	"""
	agg = copy(game)
	sv = agg.sample_values
	sv = np.swapaxes(sv, 0, -1)
	np.random.shuffle(sv)
	sv = np.swapaxes(sv, 0, -1)
	shape = list(sv.shape)
	samples = shape[-1]
	if samples % count != 0:
		sv = sv[:,:,:,:samples - (samples % count)]
	shape.append(count)
	shape[-2] /= count
	sv = np.reshape(sv, shape)
	sv = np.average(sv, axis=-1)
	agg.sample_values = sv
	agg.reset()
	agg.min_samples = agg.max_samples = shape[-2]
	return agg

def bootstrap(game, equilibrium, statistic=regret, method="resample", \
				method_args=[], points=1000):
	"""
	Returns a bootstrap distribution for the statistic.

	To run a resample regret boostrap:
		bootstrap(game, eq)
	To run a single-sample regret bootstrap:
		bootstrap(game, eq, method='single_sample')
	To run a replicator dynamics bootstrap:
		bootstrap(game, eq, replicator_dynamics)
	"""
	boot_dstr = []
	method = getattr(game, method)
	for __ in range(points):
		method(*method_args)
		boot_dstr.append(statistic(game, equilibrium))
	game.reset()
	boot_dstr.sort()
	return boot_dstr

def bootstrap_experiment(base_game_func, noise_model, statistic=regret, \
		num_games=1000, stdevs=[1.,10.,100.], sample_sizes=[5,10,20, \
		50,100,200,500], equilibrium_search=mixed_nash, bootstrap_args=[]):
	results = [{s:{} for s in stdevs} for i in range(num_games)]
	for i in range(num_games):
		base_game = base_game_func()
		RG.rescale_payoffs(base_game, 0, 100)
		for stdev in stdevs:
			sample_game = RG.add_noise(base_game, noise_model, stdev, \
									sample_sizes[-1])
			for sample_size in sample_sizes:
				subsample_game = subsample(sample_game, sample_size)
				equilibria = equilibrium_search(subsample_game)
				results[i][stdev][sample_size] = [ \
					{ \
						"profile" : eq,
						"statistic" : statistic(base_game, eq),
						"bootstrap" : bootstrap(subsample_game, eq, statistic, \
												*bootstrap_args)
					} for eq in equilibria]
	return results

def parse_args():
	parser = ArgumentParser()
	parser.add_argument("game_func", type=str, default="", choices=\
						["","congestion","local_effect","uniform_symmetric", \
						"sym_2p2s"], help="Specifies the function generating "+\
						"random base games. If empty, the script will look "+\
						"for a file with simulated game data on stdin.")
	parser.add_argument("noise_func", type=str, default="", choices=["",\
						"normal","unimodal","bimodal","u80b20","u60b40",\
						"u40b60","u20b80","gaussian_mixture"], help=\
						"Noise model to perturb sample payoffs around the "+\
						"base game payoff. May only be empty if first "+\
						"argument is also empty.")
	parser.add_argument("-game_args", type=str, nargs="*", default=[], help=\
						"Arguments to pass to game_func. Usually players "+\
						"and strategies.")
	parser.add_argument("-noise_args", type=str, nargs="*", default=[], help=\
						"Arguments to pass to noise_func. Should not include"+\
						"noise magnitude or number of samples.")
	parser.add_argument("-num_games", type=int, default=1000, help="Number "+\
						"of games to generate per stdev/subsample "+\
						"combination. Default: 1000")
	parser.add_argument("-points", type=int, default=1000, help="Number of "+\
						"bootstrap resamples per equilibrium. Default: 1000")
	parser.add_argument("-pair", type=str, default="game", choices=["game", \
						"profile", "payoff"], help="Pairing level to use for "+\
						"bootstrap resampling. Default: game")
	parser.add_argument("-agg", type=int, default=0, help="Number of samples "+\
						"to pre-aggregate. Default: 0")
	parser.add_argument("-stdevs", type=float, nargs="*", default=\
						[1.,10.,100.], help="Noise magnitude parameters "+\
						"passed to the noise model. Default: 1,10,100")
	parser.add_argument("-sample_sizes", type=int, nargs="*", default=\
						[5,10,20,100,200,500], help="Numbers of samples "+\
						"per profile at which to test the bootstrap. "+\
						"Default: 5 10 20 100 200 500")
	parser.add_argument("--single", action="store_true", help="Set to use "+\
						"the single_sample_regret function.")
	parser.add_argument("--rd", action="store_true", help="Set to compute "+\
						"bootstrap distributions of equilibrium movement by "+\
						"replicator dynamics.")
	parser.add_argument("--pure", action="store_true", help="Find an compute "+\
						"bootstrap regret distributions for pure-strategy "+\
						"(rather than role-symmetric mixed-strategy) Nash "+\
						"equilibria.")
	args = parser.parse_args()

	if args.game_func == "none":
		game = read(stdin)
		args.game_func = lambda: game
		args.noise_func = lambda g,s,c: game.subsample(c)
	else:
		assert args.noise_func != "none", "Must specify a noise model."
		game_args = []
		for a in args.game_args:
			try:
				game_args.append(int(a))
			except ValueError:
				game_args.append(float(a))
		noise_args = []
		for a in args.noise_args:
			try:
				noise_args.append(int(a))
			except ValueError:
				noise_args.append(float(a))
		args.game_func = partial(getattr(RG, args.game_func), *game_args)
		noise_func = getattr(RG, args.noise_func + "_noise")
		args.noise_func = lambda s,c: noise_func(s,c, *noise_args)

	if args.agg > 0:
		args.noise_func = lambda s,c: pre_aggregate(args.noise_func(s,c), \
											args.agg)
	assert not (args.rd and args.pure), "Must use mixed_nash for rd bootstrap"
	args.bootstrap_args = ["resample" if not args.single else "singleSample", \
						[args.pair] if not args.single else [], args.points]
	return args

def main():
	args = parse_args()
	results = bootstrap_experiment(args.game_func, args.noise_func, \
					replicator_dynamics if args.rd else regret, \
					args.num_games, args.stdevs, args.sample_sizes, \
					pure_nash if args.pure else mixed_nash, args.bootstrap_args)
	print to_JSON_str(results, indent=None)

if __name__ == "__main__":
	main()

<|endoftext|>


<commit_before>import rethinkdb as r
from tornado import ioloop, gen

r.set_loop_type("tornado")

@gen.coroutine
def print_changes(table):
    conn = yield r.connect(host="localhost", port=28015)
    feed = yield r.db('public').table(table).changes().run(conn)
    while (yield feed.fetch_next()):
        change = yield feed.next()
        print(change)

@gen.coroutine
def main():
    conn = yield r.connect(host="localhost", port=28015)
    for table in r.db('public').table_list().run(conn):
        ioloop.IOLoop.current().add_callback(print_changes, table)

if __name__ == "__main__":
    IOLoop.current().run_sync(main)
    IOLoop.current().start()<commit_msg>auto commit<commit_after>import rethinkdb as r
from tornado import ioloop, gen

r.set_loop_type("tornado")

@gen.coroutine
def print_changes(table):
    conn = yield r.connect(host="localhost", port=28015)
    feed = yield r.db('public').table(table).changes().run(conn)
    while (yield feed.fetch_next()):
        change = yield feed.next()
        print(change)

@gen.coroutine
def main():
    conn = yield r.connect(host="localhost", port=28015)
    for table in r.db('public').table_list().run(conn):
        ioloop.IOLoop.current().add_callback(print_changes, table)

if __name__ == "__main__":
    ioloop.IOLoop.current().run_sync(main)
    IOLoop.current().start()<|endoftext|>


<commit_before>import docutils

class SectNumFolder(docutils.nodes.SparseNodeVisitor):
    def __init__(self, document):
        docutils.nodes.SparseNodeVisitor.__init__(self, document)
        self.sectnums = {}

    def visit_generated(self, node):
        for i in node.parent.parent['ids']:
            self.sectnums[i]=node.parent.astext().replace(u'\xa0\xa0\xa0',' ')

class SectRefExpander(docutils.nodes.SparseNodeVisitor):
    def __init__(self, document, sectnums):
        docutils.nodes.SparseNodeVisitor.__init__(self, document)
        self.sectnums = sectnums

    def visit_reference(self, node):
        if node.get('refid', None) in self.sectnums:
            node.children=[docutils.nodes.Text('%s '%self.sectnums[node.get('refid')])]
<commit_msg>whitespace fixes<commit_after>import docutils

class SectNumFolder(docutils.nodes.SparseNodeVisitor):
    def __init__(self, document):
        docutils.nodes.SparseNodeVisitor.__init__(self, document)
        self.sectnums = {}

    def visit_generated(self, node):
        for i in node.parent.parent['ids']:
            self.sectnums[i] = node.parent.astext().replace(u'\xa0\xa0\xa0', ' ')

class SectRefExpander(docutils.nodes.SparseNodeVisitor):
    def __init__(self, document, sectnums):
        docutils.nodes.SparseNodeVisitor.__init__(self, document)
        self.sectnums = sectnums

    def visit_reference(self, node):
        if node.get('refid', None) in self.sectnums:
            node.children = [docutils.nodes.Text('%s ' % self.sectnums[node.get('refid')])]
<|endoftext|>


<commit_before>"""Sundara jāla: for a beautiful web.
Convert markdown to clean HTML5.
"""

import os
import sys
import shutil
import re
from markdown import Markdown
from bs4 import BeautifulSoup
import pygit2

DOCTYPE="<!DOCTYPE html>\n"

class Jala():
    """Convert Markdown to clean, properly formatted HTML5."""
    def __init__(self, sundara):
        self.sundara = sundara
        self.config = self.sundara.config
        self.css_path = self.config.get('sundara', 'css')
        self.js_path = self.config.get('sundara', 'js')

    def get_stylesheets(self):
        repo = pygit2.Repository(self.sundara.dir)
        return [ f.path[len(self.css_path):] for f in repo.index if (
            f.path.endswith('.css') and f.path.startswith(self.css_path)) ]

    def get_javascript(self):
        repo = pygit2.Repository(self.sundara.dir)
        return [ f.path[len(self.js_path):] for f in repo.index if (
            f.path.endswith('.js') and f.path.startswith(self.js_path)) ]

    def get_header(self):
        header = self.sundara.header + self.sundara.md_ext
        if header in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, header), "r") as md:
                return md.read()
        else:
            return str()

    def get_footer(self):
        footer = self.sundara.footer + self.sundara.md_ext
        if footer in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, footer), "r") as md:
                return md.read()
        else:
            return str()

    def get_nav(self):
        nav = self.sundara.nav + self.sundara.md_ext
        if nav in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, nav), "r") as md:
                return md.read()
        else:
            return str()

    def convert(self, md, homepage=False):
        markdown = Markdown(output_format="html5")

        # Form the HTML document.
        main = BeautifulSoup().new_tag('div',
            id=self.config.get('content', 'main'))
        # Header
        header = self.get_header()
        if header != '':
            htag1 = BeautifulSoup().new_tag('header',
                    role=self.config.get('content', 'header_role'))
            htag1['class'] = self.config.get('content', 'header')
            main.append(htag1)
            htag2 = BeautifulSoup().new_tag('div')
            htag2['class'] = self.config.get('content', 'header_content')
            main.header.append(htag2)
            bh = BeautifulSoup(markdown.convert(header), "html5lib")
            bh.html.unwrap()
            bh.body.unwrap()
            bh.head.extract()
            main.header.div.append(bh)
        # Nav
        nav = self.get_nav()
        if nav != '':
            ntag = BeautifulSoup().new_tag('nav',
                    role=self.config.get('content', 'nav_role'))
            ntag['class'] = self.config.get('content', 'nav')
            main.append(ntag)
            bn = BeautifulSoup(markdown.convert(nav), "html5lib")
            bn.html.unwrap()
            bn.body.unwrap()
            bn.head.extract()
            main.nav.append(bn)
        # Content
        bc = BeautifulSoup(markdown.convert(md), "html5lib")
        bc.html.unwrap()
        ctag1 = BeautifulSoup().new_tag('div')
        ctag1['class'] = self.config.get('content', 'container')
        bc.body.wrap(ctag1)
        ctag2 = BeautifulSoup().new_tag('div')
        ctag2['class'] = self.config.get('content', 'row')
        bc.body.wrap(ctag2)
        ctag3 = BeautifulSoup().new_tag('div')
        ctag3['class'] = self.config.get('content', 'content')
        bc.body.wrap(ctag3)
        bc.body.unwrap()
        bc.head.extract()
        main.append(bc)
        # Footer
        footer = self.get_footer()
        if footer != '':
            ftag1 = BeautifulSoup().new_tag('footer')
            ftag1['class'] = self.config.get('content', 'footer')
            main.append(ftag1)
            ftag2 = BeautifulSoup().new_tag('div')
            ftag2['class'] = self.config.get('content', 'footer_content')
            main.footer.append(ftag2)
            bf = BeautifulSoup(markdown.convert(footer), "html5lib")
            bf.html.unwrap()
            bf.body.unwrap()
            main.footer.div.append(bf)
            bf.head.extract()
        # Throw it all in the soup!
        soup = BeautifulSoup(DOCTYPE, "html5lib")
        soup.body.append(main)

        # Add meta information.
        soup.html['lang'] = self.config.get('meta', 'lang')
        soup.head.append(BeautifulSoup().new_tag('meta',
            charset=self.config.get('meta', 'encoding')))
        soup.head.append(BeautifulSoup().new_tag('title'))
        # Parse out the title formatter.
        site_name = self.config.get('meta', 'name')
        if homepage:
            soup.title.string = site_name
        else:
            soup.title.string = self.config.get('meta',
                    'title').replace('{name}', site_name).replace('{h1}',
                            soup.h1.string)
        # If description exists, add it to the homepage ONLY.
        if homepage and self.config.get('meta', 'description') != '':
                desc = BeautifulSoup().new_tag('meta',
                        content=self.config.get('meta', 'description'))
                desc['name'] = 'description'
                soup.head.append(desc)

        # Link in Bootstrap and jQuery.
        if self.config.get('style', 'bootstrap') == 'on':
            soup.head.append(BeautifulSoup().new_tag('link', rel='stylesheet',
                href='//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css'))
            soup.body.append(BeautifulSoup().new_tag('script',
                src='//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js'))
        if self.config.get('style', 'jquery') == 'on':
            soup.body.append(BeautifulSoup().new_tag('script',
                src='//code.jquery.com/jquery-1.11.0.min.js'))

        # Add custom CSS and JS.
        stylesheets = self.get_stylesheets()
        for stylesheet in stylesheets:
            # Link the file.
            soup.head.append(BeautifulSoup().new_tag('link', rel='stylesheet',
                href=os.path.join(self.config.get('style', 'css'),
                    stylesheet)))
            # Install the file.
            shutil.copy(os.path.join(self.sundara.dir, self.css_path,
                stylesheet), os.path.join(self.sundara.generate_path,
                    self.config.get('style', 'css'), stylesheet))
        scripts = self.get_javascript()
        for script in scripts:
            # Link the file.
            soup.body.append(BeautifulSoup().new_tag('script',
                src=os.path.join(self.config.get('style', 'js'),
                    script)))
            # Install the file.
            shutil.copy(os.path.join(self.sundara.dir, self.js_path,
                script), os.path.join(self.sundara.generate_path,
                        self.config.get('style', 'js'), script))

        # Prettify and split.
        html = soup.prettify().split('\n')

        # Fix indentation for non-tag lines.
        for index, line in enumerate(html):
            if index > 0 and re.match("\s*[^<\s]", line) and re.match("\s+[^<\s]", html[index - 1]):
                # if line isn't a tag and previous line is indented and not a tag
                html[index] = re.match("\s*", html[index - 1]).group() + line

        return BeautifulSoup('\n'.join(html)).prettify()
<commit_msg>Fixed Jala bugs.<commit_after>"""Sundara jāla: for a beautiful web.
Convert markdown to clean HTML5.
"""

import os
import sys
import shutil
import re
from markdown import Markdown
from bs4 import BeautifulSoup
import pygit2

DOCTYPE="<!DOCTYPE html>\n"

class Jala():
    """Convert Markdown to clean, properly formatted HTML5."""
    def __init__(self, sundara):
        self.sundara = sundara
        self.config = self.sundara.config
        self.css_path = self.config.get('sundara', 'css')
        self.js_path = self.config.get('sundara', 'js')
        self.style_css_path = self.config.get('style', 'css')
        self.style_js_path = self.config.get('style', 'js')
        if os.path.isabs(self.style_css_path):
            self.style_css = self.style_css_path[1:]
        else:
            self.style_css = self.style_css_path
        if os.path.isabs(self.style_js_path):
            self.style_js = self.style_js_path[1:]
        else:
            self.style_js = self.style_js_path

    def get_stylesheets(self):
        repo = pygit2.Repository(self.sundara.dir)
        return [ f.path[len(self.css_path):] for f in repo.index if (
            f.path.endswith('.css') and f.path.startswith(self.css_path)) ]

    def get_javascript(self):
        repo = pygit2.Repository(self.sundara.dir)
        return [ f.path[len(self.js_path):] for f in repo.index if (
            f.path.endswith('.js') and f.path.startswith(self.js_path)) ]

    def get_header(self):
        header = self.sundara.header + self.sundara.md_ext
        if header in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, header), "r") as md:
                return md.read()
        else:
            return str()

    def get_footer(self):
        footer = self.sundara.footer + self.sundara.md_ext
        if footer in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, footer), "r") as md:
                return md.read()
        else:
            return str()

    def get_nav(self):
        nav = self.sundara.nav + self.sundara.md_ext
        if nav in self.sundara.get_files():
            with open(os.path.join(self.sundara.md_path, nav), "r") as md:
                return md.read()
        else:
            return str()

    def convert(self, md, homepage=False):
        markdown = Markdown(output_format="html5")

        # Form the HTML document.
        main = BeautifulSoup().new_tag('div',
            id=self.config.get('content', 'main'))
        # Header
        header = self.get_header()
        if header != '':
            htag1 = BeautifulSoup().new_tag('header',
                    role=self.config.get('content', 'header_role'))
            htag1['class'] = self.config.get('content', 'header')
            main.append(htag1)
            htag2 = BeautifulSoup().new_tag('div')
            htag2['class'] = self.config.get('content', 'header_content')
            main.header.append(htag2)
            bh = BeautifulSoup(markdown.convert(header), "html5lib")
            bh.html.unwrap()
            bh.body.unwrap()
            bh.head.extract()
            main.header.div.append(bh)
        # Nav
        nav = self.get_nav()
        if nav != '':
            ntag = BeautifulSoup().new_tag('nav',
                    role=self.config.get('content', 'nav_role'))
            ntag['class'] = self.config.get('content', 'nav')
            main.append(ntag)
            bn = BeautifulSoup(markdown.convert(nav), "html5lib")
            bn.html.unwrap()
            bn.body.unwrap()
            bn.head.extract()
            main.nav.append(bn)
        # Content
        bc = BeautifulSoup(markdown.convert(md), "html5lib")
        bc.html.unwrap()
        ctag1 = BeautifulSoup().new_tag('div')
        ctag1['class'] = self.config.get('content', 'container')
        bc.body.wrap(ctag1)
        ctag2 = BeautifulSoup().new_tag('div')
        ctag2['class'] = self.config.get('content', 'row')
        bc.body.wrap(ctag2)
        ctag3 = BeautifulSoup().new_tag('div')
        ctag3['class'] = self.config.get('content', 'content')
        bc.body.wrap(ctag3)
        bc.body.unwrap()
        bc.head.extract()
        main.append(bc)
        # Footer
        footer = self.get_footer()
        if footer != '':
            ftag1 = BeautifulSoup().new_tag('footer')
            ftag1['class'] = self.config.get('content', 'footer')
            main.append(ftag1)
            ftag2 = BeautifulSoup().new_tag('div')
            ftag2['class'] = self.config.get('content', 'footer_content')
            main.footer.append(ftag2)
            bf = BeautifulSoup(markdown.convert(footer), "html5lib")
            bf.html.unwrap()
            bf.body.unwrap()
            main.footer.div.append(bf)
            bf.head.extract()
        # Throw it all in the soup!
        soup = BeautifulSoup(DOCTYPE, "html5lib")
        soup.body.append(main)

        # Add meta information.
        soup.html['lang'] = self.config.get('meta', 'lang')
        soup.head.append(BeautifulSoup().new_tag('meta',
            charset=self.config.get('meta', 'encoding')))
        soup.head.append(BeautifulSoup().new_tag('title'))
        # Parse out the title formatter.
        site_name = self.config.get('meta', 'name')
        if homepage:
            soup.title.string = site_name
        else:
            soup.title.string = self.config.get('meta',
                    'title').replace('{name}', site_name).replace('{h1}',
                            soup.h1.string)
        # If description exists, add it to the homepage ONLY.
        if homepage and self.config.get('meta', 'description') != '':
                desc = BeautifulSoup().new_tag('meta',
                        content=self.config.get('meta', 'description'))
                desc['name'] = 'description'
                soup.head.append(desc)

        # Link in Bootstrap and jQuery.
        if self.config.get('style', 'bootstrap') == 'on':
            soup.head.append(BeautifulSoup().new_tag('link', rel='stylesheet',
                href='//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css'))
            soup.body.append(BeautifulSoup().new_tag('script',
                src='//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js'))
        if self.config.get('style', 'jquery') == 'on':
            soup.body.append(BeautifulSoup().new_tag('script',
                src='//code.jquery.com/jquery-1.11.0.min.js'))

        # Add custom CSS and JS.
        stylesheets = self.get_stylesheets()
        for stylesheet in stylesheets:
            # Link the file.
            soup.head.append(BeautifulSoup().new_tag('link', rel='stylesheet',
                href=os.path.join(self.style_css_path, stylesheet)))
            # Install the file.
            os.makedirs(os.path.join(self.sundara.generate_path,
                self.css_path))
            shutil.copy(os.path.join(self.sundara.dir, self.css_path,
                stylesheet), os.path.join(self.sundara.generate_path,
                    self.style_css, stylesheet))
        scripts = self.get_javascript()
        for script in scripts:
            # Link the file.
            soup.body.append(BeautifulSoup().new_tag('script',
                src=os.path.join(self.style_js_path, script)))
            # Install the file.
            os.makedirs(os.path.join(self.sundara.generate_path,
                self.js_path))
            shutil.copy(os.path.join(self.sundara.dir, self.js_path,
                script), os.path.join(self.sundara.generate_path,
                        self.style_js, script))

        # Prettify and split.
        html = soup.prettify().split('\n')

        # Fix indentation for non-tag lines.
        for index, line in enumerate(html):
            if index > 0 and re.match("\s*[^<\s]", line) and re.match("\s+[^<\s]", html[index - 1]):
                # if line isn't a tag and previous line is indented and not a tag
                html[index] = re.match("\s*", html[index - 1]).group() + line

        return BeautifulSoup('\n'.join(html)).prettify()
<|endoftext|>