<commit_before>##############################################################################
#
# Copyright (c) 2004 TINY SPRL. (http://tiny.be)
#
# $Id: __init__.py 1308 2005-09-08 18:02:01Z pinky $
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contact a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler

import netsvc
from osv import fields

logger = netsvc.Logger()

opj = os.path.join
ad = tools.config['addons_path']
sys.path.insert(1,ad)

class Graph(dict):

	def addNode(self, name, deps):
		max_depth, father = 0, None
		for n in [Node(x, self) for x in deps]:
			if n.depth >= max_depth:
				father = n
				max_depth = n.depth
		if father:
			father.addChild(name)
		else:
			Node(name, self)

	def __iter__(self):
		level = 0
		done = Set(self.keys())
		while done:
			level_modules = [(name, module) for name, module in self.items() if module.depth==level]
			for name, module in level_modules:
				done.remove(name)
				yield module
			level += 1

class Singleton(object):

	def __new__(cls, name, graph):
		if name in graph:
			inst = graph[name]
		else:
			inst = object.__new__(cls)
			inst.name = name
			graph[name] = inst
		return inst

class Node(Singleton):

	def __init__(self, name, graph):
		self.graph = graph
		if not hasattr(self, 'childs'):
			self.childs = []
		if not hasattr(self, 'depth'):
			self.depth = 0

	def addChild(self, name):
		node = Node(name, self.graph)
		node.depth = self.depth + 1
		if node not in self.childs:
			self.childs.append(node)
		for attr in ('init', 'update', 'demo'):
			if hasattr(self, attr):
				setattr(node, attr, True)
		self.childs.sort(lambda x,y: cmp(x.name, y.name))

	def hasChild(self, name):
		return Node(name, self.graph) in self.childs or \
				bool([c for c in self.childs if c.hasChild(name)])

	def __setattr__(self, name, value):
		super(Singleton, self).__setattr__(name, value)
		if name in ('init', 'update', 'demo'):
			tools.config[name][self.name] = 1
			for child in self.childs:
				setattr(child, name, value)
		if name == 'depth':
			for child in self.childs:
				setattr(child, name, value + 1)

	def __iter__(self):
		return itertools.chain(iter(self.childs), *map(iter, self.childs))

	def __str__(self):
		return self._pprint()

	def _pprint(self, depth=0):
		s = '%s\n' % self.name
		for c in self.childs:
			s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
		return s

def create_graph(module_list, force=[]):
	graph = Graph()
	packages = []

	for module in module_list:
		terp_file = opj(ad, module, '__terp__.py')
		if os.path.isfile(terp_file):
			info = eval(file(terp_file).read())
			if info.get('installable', True):
				packages.append((module, info.get('depends', []), info))

	current,later = Set([p for p, dep, data in packages]), Set()
	while packages and current > later:
		package, deps, datas = packages[0]

		# if all dependencies of 'package' are already in the graph, add 'package' in the graph
		if reduce(lambda x,y: x and y in graph, deps, True):
			later.clear()
			current.remove(package)
			graph.addNode(package, deps)
			node = Node(package, graph)
			node.datas = datas
			for kind in ('init', 'demo', 'update'):
				if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
					setattr(node, kind, True)
		else:
			later.add(package)
			packages.append((package, deps, datas))
		packages = packages[1:]

	for package in later:
		logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

	return graph

def init_module_objects(cr, module_name, obj_list):
	pool = pooler.get_pool(cr.dbname)
	logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
	for obj in obj_list:
		#CHECKME: is this test useful? all objects are supposed to have an _auto_init method, right?
		#if hasattr(obj, '_auto_init'):
		if hasattr(obj, 'init'):
			obj.init(cr)
		obj._auto_init(cr)
		cr.commit()

def load_module_graph(cr, graph, status={}):
	package_todo = []
	statusi = 0
	for package in graph:
		status['progress'] = (float(statusi)+0.1)/len(graph)
		m = package.name
		logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
		sys.stdout.flush()
		pool = pooler.get_pool(cr.dbname)
		modules = pool.instanciate(m)
		cr.execute('select state, demo from ir_module_module where name=%s', (m,))
		(package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
		idref = {}
		status['progress'] = (float(statusi)+0.4)/len(graph)
		if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
			init_module_objects(cr, m, modules)
			for kind in ('init', 'update'):
				for filename in package.datas.get('%s_xml' % kind, []):
					mode = 'update'
					if hasattr(package, 'init') or package_state=='to install':
						mode = 'init'
					logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
					name, ext = os.path.splitext(filename)
					if ext == '.csv':
						tools.convert_csv_import(cr, m, filename, tools.file_open(opj(m, filename)).read(), idref, mode=mode)
					elif ext == '.sql':
						queries = tools.file_open(opj(m, filename)).read().split(';')
						for query in queries:
							new_query = ' '.join(query.split())
							if new_query:
								cr.execute(new_query)
						cr.commit()
					else:
						tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)).read(), idref, mode=mode)
			if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
				status['progress'] = (float(statusi)+0.75)/len(graph)
				for xml in package.datas.get('demo_xml', []):
					name, ext = os.path.splitext(xml)
					logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
					if ext == '.csv':
						tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref)
					else:
						tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)).read(), idref)
			package_todo.append(package.name)
			cr.execute("update ir_module_module set state='installed', demo=%s where state in ('to upgrade', 'to install') and name=%s", (str(hasattr(package, 'demo')) or package_demo, package.name))
			cr.commit()
		statusi+=1
	cr.commit()

#	pool = osv.osv.FakePool('base')
#	pool = pooler.get_pool(cr.dbname)
#	pool.get('ir.model.data')._process_end(cr, 1, package_todo)
#	cr.commit()

def register_classes():
	module_list = os.listdir(ad)
	for package in create_graph(module_list):
		m = package.name
		logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
		sys.stdout.flush()
		imp.load_module(m, *imp.find_module(m))

def load_modules(db, force_demo=False, status={}, update_module=False):
	cr = db.cursor()
	force = []
	if force_demo:
		force.append('demo')
	if update_module:
		cr.execute("select name from ir_module_module where state in ('installed', 'to install', 'to upgrade','to remove')")
	else:
		cr.execute("select name from ir_module_module where state in ('installed', 'to upgrade', 'to remove')")
	module_list = [name for (name,) in cr.fetchall()]
	graph = create_graph(module_list, force)
	load_module_graph(cr, graph, status)

	for kind in ('init', 'demo', 'update'):
		tools.config[kind]={}

	cr.commit()
	if update_module:
		cr.execute("select id,name from ir_module_module where state in ('to remove')")
		for mod_id, mod_name in cr.fetchall():
			pool = pooler.get_pool(cr.dbname)
			cr.execute('select model,res_id from ir_model_data where not noupdate and module=%s order by id desc', (mod_name,))
			for rmod,rid in cr.fetchall():
				#
				# TO BE Improved:
				#   I can not use the class_pool has _table could be defined in __init__
				#   and I can not use the pool has the module could not be loaded in the pool
				#
				uid = 1
				print rmod
				pool.get(rmod).unlink(cr, uid, [rid])
			cr.commit()
		#
		# TODO: remove menu without actions of childs
		#
		cr.execute('''delete from
				ir_ui_menu
			where
				(id not in (select parent_id from ir_ui_menu where parent_id is not null))
			and
				(id not in (select res_id from ir_values where model='ir.ui.menu'))''')

		cr.execute("update ir_module_module set state=%s where state in ('to remove')", ('uninstalled', ))
		cr.commit()
		pooler.restart_pool(cr.dbname)
	cr.close()

<commit_msg>Fix use always basename for cvs file<commit_after>##############################################################################
#
# Copyright (c) 2004 TINY SPRL. (http://tiny.be)
#
# $Id: __init__.py 1308 2005-09-08 18:02:01Z pinky $
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contact a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler

import netsvc
from osv import fields

logger = netsvc.Logger()

opj = os.path.join
ad = tools.config['addons_path']
sys.path.insert(1,ad)

class Graph(dict):

	def addNode(self, name, deps):
		max_depth, father = 0, None
		for n in [Node(x, self) for x in deps]:
			if n.depth >= max_depth:
				father = n
				max_depth = n.depth
		if father:
			father.addChild(name)
		else:
			Node(name, self)

	def __iter__(self):
		level = 0
		done = Set(self.keys())
		while done:
			level_modules = [(name, module) for name, module in self.items() if module.depth==level]
			for name, module in level_modules:
				done.remove(name)
				yield module
			level += 1

class Singleton(object):

	def __new__(cls, name, graph):
		if name in graph:
			inst = graph[name]
		else:
			inst = object.__new__(cls)
			inst.name = name
			graph[name] = inst
		return inst

class Node(Singleton):

	def __init__(self, name, graph):
		self.graph = graph
		if not hasattr(self, 'childs'):
			self.childs = []
		if not hasattr(self, 'depth'):
			self.depth = 0

	def addChild(self, name):
		node = Node(name, self.graph)
		node.depth = self.depth + 1
		if node not in self.childs:
			self.childs.append(node)
		for attr in ('init', 'update', 'demo'):
			if hasattr(self, attr):
				setattr(node, attr, True)
		self.childs.sort(lambda x,y: cmp(x.name, y.name))

	def hasChild(self, name):
		return Node(name, self.graph) in self.childs or \
				bool([c for c in self.childs if c.hasChild(name)])

	def __setattr__(self, name, value):
		super(Singleton, self).__setattr__(name, value)
		if name in ('init', 'update', 'demo'):
			tools.config[name][self.name] = 1
			for child in self.childs:
				setattr(child, name, value)
		if name == 'depth':
			for child in self.childs:
				setattr(child, name, value + 1)

	def __iter__(self):
		return itertools.chain(iter(self.childs), *map(iter, self.childs))

	def __str__(self):
		return self._pprint()

	def _pprint(self, depth=0):
		s = '%s\n' % self.name
		for c in self.childs:
			s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
		return s

def create_graph(module_list, force=[]):
	graph = Graph()
	packages = []

	for module in module_list:
		terp_file = opj(ad, module, '__terp__.py')
		if os.path.isfile(terp_file):
			info = eval(file(terp_file).read())
			if info.get('installable', True):
				packages.append((module, info.get('depends', []), info))

	current,later = Set([p for p, dep, data in packages]), Set()
	while packages and current > later:
		package, deps, datas = packages[0]

		# if all dependencies of 'package' are already in the graph, add 'package' in the graph
		if reduce(lambda x,y: x and y in graph, deps, True):
			later.clear()
			current.remove(package)
			graph.addNode(package, deps)
			node = Node(package, graph)
			node.datas = datas
			for kind in ('init', 'demo', 'update'):
				if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
					setattr(node, kind, True)
		else:
			later.add(package)
			packages.append((package, deps, datas))
		packages = packages[1:]

	for package in later:
		logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

	return graph

def init_module_objects(cr, module_name, obj_list):
	pool = pooler.get_pool(cr.dbname)
	logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
	for obj in obj_list:
		#CHECKME: is this test useful? all objects are supposed to have an _auto_init method, right?
		#if hasattr(obj, '_auto_init'):
		if hasattr(obj, 'init'):
			obj.init(cr)
		obj._auto_init(cr)
		cr.commit()

def load_module_graph(cr, graph, status={}):
	package_todo = []
	statusi = 0
	for package in graph:
		status['progress'] = (float(statusi)+0.1)/len(graph)
		m = package.name
		logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
		sys.stdout.flush()
		pool = pooler.get_pool(cr.dbname)
		modules = pool.instanciate(m)
		cr.execute('select state, demo from ir_module_module where name=%s', (m,))
		(package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
		idref = {}
		status['progress'] = (float(statusi)+0.4)/len(graph)
		if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
			init_module_objects(cr, m, modules)
			for kind in ('init', 'update'):
				for filename in package.datas.get('%s_xml' % kind, []):
					mode = 'update'
					if hasattr(package, 'init') or package_state=='to install':
						mode = 'init'
					logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
					name, ext = os.path.splitext(filename)
					if ext == '.csv':
						tools.convert_csv_import(cr, m, os.path.basename(filename), tools.file_open(opj(m, filename)).read(), idref, mode=mode)
					elif ext == '.sql':
						queries = tools.file_open(opj(m, filename)).read().split(';')
						for query in queries:
							new_query = ' '.join(query.split())
							if new_query:
								cr.execute(new_query)
						cr.commit()
					else:
						tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)).read(), idref, mode=mode)
			if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
				status['progress'] = (float(statusi)+0.75)/len(graph)
				for xml in package.datas.get('demo_xml', []):
					name, ext = os.path.splitext(xml)
					logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
					if ext == '.csv':
						tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref)
					else:
						tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)).read(), idref)
			package_todo.append(package.name)
			cr.execute("update ir_module_module set state='installed', demo=%s where state in ('to upgrade', 'to install') and name=%s", (str(hasattr(package, 'demo')) or package_demo, package.name))
			cr.commit()
		statusi+=1
	cr.commit()

#	pool = osv.osv.FakePool('base')
#	pool = pooler.get_pool(cr.dbname)
#	pool.get('ir.model.data')._process_end(cr, 1, package_todo)
#	cr.commit()

def register_classes():
	module_list = os.listdir(ad)
	for package in create_graph(module_list):
		m = package.name
		logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
		sys.stdout.flush()
		imp.load_module(m, *imp.find_module(m))

def load_modules(db, force_demo=False, status={}, update_module=False):
	cr = db.cursor()
	force = []
	if force_demo:
		force.append('demo')
	if update_module:
		cr.execute("select name from ir_module_module where state in ('installed', 'to install', 'to upgrade','to remove')")
	else:
		cr.execute("select name from ir_module_module where state in ('installed', 'to upgrade', 'to remove')")
	module_list = [name for (name,) in cr.fetchall()]
	graph = create_graph(module_list, force)
	load_module_graph(cr, graph, status)

	for kind in ('init', 'demo', 'update'):
		tools.config[kind]={}

	cr.commit()
	if update_module:
		cr.execute("select id,name from ir_module_module where state in ('to remove')")
		for mod_id, mod_name in cr.fetchall():
			pool = pooler.get_pool(cr.dbname)
			cr.execute('select model,res_id from ir_model_data where not noupdate and module=%s order by id desc', (mod_name,))
			for rmod,rid in cr.fetchall():
				#
				# TO BE Improved:
				#   I can not use the class_pool has _table could be defined in __init__
				#   and I can not use the pool has the module could not be loaded in the pool
				#
				uid = 1
				print rmod
				pool.get(rmod).unlink(cr, uid, [rid])
			cr.commit()
		#
		# TODO: remove menu without actions of childs
		#
		cr.execute('''delete from
				ir_ui_menu
			where
				(id not in (select parent_id from ir_ui_menu where parent_id is not null))
			and
				(id not in (select res_id from ir_values where model='ir.ui.menu'))''')

		cr.execute("update ir_module_module set state=%s where state in ('to remove')", ('uninstalled', ))
		cr.commit()
		pooler.restart_pool(cr.dbname)
	cr.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, re, csv

ANON = {
  "@creditmutuel;2017-12-29": "Don de S.F.",
  "@creditmutuel;2019-01-02": "Don de S.F."
}

re_not = re.compile(r'Not (available|loaded)')
re_time = re.compile(r' [0-2][0-9]:[0-5][0-9]:[0-5][0-9]$')
re_ano = re.compile(ur'(Don( récurrent)? de )(\w)\S+( +(\w).*)?$', re.I)
sub_ano = lambda x: x.group(1) + (x.group(3) + "." + (x.group(5) or "") + ".").upper()
ano = lambda x: re_ano.sub(sub_ano, x)
re_creditmut = re.compile(r'(SEPA ONLINE SAS) ONL (\d+)( DEDIBOX \2)')
def process_data(data):
    writer = csv.writer(sys.stdout)
    for line in data:
      try:
        if not line:
            continue
        if line[0] == 'id':
            writer.writerow(line)
            continue
        check = "%s;%s" % (line[0], line[2])
        if line[8] != u"Not available":
            line[9] = " ".join(line[7:9])
        for i, el in enumerate(line):
            line[i] = re_not.sub('', line[i].decode('utf-8'))
            line[i] = line[i].replace(u"Cliquer pour déplier ou plier le détail de l'opération ", "")
            line[i] = line[i].replace(u'Paiement récurrent de ', u'Don récurrent de ')
            line[i] = line[i].replace(u'Paiement de ', u'Don de ')
            line[i] = ano(line[i])
            line[i] = re_creditmut.sub(r'\1\3', line[i])
        if check in ANON:
            line[6] = ANON[check]
            line[9] = ANON[check]
        line[0] = re.sub(r'^[0-9A-Z]*@', '', line[0])
        line[2] = re_time.sub('', line[2])
        line[3] = re_time.sub('', line[3])
        if line[12] and line[12] not in ['commission', '0.00']:
            line[9] += (u' (€%.2f)' % (float(line[10]) - float(line[12]))).replace('.', ',')
        # Only keep lines with definitive date to avoid duplicates across multiple dates
        if len(line[2]) == 10:
            writer.writerow([el.encode('utf-8') for el in line])
      except Exception as e:
        print >> sys.stderr, "ERROR on line %s" % line
        raise e

if __name__ == "__main__":
    with open(sys.argv[1]) as f:
        process_data(csv.reader(f, delimiter=";"))
<commit_msg>more anon options<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys, re, csv

ANON = {
  "@creditmutuel;2017-12-29": "Don de S.F.",
  "@creditmutuel;2019-01-02": "Don de S.F."
}
ANON2 = {
  "M. R[A-Z]+ V[A-Z]+ \d+": "Don de V.R."
}

re_not = re.compile(r'Not (available|loaded)')
re_time = re.compile(r' [0-2][0-9]:[0-5][0-9]:[0-5][0-9]$')
re_ano = re.compile(ur'(Don( récurrent)? de )(\w)\S+( +(\w).*)?$', re.I)
sub_ano = lambda x: x.group(1) + (x.group(3) + "." + (x.group(5) or "") + ".").upper()
ano = lambda x: re_ano.sub(sub_ano, x)
re_creditmut = re.compile(r'(SEPA ONLINE SAS) ONL (\d+)( DEDIBOX \2)')
def process_data(data):
    writer = csv.writer(sys.stdout)
    for line in data:
      try:
        if not line:
            continue
        if line[0] == 'id':
            writer.writerow(line)
            continue
        check = "%s;%s" % (line[0], line[2])
        if line[8] != u"Not available":
            line[9] = " ".join(line[7:9])
        for i, el in enumerate(line):
            line[i] = re_not.sub('', line[i].decode('utf-8'))
            line[i] = line[i].replace(u"Cliquer pour déplier ou plier le détail de l'opération ", "")
            line[i] = line[i].replace(u'Paiement récurrent de ', u'Don récurrent de ')
            line[i] = line[i].replace(u'Paiement de ', u'Don de ')
            line[i] = ano(line[i])
            line[i] = re_creditmut.sub(r'\1\3', line[i])
        if check in ANON:
            line[6] = ANON[check]
            line[9] = ANON[check]
        for check, res in ANON2:
            if re.match(check, line[9]):
                line[6] = res
                line[9] = res
        line[0] = re.sub(r'^[0-9A-Z]*@', '', line[0])
        line[2] = re_time.sub('', line[2])
        line[3] = re_time.sub('', line[3])
        if line[12] and line[12] not in ['commission', '0.00']:
            line[9] += (u' (€%.2f)' % (float(line[10]) - float(line[12]))).replace('.', ',')
        # Only keep lines with definitive date to avoid duplicates across multiple dates
        if len(line[2]) == 10:
            writer.writerow([el.encode('utf-8') for el in line])
      except Exception as e:
        print >> sys.stderr, "ERROR on line %s" % line
        raise e

if __name__ == "__main__":
    with open(sys.argv[1]) as f:
        process_data(csv.reader(f, delimiter=";"))
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import json
import os
import pycurl
import sys
from StringIO import StringIO

import redis
from jinja2 import Template

r_server = redis.StrictRedis('127.0.0.1', db=2)
check = r_server.get("need_CSR")
if check is None:
    print("Need to run registration job.")
    sys.exit(1)

if len(sys.argv) > 1:
    need_haproxy = sys.argv[1]
else:
    need_haproxy = 'no'

# get hostname
try:
    buffer = StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, 'http://169.254.169.253/latest/meta-data/public-hostname')
    c.setopt(pycurl.CONNECTTIMEOUT, 5)
    c.setopt(c.WRITEDATA, buffer)
    c.perform()
    c.close()
    hostname = buffer.getvalue()
except:
    hostname = "localhost"

i_key = "owner-info"
json_data = r_server.get(i_key)
if json_data is not None:
    data = json.loads(json_data)
    email = data['Email']
    hostname = data['Hostname']
frontend_conf = ""
backend_conf = ""
acl = ""
redirect = ""

if check == "1":
    # get certificate
    if os.environ["ENV"] == 'AWS':
        os.system("mkdir -p /opt/certs/letsencrypt")
        os.system("cd /opt/certs && openssl req -inform pem -outform der -in server.csr -out ./letsencrypt/server.der")
        request = (
            "cd /opt/certs/letsencrypt && letsencrypt certonly --csr server.der --standalone --non-interactive --agree-tos --email %s --standalone-supported-challenges http-01" % email)
        os.system(request)
        cert_file = os.path.exists("/opt/certs/letsencrypt/0001_chain.pem")
        if not cert_file:
            sys.exit(2)
        os.system(" cd /opt/certs/letsencrypt && cat 0001_chain.pem ../server.key > ../server.bundle.pem")
        os.system("rm -rf /opt/certs/letsencrypt")
        r_server.set("need_CSR", "0")
        r_server.bgsave()
    else:
        print("Certificate is generated only for AWS env. current env is: %s" % os.environ["ENV"])
else:
    print("Don't need new certificate")

app_key = "apps"
data_apps = r_server.get(app_key)
os.system("rm -rf /opt/haproxy/haproxy.cfg")
config_template = open('/opt/madcore/bin/templates/haproxy.cfg').read()
if os.environ["ENV"] == 'AWS':
    crt_path = "/opt/certs/server.bundle.pem"
else:
    crt_path = "/etc/pki/tls/certs/server.bundle.pem"
if data_apps:
    apps = json.loads(data_apps)
    for app in apps:
        ### get service ip
        service_ip = os.system("kubectl get svc --all-namespaces | grep %s | grep %s | awk '{print $3}'" % (app["namespace"], app["service_name"]))
        if service_ip == "":
            service_ip = "127.0.0.1"
        i = "use_backend %s if { hdr_end(host) -i %s }\n    " % (
            app["name"], app["name"] + "." + data['Hostname'])
        frontend_conf += i
        i = ("backend %s\n    balance roundrobin\n    server %s %s:%s check\n  " % (
            app["name"], app["name"],service_ip, app["port"]))
        backend_conf += i
    template = Template(config_template)
    config = (
        template.render(hostname=hostname, crt_path=crt_path, subdomain=frontend_conf, backend=backend_conf)
else:
    template = Template(config_template)
    config = (template.render(hostname=hostname, crt_path=crt_path))
open("/opt/haproxy/haproxy.cfg", "w").write(config)
os.system("haproxy -f /opt/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)")
<commit_msg>fix syntax<commit_after>#!/usr/bin/env python
from __future__ import print_function

import json
import os
import pycurl
import sys
from StringIO import StringIO

import redis
from jinja2 import Template

r_server = redis.StrictRedis('127.0.0.1', db=2)
check = r_server.get("need_CSR")
if check is None:
    print("Need to run registration job.")
    sys.exit(1)

if len(sys.argv) > 1:
    need_haproxy = sys.argv[1]
else:
    need_haproxy = 'no'

# get hostname
try:
    buffer = StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, 'http://169.254.169.253/latest/meta-data/public-hostname')
    c.setopt(pycurl.CONNECTTIMEOUT, 5)
    c.setopt(c.WRITEDATA, buffer)
    c.perform()
    c.close()
    hostname = buffer.getvalue()
except:
    hostname = "localhost"

i_key = "owner-info"
json_data = r_server.get(i_key)
if json_data is not None:
    data = json.loads(json_data)
    email = data['Email']
    hostname = data['Hostname']
frontend_conf = ""
backend_conf = ""
acl = ""
redirect = ""

if check == "1":
    # get certificate
    if os.environ["ENV"] == 'AWS':
        os.system("mkdir -p /opt/certs/letsencrypt")
        os.system("cd /opt/certs && openssl req -inform pem -outform der -in server.csr -out ./letsencrypt/server.der")
        request = (
            "cd /opt/certs/letsencrypt && letsencrypt certonly --csr server.der --standalone --non-interactive --agree-tos --email %s --standalone-supported-challenges http-01" % email)
        os.system(request)
        cert_file = os.path.exists("/opt/certs/letsencrypt/0001_chain.pem")
        if not cert_file:
            sys.exit(2)
        os.system(" cd /opt/certs/letsencrypt && cat 0001_chain.pem ../server.key > ../server.bundle.pem")
        os.system("rm -rf /opt/certs/letsencrypt")
        r_server.set("need_CSR", "0")
        r_server.bgsave()
    else:
        print("Certificate is generated only for AWS env. current env is: %s" % os.environ["ENV"])
else:
    print("Don't need new certificate")

app_key = "apps"
data_apps = r_server.get(app_key)
os.system("rm -rf /opt/haproxy/haproxy.cfg")
config_template = open('/opt/madcore/bin/templates/haproxy.cfg').read()
if os.environ["ENV"] == 'AWS':
    crt_path = "/opt/certs/server.bundle.pem"
else:
    crt_path = "/etc/pki/tls/certs/server.bundle.pem"
if data_apps:
    apps = json.loads(data_apps)
    for app in apps:
        ### get service ip
        service_ip = os.system("kubectl get svc --all-namespaces | grep %s | grep %s | awk '{print $3}'" % (app["namespace"], app["service_name"]))
        if service_ip == "":
            service_ip = "127.0.0.1"
        i = "use_backend %s if { hdr_end(host) -i %s }\n    " % (
            app["name"], app["name"] + "." + data['Hostname'])
        frontend_conf += i
        i = ("backend %s\n    balance roundrobin\n    server %s %s:%s check\n  " % (
            app["name"], app["name"],service_ip, app["port"]))
        backend_conf += i
    template = Template(config_template)
    config = (template.render(hostname=hostname, crt_path=crt_path, subdomain=frontend_conf, backend=backend_conf)
else:
    template = Template(config_template)
    config = (template.render(hostname=hostname, crt_path=crt_path))
open("/opt/haproxy/haproxy.cfg", "w").write(config)
os.system("haproxy -f /opt/haproxy/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)")
<|endoftext|>


<commit_before># Ian Dennis Miller

import sys, os, platform, glob

# find all the conf files in the given directory
base_path = sys.argv[1]
files = glob.glob(os.path.join(base_path, "*.conf"))

if files:
    getmail_cmd = "/usr/bin/getmail --quiet --rcfile %s" % " --rcfile ".join(files)
    os.system(getmail_cmd)
<commit_msg>trying non-quiet cron<commit_after># Ian Dennis Miller

import sys, os, platform, glob

# find all the conf files in the given directory
base_path = sys.argv[1]
files = glob.glob(os.path.join(base_path, "*.conf"))

if files:
    getmail_cmd = "/usr/bin/getmail --rcfile %s" % " --rcfile ".join(files)
    os.system(getmail_cmd)
<|endoftext|>


<commit_before># Copyright (C) 2015 Mike Pennington, Samuel Abels
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""
A driver for Cisco Aironet Wireless Controllers
"""
import re
from Exscript.protocols.drivers.driver import Driver

_user_re     = [re.compile(r'User:\s$', re.I)]
_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')]
_tacacs_re   = re.compile(r'[\r\n]s\/key[\S ]+\r?%s' % _password_re[0].pattern)
_prompt_re   = [re.compile(r'[\r\n].+?>\s$')]
_error_re    = [re.compile(r'%Error'),
                re.compile(r'invalid input', re.I),
                re.compile(r'(?:incomplete|ambiguous) command', re.I),
                re.compile(r'connection timed out', re.I),
                re.compile(r'[^\r\n]+ not found', re.I)]

class AironetDriver(Driver):
    def __init__(self):
        Driver.__init__(self, 'aironet')
        self.user_re     = _user_re
        self.password_re = _password_re
        self.prompt_re   = _prompt_re
        #self.error_re    = _error_re

    def check_head_for_os(self, string):
        if '(Cisco Controller)' in string:
            return 60
        if _tacacs_re.search(string):
            return 50
        if _user_re[0].search(string):
            return 30
        return 0

    def init_terminal(self, conn):
        conn.execute('config pager disable')
<commit_msg>Better aironet prompt regex<commit_after># Copyright (C) 2015 Mike Pennington, Samuel Abels
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
"""
A driver for Cisco Aironet Wireless Controllers
"""
import re
from Exscript.protocols.drivers.driver import Driver

_user_re     = [re.compile(r'User:\s$', re.I)]
_password_re = [re.compile(r'(?:[\r\n]Password: ?|last resort password:)$')]
_tacacs_re   = re.compile(r'[\r\n]s\/key[\S ]+\r?%s' % _password_re[0].pattern)
_prompt_re   = [re.compile(r'[\r\n].Cisco\sController.\s>\s$')]
_error_re    = [re.compile(r'%Error'),
                re.compile(r'invalid input', re.I),
                re.compile(r'(?:incomplete|ambiguous) command', re.I),
                re.compile(r'connection timed out', re.I),
                re.compile(r'[^\r\n]+ not found', re.I)]

class AironetDriver(Driver):
    def __init__(self):
        Driver.__init__(self, 'aironet')
        self.user_re     = _user_re
        self.password_re = _password_re
        self.prompt_re   = _prompt_re
        #self.error_re    = _error_re

    def check_head_for_os(self, string):
        if '(Cisco Controller)' in string:
            return 60
        if _tacacs_re.search(string):
            return 50
        if _user_re[0].search(string):
            return 30
        return 0

    def init_terminal(self, conn):
        conn.execute('config pager disable')
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 SONATA-NFV
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""
"""
This is the main module of the Specific Manager Registry component.
"""
import logging
import time
import uuid
import yaml
from sonmanobase.plugin import ManoBasePlugin

from son_mano_specific_manager_registry import smr_engine

logging.basicConfig(level=logging.INFO)
LOG = logging.getLogger("son-mano-specific-manager-registry")
LOG.setLevel(logging.DEBUG)
logging.getLogger("son-mano-base:messaging").setLevel(logging.INFO)

class SpecificManagerRegistry(ManoBasePlugin):

    def __init__(self):

        self.version = 'v0.02'
        self.description = 'Specific Manager Registry'

        # a storage for f/ssms
        self.ssm_repo = {}

        # connect to the docker daemon
        self.smrengine = smr_engine.SMREngine()

        # register smr into the plugin manager
        super(self.__class__, self).__init__(version=self.version, description=self.description)

    def declare_subscriptions(self):
        """
        Declare topics to which we want to listen and define callback methods.
        """
        self.manoconn.register_async_endpoint(self.on_board, "specific.manager.registry.ssm.on-board")
        self.manoconn.register_async_endpoint(self.on_instantiate, "specific.manager.registry.ssm.instantiate")
        self.manoconn.register_async_endpoint(self.on_ssm_register, "specific.manager.registry.ssm.registration")
        self.manoconn.register_async_endpoint(self.on_ssm_update, "specific.manager.registry.ssm.update")
        self.manoconn.subscribe(self.on_ssm_status, "specific.manager.registry.ssm.status")

    def on_board(self, ch, method, properties, message):
        id = None
        try:
            if properties.app_id != self.name:
                message = yaml.load(message)
                if 'NSD' and 'VNFDs' in message:
                    self.ssm_onboarding(message)
                    for i in range(len(message['VNFDs'])):
                        self.fsm_onboarding(message['VNFDs'][i])
                    # return the result to SLM
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' in message and 'VNFDs' not in message:
                    self.ssm_onboarding(message)
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' not in message and 'VNFDs' in message:
                    for i in range(len(message['VNFDs'])):
                        self.fsm_onboarding(message['VNFDs'][i])
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' and 'VNFDs' not in message:
                    return yaml.dump({'status': 'Failed', 'error': 'NSD/VNFD not found'})

        except BaseException as err:
            if id is not None:
                LOG.error("'{0}' on-boarding: Failed ==> '{1}'".format(id, err))
            else:
                LOG.error("FSM/SSM on-boarding: Failed ==> '{0}'".format(err))
            return yaml.dump({'status': 'Failed', 'error': str(err)})

    def on_instantiate(self, ch, method, properties, message):

        id = None
        try:
            message = yaml.load(message)
            #image = message['NSD']['service_specific_managers'][0]['image']
            #id = message['NSD']['service_specific_managers'][0]['id']
            id = message['id']
            image = message['image']
            LOG.info('Instantiation request received for SSM id: {0}'.format(id))
            self.smrengine.start(image_name= image, ssm_name= id, host_ip= None)
            self._wait_for_ssm_registration(ssm_name= id)
            if id in self.ssm_repo.keys():
                LOG.debug("Registration: succeeded ==> '{0}' ".format(self.ssm_repo))
                return yaml.dump({'name':id,'status': 'Instantiated', 'uuid': self.ssm_repo[id]['uuid'],'error': 'None'})
            else:
                LOG.error("'{0}' instantiation: Failed ==> SSM registration in SMR failed'".format(id))
                return yaml.dump({'status':'Failed', 'uuid':'None','error': 'SSM registration in SMR failed'})
        except BaseException as err:
            if id is not None:
                LOG.error("'{0}' instantiation: Failed ==> '{1}'".format(id, err))
            else:
                LOG.error("SSM instantiation: Failed ==> '{0}'".format(err))
            return yaml.dump({'status': 'Failed', 'uuid':'None', 'error': str(err)})

    def on_ssm_register(self, ch, method, properties, message):

        try:
            message = yaml.load(str(message))
            keys = self.ssm_repo.keys()
            if message['name'] in keys:
                LOG.error("Cannot register '{0}', already exists".format(message['name']))
                result = {'status': 'Failed', 'error':"Cannot register '{0}', already exists".format(message['name'])}
            else:
                pid = str(uuid.uuid4())
                response = {
                    "status": "running",
                    "smtype": message['smtype'],
                    "sfname": message['sfname'],
                    "name": message['name'],
                    "id": message['id'],
                    "version": message['version'],
                    "description": message['description'],
                    "uuid": pid,
                    "error": None
                }
                self.ssm_repo.update({message['name']: response})

                result = response
        except BaseException as err:
            result = {'status': 'Failed', 'error': str(err)}
            LOG.exception("'{0}' registeration Failed: ".format(message['name']))
        return yaml.dump(result)

    def on_ssm_update(self, ch, method, properties, message):
        id = None
        try:
            message = yaml.load(message)
            image = message['NSD']['service_specific_managers'][0]['image']
            id = message['NSD']['service_specific_managers'][0]['id']
            LOG.info('Update request received for SSM id: {0}'.format(id))
            host_ip = ''
            try:
                list = message['VNFR']
                for x in range(len(list)):
                    if message['VNFR'][x]['virtual_deployment_units'][0]['vm_image'] == 'sonata-vfw':
                        host_ip =message['VNFR'][x]['virtual_deployment_units'][0]['vnfc_instance'][0]['connection_points'][0]['type']['address']
                #host_ip = message['VNFR'][0]['virtual_deployment_units'][0]['vnfc_instance'][0]['connection_points'][0]['type']['address']
                #message['NSR'][1]['virtual_deployment_units'][1]['vnfc_instance'][0]['connection_points'][0]['type']['address']
            except BaseException as err:
                LOG.error("'{0}' Update: Failed ==> Host IP address does not exist in the VNFR")
                return yaml.dump({'status': 'Failed', 'error': 'Host IP address does not exist in the VNFR'})
            LOG.info('vFW IP address "{0}"'.format(host_ip))
            self.smrengine.pull(image, id)
            self.smrengine.start(image_name=image, ssm_name=id, host_ip=host_ip)
            LOG.info("Waiting for '{0}' registration ...".format(id))
            self._wait_for_ssm_registration(ssm_name=id)
            if id in self.ssm_repo.keys():
                self.ssm_kill()
                LOG.debug("SSM update: succeeded ")
                return yaml.dump({'status': 'Updated', 'error': 'None'})
            else:
                LOG.error("'{0}' Update: Failed ==> SSM registration in SMR failed'".format(id))
                return yaml.dump({'status':'Failed', 'error': 'SSM registration in SMR failed'})
        except BaseException as err:
            LOG.error("'{0}' Update: Failed ==> '{1}'".format(id, err))
            return yaml.dump({'status': 'Failed', 'error': str(err)})

    def ssm_kill(self):
        self.smrengine.stop('dumb')
        self.ssm_repo['dumb']['status'] = 'killed'
        LOG.debug('dumb kill: succeeded')

    def _wait_for_ssm_registration(self, ssm_name, timeout=20, sleep_interval=5):
        c = 0
        rep = str(self.ssm_repo)
        while ssm_name not in rep and c < timeout:
            time.sleep(sleep_interval)
            c += sleep_interval

    def on_ssm_status(self, ch, method, properties, message):
        message = yaml.load(message)
        LOG.info('{0} status: {1}'.format(message['name'],message['status']))

    def ssm_onboarding(self,message):

        # on-boarding all NSD's SSMs
        for i in range(len(message['NSD']['service_specific_managers'])):
            image = message['NSD']['service_specific_managers'][i]['image']
            id = message['NSD']['service_specific_managers'][i]['id']
            LOG.info('On-boarding request received for SSM: {0}'.format(id))
            self.smrengine.pull(ssm_uri=image, ssm_name=id)
            LOG.info('On-boarding succeeded for SSM: {0}'.format(id))

    def fsm_onboarding(self, message):

        # on-boarding all VNFD's FSMs
        for i in range(len(message['VNFD']['function_specific_managers'])):
            image = message['VNFD']['function_specific_managers'][i]['image']
            id = message['VNFD']['function_specific_managers'][i]['id']
            LOG.info('On-boarding request received for FSM: {0}'.format(id))
            self.smrengine.pull(ssm_uri=image, ssm_name=id)
            LOG.info('On-boarding succeeded for FSM: {0}'.format(id))

def main():
    SpecificManagerRegistry()

if __name__ == '__main__':
    main()<commit_msg>rerun the test<commit_after>"""
Copyright (c) 2015 SONATA-NFV
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""
"""
This is the main module of the Specific Manager Registry component.
"""
import logging
import time
import uuid
import yaml
from sonmanobase.plugin import ManoBasePlugin

from son_mano_specific_manager_registry import smr_engine

logging.basicConfig(level=logging.INFO)
LOG = logging.getLogger("son-mano-specific-manager-registry")
LOG.setLevel(logging.DEBUG)
logging.getLogger("son-mano-base:messaging").setLevel(logging.INFO)

class SpecificManagerRegistry(ManoBasePlugin):

    def __init__(self):

        self.version = 'v0.02'
        self.description = 'Specific Manager Registry'

        # a storage for f/ssms
        self.ssm_repo = {}

        # connect to the docker daemon
        self.smrengine = smr_engine.SMREngine()

        # register smr into the plugin manager
        super(self.__class__, self).__init__(version=self.version, description=self.description)

    def declare_subscriptions(self):
        """
        Declare topics to which we want to listen and define callback methods.
        """
        self.manoconn.register_async_endpoint(self.on_board, "specific.manager.registry.ssm.on-board")
        self.manoconn.register_async_endpoint(self.on_instantiate, "specific.manager.registry.ssm.instantiate")
        self.manoconn.register_async_endpoint(self.on_ssm_register, "specific.manager.registry.ssm.registration")
        self.manoconn.register_async_endpoint(self.on_ssm_update, "specific.manager.registry.ssm.update")
        self.manoconn.subscribe(self.on_ssm_status, "specific.manager.registry.ssm.status")

    def on_board(self, ch, method, properties, message):
        id = None
        try:
            if properties.app_id != self.name:
                message = yaml.load(message)
                if 'NSD' and 'VNFDs' in message:
                    self.ssm_onboarding(message)
                    for i in range(len(message['VNFDs'])):
                        self.fsm_onboarding(message['VNFDs'][i])
                    # return the result to SLM
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' in message and 'VNFDs' not in message:
                    self.ssm_onboarding(message)
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' not in message and 'VNFDs' in message:
                    for i in range(len(message['VNFDs'])):
                        self.fsm_onboarding(message['VNFDs'][i])
                    return yaml.dump({'status': 'On-boarded', 'error': 'None'})

                elif 'NSD' and 'VNFDs' not in message:
                    return yaml.dump({'status': 'Failed', 'error': 'NSD/VNFD not found'})

        except BaseException as err:
            if id is not None:
                LOG.error("'{0}' on-boarding: Failed ==> '{1}'".format(id, err))
            else:
                LOG.error("FSM/SSM on-boarding: Failed ==> '{0}'".format(err))
            return yaml.dump({'status': 'Failed', 'error': str(err)})

    def on_instantiate(self, ch, method, properties, message):

        id = None
        try:
            message = yaml.load(message)
            #image = message['NSD']['service_specific_managers'][0]['image']
            #id = message['NSD']['service_specific_managers'][0]['id']
            id = message['id']
            image = message['image']
            LOG.info('Instantiation request received for SSM id: {0}'.format(id))
            self.smrengine.start(image_name= image, ssm_name= id, host_ip= None)
            self._wait_for_ssm_registration(ssm_name= id)
            if id in self.ssm_repo.keys():
                LOG.debug("Registration: succeeded ==> '{0}' ".format(self.ssm_repo))
                return yaml.dump({'name':id,'status': 'Instantiated', 'uuid': self.ssm_repo[id]['uuid'],'error': 'None'})
            else:
                LOG.error("'{0}' instantiation: Failed ==> SSM registration in SMR failed'".format(id))
                return yaml.dump({'status':'Failed', 'uuid':'None','error': 'SSM registration in SMR failed'})
        except BaseException as err:
            if id is not None:
                LOG.error("'{0}' instantiation: Failed ==> '{1}'".format(id, err))
            else:
                LOG.error("SSM instantiation: Failed ==> '{0}'".format(err))
            return yaml.dump({'status': 'Failed', 'uuid':'None', 'error': str(err)})

    def on_ssm_register(self, ch, method, properties, message):

        try:
            message = yaml.load(str(message))
            keys = self.ssm_repo.keys()
            if message['name'] in keys:
                LOG.error("Cannot register '{0}', already exists".format(message['name']))
                result = {'status': 'Failed', 'error':"Cannot register '{0}', already exists".format(message['name'])}
            else:
                pid = str(uuid.uuid4())
                response = {
                    "status": "running",
                    "smtype": message['smtype'],
                    "sfname": message['sfname'],
                    "name": message['name'],
                    "id": message['id'],
                    "version": message['version'],
                    "description": message['description'],
                    "uuid": pid,
                    "error": None
                }
                self.ssm_repo.update({message['name']: response})

                result = response
        except BaseException as err:
            result = {'status': 'Failed', 'error': str(err)}
            LOG.exception("'{0}' registeration Failed: ".format(message['name']))
        return yaml.dump(result)

    def on_ssm_update(self, ch, method, properties, message):
        id = None
        try:
            message = yaml.load(message)
            image = message['NSD']['service_specific_managers'][0]['image']
            id = message['NSD']['service_specific_managers'][0]['id']
            LOG.info('Update request received for SSM id: {0}'.format(id))
            host_ip = ''
            try:
                list = message['VNFR']
                for x in range(len(list)):
                    if message['VNFR'][x]['virtual_deployment_units'][0]['vm_image'] == 'sonata-vfw':
                        host_ip =message['VNFR'][x]['virtual_deployment_units'][0]['vnfc_instance'][0]['connection_points'][0]['type']['address']
                #host_ip = message['VNFR'][0]['virtual_deployment_units'][0]['vnfc_instance'][0]['connection_points'][0]['type']['address']
                #message['NSR'][1]['virtual_deployment_units'][1]['vnfc_instance'][0]['connection_points'][0]['type']['address']
            except BaseException as err:
                LOG.error("'{0}' Update: Failed ==> Host IP address does not exist in the VNFR")
                return yaml.dump({'status': 'Failed', 'error': 'Host IP address does not exist in the VNFR'})
            LOG.info('vFW IP address "{0}"'.format(host_ip))
            self.smrengine.pull(image, id)
            self.smrengine.start(image_name=image, ssm_name=id, host_ip=host_ip)
            LOG.info("Waiting for '{0}' registration ...".format(id))
            self._wait_for_ssm_registration(ssm_name=id)
            if id in self.ssm_repo.keys():
                self.ssm_kill()
                LOG.debug("SSM update: succeeded ")
                return yaml.dump({'status': 'Updated', 'error': 'None'})
            else:
                LOG.error("'{0}' Update: Failed ==> SSM registration in SMR failed'".format(id))
                return yaml.dump({'status':'Failed', 'error': 'SSM registration in SMR failed'})
        except BaseException as err:
            LOG.error("'{0}' Update: Failed ==> '{1}'".format(id, err))
            return yaml.dump({'status': 'Failed', 'error': str(err)})

    def ssm_kill(self):
        self.smrengine.stop('dumb')
        self.ssm_repo['dumb']['status'] = 'killed'
        LOG.debug('dumb kill: succeeded')

    def _wait_for_ssm_registration(self, ssm_name, timeout=20, sleep_interval=5):
        c = 0
        rep = str(self.ssm_repo)
        while ssm_name not in rep and c < timeout:
            time.sleep(sleep_interval)
            c += sleep_interval

    def on_ssm_status(self, ch, method, properties, message):
        message = yaml.load(message)
        LOG.info('{0} status: {1}'.format(message['name'],message['status']))

    def ssm_onboarding(self,message):

        # on-boarding all NSD's SSMs
        for i in range(len(message['NSD']['service_specific_managers'])):
            image = message['NSD']['service_specific_managers'][i]['image']
            id = message['NSD']['service_specific_managers'][i]['id']
            LOG.info('On-boarding request received for SSM: {0}'.format(id))
            self.smrengine.pull(ssm_uri=image, ssm_name=id)
            LOG.info('On-boarding succeeded for SSM: {0}'.format(id))

    def fsm_onboarding(self, message):

        # on-boarding all VNFD's FSMs
        for i in range(len(message['VNFD']['function_specific_managers'])):
            image = message['VNFD']['function_specific_managers'][i]['image']
            id = message['VNFD']['function_specific_managers'][i]['id']
            LOG.info('On-boarding request received for FSM: {0}'.format(id))
            self.smrengine.pull(ssm_uri=image, ssm_name=id)
            LOG.info('On-boarding succeeded for FSM: {0}'.format(id))

def main():
    SpecificManagerRegistry()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from openpyxl import Workbook
import sys
sys.path.append('C:/Users/조양아/Documents/GitHub/study/2018/04.10/python')
from jya_gAPI_class import GapiClass
# 이 파일과 동일한 경로에 config.py 추가 안하면 오류가 나는데, 모듈 불러올 때 config.py는 같이 안불러오나봐요??

wb = Workbook()
ws = wb.active

wb.save('../datas/hanname-excel.xlsx')

instance = GapiClass()
a = instance.getMember('planning_d')
print(a)

# 아직 다 안함<commit_msg>making excel data<commit_after>from openpyxl import Workbook
import sys
sys.path.append('C:/Users/조양아/Documents/GitHub/study/2018/04.10/python')
from jya_gAPIclass import GapiClass
# 이 파일과 동일한 경로에 config.py 추가 안하면 오류가 나는데, 모듈 불러올 때 config.py는 같이 안불러오나봐요??

wb = Workbook()
ws = wb.active

instance = GapiClass()
id_list = ['planning_d', 'test1gabia', 'jya9055']

# for id in id_list:
#     number = 0
#     for k in (1, 2, 3):
#         number = number + 1
#         ws.cell(row=k, column=1).value = number
#         ws.cell(row=k, column=2).value = instance.getMember(id)

number = 0
for k in (1, 2, 3):
    number = number + 1
    ws.cell(row=k, column=1).value = number
    for id in id_list:
        ws.cell(row=k, column=2).value = instance.getMember(id)

wb.save('../datas/hanname-excel.xlsx')

# 내일 다시!!!<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2013, Chatham Financial <oss@chathamfinancial.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: rabbitmq_plugin
short_description: Adds or removes plugins to RabbitMQ
description:
  - Enables or disables RabbitMQ plugins
version_added: "1.1"
author: '"Chris Hoffman (@chrishoffman)"'
options:
  names:
    description:
      - Comma-separated list of plugin names
    required: true
    default: null
    aliases: [name]
  new_only:
    description:
      - Only enable missing plugins
      - Does not disable plugins that are not in the names list
    required: false
    default: "no"
    choices: [ "yes", "no" ]
  state:
    description:
      - Specify if plugins are to be enabled or disabled
    required: false
    default: enabled
    choices: [enabled, disabled]
  prefix:
    description:
      - Specify a custom install prefix to a Rabbit
    required: false
    version_added: "1.3"
    default: null
'''

EXAMPLES = '''
# Enables the rabbitmq_management plugin
- rabbitmq_plugin: names=rabbitmq_management state=enabled
'''

import os

class RabbitMqPlugins(object):
    def __init__(self, module):
        self.module = module

        if module.params['prefix']:
            if os.path.isdir(os.path.join(module.params['prefix'], 'bin')):
                bin_path = os.path.join(module.params['prefix'], 'bin')
            elif os.path.isdir(os.path.join(module.params['prefix'], 'sbin')):
                bin_path = os.path.join(module.params['prefix'], 'sbin')
            else:
                # No such path exists.
                raise Exception("No binary folder in prefix %s" %
                        module.params['prefix'])

            self._rabbitmq_plugins = bin_path + "/rabbitmq-plugins"

        else:
            self._rabbitmq_plugins = module.get_bin_path('rabbitmq-plugins', True)

    def _exec(self, args, run_in_check_mode=False):
        if not self.module.check_mode or (self.module.check_mode and run_in_check_mode):
            cmd = [self._rabbitmq_plugins]
            rc, out, err = self.module.run_command(cmd + args, check_rc=True)
            return out.splitlines()
        return list()

    def get_all(self):
        return self._exec(['list', '-E', '-m'], True)

    def enable(self, name):
        self._exec(['enable', name])

    def disable(self, name):
        self._exec(['disable', name])

def main():
    arg_spec = dict(
        names=dict(required=True, aliases=['name']),
        new_only=dict(default='no', type='bool'),
        state=dict(default='enabled', choices=['enabled', 'disabled']),
        prefix=dict(required=False, default=None)
    )
    module = AnsibleModule(
        argument_spec=arg_spec,
        supports_check_mode=True
    )

    names = module.params['names'].split(',')
    new_only = module.params['new_only']
    state = module.params['state']

    rabbitmq_plugins = RabbitMqPlugins(module)
    enabled_plugins = rabbitmq_plugins.get_all()

    enabled = []
    disabled = []
    if state == 'enabled':
        if not new_only:
            for plugin in enabled_plugins:
                if plugin not in names:
                    rabbitmq_plugins.disable(plugin)
                    disabled.append(plugin)

        for name in names:
            if name not in enabled_plugins:
                rabbitmq_plugins.enable(name)
                enabled.append(name)
    else:
        for plugin in enabled_plugins:
            if plugin in names:
                rabbitmq_plugins.disable(plugin)
                disabled.append(plugin)

    changed = len(enabled) > 0 or len(disabled) > 0
    module.exit_json(changed=changed, enabled=enabled, disabled=disabled)

# import module snippets
from ansible.module_utils.basic import *
main()
<commit_msg>Ignore extra lines from Pivotal's RabbitMQ package<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2013, Chatham Financial <oss@chathamfinancial.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: rabbitmq_plugin
short_description: Adds or removes plugins to RabbitMQ
description:
  - Enables or disables RabbitMQ plugins
version_added: "1.1"
author: '"Chris Hoffman (@chrishoffman)"'
options:
  names:
    description:
      - Comma-separated list of plugin names
    required: true
    default: null
    aliases: [name]
  new_only:
    description:
      - Only enable missing plugins
      - Does not disable plugins that are not in the names list
    required: false
    default: "no"
    choices: [ "yes", "no" ]
  state:
    description:
      - Specify if plugins are to be enabled or disabled
    required: false
    default: enabled
    choices: [enabled, disabled]
  prefix:
    description:
      - Specify a custom install prefix to a Rabbit
    required: false
    version_added: "1.3"
    default: null
'''

EXAMPLES = '''
# Enables the rabbitmq_management plugin
- rabbitmq_plugin: names=rabbitmq_management state=enabled
'''

import os

class RabbitMqPlugins(object):
    def __init__(self, module):
        self.module = module

        if module.params['prefix']:
            if os.path.isdir(os.path.join(module.params['prefix'], 'bin')):
                bin_path = os.path.join(module.params['prefix'], 'bin')
            elif os.path.isdir(os.path.join(module.params['prefix'], 'sbin')):
                bin_path = os.path.join(module.params['prefix'], 'sbin')
            else:
                # No such path exists.
                raise Exception("No binary folder in prefix %s" %
                        module.params['prefix'])

            self._rabbitmq_plugins = bin_path + "/rabbitmq-plugins"

        else:
            self._rabbitmq_plugins = module.get_bin_path('rabbitmq-plugins', True)

    def _exec(self, args, run_in_check_mode=False):
        if not self.module.check_mode or (self.module.check_mode and run_in_check_mode):
            cmd = [self._rabbitmq_plugins]
            rc, out, err = self.module.run_command(cmd + args, check_rc=True)
            return out.splitlines()
        return list()

    def get_all(self):
        list_output = self._exec(['list', '-E', '-m'], True)
        plugins = []
        for plugin in list_output:
            if not plugin:
                break
            plugins.append(plugin)

        return plugins

    def enable(self, name):
        self._exec(['enable', name])

    def disable(self, name):
        self._exec(['disable', name])

def main():
    arg_spec = dict(
        names=dict(required=True, aliases=['name']),
        new_only=dict(default='no', type='bool'),
        state=dict(default='enabled', choices=['enabled', 'disabled']),
        prefix=dict(required=False, default=None)
    )
    module = AnsibleModule(
        argument_spec=arg_spec,
        supports_check_mode=True
    )

    names = module.params['names'].split(',')
    new_only = module.params['new_only']
    state = module.params['state']

    rabbitmq_plugins = RabbitMqPlugins(module)
    enabled_plugins = rabbitmq_plugins.get_all()

    enabled = []
    disabled = []
    if state == 'enabled':
        if not new_only:
            for plugin in enabled_plugins:
                if plugin not in names:
                    rabbitmq_plugins.disable(plugin)
                    disabled.append(plugin)

        for name in names:
            if name not in enabled_plugins:
                rabbitmq_plugins.enable(name)
                enabled.append(name)
    else:
        for plugin in enabled_plugins:
            if plugin in names:
                rabbitmq_plugins.disable(plugin)
                disabled.append(plugin)

    changed = len(enabled) > 0 or len(disabled) > 0
    module.exit_json(changed=changed, enabled=enabled, disabled=disabled)

# import module snippets
from ansible.module_utils.basic import *
main()
<|endoftext|>


<commit_before>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.remote.webelement import  WebElement
import time
from bs4 import BeautifulSoup
import re
from datetime import date, timedelta
from pymongo import MongoClient

class WaterScrapper:
    def __init__(self, date, countyName):
        self._date = date
        self._countyName = countyName
        self._timeDelay = 1
        self._initialTimeDelay = 10
        self._finalTimeDelay = 10
        self._url = "https://wins.mwdsc.org/Unsecured/login.aspx?Wamiuser=Wamiuser"
        self._reportLinkID = 'ctl00_ContentPlaceHolder1_gridReports_ctl05_ReportNameLinkButton'
        self._countySelectID = 'ctl00_ContentPlaceHolder1_ddlMember'
        self._meterSelectID = 'ctl00_ContentPlaceHolder1_ddlMeter'
        self._startDateID = 'ctl00_ContentPlaceHolder1_txtStart'
        self._endDateID = 'ctl00_ContentPlaceHolder1_txtEnd'
        self._previewReportButtonID = 'ctl00_ContentPlaceHolder1_PreviewRptLinkButton'
        self._externalFrameID = 'documentFrame000hb'
        self._internalFrameID = 'ceframe'
        self._containerDivClassPattern = re.compile("adi4")
        self._meterDataClassPattern = re.compile("fci4")

        self._driver = webdriver.Firefox()
        self._driver.implicitly_wait(self._initialTimeDelay) # seconds
        self._driver.get(self._url)
        # self._wait for loading form
        self._wait = WebDriverWait(self._driver, self._initialTimeDelay)
        self._iterator = 0;

    def startEngine(self):

        # Get anchor link for report form
        reportLink = self._wait.until(EC.element_to_be_clickable((By.ID,self._reportLinkID)))
        # Click report link
        reportLink.send_keys(Keys.RETURN)
        time.sleep(self._timeDelay)

        try:
            countyTag = self._wait.until(EC.presence_of_element_located((By.ID,self._countySelectID)))
            countySelectTag = Select(countyTag)
            countySelectTag.select_by_value(self._countyName)
        except Exception as e:
            return self.handleException("County Name", e)

        time.sleep(self._timeDelay)

        try:
            meterTag = self._wait.until(EC.presence_of_element_located((By.ID,self._meterSelectID)))
            meterSelectTag = Select(meterTag)
            meterSelectTag.select_by_value('All')
        except Exception as e:
            return self.handleException("Meter", e)

        time.sleep(self._timeDelay)

        if self._iterator < 1:
            startDateInputField = self._driver.find_element_by_id(self._startDateID)
            startDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

            endDateInputField = self._driver.find_element_by_id(self._endDateID)
            endDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

        # if (len(startDateInputField.get_attribute("value")) < 0 or len(endDateInputField.get_attribute("value")) < 0):
        #     return self.handleException("Missing dates", None)

        previewReportButton = self._driver.find_element_by_id(self._previewReportButtonID)
        previewReportButton.send_keys(Keys.RETURN)
        #time.sleep(self._finalTimeDelay)

        currentWindow = self._driver.current_window_handle
        popupWindow = None
        windows = self._driver.window_handles

        while len(windows) < 2:
            print "==Windows: " + str(len(windows))
            windows = self._driver.window_handles

        for window in windows:
            if currentWindow != window:
                popupWindow = window
        self._driver.switch_to.window(popupWindow)

        externalFrame = self._driver.find_element_by_name(self._externalFrameID)
        self._driver.switch_to.frame(externalFrame)
        internalFrame = self._driver.find_element_by_name(self._internalFrameID)
        self._driver.switch_to.frame(internalFrame)
        cridReportPage = self._wait.until(EC.presence_of_element_located((By.ID,'cridreportpage')))

        soup = BeautifulSoup(self._driver.page_source)
        self._driver.close()
        self._driver.switch_to.window(self._driver.window_handles[0])
        containerDivs = soup.findAll('div', {"class": self._containerDivClassPattern})

        actualSpanIndex = None
        iterator = 0
        for div in containerDivs:
            try:
                title = div["title"]
            except KeyError:
                title = ""
            if (title == "Visit the Support Site for samples, web forums, tutorials, technical briefs and more!"):
                actualSpanIndex = iterator - 1
            iterator = iterator + 1
        if actualSpanIndex == None:
            return self.handleException("Could not find frame", None)

        newSoup = BeautifulSoup(str(containerDivs[actualSpanIndex]))
        span = newSoup.find('span', {"class": self._meterDataClassPattern})
        self._iterator = self._iterator + 1
        return {'status': 'true', 'value': str(span.text.encode('utf-8'))}

    def handleException(self, message, exception):
        self._driver.get(self._url)
        self._iterator = 0;
        print "======= EXCEPTION ======="
        print "Exception caught while: " + message
        print exception
        print "======= Let's try again ======="
        return {'status': 'false'}

def main():
    client = MongoClient()
    db = client.waterdb
    fileName = 'meterData.txt'
    yesterday = date.today() - timedelta(days=1)
    dateString = str(yesterday.month) +"/"+ str(yesterday.day) + "/" + str(yesterday.year)
    scrapper = WaterScrapper(dateString, 'B')

    appender = open(fileName, 'a')
    reader = open('CountyNames.txt', 'r')

    if reader.mode == 'r' and appender.mode == 'a':
        competeListing = reader.readlines()
        for line in competeListing:
            stringList = line.split(',')
            key = stringList[0]
            valueList = stringList[1].split('\n')
            value = valueList[0]
            scrapper._countyName = key

            data = scrapper.startEngine()
            while(data['status'] == 'false'):
                time.sleep(1*5)
                data = scrapper.startEngine()

            grandTotal = re.findall('grand', data['value'], re.IGNORECASE)
            if grandTotal:
                data['value'] = 'NA'
            formattedLine = dateString + ", " + value + ", "  + data['value'] + "\n"
            print dateString + ", " + value + ", "  + data['value'] + ' AF'
            #insert data in mongo
            db.waterdb.insert({"Date":dateString,"City":value,"water":data['value']})
            appender.write(formattedLine)

    appender.close()
    reader.close()

if __name__ == "__main__": main()
<commit_msg>MongoDB + Looping from January 2<commit_after>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.remote.webelement import  WebElement
import time
from bs4 import BeautifulSoup
import re
from datetime import datetime, date, timedelta
import pymongo
from pymongo import MongoClient

class WaterScrapper:
	def __init__(self, date, countyName):
		self._date = date
		self._countyName = countyName
		self._timeDelay = 1
		self._initialTimeDelay = 10
		self._finalTimeDelay = 10
		self._url = "https://wins.mwdsc.org/Unsecured/login.aspx?Wamiuser=Wamiuser"
		self._reportLinkID = 'ctl00_ContentPlaceHolder1_gridReports_ctl05_ReportNameLinkButton'
		self._countySelectID = 'ctl00_ContentPlaceHolder1_ddlMember'
		self._meterSelectID = 'ctl00_ContentPlaceHolder1_ddlMeter'
		self._startDateID = 'ctl00_ContentPlaceHolder1_txtStart'
		self._endDateID = 'ctl00_ContentPlaceHolder1_txtEnd'
		self._previewReportButtonID = 'ctl00_ContentPlaceHolder1_PreviewRptLinkButton'
		self._externalFrameID = 'documentFrame000hb'
		self._internalFrameID = 'ceframe'
		self._containerDivClassPattern = re.compile("adi4")
		self._meterDataClassPattern = re.compile("fci4")

		self._driver = webdriver.Firefox()
		self._driver.implicitly_wait(self._initialTimeDelay) # seconds
		self._driver.get(self._url)
		# self._wait for loading form
		self._wait = WebDriverWait(self._driver, self._initialTimeDelay)
		self._iterator = 0;

	def startEngine(self):

		# Get anchor link for report form
		reportLink = self._wait.until(EC.element_to_be_clickable((By.ID,self._reportLinkID)))
		# Click report link
		reportLink.send_keys(Keys.RETURN)
		time.sleep(self._timeDelay)

		try:
			countyTag = self._wait.until(EC.presence_of_element_located((By.ID,self._countySelectID)))
			countySelectTag = Select(countyTag)
			countySelectTag.select_by_value(self._countyName)
		except Exception as e:
			return self.handleException("County Name", e)

		time.sleep(self._timeDelay)

		try:
			meterTag = self._wait.until(EC.presence_of_element_located((By.ID,self._meterSelectID)))
			meterSelectTag = Select(meterTag)
			meterSelectTag.select_by_value('All')
		except Exception as e:
			return self.handleException("Meter", e)

		time.sleep(self._timeDelay)

		if self._iterator < 1:
			startDateInputField = self._driver.find_element_by_id(self._startDateID)
			startDateInputField.send_keys(self._date)
			time.sleep(self._timeDelay)

			endDateInputField = self._driver.find_element_by_id(self._endDateID)
			endDateInputField.send_keys(self._date)
			time.sleep(self._timeDelay)

		# if (len(startDateInputField.get_attribute("value")) < 0 or len(endDateInputField.get_attribute("value")) < 0):
		#     return self.handleException("Missing dates", None)

		previewReportButton = self._driver.find_element_by_id(self._previewReportButtonID)
		previewReportButton.send_keys(Keys.RETURN)
		#time.sleep(self._finalTimeDelay)

		currentWindow = self._driver.current_window_handle
		popupWindow = None
		windows = self._driver.window_handles

		while len(windows) < 2:
			print "==Windows: " + str(len(windows))
			windows = self._driver.window_handles

		for window in windows:
			if currentWindow != window:
				popupWindow = window
		self._driver.switch_to.window(popupWindow)

		externalFrame = self._driver.find_element_by_name(self._externalFrameID)
		self._driver.switch_to.frame(externalFrame)
		internalFrame = self._driver.find_element_by_name(self._internalFrameID)
		self._driver.switch_to.frame(internalFrame)
		cridReportPage = self._wait.until(EC.presence_of_element_located((By.ID,'cridreportpage')))

		soup = BeautifulSoup(self._driver.page_source)
		self._driver.close()
		self._driver.switch_to.window(self._driver.window_handles[0])
		containerDivs = soup.findAll('div', {"class": self._containerDivClassPattern})

		actualSpanIndex = None
		iterator = 0
		for div in containerDivs:
			try:
				title = div["title"]
			except KeyError:
				title = ""
			if (title == "Visit the Support Site for samples, web forums, tutorials, technical briefs and more!"):
				actualSpanIndex = iterator - 1
			iterator = iterator + 1
		if actualSpanIndex == None:
			return self.handleException("Could not find frame", None)

		newSoup = BeautifulSoup(str(containerDivs[actualSpanIndex]))
		span = newSoup.find('span', {"class": self._meterDataClassPattern})
		self._iterator = self._iterator + 1
		return {'status': 'true', 'value': str(span.text.encode('utf-8'))}

	def handleException(self, message, exception):
		self._driver.get(self._url)
		self._iterator = 0;
		print "======= EXCEPTION ======="
		print "Exception caught while: " + message
		print exception
		print "======= Let's try again ======="
		return {'status': 'false'}

def engine(selectedDate):
	client = MongoClient('localhost', 27017)
	db = client['cs-594-project']
	meterCollection = db['meterData']
	fileName = 'meterData.txt'
	dateString = str(date.month) +"/"+ str(date.day) + "/" + str(date.year)
	scrapper = WaterScrapper(dateString, 'B')
	appender = open(fileName, 'a')
	reader = open('CountyNames.txt', 'r')
	if reader.mode == 'r' and appender.mode == 'a':
		competeListing = reader.readlines()
		for line in competeListing:
			stringList = line.split(',')
			key = stringList[0]
			valueList = stringList[1].split('\n')
			value = valueList[0]
			scrapper._countyName = key

			data = scrapper.startEngine()
			while(data['status'] == 'false'):
				time.sleep(1*5)
				data = scrapper.startEngine()

			grandTotal = re.findall('grand', data['value'], re.IGNORECASE)
			if grandTotal:
				data['value'] = 'NA'
			formattedLine = dateString + ", " + value + ", "  + data['value'] + "\n"
			print dateString + ", " + value + ", "  + data['value'] + ' AF'
			appender.write(formattedLine)
			meterCollection.insert({"Date": dateString, "City": value, "Rain": data['value']})

	appender.close()
	reader.close()

def main():
	januaryFirst = datetime(2015, 1, 1, 00,00,00).date()
	daysCount = date.today()-januaryFirst
	#print daysCount
	for iterator in range(0, daysCount.days):
		print "========"
		selectedDate = date.today() - timedelta(days=iterator)
		engine(selectedDate)
		print "========"

if __name__ == "__main__": main()
<|endoftext|>


<commit_before>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from bs4 import BeautifulSoup
import re
from datetime import date, timedelta

class WaterScrapper:
    def __init__(self, date, countyName):
        self._date = date
        self._countyName = countyName
        self._timeDelay = 1
        self._initialTimeDelay = 20 * 3
        self._finalTimeDelay = 20
        self._url = "https://wins.mwdsc.org/Unsecured/login.aspx?Wamiuser=Wamiuser"
        self._reportLinkID = 'ctl00_ContentPlaceHolder1_gridReports_ctl05_ReportNameLinkButton'
        self._countySelectID = 'ctl00_ContentPlaceHolder1_ddlMember'
        self._meterSelectID = 'ctl00_ContentPlaceHolder1_ddlMeter'
        self._startDateID = 'ctl00_ContentPlaceHolder1_txtStart'
        self._endDateID = 'ctl00_ContentPlaceHolder1_txtEnd'
        self._previewReportButtonID = 'ctl00_ContentPlaceHolder1_PreviewRptLinkButton'
        self._externalFrameID = 'documentFrame000hb'
        self._internalFrameID = 'ceframe'
        self._containerDivClassPattern = re.compile("adi4g")
        self._meterDataClassPattern = re.compile("fci4g")

        self._driver = webdriver.Firefox()
        self._driver.implicitly_wait(self._initialTimeDelay) # seconds
        self._driver.get(self._url)
        # self._wait for loading form
        self._wait = WebDriverWait(self._driver, self._initialTimeDelay)
        self._iterator = 0;

    def startEngine(self):

        # Get anchor link for report form
        reportLink = self._wait.until(EC.element_to_be_clickable((By.ID,self._reportLinkID)))
        # Click report link
        reportLink.send_keys(Keys.RETURN)
        time.sleep(self._timeDelay)

        countyTag = self._wait.until(EC.presence_of_element_located((By.ID,self._countySelectID)))
        countySelectTag = Select(countyTag)
        countySelectTag.select_by_value(self._countyName)

        meterTag = self._wait.until(EC.presence_of_element_located((By.ID,self._meterSelectID)))
        print "Meter Tag: " + str(type(meterTag))
        print "Meter Tag Name" + meterTag
        meterSelectTag = Select(meterTag)
        meterSelectTag.select_by_value('All')
        time.sleep(self._timeDelay)

        if self._iterator < 1:
            startDateInputField = self._driver.find_element_by_id(self._startDateID)
            startDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

            endDateInputField = self._driver.find_element_by_id(self._endDateID)
            endDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

        previewReportButton = self._driver.find_element_by_id(self._previewReportButtonID)
        previewReportButton.send_keys(Keys.RETURN)
        #time.sleep(self._finalTimeDelay)

        currentWindow = self._driver.current_window_handle
        popupWindow = None
        windows = self._driver.window_handles

        while len(windows) < 2:
            print "==Windows: " + str(len(windows))
            windows = self._driver.window_handles

        for window in windows:
            if currentWindow != window:
                popupWindow = window
        self._driver.switch_to.window(popupWindow)

        externalFrame = self._driver.find_element_by_name(self._externalFrameID)
        self._driver.switch_to.frame(externalFrame)
        internalFrame = self._driver.find_element_by_name(self._internalFrameID)
        self._driver.switch_to.frame(internalFrame)

        soup = BeautifulSoup(self._driver.page_source)
        self._driver.close()
        self._driver.switch_to.window(self._driver.window_handles[0])
        containerDivs = soup.findAll('div', {"class": self._containerDivClassPattern})

        actualSpanIndex = None
        iterator = 0
        for div in containerDivs:
            try:
                title = div["title"]
            except KeyError:
                title = ""
            if (title == "Visit the Support Site for samples, web forums, tutorials, technical briefs and more!"):
                actualSpanIndex = iterator - 1
            iterator = iterator + 1

        newSoup = BeautifulSoup(str(containerDivs[actualSpanIndex]))
        span = newSoup.find('span', {"class": self._meterDataClassPattern})
        self._iterator = self._iterator + 1
        return str(span.text.encode('utf-8'))

def main():
    fileName = 'meterData.txt'
    yesterday = date.today() - timedelta(days=1)
    dateString = str(yesterday.month) +"/"+ str(yesterday.day) + "/" + str(yesterday.year)
    scrapper = WaterScrapper(dateString, 'B')

    appender = open(fileName, 'a')
    reader = open('CountyNames.txt', 'r')

    if reader.mode == 'r' and appender.mode == 'a':
        competeListing = reader.readlines()
        for line in competeListing:
            stringList = line.split(',')
            key = stringList[0]
            valueList = stringList[1].split('\n')
            value = valueList[0]
            scrapper._countyName = key

            data = scrapper.startEngine()
            formattedLine = dateString + ", " + value + ", "  + data + "\n"
            print formattedLine
            appender.write(formattedLine)

    appender.close()
    reader.close()

if __name__ == "__main__": main()

<commit_msg>Check Exception<commit_after>from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.remote.webelement import  WebElement
import time
from bs4 import BeautifulSoup
import re
from datetime import date, timedelta

class WaterScrapper:
    def __init__(self, date, countyName):
        self._date = date
        self._countyName = countyName
        self._timeDelay = 1
        self._initialTimeDelay = 20
        self._finalTimeDelay = 20
        self._url = "https://wins.mwdsc.org/Unsecured/login.aspx?Wamiuser=Wamiuser"
        self._reportLinkID = 'ctl00_ContentPlaceHolder1_gridReports_ctl05_ReportNameLinkButton'
        self._countySelectID = 'ctl00_ContentPlaceHolder1_ddlMember'
        self._meterSelectID = 'ctl00_ContentPlaceHolder1_ddlMeter'
        self._startDateID = 'ctl00_ContentPlaceHolder1_txtStart'
        self._endDateID = 'ctl00_ContentPlaceHolder1_txtEnd'
        self._previewReportButtonID = 'ctl00_ContentPlaceHolder1_PreviewRptLinkButton'
        self._externalFrameID = 'documentFrame000hb'
        self._internalFrameID = 'ceframe'
        self._containerDivClassPattern = re.compile("adi4g")
        self._meterDataClassPattern = re.compile("fci4g")

        self._driver = webdriver.Firefox()
        self._driver.implicitly_wait(self._initialTimeDelay) # seconds
        self._driver.get(self._url)
        # self._wait for loading form
        self._wait = WebDriverWait(self._driver, self._initialTimeDelay)
        self._iterator = 0;

    def startEngine(self):

        # Get anchor link for report form
        reportLink = self._wait.until(EC.element_to_be_clickable((By.ID,self._reportLinkID)))
        # Click report link
        reportLink.send_keys(Keys.RETURN)
        time.sleep(self._timeDelay)

        countyTag = self._wait.until(EC.presence_of_element_located((By.ID,self._countySelectID)))
        countySelectTag = Select(countyTag)
        countySelectTag.select_by_value(self._countyName)

        time.sleep(self._timeDelay)
        meterTag = self._wait.until(EC.presence_of_element_located((By.ID,self._meterSelectID)))
        meterSelectTag = Select(meterTag)
        meterSelectTag.select_by_value('All')
        time.sleep(self._timeDelay)

        if self._iterator < 1:
            startDateInputField = self._driver.find_element_by_id(self._startDateID)
            startDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

            endDateInputField = self._driver.find_element_by_id(self._endDateID)
            endDateInputField.send_keys(self._date)
            time.sleep(self._timeDelay)

        previewReportButton = self._driver.find_element_by_id(self._previewReportButtonID)
        previewReportButton.send_keys(Keys.RETURN)
        #time.sleep(self._finalTimeDelay)

        currentWindow = self._driver.current_window_handle
        popupWindow = None
        windows = self._driver.window_handles

        while len(windows) < 2:
            print "==Windows: " + str(len(windows))
            windows = self._driver.window_handles

        for window in windows:
            if currentWindow != window:
                popupWindow = window
        self._driver.switch_to.window(popupWindow)

        externalFrame = self._driver.find_element_by_name(self._externalFrameID)
        self._driver.switch_to.frame(externalFrame)
        internalFrame = self._driver.find_element_by_name(self._internalFrameID)
        self._driver.switch_to.frame(internalFrame)
        cridReportPage = self._wait.until(EC.presence_of_element_located((By.ID,'cridreportpage')))

        soup = BeautifulSoup(self._driver.page_source)
        self._driver.close()
        self._driver.switch_to.window(self._driver.window_handles[0])
        containerDivs = soup.findAll('div', {"class": self._containerDivClassPattern})

        actualSpanIndex = None
        iterator = 0
        for div in containerDivs:
            try:
                title = div["title"]
            except KeyError:
                title = ""
            if (title == "Visit the Support Site for samples, web forums, tutorials, technical briefs and more!"):
                actualSpanIndex = iterator - 1
            iterator = iterator + 1
        if actualSpanIndex == None:
            print "Error: "
            print soup.prettify()
        newSoup = BeautifulSoup(str(containerDivs[actualSpanIndex]))
        span = newSoup.find('span', {"class": self._meterDataClassPattern})
        self._iterator = self._iterator + 1
        return str(span.text.encode('utf-8'))

def main():
    fileName = 'meterData.txt'
    yesterday = date.today() - timedelta(days=1)
    dateString = str(yesterday.month) +"/"+ str(yesterday.day) + "/" + str(yesterday.year)
    scrapper = WaterScrapper(dateString, 'B')

    appender = open(fileName, 'a')
    reader = open('CountyNames.txt', 'r')

    if reader.mode == 'r' and appender.mode == 'a':
        competeListing = reader.readlines()
        for line in competeListing:
            stringList = line.split(',')
            key = stringList[0]
            valueList = stringList[1].split('\n')
            value = valueList[0]
            scrapper._countyName = key

            data = scrapper.startEngine()
            formattedLine = dateString + ", " + value + ", "  + data + "\n"
            print formattedLine
            appender.write(formattedLine)

    appender.close()
    reader.close()

if __name__ == "__main__": main()

<|endoftext|>