<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
import json
from frappe import _
import frappe.desk.query_report
from frappe.utils import cint
from frappe.model.document import Document
from frappe.modules.export_file import export_to_files
from frappe.modules import make_boilerplate
from frappe.core.doctype.page.page import delete_custom_role
from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles
from six import iteritems

class Report(Document):
	def validate(self):
		"""only administrator can save standard report"""
		if not self.module:
			self.module = frappe.db.get_value("DocType", self.ref_doctype, "module")

		if not self.is_standard:
			self.is_standard = "No"
			if frappe.session.user=="Administrator" and getattr(frappe.local.conf, 'developer_mode',0)==1:
				self.is_standard = "Yes"

		if self.is_standard == "No" and frappe.db.get_value("Report", self.name, "is_standard") == "Yes":
			frappe.throw(_("Cannot edit a standard report. Please duplicate and create a new report"))

		if self.is_standard == "Yes" and frappe.session.user!="Administrator":
			frappe.throw(_("Only Administrator can save a standard report. Please rename and save."))

		if self.report_type in ("Query Report", "Script Report") \
			and frappe.session.user!="Administrator":
			frappe.throw(_("Only Administrator allowed to create Query / Script Reports"))

		if self.report_type == "Report Builder":
			self.update_report_json()

	def before_insert(self):
		self.set_doctype_roles()

	def on_update(self):
		self.export_doc()

	def on_trash(self):
		delete_custom_role('report', self.name)

	def set_doctype_roles(self):
		if not self.get('roles'):
			meta = frappe.get_meta(self.ref_doctype)
			roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
			self.set('roles', roles)

	def is_permitted(self):
		"""Returns true if Has Role is not set or the user is allowed."""
		from frappe.utils import has_common

		allowed = [d.role for d in frappe.get_all("Has Role", fields=["role"],
			filters={"parent": self.name})]

		custom_roles = get_custom_allowed_roles('report', self.name)
		allowed.extend(custom_roles)

		if not allowed:
			return True

		roles = frappe.get_roles()

		if has_common(roles, allowed):
			return True

	def update_report_json(self):
		if self.json:
			data = json.loads(self.json)
			data["add_total_row"] = self.add_total_row
			self.json = json.dumps(data)

	def export_doc(self):
		if frappe.flags.in_import:
			return

		if self.is_standard == 'Yes' and (frappe.local.conf.get('developer_mode') or 0) == 1:
			export_to_files(record_list=[['Report', self.name]],
				record_module=self.module)

			self.create_report_py()

	def create_report_py(self):
		if self.report_type == "Script Report":
			make_boilerplate("controller.py", self, {"name": self.name})
			make_boilerplate("controller.js", self, {"name": self.name})

	def get_data(self, filters=None, limit=None, user=None, as_dict=False):
		columns = []
		out = []

		if self.report_type in ('Query Report', 'Script Report'):
			# query and script reports
			data = frappe.desk.query_report.run(self.name, filters=filters, user=user)
			for d in data.get('columns'):
				if isinstance(d, dict):
					col = frappe._dict(d)
					if not col.fieldname:
						col.fieldname = col.label
					columns.append(col)
				else:
					fieldtype, options = "Data", None
					parts = d.split(':')
					if len(parts) > 1:
						if parts[1]:
							fieldtype, options = parts[1], None
							if fieldtype and '/' in fieldtype:
								fieldtype, options = fieldtype.split('/')

					columns.append(frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0]))

			out += data.get('result')
		else:
			# standard report
			params = json.loads(self.json)
			columns = params.get('columns')
			_filters = params.get('filters') or []

			if filters:
				for key, value in iteritems(filters):
					condition, _value = '=', value
					if isinstance(value, (list, tuple)):
						condition, _value = value
					_filters.append([key, condition, _value])

			def _format(parts):
				# sort by is saved as DocType.fieldname, covert it to sql
				return '`tab{0}`.`{1}`'.format(*parts)

			order_by = _format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order')
			if params.get('sort_by_next'):
				order_by += ', ' + _format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')

			result = frappe.get_list(self.ref_doctype,
				fields = [_format([c[1], c[0]]) for c in columns],
				filters=_filters,
				order_by = order_by,
				as_list=True,
				limit=limit,
				user=user)

			meta = frappe.get_meta(self.ref_doctype)

			columns = [meta.get_field(c[0]) or frappe._dict(label=meta.get_label(c[0]), fieldname=c[0])
				for c in columns]

			out = out + [list(d) for d in result]

		if as_dict:
			data = []
			for row in out:
				_row = frappe._dict()
				data.append(_row)
				for i, val in enumerate(row):
					_row[columns[i].get('fieldname')] = val
		else:
			data = out
		return columns, data

	@Document.whitelist
	def toggle_disable(self, disable):
		self.db_set("disabled", cint(disable))
<commit_msg>[enahnce] Fetch doctype's roles in report if developer mode is on or is standard is no (#3591)<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import frappe
import json
from frappe import _
import frappe.desk.query_report
from frappe.utils import cint
from frappe.model.document import Document
from frappe.modules.export_file import export_to_files
from frappe.modules import make_boilerplate
from frappe.core.doctype.page.page import delete_custom_role
from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles
from six import iteritems

class Report(Document):
	def validate(self):
		"""only administrator can save standard report"""
		if not self.module:
			self.module = frappe.db.get_value("DocType", self.ref_doctype, "module")

		if not self.is_standard:
			self.is_standard = "No"
			if frappe.session.user=="Administrator" and getattr(frappe.local.conf, 'developer_mode',0)==1:
				self.is_standard = "Yes"

		if self.is_standard == "No" and frappe.db.get_value("Report", self.name, "is_standard") == "Yes":
			frappe.throw(_("Cannot edit a standard report. Please duplicate and create a new report"))

		if self.is_standard == "Yes" and frappe.session.user!="Administrator":
			frappe.throw(_("Only Administrator can save a standard report. Please rename and save."))

		if self.report_type in ("Query Report", "Script Report") \
			and frappe.session.user!="Administrator":
			frappe.throw(_("Only Administrator allowed to create Query / Script Reports"))

		if self.report_type == "Report Builder":
			self.update_report_json()

	def before_insert(self):
		self.set_doctype_roles()

	def on_update(self):
		self.export_doc()

	def on_trash(self):
		delete_custom_role('report', self.name)

	def set_doctype_roles(self):
		if not self.get('roles') and self.is_standard == 'No':
			meta = frappe.get_meta(self.ref_doctype)
			roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
			self.set('roles', roles)

	def is_permitted(self):
		"""Returns true if Has Role is not set or the user is allowed."""
		from frappe.utils import has_common

		allowed = [d.role for d in frappe.get_all("Has Role", fields=["role"],
			filters={"parent": self.name})]

		custom_roles = get_custom_allowed_roles('report', self.name)
		allowed.extend(custom_roles)

		if not allowed:
			return True

		roles = frappe.get_roles()

		if has_common(roles, allowed):
			return True

	def update_report_json(self):
		if self.json:
			data = json.loads(self.json)
			data["add_total_row"] = self.add_total_row
			self.json = json.dumps(data)

	def export_doc(self):
		if frappe.flags.in_import:
			return

		if self.is_standard == 'Yes' and (frappe.local.conf.get('developer_mode') or 0) == 1:
			export_to_files(record_list=[['Report', self.name]],
				record_module=self.module)

			self.create_report_py()

	def create_report_py(self):
		if self.report_type == "Script Report":
			make_boilerplate("controller.py", self, {"name": self.name})
			make_boilerplate("controller.js", self, {"name": self.name})

	def get_data(self, filters=None, limit=None, user=None, as_dict=False):
		columns = []
		out = []

		if self.report_type in ('Query Report', 'Script Report'):
			# query and script reports
			data = frappe.desk.query_report.run(self.name, filters=filters, user=user)
			for d in data.get('columns'):
				if isinstance(d, dict):
					col = frappe._dict(d)
					if not col.fieldname:
						col.fieldname = col.label
					columns.append(col)
				else:
					fieldtype, options = "Data", None
					parts = d.split(':')
					if len(parts) > 1:
						if parts[1]:
							fieldtype, options = parts[1], None
							if fieldtype and '/' in fieldtype:
								fieldtype, options = fieldtype.split('/')

					columns.append(frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0]))

			out += data.get('result')
		else:
			# standard report
			params = json.loads(self.json)
			columns = params.get('columns')
			_filters = params.get('filters') or []

			if filters:
				for key, value in iteritems(filters):
					condition, _value = '=', value
					if isinstance(value, (list, tuple)):
						condition, _value = value
					_filters.append([key, condition, _value])

			def _format(parts):
				# sort by is saved as DocType.fieldname, covert it to sql
				return '`tab{0}`.`{1}`'.format(*parts)

			order_by = _format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order')
			if params.get('sort_by_next'):
				order_by += ', ' + _format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next')

			result = frappe.get_list(self.ref_doctype,
				fields = [_format([c[1], c[0]]) for c in columns],
				filters=_filters,
				order_by = order_by,
				as_list=True,
				limit=limit,
				user=user)

			meta = frappe.get_meta(self.ref_doctype)

			columns = [meta.get_field(c[0]) or frappe._dict(label=meta.get_label(c[0]), fieldname=c[0])
				for c in columns]

			out = out + [list(d) for d in result]

		if as_dict:
			data = []
			for row in out:
				_row = frappe._dict()
				data.append(_row)
				for i, val in enumerate(row):
					_row[columns[i].get('fieldname')] = val
		else:
			data = out
		return columns, data

	@Document.whitelist
	def toggle_disable(self, disable):
		self.db_set("disabled", cint(disable))
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division
'''
Created on Mar 14, 2012
'''
from ga_optimization_ternary.fitness_evaluators import eval_fitness_simple, eval_fitness_complex
from test.test_descrtut import defaultdict
from collections import OrderedDict
from ga_optimization_ternary.database import MAX_GOOD_LS, GOOD_CANDS_LS

__author__ = "Anubhav Jain"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Anubhav Jain"
__email__ = "ajain@lbl.gov"
__date__ = "Mar 14, 2012"

from pyevolve import G1DList, Mutators, Initializators, GAllele
from pyevolve import GSimpleGA
from pyevolve import Selectors
from pyevolve import Statistics, Crossovers
from fitness_evaluators import FitnessEvaluator
from database import Stats_Database

from scipy.interpolate import interp1d

import multiprocessing
import math

class AllFound():
    ALL_FOUND = False

def AllFoundCriteria(ga_engine):
    """ Terminate the evolution based on the raw stats
    """
    return AllFound.ALL_FOUND

class ParameterSet():

    def __init__(self, crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutation_fnc, mutation_rate, initialization_fnc, popsize, elitism_num, niching_bool):
        self.crossover_fnc = crossover_fnc
        self.fitness_fnc = fitness_fnc
        self.fitness_temp = fitness_temp
        self.popsize = popsize
        self.selection_fnc = selection_fnc
        self.mutation_fnc = mutation_fnc
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.initialization_fnc = initialization_fnc
        self.elitism_num = elitism_num
        self.niching_bool = niching_bool

    def to_dict(self):
        d = OrderedDict()
        d['crossover_fnc'] = self.crossover_fnc.__name__
        d['fitness_fnc'] = self.fitness_fnc.__name__
        d['fitness_temp'] = self.fitness_temp
        d['selection_fnc'] = self.selection_fnc.__name__
        d['mutation_fnc'] = self.mutation_fnc.__name__
        d['mutation_rate'] = self.mutation_rate
        d['tournament_rate'] = self.tournament_rate
        d['initialization_fnc'] = self.initialization_fnc.__name__
        d['popsize'] = self.popsize
        d['elitism_num'] = self.elitism_num
        d['niching'] = self.niching_bool
        return d

    def unique_key(self):
        return '-'.join([str(val) for val in self.to_dict().values()])

class StatTrack():

    def __init__(self, fe, mutation_rate, tournament_rate):
        self._candidates_tried = set()
        self._candidates_good = set()
        self.generation_ncandidates = [0]
        self.generation_ngood = [0]
        self._fitness_evaluator = fe
        self.num_breakouts = 0
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate

    def updateStats(self, generation_num, population):
        for i in population:
            cand = self._fitness_evaluator.convert_raw_to_Z((i[0], i[1], i[2]))
            self._candidates_tried.add(cand)

            if cand in GOOD_CANDS_LS:
                self._candidates_good.add(cand)

        self.generation_ncandidates.append(len(self._candidates_tried))
        self.generation_ngood.append(len(self._candidates_good))
        if len(self._candidates_good) == MAX_GOOD_LS:
            AllFound.ALL_FOUND = True

        return self.generation_ncandidates[-1] - self.generation_ncandidates[-2]

    def evolve_callback(self, ga):
        ga.getPopulation().setParams(tournamentPool=(int)(math.ceil(self.tournament_rate * len(ga.getPopulation()))))
        cands_added = self.updateStats(ga.currentGeneration, ga.getPopulation().internalPop)

        breakout_cutoff = (int)(math.ceil(0.2 * len(ga.getPopulation().internalPop)))
        #breakout_cutoff = 10
        if cands_added < breakout_cutoff:
            ga.setMutationRate(0.75)
        else:
            ga.setMutationRate(self.mutation_rate)
            self.num_breakouts += 1

        #if ga.currentGeneration == 0:
            '''
            print 'YAY'
            pop = ga.getPopulation()
            for p in pop:
                print p.__class__
            # CHANGE THE ENTIRE POPULATION "pop"
            #pop[i].genomeList = [random.random(), 0 for i in xrange(chromosome/2)]
            #newPop.evaluate()
            #newPop.sort()
            '''
        return False

    def get_interpolation_function(self):
        return interp1d(self.generation_ncandidates, self.generation_ngood)

def run_simulation(pset, max_generations):
    # Genome instance
    setOfAlleles = GAllele.GAlleles()
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))
    setOfAlleles.add(GAllele.GAlleleRange(0, 6))
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))

    # Genome instance, 1D List of 50 elements
    genome = G1DList.G1DList(3)
    genome.setParams(allele=setOfAlleles)

    #Fitness function

    fe = FitnessEvaluator(pset.fitness_fnc, pset.fitness_temp)

    st = StatTrack(fe, pset.mutation_rate, pset.tournament_rate)

    genome.crossover.set(pset.crossover_fnc)
    genome.mutator.set(pset.mutation_fnc)
    genome.evaluator.set(fe.array_to_score)
    genome.initializator.set(pset.initialization_fnc)

    ga = GSimpleGA.GSimpleGA(genome)

    ga.selector.set(pset.selection_fnc)
    ga.terminationCriteria.set(AllFoundCriteria)
    ga.stepCallback.set(st.evolve_callback)
    ga.setPopulationSize(pset.popsize)
    ga.setGenerations(max_generations)

    if pset.elitism_num > 0:
        ga.setElitism(True)
        ga.setElitismReplacement((int)(math.ceil(pset.elitism_num * pset.popsize)))
    else:
        ga.setElitism(False)

    ga.setMutationRate(pset.mutation_rate)

    # TODO: figure out niching
    ga.evolve()

    return st

def main_loop():
    ncores = 7
    clear = False
    # clear the Stats DB
    db = Stats_Database(clear=clear)
    popsizes = [16, 100, 500, 1000]  #TODO: add 250 and 2000 later, if it is needed based on the data
    fitness_fncs = [eval_fitness_simple, eval_fitness_complex]
    fitness_temps = [5, 10, 30]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform, Crossovers.G1DListCrossoverSinglePoint, Crossovers.G1DListCrossoverTwoPoint]
    selection_fncs = [Selectors.GTournamentSelector, Selectors.GRouletteWheel, Selectors.GUniformSelector]  # TODO: Rank selector is SLOW, add it later.  #TODO: add alternative Tournament selector, see if it's as good as regular tournament selector
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.1, 0.25, 0.5]
    mutation_rates = [0.01, 0.05, 0.1]
    elitisms = [0, 0.05, 0.25]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # TODO: add data-mined initializors

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for m_rate in mutation_rates:
                                        for fitness_temp in fitness_temps:
                                            for tournament_rate in tournament_rates:
                                                if (tournament_rate == tournament_rates[0] or selection_fnc == selection_fncs[0]):
                                                    all_ps.append(ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching))  # set up the parameters

    print 'the number of parameter sets is', len(all_ps)
    process_parallel(all_ps, ncores)
    # process_serial(all_ps)

def process_parameterset(ps):
    num_iterations = 15
    production = True
    max_generations = 20000  # should always work...(hopefully)

    if production:
        db = Stats_Database(clear=False)
        if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= num_iterations:
            db._stats_raw.remove({"unique_key": ps.unique_key()}, safe=True)
            for iteration in range(num_iterations):
                AllFound.ALL_FOUND = False  # reset the simulation
                stat = run_simulation(ps, max_generations)
                print stat.generation_ncandidates[-1], len(stat.generation_ngood), stat.num_breakouts, ps.to_dict(), ps.unique_key()
                if production:
                    db.add_stat_raw(ps, stat, iteration)
        else:
            print 'SKIPPING KEY' + ps.unique_key()
        # stats.append(stat)

    #if production:
    #    db.add_stats_raw(ps, stats)

    return 0

def process_parallel(all_ps, ncores):
    pool = multiprocessing.Pool(ncores)
    states = pool.map(process_parameterset, all_ps)
    state = 0 if all([s == 0 for s in states]) else -1
    print "FINISHED with state", state

def process_serial(all_ps):
    for ps in all_ps:
        process_parameterset(ps)

if __name__ == "__main__":
    main_loop()
<commit_msg>add alternate Tournament Selector as well as GRankSelector to testing<commit_after>#!/usr/bin/env python

from __future__ import division
'''
Created on Mar 14, 2012
'''
from ga_optimization_ternary.fitness_evaluators import eval_fitness_simple, eval_fitness_complex
from test.test_descrtut import defaultdict
from collections import OrderedDict
from ga_optimization_ternary.database import MAX_GOOD_LS, GOOD_CANDS_LS

__author__ = "Anubhav Jain"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Anubhav Jain"
__email__ = "ajain@lbl.gov"
__date__ = "Mar 14, 2012"

from pyevolve import G1DList, Mutators, Initializators, GAllele
from pyevolve import GSimpleGA
from pyevolve import Selectors
from pyevolve import Statistics, Crossovers
from fitness_evaluators import FitnessEvaluator
from database import Stats_Database

from scipy.interpolate import interp1d

import multiprocessing
import math

class AllFound():
    ALL_FOUND = False

def AllFoundCriteria(ga_engine):
    """ Terminate the evolution based on the raw stats
    """
    return AllFound.ALL_FOUND

class ParameterSet():

    def __init__(self, crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutation_fnc, mutation_rate, initialization_fnc, popsize, elitism_num, niching_bool):
        self.crossover_fnc = crossover_fnc
        self.fitness_fnc = fitness_fnc
        self.fitness_temp = fitness_temp
        self.popsize = popsize
        self.selection_fnc = selection_fnc
        self.mutation_fnc = mutation_fnc
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.initialization_fnc = initialization_fnc
        self.elitism_num = elitism_num
        self.niching_bool = niching_bool

    def to_dict(self):
        d = OrderedDict()
        d['crossover_fnc'] = self.crossover_fnc.__name__
        d['fitness_fnc'] = self.fitness_fnc.__name__
        d['fitness_temp'] = self.fitness_temp
        d['selection_fnc'] = self.selection_fnc.__name__
        d['mutation_fnc'] = self.mutation_fnc.__name__
        d['mutation_rate'] = self.mutation_rate
        d['tournament_rate'] = self.tournament_rate
        d['initialization_fnc'] = self.initialization_fnc.__name__
        d['popsize'] = self.popsize
        d['elitism_num'] = self.elitism_num
        d['niching'] = self.niching_bool
        return d

    def unique_key(self):
        return '-'.join([str(val) for val in self.to_dict().values()])

class StatTrack():

    def __init__(self, fe, mutation_rate, tournament_rate):
        self._candidates_tried = set()
        self._candidates_good = set()
        self.generation_ncandidates = [0]
        self.generation_ngood = [0]
        self._fitness_evaluator = fe
        self.num_breakouts = 0
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate

    def updateStats(self, generation_num, population):
        for i in population:
            cand = self._fitness_evaluator.convert_raw_to_Z((i[0], i[1], i[2]))
            self._candidates_tried.add(cand)

            if cand in GOOD_CANDS_LS:
                self._candidates_good.add(cand)

        self.generation_ncandidates.append(len(self._candidates_tried))
        self.generation_ngood.append(len(self._candidates_good))
        if len(self._candidates_good) == MAX_GOOD_LS:
            AllFound.ALL_FOUND = True

        return self.generation_ncandidates[-1] - self.generation_ncandidates[-2]

    def evolve_callback(self, ga):
        ga.getPopulation().setParams(tournamentPool=(int)(math.ceil(self.tournament_rate * len(ga.getPopulation()))))
        cands_added = self.updateStats(ga.currentGeneration, ga.getPopulation().internalPop)

        breakout_cutoff = (int)(math.ceil(0.2 * len(ga.getPopulation().internalPop)))
        #breakout_cutoff = 10
        if cands_added < breakout_cutoff:
            ga.setMutationRate(0.75)
        else:
            ga.setMutationRate(self.mutation_rate)
            self.num_breakouts += 1

        #if ga.currentGeneration == 0:
            '''
            print 'YAY'
            pop = ga.getPopulation()
            for p in pop:
                print p.__class__
            # CHANGE THE ENTIRE POPULATION "pop"
            #pop[i].genomeList = [random.random(), 0 for i in xrange(chromosome/2)]
            #newPop.evaluate()
            #newPop.sort()
            '''
        return False

    def get_interpolation_function(self):
        return interp1d(self.generation_ncandidates, self.generation_ngood)

def run_simulation(pset, max_generations):
    # Genome instance
    setOfAlleles = GAllele.GAlleles()
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))
    setOfAlleles.add(GAllele.GAlleleRange(0, 6))
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))

    # Genome instance, 1D List of 50 elements
    genome = G1DList.G1DList(3)
    genome.setParams(allele=setOfAlleles)

    #Fitness function

    fe = FitnessEvaluator(pset.fitness_fnc, pset.fitness_temp)

    st = StatTrack(fe, pset.mutation_rate, pset.tournament_rate)

    genome.crossover.set(pset.crossover_fnc)
    genome.mutator.set(pset.mutation_fnc)
    genome.evaluator.set(fe.array_to_score)
    genome.initializator.set(pset.initialization_fnc)

    ga = GSimpleGA.GSimpleGA(genome)

    ga.selector.set(pset.selection_fnc)
    ga.terminationCriteria.set(AllFoundCriteria)
    ga.stepCallback.set(st.evolve_callback)
    ga.setPopulationSize(pset.popsize)
    ga.setGenerations(max_generations)

    if pset.elitism_num > 0:
        ga.setElitism(True)
        ga.setElitismReplacement((int)(math.ceil(pset.elitism_num * pset.popsize)))
    else:
        ga.setElitism(False)

    ga.setMutationRate(pset.mutation_rate)

    # TODO: figure out niching
    ga.evolve()

    return st

def main_loop():
    ncores = 7
    clear = False
    # clear the Stats DB
    db = Stats_Database(clear=clear)
    popsizes = [16, 100, 500, 1000]  #TODO: add 250 and 2000 later, if it is needed based on the data
    fitness_fncs = [eval_fitness_simple, eval_fitness_complex]
    fitness_temps = [5, 10, 30]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform, Crossovers.G1DListCrossoverSinglePoint, Crossovers.G1DListCrossoverTwoPoint]
    selection_fncs = [Selectors.GTournamentSelector, Selectors.GTournamentSelectorAlternative, Selectors.GRankSelector, Selectors.GRouletteWheel, Selectors.GUniformSelector]  # TODO: Rank selector is SLOW, add it later.  #TODO: add alternative Tournament selector, see if it's as good as regular tournament selector
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.1, 0.25, 0.5]
    mutation_rates = [0.01, 0.05, 0.1]
    elitisms = [0, 0.05, 0.25]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # TODO: add data-mined initializors

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for m_rate in mutation_rates:
                                        for fitness_temp in fitness_temps:
                                            for tournament_rate in tournament_rates:
                                                if (tournament_rate == tournament_rates[0] or selection_fnc == selection_fncs[0] or selection_fnc == selection_fncs[1]):
                                                    all_ps.append(ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching))  # set up the parameters

    print 'the number of parameter sets is', len(all_ps)
    process_parallel(all_ps, ncores)
    # process_serial(all_ps)

def process_parameterset(ps):
    num_iterations = 15
    production = True
    max_generations = 20000  # should always work...(hopefully)

    if production:
        db = Stats_Database(clear=False)
        if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= num_iterations:
            db._stats_raw.remove({"unique_key": ps.unique_key()}, safe=True)
            for iteration in range(num_iterations):
                AllFound.ALL_FOUND = False  # reset the simulation
                stat = run_simulation(ps, max_generations)
                print stat.generation_ncandidates[-1], len(stat.generation_ngood), stat.num_breakouts, ps.to_dict(), ps.unique_key()
                if production:
                    db.add_stat_raw(ps, stat, iteration)
        else:
            print 'SKIPPING KEY' + ps.unique_key()
        # stats.append(stat)

    #if production:
    #    db.add_stats_raw(ps, stats)

    return 0

def process_parallel(all_ps, ncores):
    pool = multiprocessing.Pool(ncores)
    states = pool.map(process_parameterset, all_ps)
    state = 0 if all([s == 0 for s in states]) else -1
    print "FINISHED with state", state

def process_serial(all_ps):
    for ps in all_ps:
        process_parameterset(ps)

if __name__ == "__main__":
    main_loop()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division
'''
Created on Mar 14, 2012
'''

"""
import logging
logging.basicConfig(level=logging.DEBUG)
logging.warning('Logging enabled')
"""

from ga_optimization_ternary.fitness_evaluators import eval_fitness_simple, eval_fitness_complex,\
    eval_fitness_simple_exclusion, eval_fitness_complex_exclusion
from collections import OrderedDict
from ga_optimization_ternary.database import MAX_GOOD_LS, GOOD_CANDS_LS,\
    InitializationDB
from ga_optimization_ternary.ranked_list_optimization import get_ranked_list_goldschmidt_halffill,\
    get_excluded_list

__author__ = "Anubhav Jain"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Anubhav Jain"
__email__ = "ajain@lbl.gov"
__date__ = "Mar 14, 2012"

from pyevolve import G1DList, Mutators, Initializators, GAllele, Consts
from pyevolve import GSimpleGA
from pyevolve import Selectors
from pyevolve import Statistics, Crossovers
from fitness_evaluators import FitnessEvaluator
from database import Stats_Database

from scipy.interpolate import interp1d

import multiprocessing
import math
import copy

NUM_ITERATIONS = 20

"""
def transform_list(my_list):
    fe = FitnessEvaluator(None, None)
    t1 = [[a[0], a[2], a[1]] for a in my_list]
    t2 = [fe.convert_Z_to_raw(x) for x in t1]
    return [[a[0], a[2], a[1]] for a in t2]
"""

class AllFound():
    ALL_FOUND = False
    # initialization_dict = {"goldschmidt_halffill": transform_list(get_ranked_list_goldschmidt_halffill()), "awesome_list":transform_list(GOOD_CANDS_LS)}

def AllFoundCriteria(ga_engine):
    """ Terminate the evolution based on the raw stats
    """
    return AllFound.ALL_FOUND

class ParameterSet():

    def __init__(self, crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutation_fnc, mutation_rate, initialization_fnc, popsize, elitism_num, niching_bool, initialization, include_ridiculous=True):
        self.crossover_fnc = crossover_fnc
        self.fitness_fnc = fitness_fnc
        self.fitness_temp = fitness_temp
        self.popsize = popsize
        self.selection_fnc = selection_fnc
        self.mutation_fnc = mutation_fnc
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.initialization = initialization
        self.initialization_fnc = initialization_fnc
        self.elitism_num = elitism_num
        self.niching_bool = niching_bool
        self.include_ridiculous = include_ridiculous

    def to_dict(self):
        d = OrderedDict()
        d['crossover_fnc'] = self.crossover_fnc.__name__
        d['fitness_fnc'] = self.fitness_fnc.__name__
        d['fitness_temp'] = self.fitness_temp
        d['selection_fnc'] = self.selection_fnc.__name__
        d['mutation_fnc'] = self.mutation_fnc.__name__
        d['mutation_rate'] = self.mutation_rate
        d['tournament_rate'] = self.tournament_rate
        d['selection_overall'] = self.selection_fnc.__name__ + "-" + str(self.tournament_rate) + "-" + str(self.fitness_temp)
        d['initialization_fnc'] = self.initialization_fnc.__name__
        d['initialization'] = self.initialization
        d['popsize'] = self.popsize
        d['elitism_num'] = self.elitism_num
        d['niching'] = self.niching_bool
        d['include_ridiculous'] = self.include_ridiculous
        return d

    def unique_key(self):
        return '-'.join([str(val) for val in self.to_dict().values()])

class StatTrack():

    def __init__(self, fe, mutation_rate, tournament_rate, include_ridiculous=True):
        self._candidates_tried = set()
        self._candidates_tried_all = set()
        self._candidates_good = set()
        self.generation_ncandidates = [0]
        self.generation_ncandidates_all = [0]
        self.generation_ngood = [0]
        self._fitness_evaluator = fe
        self.num_breakouts = 0
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.include_ridiculous = include_ridiculous
        self.excluded_list = get_excluded_list()

    def updateStats(self, generation_num, population):
        for i in population:
            cand = self._fitness_evaluator.convert_raw_to_Z((i[0], i[1], i[2]))
            if self.include_ridiculous or cand not in self.excluded_list:
                self._candidates_tried.add(cand)

            self._candidates_tried_all.add(cand)

            if cand in GOOD_CANDS_LS:
                self._candidates_good.add(cand)

        self.generation_ncandidates.append(len(self._candidates_tried))
        self.generation_ncandidates_all.append(len(self._candidates_tried_all))

        self.generation_ngood.append(len(self._candidates_good))

        if len(self._candidates_good) == MAX_GOOD_LS:
            AllFound.ALL_FOUND = True

        return self.generation_ncandidates_all[-1] - self.generation_ncandidates_all[-2]

    def evolve_callback(self, ga):
        ga.getPopulation().setParams(tournamentPool=(int)(math.ceil(self.tournament_rate * len(ga.getPopulation()))))
        cands_added = self.updateStats(ga.currentGeneration, ga.getPopulation().internalPop)
        breakout_cutoff = (int)(math.ceil(0.1 * len(ga.getPopulation().internalPop)))
        if cands_added < breakout_cutoff:
            ga.setMutationRate(1.0)
            ga.setCrossoverRate(1.0)
            self.num_breakouts += 1
        else:
            ga.setMutationRate(self.mutation_rate)
            ga.setCrossoverRate(0.9)

        return False

def run_simulation(pset, max_generations, initial_list=None):
    # Genome instance
    setOfAlleles = GAllele.GAlleles()
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))
    setOfAlleles.add(GAllele.GAlleleRange(0, 6))
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))

    # Genome instance, 1D List of 50 elements
    genome = G1DList.G1DList(3)
    genome.setParams(allele=setOfAlleles)

    #Fitness function

    fe = FitnessEvaluator(pset.fitness_fnc, pset.fitness_temp)
    Consts.CDefScaleLinearMultiplier = pset.fitness_temp

    st = StatTrack(fe, pset.mutation_rate, pset.tournament_rate, include_ridiculous=pset.include_ridiculous)

    genome.crossover.set(pset.crossover_fnc)
    genome.mutator.set(pset.mutation_fnc)
    genome.evaluator.set(fe.array_to_score)
    genome.initializator.set(pset.initialization_fnc)

    ga = GSimpleGA.GSimpleGA(genome)

    ga.selector.set(pset.selection_fnc)
    ga.terminationCriteria.set(AllFoundCriteria)
    ga.stepCallback.set(st.evolve_callback)
    ga.setPopulationSize(pset.popsize)
    ga.setGenerations(max_generations)

    if pset.elitism_num > 0:
        ga.setElitism(True)
        ga.setElitismReplacement((int)(math.ceil(pset.elitism_num * pset.popsize)))
    else:
        ga.setElitism(False)

    ga.setMutationRate(pset.mutation_rate)
    # TODO: figure out niching
    stats_freq = 0
    ga.evolve(freq_stats=stats_freq, initial_list=initial_list)

    return st

def main_loop():
    ncores = 7
    clear = False
    # clear the Stats DB
    db = Stats_Database(clear=clear)
    popsizes = [20, 100, 500, 1000]
    fitness_fncs = [eval_fitness_simple, eval_fitness_complex]
    fitness_temps = [1.25, 2.5, 5, 10]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform, Crossovers.G1DListCrossoverSinglePoint, Crossovers.G1DListCrossoverTwoPoint]
    selection_fncs = [Selectors.GRouletteWheel, Selectors.GTournamentSelectorAlternative, Selectors.GUniformSelector]
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.05, 0.1, 0.25]
    mutation_rates = [0.01, 0.05, 0.1]
    elitisms = [0, 0.1, 0.5, 0.75]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()
    initializations = ["none"]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for initialization in initializations:
                                        for m_rate in mutation_rates:
                                            for fitness_temp in fitness_temps:
                                                #fitness temp only matters for roulette wheel
                                                if (fitness_temp == fitness_temps[0] or selection_fnc.__name__ == "GRouletteWheel"):
                                                    for tournament_rate in tournament_rates:
                                                        if (tournament_rate == tournament_rates[0] or selection_fnc.__name__ == "GTournamentSelectorAlternative"):
                                                                ps = ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching, initialization)
                                                                if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
                                                                    all_ps.append(ps)  # set up the parameters
                                                                else:
                                                                    print "We already have pset:", ps.unique_key()

    print 'the number of parameter sets is', len(all_ps)
    process_parallel(all_ps, ncores)
    #process_serial(all_ps)

def main_loop_exclusions():
    ncores = 4
    clear = True
    # clear the Stats DB
    db = Stats_Database(clear=clear, extension="_exclusion")
    popsizes = [100]
    fitness_fncs = [eval_fitness_complex_exclusion]
    fitness_temps = [5]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform]
    selection_fncs = [Selectors.GRouletteWheel]
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.05]
    mutation_rates = [0.05]
    elitisms = [0.5]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()
    initializations = ["none"]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for initialization in initializations:
                                        for m_rate in mutation_rates:
                                            for fitness_temp in fitness_temps:
                                                #fitness temp only matters for roulette wheel
                                                if (fitness_temp == fitness_temps[0] or selection_fnc.__name__ == "GRouletteWheel"):
                                                    for tournament_rate in tournament_rates:
                                                        if (tournament_rate == tournament_rates[0] or selection_fnc.__name__ == "GTournamentSelectorAlternative"):
                                                                ps = ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching, initialization, include_ridiculous=False)
                                                                if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
                                                                    all_ps.append(ps)  # set up the parameters
                                                                else:
                                                                    print "We already have pset:", ps.unique_key()

    print 'the number of parameter sets is', len(all_ps)
    #process_serial(all_ps)
    process_parallel(all_ps, ncores)

def process_parameterset(ps):
    production = True
    max_generations = 20000  # should always work...(hopefully)
    i_db = InitializationDB()
    iteration = 0
    extension = ""
    if not ps.include_ridiculous:
        extension = "_exclusion"
    if production:
        db = Stats_Database(clear=False, extension=extension)
        if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
            iteration = db._stats_raw.find({"unique_key": ps.unique_key()}).count()
            while iteration < NUM_ITERATIONS:
                AllFound.ALL_FOUND = False  # reset the simulation
                stat = run_simulation(ps, max_generations, initial_list=i_db.get_initial_list(iteration))
                print stat.generation_ncandidates[-1], len(stat.generation_ngood), stat.num_breakouts, ps.to_dict(), ps.unique_key()
                if production:
                    db.add_stat_raw(ps, stat, iteration)
                iteration += 1
        else:
            print 'SKIPPING KEY' + ps.unique_key()

    return 0

def process_parallel(all_ps, ncores):
    pool = multiprocessing.Pool(ncores)
    states = pool.map(process_parameterset, all_ps)
    state = 0 if all([s == 0 for s in states]) else -1
    print "FINISHED with state", state

def process_serial(all_ps):
    for ps in all_ps:
        process_parameterset(ps)

if __name__ == "__main__":
    main_loop_exclusions()
<commit_msg>run the main loop<commit_after>#!/usr/bin/env python

from __future__ import division
'''
Created on Mar 14, 2012
'''

"""
import logging
logging.basicConfig(level=logging.DEBUG)
logging.warning('Logging enabled')
"""

from ga_optimization_ternary.fitness_evaluators import eval_fitness_simple, eval_fitness_complex,\
    eval_fitness_simple_exclusion, eval_fitness_complex_exclusion
from collections import OrderedDict
from ga_optimization_ternary.database import MAX_GOOD_LS, GOOD_CANDS_LS,\
    InitializationDB
from ga_optimization_ternary.ranked_list_optimization import get_ranked_list_goldschmidt_halffill,\
    get_excluded_list

__author__ = "Anubhav Jain"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Anubhav Jain"
__email__ = "ajain@lbl.gov"
__date__ = "Mar 14, 2012"

from pyevolve import G1DList, Mutators, Initializators, GAllele, Consts
from pyevolve import GSimpleGA
from pyevolve import Selectors
from pyevolve import Statistics, Crossovers
from fitness_evaluators import FitnessEvaluator
from database import Stats_Database

from scipy.interpolate import interp1d

import multiprocessing
import math
import copy

NUM_ITERATIONS = 20

"""
def transform_list(my_list):
    fe = FitnessEvaluator(None, None)
    t1 = [[a[0], a[2], a[1]] for a in my_list]
    t2 = [fe.convert_Z_to_raw(x) for x in t1]
    return [[a[0], a[2], a[1]] for a in t2]
"""

class AllFound():
    ALL_FOUND = False
    # initialization_dict = {"goldschmidt_halffill": transform_list(get_ranked_list_goldschmidt_halffill()), "awesome_list":transform_list(GOOD_CANDS_LS)}

def AllFoundCriteria(ga_engine):
    """ Terminate the evolution based on the raw stats
    """
    return AllFound.ALL_FOUND

class ParameterSet():

    def __init__(self, crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutation_fnc, mutation_rate, initialization_fnc, popsize, elitism_num, niching_bool, initialization, include_ridiculous=True):
        self.crossover_fnc = crossover_fnc
        self.fitness_fnc = fitness_fnc
        self.fitness_temp = fitness_temp
        self.popsize = popsize
        self.selection_fnc = selection_fnc
        self.mutation_fnc = mutation_fnc
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.initialization = initialization
        self.initialization_fnc = initialization_fnc
        self.elitism_num = elitism_num
        self.niching_bool = niching_bool
        self.include_ridiculous = include_ridiculous

    def to_dict(self):
        d = OrderedDict()
        d['crossover_fnc'] = self.crossover_fnc.__name__
        d['fitness_fnc'] = self.fitness_fnc.__name__
        d['fitness_temp'] = self.fitness_temp
        d['selection_fnc'] = self.selection_fnc.__name__
        d['mutation_fnc'] = self.mutation_fnc.__name__
        d['mutation_rate'] = self.mutation_rate
        d['tournament_rate'] = self.tournament_rate
        d['selection_overall'] = self.selection_fnc.__name__ + "-" + str(self.tournament_rate) + "-" + str(self.fitness_temp)
        d['initialization_fnc'] = self.initialization_fnc.__name__
        d['initialization'] = self.initialization
        d['popsize'] = self.popsize
        d['elitism_num'] = self.elitism_num
        d['niching'] = self.niching_bool
        d['include_ridiculous'] = self.include_ridiculous
        return d

    def unique_key(self):
        return '-'.join([str(val) for val in self.to_dict().values()])

class StatTrack():

    def __init__(self, fe, mutation_rate, tournament_rate, include_ridiculous=True):
        self._candidates_tried = set()
        self._candidates_tried_all = set()
        self._candidates_good = set()
        self.generation_ncandidates = [0]
        self.generation_ncandidates_all = [0]
        self.generation_ngood = [0]
        self._fitness_evaluator = fe
        self.num_breakouts = 0
        self.mutation_rate = mutation_rate
        self.tournament_rate = tournament_rate
        self.include_ridiculous = include_ridiculous
        self.excluded_list = get_excluded_list()

    def updateStats(self, generation_num, population):
        for i in population:
            cand = self._fitness_evaluator.convert_raw_to_Z((i[0], i[1], i[2]))
            if self.include_ridiculous or cand not in self.excluded_list:
                self._candidates_tried.add(cand)

            self._candidates_tried_all.add(cand)

            if cand in GOOD_CANDS_LS:
                self._candidates_good.add(cand)

        self.generation_ncandidates.append(len(self._candidates_tried))
        self.generation_ncandidates_all.append(len(self._candidates_tried_all))

        self.generation_ngood.append(len(self._candidates_good))

        if len(self._candidates_good) == MAX_GOOD_LS:
            AllFound.ALL_FOUND = True

        return self.generation_ncandidates_all[-1] - self.generation_ncandidates_all[-2]

    def evolve_callback(self, ga):
        ga.getPopulation().setParams(tournamentPool=(int)(math.ceil(self.tournament_rate * len(ga.getPopulation()))))
        cands_added = self.updateStats(ga.currentGeneration, ga.getPopulation().internalPop)
        breakout_cutoff = (int)(math.ceil(0.1 * len(ga.getPopulation().internalPop)))
        if cands_added < breakout_cutoff:
            ga.setMutationRate(1.0)
            ga.setCrossoverRate(1.0)
            self.num_breakouts += 1
        else:
            ga.setMutationRate(self.mutation_rate)
            ga.setCrossoverRate(0.9)

        return False

def run_simulation(pset, max_generations, initial_list=None):
    # Genome instance
    setOfAlleles = GAllele.GAlleles()
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))
    setOfAlleles.add(GAllele.GAlleleRange(0, 6))
    setOfAlleles.add(GAllele.GAlleleRange(0, 51))

    # Genome instance, 1D List of 50 elements
    genome = G1DList.G1DList(3)
    genome.setParams(allele=setOfAlleles)

    #Fitness function

    fe = FitnessEvaluator(pset.fitness_fnc, pset.fitness_temp)
    Consts.CDefScaleLinearMultiplier = pset.fitness_temp

    st = StatTrack(fe, pset.mutation_rate, pset.tournament_rate, include_ridiculous=pset.include_ridiculous)

    genome.crossover.set(pset.crossover_fnc)
    genome.mutator.set(pset.mutation_fnc)
    genome.evaluator.set(fe.array_to_score)
    genome.initializator.set(pset.initialization_fnc)

    ga = GSimpleGA.GSimpleGA(genome)

    ga.selector.set(pset.selection_fnc)
    ga.terminationCriteria.set(AllFoundCriteria)
    ga.stepCallback.set(st.evolve_callback)
    ga.setPopulationSize(pset.popsize)
    ga.setGenerations(max_generations)

    if pset.elitism_num > 0:
        ga.setElitism(True)
        ga.setElitismReplacement((int)(math.ceil(pset.elitism_num * pset.popsize)))
    else:
        ga.setElitism(False)

    ga.setMutationRate(pset.mutation_rate)
    # TODO: figure out niching
    stats_freq = 0
    ga.evolve(freq_stats=stats_freq, initial_list=initial_list)

    return st

def main_loop():
    ncores = 7
    clear = False
    # clear the Stats DB
    db = Stats_Database(clear=clear)
    popsizes = [20, 100, 500, 1000]
    fitness_fncs = [eval_fitness_simple, eval_fitness_complex]
    fitness_temps = [1.25, 2.5, 5, 10]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform, Crossovers.G1DListCrossoverSinglePoint, Crossovers.G1DListCrossoverTwoPoint]
    selection_fncs = [Selectors.GRouletteWheel, Selectors.GTournamentSelectorAlternative, Selectors.GUniformSelector]
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.05, 0.1, 0.25]
    mutation_rates = [0.01, 0.05, 0.1]
    elitisms = [0, 0.1, 0.5, 0.75]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()
    initializations = ["none"]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for initialization in initializations:
                                        for m_rate in mutation_rates:
                                            for fitness_temp in fitness_temps:
                                                #fitness temp only matters for roulette wheel
                                                if (fitness_temp == fitness_temps[0] or selection_fnc.__name__ == "GRouletteWheel"):
                                                    for tournament_rate in tournament_rates:
                                                        if (tournament_rate == tournament_rates[0] or selection_fnc.__name__ == "GTournamentSelectorAlternative"):
                                                                ps = ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching, initialization)
                                                                if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
                                                                    all_ps.append(ps)  # set up the parameters
                                                                else:
                                                                    print "We already have pset:", ps.unique_key()

    print 'the number of parameter sets is', len(all_ps)
    process_parallel(all_ps, ncores)
    #process_serial(all_ps)

def main_loop_exclusions():
    ncores = 4
    clear = True
    # clear the Stats DB
    db = Stats_Database(clear=clear, extension="_exclusion")
    popsizes = [100]
    fitness_fncs = [eval_fitness_complex_exclusion]
    fitness_temps = [5]
    crossover_fncs = [Crossovers.G1DListCrossoverUniform]
    selection_fncs = [Selectors.GRouletteWheel]
    mutator_fncs = [Mutators.G1DListMutatorAllele]
    tournament_rates = [0.05]
    mutation_rates = [0.05]
    elitisms = [0.5]
    nichings = [False]  # TODO: implement True
    initialization_fncs = [Initializators.G1DListInitializatorAllele]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()
    initializations = ["none"]  # as of 10/3/2012 this no longer does anything, the initialization is done by the initializations array instead using evolve_callback()

    all_ps = []
    for popsize in popsizes:
        for fitness_fnc in fitness_fncs:
            for crossover_fnc in crossover_fncs:
                for selection_fnc in selection_fncs:
                    for mutator_fnc in mutator_fncs:
                        for elitism in elitisms:
                            for niching in nichings:
                                for initialization_fnc in initialization_fncs:
                                    for initialization in initializations:
                                        for m_rate in mutation_rates:
                                            for fitness_temp in fitness_temps:
                                                #fitness temp only matters for roulette wheel
                                                if (fitness_temp == fitness_temps[0] or selection_fnc.__name__ == "GRouletteWheel"):
                                                    for tournament_rate in tournament_rates:
                                                        if (tournament_rate == tournament_rates[0] or selection_fnc.__name__ == "GTournamentSelectorAlternative"):
                                                                ps = ParameterSet(crossover_fnc, fitness_fnc, fitness_temp, selection_fnc, tournament_rate, mutator_fnc, m_rate, initialization_fnc, popsize, elitism, niching, initialization, include_ridiculous=False)
                                                                if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
                                                                    all_ps.append(ps)  # set up the parameters
                                                                else:
                                                                    print "We already have pset:", ps.unique_key()

    print 'the number of parameter sets is', len(all_ps)
    #process_serial(all_ps)
    process_parallel(all_ps, ncores)

def process_parameterset(ps):
    production = True
    max_generations = 20000  # should always work...(hopefully)
    i_db = InitializationDB()
    iteration = 0
    extension = ""
    if not ps.include_ridiculous:
        extension = "_exclusion"
    if production:
        db = Stats_Database(clear=False, extension=extension)
        if not db._stats_raw.find({"unique_key": ps.unique_key()}).count() >= NUM_ITERATIONS:
            iteration = db._stats_raw.find({"unique_key": ps.unique_key()}).count()
            while iteration < NUM_ITERATIONS:
                AllFound.ALL_FOUND = False  # reset the simulation
                stat = run_simulation(ps, max_generations, initial_list=i_db.get_initial_list(iteration))
                print stat.generation_ncandidates[-1], len(stat.generation_ngood), stat.num_breakouts, ps.to_dict(), ps.unique_key()
                if production:
                    db.add_stat_raw(ps, stat, iteration)
                iteration += 1
        else:
            print 'SKIPPING KEY' + ps.unique_key()

    return 0

def process_parallel(all_ps, ncores):
    pool = multiprocessing.Pool(ncores)
    states = pool.map(process_parameterset, all_ps)
    state = 0 if all([s == 0 for s in states]) else -1
    print "FINISHED with state", state

def process_serial(all_ps):
    for ps in all_ps:
        process_parameterset(ps)

if __name__ == "__main__":
    main_loop()
<|endoftext|>


<commit_before>import logging
from lxml import etree as ET

from couchdbkit import ResourceNotFound
from django.core.management import BaseCommand

from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain
from corehq.apps.app_manager.models import Application, PreloadAction, CaseReferences
from corehq.apps.app_manager.util import save_xform
from corehq.apps.app_manager.xform import XForm, SESSION_USERCASE_ID, get_add_case_preloads_case_id_xpath
from corehq.toggles import NAMESPACE_DOMAIN, USER_PROPERTY_EASY_REFS

logger = logging.getLogger('app_migration')
logger.setLevel('DEBUG')

class Command(BaseCommand):
    help = '''
        Migrate apps from case management in the app builder to form builder.
        Pass either domain name(s) (to migrate all apps in the domain) or
        individual app id(s). Will skip any apps that have already been
        migrated.
    '''

    def add_arguments(self, parser):
        parser.add_argument('app_id_or_domain', nargs='+',
            help="App ID or domain name. Must be a domain name "
                 "with --usercase option.")
        parser.add_argument('--usercase', action='store_true',
            help='Migrate user properties.')
        parser.add_argument('--force', action='store_true',
            help='Migrate even if app.vellum_case_management is already true.')

    def handle(self, **options):
        domains = []
        app_ids = []
        self.force = options["force"]
        self.migrate_usercase = options["usercase"]
        for ident in options["app_id_or_domain"]:
            if not self.migrate_usercase:
                try:
                    Application.get(ident)
                    app_ids.append(ident)
                    continue
                except ResourceNotFound:
                    pass
            ids = get_app_ids_in_domain(ident)
            app_ids.extend(ids)
            if ids:
                domains.append(ident)
            logger.info('migrating {} apps in domain {}'.format(len(ids), ident))

        for app_id in app_ids:
            logger.info('migrating app {}'.format(app_id))
            self.migrate_app(app_id)

        if self.migrate_usercase:
            for domain in domains:
                USER_PROPERTY_EASY_REFS.set(domain, True, NAMESPACE_DOMAIN)
            logger.info("enabled USER_PROPERTY_EASY_REFS for domains: %s",
                ", ".join(domains))

        logger.info('done with migrate_app_to_cmitfb')

    def migrate_app(self, app_id):
        app = Application.get(app_id)
        migrate_usercase = should_migrate_usercase(app, self.migrate_usercase)
        if app.vellum_case_management and not migrate_usercase and not self.force:
            logger.info('already migrated app {}'.format(app_id))
            return

        modules = [m for m in app.modules if m.module_type == 'basic']
        for module in modules:
            forms = [f for f in module.forms if f.doc_type == 'Form']
            for form in forms:
                preloads = []
                preload = form.actions.case_preload.preload
                if preload:
                    if form.requires == 'case':
                        preloads.append({
                            "hashtag": "#case/",
                            "preloads": preload,
                        })
                    form.actions.case_preload = PreloadAction()
                usercase_preload = form.actions.usercase_preload.preload
                if migrate_usercase and usercase_preload:
                    preloads.append({
                        "hashtag": "#user/",
                        "preloads": usercase_preload,
                        "case_id_xpath": SESSION_USERCASE_ID,
                    })
                    form.actions.usercase_preload = PreloadAction()
                if preloads:
                    migrate_preloads(app, module, form, preloads)

        app.vellum_case_management = True
        app.save()

def migrate_preloads(app, module, form, preloads):
    xform = XForm(form.source)
    case_id_xpath = get_add_case_preloads_case_id_xpath(module, form)
    for kwargs in preloads:
        hashtag = kwargs.pop("hashtag")
        kwargs['case_id_xpath'] = case_id_xpath
        xform.add_case_preloads(**kwargs)
        refs = {path: [hashtag + case_property]
                for path, case_property in kwargs["preloads"].iteritems()}
        if form.case_references:
            form.case_references.load.update(refs)
        else:
            form.case_references = CaseReferences(load=refs)
    save_xform(app, form, ET.tostring(xform.xml))

def should_migrate_usercase(app, migrate_usercase):
    return migrate_usercase and any(form.actions.usercase_preload.preload
        for module in app.modules if module.module_type == 'basic'
        for form in module.forms if form.doc_type == 'Form')
<commit_msg>Migration command tweaks<commit_after>import logging
from collections import defaultdict
from lxml import etree as ET

from couchdbkit import ResourceNotFound
from django.core.management import BaseCommand

from corehq.apps.app_manager.dbaccessors import get_app_ids_in_domain
from corehq.apps.app_manager.models import Application, PreloadAction, CaseReferences
from corehq.apps.app_manager.util import save_xform
from corehq.apps.app_manager.xform import XForm, SESSION_USERCASE_ID, get_add_case_preloads_case_id_xpath
from corehq.toggles import NAMESPACE_DOMAIN, USER_PROPERTY_EASY_REFS

logger = logging.getLogger('app_migration')
logger.setLevel('DEBUG')

class Command(BaseCommand):
    help = '''
        Migrate apps from case management in the app builder to form builder.
        Pass either domain name(s) (to migrate all apps in the domain) or
        individual app id(s). Will skip any apps that have already been
        migrated.
    '''

    def add_arguments(self, parser):
        parser.add_argument('app_id_or_domain', nargs='+',
            help="App ID or domain name. Must be a domain name "
                 "with --usercase option.")
        parser.add_argument('--usercase', action='store_true',
            help='Migrate user properties.')
        parser.add_argument('--force', action='store_true',
            help='Migrate even if app.vellum_case_management is already true.')

    def handle(self, **options):
        app_ids_by_domain = defaultdict(set)
        self.force = options["force"]
        self.migrate_usercase = options["usercase"]
        for ident in options["app_id_or_domain"]:
            if not self.migrate_usercase:
                try:
                    app = Application.get(ident)
                    app_ids_by_domain[app.domain].add(ident)
                    continue
                except ResourceNotFound:
                    pass
            app_ids_by_domain[ident].update(get_app_ids_in_domain(ident))

        for domain, app_ids in sorted(app_ids_by_domain.items()):
            logger.info('migrating %s: %s apps', domain, len(app_ids))
            for app_id in app_ids:
                try:
                    migrated = self.migrate_app(app_id)
                    if migrated:
                        logger.info('migrated app %s', app_id)
                except Exception:
                    logger.exception("skipping app %s", app_id)
            if self.migrate_usercase and not USER_PROPERTY_EASY_REFS.enabled(domain):
                USER_PROPERTY_EASY_REFS.set(domain, True, NAMESPACE_DOMAIN)
                logger.info("enabled USER_PROPERTY_EASY_REFS for domain: %s", domain)

        logger.info('done with migrate_app_to_cmitfb')

    def migrate_app(self, app_id):
        app = Application.get(app_id)
        migrate_usercase = should_migrate_usercase(app, self.migrate_usercase)
        if self.migrate_usercase and not migrate_usercase:
            return False
        if app.vellum_case_management and not migrate_usercase and not self.force:
            logger.info('already migrated app {}'.format(app_id))
            return False

        modules = [m for m in app.modules if m.module_type == 'basic']
        for module in modules:
            forms = [f for f in module.forms if f.doc_type == 'Form']
            for form in forms:
                preloads = []
                preload = form.actions.case_preload.preload
                if preload:
                    if form.requires == 'case':
                        preloads.append({
                            "hashtag": "#case/",
                            "preloads": preload,
                        })
                    form.actions.case_preload = PreloadAction()
                usercase_preload = form.actions.usercase_preload.preload
                if migrate_usercase and usercase_preload:
                    preloads.append({
                        "hashtag": "#user/",
                        "preloads": usercase_preload,
                        "case_id_xpath": SESSION_USERCASE_ID,
                    })
                    form.actions.usercase_preload = PreloadAction()
                if preloads:
                    migrate_preloads(app, module, form, preloads)

        app.vellum_case_management = True
        app.save()
        return True

def migrate_preloads(app, module, form, preloads):
    xform = XForm(form.source)
    case_id_xpath = get_add_case_preloads_case_id_xpath(module, form)
    for kwargs in preloads:
        hashtag = kwargs.pop("hashtag")
        kwargs['case_id_xpath'] = case_id_xpath
        xform.add_case_preloads(**kwargs)
        refs = {path: [hashtag + case_property]
                for path, case_property in kwargs["preloads"].iteritems()}
        if form.case_references:
            form.case_references.load.update(refs)
        else:
            form.case_references = CaseReferences(load=refs)
    save_xform(app, form, ET.tostring(xform.xml))

def should_migrate_usercase(app, migrate_usercase):
    return migrate_usercase and any(form.actions.usercase_preload.preload
        for module in app.modules if module.module_type == 'basic'
        for form in module.forms if form.doc_type == 'Form')
<|endoftext|>


<commit_before>import unittest
from unittest import mock
from pathlib import Path
from tempfile import TemporaryDirectory
from contextlib import contextmanager

from cumulusci.core.exceptions import TaskOptionsError
from cumulusci.tasks.bulkdata.tests.utils import _make_task

import yaml
from sqlalchemy import create_engine

import snowfakery
from cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml import (
    GenerateAndLoadDataFromYaml,
)
from snowfakery import data_generator_runtime

if snowfakery:
    sample_yaml = Path(__file__).parent / "snowfakery/gen_npsp_standard_objects.yml"
    simple_yaml = Path(__file__).parent / "snowfakery/include_parent.yml"
    from cumulusci.tasks.bulkdata.generate_from_yaml import GenerateDataFromYaml

vanilla_mapping_file = Path(__file__).parent / "../tests/mapping_vanilla_sf.yml"

@contextmanager
def temporary_file_path(filename):
    with TemporaryDirectory() as tmpdirname:
        path = Path(tmpdirname) / filename
        yield path

@contextmanager
def temp_sqlite_database_url():
    with temporary_file_path("test.db") as path:
        yield f"sqlite:///{str(path)}"

class TestGenerateFromDataTask(unittest.TestCase):
    def assertRowsCreated(self, database_url):
        engine = create_engine(database_url)
        connection = engine.connect()
        accounts = connection.execute(f"select * from Account")
        accounts = list(accounts)
        assert accounts and accounts[0] and accounts[0][1]
        return accounts

    def test_no_options(self):
        with self.assertRaises(Exception):
            _make_task(GenerateDataFromYaml, {})

    def test_simple(self):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml,
                        "num_records": 1,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()
            self.assertRowsCreated(database_url)

    def test_inaccessible_generator_yaml(self):
        with self.assertRaises(TaskOptionsError):
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml / "junk",
                        "num_records": 10,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()

    def test_vars(self):
        with temp_sqlite_database_url() as database_url:
            with self.assertWarns(UserWarning):
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "vars": "xyzzy:foo",
                            "database_url": database_url,
                        }
                    },
                )
                task()
                self.assertRowsCreated(database_url)

    def test_generate_mapping_file(self):
        with temporary_file_path("mapping.yml") as temp_mapping:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "generate_mapping_file": temp_mapping,
                        }
                    },
                )
                task()
            mapping = yaml.safe_load(open(temp_mapping))
            assert mapping["Insert Account"]["fields"]

    def test_use_mapping_file(self):
        assert vanilla_mapping_file.exists()
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml,
                        "database_url": database_url,
                        "mapping": vanilla_mapping_file,
                    }
                },
            )
            task()
            self.assertRowsCreated(database_url)

    def test_num_records(self):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": simple_yaml,
                        "num_records": 11,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()
            assert len(self.assertRowsCreated(database_url)) == 11

    @mock.patch(
        "cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml._dataload"
    )
    def test_batching(self, _dataload):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateAndLoadDataFromYaml,
                {
                    "options": {
                        "generator_yaml": simple_yaml,
                        "num_records": 14,
                        "batch_size": 6,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                        "data_generation_task": "cumulusci.tasks.bulkdata.generate_from_yaml.GenerateDataFromYaml",
                        "reset_oids": False,
                    }
                },
            )
            task()
            assert len(_dataload.mock_calls) == 3
            task = None  # clean up db?

            engine = create_engine(database_url)
            connection = engine.connect()
            records = list(connection.execute("select * from Account"))
            connection.close()
            assert len(records) == 14 % 6  # leftovers

    def test_mismatched_options(self):
        with self.assertRaises(TaskOptionsError) as e:
            task = _make_task(
                GenerateDataFromYaml,
                {"options": {"generator_yaml": sample_yaml, "num_records": 10}},
            )
            task()
        assert "without num_records_tablename" in str(e.exception)

    def generate_continuation_data(self):
        g = data_generator_runtime.Globals()
        o = data_generator_runtime.ObjectRow(
            "Account", {"Name": "Johnston incorporated", "id": 5}
        )
        g.register_object(o, "The Company")
        for i in range(0, 5):
            # burn through 5 imaginary accounts
            g.id_manager.generate_id("Account")
        return yaml.safe_dump(g)

    def test_with_continuation_file(self):
        continuation_data = self.generate_continuation_data()
        with temp_sqlite_database_url() as database_url:
            with temporary_file_path("cont.yml") as continuation_file_path:
                with open(continuation_file_path, "w") as continuation_file:
                    continuation_file.write(continuation_data)

                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "mapping": vanilla_mapping_file,
                            "continuation_file": continuation_file_path,
                        }
                    },
                )
                task()
                rows = self.assertRowsCreated(database_url)
                assert dict(rows[0])["id"] == 6

    def test_with_nonexistent_continuation_file(self):
        with self.assertRaises(TaskOptionsError) as e:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "mapping": vanilla_mapping_file,
                            "continuation_file": "/tmp/foobar/baz/jazz/continuation.yml",
                        }
                    },
                )
                task()
                rows = self.assertRowsCreated(database_url)
                assert dict(rows[0])["id"] == 6

        assert "jazz" in str(e.exception)
        assert "does not exist" in str(e.exception)

    def test_generate_continuation_file(self):
        from snowfakery import version

        print(version.version)
        with temporary_file_path("cont.yml") as temp_continuation_file:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "generate_continuation_file": temp_continuation_file,
                        }
                    },
                )
                task()
            mapping = yaml.safe_load(open(temp_continuation_file))
            assert mapping  # internals of this file are not important to MetaCI
<commit_msg>Remove another conditional<commit_after>import unittest
from unittest import mock
from pathlib import Path
from tempfile import TemporaryDirectory
from contextlib import contextmanager

from cumulusci.core.exceptions import TaskOptionsError
from cumulusci.tasks.bulkdata.tests.utils import _make_task

import yaml
from sqlalchemy import create_engine

from cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml import (
    GenerateAndLoadDataFromYaml,
)
from snowfakery import data_generator_runtime

sample_yaml = Path(__file__).parent / "snowfakery/gen_npsp_standard_objects.yml"
simple_yaml = Path(__file__).parent / "snowfakery/include_parent.yml"
from cumulusci.tasks.bulkdata.generate_from_yaml import GenerateDataFromYaml

vanilla_mapping_file = Path(__file__).parent / "../tests/mapping_vanilla_sf.yml"

@contextmanager
def temporary_file_path(filename):
    with TemporaryDirectory() as tmpdirname:
        path = Path(tmpdirname) / filename
        yield path

@contextmanager
def temp_sqlite_database_url():
    with temporary_file_path("test.db") as path:
        yield f"sqlite:///{str(path)}"

class TestGenerateFromDataTask(unittest.TestCase):
    def assertRowsCreated(self, database_url):
        engine = create_engine(database_url)
        connection = engine.connect()
        accounts = connection.execute(f"select * from Account")
        accounts = list(accounts)
        assert accounts and accounts[0] and accounts[0][1]
        return accounts

    def test_no_options(self):
        with self.assertRaises(Exception):
            _make_task(GenerateDataFromYaml, {})

    def test_simple(self):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml,
                        "num_records": 1,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()
            self.assertRowsCreated(database_url)

    def test_inaccessible_generator_yaml(self):
        with self.assertRaises(TaskOptionsError):
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml / "junk",
                        "num_records": 10,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()

    def test_vars(self):
        with temp_sqlite_database_url() as database_url:
            with self.assertWarns(UserWarning):
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "vars": "xyzzy:foo",
                            "database_url": database_url,
                        }
                    },
                )
                task()
                self.assertRowsCreated(database_url)

    def test_generate_mapping_file(self):
        with temporary_file_path("mapping.yml") as temp_mapping:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "generate_mapping_file": temp_mapping,
                        }
                    },
                )
                task()
            mapping = yaml.safe_load(open(temp_mapping))
            assert mapping["Insert Account"]["fields"]

    def test_use_mapping_file(self):
        assert vanilla_mapping_file.exists()
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": sample_yaml,
                        "database_url": database_url,
                        "mapping": vanilla_mapping_file,
                    }
                },
            )
            task()
            self.assertRowsCreated(database_url)

    def test_num_records(self):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateDataFromYaml,
                {
                    "options": {
                        "generator_yaml": simple_yaml,
                        "num_records": 11,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                    }
                },
            )
            task()
            assert len(self.assertRowsCreated(database_url)) == 11

    @mock.patch(
        "cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml._dataload"
    )
    def test_batching(self, _dataload):
        with temp_sqlite_database_url() as database_url:
            task = _make_task(
                GenerateAndLoadDataFromYaml,
                {
                    "options": {
                        "generator_yaml": simple_yaml,
                        "num_records": 14,
                        "batch_size": 6,
                        "database_url": database_url,
                        "num_records_tablename": "Account",
                        "data_generation_task": "cumulusci.tasks.bulkdata.generate_from_yaml.GenerateDataFromYaml",
                        "reset_oids": False,
                    }
                },
            )
            task()
            assert len(_dataload.mock_calls) == 3
            task = None  # clean up db?

            engine = create_engine(database_url)
            connection = engine.connect()
            records = list(connection.execute("select * from Account"))
            connection.close()
            assert len(records) == 14 % 6  # leftovers

    def test_mismatched_options(self):
        with self.assertRaises(TaskOptionsError) as e:
            task = _make_task(
                GenerateDataFromYaml,
                {"options": {"generator_yaml": sample_yaml, "num_records": 10}},
            )
            task()
        assert "without num_records_tablename" in str(e.exception)

    def generate_continuation_data(self):
        g = data_generator_runtime.Globals()
        o = data_generator_runtime.ObjectRow(
            "Account", {"Name": "Johnston incorporated", "id": 5}
        )
        g.register_object(o, "The Company")
        for i in range(0, 5):
            # burn through 5 imaginary accounts
            g.id_manager.generate_id("Account")
        return yaml.safe_dump(g)

    def test_with_continuation_file(self):
        continuation_data = self.generate_continuation_data()
        with temp_sqlite_database_url() as database_url:
            with temporary_file_path("cont.yml") as continuation_file_path:
                with open(continuation_file_path, "w") as continuation_file:
                    continuation_file.write(continuation_data)

                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "mapping": vanilla_mapping_file,
                            "continuation_file": continuation_file_path,
                        }
                    },
                )
                task()
                rows = self.assertRowsCreated(database_url)
                assert dict(rows[0])["id"] == 6

    def test_with_nonexistent_continuation_file(self):
        with self.assertRaises(TaskOptionsError) as e:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "mapping": vanilla_mapping_file,
                            "continuation_file": "/tmp/foobar/baz/jazz/continuation.yml",
                        }
                    },
                )
                task()
                rows = self.assertRowsCreated(database_url)
                assert dict(rows[0])["id"] == 6

        assert "jazz" in str(e.exception)
        assert "does not exist" in str(e.exception)

    def test_generate_continuation_file(self):
        from snowfakery import version

        print(version.version)
        with temporary_file_path("cont.yml") as temp_continuation_file:
            with temp_sqlite_database_url() as database_url:
                task = _make_task(
                    GenerateDataFromYaml,
                    {
                        "options": {
                            "generator_yaml": sample_yaml,
                            "database_url": database_url,
                            "generate_continuation_file": temp_continuation_file,
                        }
                    },
                )
                task()
            mapping = yaml.safe_load(open(temp_continuation_file))
            assert mapping  # internals of this file are not important to MetaCI
<|endoftext|>


<commit_before>from django.contrib.auth import get_user_model
from django.core import urlresolvers
from django.utils.translation import ugettext_lazy as _
from django_cradmin import crmenu
from devilry.apps.core.models import AssignmentGroup
from devilry.devilry_group.cradmin_feedback_feed import feedbackfeedapp
from devilry.devilry_group.cradminextensions import devilrygroupcrinstance
from devilry.devilry_student.cradmin_group import projectgroupapp

class Menu(crmenu.Menu):
    def build_menu(self):
        group = self.request.cradmin_role
        self.add_headeritem(
            label=group.subject.long_name,
            url=urlresolvers.reverse('devilry_group-feedbackfeed-INDEX', kwargs={
                'roleid': group.id
            }),
            icon="angle-up")

        if group.assignment.students_can_create_groups:
            self.add(
                label=_('Project group'), url=self.appindex_url('projectgroup'),
                icon="users",
                active=self.request.cradmin_app.appname == 'projectgroup')

class CrAdminInstance(devilrygroupcrinstance.BaseDevilryGroupCrAdminInstance):
    id = 'devilry_group'
    menuclass = Menu
    roleclass = AssignmentGroup
    rolefrontpage_appname = 'feedbackfeed'

    apps = [
        ('feedbackfeed', feedbackfeedapp.App),
        ('projectgroup', projectgroupapp.App),
    ]

    def get_rolequeryset(self):
        return AssignmentGroup.objects\
            .filter_student_has_access(user=self.request.user)

    def get_titletext_for_role(self, role):
        """
        Get a short title briefly describing the given ``role``.
        Remember that the role is a User.
        """
        return str(role.id)

    @classmethod
    def matches_urlpath(cls, urlpath):
        return urlpath.startswith('/devilry_group')<commit_msg>copy-paste error..<commit_after>from django.contrib.auth import get_user_model
from django.core import urlresolvers
from django.utils.translation import ugettext_lazy as _
from django_cradmin import crmenu
from devilry.apps.core.models import AssignmentGroup
from devilry.devilry_group.cradmin_feedback_feed import feedbackfeedapp
from devilry.devilry_group.cradminextensions import devilrygroupcrinstance
from devilry.devilry_student.cradmin_group import projectgroupapp

class Menu(crmenu.Menu):
    def build_menu(self):
        group = self.request.cradmin_role
        self.add_headeritem(
            label=group.subject.long_name,
            url=self.appindex_url('feedbackfeed'),
            icon="th-list")

        if group.assignment.students_can_create_groups:
            self.add(
                label=_('Project group'),
                url=self.appindex_url('projectgroup'),
                icon="users",
                active=self.request.cradmin_app.appname == 'projectgroup')

class CrAdminInstance(devilrygroupcrinstance.BaseDevilryGroupCrAdminInstance):
    id = 'devilry_group'
    menuclass = Menu
    roleclass = AssignmentGroup
    rolefrontpage_appname = 'feedbackfeed'

    apps = [
        ('feedbackfeed', feedbackfeedapp.App),
        ('projectgroup', projectgroupapp.App),
    ]

    def get_rolequeryset(self):
        return AssignmentGroup.objects\
            .filter_student_has_access(user=self.request.user)

    def get_titletext_for_role(self, role):
        """
        Get a short title briefly describing the given ``role``.
        Remember that the role is a User.
        """
        return str(role.id)

    @classmethod
    def matches_urlpath(cls, urlpath):
        return urlpath.startswith('/devilry_group')<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011 Radim Rehurek <radimrehurek@seznam.cz>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
"""Wrappers to allow gensim models as input into Keras.
Contains various gensim based implementations which can be integrated with Keras.
"""
<commit_msg>removed 'keras_integration' folder<commit_after><|endoftext|>