<commit_before>#  -*- coding: utf-8 -*-

# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import unittest
import frappe
from frappe.custom.doctype.custom_field.custom_field import create_custom_field

class TestDB(unittest.TestCase):
	def test_get_value(self):
		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
		self.assertNotEquals(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
		self.assertEqual(frappe.db.get_value("User", {"name": ["<", "B"]}), "Administrator")
		self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator")

		self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0],
			frappe.db.get_value("User", {"name": [">", "s"]}))

		self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0],
			frappe.db.get_value("User", {"name": [">=", "t"]}))

	def test_set_value(self):
		todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert()
		todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert()

		frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1')
		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1')

		# multiple set-value
		frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')),
			'description', 'change 2')

		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2')
		self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2')

	def test_escape(self):
		frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8"))

	def test_get_single_value(self):
		frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5)
		frappe.db.commit()

		limit = frappe.db.get_single_value('System Settings', 'backup_limit')
		self.assertEqual(limit, 5)

	def test_log_touched_tables(self):
		frappe.flags.in_migrate = True
		frappe.flags.touched_tables = set()
		frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5)
		self.assertIn('tabSingles', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo = frappe.get_doc({'doctype': 'ToDo', 'description': 'Random Description'})
		todo.save()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo.description = "Another Description"
		todo.save()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		if frappe.db.db_type != "postgres":
			frappe.flags.touched_tables = set()
			frappe.db.sql("UPDATE tabToDo SET description = 'Updated Description'")
			self.assertNotIn('tabToDo SET', frappe.flags.touched_tables)
			self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo.delete()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		create_custom_field('ToDo', {'label': 'ToDo Custom Field'})

		self.assertIn('tabToDo', frappe.flags.touched_tables)
		self.assertIn('tabCustom Field', frappe.flags.touched_tables)
		frappe.flags.in_migrate = False
		frappe.flags.touched_tables.clear()

	def test_db_keywords_as_fields(self):
		"""Tests if DB keywords work as docfield names. If they're wrapped with grave accents."""
		all_keywords = {
			"mariadb": ["ADD", "ALL", "ALTER", "ANALYZE", "AND", "AS", "ASC", "AUTO_INCREMENT", "BDB", "BERKELEYDB", "BETWEEN", "BIGINT", "BINARY", "BLOB", "BOTH", "BTREE", "BY", "CASCADE", "CASE", "CHANGE", "CHAR", "CHARACTER", "CHECK", "COLLATE", "COLUMN", "COLUMNS", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DATABASES", "DAY_HOUR", "DAY_MINUTE", "DAY_SECOND", "DEC", "DECIMAL", "DEFAULT", "DELAYED", "DELETE", "DESC", "DESCRIBE", "DISTINCT", "DISTINCTROW", "DIV", "DOUBLE", "DROP", "ELSE", "ENCLOSED", "ERRORS", "ESCAPED", "EXISTS", "EXPLAIN", "FALSE", "FIELDS", "FLOAT", "FOR", "FORCE", "FOREIGN", "FROM", "FULLTEXT", "FUNCTION", "GEOMETRY", "GRANT", "GROUP", "HASH", "HAVING", "HELP", "HIGH_PRIORITY", "HOUR_MINUTE", "HOUR_SECOND", "IF", "IGNORE", "IN", "INDEX", "INFILE", "INNER", "INNODB", "INSERT", "INT", "INTEGER", "INTERVAL", "INTO", "IS", "JOIN", "KEY", "KEYS", "KILL", "LEADING", "LEFT", "LIKE", "LIMIT", "LINES", "LOAD", "LOCALTIME", "LOCALTIMESTAMP", "LOCK", "LONG", "LONGBLOB", "LONGTEXT", "LOW_PRIORITY", "MASTER_SERVER_ID", "MATCH", "MEDIUMBLOB", "MEDIUMINT", "MEDIUMTEXT", "MIDDLEINT", "MINUTE_SECOND", "MOD", "MRG_MYISAM", "NATURAL", "NOT", "NULL", "NUMERIC", "ON", "OPTIMIZE", "OPTION", "OPTIONALLY", "OR", "ORDER", "OUTER", "OUTFILE", "PRECISION", "PRIMARY", "PRIVILEGES", "PROCEDURE", "PURGE", "READ", "REAL", "REFERENCES", "REGEXP", "RENAME", "REPLACE", "REQUIRE", "RESTRICT", "RETURNS", "REVOKE", "RIGHT", "RLIKE", "RTREE", "SELECT", "SET", "SHOW", "SMALLINT", "SOME", "SONAME", "SPATIAL", "SQL_BIG_RESULT", "SQL_CALC_FOUND_ROWS", "SQL_SMALL_RESULT", "SSL", "STARTING", "STRAIGHT_JOIN", "STRIPED", "TABLE", "TABLES", "TERMINATED", "THEN", "TINYBLOB", "TINYINT", "TINYTEXT", "TO", "TRAILING", "TRUE", "TYPES", "UNION", "UNIQUE", "UNLOCK", "UNSIGNED", "UPDATE", "USAGE", "USE", "USER_RESOURCES", "USING", "VALUES", "VARBINARY", "VARCHAR", "VARCHARACTER", "VARYING", "WARNINGS", "WHEN", "WHERE", "WITH", "WRITE", "XOR", "YEAR_MONTH", "ZEROFILL"],
			"postgres": ["ABORT", "ABS", "ABSOLUTE", "ACCESS", "ACTION", "ADA", "ADD", "ADMIN", "AFTER", "AGGREGATE", "ALIAS", "ALL", "ALLOCATE", "ALTER", "ANALYSE", "ANALYZE", "AND", "ANY", "ARE", "ARRAY", "AS", "ASC", "ASENSITIVE", "ASSERTION", "ASSIGNMENT", "ASYMMETRIC", "AT", "ATOMIC", "AUTHORIZATION", "AVG", "BACKWARD", "BEFORE", "BEGIN", "BETWEEN", "BIGINT", "BINARY", "BIT", "BITVAR", "BIT_LENGTH", "BLOB", "BOOLEAN", "BOTH", "BREADTH", "BY", "CACHE", "CALL", "CALLED", "CARDINALITY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", "CATALOG_NAME", "CHAIN", "CHAR", "CHARACTER", "CHARACTERISTICS", "CHARACTER_LENGTH", "CHARACTER_SET_CATALOG", "CHARACTER_SET_NAME", "CHARACTER_SET_SCHEMA", "CHAR_LENGTH", "CHECK", "CHECKED", "CHECKPOINT", "CLASS", "CLASS_ORIGIN", "CLOB", "CLOSE", "CLUSTER", "COALESCE", "COBOL", "COLLATE", "COLLATION", "COLLATION_CATALOG", "COLLATION_NAME", "COLLATION_SCHEMA", "COLUMN", "COLUMN_NAME", "COMMAND_FUNCTION", "COMMAND_FUNCTION_CODE", "COMMENT", "COMMIT", "COMMITTED", "COMPLETION", "CONDITION_NUMBER", "CONNECT", "CONNECTION", "CONNECTION_NAME", "CONSTRAINT", "CONSTRAINTS", "CONSTRAINT_CATALOG", "CONSTRAINT_NAME", "CONSTRAINT_SCHEMA", "CONSTRUCTOR", "CONTAINS", "CONTINUE", "CONVERSION", "CONVERT", "COPY", "CORRESPONDING", "COUNT", "CREATE", "CREATEDB", "CREATEUSER", "CROSS", "CUBE", "CURRENT", "CURRENT_DATE", "CURRENT_PATH", "CURRENT_ROLE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "CURSOR", "CURSOR_NAME", "CYCLE", "DATA", "DATABASE", "DATE", "DATETIME_INTERVAL_CODE", "DATETIME_INTERVAL_PRECISION", "DAY", "DEALLOCATE", "DEC", "DECIMAL", "DECLARE", "DEFAULT", "DEFERRABLE", "DEFERRED", "DEFINED", "DEFINER", "DELETE", "DELIMITER", "DELIMITERS", "DEPTH", "DEREF", "DESC", "DESCRIBE", "DESCRIPTOR", "DESTROY", "DESTRUCTOR", "DETERMINISTIC", "DIAGNOSTICS", "DICTIONARY", "DISCONNECT", "DISPATCH", "DISTINCT", "DO", "DOMAIN", "DOUBLE", "DROP", "DYNAMIC", "DYNAMIC_FUNCTION", "DYNAMIC_FUNCTION_CODE", "EACH", "ELSE", "ENCODING", "ENCRYPTED", "END", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXCEPTION", "EXCLUSIVE", "EXEC", "EXECUTE", "EXISTING", "EXISTS", "EXPLAIN", "EXTERNAL", "EXTRACT", "FALSE", "FETCH", "FINAL", "FIRST", "FLOAT", "FOR", "FORCE", "FOREIGN", "FORTRAN", "FORWARD", "FOUND", "FREE", "FREEZE", "FROM", "FULL", "FUNCTION", "GENERAL", "GENERATED", "GET", "GLOBAL", "GO", "GOTO", "GRANT", "GRANTED", "GROUP", "GROUPING", "HANDLER", "HAVING", "HIERARCHY", "HOLD", "HOST", "HOUR", "IDENTITY", "IGNORE", "ILIKE", "IMMEDIATE", "IMMUTABLE", "IMPLEMENTATION", "IMPLICIT", "IN", "INCREMENT", "INDEX", "INDICATOR", "INFIX", "INHERITS", "INITIALIZE", "INITIALLY", "INNER", "INOUT", "INPUT", "INSENSITIVE", "INSERT", "INSTANCE", "INSTANTIABLE", "INSTEAD", "INT", "INTEGER", "INTERSECT", "INTERVAL", "INTO", "INVOKER", "IS", "ISNULL", "ISOLATION", "ITERATE", "JOIN", "KEY", "KEY_MEMBER", "KEY_TYPE", "LANCOMPILER", "LANGUAGE", "LARGE", "LAST", "LATERAL", "LEADING", "LEFT", "LENGTH", "LESS", "LEVEL", "LIKE", "LIMIT", "LISTEN", "LOAD", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", "LOCATION", "LOCATOR", "LOCK", "LOWER", "MAP", "MATCH", "MAX", "MAXVALUE", "MESSAGE_LENGTH", "MESSAGE_OCTET_LENGTH", "MESSAGE_TEXT", "METHOD", "MIN", "MINUTE", "MINVALUE", "MOD", "MODE", "MODIFIES", "MODIFY", "MODULE", "MONTH", "MORE", "MOVE", "MUMPS", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "NEW", "NEXT", "NO", "NOCREATEDB", "NOCREATEUSER", "NONE", "NOT", "NOTHING", "NOTIFY", "NOTNULL", "NULL", "NULLABLE", "NULLIF", "NUMBER", "NUMERIC", "OBJECT", "OCTET_LENGTH", "OF", "OFF", "OFFSET", "OIDS", "OLD", "ON", "ONLY", "OPEN", "OPERATION", "OPERATOR", "OPTION", "OPTIONS", "OR", "ORDER", "ORDINALITY", "OUT", "OUTER", "OUTPUT", "OVERLAPS", "OVERLAY", "OVERRIDING", "OWNER", "PAD", "PARAMETER", "PARAMETERS", "PARAMETER_MODE", "PARAMETER_NAME", "PARAMETER_ORDINAL_POSITION", "PARAMETER_SPECIFIC_CATALOG", "PARAMETER_SPECIFIC_NAME", "PARAMETER_SPECIFIC_SCHEMA", "PARTIAL", "PASCAL", "PASSWORD", "PATH", "PENDANT", "PLACING", "PLI", "POSITION", "POSTFIX", "PRECISION", "PREFIX", "PREORDER", "PREPARE", "PRESERVE", "PRIMARY", "PRIOR", "PRIVILEGES", "PROCEDURAL", "PROCEDURE", "PUBLIC", "READ", "READS", "REAL", "RECHECK", "RECURSIVE", "REF", "REFERENCES", "REFERENCING", "REINDEX", "RELATIVE", "RENAME", "REPEATABLE", "REPLACE", "RESET", "RESTRICT", "RESULT", "RETURN", "RETURNED_LENGTH", "RETURNED_OCTET_LENGTH", "RETURNED_SQLSTATE", "RETURNS", "REVOKE", "RIGHT", "ROLE", "ROLLBACK", "ROLLUP", "ROUTINE", "ROUTINE_CATALOG", "ROUTINE_NAME", "ROUTINE_SCHEMA", "ROW", "ROWS", "ROW_COUNT", "RULE", "SAVEPOINT", "SCALE", "SCHEMA", "SCHEMA_NAME", "SCOPE", "SCROLL", "SEARCH", "SECOND", "SECTION", "SECURITY", "SELECT", "SELF", "SENSITIVE", "SEQUENCE", "SERIALIZABLE", "SERVER_NAME", "SESSION", "SESSION_USER", "SET", "SETOF", "SETS", "SHARE", "SHOW", "SIMILAR", "SIMPLE", "SIZE", "SMALLINT", "SOME", "SOURCE", "SPACE", "SPECIFIC", "SPECIFICTYPE", "SPECIFIC_NAME", "SQL", "SQLCODE", "SQLERROR", "SQLEXCEPTION", "SQLSTATE", "SQLWARNING", "STABLE", "START", "STATE", "STATEMENT", "STATIC", "STATISTICS", "STDIN", "STDOUT", "STORAGE", "STRICT", "STRUCTURE", "STYLE", "SUBCLASS_ORIGIN", "SUBLIST", "SUBSTRING", "SUM", "SYMMETRIC", "SYSID", "SYSTEM", "SYSTEM_USER", "TABLE", "TABLE_NAME", "TEMP", "TEMPLATE", "TEMPORARY", "TERMINATE", "THAN", "THEN", "TIME", "TIMESTAMP", "TIMEZONE_HOUR", "TIMEZONE_MINUTE", "TO", "TOAST", "TRAILING", "TRANSACTION", "TRANSACTIONS_COMMITTED", "TRANSACTIONS_ROLLED_BACK", "TRANSACTION_ACTIVE", "TRANSFORM", "TRANSFORMS", "TRANSLATE", "TRANSLATION", "TREAT", "TRIGGER", "TRIGGER_CATALOG", "TRIGGER_NAME", "TRIGGER_SCHEMA", "TRIM", "TRUE", "TRUNCATE", "TRUSTED", "TYPE", "UNCOMMITTED", "UNDER", "UNENCRYPTED", "UNION", "UNIQUE", "UNKNOWN", "UNLISTEN", "UNNAMED", "UNNEST", "UNTIL", "UPDATE", "UPPER", "USAGE", "USER", "USER_DEFINED_TYPE_CATALOG", "USER_DEFINED_TYPE_NAME", "USER_DEFINED_TYPE_SCHEMA", "USING", "VACUUM", "VALID", "VALIDATOR", "VALUE", "VALUES", "VARCHAR", "VARIABLE", "VARYING", "VERBOSE", "VERSION", "VIEW", "VOLATILE", "WHEN", "WHENEVER", "WHERE", "WITH", "WITHOUT", "WORK", "WRITE", "YEAR", "ZONE"]
		}
		all_doctypes = []
		fieldnames = all_keywords[frappe.conf.db_type]

		def batch(iterable, n=1):
			total = len(iterable)
			for ndx in range(0, total, n):
				yield iterable[ndx:min(ndx + n, total)]

		for i, fields in enumerate(batch(fieldnames, n=20)):
			new_doctype = frappe.get_doc({
				"doctype": "DocType",
				"name": "Keywords Test {0}".format(i + 1),
				"module": "Custom",
				"custom": 1,
				"fields": [{
					"label": field.title(),
					"fieldname": field.lower(),
					"fieldtype": "Data"
				} for field in fields]
			})
			new_doctype.insert()
			all_doctypes.append(new_doctype.name)

		for doctype in all_doctypes:
			frappe.delete_doc("DocType", doctype)

<commit_msg>test: test once for 40 keyword fields instead<commit_after>#  -*- coding: utf-8 -*-

# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import unittest
import frappe
from frappe.custom.doctype.custom_field.custom_field import create_custom_field

class TestDB(unittest.TestCase):
	def test_get_value(self):
		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
		self.assertNotEquals(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
		self.assertEqual(frappe.db.get_value("User", {"name": ["<", "B"]}), "Administrator")
		self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator")

		self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0],
			frappe.db.get_value("User", {"name": [">", "s"]}))

		self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0],
			frappe.db.get_value("User", {"name": [">=", "t"]}))

	def test_set_value(self):
		todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert()
		todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert()

		frappe.db.set_value('ToDo', todo1.name, 'description', 'test_set_value change 1')
		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'test_set_value change 1')

		# multiple set-value
		frappe.db.set_value('ToDo', dict(description=('like', '%test_set_value%')),
			'description', 'change 2')

		self.assertEqual(frappe.db.get_value('ToDo', todo1.name, 'description'), 'change 2')
		self.assertEqual(frappe.db.get_value('ToDo', todo2.name, 'description'), 'change 2')

	def test_escape(self):
		frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8"))

	def test_get_single_value(self):
		frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5)
		frappe.db.commit()

		limit = frappe.db.get_single_value('System Settings', 'backup_limit')
		self.assertEqual(limit, 5)

	def test_log_touched_tables(self):
		frappe.flags.in_migrate = True
		frappe.flags.touched_tables = set()
		frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5)
		self.assertIn('tabSingles', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo = frappe.get_doc({'doctype': 'ToDo', 'description': 'Random Description'})
		todo.save()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo.description = "Another Description"
		todo.save()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		if frappe.db.db_type != "postgres":
			frappe.flags.touched_tables = set()
			frappe.db.sql("UPDATE tabToDo SET description = 'Updated Description'")
			self.assertNotIn('tabToDo SET', frappe.flags.touched_tables)
			self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		todo.delete()
		self.assertIn('tabToDo', frappe.flags.touched_tables)

		frappe.flags.touched_tables = set()
		create_custom_field('ToDo', {'label': 'ToDo Custom Field'})

		self.assertIn('tabToDo', frappe.flags.touched_tables)
		self.assertIn('tabCustom Field', frappe.flags.touched_tables)
		frappe.flags.in_migrate = False
		frappe.flags.touched_tables.clear()

	def test_db_keywords_as_fields(self):
		"""Tests if DB keywords work as docfield names. If they're wrapped with grave accents."""
		# Using random.choices, picked out a list of 40 keywords for testing
		all_keywords = {
			"mariadb": ["CHARACTER", "DELAYED", "LINES", "EXISTS", "YEAR_MONTH", "LOCALTIME", "BOTH", "MEDIUMINT", "LEFT", "BINARY", "LEFT", "DEFAULT", "KILL", "WRITE", "SQL_SMALL_RESULT", "CURRENT_TIME", "CROSS", "SET", "TABLE", "ALTER", "CURRENT_TIMESTAMP", "XOR", "CASE", "ALL", "WHERE", "INT", "TO", "SOME", "DAY_MINUTE", "ALL", "ERRORS", "OPTIMIZE", "LOCK", "REPLACE", "HIGH_PRIORITY", "VARBINARY", "HELP", "IS", "CHAR", "DESCRIBE"],
			"postgres": ["WORK", "LANCOMPILER", "REAL", "HAVING", "REPEATABLE", "DATA", "USING", "BIT", "DEALLOCATE", "SERIALIZABLE", "CURSOR", "INHERITS", "ARRAY", "TRUE", "IGNORE", "PARAMETER_MODE", "ROW", "CHECKPOINT", "SHOW", "BY", "SIZE", "SCALE", "UNENCRYPTED", "WITH", "AND", "CONVERT", "FIRST", "SCOPE", "WRITE", "INTERVAL", "CHARACTER_SET_SCHEMA", "ADD", "LANCOMPILER", "SCROLL", "PRECISION", "NULL", "WHEN", "TRANSACTION_ACTIVE", "INT", "FORTRAN"]
		}
		fields = all_keywords[frappe.conf.db_type]
		test_doctype = "Keywords Test"

		frappe.get_doc({
			"doctype": "DocType",
			"name": test_doctype,
			"module": "Custom",
			"custom": 1,
			"fields": [{
				"label": field.title(),
				"fieldname": field.lower(),
				"fieldtype": "Data"
			} for field in fields]
		}).insert()

		frappe.delete_doc("DocType", test_doctype)

<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
'''
Creates packed .js files for all source files (highlight.js and languages) in a
separate directory.
'''
import os
import re
import subprocess

REPLACES = {
    'defaultMode': 'dM',
    'case_insensitive': 'cI',
    'lexems': 'l',
    'contains': 'c',
    'keywords': 'k',
    'modes': 'm',
    'className': 'cN',
    'begin': 'b',
    'end': 'e',
    'endsWithParent': 'eW',
    'illegal': 'i',
    'excludeBegin': 'eB',
    'excludeEnd': 'eE',
    'returnBegin': 'rB',
    'returnEnd': 'rE',
    'noMarkup': 'nM',
    'relevance': 'r',
    'IDENT_RE': 'IR',
    'UNDERSCORE_IDENT_RE': 'UIR',
    'NUMBER_RE': 'NR',
    'C_NUMBER_RE': 'CNR',
    'RE_STARTERS_RE': 'RSR',
    'APOS_STRING_MODE': 'ASM',
    'QUOTE_STRING_MODE': 'QSM',
    'BACKSLASH_ESCAPE': 'BE',
    'C_LINE_COMMENT_MODE': 'CLCM',
    'C_BLOCK_COMMENT_MODE': 'CBLCLM',
    'HASH_COMMENT_MODE': 'HCM',
    'C_NUMBER_MODE': 'CNM',
}

LIBRARY_REPLACES = {
    'beginRe': 'bR',
    'endRe': 'eR',
    'illegalRe': 'iR',
    'lexemsRe': 'lR',
    'sub_modes': 'sm',
    'terminators': 't',
}

def get_compressor(tools_path):
    cmd = 'java -jar %s --type js' % os.path.join(tools_path, 'yuicompressor.jar')
    def compress(content):
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        p.stdin.write(content)
        p.stdin.close()
        content = p.stdout.read()
        p.stdout.close()
        return content
    return compress

def replace(content, s, r):
    return re.sub(r'(?<=[^\w"\'|])%s(?=[^\w"\'|])' % s, r, content)

def pack_language(content):
    for s, r in REPLACES.items():
        content = replace(content, s, r)
    return content

def pack_library(content):
    for s, r in REPLACES.items():
        content = replace(content, s, r)
    content = re.sub(r'(block|parentNode)\.cN', r'\1.className', content)
    for s, r in LIBRARY_REPLACES.items():
        content = replace(content, s, r)
    return content

def build_file(filename, in_path, out_path, packer, compressor):
    print 'Building %s' % filename
    content = open(os.path.join(in_path, filename)).read()
    content = packer(content)
    content = compressor(content)
    f = open(os.path.join(out_path, filename), 'w')
    f.write(content)
    f.close()

def build(library_path, build_path, tools_path):
    lang_path = os.path.join(library_path, 'languages')
    if not os.path.exists(build_path):
        os.makedirs(build_path)

    languages = os.listdir(lang_path)
    languages = [l for l in languages if l.endswith('.js')]

    compressor = get_compressor(tools_path)
    build_file('highlight.js', library_path, build_path, pack_library, compressor)
    for language in languages:
        build_file(language, lang_path, build_path, pack_language, compressor)

if __name__ == '__main__':
    path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    library_path = os.path.join(path, 'src')
    build_path = os.path.join(path, 'packed')
    tools_path = os.path.join(path, 'tools')
    build(library_path, build_path, tools_path)
<commit_msg>Make packing tool to work on selected languages<commit_after># -*- coding:utf-8 -*-
'''
Creates packed .js files for all source files (highlight.js and languages) in a
separate directory.
'''
import os
import sys
import re
import subprocess

REPLACES = {
    'defaultMode': 'dM',
    'case_insensitive': 'cI',
    'lexems': 'l',
    'contains': 'c',
    'keywords': 'k',
    'modes': 'm',
    'className': 'cN',
    'begin': 'b',
    'end': 'e',
    'endsWithParent': 'eW',
    'illegal': 'i',
    'excludeBegin': 'eB',
    'excludeEnd': 'eE',
    'returnBegin': 'rB',
    'returnEnd': 'rE',
    'noMarkup': 'nM',
    'relevance': 'r',
    'IDENT_RE': 'IR',
    'UNDERSCORE_IDENT_RE': 'UIR',
    'NUMBER_RE': 'NR',
    'C_NUMBER_RE': 'CNR',
    'RE_STARTERS_RE': 'RSR',
    'APOS_STRING_MODE': 'ASM',
    'QUOTE_STRING_MODE': 'QSM',
    'BACKSLASH_ESCAPE': 'BE',
    'C_LINE_COMMENT_MODE': 'CLCM',
    'C_BLOCK_COMMENT_MODE': 'CBLCLM',
    'HASH_COMMENT_MODE': 'HCM',
    'C_NUMBER_MODE': 'CNM',
}

LIBRARY_REPLACES = {
    'beginRe': 'bR',
    'endRe': 'eR',
    'illegalRe': 'iR',
    'lexemsRe': 'lR',
    'sub_modes': 'sm',
    'terminators': 't',
}

def get_compressor(tools_path):
    cmd = 'java -jar %s --type js' % os.path.join(tools_path, 'yuicompressor.jar')
    def compress(content):
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        p.stdin.write(content)
        p.stdin.close()
        content = p.stdout.read()
        p.stdout.close()
        return content
    return compress

def replace(content, s, r):
    return re.sub(r'(?<=[^\w"\'|])%s(?=[^\w"\'|])' % s, r, content)

def pack_language(content):
    for s, r in REPLACES.items():
        content = replace(content, s, r)
    return content

def pack_library(content):
    for s, r in REPLACES.items():
        content = replace(content, s, r)
    content = re.sub(r'(block|parentNode)\.cN', r'\1.className', content)
    for s, r in LIBRARY_REPLACES.items():
        content = replace(content, s, r)
    return content

def pack_file(filename, in_path, out_path, packer, compressor):
    print 'Packing %s' % filename
    content = open(os.path.join(in_path, filename)).read()
    content = packer(content)
    content = compressor(content)
    f = open(os.path.join(out_path, filename), 'w')
    f.write(content)
    f.close()

def pack(library_path, pack_path, tools_path, languages):
    lang_path = os.path.join(library_path, 'languages')
    if not os.path.exists(pack_path):
        os.makedirs(pack_path)

    files = os.listdir(lang_path)
    files = [f for f in files if f.endswith('.js')]
    if languages:
        files = [f for f in files if os.path.splitext(f)[0] in languages]

    compressor = get_compressor(tools_path)
    pack_file('highlight.js', library_path, pack_path, pack_library, compressor)
    for file in files:
        pack_file(file, lang_path, pack_path, pack_language, compressor)

if __name__ == '__main__':
    path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    library_path = os.path.join(path, 'src')
    pack_path = os.path.join(path, 'packed')
    tools_path = os.path.join(path, 'tools')
    if len(sys.argv) > 1:
        languages = set(sys.argv[1:])
    else:
        languages = set()
    pack(library_path, pack_path, tools_path, languages)
<|endoftext|>


<commit_before>import itertools, sys

def makeTotalOrdersFromLattice(partialorder,cyclic=1):
    # partialorder is list of lists of strings (extrema)
    # partialorder[i] < partialorder[i+1] for all i
    # only partial orders that are lattices are permitted
    orders = [', '.join(list(s)) for s in itertools.permutations(partialorder[0])]
    for po in partialorder[1:]:
        orders=[t + ', ' + ', '.join(list(s)) for t in orders for s in itertools.permutations(po)]
    if cyclic:
        totalorders = []
        for t in orders:
            twords = t.split(',')
            totalorders.append(t + ', ' + twords[0])
    else:
        totalorders = orders
    return totalorders

def resultsFileParser(filename):
    uniquepatterns = []
    countpatterns = []
    with open(filename,'r') as f:
        for line in f:
            pattern = line.split(':')[-1][1:-1]
            if pattern not in uniquepatterns:
                uniquepatterns.append(pattern)
                countpatterns.append(1)
            else:
                countpatterns[uniquepatterns.index(pattern)] += 1
    return uniquepatterns,countpatterns

def searchUniqueParamsInListOfPatterns(filename,listofpatterns):
    uniqueparams = set()
    with open(filename,'r') as f:
        for line in f:
            words = line.split(':')
            parameter = int(words[1].replace(',',' ').split()[0])
            pattern = words[-1][1:-1]
            if pattern in listofpatterns:
                if parameter not in uniqueparams:
                    uniqueparams.add(parameter)
    return uniqueparams

def searchFile(uniquepatterns,totalorders,lattice):
    count = 0
    for u in uniquepatterns:
        if u in totalorders:
            count+=1
    numuniq = len(uniquepatterns)
    print "Found {} patterns out of {} total patterns that are consistent with {}.".format(count,numuniq,lattice)
    return count, numuniq

def check5D_2016_01_28_originalpatterns(uniquepatterns):
    original_lattice = [ ['199 max', '177 max', '72 max','204 max'],
                ['77 max'],
                ['72 min', '199 min', '177  min', '77 min', '204 min'] ]
    for k,o in enumerate(original_lattice):
        if len(o) > 1:
            ol1=original_lattice[:k]
            ol2=original_lattice[k+1:]
            for p in o:
                q = o[:]
                q.remove(p)
                new_lattice = ol1+[q]+[[p]]+ol2
                totalorders = makeTotalOrdersFromLattice(new_lattice,cyclic=1)
                searchFile(uniquepatterns,totalorders,new_lattice)
                new_lattice = ol1+[[p]]+[q]+ol2
                totalorders = makeTotalOrdersFromLattice(new_lattice,cyclic=1)
                searchFile(uniquepatterns,totalorders,new_lattice)

def sort_by_list(X,Y,reverse=False):
    # sort Y by the sorted order of X
    newX, newY = [], []
    for (x,y) in sorted(zip(X,Y),reverse=reverse):
        newX.append(x)
        newY.append(y)
    return newX,newY

if __name__ == '__main__':
    # filename='5D_2016_01_28_stableFCs_allresults.txt'
    # listofpatterns = ['199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 199 max', '72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 204 min, 177  min, 77 min, 72 max','199 max, 72 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 199 max','199 max, 72 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 199 max','72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 77 min, 204 min, 72 max','199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 199 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max','199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 199 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 177  min, 77 min, 204 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 72 max','72 max, 199 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 72 max']
    # listofpatterns = ['199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 199 max', '72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max']
    # uniqueparams = searchUniqueParamsInListOfPatterns(filename,listofpatterns)
    # with open('uniqueparamsintop15patterns.txt','w') as uf:
    #     for u in uniqueparams:
    #         uf.write(str(u)+'\n')
    # print len(uniqueparams)
    filename='5D_2016_01_28_essential_stableFCs_allresults.txt'
    uniquepatterns, countpatterns = resultsFileParser(filename)
    counts, patterns = sort_by_list(countpatterns,uniquepatterns,True)
    with open('countedpatterns.txt','w') as cp:
        for (p,c) in zip(patterns,counts):
            cp.write(str(c) + '     '  + p+ '\n')
    print sum(counts)
    # # with open('uniquepatterns.txt','w') as uf:
    # #     for up in uniquepatterns:
    # #         uf.write(up+'\n')
    # uniquepatterns=[]
    # with open('uniquepatterns.txt','r') as uf:
    #     for line in uf:
    #         uniquepatterns.append(line[:-1])
    # # check5D_2016_01_28_originalpatterns(uniquepatterns)
    # lattice = [ ['199 max'], ['177 max', '72 max'],['204 max'],
    #             ['77 max'], ['199 min'],['72 min', '177  min', '204 min'],['77 min']]
    # totalorders = makeTotalOrdersFromLattice(lattice,cyclic=1)
    # searchFile(uniquepatterns,totalorders,lattice)<commit_msg>minor change to postprocess results<commit_after>import itertools, sys

def makeTotalOrdersFromLattice(partialorder,cyclic=1):
    # partialorder is list of lists of strings (extrema)
    # partialorder[i] < partialorder[i+1] for all i
    # only partial orders that are lattices are permitted
    orders = [', '.join(list(s)) for s in itertools.permutations(partialorder[0])]
    for po in partialorder[1:]:
        orders=[t + ', ' + ', '.join(list(s)) for t in orders for s in itertools.permutations(po)]
    if cyclic:
        totalorders = []
        for t in orders:
            twords = t.split(',')
            totalorders.append(t + ', ' + twords[0])
    else:
        totalorders = orders
    return totalorders

def resultsFileParser(filename):
    uniquepatterns = []
    countpatterns = []
    with open(filename,'r') as f:
        for line in f:
            pattern = line.split(':')[-1][1:-1]
            if pattern not in uniquepatterns:
                uniquepatterns.append(pattern)
                countpatterns.append(1)
            else:
                countpatterns[uniquepatterns.index(pattern)] += 1
    return uniquepatterns,countpatterns

def searchUniqueParamsInListOfPatterns(filename,listofpatterns):
    uniqueparams = set()
    with open(filename,'r') as f:
        for line in f:
            words = line.split(':')
            parameter = int(words[1].replace(',',' ').split()[0])
            pattern = words[-1][1:-1]
            if pattern in listofpatterns:
                if parameter not in uniqueparams:
                    uniqueparams.add(parameter)
    return uniqueparams

def searchFile(uniquepatterns,totalorders,lattice):
    count = 0
    for u in uniquepatterns:
        if u in totalorders:
            count+=1
    numuniq = len(uniquepatterns)
    print "Found {} patterns out of {} total patterns that are consistent with {}.".format(count,numuniq,lattice)
    return count, numuniq

def check5D_2016_01_28_originalpatterns(uniquepatterns):
    original_lattice = [ ['199 max', '177 max', '72 max','204 max'],
                ['77 max'],
                ['72 min', '199 min', '177  min', '77 min', '204 min'] ]
    for k,o in enumerate(original_lattice):
        if len(o) > 1:
            ol1=original_lattice[:k]
            ol2=original_lattice[k+1:]
            for p in o:
                q = o[:]
                q.remove(p)
                new_lattice = ol1+[q]+[[p]]+ol2
                totalorders = makeTotalOrdersFromLattice(new_lattice,cyclic=1)
                searchFile(uniquepatterns,totalorders,new_lattice)
                new_lattice = ol1+[[p]]+[q]+ol2
                totalorders = makeTotalOrdersFromLattice(new_lattice,cyclic=1)
                searchFile(uniquepatterns,totalorders,new_lattice)

def sort_by_list(X,Y,reverse=False):
    # sort Y by the sorted order of X
    newX, newY = [], []
    for (x,y) in sorted(zip(X,Y),reverse=reverse):
        newX.append(x)
        newY.append(y)
    return newX,newY

if __name__ == '__main__':
    # filename='5D_2016_01_28_stableFCs_allresults.txt'
    # listofpatterns = ['199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 199 max', '72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 204 min, 177  min, 77 min, 72 max','199 max, 72 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 199 max','199 max, 72 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 199 max','72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 77 min, 204 min, 72 max','199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 199 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max','199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 199 max','72 max, 199 max, 177 max, 204 max, 77 max, 199 min, 72 min, 177  min, 77 min, 204 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 72 max','72 max, 199 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 204 min, 77 min, 72 max','72 max, 199 max, 204 max, 177 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 72 max','72 max, 199 max, 177 max, 204 max, 77 max, 72 min, 199 min, 177  min, 77 min, 204 min, 72 max']
    # listofpatterns = ['199 max, 72 max, 177 max, 204 max, 77 max, 72 min, 199 min, 204 min, 177  min, 77 min, 199 max', '72 max, 199 max, 204 max, 177 max, 77 max, 199 min, 72 min, 177  min, 204 min, 77 min, 72 max']
    # uniqueparams = searchUniqueParamsInListOfPatterns(filename,listofpatterns)
    # with open('uniqueparamsintop15patterns.txt','w') as uf:
    #     for u in uniqueparams:
    #         uf.write(str(u)+'\n')
    # print len(uniqueparams)
    filename='/share/data/bcummins/parameterresults_archive/5D_2016_01_28_essential_stableFCs_allresults.txt'
    uniquepatterns, countpatterns = resultsFileParser(filename)
    counts, patterns = sort_by_list(countpatterns,uniquepatterns,True)
    with open('countedpatterns.txt','w') as cp:
        for (p,c) in zip(patterns,counts):
            cp.write(str(c) + '     '  + p+ '\n')
    print sum(counts)
    # # with open('uniquepatterns.txt','w') as uf:
    # #     for up in uniquepatterns:
    # #         uf.write(up+'\n')
    # uniquepatterns=[]
    # with open('uniquepatterns.txt','r') as uf:
    #     for line in uf:
    #         uniquepatterns.append(line[:-1])
    # # check5D_2016_01_28_originalpatterns(uniquepatterns)
    # lattice = [ ['199 max'], ['177 max', '72 max'],['204 max'],
    #             ['77 max'], ['199 min'],['72 min', '177  min', '204 min'],['77 min']]
    # totalorders = makeTotalOrdersFromLattice(lattice,cyclic=1)
    # searchFile(uniquepatterns,totalorders,lattice)<|endoftext|>


<commit_before>"""
UML model support functions.

Functions collected in this module allow to

- create more complex UML model structures
- perform specific searches and manipulations

"""

import itertools
from gaphor.UML.uml2 import *

# '<<%s>>'
STEREOTYPE_FMT = "<<%s>>"

def stereotypes_str(element, stereotypes=()):
    """
    Identify stereotypes of an UML metamodel instance and return coma
    separated stereotypes as string.

    :Parameters:
     element
        Element having stereotypes, can be None.
     stereotypes
        List of additional stereotypes, can be empty.
    """
    # generate string with stereotype names separated by coma
    if element:
        applied = (stereotype_name(st) for st in get_applied_stereotypes(element))
    else:
        applied = ()
    s = ", ".join(itertools.chain(stereotypes, applied))
    if s:
        return STEREOTYPE_FMT % s
    else:
        return ""

def stereotype_name(stereotype):
    """
    Return stereotype name suggested by UML specification. First will be
    character lowercase unless the second character is uppercase.

    :Parameters:
     stereotype
        Stereotype UML metamodel instance.
    """
    name = stereotype.name
    if not name:
        return ""
    elif len(name) > 1 and name[1].isupper():
        return name
    else:
        return name[0].lower() + name[1:]

def apply_stereotype(factory, element, stereotype):
    """
    Apply a stereotype to an element.

    :Parameters:
     factory
        UML metamodel factory.
     element
        UML metamodel class instance.
     stereotype
        UML metamodel stereotype instance.
    """
    obj = factory.create(InstanceSpecification)
    obj.classifier = stereotype
    element.appliedStereotype = obj
    return obj

def find_instances(factory, element):
    """
    Find instance specification which extend classifier `element`.
    """
    return factory.select(
        lambda e: e.isKindOf(InstanceSpecification)
        and e.classifier
        and e.classifier[0] == element
    )

def remove_stereotype(element, stereotype):
    """
    Remove a stereotype from an element.

    :Parameters:
     element
        UML metamodel element instance.
     stereotype
        UML metamodel stereotype instance.
    """
    for obj in element.appliedStereotype:
        if obj.classifier and obj.classifier[0] is stereotype:
            del element.appliedStereotype[obj]
            obj.unlink()
            break

def get_stereotypes(factory, element):
    """
    Get sorted collection of possible stereotypes for specified element.
    """
    # UML specs does not allow to extend stereotypes with stereotypes
    if isinstance(element, Stereotype):
        return ()

    cls = type(element)

    # find out names of classes, which are superclasses of element class
    names = set(c.__name__ for c in cls.__mro__ if issubclass(c, Element))

    # find stereotypes that extend element class
    classes = factory.select(lambda e: e.isKindOf(Class) and e.name in names)

    stereotypes = set(ext.ownedEnd.type for cls in classes for ext in cls.extension)
    return sorted(stereotypes, key=lambda st: st.name)

def get_applied_stereotypes(element):
    """
    Get collection of applied stereotypes to an element.
    """
    return element.appliedStereotype[:].classifier

def create_extension(factory, element, stereotype):
    """
    Extend an element with a stereotype.
    """
    ext = factory.create(Extension)
    p = factory.create(Property)
    ext_end = factory.create(ExtensionEnd)

    ext.memberEnd = p
    ext.memberEnd = ext_end
    ext.ownedEnd = ext_end
    ext_end.type = stereotype
    ext_end.aggregation = "composite"
    p.type = element
    p.name = "baseClass"
    stereotype.ownedAttribute = p

    return ext

extend_with_stereotype = create_extension

def add_slot(factory, instance, definingFeature):
    """
    Add slot to instance specification for an attribute.
    """
    slot = factory.create(Slot)
    slot.definingFeature = definingFeature
    instance.slot = slot
    return slot

def create_dependency(factory, supplier, client):
    dep = factory.create(Dependency)
    dep.supplier = supplier
    dep.client = client
    return dep

def create_realization(factory, realizingClassifier, abstraction):
    dep = factory.create(Realization)
    dep.realizingClassifier = realizingClassifier
    dep.abstraction = abstraction
    return dep

def create_generalization(factory, general, specific):
    gen = factory.create(Generalization)
    gen.general = general
    gen.specific = specific
    return gen

def create_implementation(factory, contract, implementatingClassifier):
    impl = factory.create(Implementation)
    impl.contract = contract
    impl.implementatingClassifier = implementatingClassifier
    return impl

def create_association(factory, type_a, type_b):
    """
    Create an association between two items.
    """
    assoc = factory.create(Association)
    end_a = factory.create(Property)
    end_b = factory.create(Property)
    assoc.memberEnd = end_a
    assoc.memberEnd = end_b
    end_a.type = type_a
    end_b.type = type_b
    # set default navigability (unknown)
    set_navigability(assoc, end_a, None)
    set_navigability(assoc, end_b, None)
    return assoc

def set_navigability(assoc, end, nav):
    """
    Set navigability of an association end (property).

    There are free possible values for ``nav`` parameter

     True
        association end is navigable
     False
        association end is not navigable
     None
        association end navigability is unkown

    There are two ways of specifing than an end is navigable

    - an end is in Association.navigableOwnedEnd collection
    - an end is class (interface) attribute (stored in Class.ownedAttribute
      collection)

    Let's consider the graph::

        A -----> B
          y    x

    There two association ends A.x and B.y, A.x is navigable.

    Therefore navigable association ends are constructed in following way

    - if A is a class or an interface, then A.x is an attribute owned by A
    - if A is other classifier, then association is more general
      relationship; it may mean that participating instance of B can be
      "accessed efficiently"
      - i.e. when A is a Component, then association may be some compositing
        relationship
      - when A and B are instances of Node class, then it is a
        communication path

    Therefore navigable association end may be stored as one of
    - {Class,Interface}.ownedAttribute due to their capabilities of
      editing owned members
    - Association.navigableOwnedEnd

    When an end has unknown (unspecified) navigability, then it is owned by
    association (but not by classifier).

    When an end is non-navigable, then it is just member of an association.
    """
    # remove "navigable" and "unspecified" navigation indicators first
    if type(end.type) in (Class, Interface):
        owner = end.opposite.type
        if end in owner.ownedAttribute:
            owner.ownedAttribute.remove(end)
    if end in assoc.ownedEnd:
        assoc.ownedEnd.remove(end)
    if end in assoc.navigableOwnedEnd:
        assoc.navigableOwnedEnd.remove(end)
    assert end not in assoc.ownedEnd
    assert end not in assoc.navigableOwnedEnd

    if nav is True:
        if type(end.type) in (Class, Interface):
            owner = end.opposite.type
            owner.ownedAttribute = end
        else:
            assoc.navigableOwnedEnd = end
    elif nav is None:
        assoc.ownedEnd = end
    # elif nav is False, non-navigable

def dependency_type(client, supplier):
    """
    Determine dependency type between client (tail) and supplier
    (arrowhead).

    There can be different dependencies detected automatically

    - usage when supplier is an interface
    - realization when client is component and supplier is a classifier

    If none of above is detected then standard dependency is determined.
    """
    dt = Dependency

    # test interface first as it is a classifier
    if isinstance(supplier, Interface):
        dt = Usage
    elif isinstance(client, Component) and isinstance(supplier, Classifier):
        dt = Realization

    return dt

def create_message(factory, msg, inverted=False):
    """
    Create new message based on speciied message.

    If inverted is set to True, then inverted message is created.
    """
    message = factory.create(Message)
    send = None
    receive = None

    if msg.sendEvent:
        send = factory.create(MessageOccurrenceSpecification)
        sl = msg.sendEvent.covered
        send.covered = sl
    if msg.receiveEvent:
        receive = factory.create(MessageOccurrenceSpecification)
        rl = msg.receiveEvent.covered
        receive.covered = rl

    if inverted:
        # inverted message goes in different direction, than original
        # message
        message.sendEvent = receive
        message.receiveEvent = send
    else:
        message.sendEvent = send
        message.receiveEvent = receive
    return message

# vim:sw=4:et:ai
<commit_msg>fix typos<commit_after>"""
UML model support functions.

Functions collected in this module allow to

- create more complex UML model structures
- perform specific searches and manipulations

"""

import itertools
from gaphor.UML.uml2 import *

# '<<%s>>'
STEREOTYPE_FMT = "<<%s>>"

def stereotypes_str(element, stereotypes=()):
    """
    Identify stereotypes of an UML metamodel instance and return coma
    separated stereotypes as string.

    :Parameters:
     element
        Element having stereotypes, can be None.
     stereotypes
        List of additional stereotypes, can be empty.
    """
    # generate string with stereotype names separated by coma
    if element:
        applied = (stereotype_name(st) for st in get_applied_stereotypes(element))
    else:
        applied = ()
    s = ", ".join(itertools.chain(stereotypes, applied))
    if s:
        return STEREOTYPE_FMT % s
    else:
        return ""

def stereotype_name(stereotype):
    """
    Return stereotype name suggested by UML specification. First will be
    character lowercase unless the second character is uppercase.

    :Parameters:
     stereotype
        Stereotype UML metamodel instance.
    """
    name = stereotype.name
    if not name:
        return ""
    elif len(name) > 1 and name[1].isupper():
        return name
    else:
        return name[0].lower() + name[1:]

def apply_stereotype(factory, element, stereotype):
    """
    Apply a stereotype to an element.

    :Parameters:
     factory
        UML metamodel factory.
     element
        UML metamodel class instance.
     stereotype
        UML metamodel stereotype instance.
    """
    obj = factory.create(InstanceSpecification)
    obj.classifier = stereotype
    element.appliedStereotype = obj
    return obj

def find_instances(factory, element):
    """
    Find instance specification which extend classifier `element`.
    """
    return factory.select(
        lambda e: e.isKindOf(InstanceSpecification)
        and e.classifier
        and e.classifier[0] == element
    )

def remove_stereotype(element, stereotype):
    """
    Remove a stereotype from an element.

    :Parameters:
     element
        UML metamodel element instance.
     stereotype
        UML metamodel stereotype instance.
    """
    for obj in element.appliedStereotype:
        if obj.classifier and obj.classifier[0] is stereotype:
            del element.appliedStereotype[obj]
            obj.unlink()
            break

def get_stereotypes(factory, element):
    """
    Get sorted collection of possible stereotypes for specified element.
    """
    # UML specs does not allow to extend stereotypes with stereotypes
    if isinstance(element, Stereotype):
        return ()

    cls = type(element)

    # find out names of classes, which are superclasses of element class
    names = set(c.__name__ for c in cls.__mro__ if issubclass(c, Element))

    # find stereotypes that extend element class
    classes = factory.select(lambda e: e.isKindOf(Class) and e.name in names)

    stereotypes = set(ext.ownedEnd.type for cls in classes for ext in cls.extension)
    return sorted(stereotypes, key=lambda st: st.name)

def get_applied_stereotypes(element):
    """
    Get collection of applied stereotypes to an element.
    """
    return element.appliedStereotype[:].classifier

def create_extension(factory, element, stereotype):
    """
    Extend an element with a stereotype.
    """
    ext = factory.create(Extension)
    p = factory.create(Property)
    ext_end = factory.create(ExtensionEnd)

    ext.memberEnd = p
    ext.memberEnd = ext_end
    ext.ownedEnd = ext_end
    ext_end.type = stereotype
    ext_end.aggregation = "composite"
    p.type = element
    p.name = "baseClass"
    stereotype.ownedAttribute = p

    return ext

extend_with_stereotype = create_extension

def add_slot(factory, instance, definingFeature):
    """
    Add slot to instance specification for an attribute.
    """
    slot = factory.create(Slot)
    slot.definingFeature = definingFeature
    instance.slot = slot
    return slot

def create_dependency(factory, supplier, client):
    dep = factory.create(Dependency)
    dep.supplier = supplier
    dep.client = client
    return dep

def create_realization(factory, realizingClassifier, abstraction):
    dep = factory.create(Realization)
    dep.realizingClassifier = realizingClassifier
    dep.abstraction = abstraction
    return dep

def create_generalization(factory, general, specific):
    gen = factory.create(Generalization)
    gen.general = general
    gen.specific = specific
    return gen

def create_implementation(factory, contract, implementatingClassifier):
    impl = factory.create(Implementation)
    impl.contract = contract
    impl.implementatingClassifier = implementatingClassifier
    return impl

def create_association(factory, type_a, type_b):
    """
    Create an association between two items.
    """
    assoc = factory.create(Association)
    end_a = factory.create(Property)
    end_b = factory.create(Property)
    assoc.memberEnd = end_a
    assoc.memberEnd = end_b
    end_a.type = type_a
    end_b.type = type_b
    # set default navigability (unknown)
    set_navigability(assoc, end_a, None)
    set_navigability(assoc, end_b, None)
    return assoc

def set_navigability(assoc, end, nav):
    """
    Set navigability of an association end (property).

    There are three possible values for ``nav`` parameter

     True
        association end is navigable
     False
        association end is not navigable
     None
        association end navigability is unkown

    There are two ways of specifing that an end is navigable

    - an end is in Association.navigableOwnedEnd collection
    - an end is class (interface) attribute (stored in Class.ownedAttribute
      collection)

    Let's consider the graph::

        A -----> B
          y    x

    There two association ends A.x and B.y, A.x is navigable.

    Therefore navigable association ends are constructed in following way

    - if A is a class or an interface, then A.x is an attribute owned by A
    - if A is other classifier, then association is more general
      relationship; it may mean that participating instance of B can be
      "accessed efficiently"
      - i.e. when A is a Component, then association may be some compositing
        relationship
      - when A and B are instances of Node class, then it is a
        communication path

    Therefore navigable association end may be stored as one of
    - {Class,Interface}.ownedAttribute due to their capabilities of
      editing owned members
    - Association.navigableOwnedEnd

    When an end has unknown (unspecified) navigability, then it is owned by
    association (but not by classifier).

    When an end is non-navigable, then it is just member of an association.
    """
    # remove "navigable" and "unspecified" navigation indicators first
    if type(end.type) in (Class, Interface):
        owner = end.opposite.type
        if end in owner.ownedAttribute:
            owner.ownedAttribute.remove(end)
    if end in assoc.ownedEnd:
        assoc.ownedEnd.remove(end)
    if end in assoc.navigableOwnedEnd:
        assoc.navigableOwnedEnd.remove(end)
    assert end not in assoc.ownedEnd
    assert end not in assoc.navigableOwnedEnd

    if nav is True:
        if type(end.type) in (Class, Interface):
            owner = end.opposite.type
            owner.ownedAttribute = end
        else:
            assoc.navigableOwnedEnd = end
    elif nav is None:
        assoc.ownedEnd = end
    # elif nav is False, non-navigable

def dependency_type(client, supplier):
    """
    Determine dependency type between client (tail) and supplier
    (arrowhead).

    There can be different dependencies detected automatically

    - usage when supplier is an interface
    - realization when client is component and supplier is a classifier

    If none of above is detected then standard dependency is determined.
    """
    dt = Dependency

    # test interface first as it is a classifier
    if isinstance(supplier, Interface):
        dt = Usage
    elif isinstance(client, Component) and isinstance(supplier, Classifier):
        dt = Realization

    return dt

def create_message(factory, msg, inverted=False):
    """
    Create new message based on speciied message.

    If inverted is set to True, then inverted message is created.
    """
    message = factory.create(Message)
    send = None
    receive = None

    if msg.sendEvent:
        send = factory.create(MessageOccurrenceSpecification)
        sl = msg.sendEvent.covered
        send.covered = sl
    if msg.receiveEvent:
        receive = factory.create(MessageOccurrenceSpecification)
        rl = msg.receiveEvent.covered
        receive.covered = rl

    if inverted:
        # inverted message goes in different direction, than original
        # message
        message.sendEvent = receive
        message.receiveEvent = send
    else:
        message.sendEvent = send
        message.receiveEvent = receive
    return message

# vim:sw=4:et:ai
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""Tools making everyday plotting tasks easier."""

from __future__ import division, print_function
import numpy as np
from matplotlib import pyplot as pl

def plot(function, intervall, num=500, axis=None, **kwargs):
    """Plots the function f on the axisis axis on the intervall (xmin, xmaxis)

    :param function: Functions or list of function to plot
    :param intervall: Intervall to plot function on (xmin, xmaxis)
    :param num: Number of points used for the plot (default 500)
    :param axis: Axis to plot on (default current axisis)
    :returns: Plot (or list of plots)

    """
    if hasattr(function, '__iter__'):
        return [plot(f, intervall, num, axis, **kwargs) for f in function]
    else:
        x = np.linspace(*intervall, num=num)
        axis = pl.gca() if axis is None else axis
        return axis.plot(x, function(x), **kwargs)

def imshow(img, ax=None, **kwargs):
    """Shows the image `img` passed as numpy array in a much prettier way

    :param np.ndarray img: Image to show passed as RGB or grayscale image

    """
    ax = ax if ax is not None else pl.gca()
    ax.grid(False)
    ax.set_xticklabels([])
    ax.set_yticklabels([])

    pl.imshow(img, **kwargs)
    pl.axis((0, img.shape[1], img.shape[0], 0))
    pl.show()
<commit_msg>Function for showing multiple images<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""Tools making everyday plotting tasks easier."""

from __future__ import division, print_function

from itertools import izip
from matplotlib import pyplot as pl

import numpy as np

def plot(function, intervall, num=500, axis=None, **kwargs):
    """Plots the function f on the axisis axis on the intervall (xmin, xmaxis)

    :param function: Functions or list of function to plot
    :param intervall: Intervall to plot function on (xmin, xmaxis)
    :param num: Number of points used for the plot (default 500)
    :param axis: Axis to plot on (default current axisis)
    :returns: Plot (or list of plots)

    """
    if hasattr(function, '__iter__'):
        return [plot(f, intervall, num, axis, **kwargs) for f in function]
    else:
        x = np.linspace(*intervall, num=num)
        axis = pl.gca() if axis is None else axis
        return axis.plot(x, function(x), **kwargs)

def imshow(img, ax=None, show=True, **kwargs):
    """Shows the image `img` passed as numpy array in a much prettier way

    :param np.ndarray img: Image to show passed as RGB or grayscale image
    :param ax: Axis to use for plot (default: current axis)
    :param bool show: Whether to call pl.show() afterwards

    """
    if ax is None:
        ax = pl.gca()

    ax.grid(False)
    ax.set_xticklabels([])
    ax.set_yticklabels([])

    ax.imshow(img, **kwargs)
    ax.axis((0, img.shape[1], img.shape[0], 0))
    if show:
        pl.show()

def imsshow(imgs, layout='h', show=True, **kwargs):
    """Plots a grid of images

    :param imgs: List of images as 2D arrays
    :param layout: 'h' for horizontal layout; 'v' for vertical
    :param bool show: Whether to call pl.show() afterwards
    :param kwargs: Further parameteres for imshow
    :returns: List of axes

    """
    if layout == 'h':
        axlist = [pl.subplot(1, len(imgs), i + 1) for i in range(len(imgs))]
    elif layout == 'v':
        axlist = [pl.subplot(len(imgs), 1, i + 1) for i in range(len(imgs))]
    else:
        raise AttributeError(str(layout) + " is not a valid layout.")

    for ax, img in izip(axlist, imgs):
        imshow(img, ax=ax, show=False)

    if show:
        pl.show()

    return axlist
<|endoftext|>


<commit_before>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from mongokit import IS

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Node, GSystemType
from gnowsys_ndf.ndf.models import Group
from gnowsys_ndf.ndf.models import DATA_TYPE_CHOICES, QUIZ_TYPE_CHOICES_TU
from gnowsys_ndf.settings import GAPPS
from gnowsys_ndf.settings import META_TYPE

####################################################################################################################
#global variables
db = get_database()
collection = db[Node.collection_name]

#append gsystem_type name to create factory GSystemType
factory_gsystem_types = ['Twist', 'Reply', 'Author', 'Shelf', 'QuizItem', 'Pandora_video']

#fill attribute name,data_type,gsystem_type name in bellow dict to create factory Attribute Type
factory_attribute_types = [{'quiz_type':{'gsystem_names_list':['QuizItem'], 'data_type':'str(QUIZ_TYPE_CHOICES_TU)'}},{'options':{'gsystem_names_list':['QuizItem'], 'data_type':'"[" + DATA_TYPE_CHOICES[6] + "]"'}}, {'correct_answer':{'gsystem_names_list':['QuizItem'], 'data_type':'"[" + DATA_TYPE_CHOICES[6] + "]"'}}, {'start_time':{'gsystem_names_list':['QuizItem','Forum'], 'data_type':'DATA_TYPE_CHOICES[9]'}}, {'end_time':{'gsystem_names_list':['QuizItem','Forum'], 'data_type':'DATA_TYPE_CHOICES[9]'}}, {'module_set_md5':{'gsystem_names_list':['Module'], 'data_type':'u""'}},{'source_id':{'gsystem_names_list':['Pandora_video'], 'data_type':'u""'}} ]

#fill relation_type_name,inverse_name,subject_type,object_type in bellow dict to create factory Relation Type
factory_relation_types = [{'has_shelf':{'subject_type':['Author'],'object_type':['Shelf'], 'inverse_name':'shelf_of'}}, {'translation_of':{'subject_type':['Page'],'object_type':['Page'], 'inverse_name':'translation_of'}}, {'has_module':{'subject_type':['Page'],'object_type':['Module'], 'inverse_name':'generated_from'}}, {'has_profile_pic':{'subject_type':['Author'],'object_type':['Image'], 'inverse_name':'profile_pic_of'}}]

#for taking input json file
import json
import sys

filename = sys.argv[-1]
print filename,"test"
f = filename.split("/")
if f[-1] == "ATs.json" or f[-1] == "RTs.json" or f[-1] == "STs.json":
    json_file = open(filename)
else:
    json_file = ""

class Command(BaseCommand):
    help = "Based on GAPPS list, inserts few documents (consisting of dummy values) into the following collections:\n\n" \
           "\tGSystemType, AttributeType"

    def handle(self, *args, **options):
        user_id = 1
        node_doc = None
        meta_type_name = META_TYPE[0]
        meta_type = collection.GSystemType.one({'$and':[{'_type':'MetaType'},{'name':meta_type_name}]}) # getting MetaType Object
        if meta_type == None:
            meta_type = create_meta_type(user_id) #creating MetaType

        for each in GAPPS:
            node_doc = collection.GSystemType.one({'$and':[{'_type':'GSystemType'},{'name':each}]})
            if (node_doc == None or each != node_doc['name']):
                gst_node=collection.GSystemType()
                gst_node.name = unicode(each)
                gst_node.created_by = user_id
                gst_node.modified_by = user_id
                if user_id not in gst_node.contributors:
                    gst_node.contributors.append(user_id)
                gst_node.member_of.append(meta_type._id) # appending metatype to the GSystemType
                gst_node.save()
            elif(meta_type._id not in node_doc.member_of):
                 node_doc.member_of.append(meta_type._id)
                 node_doc.save()

        #Create default group 'home'
        node_doc =collection.GSystemType.one({'$and':[{'_type': u'Group'},{'name': u'home'}]})
        if node_doc is None:
            gs_node = collection.Group()
            gs_node.name = u'home'
            gs_node.created_by = user_id
            gs_node.modified_by = user_id
            if user_id not in gs_node.contributors:
                gs_node.contributors.append(user_id)
            gs_node.member_of.append(collection.Node.one({'name': "Group"})._id)
            gs_node.disclosure_policy=u'DISCLOSED_TO_MEM'
            gs_node.subscription_policy=u'OPEN'
            gs_node.visibility_policy=u'ANNOUNCED'
            gs_node.encryption_policy=u'NOT_ENCRYPTED'
            gs_node.group_type= u'PUBLIC'
            gs_node.edit_policy =u'NON_EDITABLE'
            gs_node.save()

        #creating factory GSystemType's
        create_sts(factory_gsystem_types,user_id)
        #creating factory RelationType's
        create_rts(factory_relation_types,user_id)

        #creating factory AttributeType's
        create_ats(factory_attribute_types,user_id)

        #creating  AttributeType's, RelationType's and SystemType's by json file as input
        if json_file:
            a = json_file.name.split('/')
            if a[-1] == 'ATs.json':
                json_data = json.loads(json_file.read())
                create_ats(json_data,user_id)
            elif a[-1] == 'RTs.json':
                json_data = json.loads(json_file.read())
                #print json_data,"Test_RTS json"
                create_rts(json_data,user_id)
            elif a[-1] == "STs.json":
                json_data = json.loads(json_file.read())
                #print json_data,"Test_STs json"
                create_sts(json_data,user_id)
            else:
                print 'file name should be ATs.json,STs.json or RTs.json to load Ats,STs or RTs of json'

        # # Retrieve 'Quiz' GSystemType's id -- in order to append it to 'meta_type_set' for 'QuizItem' GSystemType
        quiz_type = collection.GSystemType.one({'_type': u'GSystemType', 'name': u'Quiz'})

        quiz_item_type = collection.Node.one({'_type': u'GSystemType', 'name': u'QuizItem'})

        # Append quiz_type, options & correct_answer to attribute_type_set of 'QuizItem'
        if not quiz_item_type.attribute_type_set:
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'quiz_type'}))
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'options'}))
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'correct_answer'}))
            quiz_item_type.save()

        # Append start_time & end_time to attribute_type_set of 'Quiz'
        if not quiz_type.attribute_type_set:
            quiz_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'start_time'}))
            quiz_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'end_time'}))
            quiz_type.save()

        # --- End of handle() ---

def create_meta_type(user_id):
    '''
    creating meta_type in database
    '''
    meta = collection.MetaType()
    meta.name = META_TYPE[0]
    meta.created_by = user_id # default hardcode
    meta.modified_by = user_id
    if user_id not in meta.contributors:
        meta.contributors.append(user_id)
    meta.save()
    return meta

def create_gsystem_type(st_name, user_id):
    '''
    creating factory GSystemType's
    '''
    node = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name':st_name}]})
    if node is None:
        try:
            gs_node = collection.GSystemType()
            gs_node.name = unicode(st_name)
            gs_node.created_by = user_id
            gs_node.modified_by = user_id
            if user_id not in gs_node.contributors:
                gs_node.contributors.append(user_id)
            gs_node.save()
            print 'created', st_name, 'as', 'GSystemType'
        except Exception as e:
            print 'GsystemType',st_name,'fails to create because:',e
    else:
        print 'GSystemType',st_name,'already created'

def create_attribute_type(at_name, user_id, data_type, system_type_id_list):
    '''
    creating factory AttributeType's
    '''
    node = collection.Node.one({'$and':[{'_type': u'AttributeType'},{'name':at_name}]})
    if node is None:
        try:
            at = collection.AttributeType()
            at.name = unicode(at_name)
            at.created_by = user_id
            at.modified_by = user_id
            if user_id not in at.contributors:
                at.contributors.append(user_id)
            at.data_type = data_type
            for each in system_type_id_list:
                at.subject_type.append(each)
            at.save()
            print 'created', at_name, 'as', 'AttributeType'
        except Exception as e:
            print 'AttributeType',at_name,'fails to create because:',e
    else:
        print 'AttributeType',at_name,'already created'

def create_relation_type(rt_name, inverse_name, user_id, subject_type_id_list, object_type_id_list):
    '''
    creating factory RelationType's
    '''
    rt_node = collection.RelationType.one({'_type': u'RelationType', 'name': rt_name})
    if rt_node is None:
        try:
            rt_node = collection.RelationType()
            rt_node.name = unicode(rt_name)
            rt_node.inverse_name = unicode(inverse_name)
            for st_id in subject_type_id_list:
                rt_node.subject_type.append(st_id)
            for ot_id in object_type_id_list:
                rt_node.object_type.append(ot_id)
            rt_node.created_by = user_id
            rt_node.modified_by = user_id
            if user_id not in rt_node.contributors:
                rt_node.contributors.append(user_id)
            rt_node.save()
            print 'created', rt_name, 'as', 'RelationType'
        except Exception as e:
            print 'RelationType',rt_name,'fails to create because:',e
    else:
        print 'RelationType',rt_node.name,'already created'

def create_ats(factory_attribute_types,user_id):
    for each in factory_attribute_types:
        gsystem_id_list = []
        for key,value in each.items():
            at_name = key
            data_type = value['data_type']
            for e in value['gsystem_names_list']:
                node = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': e}]})
                if node is not None:
                    gsystem_id_list.append(node._id)
                else:
                    print e,"as GSystemType not present in database"
        create_attribute_type(at_name, user_id, data_type, gsystem_id_list)

def create_rts(factory_relation_types,user_id):
    for each in factory_relation_types:
        subject_type_id_list = []
        object_type_id_list = []
        for key,value in each.items():
            at_name = key
            inverse_name = value['inverse_name']
            for s in value['subject_type']:
                node_s = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': s}]})
                subject_type_id_list.append(node_s._id)
            for rs in value['object_type']:
                node_rs = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': rs}]})
                object_type_id_list.append(node_rs._id)
        create_relation_type(at_name, inverse_name, user_id, subject_type_id_list, object_type_id_list)

def create_sts(factory_gsystem_types,user_id):
    for each in factory_gsystem_types:
        create_gsystem_type(each, user_id)

# Update type_of field to list
type_of_cursor=collection.find({'type_of':{'$exists':True}})
for object_cur in type_of_cursor:
    if type(object_cur['type_of']) == ObjectId or object_cur['type_of'] == None:
	if type(object_cur['type_of']) == ObjectId :
		collection.update({'_id':object_cur['_id']},{'$set':{'type_of':[object_cur['type_of']]}})
	else :
		collection.update({'_id':object_cur['_id']},{'$set':{'type_of':[]}})

# ===============================================================================================

# Removes n attribute if created accidently in existsing documents
collection.update({'n': {'$exists': True}}, {'$unset': {'n': ""}}, upsert=False, multi=True)

# Updates wherever modified_by field is None with default value as either first contributor or the creator of the resource
modified_by_cur = collection.Node.find({'_type': {'$nin': ['GAttribute', 'GRelation']}, 'modified_by': None})
if modified_by_cur.count > 0:
    for n in modified_by_cur:
        if n.contributors:
            collection.update({'_id': n._id}, {'$set': {'modified_by': n.contributors[0]}}, upsert=False, multi=False)
        else:
            if n.created_by:
                collection.update({'_id': n._id}, {'$set': {'modified_by': n.created_by, 'contributors': [n.created_by]}}, upsert=False, multi=False)
            else:
                print "\n Please set created_by value for node (", n._id, " -- ", n._type, " : ", n.name, ")\n"

# Updating faulty modified_by and contributors values (in case of user-group and file documents)
cur = collection.Node.find({'modified_by': {'$exists': True}})
for n in cur:
    # By faulty, it means modified_by and contributors has 1 as their values
    # 1 stands for superuser
    # Instead of this value should be the creator of that resource
    # (even this is applicable only if created_by field of that resource holds some value)
    if not n.created_by:
        print "\n Please set created_by value for node (", n._id, " -- ", n._type, " : ", n.name, ")"
    else:
        if n.created_by not in n.contributors:
            collection.update({'_id': n._id}, {'$set': {'modified_by': n.created_by, 'contributors': [n.created_by]} }, upsert=False, multi=False)

# For delete the profile_pic as GST
profile_pic_obj = collection.Node.one({'_type': 'GSystemType','name': u'profile_pic'})
if profile_pic_obj:
	profile_pic_obj.delete()
	print "Deleted GST document of profile_pic"<commit_msg>visited_location field added into Author class with default value sets to []<commit_after>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from mongokit import IS

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import Node, GSystemType
from gnowsys_ndf.ndf.models import Group
from gnowsys_ndf.ndf.models import DATA_TYPE_CHOICES, QUIZ_TYPE_CHOICES_TU
from gnowsys_ndf.settings import GAPPS
from gnowsys_ndf.settings import META_TYPE

####################################################################################################################
#global variables
db = get_database()
collection = db[Node.collection_name]

#append gsystem_type name to create factory GSystemType
factory_gsystem_types = ['Twist', 'Reply', 'Author', 'Shelf', 'QuizItem', 'Pandora_video']

#fill attribute name,data_type,gsystem_type name in bellow dict to create factory Attribute Type
factory_attribute_types = [{'quiz_type':{'gsystem_names_list':['QuizItem'], 'data_type':'str(QUIZ_TYPE_CHOICES_TU)'}},{'options':{'gsystem_names_list':['QuizItem'], 'data_type':'"[" + DATA_TYPE_CHOICES[6] + "]"'}}, {'correct_answer':{'gsystem_names_list':['QuizItem'], 'data_type':'"[" + DATA_TYPE_CHOICES[6] + "]"'}}, {'start_time':{'gsystem_names_list':['QuizItem','Forum'], 'data_type':'DATA_TYPE_CHOICES[9]'}}, {'end_time':{'gsystem_names_list':['QuizItem','Forum'], 'data_type':'DATA_TYPE_CHOICES[9]'}}, {'module_set_md5':{'gsystem_names_list':['Module'], 'data_type':'u""'}},{'source_id':{'gsystem_names_list':['Pandora_video'], 'data_type':'u""'}} ]

#fill relation_type_name,inverse_name,subject_type,object_type in bellow dict to create factory Relation Type
factory_relation_types = [{'has_shelf':{'subject_type':['Author'],'object_type':['Shelf'], 'inverse_name':'shelf_of'}}, {'translation_of':{'subject_type':['Page'],'object_type':['Page'], 'inverse_name':'translation_of'}}, {'has_module':{'subject_type':['Page'],'object_type':['Module'], 'inverse_name':'generated_from'}}, {'has_profile_pic':{'subject_type':['Author'],'object_type':['Image'], 'inverse_name':'profile_pic_of'}}]

#for taking input json file
import json
import sys

filename = sys.argv[-1]
print filename,"test"
f = filename.split("/")
if f[-1] == "ATs.json" or f[-1] == "RTs.json" or f[-1] == "STs.json":
    json_file = open(filename)
else:
    json_file = ""

class Command(BaseCommand):
    help = "Based on GAPPS list, inserts few documents (consisting of dummy values) into the following collections:\n\n" \
           "\tGSystemType, AttributeType"

    def handle(self, *args, **options):
        user_id = 1
        node_doc = None
        meta_type_name = META_TYPE[0]
        meta_type = collection.GSystemType.one({'$and':[{'_type':'MetaType'},{'name':meta_type_name}]}) # getting MetaType Object
        if meta_type == None:
            meta_type = create_meta_type(user_id) #creating MetaType

        for each in GAPPS:
            node_doc = collection.GSystemType.one({'$and':[{'_type':'GSystemType'},{'name':each}]})
            if (node_doc == None or each != node_doc['name']):
                gst_node=collection.GSystemType()
                gst_node.name = unicode(each)
                gst_node.created_by = user_id
                gst_node.modified_by = user_id
                if user_id not in gst_node.contributors:
                    gst_node.contributors.append(user_id)
                gst_node.member_of.append(meta_type._id) # appending metatype to the GSystemType
                gst_node.save()
            elif(meta_type._id not in node_doc.member_of):
                 node_doc.member_of.append(meta_type._id)
                 node_doc.save()

        #Create default group 'home'
        node_doc =collection.GSystemType.one({'$and':[{'_type': u'Group'},{'name': u'home'}]})
        if node_doc is None:
            gs_node = collection.Group()
            gs_node.name = u'home'
            gs_node.created_by = user_id
            gs_node.modified_by = user_id
            if user_id not in gs_node.contributors:
                gs_node.contributors.append(user_id)
            gs_node.member_of.append(collection.Node.one({'name': "Group"})._id)
            gs_node.disclosure_policy=u'DISCLOSED_TO_MEM'
            gs_node.subscription_policy=u'OPEN'
            gs_node.visibility_policy=u'ANNOUNCED'
            gs_node.encryption_policy=u'NOT_ENCRYPTED'
            gs_node.group_type= u'PUBLIC'
            gs_node.edit_policy =u'NON_EDITABLE'
            gs_node.save()

        #creating factory GSystemType's
        create_sts(factory_gsystem_types,user_id)
        #creating factory RelationType's
        create_rts(factory_relation_types,user_id)

        #creating factory AttributeType's
        create_ats(factory_attribute_types,user_id)

        #creating  AttributeType's, RelationType's and SystemType's by json file as input
        if json_file:
            a = json_file.name.split('/')
            if a[-1] == 'ATs.json':
                json_data = json.loads(json_file.read())
                create_ats(json_data,user_id)
            elif a[-1] == 'RTs.json':
                json_data = json.loads(json_file.read())
                #print json_data,"Test_RTS json"
                create_rts(json_data,user_id)
            elif a[-1] == "STs.json":
                json_data = json.loads(json_file.read())
                #print json_data,"Test_STs json"
                create_sts(json_data,user_id)
            else:
                print 'file name should be ATs.json,STs.json or RTs.json to load Ats,STs or RTs of json'

        # # Retrieve 'Quiz' GSystemType's id -- in order to append it to 'meta_type_set' for 'QuizItem' GSystemType
        quiz_type = collection.GSystemType.one({'_type': u'GSystemType', 'name': u'Quiz'})

        quiz_item_type = collection.Node.one({'_type': u'GSystemType', 'name': u'QuizItem'})

        # Append quiz_type, options & correct_answer to attribute_type_set of 'QuizItem'
        if not quiz_item_type.attribute_type_set:
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'quiz_type'}))
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'options'}))
            quiz_item_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'correct_answer'}))
            quiz_item_type.save()

        # Append start_time & end_time to attribute_type_set of 'Quiz'
        if not quiz_type.attribute_type_set:
            quiz_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'start_time'}))
            quiz_type.attribute_type_set.append(collection.Node.one({'_type': u'AttributeType', 'name': u'end_time'}))
            quiz_type.save()

        # --- End of handle() ---

def create_meta_type(user_id):
    '''
    creating meta_type in database
    '''
    meta = collection.MetaType()
    meta.name = META_TYPE[0]
    meta.created_by = user_id # default hardcode
    meta.modified_by = user_id
    if user_id not in meta.contributors:
        meta.contributors.append(user_id)
    meta.save()
    return meta

def create_gsystem_type(st_name, user_id):
    '''
    creating factory GSystemType's
    '''
    node = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name':st_name}]})
    if node is None:
        try:
            gs_node = collection.GSystemType()
            gs_node.name = unicode(st_name)
            gs_node.created_by = user_id
            gs_node.modified_by = user_id
            if user_id not in gs_node.contributors:
                gs_node.contributors.append(user_id)
            gs_node.save()
            print 'created', st_name, 'as', 'GSystemType'
        except Exception as e:
            print 'GsystemType',st_name,'fails to create because:',e
    else:
        print 'GSystemType',st_name,'already created'

def create_attribute_type(at_name, user_id, data_type, system_type_id_list):
    '''
    creating factory AttributeType's
    '''
    node = collection.Node.one({'$and':[{'_type': u'AttributeType'},{'name':at_name}]})
    if node is None:
        try:
            at = collection.AttributeType()
            at.name = unicode(at_name)
            at.created_by = user_id
            at.modified_by = user_id
            if user_id not in at.contributors:
                at.contributors.append(user_id)
            at.data_type = data_type
            for each in system_type_id_list:
                at.subject_type.append(each)
            at.save()
            print 'created', at_name, 'as', 'AttributeType'
        except Exception as e:
            print 'AttributeType',at_name,'fails to create because:',e
    else:
        print 'AttributeType',at_name,'already created'

def create_relation_type(rt_name, inverse_name, user_id, subject_type_id_list, object_type_id_list):
    '''
    creating factory RelationType's
    '''
    rt_node = collection.RelationType.one({'_type': u'RelationType', 'name': rt_name})
    if rt_node is None:
        try:
            rt_node = collection.RelationType()
            rt_node.name = unicode(rt_name)
            rt_node.inverse_name = unicode(inverse_name)
            for st_id in subject_type_id_list:
                rt_node.subject_type.append(st_id)
            for ot_id in object_type_id_list:
                rt_node.object_type.append(ot_id)
            rt_node.created_by = user_id
            rt_node.modified_by = user_id
            if user_id not in rt_node.contributors:
                rt_node.contributors.append(user_id)
            rt_node.save()
            print 'created', rt_name, 'as', 'RelationType'
        except Exception as e:
            print 'RelationType',rt_name,'fails to create because:',e
    else:
        print 'RelationType',rt_node.name,'already created'

def create_ats(factory_attribute_types,user_id):
    for each in factory_attribute_types:
        gsystem_id_list = []
        for key,value in each.items():
            at_name = key
            data_type = value['data_type']
            for e in value['gsystem_names_list']:
                node = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': e}]})
                if node is not None:
                    gsystem_id_list.append(node._id)
                else:
                    print e,"as GSystemType not present in database"
        create_attribute_type(at_name, user_id, data_type, gsystem_id_list)

def create_rts(factory_relation_types,user_id):
    for each in factory_relation_types:
        subject_type_id_list = []
        object_type_id_list = []
        for key,value in each.items():
            at_name = key
            inverse_name = value['inverse_name']
            for s in value['subject_type']:
                node_s = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': s}]})
                subject_type_id_list.append(node_s._id)
            for rs in value['object_type']:
                node_rs = collection.Node.one({'$and':[{'_type': u'GSystemType'},{'name': rs}]})
                object_type_id_list.append(node_rs._id)
        create_relation_type(at_name, inverse_name, user_id, subject_type_id_list, object_type_id_list)

def create_sts(factory_gsystem_types,user_id):
    for each in factory_gsystem_types:
        create_gsystem_type(each, user_id)

# Update type_of field to list
type_of_cursor=collection.find({'type_of':{'$exists':True}})
for object_cur in type_of_cursor:
    if type(object_cur['type_of']) == ObjectId or object_cur['type_of'] == None:
	if type(object_cur['type_of']) == ObjectId :
		collection.update({'_id':object_cur['_id']},{'$set':{'type_of':[object_cur['type_of']]}})
	else :
		collection.update({'_id':object_cur['_id']},{'$set':{'type_of':[]}})

# ===============================================================================================

# Removes n attribute if created accidently in existsing documents
collection.update({'n': {'$exists': True}}, {'$unset': {'n': ""}}, upsert=False, multi=True)

# Updates wherever modified_by field is None with default value as either first contributor or the creator of the resource
modified_by_cur = collection.Node.find({'_type': {'$nin': ['GAttribute', 'GRelation']}, 'modified_by': None})
if modified_by_cur.count > 0:
    for n in modified_by_cur:
        if n.contributors:
            collection.update({'_id': n._id}, {'$set': {'modified_by': n.contributors[0]}}, upsert=False, multi=False)
        else:
            if n.created_by:
                collection.update({'_id': n._id}, {'$set': {'modified_by': n.created_by, 'contributors': [n.created_by]}}, upsert=False, multi=False)
            else:
                print "\n Please set created_by value for node (", n._id, " -- ", n._type, " : ", n.name, ")\n"

# Updating faulty modified_by and contributors values (in case of user-group and file documents)
cur = collection.Node.find({'modified_by': {'$exists': True}})
for n in cur:
    # By faulty, it means modified_by and contributors has 1 as their values
    # 1 stands for superuser
    # Instead of this value should be the creator of that resource
    # (even this is applicable only if created_by field of that resource holds some value)
    if not n.created_by:
        print "\n Please set created_by value for node (", n._id, " -- ", n._type, " : ", n.name, ")"
    else:
        if n.created_by not in n.contributors:
            collection.update({'_id': n._id}, {'$set': {'modified_by': n.created_by, 'contributors': [n.created_by]} }, upsert=False, multi=False)

# For delete the profile_pic as GST
profile_pic_obj = collection.Node.one({'_type': 'GSystemType','name': u'profile_pic'})
if profile_pic_obj:
	profile_pic_obj.delete()
	print "Deleted GST document of profile_pic"

# For adding visited_location field (default value set as []) in User Groups.
try:
    author = collection.Node.one({'_type': "GSystemType", 'name': "Author"})
    if author:
        cur = collection.Node.find({'_type': "Group", 'member_of': author._id, 'visited_location': {'$exists': False}})

        if cur.count():
            print "\n"
            for each in cur:
                collection.update({'_type': "Group", '_id': each._id}, {'$set': {'visited_location': []}}, upsert=False, multi=True)
                print " 'visited_location' field added to Author group (" + each.name + ")\n"

    else:
        error_message = "\n Exception while creating 'visited_location' field in Author class.\n Author GSystemType doesn't exists!!!\n"
        raise Exception(error_message)

except Exception as e:
    print str(e)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
/***************************************************************************
 ORStools
                                 A QGIS plugin
 falk
                              -------------------
        begin                : 2017-02-01
        git sha              : $Format:%H$
        copyright            : (C) 2017 by Nils Nolde
        email                : nils.nolde@gmail.com
 ***************************************************************************/

 This plugin provides access to the various APIs from OpenRouteService
 (http://openrouteservice.readthedocs.io/en/1.0/api.html), developed and
 maintained by GIScience team at University of Heidelberg, Germany. By using
 this plugin you agree to the ORS terms of service
 (http://openrouteservice.readthedocs.io/en/1.0/tos.html#terms-of-service).

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
from PyQt4.QtCore import *
from PyQt4.QtGui import *
# Initialize Qt resources from file resources.py
import resources_rc
# Import the code for the dialog
from ors_tools_dialog import ORStoolsDialog
import os.path

import ors_tools_access
import ors_tools_routing

from qgis.core import *
import qgis.gui
import qgis.utils

import logging

logging.basicConfig(format='%(levelname)s:%(message)s', level = logging.INFO)

class ORStools:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface
        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)
        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir,
            'i18n',
            'ORStools_{}.qm'.format(locale))

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)

            if qVersion() > '4.3.3':
                QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&ORS Tools')
        # TODO: We are going to let the user set this up in a future iteration
        self.toolbar = self.iface.addToolBar(u'ORStools')
        self.toolbar.setObjectName(u'ORStools')

        #custom __init__ declarations
        self.dlg = ORStoolsDialog()

        self.canvas = qgis.utils.iface.mapCanvas()
        self.script_dir = os.path.dirname(os.path.abspath(__file__))

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('ORStools', message)

    def add_action(
        self,
        icon_path,
        text,
        callback,
        enabled_flag=True,
        add_to_menu=True,
        add_to_toolbar=True,
        status_tip=None,
        whats_this=None,
        parent=None):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        # Create the dialog (after translation) and keep reference

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            self.toolbar.addAction(action)

        if add_to_menu:
            self.iface.addPluginToMenu(
                self.menu,
                action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/ORStools/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'ORS Tools'),
            callback=self.run,
            parent=self.iface.mainWindow())

        self.dlg.api_key.textChanged.connect(self.keyWriter)

        self.dlg.key_order.setText("Get Key!")
        self.dlg.key_order.connect(self.dlg.key_order, SIGNAL("linkActivated(QString)"), self.OpenURL)
        self.dlg.header_2.linkActivated.connect(self.OpenURL)
        self.dlg.header_3.linkActivated.connect(self.OpenURL)

    def OpenURL(self, URL):
          QDesktopServices().openUrl(QUrl(URL))

    def unload(self):
        self.dlg.close()
        QApplication.restoreOverrideCursor()

    def run(self):
        """Run method that performs all the real work"""

        # Populate the api key lineEdit widget
        with open(os.path.join(self.script_dir, "apikey.txt")) as key:
            self.dlg.api_key.setText(key.read())

        # Initiate analysis classes
        self.access_anal = ors_tools_access.accessAnalysis(self.dlg)
        self.route_anal = ors_tools_routing.routing(self.dlg)

        self.dlg.setFixedSize(self.dlg.size())

        self.dlg.show()

        # show the dialog
        # Run the dialog event loop
        result = self.dlg.exec_()
        # See if OK was pressed
        if result:
            if self.dlg.tabWidget.currentIndex() == 0 and self.dlg.use_layer.isChecked():
                self.access_anal.iterAnalysis()

            elif self.dlg.tabWidget.currentIndex() == 1:
                self.route_anal.route()
        else:
            self.unload()

    def keyWriter(self):
        with open(os.path.join(self.script_dir, "apikey.txt"), 'w') as key:
            return key.write(self.dlg.api_key.text())

def CheckCRS(self,crs):
    check = True
    if crs != "EPSG:4326":
        msg = "CRS is {}. Must be EPSG:4326 (WGS84)".format(crs)
        qgis.utils.iface.messageBar().pushMessage(msg, level = qgis.gui.QgsMessageBar.CRITICAL)
        check = False
    return check

<commit_msg>Delete ors_tools.py<commit_after><|endoftext|>