<commit_before># acsn2hugo.py -- Convert ACSN gene to HUGO gene
#
# jean-daniel.granet@mines-paristech.fr

import sys
import argparse

def main():
    # get arguments :
    parser = argparse.ArgumentParser(description='Convert ACSN gene to HUGO gene')
    parser.add_argument('acsn', help='ACSN to convert in .sif format')
    parser.add_argument('curated', help='reference file in gmt format')
    parser.add_argument('hugo', help='HUGO file to create')
    args = parser.parse_args()

    curated = dict()
    # read the curated file and save into a dictionary :
    # the key is the acsn gene name and the value is the hugo gene name
    # curated [ "acsn entity name" ] = "hugo\tgene\tnames\tin\tthe\tacsn\tentity"
    with open(args.curated, 'r') as fdCurated:
        for line in fdCurated:
            line_split = line.split('\tna\t')
            curated[line_split[0].strip()] = line_split[1].strip()
        fdCurated.close()

    # read the acsn network file and write the corresponding hugo network into the output file
    with open(args.acsn, 'r') as fdAcsn:
        with open(args.hugo, 'w') as fdHugo:
            for line in fdAcsn:
                line_split = line.split('\t')
                geneA = curated[line_split[0].strip()].split()
                geneB = curated[line_split[2].strip()].split()
                for elemA in geneA:
                    for elemB in geneB:
                        fdHugo.write("%s\n" % "\n".join(["%s %s %s" % (elemA, line_split[1], elemB)]))
            fdHugo.close()
        fdAcsn.close()

if __name__ == "__main__":
    main()
<commit_msg>Add infos on curated.gmt file structure<commit_after># acsn2hugo.py -- Convert ACSN gene to HUGO gene
#
# jean-daniel.granet@mines-paristech.fr

import sys
import argparse

def main():
    # get arguments :
    parser = argparse.ArgumentParser(description='Convert ACSN gene to HUGO gene')
    parser.add_argument('acsn', help='ACSN to convert in .sif format')
    parser.add_argument('curated', help='reference file in gmt format')
    parser.add_argument('hugo', help='HUGO file to create')
    args = parser.parse_args()

    curated = dict()
    # read the curated file and save into a dictionary :
    # the key is the acsn gene name and the value is the hugo gene name
    # curated [ "acsn entity name" ] = "hugo\tgene\tnames\tin\tthe\tacsn\tentity"
    with open(args.curated, 'r') as fdCurated:
        # File structure : acsn entity name\t description (=na)\tgene\list\in\tHUGO\symbol
        for line in fdCurated:
            line_split = line.split('\tna\t')
            curated[line_split[0].strip()] = line_split[1].strip()
        fdCurated.close()

    # read the acsn network file and write the corresponding hugo network into the output file
    with open(args.acsn, 'r') as fdAcsn:
        with open(args.hugo, 'w') as fdHugo:
            for line in fdAcsn:
                line_split = line.split('\t')
                geneA = curated[line_split[0].strip()].split()
                geneB = curated[line_split[2].strip()].split()
                for elemA in geneA:
                    for elemB in geneB:
                        fdHugo.write("%s\n" % "\n".join(["%s %s %s" % (elemA, line_split[1], elemB)]))
            fdHugo.close()
        fdAcsn.close()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import requests

class Session(object):

    def __init__(self, token=None):
        self.next_call = None
        self.session = requests.Session()
        self.token = token

    def query(self, end_point_str, param_dict=None):

        if param_dict is not None:
            param_dict.update({"token": self.token})
        else:
            param_dict = {
                "format": "json",
                "token": self.token
            }

        response = self.session.get("http://webhose.io/" + end_point_str, params=param_dict)
        if response.status_code != 200:
            raise Exception(response.text)

        _output = response.json()
        self.next_call = _output['next']
        return _output

    def get_next(self):
        return self.query(self.next_call[1:])

__session = Session()

def config(token):
    __session.token = token

def query(end_point_str, param_dict=None):
    return __session.query(end_point_str, param_dict)

def get_next():
    return __session.get_next()

<commit_msg>update the format<commit_after>import requests

class Session(object):

    def __init__(self, token=None):
        self.next_call = None
        self.session = requests.Session()
        self.token = token

    def query(self, end_point_str, param_dict=None):

        if param_dict is not None:
            param_dict.update({"token": self.token})
            param_dict.update({"format": "json"})
        else:
            param_dict = {
                "format": "json",
                "token": self.token
            }

        response = self.session.get("http://webhose.io/" + end_point_str, params=param_dict)
        if response.status_code != 200:
            raise Exception(response.text)

        _output = response.json()
        self.next_call = _output['next']
        return _output

    def get_next(self):
        return self.query(self.next_call[1:])

__session = Session()

def config(token):
    __session.token = token

def query(end_point_str, param_dict=None):
    return __session.query(end_point_str, param_dict)

def get_next():
    return __session.get_next()

<|endoftext|>


<commit_before>"""
Advanced utilities for traversing nesting/hierarchical Dimensioned
objects either to inspect the structure of their declared dimensions
or mutate the matching elements.
"""

from operator import itemgetter

from .dimension import Dimension

def create_ndkey(length, indexes, values):
    key = [None] * length
    for i, v in zip(indexes, values):
        key[i] = v
    return tuple(key)

def uniform(obj):
    """
    Finds all common dimension keys in the object including subsets of
    dimensions. If there are is no common subset of dimensions, None
    is returned.
    """
    dim_groups = obj.traverse(lambda x: tuple(x.key_dimensions),
                              ('HoloMap',))
    if dim_groups:
        return all(set(g1) <= set(g2) or set(g1) >= set(g2)
                   for g1 in dim_groups for g2 in dim_groups)
    return True

def unique_dimkeys(obj, default_dim='Frame'):
    """
    Finds all common dimension keys in the object including subsets of
    dimensions. If there are is no common subset of dimensions, None
    is returned.

    Returns the list of dimensions followed by the list of unique
    keys.
    """
    from .ndmapping import NdMapping
    key_dims = obj.traverse(lambda x: (tuple(x.key_dimensions),
                                       list(x.data.keys())), ('HoloMap',))
    if not key_dims:
        return [Dimension(default_dim)], [(0,)]
    dim_groups, keys = zip(*sorted(key_dims, key=lambda x: -len(x[0])))
    subset = all(set(g1) <= set(g2) or set(g1) >= set(g2)
               for g1 in dim_groups for g2 in dim_groups)
    # Find unique keys
    all_dims = sorted({dim for dim_group in dim_groups
                       for dim in dim_group},
                      key=lambda x: dim_groups[0].index(x))
    ndims = len(all_dims)
    unique_keys = []
    for group, keys in key_dims:
        dim_idxs = [all_dims.index(dim) for dim in group]
        for k in keys:
            matches = [item for item in unique_keys
                       if k == (itemgetter(*dim_idxs)(item)
                                if len(dim_idxs) != 1 else (item[dim_idxs[0]]),)]
            if not matches:
                unique_keys.append(create_ndkey(ndims, dim_idxs, k))
    sorted_keys = NdMapping({key: None for key in unique_keys},
                            key_dimensions=all_dims).data.keys()
    if subset:
        return all_dims, list(sorted_keys)
    else:
        return [default_dim], [i for i in range(len(unique_keys))]
<commit_msg>Fixed unique_dimkeys in traversal<commit_after>"""
Advanced utilities for traversing nesting/hierarchical Dimensioned
objects either to inspect the structure of their declared dimensions
or mutate the matching elements.
"""

from operator import itemgetter

from .dimension import Dimension

def create_ndkey(length, indexes, values):
    key = [None] * length
    for i, v in zip(indexes, values):
        key[i] = v
    return tuple(key)

def uniform(obj):
    """
    Finds all common dimension keys in the object including subsets of
    dimensions. If there are is no common subset of dimensions, None
    is returned.
    """
    dim_groups = obj.traverse(lambda x: tuple(x.key_dimensions),
                              ('HoloMap',))
    if dim_groups:
        return all(set(g1) <= set(g2) or set(g1) >= set(g2)
                   for g1 in dim_groups for g2 in dim_groups)
    return True

def unique_dimkeys(obj, default_dim='Frame'):
    """
    Finds all common dimension keys in the object including subsets of
    dimensions. If there are is no common subset of dimensions, None
    is returned.

    Returns the list of dimensions followed by the list of unique
    keys.
    """
    from .ndmapping import NdMapping
    key_dims = obj.traverse(lambda x: (tuple(x.key_dimensions),
                                       list(x.data.keys())), ('HoloMap',))
    if not key_dims:
        return [Dimension(default_dim)], [(0,)]
    dim_groups, keys = zip(*sorted(key_dims, key=lambda x: -len(x[0])))
    subset = all(set(g1) <= set(g2) or set(g1) >= set(g2)
                 for g1 in dim_groups for g2 in dim_groups)
    # Find unique keys
    all_dims = sorted({dim for dim_group in dim_groups
                       for dim in dim_group},
                      key=lambda x: dim_groups[0].index(x))
    ndims = len(all_dims)
    unique_keys = []
    for group, keys in key_dims:
        dim_idxs = [all_dims.index(dim) for dim in group]
        for k in keys:
            matches = [item for item in unique_keys
                       if k == (itemgetter(*dim_idxs)(item)[0]
                                if len(dim_idxs) != 1 else (item[dim_idxs[0]]),)]
            if not matches:
                unique_keys.append(create_ndkey(ndims, dim_idxs, k))
    sorted_keys = NdMapping({key: None for key in unique_keys},
                            key_dimensions=all_dims).data.keys()
    if subset:
        return all_dims, list(sorted_keys)
    else:
        return [default_dim], [i for i in range(len(unique_keys))]
<|endoftext|>


<commit_before># Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
#
# MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

from __future__ import unicode_literals

"""
Contributing:
1. Add the .csv file
2. Run import
3. Then run translate
"""

import webnotes
import os
import codecs
import json
import re
from csv import reader
from webnotes.modules import get_doc_path

messages = {}

def translate(lang=None):
	languages = [lang]
	if lang=="all" or lang==None:
		languages = get_all_languages()

	print "Extracting / updating translatable strings..."
	build_message_files()

	print "Compiling messages in one file..."
	export_messages(lang, '_lang_tmp.csv')

	for lang in languages:
		if lang != "en":
			filename = 'app/translations/'+lang+'.csv'
			print "For " + lang + ":"
			print "Translating via Google Translate..."
			google_translate(lang, '_lang_tmp.csv', filename)
			print "Updating language files..."
			import_messages(lang, filename)

	print "Deleting temp file..."
	os.remove('_lang_tmp.csv')

def get_all_languages():
	return [f[:-4] for f in os.listdir("app/translations") if f.endswith(".csv")]

def update_translations():
	"""
	compare language file timestamps with last updated timestamps in `.wnf-lang-status`
	if timestamps are missing / changed, build new `.json` files in the `lang folders`
	"""
	langstatus = {}
	languages = get_all_languages()
	message_updated = False
	status_file_path = "app/.wnf-lang-status"

	if os.path.exists(status_file_path):
		with open(status_file_path, "r") as langstatusfile:
			langstatus = eval(langstatusfile.read())

	for lang in languages:
		filename = 'app/translations/'+lang+'.csv'
		if langstatus.get(lang, None)!=os.path.getmtime(filename):
			print "Setting up lang files for " + lang + "..."
			if not message_updated:
				print "Extracting / updating translatable strings..."
				build_message_files()
				message_updated = True
			print "Writing translations..."
			import_messages(lang, filename)
			langstatus[lang] = os.path.getmtime(filename)

	with open(status_file_path, "w") as langstatusfile:
		langstatus = langstatusfile.write(str(langstatus))

def build_message_files():
	"""build from doctypes, pages, database and framework"""
	if not webnotes.conn:
		webnotes.connect()

	build_for_pages('lib/core')
	build_for_pages('app')

	build_from_doctype_code('lib/core')
	build_from_doctype_code('app')

	# doctype
	build_from_database()

	build_for_framework('lib/webnotes', 'py', with_doctype_names=True)
	build_for_framework('lib/public/js/wn', 'js')
	build_for_framework('app/public/js', 'js', with_doctype_names=True)

def build_for_pages(path):
	"""make locale files for framework py and js (all)"""
	from webnotes.modules import get_doc_path
	messages = []
	for (basepath, folders, files) in os.walk(path):
		if os.path.basename(os.path.dirname(basepath))=="page":
			messages_js, messages_py = [], []
			for fname in files:
				if fname.endswith('.js'):
					messages_js += get_message_list(os.path.join(basepath, fname))
				if fname.endswith('.py'):
					messages_py += get_message_list(os.path.join(basepath, fname))
			if messages_js:
				write_messages_file(basepath, messages_js, "js")
			if messages_py:
				write_messages_file(basepath, messages_py, "py")

def build_from_database():
	"""make doctype labels, names, options, descriptions"""
	def get_select_options(doc):
		if doc.doctype=="DocField" and doc.fieldtype=='Select' and doc.options \
			and not doc.options.startswith("link:") \
			and not doc.options.startswith("attach_files:"):
			return doc.options.split('\n')
		else:
			return []

	build_for_doc_from_database(webnotes._dict({
		"doctype": "DocType",
		"module_field": "module",
		"DocType": ["name", "description", "module"],
		"DocField": ["label", "description"],
		"custom": get_select_options
	}))

def build_for_doc_from_database(fields):
	for item in webnotes.conn.sql("""select name from `tab%s`""" % fields.doctype, as_dict=1):
		messages = []
		doclist = webnotes.bean(fields.doctype, item.name).doclist

		for doc in doclist:
			if doc.doctype in fields:
				messages += map(lambda x: x in fields[doc.doctype] and doc.fields.get(x) or None,
					doc.fields.keys())

			if fields.custom:
				messages += fields.custom(doc)

		doc = doclist[0]
		if doc.fields.get(fields.module_field):
			doctype_path = get_doc_path(doc.fields[fields.module_field],
				doc.doctype, doc.name)
			write_messages_file(doctype_path, messages, 'doc')

def build_for_framework(path, mtype, with_doctype_names = False):
	"""make locale files for framework py and js (all)"""
	messages = []
	for (basepath, folders, files) in os.walk(path):
		for fname in files:
			if fname.endswith('.' + mtype):
				messages += get_message_list(os.path.join(basepath, fname))

	# append module & doctype names
	if with_doctype_names:
		for m in webnotes.conn.sql("""select name, module from `tabDocType`"""):
			messages.append(m[0])
			messages.append(m[1])

	# append labels from config.json
	config = webnotes.get_config()
	for moduleinfo in config["modules"].values():
		if moduleinfo.get("label"):
			messages.append(moduleinfo["label"])

	if messages:
		write_messages_file(path, messages, mtype)

def build_from_doctype_code(path):
	"""walk and make locale files in all folders"""
	for (basepath, folders, files) in os.walk(path):
		messagespy = []
		messagesjs = []
		for fname in files:
			if fname.endswith('py'):
				messagespy += get_message_list(os.path.join(basepath, fname))
			if fname.endswith('js'):
				messagesjs += get_message_list(os.path.join(basepath, fname))

		if messagespy:
			write_messages_file(basepath, messagespy, 'py')

		if messagespy:
			write_messages_file(basepath, messagesjs, 'js')

def get_message_list(path):
	"""get list of messages from a code file"""
	import re
	messages = []
	with open(path, 'r') as sourcefile:
		txt = sourcefile.read()
		messages += re.findall('_\("([^"]*)"\)', txt)
		messages += re.findall("_\('([^']*)'\)", txt)
		messages += re.findall('_\("{3}([^"]*)"{3}\)', txt, re.S)

	return messages

def write_messages_file(path, messages, mtype):
	"""write messages to translation file"""
	if not os.path.exists(path):
		return

	if not os.path.exists(os.path.join(path, 'locale')):
		os.makedirs(os.path.join(path, 'locale'))

	fname = os.path.join(path, 'locale', '_messages_' + mtype + '.json')
	messages = list(set(messages))
	filtered = []
	for m in messages:
		if m and re.search('[a-zA-Z]+', m):
			filtered.append(m)
	with open(fname, 'w') as msgfile:
		msgfile.write(json.dumps(filtered, indent=1))

def export_messages(lang, outfile):
	"""get list of all messages"""
	messages = {}
	# extract messages
	for (basepath, folders, files) in os.walk('.'):
		def _get_messages(messages, basepath, mtype):
			mlist = get_messages(basepath, mtype)
			if not mlist:
				return

			# update messages with already existing translations
			langdata = get_lang_data(basepath, lang, mtype)
			for m in mlist:
				if not messages.get(m):
					messages[m] = langdata.get(m, "")

		if os.path.basename(basepath)=='locale':
			_get_messages(messages, basepath, 'doc')
			_get_messages(messages, basepath, 'py')
			_get_messages(messages, basepath, 'js')

	# remove duplicates
	if outfile:
		from csv import writer
		with open(outfile, 'w') as msgfile:
			w = writer(msgfile)
			keys = messages.keys()
			keys.sort()
			for m in keys:

				w.writerow([m.encode('utf-8'), messages.get(m, '').encode('utf-8')])

def import_messages(lang, infile):
	"""make individual message files for each language"""
	data = dict(get_all_messages_from_file(infile))

	for (basepath, folders, files) in os.walk('.'):
		def _update_lang_file(mtype):
			"""create a langauge file for the given message type"""
			messages = get_messages(basepath, mtype)
			if not messages: return

			# read existing
			langdata = get_lang_data(basepath, lang, mtype)

			# update fresh
			for m in messages:
				if data.get(m):
					langdata[m] = data.get(m)

			if langdata:
				# write new langfile
				langfilename = os.path.join(basepath, lang + '-' + mtype + '.json')
				with open(langfilename, 'w') as langfile:
					langfile.write(json.dumps(langdata, indent=1, sort_keys=True).encode('utf-8'))
				#print 'wrote ' + langfilename

		if os.path.basename(basepath)=='locale':
			# make / update lang files for each type of message file (doc, js, py)
			# example: hi-doc.json, hi-js.json, hi-py.json
			_update_lang_file('doc')
			_update_lang_file('js')
			_update_lang_file('py')

def get_doc_messages(module, doctype, name):
	return get_lang_data(get_doc_path(module, doctype, name), None, 'doc')

def get_lang_data(basepath, lang, mtype):
	"""get language dict from langfile"""

	# add "locale" folder if reqd
	if os.path.basename(basepath) != 'locale':
		basepath = os.path.join(basepath, 'locale')

	if not lang: lang = webnotes.lang

	path = os.path.join(basepath, lang + '-' + mtype + '.json')

	langdata = {}
	if os.path.exists(path):
		with codecs.open(path, 'r', 'utf-8') as langfile:
			langdata = json.loads(langfile.read())

	return langdata

def get_messages(basepath, mtype):
	"""load list of messages from _message files"""
	# get message list
	path = os.path.join(basepath, '_messages_' + mtype + '.json')
	messages = []
	if os.path.exists(path):
		with open(path, 'r') as msgfile:
			messages = json.loads(msgfile.read())

	return messages

def update_lang_js(jscode, path):
	return jscode + "\n\n$.extend(wn._messages, %s)" % \
		json.dumps(get_lang_data(path, webnotes.lang, 'js'))

def get_all_messages_from_file(path):
	with codecs.open(path, 'r', 'utf-8') as msgfile:
		data = msgfile.read()
		data = reader([r.encode('utf-8') for r in data.splitlines()])
		newdata = []
		for row in data:
			newrow = []
			for val in row:
				newrow.append(unicode(val, 'utf-8'))
			newdata.append(newrow)

	return newdata

def google_translate(lang, infile, outfile):
	"""translate objects using Google API. Add you own API key for translation"""
	data = get_all_messages_from_file(infile)

	import requests, conf

	# update existing translations
	if os.path.exists(outfile):
		with codecs.open(outfile, "r", "utf-8") as oldfile:
			old_data = oldfile.read()
			old_translations = dict(reader([r.encode('utf-8').strip() for r in old_data.splitlines()]))

	with open(outfile, 'w') as msgfile:
		from csv import writer
		w = writer(msgfile)
		for row in data:
			if row[0] and row[0].strip():
				if old_translations.get(row[0].strip()):
					row[1] = old_translations[row[0].strip()]
				else:
					print 'translating: ' + row[0]
					response = requests.get("""https://www.googleapis.com/language/translate/v2""",
						params = {
							"key": conf.google_api_key,
							"source": "en",
							"target": lang,
							"q": row[0]
						})

					if "error" in response.json:
						print response.json

					row[1] = response.json["data"]["translations"][0]["translatedText"]
					if not row[1]:
						row[1] = row[0] # google unable to translate!

					row[1] = row[1].encode('utf-8')

				row[0] = row[0].encode('utf-8')
				w.writerow(row)

<commit_msg>[translations] [minor] updated translations.py<commit_after># Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
#
# MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

from __future__ import unicode_literals

"""
Contributing:
1. Add the .csv file
2. Run import
3. Then run translate
"""

import webnotes
import os
import codecs
import json
import re
from csv import reader
from webnotes.modules import get_doc_path

messages = {}

def translate(lang=None):
	languages = [lang]
	if lang=="all" or lang==None:
		languages = get_all_languages()

	print "Extracting / updating translatable strings..."
	build_message_files()

	print "Compiling messages in one file..."
	export_messages(lang, '_lang_tmp.csv')

	for lang in languages:
		if lang != "en":
			filename = 'app/translations/'+lang+'.csv'
			print "For " + lang + ":"
			print "Translating via Google Translate..."
			google_translate(lang, '_lang_tmp.csv', filename)
			print "Updating language files..."
			import_messages(lang, filename)

	print "Deleting temp file..."
	os.remove('_lang_tmp.csv')

def get_all_languages():
	return [f[:-4] for f in os.listdir("app/translations") if f.endswith(".csv")]

def update_translations():
	"""
	compare language file timestamps with last updated timestamps in `.wnf-lang-status`
	if timestamps are missing / changed, build new `.json` files in the `lang folders`
	"""
	langstatus = {}
	languages = get_all_languages()
	message_updated = False
	status_file_path = "app/.wnf-lang-status"

	if os.path.exists(status_file_path):
		with open(status_file_path, "r") as langstatusfile:
			langstatus = eval(langstatusfile.read())

	for lang in languages:
		filename = 'app/translations/'+lang+'.csv'
		if langstatus.get(lang, None)!=os.path.getmtime(filename):
			print "Setting up lang files for " + lang + "..."
			if not message_updated:
				print "Extracting / updating translatable strings..."
				build_message_files()
				message_updated = True
			print "Writing translations..."
			import_messages(lang, filename)
			langstatus[lang] = os.path.getmtime(filename)

	with open(status_file_path, "w") as langstatusfile:
		langstatus = langstatusfile.write(str(langstatus))

def build_message_files():
	"""build from doctypes, pages, database and framework"""
	if not webnotes.conn:
		webnotes.connect()

	build_for_pages('lib/core')
	build_for_pages('app')

	build_from_doctype_code('lib/core')
	build_from_doctype_code('app')

	# doctype
	build_from_database()

	build_for_framework('lib/webnotes', 'py', with_doctype_names=True)
	build_for_framework('lib/public/js/wn', 'js')
	build_for_framework('app/public/js', 'js', with_doctype_names=True)

def build_for_pages(path):
	"""make locale files for framework py and js (all)"""
	messages = []
	for (basepath, folders, files) in os.walk(path):
		if os.path.basename(os.path.dirname(basepath))=="page":
			messages_js, messages_py = [], []
			for fname in files:
				if fname.endswith('.js'):
					messages_js += get_message_list(os.path.join(basepath, fname))
				if fname.endswith('.py'):
					messages_py += get_message_list(os.path.join(basepath, fname))
			if messages_js:
				write_messages_file(basepath, messages_js, "js")
			if messages_py:
				write_messages_file(basepath, messages_py, "py")

def build_from_database():
	"""make doctype labels, names, options, descriptions"""
	def get_select_options(doc):
		if doc.doctype=="DocField" and doc.fieldtype=='Select' and doc.options \
			and not doc.options.startswith("link:") \
			and not doc.options.startswith("attach_files:"):
			return doc.options.split('\n')
		else:
			return []

	build_for_doc_from_database(webnotes._dict({
		"doctype": "DocType",
		"module_field": "module",
		"DocType": ["name", "description", "module"],
		"DocField": ["label", "description"],
		"custom": get_select_options
	}))

def build_for_doc_from_database(fields):
	for item in webnotes.conn.sql("""select name from `tab%s`""" % fields.doctype, as_dict=1):
		messages = []
		doclist = webnotes.bean(fields.doctype, item.name).doclist

		for doc in doclist:
			if doc.doctype in fields:
				messages += map(lambda x: x in fields[doc.doctype] and doc.fields.get(x) or None,
					doc.fields.keys())

			if fields.custom:
				messages += fields.custom(doc)

		doc = doclist[0]
		if doc.fields.get(fields.module_field):
			doctype_path = get_doc_path(doc.fields[fields.module_field],
				doc.doctype, doc.name)
			write_messages_file(doctype_path, messages, 'doc')

def build_for_framework(path, mtype, with_doctype_names = False):
	"""make locale files for framework py and js (all)"""
	messages = []
	for (basepath, folders, files) in os.walk(path):
		for fname in files:
			if fname.endswith('.' + mtype):
				messages += get_message_list(os.path.join(basepath, fname))

	# append module & doctype names
	if with_doctype_names:
		for m in webnotes.conn.sql("""select name, module from `tabDocType`"""):
			messages.append(m[0])
			messages.append(m[1])

	# append labels from config.json
	config = webnotes.get_config()
	for moduleinfo in config["modules"].values():
		if moduleinfo.get("label"):
			messages.append(moduleinfo["label"])

	if messages:
		write_messages_file(path, messages, mtype)

def build_from_doctype_code(path):
	"""walk and make locale files in all folders"""
	for (basepath, folders, files) in os.walk(path):
		messagespy = []
		messagesjs = []
		for fname in files:
			if fname.endswith('py'):
				messagespy += get_message_list(os.path.join(basepath, fname))
			if fname.endswith('js'):
				messagesjs += get_message_list(os.path.join(basepath, fname))

		if messagespy:
			write_messages_file(basepath, messagespy, 'py')

		if messagespy:
			write_messages_file(basepath, messagesjs, 'js')

def get_message_list(path):
	"""get list of messages from a code file"""
	import re
	messages = []
	with open(path, 'r') as sourcefile:
		txt = sourcefile.read()
		messages += re.findall('_\("([^"]*)"\)', txt)
		messages += re.findall("_\('([^']*)'\)", txt)
		messages += re.findall('_\("{3}([^"]*)"{3}\)', txt, re.S)

	return messages

def write_messages_file(path, messages, mtype):
	"""write messages to translation file"""
	if not os.path.exists(path):
		return

	if not os.path.exists(os.path.join(path, 'locale')):
		os.makedirs(os.path.join(path, 'locale'))

	fname = os.path.join(path, 'locale', '_messages_' + mtype + '.json')
	messages = list(set(messages))
	filtered = []
	for m in messages:
		if m and re.search('[a-zA-Z]+', m):
			filtered.append(m)
	with open(fname, 'w') as msgfile:
		msgfile.write(json.dumps(filtered, indent=1))

def export_messages(lang, outfile):
	"""get list of all messages"""
	messages = {}
	# extract messages
	for (basepath, folders, files) in os.walk('.'):
		def _get_messages(messages, basepath, mtype):
			mlist = get_messages(basepath, mtype)
			if not mlist:
				return

			# update messages with already existing translations
			langdata = get_lang_data(basepath, lang, mtype)
			for m in mlist:
				if not messages.get(m):
					messages[m] = langdata.get(m, "")

		if os.path.basename(basepath)=='locale':
			_get_messages(messages, basepath, 'doc')
			_get_messages(messages, basepath, 'py')
			_get_messages(messages, basepath, 'js')

	# remove duplicates
	if outfile:
		from csv import writer
		with open(outfile, 'w') as msgfile:
			w = writer(msgfile)
			keys = messages.keys()
			keys.sort()
			for m in keys:

				w.writerow([m.encode('utf-8'), messages.get(m, '').encode('utf-8')])

def import_messages(lang, infile):
	"""make individual message files for each language"""
	data = dict(get_all_messages_from_file(infile))

	for (basepath, folders, files) in os.walk('.'):
		def _update_lang_file(mtype):
			"""create a langauge file for the given message type"""
			messages = get_messages(basepath, mtype)
			if not messages: return

			# read existing
			langdata = get_lang_data(basepath, lang, mtype)

			# update fresh
			for m in messages:
				if data.get(m):
					langdata[m] = data.get(m)

			if langdata:
				# write new langfile
				langfilename = os.path.join(basepath, lang + '-' + mtype + '.json')
				with open(langfilename, 'w') as langfile:
					langfile.write(json.dumps(langdata, indent=1, sort_keys=True).encode('utf-8'))
				#print 'wrote ' + langfilename

		if os.path.basename(basepath)=='locale':
			# make / update lang files for each type of message file (doc, js, py)
			# example: hi-doc.json, hi-js.json, hi-py.json
			_update_lang_file('doc')
			_update_lang_file('js')
			_update_lang_file('py')

def get_doc_messages(module, doctype, name):
	return get_lang_data(get_doc_path(module, doctype, name), None, 'doc')

def get_lang_data(basepath, lang, mtype):
	"""get language dict from langfile"""

	# add "locale" folder if reqd
	if os.path.basename(basepath) != 'locale':
		basepath = os.path.join(basepath, 'locale')

	if not lang: lang = webnotes.lang

	path = os.path.join(basepath, lang + '-' + mtype + '.json')

	langdata = {}
	if os.path.exists(path):
		with codecs.open(path, 'r', 'utf-8') as langfile:
			langdata = json.loads(langfile.read())

	return langdata

def get_messages(basepath, mtype):
	"""load list of messages from _message files"""
	# get message list
	path = os.path.join(basepath, '_messages_' + mtype + '.json')
	messages = []
	if os.path.exists(path):
		with open(path, 'r') as msgfile:
			messages = json.loads(msgfile.read())

	return messages

def update_lang_js(jscode, path):
	return jscode + "\n\n$.extend(wn._messages, %s)" % \
		json.dumps(get_lang_data(path, webnotes.lang, 'js'))

def get_all_messages_from_file(path):
	with codecs.open(path, 'r', 'utf-8') as msgfile:
		data = msgfile.read()
		data = reader([r.encode('utf-8') for r in data.splitlines()])
		newdata = []
		for row in data:
			newrow = []
			for val in row:
				newrow.append(unicode(val, 'utf-8'))
			newdata.append(newrow)

	return newdata

def google_translate(lang, infile, outfile):
	"""translate objects using Google API. Add you own API key for translation"""
	data = get_all_messages_from_file(infile)

	import requests, conf

	# update existing translations
	if os.path.exists(outfile):
		with codecs.open(outfile, "r", "utf-8") as oldfile:
			old_data = oldfile.read()
			old_translations = dict(reader([r.encode('utf-8').strip() for r in old_data.splitlines()]))

	with open(outfile, 'w') as msgfile:
		from csv import writer
		w = writer(msgfile)
		for row in data:
			if row[0] and row[0].strip():
				if old_translations.get(row[0].strip()):
					row[1] = old_translations[row[0].strip()]
				else:
					print 'translating: ' + row[0]
					response = requests.get("""https://www.googleapis.com/language/translate/v2""",
						params = {
							"key": conf.google_api_key,
							"source": "en",
							"target": lang,
							"q": row[0]
						})

					if "error" in response.json:
						print response.json

					row[1] = response.json["data"]["translations"][0]["translatedText"]
					if not row[1]:
						row[1] = row[0] # google unable to translate!

					row[1] = row[1].encode('utf-8')

				row[0] = row[0].encode('utf-8')
				w.writerow(row)

<|endoftext|>


<commit_before>import cupy
from cupy import _core

_packbits_kernel = {
    'big': _core.ElementwiseKernel(
        'raw T myarray, raw int32 myarray_size', 'uint8 packed',
        '''for (int j = 0; j < 8; ++j) {
                    int k = i * 8 + j;
                    int bit = k < myarray_size && myarray[k] != 0;
                    packed |= bit << (7 - j);
                }''',
        'cupy_packbits_kernel'
    ),
    'little': _core.ElementwiseKernel(
        'raw T myarray, raw int32 myarray_size', 'uint8 packed',
        '''for (int j = 0; j < 8; ++j) {
                    int k = i * 8 + j;
                    int bit = k < myarray_size && myarray[k] != 0;
                    packed |= bit << j;
                }''',
        'cupy_packbits_kernel'
    )
}

def packbits(myarray, bitorder='big'):
    """Packs the elements of a binary-valued array into bits in a uint8 array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.
        bitorder (str, optional): bit order to use when packing the array, allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The packed array.

    .. note::
        When the input array is empty, this function returns a copy of it,
        i.e., the type of the output array is not necessarily always uint8.
        This exactly follows the NumPy's behaviour (as of version 1.11),
        alghough this is inconsistent to the documentation.

    .. seealso:: :func:`numpy.packbits`
    """
    if myarray.dtype.kind not in 'biu':
        raise TypeError(
            'Expected an input array of integer or boolean data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    myarray = myarray.ravel()
    packed_size = (myarray.size + 7) // 8
    packed = cupy.zeros((packed_size,), dtype=cupy.uint8)
    return _packbits_kernel[bitorder](myarray, myarray.size, packed)

_unpackbits_kernel = {
    'big': _core.ElementwiseKernel(
        'raw uint8 myarray', 'T unpacked',
        'unpacked = (myarray[i / 8] >> (7 - i % 8)) & 1;',
        'cupy_unpackbits_kernel'
    ),
    'little': _core.ElementwiseKernel(
        'raw uint8 myarray', 'T unpacked',
        'unpacked = (myarray[i / 8] >> (i % 8)) & 1;',
        'cupy_unpackbits_kernel'
    )
}

def unpackbits(myarray, bitorder='big'):
    """Unpacks elements of a uint8 array into a binary-valued output array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.
        bitorder (str, optional): bit order to use when unpacking the array, allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The unpacked array.

    .. seealso:: :func:`numpy.unpackbits`
    """
    if myarray.dtype != cupy.uint8:
        raise TypeError('Expected an input array of unsigned byte data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    unpacked = cupy.ndarray((myarray.size * 8), dtype=cupy.uint8)
    return _unpackbits_kernel[bitorder](myarray, unpacked)
<commit_msg>fix doc flake8 error.<commit_after>import cupy
from cupy import _core

_packbits_kernel = {
    'big': _core.ElementwiseKernel(
        'raw T myarray, raw int32 myarray_size', 'uint8 packed',
        '''for (int j = 0; j < 8; ++j) {
                    int k = i * 8 + j;
                    int bit = k < myarray_size && myarray[k] != 0;
                    packed |= bit << (7 - j);
                }''',
        'cupy_packbits_kernel'
    ),
    'little': _core.ElementwiseKernel(
        'raw T myarray, raw int32 myarray_size', 'uint8 packed',
        '''for (int j = 0; j < 8; ++j) {
                    int k = i * 8 + j;
                    int bit = k < myarray_size && myarray[k] != 0;
                    packed |= bit << j;
                }''',
        'cupy_packbits_kernel'
    )
}

def packbits(myarray, bitorder='big'):
    """Packs the elements of a binary-valued array into bits in a uint8 array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.
        bitorder (str, optional): bit order to use when packing the array,
            allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The packed array.

    .. note::
        When the input array is empty, this function returns a copy of it,
        i.e., the type of the output array is not necessarily always uint8.
        This exactly follows the NumPy's behaviour (as of version 1.11),
        alghough this is inconsistent to the documentation.

    .. seealso:: :func:`numpy.packbits`
    """
    if myarray.dtype.kind not in 'biu':
        raise TypeError(
            'Expected an input array of integer or boolean data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    myarray = myarray.ravel()
    packed_size = (myarray.size + 7) // 8
    packed = cupy.zeros((packed_size,), dtype=cupy.uint8)
    return _packbits_kernel[bitorder](myarray, myarray.size, packed)

_unpackbits_kernel = {
    'big': _core.ElementwiseKernel(
        'raw uint8 myarray', 'T unpacked',
        'unpacked = (myarray[i / 8] >> (7 - i % 8)) & 1;',
        'cupy_unpackbits_kernel'
    ),
    'little': _core.ElementwiseKernel(
        'raw uint8 myarray', 'T unpacked',
        'unpacked = (myarray[i / 8] >> (i % 8)) & 1;',
        'cupy_unpackbits_kernel'
    )
}

def unpackbits(myarray, bitorder='big'):
    """Unpacks elements of a uint8 array into a binary-valued output array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.
        bitorder (str, optional): bit order to use when unpacking the array,
            allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The unpacked array.

    .. seealso:: :func:`numpy.unpackbits`
    """
    if myarray.dtype != cupy.uint8:
        raise TypeError('Expected an input array of unsigned byte data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    unpacked = cupy.ndarray((myarray.size * 8), dtype=cupy.uint8)
    return _unpackbits_kernel[bitorder](myarray, unpacked)
<|endoftext|>


<commit_before>'''
HolyGrail (c) University of Manchester 2015

HolyGrail is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
import sys

import climate

import holygrail
import synbiochem.ann

def classify(sample_size, struct_patterns, split, hidden_layers=None,
             learning_rate=0.01, momentum=0.5, patience=5,
             min_improvement=0.005, validate_every=1, batch_size=5,
             hidden_dropout=0.0, input_dropout=0.0, scale=(0.1, 0.9)):
    '''Classification of peptides, specified by structure patterns as
    regexps.'''
    climate.enable_default_logging()

    # Get random peptides that match structure patterns from PDB:
    pdb_data = holygrail.get_pdb_data(sample_size, struct_patterns)

    # Convert peptides to inputs, based on amino acid properties:
    x_data = holygrail.get_input_data([i[0] for v in pdb_data.values()
                                       for i in v],
                                      scale=scale)

    # Convert classess to outputs (these are based on structure patterns):
    y_data = [i for k, v in pdb_data.iteritems() for i in [k] * len(v)]

    # Randomise input and output data order:
    x_data, y_data = synbiochem.ann.randomise_order(x_data, y_data)

    # Split data into training and classifying:
    ind = int(split * len(x_data))

    # Perform classification:
    classifier = synbiochem.ann.Classifier(x_data[:ind], y_data[:ind])
    classifier.train(hidden_layers=hidden_layers,  learning_rate=learning_rate,
                     momentum=momentum, patience=patience,
                     min_improvement=min_improvement,
                     validate_every=validate_every, batch_size=batch_size,
                     hidden_dropout=hidden_dropout,
                     input_dropout=input_dropout)

    return classifier.classify(x_data[ind:], y_data[ind:])[1:]

def main(argv):
    '''main method.'''
    classification = classify(int(argv[1]),
                              argv[4:],
                              float(argv[2]),
                              [int(argv[3])])

    for output in classification:
        print output

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Turned classification from a module to a class.<commit_after>'''
HolyGrail (c) University of Manchester 2015

HolyGrail is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
import sys

import climate

import holygrail
import synbiochem.ann

class Classifier(object):

    def __init__(self, sample_size, struct_patterns, split, scale=(0.1, 0.9)):
        '''Constructor.'''
        climate.enable_default_logging()

        # Get random peptides that match structure patterns from PDB:
        pdb_data = holygrail.get_pdb_data(sample_size, struct_patterns)

        # Convert peptides to inputs, based on amino acid properties:
        x_data = holygrail.get_input_data([i[0] for v in pdb_data.values()
                                           for i in v],
                                          scale=scale)

        # Convert classes to outputs (these are based on structure patterns):
        y_data = [i for k, v in pdb_data.iteritems() for i in [k] * len(v)]

        # Randomise input and output data order:
        x_data, y_data = synbiochem.ann.randomise_order(x_data, y_data)

        # Split data into training and testing:
        ind = int(split * len(x_data))
        self.__x_data_train = x_data[:ind]
        self.__y_data_train = y_data[:ind]
        self.__x_data_test = x_data[ind:]
        self.__y_data_test = y_data[ind:]

    def classify(self, hidden_layers=None, learning_rate=0.01, momentum=0.5,
                 patience=5, min_improvement=0.005, validate_every=1,
                 batch_size=5, hidden_dropout=0.0, input_dropout=0.0):
        '''Classification of peptides, specified by structure patterns as
        regexps.'''

        # Perform classification:
        classifier = synbiochem.ann.Classifier(self.__x_data_train,
                                               self.__y_data_train)
        classifier.train(hidden_layers=hidden_layers,
                         learning_rate=learning_rate, momentum=momentum,
                         patience=patience, min_improvement=min_improvement,
                         validate_every=validate_every, batch_size=batch_size,
                         hidden_dropout=hidden_dropout,
                         input_dropout=input_dropout)

        return classifier.classify(self.__x_data_test, self.__y_data_test)[1:]

def main(argv):
    '''main method.'''
    classifier = Classifier(int(argv[1]), argv[4:], float(argv[2]))
    classification = classifier.classify(hidden_layers=[int(argv[3])])

    for output in classification:
        print output

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

##***** BEGIN LICENSE BLOCK *****
##Version: MPL 1.1
##
##The contents of this file are subject to the Mozilla Public License Version
##1.1 (the "License"); you may not use this file except in compliance with
##the License. You may obtain a copy of the License at
##http:##www.mozilla.org/MPL/
##
##Software distributed under the License is distributed on an "AS IS" basis,
##WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
##for the specific language governing rights and limitations under the
##License.
##
##The Original Code is the AllegroGraph Java Client interface.
##
##The Original Code was written by Franz Inc.
##Copyright (C) 2006 Franz Inc.  All Rights Reserved.
##
##***** END LICENSE BLOCK *****

from franz.openrdf.exceptions import *
from franz.openrdf.model.value import URI
from franz.openrdf.model.valuefactory import ValueFactory
from franz.openrdf.repository.repositoryconnection import RepositoryConnection

# * A Sesame repository that contains RDF data that can be queried and updated.
# * Access to the repository can be acquired by openening a connection to it.
# * This connection can then be used to query and/or update the contents of the
# * repository. Depending on the implementation of the repository, it may or may
# * not support multiple concurrent connections.
# *

# * Please note that a repository needs to be initialized before it can be used
# * and that it should be shut down before it is discarded/garbage collected.
# * Forgetting the latter can result in loss of data (depending on the Repository
# * implementation)!
class Repository:
    RENEW = 'renew'
    ACCESS = 'access'
    OPEN = 'open'
    CREATE = 'create'
    REPLACE = 'replace'

    def __init__(self, catalog, database_name, access_verb):
        self.catalog = catalog
        self.mini_catalog = catalog.mini_catalog
        self.mini_repository = None
        self.database_name = database_name
        self.access_verb = access_verb
        ## system state fields:
        self.connection = None
        self.value_factory = None
        self.inlined_predicates = {}
        self.inlined_datatypes = {}

    def getDatabaseName(self):
        """
        Return the name of the database (remote triple store) that this repository is
        interfacing with.
        """
        return self.database_name

    def _attach_to_mini_repository(self):
        """
        Create a mini-repository and execute a RENEW, OPEN, CREATE, or ACCESS.

        TODO: FIGURE OUT WHAT 'REPLACE' DOES
        """
        clearIt = False
        dbName = self.database_name
        conn = self.mini_catalog
        if self.access_verb == Repository.RENEW:
            if dbName in conn.listTripleStores():
                ## not nice, since someone else probably has it open:
                clearIt = True
            else:
                conn.createTripleStore(dbName)
        elif self.access_verb == Repository.CREATE:
            if dbName in conn.listTripleStores():
                raise ServerException(
                    "Can't create triple store named '%s' because a store with that name already exists.",
                    dbName)
            conn.createTripleStore(dbName)
        elif self.access_verb == Repository.OPEN:
            if not dbName in conn.listTripleStores():
                raise ServerException(
                    "Can't open a triple store named '%s' because there is none.", dbName)
        elif self.access_verb == Repository.ACCESS:
            if not dbName in conn.listTripleStores():
                conn.createTripleStore(dbName)
        self.mini_repository = conn.getRepository(dbName)
        ## we are done unless a RENEW requires us to clear the store
        if clearIt:
            self.mini_repository.deleteMatchingStatements(None, None, None, None)

    def initialize(self):
        """
        Initializes this repository. A repository needs to be initialized before
        it can be used.
        """
        self._attach_to_mini_repository()
        ## EXPERIMENTATION WITH INITIALIZING AN ENVIRONMENT.  DIDN'T LOOK RIGHT - RMM
#        self.environment = self.mini_repository.createEnvironment()
#        print "ENV", self.environment
#        self.mini_repository.deleteEnvironment(self.environment)
#        print "ENV AfTER", self.mini_repository.listEnvironments()

    def indexTriples(self, all=False, asynchronous=False):
        """
        Index the newly-added triples in the store.  This should be done after every
        significant-sized load of triples into the store.
        If 'all', re-index all triples in the store.  If 'asynchronous', spawn
        the indexing task as a separate thread, and don't wait for it to complete.
        Note. Upon version 4.0, calling this will no longer be necessary.
        """
        self.mini_repository.indexStatements(all=all)

    def registerFreeTextPredicate(self,uri=None, namespace=None, localname=None):
        """
        Register a predicate 'uri' (or 'namespace'+'localname'), telling the RDF store to index
        text keywords belonging to strings in object position in the corresponding
        triples/statements.  This is needed to make the  fti:match  operator
        work properly.
        """
        uri = uri or (namespace + localname)
        self.mini_repository.registerFreeTextPredicate("<%s>" % uri)

    def _translate_inlined_type(self, type):
        if type == "int": return "int"
        elif type == "datetime": return "date-time"
        elif type == "float": return "float"
        else:
            raise IllegalArgumentException("Unknown inlined type '%s'\n.  Legal types are " +
                    "'int', 'float', and 'datetime'")

    def registerInlinedDatatype(self, predicate=None, datatype=None, inlinedType=None):
        """
        Register an inlined datatype.  If 'predicate', then object arguments to triples
        with that predicate will use an inlined encoding of type 'inlinedType' in their
        internal representation.
        If 'datatype', then typed literal objects with a datatype matching 'datatype' will
        use an inlined encoding of type 'inlinedType'.
        """
        predicate = predicate.getURI() if isinstance(predicate, URI) else predicate
        datatype = datatype.getURI() if isinstance(datatype, URI) else datatype
        if predicate:
            if not inlinedType:
                raise IllegalArgumentException("Missing 'inlinedType' parameter in call to 'registerInlinedDatatype'")
            lispType = self._translate_inlined_type(inlinedType)
            mapping = [predicate, lispType, "predicate"]
            self.inlined_predicates[predicate] = lispType
        elif datatype:
            lispType = self._translate_inlined_type(inlinedType or datatype)
            mapping = [datatype, lispType, "datatype"]
            self.inlined_datatypes[datatype] = lispType
        if predicate:
            self.mini_repository.addMappedPredicate("<%s>" % predicate, lispType)
        else:
            self.mini_repository.addMappedType("<%s>" % datatype, lispType)

    def shutDown(self):
        """
        Shuts the store down, releasing any resources that it keeps hold of.
        Once shut down, the store can no longer be used.

        TODO: WE COULD PRESUMABLY ADD SOME LOGIC TO MAKE A RESTART POSSIBLE, ALTHOUGH
        THE ACCESS OPTION MIGHT NOT MAKE SENSE THE SECOND TIME AROUND (KILLING THAT IDEA!)
        """
        self.mini_catalog = None
        self.mini_repository = None

    def isWritable(self):
        """
        Checks whether this store is writable, i.e. if the data contained in
        this store can be changed. The writability of the store is
        determined by the writability of the Sail that this store operates
        on.
        """
        return self.mini_repository.is_writable()

    def getConnection(self):
        """
        Opens a connection to this store that can be used for querying and
        updating the contents of the store. Created connections need to be
        closed to make sure that any resources they keep hold of are released. The
        best way to do this is to use a try-finally-block
        """
        if not self.connection:
            self.connection = RepositoryConnection(self)
        return self.connection

    def getValueFactory(self):
        """
        Return a ValueFactory for this store
        """
        if not self.value_factory:
            self.value_factory = ValueFactory(self)
        return self.value_factory

    <commit_msg>Added quote_plus to repository name logic to permit use of repositories with blanks in their names.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

##***** BEGIN LICENSE BLOCK *****
##Version: MPL 1.1
##
##The contents of this file are subject to the Mozilla Public License Version
##1.1 (the "License"); you may not use this file except in compliance with
##the License. You may obtain a copy of the License at
##http:##www.mozilla.org/MPL/
##
##Software distributed under the License is distributed on an "AS IS" basis,
##WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
##for the specific language governing rights and limitations under the
##License.
##
##The Original Code is the AllegroGraph Java Client interface.
##
##The Original Code was written by Franz Inc.
##Copyright (C) 2006 Franz Inc.  All Rights Reserved.
##
##***** END LICENSE BLOCK *****

from franz.openrdf.exceptions import *
from franz.openrdf.model.value import URI
from franz.openrdf.model.valuefactory import ValueFactory
from franz.openrdf.repository.repositoryconnection import RepositoryConnection

import urllib

# * A Sesame repository that contains RDF data that can be queried and updated.
# * Access to the repository can be acquired by openening a connection to it.
# * This connection can then be used to query and/or update the contents of the
# * repository. Depending on the implementation of the repository, it may or may
# * not support multiple concurrent connections.
# *

# * Please note that a repository needs to be initialized before it can be used
# * and that it should be shut down before it is discarded/garbage collected.
# * Forgetting the latter can result in loss of data (depending on the Repository
# * implementation)!
class Repository:
    RENEW = 'renew'
    ACCESS = 'access'
    OPEN = 'open'
    CREATE = 'create'
    REPLACE = 'replace'

    def __init__(self, catalog, database_name, access_verb):
        self.catalog = catalog
        self.mini_catalog = catalog.mini_catalog
        self.mini_repository = None
        self.database_name = database_name
        self.access_verb = access_verb
        ## system state fields:
        self.connection = None
        self.value_factory = None
        self.inlined_predicates = {}
        self.inlined_datatypes = {}

    def getDatabaseName(self):
        """
        Return the name of the database (remote triple store) that this repository is
        interfacing with.
        """
        return self.database_name

    def _attach_to_mini_repository(self):
        """
        Create a mini-repository and execute a RENEW, OPEN, CREATE, or ACCESS.

        TODO: FIGURE OUT WHAT 'REPLACE' DOES
        """
        clearIt = False
        quotedDbName = urllib.quote_plus(self.database_name)
        conn = self.mini_catalog
        if self.access_verb == Repository.RENEW:
            if quotedDbName in conn.listTripleStores():
                ## not nice, since someone else probably has it open:
                clearIt = True
            else:
                conn.createTripleStore(quotedDbName)
        elif self.access_verb == Repository.CREATE:
            if quotedDbName in conn.listTripleStores():
                raise ServerException(
                    "Can't create triple store named '%s' because a store with that name already exists.",
                    quotedDbName)
            conn.createTripleStore(quotedDbName)
        elif self.access_verb == Repository.OPEN:
            if not quotedDbName in conn.listTripleStores():
                raise ServerException(
                    "Can't open a triple store named '%s' because there is none.", quotedDbName)
        elif self.access_verb == Repository.ACCESS:
            if not quotedDbName in conn.listTripleStores():
                conn.createTripleStore(quotedDbName)
        self.mini_repository = conn.getRepository(quotedDbName)
        ## we are done unless a RENEW requires us to clear the store
        if clearIt:
            self.mini_repository.deleteMatchingStatements(None, None, None, None)

    def initialize(self):
        """
        Initializes this repository. A repository needs to be initialized before
        it can be used.
        """
        self._attach_to_mini_repository()
        ## EXPERIMENTATION WITH INITIALIZING AN ENVIRONMENT.  DIDN'T LOOK RIGHT - RMM
#        self.environment = self.mini_repository.createEnvironment()
#        print "ENV", self.environment
#        self.mini_repository.deleteEnvironment(self.environment)
#        print "ENV AfTER", self.mini_repository.listEnvironments()

    def indexTriples(self, all=False, asynchronous=False):
        """
        Index the newly-added triples in the store.  This should be done after every
        significant-sized load of triples into the store.
        If 'all', re-index all triples in the store.  If 'asynchronous', spawn
        the indexing task as a separate thread, and don't wait for it to complete.
        Note. Upon version 4.0, calling this will no longer be necessary.
        """
        self.mini_repository.indexStatements(all=all)

    def registerFreeTextPredicate(self,uri=None, namespace=None, localname=None):
        """
        Register a predicate 'uri' (or 'namespace'+'localname'), telling the RDF store to index
        text keywords belonging to strings in object position in the corresponding
        triples/statements.  This is needed to make the  fti:match  operator
        work properly.
        """
        uri = uri or (namespace + localname)
        self.mini_repository.registerFreeTextPredicate("<%s>" % uri)

    def _translate_inlined_type(self, type):
        if type == "int": return "int"
        elif type == "datetime": return "date-time"
        elif type == "float": return "float"
        else:
            raise IllegalArgumentException("Unknown inlined type '%s'\n.  Legal types are " +
                    "'int', 'float', and 'datetime'")

    def registerInlinedDatatype(self, predicate=None, datatype=None, inlinedType=None):
        """
        Register an inlined datatype.  If 'predicate', then object arguments to triples
        with that predicate will use an inlined encoding of type 'inlinedType' in their
        internal representation.
        If 'datatype', then typed literal objects with a datatype matching 'datatype' will
        use an inlined encoding of type 'inlinedType'.
        """
        predicate = predicate.getURI() if isinstance(predicate, URI) else predicate
        datatype = datatype.getURI() if isinstance(datatype, URI) else datatype
        if predicate:
            if not inlinedType:
                raise IllegalArgumentException("Missing 'inlinedType' parameter in call to 'registerInlinedDatatype'")
            lispType = self._translate_inlined_type(inlinedType)
            mapping = [predicate, lispType, "predicate"]
            self.inlined_predicates[predicate] = lispType
        elif datatype:
            lispType = self._translate_inlined_type(inlinedType or datatype)
            mapping = [datatype, lispType, "datatype"]
            self.inlined_datatypes[datatype] = lispType
        if predicate:
            self.mini_repository.addMappedPredicate("<%s>" % predicate, lispType)
        else:
            self.mini_repository.addMappedType("<%s>" % datatype, lispType)

    def shutDown(self):
        """
        Shuts the store down, releasing any resources that it keeps hold of.
        Once shut down, the store can no longer be used.

        TODO: WE COULD PRESUMABLY ADD SOME LOGIC TO MAKE A RESTART POSSIBLE, ALTHOUGH
        THE ACCESS OPTION MIGHT NOT MAKE SENSE THE SECOND TIME AROUND (KILLING THAT IDEA!)
        """
        self.mini_catalog = None
        self.mini_repository = None

    def isWritable(self):
        """
        Checks whether this store is writable, i.e. if the data contained in
        this store can be changed. The writability of the store is
        determined by the writability of the Sail that this store operates
        on.
        """
        return self.mini_repository.is_writable()

    def getConnection(self):
        """
        Opens a connection to this store that can be used for querying and
        updating the contents of the store. Created connections need to be
        closed to make sure that any resources they keep hold of are released. The
        best way to do this is to use a try-finally-block
        """
        if not self.connection:
            self.connection = RepositoryConnection(self)
        return self.connection

    def getValueFactory(self):
        """
        Return a ValueFactory for this store
        """
        if not self.value_factory:
            self.value_factory = ValueFactory(self)
        return self.value_factory

    <|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import uuid

import six
import yaml
import requests
from mistralclient.api import client as mistral
from oslo.config import cfg

from st2common.constants.action import LIVEACTION_STATUS_RUNNING
from st2actions.runners import AsyncActionRunner
from st2actions.runners.mistral import utils
from st2common import log as logging

LOG = logging.getLogger(__name__)

def get_runner():
    return MistralRunner(str(uuid.uuid4()))

class MistralRunner(AsyncActionRunner):

    url = cfg.CONF.workflow.url

    def __init__(self, runner_id):
        super(MistralRunner, self).__init__(runner_id=runner_id)
        self._on_behalf_user = cfg.CONF.system_user.user
        self._client = mistral.client(mistral_url='%s/v2' % self.url)

    def pre_run(self):
        pass

    @staticmethod
    def _check_name(action_ref, is_workbook, def_dict):
        # If workbook, change the value of the "name" key.
        if is_workbook:
            if def_dict.get('name') != action_ref:
                raise Exception('Name of the workbook must be the same as the '
                                'fully qualified action name "%s".' % action_ref)
        # If workflow, change the key name of the workflow.
        else:
            workflow_name = [k for k, v in six.iteritems(def_dict) if k != 'version'][0]
            if workflow_name != action_ref:
                raise Exception('Name of the workflow must be the same as the '
                                'fully qualified action name "%s".' % action_ref)

    def _save_workbook(self, name, def_yaml):
        # If the workbook is not found, the mistral client throws a generic API exception.
        try:
            # Update existing workbook.
            wb = self._client.workbooks.get(name)
        except:
            # Delete if definition was previously a workflow.
            # If not found, an API exception is thrown.
            try:
                self._client.workflows.delete(name)
            except:
                pass

            # Create the new workbook.
            wb = self._client.workbooks.create(def_yaml)

        # Update the workbook definition.
        if wb.definition != def_yaml:
            self._client.workbooks.update(def_yaml)

    def _save_workflow(self, name, def_yaml):
        # If the workflow is not found, the mistral client throws a generic API exception.
        try:
            # Update existing workbook.
            wf = self._client.workflows.get(name)
        except:
            # Delete if definition was previously a workbook.
            # If not found, an API exception is thrown.
            try:
                self._client.workbooks.delete(name)
            except:
                pass

            # Create the new workflow.
            wf = self._client.workflows.create(def_yaml)[0]

        # Update the workflow definition.
        if wf.definition != def_yaml:
            self._client.workflows.update(def_yaml)

    def _find_default_workflow(self, def_dict):
        num_workflows = len(def_dict['workflows'].keys())

        if num_workflows > 1:
            fully_qualified_wf_name = self.runner_parameters.get('workflow')
            if not fully_qualified_wf_name:
                raise ValueError('Workbook definition is detected. '
                                 'Default workflow cannot be determined.')

            wf_name = fully_qualified_wf_name[fully_qualified_wf_name.rindex('.') + 1:]
            if wf_name not in def_dict['workflows']:
                raise ValueError('Unable to find the workflow "%s" in the workbook.'
                                 % fully_qualified_wf_name)

            return fully_qualified_wf_name
        elif num_workflows == 1:
            return '%s.%s' % (def_dict['name'], def_dict['workflows'].keys()[0])
        else:
            raise Exception('There are no workflows in the workbook.')

    def run(self, action_parameters):
        try:
            # Test the connection
            self._client.workflows.list()
        except requests.exceptions.ConnectionError:
            msg = ('Failed to connect to mistral on %s. Make sure that mistral is running '
                   ' and that the url is set correctly in the config.' % (self.url))
            raise Exception(msg)

        # Setup inputs for the workflow execution.
        inputs = self.runner_parameters.get('context', dict())
        inputs.update(action_parameters)

        endpoint = 'http://%s:%s/v1/actionexecutions' % (cfg.CONF.api.host, cfg.CONF.api.port)

        st2_execution_context = {
            'endpoint': endpoint,
            'parent': self.liveaction_id
        }

        options = {
            'env': {
                '__actions': {
                    'st2.action': {
                        'st2_context': st2_execution_context
                    }
                }
            }
        }

        # Get workbook/workflow definition from file.
        with open(self.entry_point, 'r') as def_file:
            def_yaml = def_file.read()

        def_dict = yaml.safe_load(def_yaml)
        is_workbook = ('workflows' in def_dict)

        if not is_workbook:
            # Non-workbook definition containing multiple workflows is not supported.
            if len([k for k, _ in six.iteritems(def_dict) if k != 'version']) != 1:
                raise Exception('Workflow (not workbook) definition is detected. '
                                'Multiple workflows is not supported.')

        action_ref = '%s.%s' % (self.action.pack, self.action.name)
        self._check_name(action_ref, is_workbook, def_dict)
        def_dict_xformed = utils.transform_definition(def_dict)
        def_yaml_xformed = yaml.safe_dump(def_dict_xformed, default_flow_style=False)

        # Save workbook/workflow definition.
        if is_workbook:
            self._save_workbook(action_ref, def_yaml_xformed)
            default_workflow = self._find_default_workflow(def_dict_xformed)
            execution = self._client.executions.create(default_workflow,
                                                       workflow_input=inputs,
                                                       **options)
        else:
            self._save_workflow(action_ref, def_yaml_xformed)
            execution = self._client.executions.create(action_ref,
                                                       workflow_input=inputs,
                                                       **options)

        status = LIVEACTION_STATUS_RUNNING
        partial_results = {'tasks': []}

        current_context = {
            'execution_id': str(execution.id),
            'workflow_name': execution.workflow_name
        }

        exec_context = self.context
        exec_context = self._build_mistral_context(exec_context, current_context)
        LOG.info('Mistral query context is %s' % exec_context)

        return (status, partial_results, exec_context)

    @staticmethod
    def _build_mistral_context(parent, current):
        context = dict()

        if not parent:
            context['mistral'] = current
        else:
            if 'mistral' in parent.keys():
                orig_parent_context = parent.get('mistral', dict())
                actual_parent = dict()
                if 'workflow_name' in orig_parent_context.keys():
                    actual_parent['workflow_name'] = orig_parent_context['workflow_name']
                    del orig_parent_context['workflow_name']
                if 'execution_id' in orig_parent_context.keys():
                    actual_parent['execution_id'] = orig_parent_context['execution_id']
                    del orig_parent_context['execution_id']
                context['mistral'] = orig_parent_context
                context['mistral'].update(current)
                context['mistral']['parent'] = actual_parent
            else:
                context['mistral'] = current

        return context
<commit_msg>Make a deep copy of exec context<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import uuid

import six
import yaml
import requests
from mistralclient.api import client as mistral
from oslo.config import cfg

from st2common.constants.action import LIVEACTION_STATUS_RUNNING
from st2actions.runners import AsyncActionRunner
from st2actions.runners.mistral import utils
from st2common import log as logging

LOG = logging.getLogger(__name__)

def get_runner():
    return MistralRunner(str(uuid.uuid4()))

class MistralRunner(AsyncActionRunner):

    url = cfg.CONF.workflow.url

    def __init__(self, runner_id):
        super(MistralRunner, self).__init__(runner_id=runner_id)
        self._on_behalf_user = cfg.CONF.system_user.user
        self._client = mistral.client(mistral_url='%s/v2' % self.url)

    def pre_run(self):
        pass

    @staticmethod
    def _check_name(action_ref, is_workbook, def_dict):
        # If workbook, change the value of the "name" key.
        if is_workbook:
            if def_dict.get('name') != action_ref:
                raise Exception('Name of the workbook must be the same as the '
                                'fully qualified action name "%s".' % action_ref)
        # If workflow, change the key name of the workflow.
        else:
            workflow_name = [k for k, v in six.iteritems(def_dict) if k != 'version'][0]
            if workflow_name != action_ref:
                raise Exception('Name of the workflow must be the same as the '
                                'fully qualified action name "%s".' % action_ref)

    def _save_workbook(self, name, def_yaml):
        # If the workbook is not found, the mistral client throws a generic API exception.
        try:
            # Update existing workbook.
            wb = self._client.workbooks.get(name)
        except:
            # Delete if definition was previously a workflow.
            # If not found, an API exception is thrown.
            try:
                self._client.workflows.delete(name)
            except:
                pass

            # Create the new workbook.
            wb = self._client.workbooks.create(def_yaml)

        # Update the workbook definition.
        if wb.definition != def_yaml:
            self._client.workbooks.update(def_yaml)

    def _save_workflow(self, name, def_yaml):
        # If the workflow is not found, the mistral client throws a generic API exception.
        try:
            # Update existing workbook.
            wf = self._client.workflows.get(name)
        except:
            # Delete if definition was previously a workbook.
            # If not found, an API exception is thrown.
            try:
                self._client.workbooks.delete(name)
            except:
                pass

            # Create the new workflow.
            wf = self._client.workflows.create(def_yaml)[0]

        # Update the workflow definition.
        if wf.definition != def_yaml:
            self._client.workflows.update(def_yaml)

    def _find_default_workflow(self, def_dict):
        num_workflows = len(def_dict['workflows'].keys())

        if num_workflows > 1:
            fully_qualified_wf_name = self.runner_parameters.get('workflow')
            if not fully_qualified_wf_name:
                raise ValueError('Workbook definition is detected. '
                                 'Default workflow cannot be determined.')

            wf_name = fully_qualified_wf_name[fully_qualified_wf_name.rindex('.') + 1:]
            if wf_name not in def_dict['workflows']:
                raise ValueError('Unable to find the workflow "%s" in the workbook.'
                                 % fully_qualified_wf_name)

            return fully_qualified_wf_name
        elif num_workflows == 1:
            return '%s.%s' % (def_dict['name'], def_dict['workflows'].keys()[0])
        else:
            raise Exception('There are no workflows in the workbook.')

    def run(self, action_parameters):
        try:
            # Test the connection
            self._client.workflows.list()
        except requests.exceptions.ConnectionError:
            msg = ('Failed to connect to mistral on %s. Make sure that mistral is running '
                   ' and that the url is set correctly in the config.' % (self.url))
            raise Exception(msg)

        # Setup inputs for the workflow execution.
        inputs = self.runner_parameters.get('context', dict())
        inputs.update(action_parameters)

        endpoint = 'http://%s:%s/v1/actionexecutions' % (cfg.CONF.api.host, cfg.CONF.api.port)

        st2_execution_context = {
            'endpoint': endpoint,
            'parent': self.liveaction_id
        }

        options = {
            'env': {
                '__actions': {
                    'st2.action': {
                        'st2_context': st2_execution_context
                    }
                }
            }
        }

        # Get workbook/workflow definition from file.
        with open(self.entry_point, 'r') as def_file:
            def_yaml = def_file.read()

        def_dict = yaml.safe_load(def_yaml)
        is_workbook = ('workflows' in def_dict)

        if not is_workbook:
            # Non-workbook definition containing multiple workflows is not supported.
            if len([k for k, _ in six.iteritems(def_dict) if k != 'version']) != 1:
                raise Exception('Workflow (not workbook) definition is detected. '
                                'Multiple workflows is not supported.')

        action_ref = '%s.%s' % (self.action.pack, self.action.name)
        self._check_name(action_ref, is_workbook, def_dict)
        def_dict_xformed = utils.transform_definition(def_dict)
        def_yaml_xformed = yaml.safe_dump(def_dict_xformed, default_flow_style=False)

        # Save workbook/workflow definition.
        if is_workbook:
            self._save_workbook(action_ref, def_yaml_xformed)
            default_workflow = self._find_default_workflow(def_dict_xformed)
            execution = self._client.executions.create(default_workflow,
                                                       workflow_input=inputs,
                                                       **options)
        else:
            self._save_workflow(action_ref, def_yaml_xformed)
            execution = self._client.executions.create(action_ref,
                                                       workflow_input=inputs,
                                                       **options)

        status = LIVEACTION_STATUS_RUNNING
        partial_results = {'tasks': []}

        current_context = {
            'execution_id': str(execution.id),
            'workflow_name': execution.workflow_name
        }

        exec_context = copy.deepcopy(self.context)
        exec_context = self._build_mistral_context(exec_context, current_context)
        LOG.info('Mistral query context is %s' % exec_context)

        return (status, partial_results, exec_context)

    @staticmethod
    def _build_mistral_context(parent, current):
        context = dict()

        if not parent:
            context['mistral'] = current
        else:
            if 'mistral' in parent.keys():
                orig_parent_context = parent.get('mistral', dict())
                actual_parent = dict()
                if 'workflow_name' in orig_parent_context.keys():
                    actual_parent['workflow_name'] = orig_parent_context['workflow_name']
                    del orig_parent_context['workflow_name']
                if 'execution_id' in orig_parent_context.keys():
                    actual_parent['execution_id'] = orig_parent_context['execution_id']
                    del orig_parent_context['execution_id']
                context['mistral'] = orig_parent_context
                context['mistral'].update(current)
                context['mistral']['parent'] = actual_parent
            else:
                context['mistral'] = current

        return context
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.core.management.base import BaseCommand

from stardate.models import Blog, Post

class Command(BaseCommand):
    """
    Usage:
    ./manage.py import_posts [username ...]

    When importing posts from the backend, a backend should check that
    there is new data on the backend itself, if there is new data, then
    the backend may fetch new data to be updated.

    """
    args = '<username username ...>'
    help = 'Imports posts for the specified blogs'

    def handle(self, *args, **kwargs):
        if args:
            for username in args:
                try:
                    user = User.objects.get(username=username)
                    for blog in Blog.objects.filter(owner=user.id):
                        blog.save_post_objects()
                except User.DoesNotExist:
                    print u'User with username %s does not exist.' % username
        else:
            for user in User.objects.all():
                try:
                    for blog in Blog.objects.filter(owner=user.id):
                        path = blog.get_backend_choice()
                        print 'Processing %s, %s' % (blog.name, path)
                        try:
                            for post in blog.backend.get_posts(path):
                                post.update(blog_id=blog.id)
                                obj, created = Post.objects.get_or_create(
                                    stardate=post['stardate'])
                                obj.__dict__.update(**post)
                                obj.save()
                        except:
                            raise
                except:
                    raise
<commit_msg>update a post if not created<commit_after>from django.contrib.auth.models import User
from django.core.management.base import BaseCommand

from stardate.models import Blog, Post

class Command(BaseCommand):
    """
    Usage:
    ./manage.py import_posts [username ...]

    When importing posts from the backend, a backend should check that
    there is new data on the backend itself, if there is new data, then
    the backend may fetch new data to be updated.

    """
    args = '<username username ...>'
    help = 'Imports posts for the specified blogs'

    def handle(self, *args, **kwargs):
        if args:
            for username in args:
                try:
                    user = User.objects.get(username=username)
                    for blog in Blog.objects.filter(owner=user.id):
                        blog.save_post_objects()
                except User.DoesNotExist:
                    print u'User with username %s does not exist.' % username
        else:
            for user in User.objects.all():
                try:
                    for blog in Blog.objects.filter(owner=user.id):
                        path = blog.get_backend_choice()
                        print 'Processing %s, %s' % (blog.name, path)
                        try:
                            for post in blog.backend.get_posts(path):
                                post.update(blog_id=blog.id)
                                obj, created = Post.objects.get_or_create(**post)
                                if not created:
                                    obj.__dict__.update(**post)
                                    obj.save()
                        except:
                            raise
                except:
                    raise
<|endoftext|>