<commit_before># Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

# snippet-sourcedescription:[assume_role.py demonstrates how to use the Boto3 interface to call AssumeRole. The temporary security credentials returned by AssumeRole are then used to list all Amazon S3 buckets in the account.]
# snippet-service:[iam]
# snippet-keyword:[Python]
# snippet-keyword:[AWS Identity and Access Management (IAM)]
# snippet-keyword:[Code Sample]
# snippet-keyword:[AssumeRole]
# snippet-sourcetype:[full-example]
# snippet-sourcedate:[2019-01-22]
# snippet-sourceauthor:[stephswo (AWS)]
# snippet-start:[iam.python.assume_role.complete]

import boto3

# The calls to AWS STS AssumeRole must be signed with the access key ID
# and secret access key of an existing IAM user or by using existing temporary
# credentials such as those from antoher role. (You cannot call AssumeRole
# with the access key for the root account.) The credentials can be in
# environment variables or in a configuration file and will be discovered
# automatically by the boto3.client() function. For more information, see the
# Python SDK documentation:
# http://boto3.readthedocs.io/en/latest/reference/services/sts.html#client

# create an STS client object that represents a live connection to the
# STS service
sts_client = boto3.client('sts')

# Call the assume_role method of the STSConnection object and pass the role
# ARN and a role session name.
assumed_role_object=sts_client.assume_role(
    RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role",
    RoleSessionName="AssumeRoleSession1"
)

# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials=assumed_role_object['Credentials']

# Use the temporary credentials that AssumeRole returns to make a
# connection to Amazon S3
s3_resource=boto3.resource(
    's3',
    aws_access_key_id=credentials['AccessKeyId'],
    aws_secret_access_key=credentials['SecretAccessKey'],
    aws_session_token=credentials['SessionToken'],
)

# Use the Amazon S3 resource object that is now configured with the
# credentials to access your S3 buckets.
for bucket in s3_resource.buckets.all():
    print(bucket.name)

# snippet-end:[iam.python.assume_role.complete]
<commit_msg>Update link to boto<commit_after># Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

# snippet-sourcedescription:[assume_role.py demonstrates how to use the Boto3 interface to call AssumeRole. The temporary security credentials returned by AssumeRole are then used to list all Amazon S3 buckets in the account.]
# snippet-service:[iam]
# snippet-keyword:[Python]
# snippet-keyword:[AWS Identity and Access Management (IAM)]
# snippet-keyword:[Code Sample]
# snippet-keyword:[AssumeRole]
# snippet-sourcetype:[full-example]
# snippet-sourcedate:[2019-01-22]
# snippet-sourceauthor:[stephswo (AWS)]
# snippet-start:[iam.python.assume_role.complete]

import boto3

# The calls to AWS STS AssumeRole must be signed with the access key ID
# and secret access key of an existing IAM user or by using existing temporary
# credentials such as those from antoher role. (You cannot call AssumeRole
# with the access key for the root account.) The credentials can be in
# environment variables or in a configuration file and will be discovered
# automatically by the boto3.client() function. For more information, see the
# Python SDK documentation:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#client

# create an STS client object that represents a live connection to the
# STS service
sts_client = boto3.client('sts')

# Call the assume_role method of the STSConnection object and pass the role
# ARN and a role session name.
assumed_role_object=sts_client.assume_role(
    RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role",
    RoleSessionName="AssumeRoleSession1"
)

# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials=assumed_role_object['Credentials']

# Use the temporary credentials that AssumeRole returns to make a
# connection to Amazon S3
s3_resource=boto3.resource(
    's3',
    aws_access_key_id=credentials['AccessKeyId'],
    aws_secret_access_key=credentials['SecretAccessKey'],
    aws_session_token=credentials['SessionToken'],
)

# Use the Amazon S3 resource object that is now configured with the
# credentials to access your S3 buckets.
for bucket in s3_resource.buckets.all():
    print(bucket.name)

# snippet-end:[iam.python.assume_role.complete]
<|endoftext|>


<commit_before>'''
Created on Mar 18, 2016
This module is responsible for converting the tasks csv file to tasks objects and sends them out to the storage service
@author: auerbach
'''
from SuperGLU.Core.MessagingGateway import BaseService
from SuperGLU.Services.StudentModel.PersistentData import SerializableTask,\
    SerializableAssistmentsItem

class CSVReader (BaseService):

    LINE_DELIMITER = '\n'
    CELL_DELIMITER = ','
    PIPE_DELIMITER = '|||'

    def receiveMessage(self, msg):
        #TODO: implementation
        raise NotImplementedError

    def processCSVFile(self, csvString):
        #List of strings
        lines = csvString.split(sep=self.LINE_DELIMITER)

        #list of tasks
        result = []

        #remove the line containing the labels
        lines.pop(0)

        for line in lines:
            cells = line.split(sep=self.CELL_DELIMITER)

            task = SerializableTask()

            kcCell = cells[10]
            kcs = kcCell.split(self.PIPE_DELIMITER)
            task._kcs = kcs

            task._name = cells[2]
            task._taskId = cells[1]
            task._baseURL = cells[6]

            #if there is an assismentsItem associated with this task
            if cells[11] is not None and cells[11] is not '':
                assistmentsItem = SerializableAssistmentsItem()
                assistmentsItem._itemID = cells[11]
                assistmentsItem._problemSetID = cells[12]
                assistmentsItem._problemSetName = cells[13]
                assistmentsItem._assignments = []

                if cells[14] is not None and cells[14] is not '':
                    assistmentsItem._assignments.append((cells[14], cells[15], cells[16]))
                if cells[17] is not None and cells[17] is not '':
                    assistmentsItem._assignments.append((cells[17], cells[18], cells[19]))
                if cells[20] is not None and cells[20] is not '':
                    assistmentsItem._assignments.append((cells[20], cells[21], cells[22]))

                task._assistmentsItem = assistmentsItem
            else:
                task._assistmentsItem = None

            result.append(task)

        return result
            <commit_msg>started work on the messaging portion of the CSVReader<commit_after>'''
Created on Mar 18, 2016
This module is responsible for converting the tasks csv file to tasks objects and sends them out to the storage service
@author: auerbach
'''
from SuperGLU.Core.MessagingGateway import BaseService
from SuperGLU.Services.StudentModel.PersistentData import SerializableTask, SerializableAssistmentsItem
from SuperGLU.Core.FIPA.SpeechActs import INFORM_ACT
from SuperGLU.Util.ErrorHandling import logInfo
from SuperGLU.Core.Messaging import Message
from SuperGLU.Services.Authentication.UserDataService import VALUE_VERB
from SuperGLU.Util.Serialization import serializeObject

CSV_READER_SERVICE_NAME = "CSV Reader Service"

class CSVReader (BaseService):

    LINE_DELIMITER = '\n'
    CELL_DELIMITER = ','
    PIPE_DELIMITER = '|||'

    def receiveMessage(self, msg):
        #depending on the content of the message react differently
        logInfo('Entering CSVReader.receiveMessage', 5)

        reply = None
        #Only considering
        if msg.getSpeechAct() == INFORM_ACT:

            if True:
                logInfo('{0} is processing a {1},{2} message'.format(CSV_READER_SERVICE_NAME, CSV_READER_SERVICE_NAME, INFORM_ACT), 4)

                csvString = msg.getResult()
                taskList = self.processCSVFile(csvString)

                reply = Message()
                reply.setSpeechAct(INFORM_ACT)
                reply.setVerb(VALUE_VERB)
                reply.setObject(taskList)

        if reply is not None:
            logInfo('{0} is broadcasting a {1}, {2} message'.format(CSV_READER_SERVICE_NAME, INFORM_ACT, VALUE_VERB), 4)
            self.sendMessage(reply)

    def processCSVFile(self, csvString):
        #List of strings
        logInfo('Entering CSVReader.processCSVFile', 4)
        lines = csvString.split(sep=self.LINE_DELIMITER)

        #list of tasks
        result = []

        #remove the line containing the labels
        lines.pop(0)

        for line in lines:

            logInfo('{0} is splitting the line into cells'.format(CSV_READER_SERVICE_NAME), 5)
            cells = line.split(sep=self.CELL_DELIMITER)

            task = SerializableTask()

            logInfo('{0} is extracting the knowledge components'.format(CSV_READER_SERVICE_NAME), 5)
            kcCell = cells[10]
            kcs = kcCell.split(self.PIPE_DELIMITER)

            logInfo('{0} is constructing the next serializable task object'.format(CSV_READER_SERVICE_NAME), 5)
            task._kcs = kcs

            task._name = cells[2]
            task._taskId = cells[1]
            task._baseURL = cells[6]

            #if there is an assismentsItem associated with this task
            if cells[11] is not None and cells[11] is not '':
                assistmentsItem = SerializableAssistmentsItem()
                assistmentsItem._itemID = cells[11]
                assistmentsItem._problemSetID = cells[12]
                assistmentsItem._problemSetName = cells[13]
                assistmentsItem._assignments = []

                logInfo('{0} is checking for assistments data'.format(CSV_READER_SERVICE_NAME), 5)
                if cells[14] is not None and cells[14] is not '':
                    assistmentsItem._assignments.append((cells[14], cells[15], cells[16]))
                if cells[17] is not None and cells[17] is not '':
                    assistmentsItem._assignments.append((cells[17], cells[18], cells[19]))
                if cells[20] is not None and cells[20] is not '':
                    assistmentsItem._assignments.append((cells[20], cells[21], cells[22]))

                task._assistmentsItem = assistmentsItem
            else:
                task._assistmentsItem = None

            #this could cause performance trouble so only use this log if you need to.
            #logInfo('{0} constructed task: {1}'.format(CSV_READER_SERVICE_NAME, serializeObject(task)), 5)
            logInfo('{0} constructed task with name: {1}'.format(CSV_READER_SERVICE_NAME, task._name), 5)
            result.append(task)

        logInfo('Exiting CSVReader.processCSVFile', 4)
        return result
            <|endoftext|>


<commit_before>import Queue
import threading
import json
import abc
import docker
import os.path
from common.base import INGIniousConfiguration

class JobManager (threading.Thread):
    """ Abstract thread class that runs the jobs that are in the queue """
    def __init__(self):
        threading.Thread.__init__(self)
    def run(self):
        while True:
            # Retrieves the task from the queue
            jobId,task,inputdata,callback = main_queue.get()

            # Base dictonnary with output
            basedict = {"task":task,"input":inputdata}

            # Check task answer that do not need emulation
            first_result,need_emul,first_text,first_problems = task.checkAnswer(inputdata)
            finaldict = basedict.copy()
            finaldict.update({"result": ("success" if first_result else "failed")})
            if first_text != None:
                finaldict["text"] = first_text
            if first_problems:
                finaldict["problems"] = first_problems

            # Launch the emulation
            if need_emul:
                try:
                    emul_result = self.runJob(jobId, task, {"limits": task.getLimits(), "input": inputdata})
                    print json.dumps(emul_result, sort_keys=True, indent=4, separators=(',', ': '))
                except Exception as inst:
                    emul_result = {"result":"error","text":"The grader did not gave any output. This can be because you used too much memory."}

                if finaldict['result'] not in ["error","failed","success","timeout","overflow"]:
                    finaldict['result'] = "error"

                if emul_result["result"] not in ["error","timeout","overflow"]:
                    # Merge results
                    novmDict = finaldict
                    finaldict = emul_result

                    finaldict["result"] = "success" if novmDict["result"] == "success" and finaldict["result"] == "success" else "failed"
                    if "text" in finaldict and "text" in novmDict:
                        finaldict["text"] = finaldict["text"]+"\n"+"\n".join(novmDict["text"])
                    elif "text" not in finaldict and "text" in novmDict:
                        finaldict["text"] = "\n".join(novmDict["text"])

                    if "problems" in finaldict and "problems" in novmDict:
                        for p in novmDict["problems"]:
                            if p in finaldict["problems"]:
                                finaldict["problems"][p] = finaldict["problems"][p] + "\n" + novmDict["problems"][p]
                            else:
                                finaldict["problems"][p] = novmDict["problems"][p]
                    elif "problems" not in finaldict and "problems" in novmDict:
                        finaldict["problems"] = novmDict["problems"]
                elif emul_result["result"] in ["error","timeout","overflow"] and "text" in emul_result:
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":emul_result["text"]})
                elif emul_result["result"] == "error":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"An unknown internal error occured"})
                elif emul_result["result"] == "timeout":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"Your code took too much time to execute"})
                elif emul_result["result"] == "overflow":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"Your code took too much memory or disk"})

            main_dict[jobId] = finaldict
            if callback != None:
                callback(jobId)
    @abc.abstractmethod
    def runJob(self, jobId, task, inputdata):
        pass

class DockerJobManager (JobManager):
    def __init__(self):
        JobManager.__init__(self)
        self.docker = docker.Client(base_url=INGIniousConfiguration["dockerServerUrl"])
        self.buildAllContainers()
    def buildAllContainers(self):
        """ Ensures all containers are up to date """
        print "- Building containers"
        containers = [ f for f in os.listdir(INGIniousConfiguration["containersDirectory"]) if os.path.isdir(os.path.join(INGIniousConfiguration["containersDirectory"], f)) and os.path.isfile(os.path.join(INGIniousConfiguration["containersDirectory"], f, "Dockerfile"))]
        for container in containers:
            print "\tbuilding "+container
            try:
                self.buildContainer(container)
            except Exception as inst:
                print "\tthere was an error while building the container:"
                print "\t\t"+str(inst)
        print "- Containers have been built"
    def buildContainer(self,container):
        """ Ensures a container is up to date """
        r=self.docker.build(path=os.path.join(INGIniousConfiguration["containersDirectory"],container),tag=INGIniousConfiguration["containerPrefix"]+container)
        for i in r:
            if i == "\n" or i == "\r\n":
                continue
            try:
                j = json.loads(i)
            except:
                raise Exception("Error while building "+container+": can't read Docker output")
            if 'error' in j:
                raise Exception("Error while building "+container+": Docker returned error"+j["error"])

    def getSockets(self,containerId):
        """ Utility function to get stdin of a container """
        return self.docker.attach_socket(containerId,{'stdin': 1, 'stream': 1})

    def runJob(self, jobId, task, inputdata):
        """ Runs the job by launching a container """
        #limits: currently we only supports time and memory limits.
        #Memory is the memory used by the VM, in megabytes, and time is the time taken by the script (not the VM!) in seconds
        memLimit = task.getLimits()["memory"]
        if memLimit < 20:
            memLimit = 20
        elif memLimit > 500:
            memLimit = 500

        response = self.docker.create_container(
            INGIniousConfiguration["containerPrefix"]+task.getEnvironment(),
            stdin_open=True,
            network_disabled=True,
            volumes={'/ro/task':{}},
            mem_limit=memLimit*1024*1024
        )
        containerId = response["Id"]
        self.docker.start(containerId, binds={os.path.abspath(os.path.join(INGIniousConfiguration["tasksDirectory"],task.getCourseId(),task.getId())):{'ro':True,'bind':'/ro/task'}})
        self.getSockets(containerId).send(json.dumps(inputdata)+"\n")
        self.docker.wait(containerId)
        stdout = str(self.docker.logs(containerId, stdout=True, stderr=False))
        stderr = str(self.docker.logs(containerId, stdout=False, stderr=True))
        return json.loads(stdout)

def addJob(task, inputdata, callback = None):
    """ Add a job in the queue and returns a job id.
        task is a Task instance and inputdata is the input as a dictionary
        callback is a function (that can be None) that will be called ASYNC when the job is done.
        The callback receives the jobId as argument"""

    # Put task in the job queue
    addJob.cur_id  += 1
    jobId = addJob.cur_id
    main_queue.put((jobId,task,inputdata,callback))
    main_dict[jobId] = None

    # Returns the jobId
    return jobId

def isRunning(jobId):
    """ Tells if a job given by job id is running/in queue """
    if main_dict.has_key(jobId):
        return main_dict[jobId] == None
    else:
        return False

def isDone(jobId):
    """ Tells if a job given y job id is done and its result is available """
    if main_dict.has_key(jobId):
        return main_dict[jobId] != None
    else:
        return False

def getResult(jobId):
    """ Returns the result of a job given by a job id or None if the job is not finished/in queue.
        If the job is finished, subsequent call to getResult will return None (job is deleted)
        Results are dictionnaries with content similar to:
        {
            "task":task, #mandatory
            "input":inputdata,#mandatory
            "result": "error", #mandatory
            "text": "Error message to be displayed on the top of the exercice",
            "problems":{"pb1":"Error message for pb1"},
            "archive":"archive in base 64"
        }

        available result type are
        * error: VM crashed
        * failed: student made an error in his answers
        * success: student solved the exercice
        * timeout: student's code has timeout
        * overflow: memory or disk overflow
    """
    result = None

    # Delete result from dictionary if there is sth
    if main_dict.has_key(jobId) and (not main_dict[jobId] == None):
        result = main_dict[jobId]
        del main_dict[jobId]

    return result

# Initialization
addJob.cur_id = 0 # static variable
main_queue = Queue.Queue()
main_dict = {}

# Launch the main thread
main_thread = DockerJobManager()
main_thread.daemon = True
main_thread.start()
<commit_msg>Delete intermediate containers. Should save a lot of disk space ;-)<commit_after>import Queue
import threading
import json
import abc
import docker
import os.path
from common.base import INGIniousConfiguration

class JobManager (threading.Thread):
    """ Abstract thread class that runs the jobs that are in the queue """
    def __init__(self):
        threading.Thread.__init__(self)
    def run(self):
        while True:
            # Retrieves the task from the queue
            jobId,task,inputdata,callback = main_queue.get()

            # Base dictonnary with output
            basedict = {"task":task,"input":inputdata}

            # Check task answer that do not need emulation
            first_result,need_emul,first_text,first_problems = task.checkAnswer(inputdata)
            finaldict = basedict.copy()
            finaldict.update({"result": ("success" if first_result else "failed")})
            if first_text != None:
                finaldict["text"] = first_text
            if first_problems:
                finaldict["problems"] = first_problems

            # Launch the emulation
            if need_emul:
                try:
                    emul_result = self.runJob(jobId, task, {"limits": task.getLimits(), "input": inputdata})
                    print json.dumps(emul_result, sort_keys=True, indent=4, separators=(',', ': '))
                except Exception as inst:
                    emul_result = {"result":"error","text":"The grader did not gave any output. This can be because you used too much memory."}

                if finaldict['result'] not in ["error","failed","success","timeout","overflow"]:
                    finaldict['result'] = "error"

                if emul_result["result"] not in ["error","timeout","overflow"]:
                    # Merge results
                    novmDict = finaldict
                    finaldict = emul_result

                    finaldict["result"] = "success" if novmDict["result"] == "success" and finaldict["result"] == "success" else "failed"
                    if "text" in finaldict and "text" in novmDict:
                        finaldict["text"] = finaldict["text"]+"\n"+"\n".join(novmDict["text"])
                    elif "text" not in finaldict and "text" in novmDict:
                        finaldict["text"] = "\n".join(novmDict["text"])

                    if "problems" in finaldict and "problems" in novmDict:
                        for p in novmDict["problems"]:
                            if p in finaldict["problems"]:
                                finaldict["problems"][p] = finaldict["problems"][p] + "\n" + novmDict["problems"][p]
                            else:
                                finaldict["problems"][p] = novmDict["problems"][p]
                    elif "problems" not in finaldict and "problems" in novmDict:
                        finaldict["problems"] = novmDict["problems"]
                elif emul_result["result"] in ["error","timeout","overflow"] and "text" in emul_result:
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":emul_result["text"]})
                elif emul_result["result"] == "error":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"An unknown internal error occured"})
                elif emul_result["result"] == "timeout":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"Your code took too much time to execute"})
                elif emul_result["result"] == "overflow":
                    finaldict = basedict.copy()
                    finaldict.update({"result":emul_result["result"],"text":"Your code took too much memory or disk"})

            main_dict[jobId] = finaldict
            if callback != None:
                callback(jobId)
    @abc.abstractmethod
    def runJob(self, jobId, task, inputdata):
        pass

class DockerJobManager (JobManager):
    def __init__(self):
        JobManager.__init__(self)
        self.docker = docker.Client(base_url=INGIniousConfiguration["dockerServerUrl"])
        self.buildAllContainers()
    def buildAllContainers(self):
        """ Ensures all containers are up to date """
        print "- Building containers"
        containers = [ f for f in os.listdir(INGIniousConfiguration["containersDirectory"]) if os.path.isdir(os.path.join(INGIniousConfiguration["containersDirectory"], f)) and os.path.isfile(os.path.join(INGIniousConfiguration["containersDirectory"], f, "Dockerfile"))]
        for container in containers:
            print "\tbuilding "+container
            try:
                self.buildContainer(container)
            except Exception as inst:
                print "\tthere was an error while building the container:"
                print "\t\t"+str(inst)
        print "- Containers have been built"
    def buildContainer(self,container):
        """ Ensures a container is up to date """
        r=self.docker.build(path=os.path.join(INGIniousConfiguration["containersDirectory"],container),tag=INGIniousConfiguration["containerPrefix"]+container,rm=True)
        for i in r:
            if i == "\n" or i == "\r\n":
                continue
            try:
                j = json.loads(i)
            except:
                raise Exception("Error while building "+container+": can't read Docker output")
            if 'error' in j:
                raise Exception("Error while building "+container+": Docker returned error"+j["error"])

    def getSockets(self,containerId):
        """ Utility function to get stdin of a container """
        return self.docker.attach_socket(containerId,{'stdin': 1, 'stream': 1})

    def runJob(self, jobId, task, inputdata):
        """ Runs the job by launching a container """
        #limits: currently we only supports time and memory limits.
        #Memory is the memory used by the VM, in megabytes, and time is the time taken by the script (not the VM!) in seconds
        memLimit = task.getLimits()["memory"]
        if memLimit < 20:
            memLimit = 20
        elif memLimit > 500:
            memLimit = 500

        response = self.docker.create_container(
            INGIniousConfiguration["containerPrefix"]+task.getEnvironment(),
            stdin_open=True,
            network_disabled=True,
            volumes={'/ro/task':{}},
            mem_limit=memLimit*1024*1024
        )
        containerId = response["Id"]
        self.docker.start(containerId, binds={os.path.abspath(os.path.join(INGIniousConfiguration["tasksDirectory"],task.getCourseId(),task.getId())):{'ro':True,'bind':'/ro/task'}})
        self.getSockets(containerId).send(json.dumps(inputdata)+"\n")
        self.docker.wait(containerId)
        stdout = str(self.docker.logs(containerId, stdout=True, stderr=False))
        stderr = str(self.docker.logs(containerId, stdout=False, stderr=True))
        return json.loads(stdout)

def addJob(task, inputdata, callback = None):
    """ Add a job in the queue and returns a job id.
        task is a Task instance and inputdata is the input as a dictionary
        callback is a function (that can be None) that will be called ASYNC when the job is done.
        The callback receives the jobId as argument"""

    # Put task in the job queue
    addJob.cur_id  += 1
    jobId = addJob.cur_id
    main_queue.put((jobId,task,inputdata,callback))
    main_dict[jobId] = None

    # Returns the jobId
    return jobId

def isRunning(jobId):
    """ Tells if a job given by job id is running/in queue """
    if main_dict.has_key(jobId):
        return main_dict[jobId] == None
    else:
        return False

def isDone(jobId):
    """ Tells if a job given y job id is done and its result is available """
    if main_dict.has_key(jobId):
        return main_dict[jobId] != None
    else:
        return False

def getResult(jobId):
    """ Returns the result of a job given by a job id or None if the job is not finished/in queue.
        If the job is finished, subsequent call to getResult will return None (job is deleted)
        Results are dictionnaries with content similar to:
        {
            "task":task, #mandatory
            "input":inputdata,#mandatory
            "result": "error", #mandatory
            "text": "Error message to be displayed on the top of the exercice",
            "problems":{"pb1":"Error message for pb1"},
            "archive":"archive in base 64"
        }

        available result type are
        * error: VM crashed
        * failed: student made an error in his answers
        * success: student solved the exercice
        * timeout: student's code has timeout
        * overflow: memory or disk overflow
    """
    result = None

    # Delete result from dictionary if there is sth
    if main_dict.has_key(jobId) and (not main_dict[jobId] == None):
        result = main_dict[jobId]
        del main_dict[jobId]

    return result

# Initialization
addJob.cur_id = 0 # static variable
main_queue = Queue.Queue()
main_dict = {}

# Launch the main thread
main_thread = DockerJobManager()
main_thread.daemon = True
main_thread.start()
<|endoftext|>


<commit_before>#
# Copyright (C) 2012-2019 Red Hat, Inc.
#
# Licensed under the GNU Lesser General Public License Version 2.1
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

from __future__ import absolute_import

import copy
import os
import sys
import unittest

from . import base
import libdnf
import hawkey
import hawkey.test

class TestSackTest(base.TestCase):
    def test_sanity(self):
        assert(self.repo_dir)
        sack = base.TestSack(repo_dir=self.repo_dir)
        self.assertEqual(len(sack), 0)

    def test_load_rpm(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        self.assertEqual(len(sack), hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    def test_load_yum(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        sack.load_repo(build_cache=True)
        self.assertEqual(len(sack), hawkey.test.EXPECT_YUM_NSOLVABLES +
                         hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    # Loading test using hawkey.Repo
    def test_load_yum_hawkey_Repo(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        sack.load_repo_hawkey_Repo(build_cache=True)
        self.assertEqual(len(sack), hawkey.test.EXPECT_YUM_NSOLVABLES +
                         hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    def test_cache_dir(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        self.assertTrue(sack.cache_dir.startswith("/tmp/pyhawkey"))

class BasicTest(unittest.TestCase):
    def test_creation(self):
        hawkey.Sack(arch="noarch")
        hawkey.Sack(arch="x86_64")

    def test_deepcopy(self):
        sack = hawkey.Sack()
        self.assertRaises(NotImplementedError, copy.deepcopy, sack)

    def test_creation_dir(self):
        sack = hawkey.Sack()
        self.assertFalse(os.access(sack.cache_dir, os.F_OK))
        sack = hawkey.Sack(make_cache_dir=True)
        self.assertTrue(os.access(sack.cache_dir, os.F_OK))
        self.assertRaises(IOError, hawkey.Sack, "", make_cache_dir=True)

    def test_failed_load(self):
        sack = hawkey.Sack(cachedir=base.cachedir)
        self._conf = libdnf.conf.ConfigMain()
        repo_conf = libdnf.conf.ConfigRepo(self._conf)
        repo_conf.this.disown()  # _repo will be the owner of _config
        repo = libdnf.repo.Repo("name", repo_conf)
        self.assertRaises(IOError, sack.load_repo, repo)
        sack = hawkey.Sack()

    # Loading test using hawkey.Repo
    def test_failed_load_hawkey_Repo(self):
        sack = hawkey.Sack(cachedir=base.cachedir)
        repo = hawkey.Repo("name")
        self.assertRaises(IOError, sack.load_repo, repo)
        sack = hawkey.Sack()

    def test_unicoded_cachedir(self):
        # does not raise UnicodeEncodeError
        hawkey.Sack(cachedir=u"unicod\xe9")

    def test_evr_cmp(self):
        sack = hawkey.Sack()
        self.assertEqual(sack.evr_cmp("3:3.10-4", "3:3.10-4"), 0)
        self.assertLess(sack.evr_cmp("3.10-4", "3.10-5"), 0)
        self.assertGreater(sack.evr_cmp("3.11-4", "3.10-5"), 0)
        self.assertGreater(sack.evr_cmp("1:3.10-4", "3.10-5"), 0)

    def test_all_arch(self):
        sack = hawkey.Sack(arch="x86_64")
        # greater than noarch as it has picked up the default architecture
        self.assertGreater(len(sack.list_arches()), 1)
        sack2 = hawkey.Sack(all_arch=True)
        self.assertEqual(len(sack2.list_arches()), 0)

class PackageWrappingTest(base.TestCase):
    class MyPackage(hawkey.Package):
        def __init__(self, initobject, myval):
            super(PackageWrappingTest.MyPackage, self).__init__(initobject)
            self.initobject = initobject
            self.myval = myval

    def test_wrapping(self):
        sack = base.TestSack(repo_dir=self.repo_dir,
                                    PackageClass=self.MyPackage,
                                    package_userdata=42)
        sack.load_system_repo()
        pkg = sack.create_package(2)
        # it is the correct type:
        self.assertIsInstance(pkg, self.MyPackage)
        self.assertIsInstance(pkg, hawkey.Package)
        # it received userdata:
        self.assertEqual(pkg.myval, 42)
        # the common attributes are working:
        self.assertEqual(pkg.name, "baby")
<commit_msg>[tests] Don't use user cachedir in Python tests.<commit_after>#
# Copyright (C) 2012-2019 Red Hat, Inc.
#
# Licensed under the GNU Lesser General Public License Version 2.1
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

from __future__ import absolute_import

import copy
import os
import shutil
import sys
import tempfile
import unittest

from . import base
import libdnf
import hawkey
import hawkey.test

class TestSackTest(base.TestCase):
    def test_sanity(self):
        assert(self.repo_dir)
        sack = base.TestSack(repo_dir=self.repo_dir)
        self.assertEqual(len(sack), 0)

    def test_load_rpm(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        self.assertEqual(len(sack), hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    def test_load_yum(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        sack.load_repo(build_cache=True)
        self.assertEqual(len(sack), hawkey.test.EXPECT_YUM_NSOLVABLES +
                         hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    # Loading test using hawkey.Repo
    def test_load_yum_hawkey_Repo(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        sack.load_system_repo()
        sack.load_repo_hawkey_Repo(build_cache=True)
        self.assertEqual(len(sack), hawkey.test.EXPECT_YUM_NSOLVABLES +
                         hawkey.test.EXPECT_SYSTEM_NSOLVABLES)

    def test_cache_dir(self):
        sack = base.TestSack(repo_dir=self.repo_dir)
        self.assertTrue(sack.cache_dir.startswith("/tmp/pyhawkey"))

class BasicTest(unittest.TestCase):
    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(prefix="libdnf_test_")
        self.cache_dir = os.path.join(self.temp_dir, "cache")

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def test_creation(self):
        hawkey.Sack(arch="noarch")
        hawkey.Sack(arch="x86_64")

    def test_deepcopy(self):
        sack = hawkey.Sack()
        self.assertRaises(NotImplementedError, copy.deepcopy, sack)

    def test_creation_dir(self):
        sack = hawkey.Sack(cachedir=self.cache_dir)
        self.assertFalse(os.access(sack.cache_dir, os.F_OK))
        sack = hawkey.Sack(make_cache_dir=True)
        self.assertTrue(os.access(sack.cache_dir, os.F_OK))
        self.assertRaises(IOError, hawkey.Sack, "", make_cache_dir=True)

    def test_failed_load(self):
        sack = hawkey.Sack(cachedir=base.cachedir)
        self._conf = libdnf.conf.ConfigMain()
        repo_conf = libdnf.conf.ConfigRepo(self._conf)
        repo_conf.this.disown()  # _repo will be the owner of _config
        repo = libdnf.repo.Repo("name", repo_conf)
        self.assertRaises(IOError, sack.load_repo, repo)
        sack = hawkey.Sack()

    # Loading test using hawkey.Repo
    def test_failed_load_hawkey_Repo(self):
        sack = hawkey.Sack(cachedir=base.cachedir)
        repo = hawkey.Repo("name")
        self.assertRaises(IOError, sack.load_repo, repo)
        sack = hawkey.Sack()

    def test_unicoded_cachedir(self):
        # does not raise UnicodeEncodeError
        hawkey.Sack(cachedir=u"unicod\xe9")

    def test_evr_cmp(self):
        sack = hawkey.Sack()
        self.assertEqual(sack.evr_cmp("3:3.10-4", "3:3.10-4"), 0)
        self.assertLess(sack.evr_cmp("3.10-4", "3.10-5"), 0)
        self.assertGreater(sack.evr_cmp("3.11-4", "3.10-5"), 0)
        self.assertGreater(sack.evr_cmp("1:3.10-4", "3.10-5"), 0)

    def test_all_arch(self):
        sack = hawkey.Sack(arch="x86_64")
        # greater than noarch as it has picked up the default architecture
        self.assertGreater(len(sack.list_arches()), 1)
        sack2 = hawkey.Sack(all_arch=True)
        self.assertEqual(len(sack2.list_arches()), 0)

class PackageWrappingTest(base.TestCase):
    class MyPackage(hawkey.Package):
        def __init__(self, initobject, myval):
            super(PackageWrappingTest.MyPackage, self).__init__(initobject)
            self.initobject = initobject
            self.myval = myval

    def test_wrapping(self):
        sack = base.TestSack(repo_dir=self.repo_dir,
                                    PackageClass=self.MyPackage,
                                    package_userdata=42)
        sack.load_system_repo()
        pkg = sack.create_package(2)
        # it is the correct type:
        self.assertIsInstance(pkg, self.MyPackage)
        self.assertIsInstance(pkg, hawkey.Package)
        # it received userdata:
        self.assertEqual(pkg.myval, 42)
        # the common attributes are working:
        self.assertEqual(pkg.name, "baby")
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
This module contains functions that are not normally needed
to run the platform, but that can be useful during development,
or to cleanup various things in the database.

These functions are not integrated in the rest of the platform,
so running them involves starting a Django shell ("python manage.py shell"),
importing this module and running the function manually.
"""

from __future__ import unicode_literals

from collections import defaultdict

import backend.crossref
from backend.romeo import fetch_publisher
from backend.tasks import change_publisher_oa_status
from papers.models import Name
from papers.models import NameVariant
from papers.models import OaiRecord
from papers.models import Paper
from papers.models import Researcher
from papers.utils import sanitize_html
from publishers.models import AliasPublisher
from publishers.models import Publisher

def cleanup_researchers():
    """
    Deletes all the researchers who have not authored any paper.
    """
    deleted_count = 0
    for p in Researcher.objects.all():
        nb_papers = p.papers.count()
        if not nb_papers:
            deleted_count += 1
            p.delete()
    print "Deleted "+str(deleted_count)+" researchers"

def cleanup_names(dry_run=False):
    """
    Deletes all the names that are not linked to any researcher
    """
    deleted_count = 0
    for n in Name.objects.all():
        if NameVariant.objects.filter(name=n).count() == 0:
            deleted_count += 1
            if not dry_run:
                n.delete()
    print "Deleted "+str(deleted_count)+" names"

def merge_names(fro, to):
    """
    Merges the name object 'fro' into 'to
    """
    Researcher.objects.filter(name_id=fro.id).update(name_id=to.id)
    NameVariant.objects.filter(name_id=fro.id).update(name_id=to.id)
    fro.delete()

def update_paper_statuses():
    """
    Should only be run if something went wrong,
    the backend is supposed to update the fields by itself
    """
    papers = Paper.objects.all()
    for p in papers:
        p.update_availability()

def cleanup_titles():
    """
    Run HTML sanitizing on all the titles of the papers
    (this is normally done on creation of the papers, but
    not for old dumps of the database)
    """
    papers = Paper.objects.all()
    for p in papers:
        p.title = sanitize_html(p.title)
        p.save(update_fields=['title'])

def cleanup_abstracts():
    """
    Run HTML sanitizing on the abstracts
    (this is normally done on creation of the papers, but
    not for old dumps of the database)
    """
    for p in OaiRecord.objects.all():
        if p.description:
            new_abstract = sanitize_html(p.description)
            if new_abstract != p.description:
                p.description = new_abstract
                p.save()

def recompute_fingerprints():
    """
    Recomputes the fingerprints of all papers, merging
    those who end up having the same fingerprint
    """
    merged = 0
    for idx, p in enumerate(Paper.objects.all()):
        if idx % 100 == 0:
            print idx
        p.fingerprint = 'blah'
        match = p.recompute_fingerprint_and_merge_if_needed()
        if match is not None:
            merged += 1
    print "%d papers merged" % merged

def find_collisions():
    """
    Recomputes all the fingerprints and reports those which would be
    merged by recompute_fingerprints()
    """
    dct = defaultdict(set)
    for p in Paper.objects.all():
        fp = p.new_fingerprint()
        dct[fp].add(p)

    for fp, s in dct.items():
        if len(s) > 1:
            first = True
            for paper in s:
                if first:
                    first = False
                    print "### "+paper.plain_fingerprint()
                print paper.title
                print paper.bare_author_names()

def create_publisher_aliases(erase_existing=True):
    # TODO: this might be more efficient with aggregates?
    counts = defaultdict(int)
    for p in OaiRecord.objects.all():
        if p.publisher_id:
            pair = (p.publisher_name, p.publisher_id)
            counts[pair] += 1

    if erase_existing:
        AliasPublisher.objects.all().delete()

    for (name, pk) in counts:
        if not erase_existing:
            alias = AliasPublisher.objects.get_or_create(
                name=name, publisher_id=pk)
        else:
            alias = AliasPublisher(name=name, publisher_id=pk)
        alias.count = counts[(name, pk)]
        alias.save()

def refetch_publishers():
    """
    Tries to assign publishers to OaiRecords without Journals
    """
    for p in OaiRecord.objects.filter(publisher__isnull=True, publisher_name__isnull=False):
        publisher = fetch_publisher(p.publisher_name)
        if publisher:
            p.publisher = publisher
            p.save(update_fields=['publisher'])
            p.paper.update_availability()

def refetch_containers():
    """
    Tries to assign containers to OaiRecords without containers
    """
    # TODO
    for p in OaiRecord.objects.filter(container__isnull=True):
        metadata = backend.crossref.fetch_metadata_by_DOI(p.doi)
        if metadata is None:
            continue
        p.container = metadata.get('container-title')
        if p.container:
            p.save()

def recompute_publisher_policies():
    """
    Recomputes the publisher policy according to some possibly new criteria
    """
    for p in Publisher.objects.all():
        change_publisher_oa_status(p.pk, p.classify_oa_status())
<commit_msg>Add maintenance method to clean up researcher ids in papers<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
This module contains functions that are not normally needed
to run the platform, but that can be useful during development,
or to cleanup various things in the database.

These functions are not integrated in the rest of the platform,
so running them involves starting a Django shell ("python manage.py shell"),
importing this module and running the function manually.
"""

from __future__ import unicode_literals

from collections import defaultdict
from bulk_update.helper import bulk_update

import backend.crossref
from backend.romeo import fetch_publisher
from backend.tasks import change_publisher_oa_status
from papers.models import Name
from papers.models import NameVariant
from papers.models import OaiRecord
from papers.models import Paper
from papers.models import Researcher
from papers.utils import sanitize_html
from publishers.models import AliasPublisher
from publishers.models import Publisher

def cleanup_researchers():
    """
    Deletes all the researchers who have not authored any paper.
    """
    deleted_count = 0
    for p in Researcher.objects.all():
        nb_papers = p.papers.count()
        if not nb_papers:
            deleted_count += 1
            p.delete()
    print "Deleted "+str(deleted_count)+" researchers"

def cleanup_names(dry_run=False):
    """
    Deletes all the names that are not linked to any researcher
    """
    deleted_count = 0
    for n in Name.objects.all():
        if NameVariant.objects.filter(name=n).count() == 0:
            deleted_count += 1
            if not dry_run:
                n.delete()
    print "Deleted "+str(deleted_count)+" names"

def merge_names(fro, to):
    """
    Merges the name object 'fro' into 'to
    """
    Researcher.objects.filter(name_id=fro.id).update(name_id=to.id)
    NameVariant.objects.filter(name_id=fro.id).update(name_id=to.id)
    fro.delete()

def update_paper_statuses():
    """
    Should only be run if something went wrong,
    the backend is supposed to update the fields by itself
    """
    papers = Paper.objects.all()
    for p in papers:
        p.update_availability()

def cleanup_titles():
    """
    Run HTML sanitizing on all the titles of the papers
    (this is normally done on creation of the papers, but
    not for old dumps of the database)
    """
    papers = Paper.objects.all()
    for p in papers:
        p.title = sanitize_html(p.title)
        p.save(update_fields=['title'])

def cleanup_abstracts():
    """
    Run HTML sanitizing on the abstracts
    (this is normally done on creation of the papers, but
    not for old dumps of the database)
    """
    for p in OaiRecord.objects.all():
        if p.description:
            new_abstract = sanitize_html(p.description)
            if new_abstract != p.description:
                p.description = new_abstract
                p.save()

def recompute_fingerprints():
    """
    Recomputes the fingerprints of all papers, merging
    those who end up having the same fingerprint
    """
    merged = 0
    for idx, p in enumerate(Paper.objects.all()):
        if idx % 100 == 0:
            print idx
        p.fingerprint = 'blah'
        match = p.recompute_fingerprint_and_merge_if_needed()
        if match is not None:
            merged += 1
    print "%d papers merged" % merged

def find_collisions():
    """
    Recomputes all the fingerprints and reports those which would be
    merged by recompute_fingerprints()
    """
    dct = defaultdict(set)
    for p in Paper.objects.all():
        fp = p.new_fingerprint()
        dct[fp].add(p)

    for fp, s in dct.items():
        if len(s) > 1:
            first = True
            for paper in s:
                if first:
                    first = False
                    print "### "+paper.plain_fingerprint()
                print paper.title
                print paper.bare_author_names()

def create_publisher_aliases(erase_existing=True):
    # TODO: this might be more efficient with aggregates?
    counts = defaultdict(int)
    for p in OaiRecord.objects.all():
        if p.publisher_id:
            pair = (p.publisher_name, p.publisher_id)
            counts[pair] += 1

    if erase_existing:
        AliasPublisher.objects.all().delete()

    for (name, pk) in counts:
        if not erase_existing:
            alias = AliasPublisher.objects.get_or_create(
                name=name, publisher_id=pk)
        else:
            alias = AliasPublisher(name=name, publisher_id=pk)
        alias.count = counts[(name, pk)]
        alias.save()

def refetch_publishers():
    """
    Tries to assign publishers to OaiRecords without Journals
    """
    for p in OaiRecord.objects.filter(publisher__isnull=True, publisher_name__isnull=False):
        publisher = fetch_publisher(p.publisher_name)
        if publisher:
            p.publisher = publisher
            p.save(update_fields=['publisher'])
            p.paper.update_availability()

def refetch_containers():
    """
    Tries to assign containers to OaiRecords without containers
    """
    # TODO
    for p in OaiRecord.objects.filter(container__isnull=True):
        metadata = backend.crossref.fetch_metadata_by_DOI(p.doi)
        if metadata is None:
            continue
        p.container = metadata.get('container-title')
        if p.container:
            p.save()

def recompute_publisher_policies():
    """
    Recomputes the publisher policy according to some possibly new criteria
    """
    for p in Publisher.objects.all():
        change_publisher_oa_status(p.pk, p.classify_oa_status())

def cleanup_paper_researcher_ids():
    """
    Ensures that all researcher_ids in Papers link to actual researchers
    """
    researcher_ids = set(Researcher.objects.all().values_list('id', flat=True))
    bs = 1000
    curid = 0
    found = True
    while found:
        found = False
        batch = []
        for p in Paper.objects.filter(id__gt=curid).order_by('id')[:bs]:
            curid = p.id
            found = True
            modified = False
            for i, author in enumerate(p.authors_list):
                rid = author['researcher_id']
                if (rid is not None and rid not in researcher_ids):
                    p.authors_list[i]['researcher_id'] = None
                modified = True
            if modified:
                batch.append(p)
        print "Updating %d papers" % len(batch)
        bulk_update(batch)
<|endoftext|>


<commit_before>from django.db import models

from app.models.account import *

class Shed(models.Model):
    name = models.CharField(max_length=100, unique=True, blank=False)
    registered_on = models.DateTimeField(auto_now_add=True)

    # Foreign keys
    owner = models.ForeignKey(User)
    address = models.ForeignKey(Address)

    @property
    def share_zone(self):
        return (self.address.zip[:5]).ljust(9, '0')

    class Meta:
        app_label = 'app'

    def __str__(self):
        return self.name<commit_msg>Change to share zone property of shed which follows new convention<commit_after>from django.db import models

from app.models.account import *

class Shed(models.Model):
    name = models.CharField(max_length=100, unique=True, blank=False)
    registered_on = models.DateTimeField(auto_now_add=True)

    # Foreign keys
    owner = models.ForeignKey(User)
    address = models.ForeignKey(Address)

    @property
    def share_zone(self):
        return self.address.share_zone

    class Meta:
        app_label = 'app'

    def __str__(self):
        return self.name<|endoftext|>


<commit_before># this requires both the hierarchical and nonhierarchical MCMCs to have been run
# also requires the posterior predictive CDFs to have been computed in the hierarchical case

import doseresponse as dr
import matplotlib
#matplotlib.rc('font', family='ubuntu')
#matplotlib.use('Agg')
import matplotlib.pyplot as plt
import argparse
import itertools as it
import numpy as np
import sys
import numpy.random as npr
import scipy.stats as st

seed = 1
npr.seed(seed)

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--concs", nargs="+", type=float, help="drug concentrations at which to find predicted responses",default=[])
parser.add_argument("-a", "--all", action='store_true', help='plot histograms from hierarchical MCMC on all drugs and channels', default=False)
parser.add_argument("-nc", "--num-cores", type=int, help="number of cores to parallelise drug/channel combinations", default=1)
parser.add_argument("-Ne", "--num-expts", type=int, help="how many experiments to fit to, otherwise will fit to all in the data file", default=0)
parser.add_argument("--data-file", type=str, help="csv file from which to read in data, in same format as provided crumb_data.csv")
parser.add_argument("--fix-hill", action='store_true', help="fix Hill=1 through fitting and MCMC",default=False)
parser.add_argument("-T", "--num-samples", type=int, help="number of samples to plot", required=True)
parser.add_argument("-H", "--num-hist-samples", type=int, help="number of samples for histogram --- should be much higher, really", required=True)
args = parser.parse_args()

dr.setup(args.data_file)

drugs_to_run, channels_to_run = dr.list_drug_channel_options(args.all)

colors = ['#a6cee3','#1f78b4','#b2df8a','#33a02c','#fb9a99']

def run(drug_channel):
    drug,channel = drug_channel
    print "\n\n{} + {}\n\n".format(drug,channel)

    num_expts, experiment_numbers, experiments = dr.load_crumb_data(drug,channel)
    if (0 < args.num_expts < num_expts):
        num_expts = args.num_expts

    drug, channel, output_dir, chain_dir, figs_dir, chain_file = dr.hierarchical_output_dirs_and_chain_file(drug,channel,num_expts)

    chain = np.loadtxt(chain_file)
    end = chain.shape[0]
    burn = end/4

    pic50_samples = np.zeros(args.num_hist_samples)
    hill_samples = np.zeros(args.num_hist_samples)
    rand_idx = npr.randint(burn, end, args.num_hist_samples)
    for t in xrange(args.num_hist_samples):
        alpha, beta, mu, s = chain[rand_idx[t], :4]
        hill_samples[t] = st.fisk.rvs(c=beta, scale=alpha, loc=0)
        pic50_samples[t] = st.logistic.rvs(mu, s)

    fig = plt.figure(figsize=(11,7))

    ax1 = fig.add_subplot(231)
    ax1.grid()
    xmin = -4
    xmax = 3
    concs = np.logspace(xmin,xmax,101)
    ax1.set_xscale('log')
    ax1.set_ylim(0,100)
    ax1.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax1.set_ylabel(r'% {} block'.format(channel))
    ax1.set_title('A. Hierarchical predicted\nfuture experiments')
    ax1.set_xlim(10**xmin,10**xmax)

    for expt in experiment_numbers:
        ax1.scatter(experiments[expt][:,0],experiments[expt][:,1],label='Expt {}'.format(expt+1),color=colors[expt],s=100,zorder=10)

    for i, conc in enumerate(args.concs):
        ax1.axvline(conc,color=colors[3+i],lw=2,label=r"{} $\mu$M".format(conc),alpha=0.8)

    subset_idx = npr.randint(0, args.num_hist_samples, args.num_samples)
    for i in xrange(args.num_samples):  # only plot the first T of the H samples (should be fine because they're all randomly selected)
        ax1.plot(concs,dr.dose_response_model(concs, hill_samples[i], dr.pic50_to_ic50(pic50_samples[i])),color='black', alpha=0.01)
    ax1.legend(loc=2,fontsize=10)

    ax2 = fig.add_subplot(234)
    ax2.set_xlim(0,100)
    ax2.set_xlabel(r'% {} block'.format(channel))
    ax2.set_ylabel(r'Probability density')
    ax2.grid()
    for i, conc in enumerate(args.concs):
        ax2.hist(dr.dose_response_model(conc,hill_samples,dr.pic50_to_ic50(pic50_samples)),bins=50,normed=True,color=colors[3+i],alpha=0.8,edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))

    ax2.set_title('D. Hierarchical predicted\nfuture experiments')
    ax2.legend(loc=2,fontsize=10)

    ax3 = fig.add_subplot(232,sharey=ax1)
    ax3.grid()
    xmin = -4
    xmax = 3
    concs = np.logspace(xmin,xmax,101)
    ax3.set_xscale('log')
    ax3.set_ylim(0,100)
    ax3.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax3.set_title('B. Hierarchical inferred\nunderlying effects')
    ax3.set_xlim(10**xmin,10**xmax)

    for expt in experiment_numbers:
        ax3.scatter(experiments[expt][:,0],experiments[expt][:,1],label='Expt {}'.format(expt+1),color=colors[expt],s=100,zorder=10)

    num_samples = 1000
    alpha_indices = npr.randint(burn,end,num_samples)
    alpha_samples = chain[alpha_indices,0]
    mu_samples = chain[alpha_indices,2]
    for i, conc in enumerate(args.concs):
        ax3.axvline(conc,color=colors[3+i],lw=2,label=r"{} $\mu$M".format(conc),alpha=0.8)
    for i in xrange(num_samples):
        ax3.plot(concs,dr.dose_response_model(concs,alpha_samples[i],dr.pic50_to_ic50(mu_samples[i])),color='black',alpha=0.01)
    ax3.legend(loc=2,fontsize=10)
    ax4 = fig.add_subplot(235,sharey=ax2)
    ax4.set_xlim(0,100)
    ax4.set_xlabel(r'% {} block'.format(channel))
    ax4.grid()

    args.num_hist_samples = 100000
    hist_indices = npr.randint(burn,end,args.num_hist_samples)
    alphas = chain[hist_indices,0]
    mus = chain[hist_indices,2]

    for i, conc in enumerate(args.concs):
        ax4.hist(dr.dose_response_model(conc,alphas,dr.pic50_to_ic50(mus)),bins=50,normed=True,color=colors[3+i],alpha=0.8,edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))
    ax4.set_title('E. Hierarchical inferred\nunderlying effects')

    plt.setp(ax3.get_yticklabels(), visible=False)
    plt.setp(ax4.get_yticklabels(), visible=False)

    # now plot non-hierarchical

    num_params = 3
    drug,channel,chain_file,figs_dir = dr.nonhierarchical_chain_file_and_figs_dir(drug, channel, args.fix_hill)
    chain = np.loadtxt(chain_file,usecols=range(num_params-1)) # not interested in log-target values right now
    end = chain.shape[0]
    burn = end/4

    num_samples = 1000
    sample_indices = npr.randint(burn,end,num_samples)
    samples = chain[sample_indices,:]

    ax5 = fig.add_subplot(233,sharey=ax1)
    ax5.grid()
    plt.setp(ax5.get_yticklabels(), visible=False)
    xmin = -4
    xmax = 4
    concs = np.logspace(xmin,xmax,101)
    ax5.set_xscale('log')
    ax5.set_ylim(0,100)
    ax5.set_xlim(10**xmin,10**xmax)
    ax5.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax5.set_title('C. Single-level inferred\neffects')
    ax5.legend(fontsize=10)

    for expt in experiment_numbers:
        if expt==1:
            ax5.scatter(experiments[expt][:,0],experiments[expt][:,1],color='orange',s=100,label='All expts',zorder=10)
        else:
            ax5.scatter(experiments[expt][:,0],experiments[expt][:,1],color='orange',s=100,zorder=10)

    for i, conc in enumerate(args.concs):
        ax5.axvline(conc,color=colors[3+i],alpha=0.8,lw=2,label=r"{} $\mu$M".format(conc))
    for i in xrange(num_samples):
        ax5.plot(concs,dr.dose_response_model(concs,samples[i,0],dr.pic50_to_ic50(samples[i,1])),color='black',alpha=0.01)
    ax5.legend(loc=2,fontsize=10)

    args.num_hist_samples = 50000
    sample_indices = npr.randint(burn,end,args.num_hist_samples)
    samples = chain[sample_indices,:]
    ax6 = fig.add_subplot(236,sharey=ax2)
    ax6.set_xlim(0,100)
    ax6.set_xlabel(r'% {} block'.format(channel))
    plt.setp(ax6.get_yticklabels(), visible=False)
    ax6.grid()
    for i, conc in enumerate(args.concs):
        ax6.hist(dr.dose_response_model(conc,samples[:,0],dr.pic50_to_ic50(samples[:,1])),bins=50,normed=True,alpha=0.8,color=colors[3+i],edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))
    ax6.set_title('F. Single-level inferred\neffects')

    ax2.legend(loc=2,fontsize=10)

    plot_dir = dr.all_predictions_dir(drug,channel)

    fig.tight_layout()
    png_file = plot_dir+'{}_{}_all_predictions_corrected.png'.format(drug,channel)
    print png_file
    fig.savefig(png_file)
    pdf_file = plot_dir+'{}_{}_all_predictions_corrected.pdf'.format(drug,channel)
    print pdf_file
    fig.savefig(pdf_file) # uncomment to save as pdf, or change extension to whatever you want

    plt.close()

    print "Figures saved in", plot_dir

drugs_channels = it.product(drugs_to_run,channels_to_run)
if (args.num_cores<=1) or (len(drugs_to_run)==1):
    for drug_channel in drugs_channels:
        #run(drug_channel)

        # try/except is good when running multiple plots and leaving them overnight,say
        # if one or more crash then the others will survive!
        # however, if you need more "control", comment out the try/except, and uncomment the other run(drug_channel) line
        try:
            run(drug_channel)
        except Exception,e:
            print e
            print "Failed to run {} + {}!".format(drug_channel[0],drug_channel[1])
# run multiple plots in parallel
elif (args.num_cores>1):
    import multiprocessing as mp
    num_cores = min(args.num_cores, mp.cpu_count()-1)
    pool = mp.Pool(processes=num_cores)
    pool.map_async(run,drugs_channels).get(9999999)
    pool.close()
    pool.join()
<commit_msg>Plot full figure<commit_after># this requires both the hierarchical and nonhierarchical MCMCs to have been run
# also requires the posterior predictive CDFs to have been computed in the hierarchical case

import doseresponse as dr
import matplotlib
#matplotlib.rc('font', family='ubuntu')
#matplotlib.use('Agg')
import matplotlib.pyplot as plt
import argparse
import itertools as it
import numpy as np
import sys
import numpy.random as npr
import scipy.stats as st

seed = 1
npr.seed(seed)

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--concs", nargs="+", type=float, help="drug concentrations at which to find predicted responses",default=[])
parser.add_argument("-a", "--all", action='store_true', help='plot histograms from hierarchical MCMC on all drugs and channels', default=False)
parser.add_argument("-nc", "--num-cores", type=int, help="number of cores to parallelise drug/channel combinations", default=1)
parser.add_argument("-Ne", "--num-expts", type=int, help="how many experiments to fit to, otherwise will fit to all in the data file", default=0)
parser.add_argument("--data-file", type=str, help="csv file from which to read in data, in same format as provided crumb_data.csv")
parser.add_argument("--fix-hill", action='store_true', help="fix Hill=1 through fitting and MCMC",default=False)
parser.add_argument("-T", "--num-samples", type=int, help="number of samples to plot", required=True)
parser.add_argument("-H", "--num-hist-samples", type=int, help="number of samples for histogram --- should be much higher, really", required=True)
args = parser.parse_args()

dr.setup(args.data_file)

drugs_to_run, channels_to_run = dr.list_drug_channel_options(args.all)

colors = ['#a6cee3','#1f78b4','#b2df8a','#33a02c','#fb9a99']

def run(drug_channel):
    drug,channel = drug_channel
    print "\n\n{} + {}\n\n".format(drug,channel)

    num_expts, experiment_numbers, experiments = dr.load_crumb_data(drug,channel)
    if (0 < args.num_expts < num_expts):
        num_expts = args.num_expts

    drug, channel, output_dir, chain_dir, figs_dir, chain_file = dr.hierarchical_output_dirs_and_chain_file(drug,channel,num_expts)

    chain = np.loadtxt(chain_file)
    end = chain.shape[0]
    burn = end/4

    pic50_samples = np.zeros(args.num_hist_samples)
    hill_samples = np.zeros(args.num_hist_samples)
    rand_idx = npr.randint(burn, end, args.num_hist_samples)
    for t in xrange(args.num_hist_samples):
        alpha, beta, mu, s = chain[rand_idx[t], :4]
        hill_samples[t] = st.fisk.rvs(c=beta, scale=alpha, loc=0)
        pic50_samples[t] = st.logistic.rvs(mu, s)

    fig = plt.figure(figsize=(11,7))

    ax1 = fig.add_subplot(231)
    ax1.grid()
    xmin = -4
    xmax = 3
    concs = np.logspace(xmin,xmax,101)
    ax1.set_xscale('log')
    ax1.set_ylim(0,100)
    ax1.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax1.set_ylabel(r'% {} block'.format(channel))
    ax1.set_title('A. Hierarchical predicted\nfuture experiments')
    ax1.set_xlim(10**xmin,10**xmax)

    for expt in experiment_numbers:
        ax1.scatter(experiments[expt][:,0],experiments[expt][:,1],label='Expt {}'.format(expt+1),color=colors[expt],s=100,zorder=10)

    for i, conc in enumerate(args.concs):
        ax1.axvline(conc,color=colors[3+i],lw=2,label=r"{} $\mu$M".format(conc),alpha=0.8)

    subset_idx = npr.randint(0, args.num_hist_samples, args.num_samples)
    for i in xrange(args.num_samples):  # only plot the first T of the H samples (should be fine because they're all randomly selected)
        ax1.plot(concs,dr.dose_response_model(concs, hill_samples[i], dr.pic50_to_ic50(pic50_samples[i])),color='black', alpha=0.01)
    ax1.legend(loc=2,fontsize=10)

    ax2 = fig.add_subplot(234)
    ax2.set_xlim(0,100)
    ax2.set_xlabel(r'% {} block'.format(channel))
    ax2.set_ylabel(r'Probability density')
    ax2.grid()
    for i, conc in enumerate(args.concs):
        ax2.hist(dr.dose_response_model(conc,hill_samples,dr.pic50_to_ic50(pic50_samples)),bins=50,normed=True,color=colors[3+i],alpha=0.8,edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))

    ax2.set_title('D. Hierarchical predicted\nfuture experiments')
    ax2.legend(loc=2,fontsize=10)

    ax3 = fig.add_subplot(232,sharey=ax1)
    ax3.grid()
    xmin = -4
    xmax = 3
    concs = np.logspace(xmin,xmax,101)
    ax3.set_xscale('log')
    ax3.set_ylim(0,100)
    ax3.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax3.set_title('B. Hierarchical inferred\nunderlying effects')
    ax3.set_xlim(10**xmin,10**xmax)

    for expt in experiment_numbers:
        ax3.scatter(experiments[expt][:,0],experiments[expt][:,1],label='Expt {}'.format(expt+1),color=colors[expt],s=100,zorder=10)

    num_samples = 1000
    alpha_indices = npr.randint(burn,end,num_samples)
    alpha_samples = chain[alpha_indices,0]
    mu_samples = chain[alpha_indices,2]
    for i, conc in enumerate(args.concs):
        ax3.axvline(conc,color=colors[3+i],lw=2,label=r"{} $\mu$M".format(conc),alpha=0.8)
    for i in xrange(num_samples):
        ax3.plot(concs,dr.dose_response_model(concs,alpha_samples[i],dr.pic50_to_ic50(mu_samples[i])),color='black',alpha=0.01)
    ax3.legend(loc=2,fontsize=10)
    ax4 = fig.add_subplot(235,sharey=ax2)
    ax4.set_xlim(0,100)
    ax4.set_xlabel(r'% {} block'.format(channel))
    ax4.grid()

    args.num_hist_samples = 100000
    hist_indices = npr.randint(burn,end,args.num_hist_samples)
    alphas = chain[hist_indices,0]
    mus = chain[hist_indices,2]

    for i, conc in enumerate(args.concs):
        ax4.hist(dr.dose_response_model(conc,alphas,dr.pic50_to_ic50(mus)),bins=50,normed=True,color=colors[3+i],alpha=0.8,edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))
    ax4.set_title('E. Hierarchical inferred\nunderlying effects')

    plt.setp(ax3.get_yticklabels(), visible=False)
    plt.setp(ax4.get_yticklabels(), visible=False)

    # now plot non-hierarchical

    num_params = 3
    temperature = 1
    if args.fix_hill:
        model = 1
    else:
        model = 2
    drug,channel,chain_file,figs_dir = dr.nonhierarchical_chain_file_and_figs_dir(model, drug, channel, temperature)
    chain = np.loadtxt(chain_file,usecols=range(num_params-1)) # not interested in log-target values right now
    end = chain.shape[0]
    burn = end/4

    num_samples = 1000
    sample_indices = npr.randint(burn,end,num_samples)
    samples = chain[sample_indices,:]

    ax5 = fig.add_subplot(233,sharey=ax1)
    ax5.grid()
    plt.setp(ax5.get_yticklabels(), visible=False)
    xmin = -4
    xmax = 4
    concs = np.logspace(xmin,xmax,101)
    ax5.set_xscale('log')
    ax5.set_ylim(0,100)
    ax5.set_xlim(10**xmin,10**xmax)
    ax5.set_xlabel(r'{} concentration ($\mu$M)'.format(drug))
    ax5.set_title('C. Single-level inferred\neffects')
    ax5.legend(fontsize=10)

    for expt in experiment_numbers:
        if expt==1:
            ax5.scatter(experiments[expt][:,0],experiments[expt][:,1],color='orange',s=100,label='All expts',zorder=10)
        else:
            ax5.scatter(experiments[expt][:,0],experiments[expt][:,1],color='orange',s=100,zorder=10)

    for i, conc in enumerate(args.concs):
        ax5.axvline(conc,color=colors[3+i],alpha=0.8,lw=2,label=r"{} $\mu$M".format(conc))
    for i in xrange(num_samples):
        ax5.plot(concs,dr.dose_response_model(concs,samples[i,0],dr.pic50_to_ic50(samples[i,1])),color='black',alpha=0.01)
    ax5.legend(loc=2,fontsize=10)

    args.num_hist_samples = 50000
    sample_indices = npr.randint(burn,end,args.num_hist_samples)
    samples = chain[sample_indices,:]
    ax6 = fig.add_subplot(236,sharey=ax2)
    ax6.set_xlim(0,100)
    ax6.set_xlabel(r'% {} block'.format(channel))
    plt.setp(ax6.get_yticklabels(), visible=False)
    ax6.grid()
    for i, conc in enumerate(args.concs):
        ax6.hist(dr.dose_response_model(conc,samples[:,0],dr.pic50_to_ic50(samples[:,1])),bins=50,normed=True,alpha=0.8,color=colors[3+i],edgecolor='none',label=r"{} $\mu$M {}".format(conc,drug))
    ax6.set_title('F. Single-level inferred\neffects')

    ax2.legend(loc=2,fontsize=10)

    plot_dir = dr.all_predictions_dir(drug,channel)

    fig.tight_layout()
    png_file = plot_dir+'{}_{}_all_predictions_corrected.png'.format(drug,channel)
    print png_file
    fig.savefig(png_file)
    pdf_file = plot_dir+'{}_{}_all_predictions_corrected.pdf'.format(drug,channel)
    print pdf_file
    fig.savefig(pdf_file) # uncomment to save as pdf, or change extension to whatever you want

    plt.close()

    print "Figures saved in", plot_dir

drugs_channels = it.product(drugs_to_run,channels_to_run)
if (args.num_cores<=1) or (len(drugs_to_run)==1):
    for drug_channel in drugs_channels:
        #run(drug_channel)

        # try/except is good when running multiple plots and leaving them overnight,say
        # if one or more crash then the others will survive!
        # however, if you need more "control", comment out the try/except, and uncomment the other run(drug_channel) line
        try:
            run(drug_channel)
        except Exception,e:
            print e
            print "Failed to run {} + {}!".format(drug_channel[0],drug_channel[1])
# run multiple plots in parallel
elif (args.num_cores>1):
    import multiprocessing as mp
    num_cores = min(args.num_cores, mp.cpu_count()-1)
    pool = mp.Pool(processes=num_cores)
    pool.map_async(run,drugs_channels).get(9999999)
    pool.close()
    pool.join()
<|endoftext|>


<commit_before># Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import nnabla.utils.converter
import os

def dump_command(args):
    resolve_file_format(args, args.files)

    if args.import_format not in nnabla.utils.converter.formats.import_name:
        print('Import format ({}) is not supported.'.format(args.import_format))
        return False

    nnabla.utils.converter.dump_files(args, args.files)
    return True

def nnb_template_command(args):
    if len(args.files) >= 2:
        output = args.files.pop()
        resolve_file_format(args, args.files)

        if args.import_format not in nnabla.utils.converter.formats.import_name:
            print('Import format ({}) is not supported.'.format(args.import_format))
            return False

        nnabla.utils.converter.nnb_template(args, args.files, output)
        return True

    print('Input and Output arg is mandatory.')
    return False

def resolve_file_format(args, import_files, export_file=None):
    if len(import_files) == 1:
        input_ext = os.path.splitext(import_files[0])[1]
        if input_ext == '.nnp':
            args.import_format = 'NNP'
        elif input_ext == '.onnx':
            args.import_format = 'ONNX'
        elif input_ext == '.pb':
            args.import_format = "TF_PB"
        elif input_ext == '.ckpt':
            args.import_format = "TF_CKPT_V1"
        elif input_ext == '.meta':
            args.import_format = "TF_CKPT_V2"

    if export_file and not os.path.isdir(export_file):
        output_ext = os.path.splitext(export_file)[1]
        if output_ext == '.nnp':
            args.export_format == 'NNP'
        elif output_ext == '.nnb':
            args.export_format == 'NNB'
        elif output_ext == '.onnx':
            args.export_format = 'ONNX'
        elif output_ext == '.pb':
            args.export_format = 'TF_PB'
        elif output_ext == '.tflite':
            args.export_format = 'TFLITE'

    if args.import_format in ['ONNX', 'TF_CKPT_V1', 'TF_CKPT_V2', 'TF_PB'] or \
            args.export_format in ['ONNX', 'TF_PB', 'TFLITE']:
        try:
            import nnabla.utils.converter.onnx
            import nnabla.utils.converter.tensorflow
        except ImportError:
            raise ImportError(
                'nnabla_converter python package is not found, install nnabla_converter package with "pip install nnabla_converter"')

def convert_command(args):
    if len(args.files) >= 2:
        output = args.files.pop()
        resolve_file_format(args, args.files, output)

        if args.import_format not in nnabla.utils.converter.formats.import_name:
            print('Import format ({}) is not supported.'.format(args.import_format))
            return False

        if args.export_format not in nnabla.utils.converter.formats.export_name:
            print('Export format ({}) is not supported.'.format(args.export_format))
            return False

        nnabla.utils.converter.convert_files(args, args.files, output)
        return True

    print('Input and Output arg is mandatory.')
    return False

def add_convert_command(subparsers):

    def add_import_arg(parser):
        parser.add_argument('-I', '--import-format', type=str, default='NNP',
                            help='[import] import format. (one of [{}])'.format(import_formats_string))
        parser.add_argument('-E', '--nnp-import-executor-index', type=int, default=None,
                            help='[import][NNP] import only specified executor.')
        parser.add_argument('--nnp-exclude-preprocess', action='store_true',
                            help='[import][NNP] EXPERIMENTAL exclude preprocess functions when import.')
        parser.add_argument('--nnp-no-expand-network', action='store_true',
                            help='[import][NNP] expand network with repeat or recurrent.')

    import_formats_string = ','.join(
        nnabla.utils.converter.formats.import_name)

    ################################################################################
    # Dump Network.
    subparser = subparsers.add_parser(
        'dump', help='Dump network with supported format.')
    subparser.add_argument('-v', '--dump-verbose', action='store_true',
                           help='[dump] verbose output.')
    subparser.add_argument('-F', '--dump-functions', action='store_true',
                           help='[dump] dump function list.')
    subparser.add_argument('-V', '--dump-variables', action='store_true',
                           help='[dump] dump variable list.')
    subparser.add_argument('--dump-limit', type=int, default=-1,
                           help='[dump] limit num of items.')
    subparser.add_argument('-n', '--dump-variable-name', type=str, default=None,
                           help='[dump] Specific variable name to display.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File or directory name(s) to convert.')
    # import option
    add_import_arg(subparser)
    subparser.set_defaults(func=dump_command)

    ################################################################################
    # Generate NNB template
    subparser = subparsers.add_parser(
        'nnb_template', help='Generate NNB config file template.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File to generate NNB config file template. The last is setting yaml file.')
    # import option
    add_import_arg(subparser)

    subparser.add_argument('-b', '--batch-size', type=int, default=-1,
                           help='[export] overwrite batch size.')
    subparser.add_argument('-T', '--default-variable-type', type=str, nargs=1, default=['FLOAT32'],
                           help='Default type of variable')
    subparser.add_argument('--api', type=int, default=-1,
                           help='Set API Level to convert to, default is highest API Level.')

    subparser.set_defaults(func=nnb_template_command)

    ################################################################################
    # Converter
    subparser = subparsers.add_parser('convert', help='File format converter.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File or directory name(s) to convert. \
                           (When convert ckpt format of the tensorflow model, \
                           If the version of the checkpoint is V1, need to enter the `.ckpt` file, \
                           otherwise need to enter the `.meta` file.)')
    # import option
    subparser.add_argument('--outputs', type=str, default=None,
                           help='[import][tensorflow] The name(s) of the output nodes, comma separated. \
                           Only needed when convert CKPT format.')
    subparser.add_argument('--inputs', type=str, default=None,
                           help='[import][tensorflow] The name(s) of the input nodes, comma separated. \
                           Only needed when convert CKPT format.')
    add_import_arg(subparser)

    # export option
    export_formats_string = ','.join(
        nnabla.utils.converter.formats.export_name)
    subparser.add_argument('-O', '--export-format', type=str, default='NNP',
                           help='[export] export format. (one of [{}])'.format(export_formats_string))
    subparser.add_argument('-f', '--force', action='store_true',
                           help='[export] overwrite output file.')
    subparser.add_argument('-b', '--batch-size', type=int, default=-1,
                           help='[export] overwrite batch size.')
    subparser.add_argument('-S', '--split', type=str, default=None,
                           help='[export] This option need to set  "-E" option.' +
                           'Split executor with specified index. ' +
                           '(eg. "1-9", "1-2,5-")')
    subparser.add_argument('-d', '--define_version', type=str, default=None,
                           help='[export][ONNX] define onnx opset version. e.g. opset_6' + '\n' +
                           '[export][ONNX] define convert to onnx for SNPE. e.g. opset_snpe' + '\n' +
                           '[export][ONNX] define convert to onnx for TensorRT. e.g. opset_tensorrt' + '\n' +
                           '[export][NNB] define binary format version. e.g. nnb_3')
    subparser.add_argument('--enable-optimize-pb', action='store_true',
                           help='[export][tensorflow] enable optimization when export to pb or tflite.')

    # For config function list
    subparser.add_argument('-c', '--config', type=str, default=None,
                           help='[export] config target function list.')

    # NNP
    subparser.add_argument('--nnp-parameter-h5', action='store_true',
                           help='[export][NNP] store parameter with h5 format')
    subparser.add_argument('--nnp-parameter-nntxt', action='store_true',
                           help='[export][NNP] store parameter into nntxt')
    subparser.add_argument('--nnp-exclude-parameter', action='store_true',
                           help='[export][NNP] output without parameter')

    # Both NNB and CSRC
    subparser.add_argument('-T', '--default-variable-type', type=str, nargs=1, default=['FLOAT32'],
                           help='Default type of variable')
    subparser.add_argument('-s', '--settings', type=str, nargs=1, default=None,
                           help='Settings in YAML format file.')
    subparser.add_argument('--api', type=int, default=-1,
                           help='Set API Level to convert to, default is highest API Level.')

    subparser.set_defaults(func=convert_command)
<commit_msg>Fix code typo<commit_after># Copyright (c) 2017 Sony Corporation. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import nnabla.utils.converter
import os

def dump_command(args):
    resolve_file_format(args, args.files)

    if args.import_format not in nnabla.utils.converter.formats.import_name:
        print('Import format ({}) is not supported.'.format(args.import_format))
        return False

    nnabla.utils.converter.dump_files(args, args.files)
    return True

def nnb_template_command(args):
    if len(args.files) >= 2:
        output = args.files.pop()
        resolve_file_format(args, args.files)

        if args.import_format not in nnabla.utils.converter.formats.import_name:
            print('Import format ({}) is not supported.'.format(args.import_format))
            return False

        nnabla.utils.converter.nnb_template(args, args.files, output)
        return True

    print('Input and Output arg is mandatory.')
    return False

def resolve_file_format(args, import_files, export_file=None):
    if len(import_files) == 1:
        input_ext = os.path.splitext(import_files[0])[1]
        if input_ext == '.nnp':
            args.import_format = 'NNP'
        elif input_ext == '.onnx':
            args.import_format = 'ONNX'
        elif input_ext == '.pb':
            args.import_format = "TF_PB"
        elif input_ext == '.ckpt':
            args.import_format = "TF_CKPT_V1"
        elif input_ext == '.meta':
            args.import_format = "TF_CKPT_V2"

    if export_file and not os.path.isdir(export_file):
        output_ext = os.path.splitext(export_file)[1]
        if output_ext == '.nnp':
            args.export_format = 'NNP'
        elif output_ext == '.nnb':
            args.export_format = 'NNB'
        elif output_ext == '.onnx':
            args.export_format = 'ONNX'
        elif output_ext == '.pb':
            args.export_format = 'TF_PB'
        elif output_ext == '.tflite':
            args.export_format = 'TFLITE'

    if args.import_format in ['ONNX', 'TF_CKPT_V1', 'TF_CKPT_V2', 'TF_PB'] or \
            args.export_format in ['ONNX', 'TF_PB', 'TFLITE']:
        try:
            import nnabla.utils.converter.onnx
            import nnabla.utils.converter.tensorflow
        except ImportError:
            raise ImportError(
                'nnabla_converter python package is not found, install nnabla_converter package with "pip install nnabla_converter"')

def convert_command(args):
    if len(args.files) >= 2:
        output = args.files.pop()
        resolve_file_format(args, args.files, output)

        if args.import_format not in nnabla.utils.converter.formats.import_name:
            print('Import format ({}) is not supported.'.format(args.import_format))
            return False

        if args.export_format not in nnabla.utils.converter.formats.export_name:
            print('Export format ({}) is not supported.'.format(args.export_format))
            return False

        nnabla.utils.converter.convert_files(args, args.files, output)
        return True

    print('Input and Output arg is mandatory.')
    return False

def add_convert_command(subparsers):

    def add_import_arg(parser):
        parser.add_argument('-I', '--import-format', type=str, default='NNP',
                            help='[import] import format. (one of [{}])'.format(import_formats_string))
        parser.add_argument('-E', '--nnp-import-executor-index', type=int, default=None,
                            help='[import][NNP] import only specified executor.')
        parser.add_argument('--nnp-exclude-preprocess', action='store_true',
                            help='[import][NNP] EXPERIMENTAL exclude preprocess functions when import.')
        parser.add_argument('--nnp-no-expand-network', action='store_true',
                            help='[import][NNP] expand network with repeat or recurrent.')

    import_formats_string = ','.join(
        nnabla.utils.converter.formats.import_name)

    ################################################################################
    # Dump Network.
    subparser = subparsers.add_parser(
        'dump', help='Dump network with supported format.')
    subparser.add_argument('-v', '--dump-verbose', action='store_true',
                           help='[dump] verbose output.')
    subparser.add_argument('-F', '--dump-functions', action='store_true',
                           help='[dump] dump function list.')
    subparser.add_argument('-V', '--dump-variables', action='store_true',
                           help='[dump] dump variable list.')
    subparser.add_argument('--dump-limit', type=int, default=-1,
                           help='[dump] limit num of items.')
    subparser.add_argument('-n', '--dump-variable-name', type=str, default=None,
                           help='[dump] Specific variable name to display.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File or directory name(s) to convert.')
    # import option
    add_import_arg(subparser)
    subparser.set_defaults(func=dump_command)

    ################################################################################
    # Generate NNB template
    subparser = subparsers.add_parser(
        'nnb_template', help='Generate NNB config file template.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File to generate NNB config file template. The last is setting yaml file.')
    # import option
    add_import_arg(subparser)

    subparser.add_argument('-b', '--batch-size', type=int, default=-1,
                           help='[export] overwrite batch size.')
    subparser.add_argument('-T', '--default-variable-type', type=str, nargs=1, default=['FLOAT32'],
                           help='Default type of variable')
    subparser.add_argument('--api', type=int, default=-1,
                           help='Set API Level to convert to, default is highest API Level.')

    subparser.set_defaults(func=nnb_template_command)

    ################################################################################
    # Converter
    subparser = subparsers.add_parser('convert', help='File format converter.')
    subparser.add_argument('files', metavar='FILE', type=str, nargs='+',
                           help='File or directory name(s) to convert. \
                           (When convert ckpt format of the tensorflow model, \
                           If the version of the checkpoint is V1, need to enter the `.ckpt` file, \
                           otherwise need to enter the `.meta` file.)')
    # import option
    subparser.add_argument('--outputs', type=str, default=None,
                           help='[import][tensorflow] The name(s) of the output nodes, comma separated. \
                           Only needed when convert CKPT format.')
    subparser.add_argument('--inputs', type=str, default=None,
                           help='[import][tensorflow] The name(s) of the input nodes, comma separated. \
                           Only needed when convert CKPT format.')
    add_import_arg(subparser)

    # export option
    export_formats_string = ','.join(
        nnabla.utils.converter.formats.export_name)
    subparser.add_argument('-O', '--export-format', type=str, default='NNP',
                           help='[export] export format. (one of [{}])'.format(export_formats_string))
    subparser.add_argument('-f', '--force', action='store_true',
                           help='[export] overwrite output file.')
    subparser.add_argument('-b', '--batch-size', type=int, default=-1,
                           help='[export] overwrite batch size.')
    subparser.add_argument('-S', '--split', type=str, default=None,
                           help='[export] This option need to set  "-E" option.' +
                           'Split executor with specified index. ' +
                           '(eg. "1-9", "1-2,5-")')
    subparser.add_argument('-d', '--define_version', type=str, default=None,
                           help='[export][ONNX] define onnx opset version. e.g. opset_6' + '\n' +
                           '[export][ONNX] define convert to onnx for SNPE. e.g. opset_snpe' + '\n' +
                           '[export][ONNX] define convert to onnx for TensorRT. e.g. opset_tensorrt' + '\n' +
                           '[export][NNB] define binary format version. e.g. nnb_3')
    subparser.add_argument('--enable-optimize-pb', action='store_true',
                           help='[export][tensorflow] enable optimization when export to pb or tflite.')

    # For config function list
    subparser.add_argument('-c', '--config', type=str, default=None,
                           help='[export] config target function list.')

    # NNP
    subparser.add_argument('--nnp-parameter-h5', action='store_true',
                           help='[export][NNP] store parameter with h5 format')
    subparser.add_argument('--nnp-parameter-nntxt', action='store_true',
                           help='[export][NNP] store parameter into nntxt')
    subparser.add_argument('--nnp-exclude-parameter', action='store_true',
                           help='[export][NNP] output without parameter')

    # Both NNB and CSRC
    subparser.add_argument('-T', '--default-variable-type', type=str, nargs=1, default=['FLOAT32'],
                           help='Default type of variable')
    subparser.add_argument('-s', '--settings', type=str, nargs=1, default=None,
                           help='Settings in YAML format file.')
    subparser.add_argument('--api', type=int, default=-1,
                           help='Set API Level to convert to, default is highest API Level.')

    subparser.set_defaults(func=convert_command)
<|endoftext|>