<commit_before>#  Copyright 2007-2014 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

__author__ = 'Rajiv Mayani'

import logging

import hashlib

from flask import g, request, make_response

from Pegasus.service import cache
from Pegasus.service.monitoring import monitoring_routes
from Pegasus.service.monitoring.queries import MasterWorkflowQueries
from Pegasus.service.monitoring.serializer import RootWorkflowSerializer

log = logging.getLogger(__name__)

JSON_HEADER = {'Content-Type': 'application/json'}

@monitoring_routes.url_value_preprocessor
def pull_m_wf_id(endpoint, values):
    """
    If the requested endpoint contains a value for m_wf_id variable then extract it and set it in g.m_wf_id.
    """
    if values and 'm_wf_id' in values:
        g.m_wf_id = values['m_wf_id']

@monitoring_routes.before_request
def compute_stampede_db_url():
    """
    If the requested endpoint requires connecting to a STAMPEDE database, then determine STAMPEDE DB URL and store it
    in g.stampede_db_url
    """
    if '/workflow' not in request.path or 'm_wf_id' not in g:
        return

    md5sum = hashlib.md5()
    md5sum.update(g.master_db_url)
    m_wf_id = g.m_wf_id

    del g.m_wf_id

    def _get_cache_key(key_suffix):
        return '%s.%s' % (md5sum.hexdigest(), key_suffix)

    cache_key = _get_cache_key(m_wf_id)

    if cache.get(cache_key):
        log.debug('Cache Hit: compute_stampede_db_url %s' % cache_key)
        g.stampede_db_url = cache.get(cache_key).db_url

    else:
        log.debug('Cache Miss: compute_stampede_db_url %s' % cache_key)
        queries = MasterWorkflowQueries(g.master_db_url)
        root_workflow = queries.get_root_workflow(m_wf_id)
        queries.close()

        g.stampede_db_url = root_workflow.db_url

        cache.set(_get_cache_key(root_workflow.wf_id), root_workflow, timeout=600)
        cache.set(_get_cache_key(root_workflow.wf_uuid), root_workflow, timeout=600)

@monitoring_routes.before_request
def get_query_args():
    g.query_args = {}

    def to_int(q_arg, value):
        try:
            return int(value)
        except ValueError:
            log.exception('Query Argument %s = %s is not a valid int' % (q_arg, value))
            abort(400)

    def to_str(q_arg, value):
        return value

    def to_bool(q_arg, value):
        return True if value.lower() == 'true' else False

    query_args = {
        'start-index': to_int,
        'max-results': to_int,
        'query': to_str,
        'order': to_str,
        'recent': to_bool,
        'pretty-print': to_bool
    }

    for arg, cast in query_args.iteritems():
        if arg in request.args:
            g.query_args[arg.replace('-', '_')] = cast(arg, request.args.get(arg))

"""
Root Workflow

{
    "wf_id"             : int:wf_id,
    "wf_uuid"           : string:wf_uuid,
    "submit_hostname"   : string:submit_hostname,
    "submit_dir"        : string:submit_dir,
    "planner_arguments" : string:planner_arguments,
    "planner_version"   : string:planner_version,
    "user"              : string:user,
    "grid_dn"           : string:grid_dn,
    "dax_label"         : string:dax_label,
    "dax_version"       : string:dax_version,
    "dax_file"          : string:dax_file,
    "dag_file_name"     : string:dag_file_name,
    "timestamp"         : int:timestamp,
    "workflow_state"    : object:workflow_state,
    "_links"            : {
        "workflow" : href:workflow
    }
}
"""

@monitoring_routes.route('/user/<string:username>/root')
@monitoring_routes.route('/user/<string:username>/root/query', methods=['POST'])
def get_root_workflows(username):
    """
    Returns a collection of root level workflows.

    :query int start-index: Return results starting from record <start-index> (0 indexed)
    :query int max-results: Return a maximum of <max-results> records
    :query string query: Search criteria
    :query string order: Sorting criteria
    :query boolean pretty-print: Return formatted JSON response.

    :statuscode 200: OK
    :statuscode 204: No content; when no workflows found.
    :statuscode 400: Bad request
    :statuscode 401: Authentication failure
    :statuscode 403: Authorization failure

    :return type: Collection
    :return resource: Root Workflow
    """
    queries = MasterWorkflowQueries(g.master_db_url)
    records, total_records, total_filtered = queries.get_root_workflows(**g.query_args)

    if total_records == 0:
        log.debug('Total records is 0; returning HTTP 204 No content')
        return make_response('', 204, JSON_HEADER)

    #
    # Generate JSON Response
    #
    serializer = RootWorkflowSerializer(**g.query_args)
    response_json = serializer.encode_collection(records, total_records, total_filtered)

    return make_response(response_json, 200, JSON_HEADER)

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>')
def get_root_workflow(username, m_wf_id):
    """
    Returns root level workflow identified by m_wf_id.

    :query boolean pretty-print: Return formatted JSON response.

    :statuscode 200: OK
    :statuscode 401: Authentication failure
    :statuscode 403: Authorization failure
    :statuscode 404: Not found

    :return type: Record
    :return resource: Root Workflow
    """
    pass

"""
Workflow

{
    "wf_id"             : int:wf_id,
    "root_wf_id"        : int:root_wf_id,
    "parent_wf_id"      : int:parent_wf_id,
    "wf_uuid"           : string:wf_uuid,
    "submit_hostname"   : string:submit_hostname,
    "submit_dir"        : string:submit_dir,
    "planner_arguments" : string:planner_arguments,
    "planner_version"   : string:planner_version,
    "user"              : string:user,
    "grid_dn"           : string:grid_dn,
    "dax_label"         : string:dax_label,
    "dax_version"       : string:dax_version,
    "dax_file"          : string:dax_file,
    "dag_file_name"     : string:dag_file_name,
    "timestamp"         : int:timestamp,
    "_links"            : {
        "workflow_state" : href:workflow_state,
        "job"            : href:job,
        "task"           : href:task,
        "host"           : href:host,
        "invocation"     : href:invocation
    }
}
"""

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>/workflow')
@monitoring_routes.route('/user/<string:username>/root//<string:m_wf_id>/workflow/query', methods=['POST'])
def get_workflows(username, m_wf_id):
    pass

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>/workflow/<string:wf_id>')
def get_workflow(username, m_wf_id, wf_id):
    pass

"""
{
    "state"         : string:state,
    "status"        : int:status,
    "restart_count" : int:restart_count,
    "timestamp"     : datetime:timestamp,
    "_links"          : {
        "workflow"     : "<href:workflow>"
    }
}
"""

"""
Job

{
    "job_id"      : int: job_id,
    "exec_job_id" : string: exec_job_id,
    "submit_file" : string: submit_file,
    "type_desc"   : string: type_desc,
    "max_retries" : int: max_retries,
    "clustered"   : bool: clustered,
    "task_count"  : int: task_count,
    "executable"  : string: executable,
    "argv"        : string: argv,
    "_links"      : {
        "workflow"     : href:workflow,
        "task"         : href:task,
        "job_instance" : href:job_instance
    }
}
"""

"""
Host

    {
    "host_id"      : int:host_id,
    "site_name"    : string:site_name,
    "hostname"     : string:hostname,
    "ip"           : string:ip,
    "uname"        : string:uname,
    "total_memory" : string:total_memory,
    "_links"       : {
        "workflow"     : href:workflow,
        "job_instance" : href:job_instance
    }
}
"""

"""
Job State

{
    "job_instance_id"     : "<int:job_instance_id>",
    "state"               : "<string:state>",
    "jobstate_submit_seq" : "<int:jobstate_submit_seq>",
    "timestamp"           : "<int:timestamp>",
    "_links"              : {
        "job_instance" : "href:job_instance"
    }
}
"""

"""
Task

{
    "task_id"        : int:task_id,
    "abs_task_id"    : string:abs_task_id,
    "type_desc"      : string: type_desc,
    "transformation" : string:transformation,
    "argv"           : string:argv,
    "_links"         : {
        "workflow" : href:workflow,
        "job"      : href:job
    }
}
"""

"""
Job Instance

{
    "job_instance_id"   : int:job_instance_id,
    "host_id"           : int:host_id,
    "job_submit_seq"    : int:job_submit_seq,
    "sched_id"          : string:sched_id,
    "site_name"         : string:site_name,
    "user"              : string:user,
    "work_dir"          : string:work_dir,
    "cluster_start"     : int:cluster_start,
    "cluster_duration"  : int:cluster_duration,
    "local_duration"    : int:local_duration,
    "subwf_id"          : int:subwf_id,
    "stdout_text"       : string:stdout_text,
    "stderr_text"       : string:stderr_text,
    "stdin_file"        : string:stdin_file,
    "stdout_file"       : string:stdout_file,
    "stderr_file"       : string:stderr_file,
    "multiplier_factor" : int:multiplier_factor,
    "_links"            : {
        "job_state"  : href:job_state,
        "host"       : href:host,
        "invocation" : href:invocation,
        "job"        : href:job
    }
}
"""

"""
Invocation

{
    "invocation_id"   : int:invocation_id,
    "job_instance_id" : int:job_instance_id,
    "abs_task_id"     : string:abs_task_id,
    "task_submit_seq" : int:task_submit_seq,
    "start_time"      : int:start_time,
    "remote_duration" : int:remote_duration,
    "remote_cpu_time" : int:remote_cpu_time,
    "exitcode"        : int:exitcode,
    "transformation"  : string:transformation,
    "executable"      : string:executable,
    "argv"            : string:argv,
    "_links"          : {
        "workflow"     : href:workflow,
        "job_instance" : href:job_instance
    }
}
"""

@monitoring_routes.errorhandler(404)
def page_not_found(error):
    pass

@monitoring_routes.errorhandler(BaseException)
def master_database_missing(error):
    pass
<commit_msg>Set m_wf_id to point to wf_uuid as it will be required for filtering during get_workflows<commit_after>#  Copyright 2007-2014 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

__author__ = 'Rajiv Mayani'

import logging

import hashlib

from flask import g, request, make_response

from Pegasus.service import cache
from Pegasus.service.monitoring import monitoring_routes
from Pegasus.service.monitoring.queries import MasterWorkflowQueries
from Pegasus.service.monitoring.serializer import RootWorkflowSerializer

log = logging.getLogger(__name__)

JSON_HEADER = {'Content-Type': 'application/json'}

@monitoring_routes.url_value_preprocessor
def pull_m_wf_id(endpoint, values):
    """
    If the requested endpoint contains a value for m_wf_id variable then extract it and set it in g.m_wf_id.
    """
    if values and 'm_wf_id' in values:
        g.m_wf_id = values['m_wf_id']

@monitoring_routes.before_request
def compute_stampede_db_url():
    """
    If the requested endpoint requires connecting to a STAMPEDE database, then determine STAMPEDE DB URL and store it
    in g.stampede_db_url. Also, set g.m_wf_id to be the root workflow's uuid
    """
    if '/workflow' not in request.path or 'm_wf_id' not in g:
        return

    md5sum = hashlib.md5()
    md5sum.update(g.master_db_url)
    m_wf_id = g.m_wf_id

    def _get_cache_key(key_suffix):
        return '%s.%s' % (md5sum.hexdigest(), key_suffix)

    cache_key = _get_cache_key(m_wf_id)

    if cache.get(cache_key):
        log.debug('Cache Hit: compute_stampede_db_url %s' % cache_key)
        root_workflow = cache.get(cache_key)

    else:
        log.debug('Cache Miss: compute_stampede_db_url %s' % cache_key)
        queries = MasterWorkflowQueries(g.master_db_url)
        root_workflow = queries.get_root_workflow(m_wf_id)
        queries.close()

        cache.set(_get_cache_key(root_workflow.wf_id), root_workflow, timeout=600)
        cache.set(_get_cache_key(root_workflow.wf_uuid), root_workflow, timeout=600)

    g.m_wf_id = root_workflow.wf_uuid
    g.stampede_db_url = root_workflow.db_url

@monitoring_routes.before_request
def get_query_args():
    g.query_args = {}

    def to_int(q_arg, value):
        try:
            return int(value)
        except ValueError:
            log.exception('Query Argument %s = %s is not a valid int' % (q_arg, value))
            abort(400)

    def to_str(q_arg, value):
        return value

    def to_bool(q_arg, value):
        return True if value.lower() == 'true' else False

    query_args = {
        'start-index': to_int,
        'max-results': to_int,
        'query': to_str,
        'order': to_str,
        'recent': to_bool,
        'pretty-print': to_bool
    }

    for arg, cast in query_args.iteritems():
        if arg in request.args:
            g.query_args[arg.replace('-', '_')] = cast(arg, request.args.get(arg))

"""
Root Workflow

{
    "wf_id"             : int:wf_id,
    "wf_uuid"           : string:wf_uuid,
    "submit_hostname"   : string:submit_hostname,
    "submit_dir"        : string:submit_dir,
    "planner_arguments" : string:planner_arguments,
    "planner_version"   : string:planner_version,
    "user"              : string:user,
    "grid_dn"           : string:grid_dn,
    "dax_label"         : string:dax_label,
    "dax_version"       : string:dax_version,
    "dax_file"          : string:dax_file,
    "dag_file_name"     : string:dag_file_name,
    "timestamp"         : int:timestamp,
    "workflow_state"    : object:workflow_state,
    "_links"            : {
        "workflow" : href:workflow
    }
}
"""

@monitoring_routes.route('/user/<string:username>/root')
@monitoring_routes.route('/user/<string:username>/root/query', methods=['POST'])
def get_root_workflows(username):
    """
    Returns a collection of root level workflows.

    :query int start-index: Return results starting from record <start-index> (0 indexed)
    :query int max-results: Return a maximum of <max-results> records
    :query string query: Search criteria
    :query string order: Sorting criteria
    :query boolean pretty-print: Return formatted JSON response.

    :statuscode 200: OK
    :statuscode 204: No content; when no workflows found.
    :statuscode 400: Bad request
    :statuscode 401: Authentication failure
    :statuscode 403: Authorization failure

    :return type: Collection
    :return resource: Root Workflow
    """
    queries = MasterWorkflowQueries(g.master_db_url)
    records, total_records, total_filtered = queries.get_root_workflows(**g.query_args)

    if total_records == 0:
        log.debug('Total records is 0; returning HTTP 204 No content')
        return make_response('', 204, JSON_HEADER)

    #
    # Generate JSON Response
    #
    serializer = RootWorkflowSerializer(**g.query_args)
    response_json = serializer.encode_collection(records, total_records, total_filtered)

    return make_response(response_json, 200, JSON_HEADER)

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>')
def get_root_workflow(username, m_wf_id):
    """
    Returns root level workflow identified by m_wf_id.

    :query boolean pretty-print: Return formatted JSON response.

    :statuscode 200: OK
    :statuscode 401: Authentication failure
    :statuscode 403: Authorization failure
    :statuscode 404: Not found

    :return type: Record
    :return resource: Root Workflow
    """
    pass

"""
Workflow

{
    "wf_id"             : int:wf_id,
    "root_wf_id"        : int:root_wf_id,
    "parent_wf_id"      : int:parent_wf_id,
    "wf_uuid"           : string:wf_uuid,
    "submit_hostname"   : string:submit_hostname,
    "submit_dir"        : string:submit_dir,
    "planner_arguments" : string:planner_arguments,
    "planner_version"   : string:planner_version,
    "user"              : string:user,
    "grid_dn"           : string:grid_dn,
    "dax_label"         : string:dax_label,
    "dax_version"       : string:dax_version,
    "dax_file"          : string:dax_file,
    "dag_file_name"     : string:dag_file_name,
    "timestamp"         : int:timestamp,
    "_links"            : {
        "workflow_state" : href:workflow_state,
        "job"            : href:job,
        "task"           : href:task,
        "host"           : href:host,
        "invocation"     : href:invocation
    }
}
"""

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>/workflow')
@monitoring_routes.route('/user/<string:username>/root//<string:m_wf_id>/workflow/query', methods=['POST'])
def get_workflows(username, m_wf_id):
    pass

@monitoring_routes.route('/user/<string:username>/root/<string:m_wf_id>/workflow/<string:wf_id>')
def get_workflow(username, m_wf_id, wf_id):
    pass

"""
{
    "state"         : string:state,
    "status"        : int:status,
    "restart_count" : int:restart_count,
    "timestamp"     : datetime:timestamp,
    "_links"          : {
        "workflow"     : "<href:workflow>"
    }
}
"""

"""
Job

{
    "job_id"      : int: job_id,
    "exec_job_id" : string: exec_job_id,
    "submit_file" : string: submit_file,
    "type_desc"   : string: type_desc,
    "max_retries" : int: max_retries,
    "clustered"   : bool: clustered,
    "task_count"  : int: task_count,
    "executable"  : string: executable,
    "argv"        : string: argv,
    "_links"      : {
        "workflow"     : href:workflow,
        "task"         : href:task,
        "job_instance" : href:job_instance
    }
}
"""

"""
Host

    {
    "host_id"      : int:host_id,
    "site_name"    : string:site_name,
    "hostname"     : string:hostname,
    "ip"           : string:ip,
    "uname"        : string:uname,
    "total_memory" : string:total_memory,
    "_links"       : {
        "workflow"     : href:workflow,
        "job_instance" : href:job_instance
    }
}
"""

"""
Job State

{
    "job_instance_id"     : "<int:job_instance_id>",
    "state"               : "<string:state>",
    "jobstate_submit_seq" : "<int:jobstate_submit_seq>",
    "timestamp"           : "<int:timestamp>",
    "_links"              : {
        "job_instance" : "href:job_instance"
    }
}
"""

"""
Task

{
    "task_id"        : int:task_id,
    "abs_task_id"    : string:abs_task_id,
    "type_desc"      : string: type_desc,
    "transformation" : string:transformation,
    "argv"           : string:argv,
    "_links"         : {
        "workflow" : href:workflow,
        "job"      : href:job
    }
}
"""

"""
Job Instance

{
    "job_instance_id"   : int:job_instance_id,
    "host_id"           : int:host_id,
    "job_submit_seq"    : int:job_submit_seq,
    "sched_id"          : string:sched_id,
    "site_name"         : string:site_name,
    "user"              : string:user,
    "work_dir"          : string:work_dir,
    "cluster_start"     : int:cluster_start,
    "cluster_duration"  : int:cluster_duration,
    "local_duration"    : int:local_duration,
    "subwf_id"          : int:subwf_id,
    "stdout_text"       : string:stdout_text,
    "stderr_text"       : string:stderr_text,
    "stdin_file"        : string:stdin_file,
    "stdout_file"       : string:stdout_file,
    "stderr_file"       : string:stderr_file,
    "multiplier_factor" : int:multiplier_factor,
    "_links"            : {
        "job_state"  : href:job_state,
        "host"       : href:host,
        "invocation" : href:invocation,
        "job"        : href:job
    }
}
"""

"""
Invocation

{
    "invocation_id"   : int:invocation_id,
    "job_instance_id" : int:job_instance_id,
    "abs_task_id"     : string:abs_task_id,
    "task_submit_seq" : int:task_submit_seq,
    "start_time"      : int:start_time,
    "remote_duration" : int:remote_duration,
    "remote_cpu_time" : int:remote_cpu_time,
    "exitcode"        : int:exitcode,
    "transformation"  : string:transformation,
    "executable"      : string:executable,
    "argv"            : string:argv,
    "_links"          : {
        "workflow"     : href:workflow,
        "job_instance" : href:job_instance
    }
}
"""

@monitoring_routes.errorhandler(404)
def page_not_found(error):
    pass

@monitoring_routes.errorhandler(BaseException)
def master_database_missing(error):
    pass
<|endoftext|>


<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os
from toil.lib.bioio import getTempFile
from toil.common import Toil, ToilContextManagerException
from toil.job import Job
from toil.test import ToilTest, slow

@slow
class ToilContextManagerTest(ToilTest):
    def setUp(self):
        self.exportPath = getTempFile()

    def tearDown(self):
        os.remove(self.exportPath)

    def testContextManger(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = 'INFO'
        with Toil(options) as toil:
            toil.start(HelloWorld())

    def testNoContextManger(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = 'INFO'
        toil = Toil(options)
        self.assertRaises(ToilContextManagerException, toil.start, HelloWorld())

    def testExportAfterFailedExport(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        try:
            with Toil(options) as toil:
                _ = toil.start(HelloWorld())
                # oh no, an error! :(
                raise RuntimeError("we died after workflow completion but before our export finished")
        except RuntimeError:
            pass

        options.restart = True
        with Toil(options) as toil:
            fileID = toil.restart()
            print(fileID)
            # Hopefully the error didn't cause us to lose all our work!
            toil.exportFile(fileID, 'file://' + self.exportPath)
        with open(self.exportPath) as f:
            # The file should have all our content
            self.assertEquals(f.read(), "Hello, World!")

class HelloWorld(Job):
    def __init__(self):
        Job.__init__(self, memory=100000, cores=2, disk='1M')

    def run(self, fileStore):
        fileID = self.addChildJobFn(childFn, cores=1, memory='1M', disk='1M').rv()
        return self.addFollowOn(FollowOn(fileID)).rv()

def childFn(job):
    with job.fileStore.writeGlobalFileStream() as (fH, fileID):
        fH.write("Hello, World!")
        return fileID

class FollowOn(Job):
    def __init__(self, fileId):
        Job.__init__(self)
        self.fileId = fileId

    def run(self, fileStore):
        tempDir = fileStore.getLocalTempDir()
        tempFilePath = "/".join([tempDir, 'LocalCopy'])
        with fileStore.readGlobalFileStream(self.fileId) as globalFile:
            with open(tempFilePath, "w") as localFile:
                localFile.write(globalFile.read())
        return self.fileId
<commit_msg>Handle file stream binary-ness in ToilContextManagerTest<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os
from toil.lib.bioio import getTempFile
from toil.common import Toil, ToilContextManagerException
from toil.job import Job
from toil.test import ToilTest, slow

@slow
class ToilContextManagerTest(ToilTest):
    def setUp(self):
        self.exportPath = getTempFile()

    def tearDown(self):
        os.remove(self.exportPath)

    def testContextManger(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = 'INFO'
        with Toil(options) as toil:
            toil.start(HelloWorld())

    def testNoContextManger(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = 'INFO'
        toil = Toil(options)
        self.assertRaises(ToilContextManagerException, toil.start, HelloWorld())

    def testExportAfterFailedExport(self):
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        try:
            with Toil(options) as toil:
                _ = toil.start(HelloWorld())
                # oh no, an error! :(
                raise RuntimeError("we died after workflow completion but before our export finished")
        except RuntimeError:
            pass

        options.restart = True
        with Toil(options) as toil:
            fileID = toil.restart()
            print(fileID)
            # Hopefully the error didn't cause us to lose all our work!
            toil.exportFile(fileID, 'file://' + self.exportPath)
        with open(self.exportPath) as f:
            # The file should have all our content
            self.assertEquals(f.read(), "Hello, World!")

class HelloWorld(Job):
    def __init__(self):
        Job.__init__(self, memory=100000, cores=2, disk='1M')

    def run(self, fileStore):
        fileID = self.addChildJobFn(childFn, cores=1, memory='1M', disk='1M').rv()
        return self.addFollowOn(FollowOn(fileID)).rv()

def childFn(job):
    with job.fileStore.writeGlobalFileStream() as (fH, fileID):
        fH.write("Hello, World!".encode('utf-8'))
        return fileID

class FollowOn(Job):
    def __init__(self, fileId):
        Job.__init__(self)
        self.fileId = fileId

    def run(self, fileStore):
        tempDir = fileStore.getLocalTempDir()
        tempFilePath = "/".join([tempDir, 'LocalCopy'])
        with fileStore.readGlobalFileStream(self.fileId) as globalFile:
            with open(tempFilePath, "wb") as localFile:
                localFile.write(globalFile.read())
        return self.fileId
<|endoftext|>


<commit_before>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import os

from cloudify.constants import LOCAL_IP_KEY, MANAGER_IP_KEY, MANAGER_REST_PORT_KEY

def get_local_ip():
    return os.environ[LOCAL_IP_KEY]

def get_manager_ip():
    return os.environ[MANAGER_IP_KEY]

def get_manager_rest_service_port():
    return int(os.environ[MANAGER_REST_PORT_KEY])

def get_cosmo_properties():
    return {
        "management_ip": get_manager_ip(),
        "ip": get_local_ip()
    }
<commit_msg>CLOUDIFY-2370 fix flake<commit_after>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import os

from cloudify.constants import LOCAL_IP_KEY, MANAGER_IP_KEY, \
    MANAGER_REST_PORT_KEY

def get_local_ip():
    return os.environ[LOCAL_IP_KEY]

def get_manager_ip():
    return os.environ[MANAGER_IP_KEY]

def get_manager_rest_service_port():
    return int(os.environ[MANAGER_REST_PORT_KEY])

def get_cosmo_properties():
    return {
        "management_ip": get_manager_ip(),
        "ip": get_local_ip()
    }
<|endoftext|>


<commit_before>
from gasistafelice.base.models import Person
from gasistafelice.admin.models import *

class GASAdmin_GAS(GASAdmin):
    change_list_template = 'gas_admin/change_list.html'

    def queryset(self, request):
        qs = super(GASAdmin_GAS, self).queryset(request)
        p = Person.objects.get(user=request.user)
        rv = qs.filter(pk__in=p.gasmember_set.values('pk'))
        return rv

class GASAdmin_GASSupplierOrder(GASSupplierOrderAdmin):

    change_list_template = 'gas_admin/change_list.html'

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if db_field.name == "gas":
            gas_qs = gas_models.GAS.objects.filter(gasmember__in=[request.user.person])
            kwargs["queryset"] = gas_qs
        elif db_field.name == "supplier":
            gas_qs = gas_models.GAS.objects.filter(gasmember__in=[request.user.person])
            supplier_qs = supplier_models.Supplier.objects.filter(gas__in=gas_qs)
            kwargs["queryset"] = supplier_qs
        return super(GASSupplierOrderAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

    def queryset(self, request):
        qs = super(GASAdmin_GASSupplierOrder, self).queryset(request)
        p = Person.objects.get(user=request.user)
        rv = qs.filter(gas__isnull=True) #in=p.gasmember_set.all())
        return rv

<commit_msg>Permission integration<commit_after>
from gasistafelice.base.models import Person
from gasistafelice.admin.models import *

from gasistafelice.auth import EDIT, CREATE

class GASAdmin_GAS(GASAdmin):
    change_list_template = 'gas_admin/change_list.html'

    def has_change_permission(self, request, obj=None):
        if obj is None:
            rv = super(GASAdmin, self).has_change_permission(request, None)
        else:
            rv = request.user.has_perm(EDIT, obj=obj)
        return rv

    def queryset(self, request):
        qs = super(GASAdmin_GAS, self).queryset(request)
        p = Person.objects.get(user=request.user)
        rv = qs.filter(pk__in=p.gasmember_set.values('pk'))
        return rv

class GASAdmin_GASSupplierOrder(GASSupplierOrderAdmin):

    change_list_template = 'gas_admin/change_list.html'

    def has_add_permission(self, request, obj=None):
        if obj is None:
            rv = super(GASSupplierOrderAdmin, self).has_add_permission(request, None)
        else:
            rv = request.user.has_perm(CREATE, obj=obj)
        return rv

    def has_change_permission(self, request, obj=None):
        if obj is None:
            rv = super(GASSupplierOrderAdmin, self).has_change_permission(request, None)
        else:
            rv = request.user.has_perm(EDIT, obj=obj)
        return rv

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if db_field.name == "gas":
            gas_qs = gas_models.GAS.objects.filter(gasmember__in=[request.user.person])
            kwargs["queryset"] = gas_qs
        elif db_field.name == "supplier":
            gas_qs = gas_models.GAS.objects.filter(gasmember__in=[request.user.person])
            supplier_qs = supplier_models.Supplier.objects.filter(gas__in=gas_qs)
            kwargs["queryset"] = supplier_qs
        return super(GASSupplierOrderAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)

    def queryset(self, request):
        qs = super(GASAdmin_GASSupplierOrder, self).queryset(request)
        p = Person.objects.get(user=request.user)
        rv = qs.filter(gas__in=p.gasmember_set.all())
        return rv

<|endoftext|>


<commit_before>try:
    idalink = __import__('idalink').idalink
except ImportError:
    idalink = None

from ..errors import CLEError
from ..backends import Backend

import logging
l = logging.getLogger("cle.idabin")

__all__ = ('IDABin',)

class IDABin(Backend):
    """
    Get information from binaries using IDA.
    """
    def __init__(self, binary, *args, **kwargs):
        if idalink is None:
            raise CLEError("Install the idalink module to use the IDABin backend!")

        super(IDABin, self).__init__(binary, *args, **kwargs)

        if self.arch is None:
            raise CLEError("You must specify a custom_arch in order to use the IDABin backend")

        ida_prog = "idal64" # We don't really need 32 bit idal, do we ?
        processor_type = self.arch.ida_processor

        l.debug("Loading binary %s using IDA with arch %s", self.binary, processor_type)

        self.ida_path = Loader._make_tmp_copy(self.binary)
        try:
            self.ida = idalink(self.ida_path, ida_prog=ida_prog,
                                       processor_type=processor_type).link
        except idalink.IDALinkError as e:
            raise CLEError("IDALink returned error: %s" % e.message)

        self.BADADDR = self.ida.idc.BADADDR
        l.info('Loading memory from ida, this will take a minute...')
        memcache = self.ida.memory

        for segaddr in self.ida.idautils.Segments():
            segend = self.ida.idc.SegEnd(segaddr)
            string = ''.join(memcache[i] if i in memcache else '\0' for i in xrange(segaddr, segend))
            self.memory.add_backer(segaddr, string)

        self.got_begin = None
        self.got_end = None
        self.raw_imports = {}
        self.current_module_name = None

        self.imports = self._get_imports()
        self.resolved_imports = {}
        self.linking = self._get_linking_type()

        self.exports = self._get_exports()

        l.warning('The IDABin module is not well supported. Good luck!')

    supported_filetypes = ['elf', 'pe', 'mach-o']

    def in_which_segment(self, addr):
        """
        Return the segment name at address `addr` (IDA).
        """
        seg = self.ida.idc.SegName(addr)
        if len(seg) == 0:
            seg = "unknown"
        return seg

    def _find_got(self):
        """
        Locate the section (e.g. .got) that should be updated when relocating functions (that's where we want to
        write absolute addresses).
        """
        sec_name = self.arch.got_section_name
        self.got_begin = None
        self.got_end = None

        for seg in self.ida.idautils.Segments():
            name = self.ida.idc.SegName(seg)
            if name == sec_name:
                self.got_begin = self.ida.idc.SegStart(seg)
                self.got_end = self.ida.idc.SegEnd(seg)

        # If we reach this point, we should have the addresses
        if self.got_begin is None or self.got_end is None:
            l.warning("No section %s, is this a static binary ? (or stripped)", sec_name)
            return False
        return True

    def _in_proper_section(self, addr):
        """
        Is `addr` in the proper section for this architecture ?
        """
        return self.got_begin < addr < self.got_end

    def function_name(self, addr):
        """
        Return the function name at address `addr` (IDA).
        """
        name = self.ida.idc.GetFunctionName(addr)
        if len(name) == 0:
            name = "UNKNOWN"
        return name

    def _lookup_symbols(self, symbols):
        """
        Resolves a bunch of symbols denoted by the list `symbols`.

        :returns: A dict of the form {symb:addr}.
        """
        addrs = {}

        for sym in symbols:
            addr = self.get_symbol_addr(sym)
            if not addr:
                l.debug("Symbol %s was not found (IDA)", sym)
                continue
            addrs[sym] = addr
        return addrs

    def get_symbol_addr(self, sym):
        """
        Get the address of the symbol `sym` from IDA.

        :returns: An address.
        """
        #addr = self.ida.idaapi.get_name_ea(self.ida.idc.BADADDR, sym)
        addr = self.ida.idc.LocByName(sym)
        if addr == self.BADADDR:
            addr = None
        return addr

    def _get_exports(self):
        """
        Get the binary exports names from IDA and return a list.
        """
        exports = {}
        for item in list(self.ida.idautils.Entries()):
            name = item[-1]
            if name is None:
                continue
            ea = item[1]
            exports[name] = ea
            #l.debug("\t export %s 0x@%x" % (name, ea))
        return exports

    def _get_ida_imports(self):
        """
        Extract imports from binary (IDA).
        """
        l.warning("TODO: improve this: IDA mixes functions and global data in exports, this will cause issues.")
        import_modules_count = self.ida.idaapi.get_import_module_qty()
        self.raw_imports = {}

        for i in xrange(0, import_modules_count):
            self.current_module_name = self.ida.idaapi.get_import_module_name(i)
            self.ida.idaapi.enum_import_names(i, self._import_entry_callback)

    def _import_entry_callback(self, ea, name, entry_ord): # pylint: disable=unused-argument
        """
        Callback function for IDA's enum_import_names.
        """
        self.raw_imports[name] = ea
        return True

    def _get_imports(self):
        """
        Extract imports from the binary. This uses the exports we get from IDA and then tries to find the GOT
        entries related to them.

        :returns:   a dict of the form {import:got_address}.
        """
        # Get the list of imports from IDA
        self._get_ida_imports()

        # Static binary
        if len(self.raw_imports) == 0:
            l.info("This is a static binary.")
            return

        # Locate the GOT on this architecture. If we can't, let's just default
        # to IDA's imports (which gives stub addresses instead).
        if not self._find_got():
            l.warning("We could not identify the GOT section. This looks like a stripped binary. IDA'll probably give "
                      "us PLT stubs instead, so keep in mind that Ld.find_symbol_got_entry() and friends won't work "
                      "with actual GOT addresses. If that's a problem, use the ELF backend instead.")
            return self.raw_imports

        # Then process it to get the correct addresses
        imports = {}
        for name, ea in self.raw_imports.iteritems():
            # If this architecture uses the plt directly, then we need to look
            # in the code segment.
            if self.arch.got_section_name == '.plt':
                lst = list(self.ida.idautils.CodeRefsTo(ea, 1))
            else:
                lst = list(self.ida.idautils.DataRefsTo(ea))

            for addr in lst:
                if self._in_proper_section(addr) and addr != self.BADADDR:
                    imports[name] = addr
                    l.debug("\t -> has import %s - GOT entry @ 0x%x", name, addr)
        return imports

    def get_min_addr(self):
        """
        Get the min address of the binary (IDA).
        """
        nm = self.ida.idc.NextAddr(0)
        pm = self.ida.idc.PrevAddr(nm)

        if pm == self.BADADDR:
            return nm
        else:
            return pm

    def get_max_addr(self):
        """
        Get the max address of the binary (IDA).
        """
        pm = self.ida.idc.PrevAddr(self.ida.idc.MAXADDR)
        nm = self.ida.idc.NextAddr(pm)

        if nm == self.BADADDR:
            return pm
        else:
            return nm

    @property
    def entry(self):
        if self._custom_entry_point is not None:
            return self._custom_entry_point + self.rebase_addr
        return self.ida.idc.BeginEA() + self.rebase_addr

    def resolve_import_dirty(self, sym, new_val):
        """
        Resolve import for symbol `sym` the dirty way, i.e. find all references to it in the code and replace it with
        the address `new_val` inline (instead of updating GOT slots). Don't use this unless you really have to, use
        :func:`resolve_import_with` instead.
        """

        #l.debug("\t %s resolves to 0x%x", sym, new_val)

        # Try IDA's _ptr
        plt_addr = self.get_symbol_addr(sym + "_ptr")
        if plt_addr:
            self.memory.write_addr_at(plt_addr, new_val)
            return

        # Try the __imp_name
        plt_addr = self.get_symbol_addr("__imp_" + sym)
        if plt_addr:
            for addr in self.ida.idautils.DataRefsTo(plt_addr):
                self.memory.write_addr_at(addr, new_val)
            return

        # Try the normal name
        plt_addr = self.get_symbol_addr(sym)
        if plt_addr:
            addrlist = list(self.ida.idautils.DataRefsTo(plt_addr))
            # If not datarefs, try coderefs. It can happen on PPC
            if len(addrlist) == 0:
                addrlist = list(self.ida.idautils.CodeRefsTo(plt_addr))
            for addr in addrlist:
                self.memory.write_addr_at(addr, new_val)
            return

        # If none of them has an address, that's a problem
        l.warning("Could not find references to symbol %s (IDA)", sym)

    def set_got_entry(self, name, newaddr):
        """
        Resolve import `name` with address `newaddr`. That is, update the GOT entry for `name` with `newaddr`.
        """
        if name not in self.imports:
            l.warning("%s not in imports", name)
            return

        addr = self.imports[name]
        self.memory.write_addr_at(addr, newaddr)

    def is_thumb(self, addr):
        """
        Is the address `addr` in thumb mode ? (ARM).
        """
        if not "arm" in self.arch:
            return False
        return self.ida.idc.GetReg(addr, "T") == 1

    def get_strings(self):
        """
        Extract strings from binary (IDA).

        :returns:   An array of strings.
        """
        ss = self.ida.idautils.Strings()
        string_list = []
        for s in ss:
            t_entry = (s.ea, str(s), s.length)
            string_list.append(t_entry)
        return string_list

    def _get_linking_type(self):
        """
        Returns whether a binary is statically or dynamically linked based on its imports.
        """
        # TODO: this is not the best, and with the Elf class we actually look for the presence of a dynamic table. We
        # should do it with IDA too.

        if len(self.raw_imports) == 0:
            return "static"
        else:
            return "dynamic"

from ..loader import Loader
<commit_msg>Duck type the IDABin backend as a MetaELF<commit_after>try:
    idalink = __import__('idalink').idalink
except ImportError:
    idalink = None

from ..errors import CLEError
from ..backends import Backend

import logging
l = logging.getLogger("cle.idabin")

__all__ = ('IDABin',)

class IDABin(Backend):
    """
    Get information from binaries using IDA.
    """
    def __init__(self, binary, *args, **kwargs):
        if idalink is None:
            raise CLEError("Install the idalink module to use the IDABin backend!")

        super(IDABin, self).__init__(binary, *args, **kwargs)

        if self.arch is None:
            raise CLEError("You must specify a custom_arch in order to use the IDABin backend")

        ida_prog = "idal64" # We don't really need 32 bit idal, do we ?
        processor_type = self.arch.ida_processor

        l.debug("Loading binary %s using IDA with arch %s", self.binary, processor_type)

        self.ida_path = Loader._make_tmp_copy(self.binary)
        try:
            self.ida = idalink(self.ida_path, ida_prog=ida_prog,
                                       processor_type=processor_type).link
        except idalink.IDALinkError as e:
            raise CLEError("IDALink returned error: %s" % e.message)

        self.BADADDR = self.ida.idc.BADADDR
        l.info('Loading memory from ida, this will take a minute...')
        memcache = self.ida.memory

        for segaddr in self.ida.idautils.Segments():
            segend = self.ida.idc.SegEnd(segaddr)
            string = ''.join(memcache[i] if i in memcache else '\0' for i in xrange(segaddr, segend))
            self.memory.add_backer(segaddr, string)

        self.got_begin = None
        self.got_end = None
        self.raw_imports = {}
        self.current_module_name = None

        self.imports = self._get_imports()
        self.resolved_imports = {}
        self.linking = self._get_linking_type()

        self.exports = self._get_exports()

        l.warning('The IDABin module is not well supported. Good luck!')

    supported_filetypes = ['elf', 'pe', 'mach-o']

    def in_which_segment(self, addr):
        """
        Return the segment name at address `addr` (IDA).
        """
        seg = self.ida.idc.SegName(addr)
        if len(seg) == 0:
            seg = "unknown"
        return seg

    def _find_got(self):
        """
        Locate the section (e.g. .got) that should be updated when relocating functions (that's where we want to
        write absolute addresses).
        """
        sec_name = self.arch.got_section_name
        self.got_begin = None
        self.got_end = None

        for seg in self.ida.idautils.Segments():
            name = self.ida.idc.SegName(seg)
            if name == sec_name:
                self.got_begin = self.ida.idc.SegStart(seg)
                self.got_end = self.ida.idc.SegEnd(seg)

        # If we reach this point, we should have the addresses
        if self.got_begin is None or self.got_end is None:
            l.warning("No section %s, is this a static binary ? (or stripped)", sec_name)
            return False
        return True

    def _in_proper_section(self, addr):
        """
        Is `addr` in the proper section for this architecture ?
        """
        return self.got_begin < addr < self.got_end

    def function_name(self, addr):
        """
        Return the function name at address `addr` (IDA).
        """
        name = self.ida.idc.GetFunctionName(addr)
        if len(name) == 0:
            name = "UNKNOWN"
        return name

    def _lookup_symbols(self, symbols):
        """
        Resolves a bunch of symbols denoted by the list `symbols`.

        :returns: A dict of the form {symb:addr}.
        """
        addrs = {}

        for sym in symbols:
            addr = self.get_symbol_addr(sym)
            if not addr:
                l.debug("Symbol %s was not found (IDA)", sym)
                continue
            addrs[sym] = addr
        return addrs

    def get_symbol_addr(self, sym):
        """
        Get the address of the symbol `sym` from IDA.

        :returns: An address.
        """
        #addr = self.ida.idaapi.get_name_ea(self.ida.idc.BADADDR, sym)
        addr = self.ida.idc.LocByName(sym)
        if addr == self.BADADDR:
            addr = None
        return addr

    def _get_exports(self):
        """
        Get the binary exports names from IDA and return a list.
        """
        exports = {}
        for item in list(self.ida.idautils.Entries()):
            name = item[-1]
            if name is None:
                continue
            ea = item[1]
            exports[name] = ea
            #l.debug("\t export %s 0x@%x" % (name, ea))
        return exports

    def _get_ida_imports(self):
        """
        Extract imports from binary (IDA).
        """
        l.warning("TODO: improve this: IDA mixes functions and global data in exports, this will cause issues.")
        import_modules_count = self.ida.idaapi.get_import_module_qty()
        self.raw_imports = {}

        for i in xrange(0, import_modules_count):
            self.current_module_name = self.ida.idaapi.get_import_module_name(i)
            self.ida.idaapi.enum_import_names(i, self._import_entry_callback)

    def _import_entry_callback(self, ea, name, entry_ord): # pylint: disable=unused-argument
        """
        Callback function for IDA's enum_import_names.
        """
        self.raw_imports[name] = ea
        return True

    def _get_imports(self):
        """
        Extract imports from the binary. This uses the exports we get from IDA and then tries to find the GOT
        entries related to them.

        :returns:   a dict of the form {import:got_address}.
        """
        # Get the list of imports from IDA
        self._get_ida_imports()

        # Static binary
        if len(self.raw_imports) == 0:
            l.info("This is a static binary.")
            return

        # Locate the GOT on this architecture. If we can't, let's just default
        # to IDA's imports (which gives stub addresses instead).
        if not self._find_got():
            l.warning("We could not identify the GOT section. This looks like a stripped binary. IDA'll probably give "
                      "us PLT stubs instead, so keep in mind that Ld.find_symbol_got_entry() and friends won't work "
                      "with actual GOT addresses. If that's a problem, use the ELF backend instead.")
            return self.raw_imports

        # Then process it to get the correct addresses
        imports = {}
        for name, ea in self.raw_imports.iteritems():
            # If this architecture uses the plt directly, then we need to look
            # in the code segment.
            if self.arch.got_section_name == '.plt':
                lst = list(self.ida.idautils.CodeRefsTo(ea, 1))
            else:
                lst = list(self.ida.idautils.DataRefsTo(ea))

            for addr in lst:
                if self._in_proper_section(addr) and addr != self.BADADDR:
                    imports[name] = addr
                    l.debug("\t -> has import %s - GOT entry @ 0x%x", name, addr)
        return imports

    def get_min_addr(self):
        """
        Get the min address of the binary (IDA).
        """
        nm = self.ida.idc.NextAddr(0)
        pm = self.ida.idc.PrevAddr(nm)

        if pm == self.BADADDR:
            return nm
        else:
            return pm

    def get_max_addr(self):
        """
        Get the max address of the binary (IDA).
        """
        pm = self.ida.idc.PrevAddr(self.ida.idc.MAXADDR)
        nm = self.ida.idc.NextAddr(pm)

        if nm == self.BADADDR:
            return pm
        else:
            return nm

    @property
    def entry(self):
        if self._custom_entry_point is not None:
            return self._custom_entry_point + self.rebase_addr
        return self.ida.idc.BeginEA() + self.rebase_addr

    def resolve_import_dirty(self, sym, new_val):
        """
        Resolve import for symbol `sym` the dirty way, i.e. find all references to it in the code and replace it with
        the address `new_val` inline (instead of updating GOT slots). Don't use this unless you really have to, use
        :func:`resolve_import_with` instead.
        """

        #l.debug("\t %s resolves to 0x%x", sym, new_val)

        # Try IDA's _ptr
        plt_addr = self.get_symbol_addr(sym + "_ptr")
        if plt_addr:
            self.memory.write_addr_at(plt_addr, new_val)
            return

        # Try the __imp_name
        plt_addr = self.get_symbol_addr("__imp_" + sym)
        if plt_addr:
            for addr in self.ida.idautils.DataRefsTo(plt_addr):
                self.memory.write_addr_at(addr, new_val)
            return

        # Try the normal name
        plt_addr = self.get_symbol_addr(sym)
        if plt_addr:
            addrlist = list(self.ida.idautils.DataRefsTo(plt_addr))
            # If not datarefs, try coderefs. It can happen on PPC
            if len(addrlist) == 0:
                addrlist = list(self.ida.idautils.CodeRefsTo(plt_addr))
            for addr in addrlist:
                self.memory.write_addr_at(addr, new_val)
            return

        # If none of them has an address, that's a problem
        l.warning("Could not find references to symbol %s (IDA)", sym)

    def set_got_entry(self, name, newaddr):
        """
        Resolve import `name` with address `newaddr`. That is, update the GOT entry for `name` with `newaddr`.
        """
        if name not in self.imports:
            l.warning("%s not in imports", name)
            return

        addr = self.imports[name]
        self.memory.write_addr_at(addr, newaddr)

    def is_thumb(self, addr):
        """
        Is the address `addr` in thumb mode ? (ARM).
        """
        if not "arm" in self.arch:
            return False
        return self.ida.idc.GetReg(addr, "T") == 1

    def get_strings(self):
        """
        Extract strings from binary (IDA).

        :returns:   An array of strings.
        """
        ss = self.ida.idautils.Strings()
        string_list = []
        for s in ss:
            t_entry = (s.ea, str(s), s.length)
            string_list.append(t_entry)
        return string_list

    def _get_linking_type(self):
        """
        Returns whether a binary is statically or dynamically linked based on its imports.
        """
        # TODO: this is not the best, and with the Elf class we actually look for the presence of a dynamic table. We
        # should do it with IDA too.

        if len(self.raw_imports) == 0:
            return "static"
        else:
            return "dynamic"

    # must be able to duck type as a MetaELF subclass

    @property
    def plt(self):
        # I know there's a way to do this but BOY do I not want to do it right now
        return {}

    @property
    def reverse_plt(self):
        return {}

    def get_call_stub_addr(self, name):
        return None

    @property
    def is_ppc64_abiv1(self):
        # IDA 6.9 segfaults when loading ppc64 abiv1 binaries so....
        return False

from ..loader import Loader
<|endoftext|>


<commit_before>#! /usr/bin/env python3

from argparse import ArgumentParser
from subprocess import Popen, call, check_output
from itertools import permutations
from collections import defaultdict
import csv

parser = ArgumentParser(description="Compare many binaries to each other",)
parser.add_argument('-f', '--file', help="File listing binaries, one per line")
parser.add_argument('-o', '--output', help="The base filename for the outputs")
parser.add_argument('-n', '--number', help="The n value for n-grams")

args = parser.parse_args()

input_file = open(args.file, "r")
n = args.number
output_file = open(args.output+"."+n+".out", "w")
file_list = list()

grams = list()
files = list()

# Read through the input file and run each binary through the n-gram program
for line in input_file:
    line = line.strip()
    # Run through n-gram parser
    out_name = line+"."+n+"gram"
    cmd = ["n_grams_bytes.py", "-n", n, "-o", out_name, line]
    call(cmd)
    grams.append(out_name)
    files.append(out_name)
input_file.close()

table = defaultdict(dict)
#Now, run each binary against every other binary
for first, second in permutations(grams, 2):
    cmd = ["compare_cluster.py", "-f", first, "-s", second]
    output = float(check_output(cmd))
    table[first][second] = output
    table[first][first] = 1.0

cw = csv.writer(output_file)
files.insert(0, "")
#Write the header to the file
cw.writerow(files)
del files[0]

#Go through the table and print out the cosine scores to fit a table.
print(table)
print(files)
for key in files:
    line = list()
    line.append(key)
    for item in files:
        print("%s, %s" % (key, item))
        line.append(table[key][item])
    cw.writerow(line)
<commit_msg>Prepended './' to the cmd calls because I forgot not everyone has . in their PATH<commit_after>#! /usr/bin/env python3

from argparse import ArgumentParser
from subprocess import Popen, call, check_output
from itertools import permutations
from collections import defaultdict
import csv

parser = ArgumentParser(description="Compare many binaries to each other",)
parser.add_argument('-f', '--file', help="File listing binaries, one per line")
parser.add_argument('-o', '--output', help="The base filename for the outputs")
parser.add_argument('-n', '--number', help="The n value for n-grams")

args = parser.parse_args()

input_file = open(args.file, "r")
n = args.number
output_file = open(args.output+"."+n+".out", "w")
file_list = list()

grams = list()
files = list()

# Read through the input file and run each binary through the n-gram program
for line in input_file:
    line = line.strip()
    # Run through n-gram parser
    out_name = line+"."+n+"gram"
    cmd = ["./n_grams_bytes.py", "-n", n, "-o", out_name, line]
    call(cmd)
    grams.append(out_name)
    files.append(out_name)
input_file.close()

table = defaultdict(dict)
#Now, run each binary against every other binary
for first, second in permutations(grams, 2):
    cmd = ["./compare_cluster.py", "-f", first, "-s", second]
    output = float(check_output(cmd))
    table[first][second] = output
    table[first][first] = 1.0

cw = csv.writer(output_file)
files.insert(0, "")
#Write the header to the file
cw.writerow(files)
del files[0]

#Go through the table and print out the cosine scores to fit a table.
print(table)
print(files)
for key in files:
    line = list()
    line.append(key)
    for item in files:
        print("%s, %s" % (key, item))
        line.append(table[key][item])
    cw.writerow(line)
<|endoftext|>


<commit_before>from bottypes.command import *
from bottypes.command_descriptor import *
from bottypes.invalid_command import *
from handlers.handler_factory import *
from handlers.base_handler import *
from addons.syscalls.syscallinfo import *
from util.util import *

import wolframalpha
import re

class QuestionCommand(Command):
    """Asks wolfram alpha a question and shows the anser."""

    def execute(self, slack_wrapper, args, channel_id, user_id):
        """Execute the Qeuestion command."""
        app_id = HandlerFactory.botserver.get_config_option("wolfram_app_id")

        if app_id:
            try:
                question = " ".join(args)
                client = wolframalpha.Client(app_id)
                res = client.query(question)
                answer = next(res.results)

                if answer:
                    response = answer.text

                    WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
                else:
                    response = "Wolfram Alpha doesn't seem to understand you. What a pity :("

                    WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
            except:
                response = "Wolfram Alpha doesn't seem to understand you :("

                WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
        else:
            response = "It seems you have no valid wolfram alpha app id. Contact an admin about this..."

            WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)

class WolframHandler(BaseHandler):
    """
    Handles questions for wolfram alpha engine.

    Commands :
    # Ask wolfram alpha a question
    !wolfram ask
    """

    @staticmethod
    def send_message(slack_wrapper, channel_id, user_id, msg):
        """Send message to user or channel, depending on configuration."""
        slack_wrapper.post_message(channel_id, msg)

    def __init__(self):
        self.commands = {
            "ask": CommandDesc(QuestionCommand, "Ask wolfram alpha a question", ["question"], None, False),
        }

HandlerFactory.register("wolfram", WolframHandler())
<commit_msg>Add error handling for invalid app id<commit_after>from bottypes.command import *
from bottypes.command_descriptor import *
from bottypes.invalid_command import *
from handlers.handler_factory import *
from handlers.base_handler import *
from addons.syscalls.syscallinfo import *
from util.util import *

import wolframalpha
import re

class QuestionCommand(Command):
    """Asks wolfram alpha a question and shows the anser."""

    def execute(self, slack_wrapper, args, channel_id, user_id):
        """Execute the Qeuestion command."""
        app_id = HandlerFactory.botserver.get_config_option("wolfram_app_id")

        if app_id:
            try:
                question = " ".join(args)
                client = wolframalpha.Client(app_id)
                res = client.query(question)

                answer = next(res.results, None)

                if answer:
                    response = answer.text

                    WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
                else:
                    response = "Wolfram Alpha doesn't seem to know the answer for this :("

                    WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
            except Exception as ex:
                if "Invalid appid" in str(ex):
                    response = "Wolfram Alpha app id doesn't seem to be correct (or api is choking)..."
                else:
                    response = "Wolfram Alpha doesn't seem to understand you :("

                WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)
        else:
            response = "It seems you have no valid wolfram alpha app id. Contact an admin about this..."

            WolframHandler.send_message(slack_wrapper, channel_id, user_id, response)

class WolframHandler(BaseHandler):
    """
    Handles questions for wolfram alpha engine.

    Commands :
    # Ask wolfram alpha a question
    !wolfram ask
    """

    @staticmethod
    def send_message(slack_wrapper, channel_id, user_id, msg):
        """Send message to user or channel, depending on configuration."""
        slack_wrapper.post_message(channel_id, msg)

    def __init__(self):
        self.commands = {
            "ask": CommandDesc(QuestionCommand, "Ask wolfram alpha a question", ["question"], None, False),
        }

HandlerFactory.register("wolfram", WolframHandler())
<|endoftext|>


<commit_before>__all__ = ('version', '__version__')

version = (0, 13, 9)
__version__ = '.'.join(str(x) for x in version)
<commit_msg>Bump version to 0.13.9.1<commit_after>__all__ = ('version', '__version__')

version = (0, 13, 9, 1)
__version__ = '.'.join(str(x) for x in version)
<|endoftext|>


<commit_before>## @package csnDependencies
# Definition of the dependencies Manager class.
import csnProject
import OrderedSet
import logging

class DependencyError(StandardError):
    """ Used when there is a cyclic dependency between CSnake projects. """
    pass

class Manager:
    """ Dependency checker class. """
    def __init__(self, _project):
        self.project = _project
        self.projects = OrderedSet.OrderedSet()
        self.projectsNonRequired = OrderedSet.OrderedSet()
        self.projectsIncludedInSolution = OrderedSet.OrderedSet()
        self.useBefore = []
        self.isTopLevel = False
        # logger
        self.__logger = logging.getLogger("CSnake")

    def AddProjects(self, _projects, _dependency = True, _includeInSolution = True):
        for project in _projects:
            projectToAdd = csnProject.ToProject(project)
            if projectToAdd in self.GetProjects():
                continue

            if self.project is projectToAdd:
                raise DependencyError, "Project %s cannot be added to itself" % (projectToAdd.name)

            if not projectToAdd in self.projects:
                #if _dependency and projectToAdd.dependenciesManager.DependsOn(self.project):
                #    raise DependencyError, "Circular dependency detected during %s.AddProjects(%s)" % (self.project.name, projectToAdd.name)
                self.projects.add( projectToAdd )
                if not _dependency:
                    self.projectsNonRequired.add( projectToAdd )
                if _includeInSolution:
                    self.projectsIncludedInSolution.add( projectToAdd )

    def DependsOn(self, _otherProject, _skipList = None):
        """
        Returns true if self is (directly or indirectly) dependent upon _otherProject.
        _otherProject - May be a project, or a function returning a project.
        _skipList - Used to not process project twice during the recursion (also prevents infinite loops).
        """
        if _skipList is None:
            _skipList = []

        otherProject = csnProject.ToProject(_otherProject)
        assert not self.project in _skipList, "\n\nError: %s should not be in stoplist" % (self.project.name)
        _skipList.append(self.project)
        for requiredProject in self.GetProjects(_onlyRequiredProjects = 1):
            if requiredProject in _skipList:
                continue
            if requiredProject is otherProject or requiredProject.dependenciesManager.DependsOn(otherProject, _skipList ):
                return True
        return False

    def GetProjects(self, _recursive = False, _onlyRequiredProjects = False, _includeSelf = False, _onlyPublicDependencies = False,
            _onlyNonRequiredProjects = False, _filter = True, _skipList = [], _stack = [], _cache = None):
        """
        Returns list of all projects associated with this project.
        _recursive -- If true, returns not only child projects but all projects in the tree below this project.
        _onlyRequiredProjects -- If true, only projects that this project requires are returned.
        """
        if _cache is None:
            _cache = dict()
        _recursive = bool(_recursive)
        _onlyRequiredProjects = bool(_onlyRequiredProjects)
        _includeSelf = bool(_includeSelf)
        _onlyPublicDependencies = bool(_onlyPublicDependencies)
        _onlyNonRequiredProjects = bool(_onlyNonRequiredProjects)
        _filter = bool(_filter)
        if self.project in _stack:
            if _onlyRequiredProjects:
                namelist = []
                foundStart = False
                for project in _stack:
                    if project == self.project:
                        foundStart = True
                    if foundStart:
                        namelist.append(project.name)
                namelist.append(self.project.name)
                self.__logger.Warn("Cyclic dependency: %s" % namelist)
            return OrderedSet.OrderedSet()
        _stack = _stack + [self.project]

        if self.project in _cache:
            if _includeSelf:
                return _cache[self.project] + [self.project]
            else:
                return _cache[self.project]

        directDependencies = OrderedSet.OrderedSet()
        toSubtract = OrderedSet.OrderedSet()
        toAdd = OrderedSet.OrderedSet()
        if _onlyNonRequiredProjects:
            toAdd.update(self.projectsNonRequired)
        else:
            toAdd.update(self.projects)
        if _filter:
            toAddFiltered = OrderedSet.OrderedSet()
            toAddFiltered.update([project for project in toAdd if not (project.MatchesFilter() and project in self.projectsNonRequired)])
            toAdd = toAddFiltered

        if _onlyRequiredProjects:
            toSubtract.update(self.projectsNonRequired)
        directDependencies.update(toAdd - toSubtract)

        if _recursive:
            _cache[self.project] = OrderedSet.OrderedSet()
            for project in directDependencies:
                _cache[self.project].update(
                    project.dependenciesManager.GetProjects(
                        _recursive = _recursive,
                        _onlyRequiredProjects = _onlyRequiredProjects,
                        _onlyPublicDependencies = _onlyPublicDependencies,
                        _includeSelf = True,
                        _onlyNonRequiredProjects = _onlyNonRequiredProjects,
                        _filter = _filter,
                        _skipList = _skipList,
                        _stack = _stack,
                        _cache = _cache
                    )
                )
        else:
            _cache[self.project] = directDependencies

        if _includeSelf:
            return _cache[self.project] + [self.project]
        else:
            return _cache[self.project]

    def UseBefore(self, _otherProject):
        """
        Sets a flag that says that self must be used before _otherProjects in a cmake file.
        Throws DependencyError if _otherProject wants to be used before self.
        _otherProject - May be a project, or a function returning a project.
        """
        otherProject = csnProject.ToProject(_otherProject)
        if otherProject.dependenciesManager.WantsToBeUsedBefore(self.project):
            raise DependencyError, "Cyclic use-before relation between %s and %s" % (self.project.name, otherProject.name)
        self.useBefore.append(otherProject)

    def WantsToBeUsedBefore(self, _otherProject):
        """
        Return true if self wants to be used before _otherProject.
        _otherProject - May be a project, or a function returning a project.
        """
        otherProject = csnProject.ToProject(_otherProject)

        #if otherProject in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
        #    return 1
        #if self.project in otherProject.dependenciesManager.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
        #    return 0

        if self.project is otherProject:
            return 0

        if otherProject in self.useBefore:
            return 1

        for requiredProject in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
            if otherProject in requiredProject.dependenciesManager.useBefore:
                return 1

        return 0

    def ProjectsToUse(self):
        """
        Determine a list of projects that must be used (meaning: include the config and use file) to generate this project.
        Note that self is always the last project in this list.
        The list is sorted in the correct order, using Project.WantsToBeUsedBefore.
        """
        result = []

        projectsToUse = [project for project in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1)]
        assert not self in projectsToUse, "\n\nError: %s should not be in projectsToUse" % (self.project.name)
        projectsToUse.append(self.project)

        (count, maxCount) = (0, 1)
        for i in range(len(projectsToUse)):
            maxCount = maxCount * (i+1)

        while (len(projectsToUse)):
            assert count < maxCount
            count += 1
            project = projectsToUse.pop()

            # check if we must skip this project for now, because another project must be used before this one
            skipThisProjectForNow = 0
            for otherProject in projectsToUse:
                if otherProject.dependenciesManager.WantsToBeUsedBefore(project):
                    assert not otherProject is self.project, "\n\nLogical error: %s cannot be used before %s" % (self.project.name, project.name)
                    skipThisProjectForNow = 1
            if skipThisProjectForNow:
                projectsToUse.insert(0, project)
                continue
            else:
                result.append(project)

        # ensure that self.project is the first entry in the result
        result.remove(self.project)
        result.insert(0,self.project)
        return result

    def WriteDependencyStructureToXML(self, filename):
        """
        Writes an xml file containing the dependency structure for this project and its dependency projects.
        """
        f = open(filename, 'w')
        self.WriteDependencyStructureToXMLImp(f)
        f.close()

    def WriteDependencyStructureToXMLImp(self, f, indent = 0):
        """
        Helper function.
        """
        for _ in range(indent):
            f.write(' ')
        f.write("<%s>" % self.project.name)
        for project in self.project.GetProjects(_onlyRequiredProjects = 1):
            project.dependenciesManager.WriteDependencyStructureToXMLImp(f, indent + 4)
        f.write("</%s>\n" % self.project.name)

    def Dump(self):
        return { \
            "topLevel" : self.isTopLevel, \
            "projects" : sorted([x.name for x in self.projects]), \
            "projectsNonRequired" : sorted([x.name for x in self.projectsNonRequired]), \
            "projectsInSolution" : sorted([x.name for x in self.projectsIncludedInSolution]), \
            "useBefore" : sorted([x.name for x in self.useBefore]) \
        }

<commit_msg>Updated documentation.<commit_after>## @package csnDependencies
# Definition of the dependencies Manager class.
import csnProject
import OrderedSet
import logging

class DependencyError(StandardError):
    """ Used when there is a cyclic dependency between CSnake projects. """
    pass

class Manager:
    """ Dependency checker class. """
    def __init__(self, _project):
        self.project = _project
        self.projects = OrderedSet.OrderedSet()
        self.projectsNonRequired = OrderedSet.OrderedSet()
        self.projectsIncludedInSolution = OrderedSet.OrderedSet()
        self.useBefore = []
        self.isTopLevel = False
        # logger
        self.__logger = logging.getLogger("CSnake")

    def AddProjects(self, _projects, _dependency = True, _includeInSolution = True):
        for project in _projects:
            projectToAdd = csnProject.ToProject(project)
            if projectToAdd in self.GetProjects():
                continue

            if self.project is projectToAdd:
                raise DependencyError, "Project %s cannot be added to itself" % (projectToAdd.name)

            if not projectToAdd in self.projects:
                #if _dependency and projectToAdd.dependenciesManager.DependsOn(self.project):
                #    raise DependencyError, "Circular dependency detected during %s.AddProjects(%s)" % (self.project.name, projectToAdd.name)
                self.projects.add( projectToAdd )
                if not _dependency:
                    self.projectsNonRequired.add( projectToAdd )
                if _includeInSolution:
                    self.projectsIncludedInSolution.add( projectToAdd )

    def DependsOn(self, _otherProject, _skipList = None):
        """
        Returns true if self is (directly or indirectly) dependent upon _otherProject.
        _otherProject - May be a project, or a function returning a project.
        _skipList - Used to not process project twice during the recursion (also prevents infinite loops).
        """
        if _skipList is None:
            _skipList = []

        otherProject = csnProject.ToProject(_otherProject)
        assert not self.project in _skipList, "\n\nError: %s should not be in stoplist" % (self.project.name)
        _skipList.append(self.project)
        for requiredProject in self.GetProjects(_onlyRequiredProjects = 1):
            if requiredProject in _skipList:
                continue
            if requiredProject is otherProject or requiredProject.dependenciesManager.DependsOn(otherProject, _skipList ):
                return True
        return False

    def GetProjects(self, _recursive = False, _onlyRequiredProjects = False, _includeSelf = False, _onlyPublicDependencies = False,
            _onlyNonRequiredProjects = False, _filter = True, _skipList = [], _stack = [], _cache = None):
        """
        Returns list of all projects associated with this project.
        _recursive -- If true, returns not only child projects but all projects in the tree below this project.
        _onlyRequiredProjects -- If true, only projects that this project requires are returned.
        _includeSelf
        _onlyPublicDependencies
        _onlyNonRequiredProjects
        _filter -- use the filter or not
        _stack -- do not touch, only used internally in recursive loops
        _cache -- cache for intermediate results, can be provided as external dictionary to be shared between calls
        """
        if _cache is None:
            _cache = dict()
        _recursive = bool(_recursive)
        _onlyRequiredProjects = bool(_onlyRequiredProjects)
        _includeSelf = bool(_includeSelf)
        _onlyPublicDependencies = bool(_onlyPublicDependencies)
        _onlyNonRequiredProjects = bool(_onlyNonRequiredProjects)
        _filter = bool(_filter)
        if self.project in _stack:
            if _onlyRequiredProjects:
                namelist = []
                foundStart = False
                for project in _stack:
                    if project == self.project:
                        foundStart = True
                    if foundStart:
                        namelist.append(project.name)
                namelist.append(self.project.name)
                self.__logger.Warn("Cyclic dependency: %s" % namelist)
            return OrderedSet.OrderedSet()
        _stack = _stack + [self.project]

        if self.project in _cache:
            if _includeSelf:
                return _cache[self.project] + [self.project]
            else:
                return _cache[self.project]

        directDependencies = OrderedSet.OrderedSet()
        toSubtract = OrderedSet.OrderedSet()
        toAdd = OrderedSet.OrderedSet()
        if _onlyNonRequiredProjects:
            toAdd.update(self.projectsNonRequired)
        else:
            toAdd.update(self.projects)
        if _filter:
            toAddFiltered = OrderedSet.OrderedSet()
            toAddFiltered.update([project for project in toAdd if not (project.MatchesFilter() and project in self.projectsNonRequired)])
            toAdd = toAddFiltered

        if _onlyRequiredProjects:
            toSubtract.update(self.projectsNonRequired)
        directDependencies.update(toAdd - toSubtract)

        if _recursive:
            _cache[self.project] = OrderedSet.OrderedSet()
            for project in directDependencies:
                _cache[self.project].update(
                    project.dependenciesManager.GetProjects(
                        _recursive = _recursive,
                        _onlyRequiredProjects = _onlyRequiredProjects,
                        _onlyPublicDependencies = _onlyPublicDependencies,
                        _includeSelf = True,
                        _onlyNonRequiredProjects = _onlyNonRequiredProjects,
                        _filter = _filter,
                        _skipList = _skipList,
                        _stack = _stack,
                        _cache = _cache
                    )
                )
        else:
            _cache[self.project] = directDependencies

        if _includeSelf:
            return _cache[self.project] + [self.project]
        else:
            return _cache[self.project]

    def UseBefore(self, _otherProject):
        """
        Sets a flag that says that self must be used before _otherProjects in a cmake file.
        Throws DependencyError if _otherProject wants to be used before self.
        _otherProject - May be a project, or a function returning a project.
        """
        otherProject = csnProject.ToProject(_otherProject)
        if otherProject.dependenciesManager.WantsToBeUsedBefore(self.project):
            raise DependencyError, "Cyclic use-before relation between %s and %s" % (self.project.name, otherProject.name)
        self.useBefore.append(otherProject)

    def WantsToBeUsedBefore(self, _otherProject):
        """
        Return true if self wants to be used before _otherProject.
        _otherProject - May be a project, or a function returning a project.
        """
        otherProject = csnProject.ToProject(_otherProject)

        #if otherProject in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
        #    return 1
        #if self.project in otherProject.dependenciesManager.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
        #    return 0

        if self.project is otherProject:
            return 0

        if otherProject in self.useBefore:
            return 1

        for requiredProject in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1):
            if otherProject in requiredProject.dependenciesManager.useBefore:
                return 1

        return 0

    def ProjectsToUse(self):
        """
        Determine a list of projects that must be used (meaning: include the config and use file) to generate this project.
        Note that self is always the last project in this list.
        The list is sorted in the correct order, using Project.WantsToBeUsedBefore.
        """
        result = []

        projectsToUse = [project for project in self.GetProjects(_recursive = 1, _onlyRequiredProjects = 1)]
        assert not self in projectsToUse, "\n\nError: %s should not be in projectsToUse" % (self.project.name)
        projectsToUse.append(self.project)

        (count, maxCount) = (0, 1)
        for i in range(len(projectsToUse)):
            maxCount = maxCount * (i+1)

        while (len(projectsToUse)):
            assert count < maxCount
            count += 1
            project = projectsToUse.pop()

            # check if we must skip this project for now, because another project must be used before this one
            skipThisProjectForNow = 0
            for otherProject in projectsToUse:
                if otherProject.dependenciesManager.WantsToBeUsedBefore(project):
                    assert not otherProject is self.project, "\n\nLogical error: %s cannot be used before %s" % (self.project.name, project.name)
                    skipThisProjectForNow = 1
            if skipThisProjectForNow:
                projectsToUse.insert(0, project)
                continue
            else:
                result.append(project)

        # ensure that self.project is the first entry in the result
        result.remove(self.project)
        result.insert(0,self.project)
        return result

    def WriteDependencyStructureToXML(self, filename):
        """
        Writes an xml file containing the dependency structure for this project and its dependency projects.
        """
        f = open(filename, 'w')
        self.WriteDependencyStructureToXMLImp(f)
        f.close()

    def WriteDependencyStructureToXMLImp(self, f, indent = 0):
        """
        Helper function.
        """
        for _ in range(indent):
            f.write(' ')
        f.write("<%s>" % self.project.name)
        for project in self.project.GetProjects(_onlyRequiredProjects = 1):
            project.dependenciesManager.WriteDependencyStructureToXMLImp(f, indent + 4)
        f.write("</%s>\n" % self.project.name)

    def Dump(self):
        return { \
            "topLevel" : self.isTopLevel, \
            "projects" : sorted([x.name for x in self.projects]), \
            "projectsNonRequired" : sorted([x.name for x in self.projectsNonRequired]), \
            "projectsInSolution" : sorted([x.name for x in self.projectsIncludedInSolution]), \
            "useBefore" : sorted([x.name for x in self.useBefore]) \
        }

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from baseframe import __
from baseframe.forms import Form, MarkdownField
import wtforms
import wtforms.fields.html5
from wtforms.ext.sqlalchemy.fields import QuerySelectField

__all__ = ['ProposalForm']

class ProposalForm(Form):
    speaking = wtforms.RadioField(__("Are you speaking?"), coerce=int,
        choices=[(1, __(u"I will be speaking")),
                 (0, __(u"I’m proposing a topic for someone to speak on"))])
    title = wtforms.TextField(__("Title"), validators=[wtforms.validators.Required()],
        description=__("The title of your session"))
    section = QuerySelectField(__("Section"), get_label='title', validators=[wtforms.validators.Required()],
        widget=wtforms.widgets.ListWidget(prefix_label=False), option_widget=wtforms.widgets.RadioInput())
    objective = MarkdownField(__("Objective"), validators=[wtforms.validators.Required()],
        description=__("What is the expected benefit for someone attending this?"))
    session_type = wtforms.RadioField(__("Session type"), validators=[wtforms.validators.Required()], choices=[
        ('Lecture', __("Lecture")),
        ('Demo', __("Demo")),
        ('Tutorial', __("Tutorial")),
        ('Workshop', __("Workshop")),
        ('Discussion', __("Discussion")),
        ('Panel', __("Panel")),
        ])
    technical_level = wtforms.RadioField(__("Technical level"), validators=[wtforms.validators.Required()], choices=[
        ('Beginner', __("Beginner")),
        ('Intermediate', __("Intermediate")),
        ('Advanced', __("Advanced")),
        ])
    description = MarkdownField(__("Description"), validators=[wtforms.validators.Required()],
        description=__("A detailed description of the session"))
    requirements = MarkdownField(__("Requirements"),
        description=__("For workshops, what must participants bring to the session?"))
    slides = wtforms.fields.html5.URLField(__("Slides"), validators=[wtforms.validators.Optional(), wtforms.validators.URL()],
        description=__("Link to your slides. These can be just an outline initially. "
            "If you provide a Slideshare link, we'll embed slides in the page"))
    links = wtforms.TextAreaField(__("Links"),
        description=__("Other links, one per line. Provide links to your profile and "
            "slides and videos from your previous sessions; anything that'll help "
            "folks decide if they want to attend your session"))
    bio = MarkdownField(__("Speaker bio"), validators=[wtforms.validators.Required()],
        description=__("Tell us why you are the best person to be taking this session"))
    email = wtforms.fields.html5.EmailField(__("Your email address"), validators=[wtforms.validators.Required()],
        description=__("An email address we can contact you at. "
            "Not displayed anywhere"))
    phone = wtforms.TextField(__("Phone number"), validators=[wtforms.validators.Required()],
        description=__("A phone number we can call you at to discuss your proposal, if required. "
            "Will not be displayed"))
    location = wtforms.TextField(__("Your location"), validators=[wtforms.validators.Required(), wtforms.validators.Length(max=80)],
        description=__("Your location, to help plan for travel arrangement"))
<commit_msg>Change wording for location field.<commit_after># -*- coding: utf-8 -*-

from baseframe import __
from baseframe.forms import Form, MarkdownField
import wtforms
import wtforms.fields.html5
from wtforms.ext.sqlalchemy.fields import QuerySelectField

__all__ = ['ProposalForm']

class ProposalForm(Form):
    speaking = wtforms.RadioField(__("Are you speaking?"), coerce=int,
        choices=[(1, __(u"I will be speaking")),
                 (0, __(u"I’m proposing a topic for someone to speak on"))])
    title = wtforms.TextField(__("Title"), validators=[wtforms.validators.Required()],
        description=__("The title of your session"))
    section = QuerySelectField(__("Section"), get_label='title', validators=[wtforms.validators.Required()],
        widget=wtforms.widgets.ListWidget(prefix_label=False), option_widget=wtforms.widgets.RadioInput())
    objective = MarkdownField(__("Objective"), validators=[wtforms.validators.Required()],
        description=__("What is the expected benefit for someone attending this?"))
    session_type = wtforms.RadioField(__("Session type"), validators=[wtforms.validators.Required()], choices=[
        ('Lecture', __("Lecture")),
        ('Demo', __("Demo")),
        ('Tutorial', __("Tutorial")),
        ('Workshop', __("Workshop")),
        ('Discussion', __("Discussion")),
        ('Panel', __("Panel")),
        ])
    technical_level = wtforms.RadioField(__("Technical level"), validators=[wtforms.validators.Required()], choices=[
        ('Beginner', __("Beginner")),
        ('Intermediate', __("Intermediate")),
        ('Advanced', __("Advanced")),
        ])
    description = MarkdownField(__("Description"), validators=[wtforms.validators.Required()],
        description=__("A detailed description of the session"))
    requirements = MarkdownField(__("Requirements"),
        description=__("For workshops, what must participants bring to the session?"))
    slides = wtforms.fields.html5.URLField(__("Slides"), validators=[wtforms.validators.Optional(), wtforms.validators.URL()],
        description=__("Link to your slides. These can be just an outline initially. "
            "If you provide a Slideshare link, we'll embed slides in the page"))
    links = wtforms.TextAreaField(__("Links"),
        description=__("Other links, one per line. Provide links to your profile and "
            "slides and videos from your previous sessions; anything that'll help "
            "folks decide if they want to attend your session"))
    bio = MarkdownField(__("Speaker bio"), validators=[wtforms.validators.Required()],
        description=__("Tell us why you are the best person to be taking this session"))
    email = wtforms.fields.html5.EmailField(__("Your email address"), validators=[wtforms.validators.Required()],
        description=__("An email address we can contact you at. "
            "Not displayed anywhere"))
    phone = wtforms.TextField(__("Phone number"), validators=[wtforms.validators.Required()],
        description=__("A phone number we can call you at to discuss your proposal, if required. "
            "Will not be displayed"))
    location = wtforms.TextField(__("Your location"), validators=[wtforms.validators.Required(), wtforms.validators.Length(max=80)],
        description=__("Your location, to help plan for your travel if required"))
<|endoftext|>