<commit_before># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.

from ..time_interval import TimeInterval, TimeIntervals, RelativeTimeInterval, parse_time_tuple
from ..utils import Hashable, utcnow
from . import StreamView, StreamId
from ..models import TimeIntervalModel, StreamDefinitionModel

from collections import Iterable
from mongoengine.context_managers import switch_db
from mongoengine.errors import NotUniqueError, DoesNotExist
import logging

class Stream(Hashable):
    """
    Stream reference class
    """
    def __init__(self, channel, stream_id, calculated_intervals, sandbox):
        """
        :param channel: The channel to which this stream belongs
        :param stream_id: The unique identifier for this string
        :param calculated_intervals: The time intervals in which this has been calculated
        :param sandbox: The sandbox in which this stream lives
        :type channel: BaseChannel
        :type stream_id: StreamId
        :type calculated_intervals: TimeIntervals, None
        :type sandbox: str, unicode, None
        """
        self.channel = channel
        if not isinstance(stream_id, StreamId):
            raise TypeError(type(stream_id))
        self.stream_id = stream_id
        # self.get_results_func = get_results_func
        self._calculated_intervals = None
        if calculated_intervals:
            if not isinstance(calculated_intervals, TimeIntervals):
                raise TypeError(type(calculated_intervals))
            self._calculated_intervals = calculated_intervals
        else:
            self._calculated_intervals = TimeIntervals()
        self.tool_reference = None  # needed to traverse the graph outside of workflows
        self.sandbox = sandbox
        self._node = None  # Back reference to node

        # Here we define the output type. When modifiers are applied, this changes
        # self.output_format = 'doc_gen'

    def set_tool_reference(self, tool_reference):  # needed to traverse the graph outside of workflows
        self.tool_reference = tool_reference

    def __str__(self):
        return "{}(stream_id={}, channel_id={})".format(
            self.__class__.__name__,
            self.stream_id,
            self.channel.channel_id)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        return str(self) == str(other)

    @property
    def parent_node(self):
        return self._node

    @parent_node.setter
    def parent_node(self, node):
        self._node = node

    @property
    def calculated_intervals(self):
        """
        Get the calculated intervals
        This will be read from the stream_status collection if it's in the database channel
        :return: The calculated intervals
        """
        return self._calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, value):
        """
        Set the calculated intervals
        This will be written to the stream_status collection if it's in the database channel
        :param value: The calculated intervals
        :type value: TimeIntervals, TimeInterval, list[TimeInterval]
        """
        if not value:
            self._calculated_intervals = TimeIntervals()
            return

        if isinstance(value, TimeInterval):
            value = TimeIntervals([value])
        elif isinstance(value, TimeIntervals):
            pass
        elif isinstance(value, list):
            value = TimeIntervals(value)
        else:
            raise TypeError("Expected list/TimeInterval/TimeIntervals, got {}".format(type(value)))

        for interval in value:
            if interval.end > utcnow():
                raise ValueError("Calculated intervals should not be in the future")

        self._calculated_intervals = value

    @property
    def writer(self):
        return self.channel.get_stream_writer(self)

    def window(self, time_interval=None, force_calculation=False):
        """
        Gets a view on this stream for the time interval given
        :param time_interval: either a TimeInterval object or (start, end) tuple of type str or datetime
        :param force_calculation: Whether we should force calculation for this stream view if data does not exist
        :type time_interval: None | Iterable | TimeInterval
        :type force_calculation: bool
        :return: a stream view object
        """
        if not time_interval:
            if self.calculated_intervals:
                time_interval = self.calculated_intervals[-1]
            else:
                raise ValueError("No calculations have been performed and no time interval was provided")
        elif isinstance(time_interval, TimeInterval):
            time_interval = TimeInterval(time_interval.start, time_interval.end)
        elif isinstance(time_interval, Iterable):
            time_interval = parse_time_tuple(*time_interval)
            if isinstance(time_interval, RelativeTimeInterval):
                raise NotImplementedError
        elif isinstance(time_interval, RelativeTimeInterval):
            raise NotImplementedError
        else:
            raise TypeError("Expected TimeInterval or (start, end) tuple of type str or datetime, got {}"
                            .format(type(time_interval)))
        return StreamView(stream=self, time_interval=time_interval, force_calculation=force_calculation)

class DatabaseStream(Stream):
    """
    Simple subclass that overrides the calculated intervals property
    """
    def __init__(self, channel, stream_id, calculated_intervals, last_accessed, last_updated, sandbox,
                 mongo_model=None):
        super(DatabaseStream, self).__init__(
            channel=channel,
            stream_id=stream_id,
            calculated_intervals=None,  # TODO: probably no point in having the actual calculated intervals here
            sandbox=sandbox)

        if mongo_model:
            self._calculated_intervals = self.mongo_model.get_calculated_intervals()
        else:
            # First try to load it from the database
            try:
                self.load()
            except DoesNotExist:
                self.mongo_model = StreamDefinitionModel(
                    stream_id=self.stream_id.as_dict(),
                    channel_id=self.channel.channel_id,
                    calculated_intervals=calculated_intervals,
                    last_accessed=last_accessed,
                    last_updated=last_updated,
                    sandbox=self.sandbox)
                self.save()

    def load(self):
        """
        Load the stream definition from the database
        :return: None
        """
        with switch_db(StreamDefinitionModel, 'hyperstream'):
            self.mongo_model = StreamDefinitionModel.objects.get(__raw__=self.stream_id.as_raw())
            self._calculated_intervals = self.mongo_model.get_calculated_intervals()

    def save(self):
        """
        Saves the stream definition to the database. This assumes that the definition doesn't already exist, and will
        raise an exception if it does.
        :type upsert: bool
        :return: None
        """
        with switch_db(StreamDefinitionModel, 'hyperstream'):
            self.mongo_model.save()

    @property
    def calculated_intervals(self):
        """
        Gets the calculated intervals from the database
        :return: The calculated intervals
        """
        if self._calculated_intervals is None:
            logging.debug("get calculated intervals")
            self.load()
            return self.mongo_model.get_calculated_intervals()
        return self._calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, intervals):
        """
        Updates the calculated intervals in the database. Performs an upsert
        :param intervals: The calculated intervals
        :return: None
        """
        logging.debug("set calculated intervals")
        self.mongo_model.set_calculated_intervals(intervals)
        self.save()
        self._calculated_intervals = intervals

    @property
    def last_accessed(self):
        """
        Gets the last accessed time from the database
        :return: The last accessed time
        """
        self.load()
        return self.mongo_model.last_accessed

    @last_accessed.setter
    def last_accessed(self, dt):
        """
        Updates the last accessed time in the database. Performs an upsert
        :param dt: The last accessed time
        :return: None
        """
        self.mongo_model.last_accessed = dt

    @property
    def last_updated(self):
        """
        Gets the last updated time from the database
        :return: The last updated time
        """
        self.load()
        return self.mongo_model.last_updated

    @last_updated.setter
    def last_updated(self, dt):
        """
        Updates the last updated time in the database. Performs an upsert
        :param dt: The last updated time
        :return: None
        """
        self.mongo_model.last_updated = dt

class AssetStream(DatabaseStream):
    """
    Simple subclass that overrides the calculated intervals property
    """
    @property
    def calculated_intervals(self):
        return super(AssetStream, self).calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, intervals):
        """
        Updates the calculated intervals in the database. Performs an upsert
        :param intervals: The calculated intervals
        :return: None
        """
        if len(intervals) > 1:
            raise ValueError("Only single calculated interval valid for AssetStream")
        super(AssetStream, self.__class__).calculated_intervals.fset(self, intervals)
<commit_msg>small fix<commit_after># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.

from ..time_interval import TimeInterval, TimeIntervals, RelativeTimeInterval, parse_time_tuple
from ..utils import Hashable, utcnow
from . import StreamView, StreamId
from ..models import TimeIntervalModel, StreamDefinitionModel

from collections import Iterable
from mongoengine.context_managers import switch_db
from mongoengine.errors import NotUniqueError, DoesNotExist
import logging

class Stream(Hashable):
    """
    Stream reference class
    """
    def __init__(self, channel, stream_id, calculated_intervals, sandbox):
        """
        :param channel: The channel to which this stream belongs
        :param stream_id: The unique identifier for this string
        :param calculated_intervals: The time intervals in which this has been calculated
        :param sandbox: The sandbox in which this stream lives
        :type channel: BaseChannel
        :type stream_id: StreamId
        :type calculated_intervals: TimeIntervals, None
        :type sandbox: str, unicode, None
        """
        self.channel = channel
        if not isinstance(stream_id, StreamId):
            raise TypeError(type(stream_id))
        self.stream_id = stream_id
        # self.get_results_func = get_results_func
        self._calculated_intervals = None
        if calculated_intervals:
            if not isinstance(calculated_intervals, TimeIntervals):
                raise TypeError(type(calculated_intervals))
            self._calculated_intervals = calculated_intervals
        else:
            self._calculated_intervals = TimeIntervals()
        self.tool_reference = None  # needed to traverse the graph outside of workflows
        self.sandbox = sandbox
        self._node = None  # Back reference to node

        # Here we define the output type. When modifiers are applied, this changes
        # self.output_format = 'doc_gen'

    def set_tool_reference(self, tool_reference):  # needed to traverse the graph outside of workflows
        self.tool_reference = tool_reference

    def __str__(self):
        return "{}(stream_id={}, channel_id={})".format(
            self.__class__.__name__,
            self.stream_id,
            self.channel.channel_id)

    def __repr__(self):
        return str(self)

    def __eq__(self, other):
        return str(self) == str(other)

    @property
    def parent_node(self):
        return self._node

    @parent_node.setter
    def parent_node(self, node):
        self._node = node

    @property
    def calculated_intervals(self):
        """
        Get the calculated intervals
        This will be read from the stream_status collection if it's in the database channel
        :return: The calculated intervals
        """
        return self._calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, value):
        """
        Set the calculated intervals
        This will be written to the stream_status collection if it's in the database channel
        :param value: The calculated intervals
        :type value: TimeIntervals, TimeInterval, list[TimeInterval]
        """
        if not value:
            self._calculated_intervals = TimeIntervals()
            return

        if isinstance(value, TimeInterval):
            value = TimeIntervals([value])
        elif isinstance(value, TimeIntervals):
            pass
        elif isinstance(value, list):
            value = TimeIntervals(value)
        else:
            raise TypeError("Expected list/TimeInterval/TimeIntervals, got {}".format(type(value)))

        for interval in value:
            if interval.end > utcnow():
                raise ValueError("Calculated intervals should not be in the future")

        self._calculated_intervals = value

    @property
    def writer(self):
        return self.channel.get_stream_writer(self)

    def window(self, time_interval=None, force_calculation=False):
        """
        Gets a view on this stream for the time interval given
        :param time_interval: either a TimeInterval object or (start, end) tuple of type str or datetime
        :param force_calculation: Whether we should force calculation for this stream view if data does not exist
        :type time_interval: None | Iterable | TimeInterval
        :type force_calculation: bool
        :return: a stream view object
        """
        if not time_interval:
            if self.calculated_intervals:
                time_interval = self.calculated_intervals[-1]
            else:
                raise ValueError("No calculations have been performed and no time interval was provided")
        elif isinstance(time_interval, TimeInterval):
            time_interval = TimeInterval(time_interval.start, time_interval.end)
        elif isinstance(time_interval, Iterable):
            time_interval = parse_time_tuple(*time_interval)
            if isinstance(time_interval, RelativeTimeInterval):
                raise NotImplementedError
        elif isinstance(time_interval, RelativeTimeInterval):
            raise NotImplementedError
        else:
            raise TypeError("Expected TimeInterval or (start, end) tuple of type str or datetime, got {}"
                            .format(type(time_interval)))
        return StreamView(stream=self, time_interval=time_interval, force_calculation=force_calculation)

class DatabaseStream(Stream):
    """
    Simple subclass that overrides the calculated intervals property
    """
    def __init__(self, channel, stream_id, calculated_intervals, last_accessed, last_updated, sandbox,
                 mongo_model=None):
        super(DatabaseStream, self).__init__(
            channel=channel,
            stream_id=stream_id,
            calculated_intervals=None,  # TODO: probably no point in having the actual calculated intervals here
            sandbox=sandbox)

        if mongo_model:
            self.mongo_model = mongo_model
            self._calculated_intervals = self.mongo_model.get_calculated_intervals()
        else:
            # First try to load it from the database
            try:
                self.load()
            except DoesNotExist:
                self.mongo_model = StreamDefinitionModel(
                    stream_id=self.stream_id.as_dict(),
                    channel_id=self.channel.channel_id,
                    calculated_intervals=calculated_intervals,
                    last_accessed=last_accessed,
                    last_updated=last_updated,
                    sandbox=self.sandbox)
                self.save()

    def load(self):
        """
        Load the stream definition from the database
        :return: None
        """
        with switch_db(StreamDefinitionModel, 'hyperstream'):
            self.mongo_model = StreamDefinitionModel.objects.get(__raw__=self.stream_id.as_raw())
            self._calculated_intervals = self.mongo_model.get_calculated_intervals()

    def save(self):
        """
        Saves the stream definition to the database. This assumes that the definition doesn't already exist, and will
        raise an exception if it does.
        :type upsert: bool
        :return: None
        """
        with switch_db(StreamDefinitionModel, 'hyperstream'):
            self.mongo_model.save()

    @property
    def calculated_intervals(self):
        """
        Gets the calculated intervals from the database
        :return: The calculated intervals
        """
        if self._calculated_intervals is None:
            logging.debug("get calculated intervals")
            self.load()
            return self.mongo_model.get_calculated_intervals()
        return self._calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, intervals):
        """
        Updates the calculated intervals in the database. Performs an upsert
        :param intervals: The calculated intervals
        :return: None
        """
        logging.debug("set calculated intervals")
        self.mongo_model.set_calculated_intervals(intervals)
        self.save()
        self._calculated_intervals = intervals

    @property
    def last_accessed(self):
        """
        Gets the last accessed time from the database
        :return: The last accessed time
        """
        self.load()
        return self.mongo_model.last_accessed

    @last_accessed.setter
    def last_accessed(self, dt):
        """
        Updates the last accessed time in the database. Performs an upsert
        :param dt: The last accessed time
        :return: None
        """
        self.mongo_model.last_accessed = dt

    @property
    def last_updated(self):
        """
        Gets the last updated time from the database
        :return: The last updated time
        """
        self.load()
        return self.mongo_model.last_updated

    @last_updated.setter
    def last_updated(self, dt):
        """
        Updates the last updated time in the database. Performs an upsert
        :param dt: The last updated time
        :return: None
        """
        self.mongo_model.last_updated = dt

class AssetStream(DatabaseStream):
    """
    Simple subclass that overrides the calculated intervals property
    """
    @property
    def calculated_intervals(self):
        return super(AssetStream, self).calculated_intervals

    @calculated_intervals.setter
    def calculated_intervals(self, intervals):
        """
        Updates the calculated intervals in the database. Performs an upsert
        :param intervals: The calculated intervals
        :return: None
        """
        if len(intervals) > 1:
            raise ValueError("Only single calculated interval valid for AssetStream")
        super(AssetStream, self.__class__).calculated_intervals.fset(self, intervals)
<|endoftext|>


<commit_before>import threading
import Pyro4

class PyroServiceObj(threading.Thread):
    def __init__(self, service_name):
        super(PyroServiceObj, self).__init__()
        self.service_name = service_name
        self.uri = None
        self.pyro_daemon = None

    def run(self):
        self.launch_service_msg_loop()

    def launch_service_msg_loop(self):
        if self.pyro_daemon is None:
            self.pyro_daemon = Pyro4.Daemon(port=8018)
            self.uri = self.pyro_daemon.register(self, self.service_name)
            print self.uri
            self.pyro_daemon.requestLoop(loopCondition=self.still_running)

    def still_running(self):
        print "still running"
        return True

    def pyro_shutdown(self):
        print 'shutting down daemon'
        self.pyro_daemon.shutdown()
        print 'shutdown complete'<commit_msg>Remove debug print.<commit_after>import threading
import Pyro4

class PyroServiceObj(threading.Thread):
    def __init__(self, service_name):
        super(PyroServiceObj, self).__init__()
        self.service_name = service_name
        self.uri = None
        self.pyro_daemon = None

    def run(self):
        self.launch_service_msg_loop()

    def launch_service_msg_loop(self):
        if self.pyro_daemon is None:
            self.pyro_daemon = Pyro4.Daemon(port=8018)
            self.uri = self.pyro_daemon.register(self, self.service_name)
            print self.uri
            #self.pyro_daemon.requestLoop(loopCondition=self.still_running)
            self.pyro_daemon.requestLoop()

    def still_running(self):
        print "still running"
        return True

    def pyro_shutdown(self):
        print 'shutting down daemon'
        self.pyro_daemon.shutdown()
        print 'shutdown complete'<|endoftext|>


<commit_before>import json
import pandas as pd
from sqlalchemy.sql import table, column
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.useradaptor import ExperimentAdaptor
from igf_data.igfdb.igfTables import Experiment, Run, Run_attribute, Seqrun

class RunAdaptor(BaseAdaptor):
  '''
   An adaptor class for Run and Run_attribute tables
  '''

  def store_run_and_attribute_data(self, data):
    '''
    A method for dividing and storing data to run and attribute table
    '''
    (run_data, run_attr_data)=self.divide_data_to_table_and_attribute(data=data)

    try:
      seqrun_map_function=lambda x: self.map_foreign_table_and_store_attribute(data=x, \
                                                                               lookup_table=Seqrun, \
                                                                               lookup_column_name='seqrun_igf_id', \
                                                                               target_column_name='seqrun_id')        # prepare seqrun mapping function
      new_run_data=run_data.apply(seqrun_map_function, axis=1)                                                        # map seqrun id
      self.store_run_data(data=new_run_data)                                                                              # store run
      exp_map_function=lambda x: self.map_foreign_table_and_store_attribute(data=x, \
                                                                            lookup_table=Experiment, \
                                                                            lookup_column_name='experiment_igf_id', \
                                                                            target_column_name='experiment_id')       # prepare the function
      new_run_attr_data=run_attr_data.apply(exp_map_function, axis=1)                                                 # map foreign key id
      self.store_run_attributes(data=new_run_attr_data)                                                               # store run attributes
      self.commit_session()                                                                                # save changes to database
    except:
      self.rollback_session()
      raise

  def divide_data_to_table_and_attribute(self, data, required_column='run_igf_id', attribute_name_column='attribute_name', attribute_value_column='attribute_value'):
    '''
    A method for separating data for Run and Run_attribute tables
    required params:
    required_column: column name to add to the attribute data
    attribute_name_column: label for attribute name column
    attribute_value_column: label for attribute value column

    It returns two pandas dataframes, one for Run and another for Run_attribute table

    '''
    if not isinstance(data, pd.DataFrame):
      data=pd.DataFrame(data)

    run_columns=self.get_table_columns(table_name=Run, excluded_columns=['run_id', 'seqrun_id'])                      # get required columns for run table
    (run_df, run_attr_df)=super(RunAdaptor, self).divide_data_to_table_and_attribute( \
                                                      data=data, \
    	                                              required_column=required_column, \
    	                                              table_columns=project_columns,  \
                                                      attribute_name_column=attribute_name_column, \
                                                      attribute_value_column=attribute_value_column \
                                                    )                                                                 # divide data to run and attribute table
    return (run_df, run_attr_df)

  def store_run_data(self, data):
    '''
    Load data to Run table
    '''
    try:
      self.store_records(table=Run, data=data)
    except:
      raise

  def store_run_attributes(self, data, run_id=''):
    '''
    A method for storing data to Run_attribute table
    '''
    try:
      self.store_attributes(attribute_table=Run_attribute, linked_column='run_id', db_id=run_id, data=data)
    except:
      raise

  def fetch_run_records_igf_id(self, run_igf_id, target_column_name='run_igf_id'):
    '''
    A method for fetching data for Run table
    required params:
    run_igf_id: an igf id
    target_column_name: a column name, default run_igf_id
    '''
    try:
      column=[column for column in Run.__table__.columns \
                       if column.key == target_column_name][0]
      run=self.fetch_records_by_column(table=Run, \
      	                                   column_name=column, \
      	                                   column_id=run_igf_id, \
      	                                   output_mode='one')
      return run
    except:
      raise

<commit_msg>fixing run attribute store in adaptor<commit_after>import json
import pandas as pd
from sqlalchemy.sql import table, column
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.useradaptor import ExperimentAdaptor
from igf_data.igfdb.igfTables import Experiment, Run, Run_attribute, Seqrun

class RunAdaptor(BaseAdaptor):
  '''
   An adaptor class for Run and Run_attribute tables
  '''

  def store_run_and_attribute_data(self, data):
    '''
    A method for dividing and storing data to run and attribute table
    '''
    (run_data, run_attr_data)=self.divide_data_to_table_and_attribute(data=data)

    try:
      seqrun_map_function=lambda x: self.map_foreign_table_and_store_attribute( \
                                             data=x, \
                                             lookup_table=Seqrun, \
                                             lookup_column_name='seqrun_igf_id', \
                                             target_column_name='seqrun_id')              # prepare seqrun mapping function
      new_run_data=run_data.apply(seqrun_map_function, axis=1)                            # map seqrun id
      exp_map_function=lambda x: self.map_foreign_table_and_store_attribute(\
                                             data=x, \
                                             lookup_table=Experiment, \
                                             lookup_column_name='experiment_igf_id', \
                                             target_column_name='experiment_id')          # prepare experiment mapping function
      new_run_data=new_run_data.apply(exp_map_function, axis=1)                           # map experiment id
      self.store_run_data(data=new_run_data)                                              # store run
      run_map_function=lambda x: self.map_foreign_table_and_store_attribute(\
                                             data=x, \
                                             lookup_table=Run, \
                                             lookup_column_name='run_igf_id', \
                                             target_column_name='run_id')                 # prepare run mapping function
      new_run_attr_data=run_attr_data.apply(run_map_function, axis=1)
      self.store_run_attributes(data=new_run_attr_data)                                   # store run attributes
      self.commit_session()                                                               # save changes to database
    except:
      self.rollback_session()
      raise

  def divide_data_to_table_and_attribute(self, data, required_column='run_igf_id', attribute_name_column='attribute_name', attribute_value_column='attribute_value'):
    '''
    A method for separating data for Run and Run_attribute tables
    required params:
    required_column: column name to add to the attribute data
    attribute_name_column: label for attribute name column
    attribute_value_column: label for attribute value column

    It returns two pandas dataframes, one for Run and another for Run_attribute table

    '''
    if not isinstance(data, pd.DataFrame):
      data=pd.DataFrame(data)

    run_columns=self.get_table_columns(table_name=Run, excluded_columns=['run_id', 'seqrun_id', 'experiment_id'])     # get required columns for run table
    (run_df, run_attr_df)=super(RunAdaptor, self).divide_data_to_table_and_attribute( \
                                                      data=data, \
    	                                              required_column=required_column, \
    	                                              table_columns=project_columns,  \
                                                      attribute_name_column=attribute_name_column, \
                                                      attribute_value_column=attribute_value_column \
                                                    )                                                                 # divide data to run and attribute table
    return (run_df, run_attr_df)

  def store_run_data(self, data):
    '''
    Load data to Run table
    '''
    try:
      self.store_records(table=Run, data=data)
    except:
      raise

  def store_run_attributes(self, data, run_id=''):
    '''
    A method for storing data to Run_attribute table
    '''
    try:
      self.store_attributes(attribute_table=Run_attribute, linked_column='run_id', db_id=run_id, data=data)
    except:
      raise

  def fetch_run_records_igf_id(self, run_igf_id, target_column_name='run_igf_id'):
    '''
    A method for fetching data for Run table
    required params:
    run_igf_id: an igf id
    target_column_name: a column name, default run_igf_id
    '''
    try:
      column=[column for column in Run.__table__.columns \
                       if column.key == target_column_name][0]
      run=self.fetch_records_by_column(table=Run, \
      	                                   column_name=column, \
      	                                   column_id=run_igf_id, \
      	                                   output_mode='one')
      return run
    except:
      raise

<|endoftext|>


<commit_before>import numpy as np
from sympy import Add, Expr, Mul, Symbol, preorder_traversal

from ...utils import flatten_list

class StrongForm(object):

    def __init__(self, eqn):
        self.eqn = eqn

    def _find_obj(self, node, func, return_first=False):
        ret_set = set()
        if func(node):
            ret_set.add(node)
        if return_first and len(ret_set):
            return ret_set
        for arg in node.args:
            ret_set.update(self._find_obj(arg, func, return_first))
            if return_first and len(ret_set):
                return ret_set
        return ret_set

    def _find_obj_by_name(self, node, name):
        """Returns the first object with name"""
        func = lambda n: hasattr(n, "name") and n.name == name
        ret = self._find_obj(node, func, return_first=True)
        if len(ret):
            ret = ret.pop()
        else:
            ret = None
        return ret

    def _find_obj_by_type(self, node, type):
        """Returns all variables in node of given type"""
        func = lambda n: isinstance(n, type)
        return self._find_obj(node, func)

    def __getattr__(self, key):
        obj = self._find_obj_by_name(self.eqn, key)
        if obj is None:
            raise AttributeError("%s not found in 'StrongForm' or %s" % (key, self.eqn))
        return obj

    def __setattr__(self, key, val):
        if key != "eqn":
            try:
                obj = self._find_obj_by_name(self.eqn, key)
                if hasattr(obj, "_set"):
                    obj._set(val)
                    return
            except AttributeError:
                pass
        super(StrongForm, self).__setattr__(key, val)

    def variables(self):
        return self._find_obj_by_type(self.eqn, Variable)

    def separate_by_order(self):
        ret_dict = {}

        #TODO: Pretty gorpy, should probably use a dynamic programming solution
        def _order_visitor(node):
            if isinstance(node, Add):
                return max(map(_order_visitor, node.args))
            elif isinstance(node, Mul):
                return sum(map(_order_visitor, node.args))
            elif isinstance(node, Operator):
                return _order_visitor(node.args[0]) + node.differential_order
            else:
                return 0

        if isinstance(self.eqn, Add):
            for arg in self.eqn.args:
                order = _order_visitor(arg)
                ret_dict[order] = ret_dict.get(order, 0) + arg
        else:
            order = _order_visitor(arg)
            ret_dict[order] = self.eqn

        return ret_dict

    def _extract_advection(self, order_dict):
        first_order = order_dict.get(1, 0)
        divs = filter(lambda x: isinstance(x, div), preorder_traversal(first_order))
        div_args = map(lambda d: d.args[0], divs)
        return Add(*div_args)

    def _find_grad_coefficient(self, node):
        if isinstance(node, Add):
            return Add(*map(self._find_grad_coefficient, node.args))
        if isinstance(node, Mul):
            grad_pos = []
            for i in range(len(node.args)):
                if isinstance(node.args[i], grad):
                    grad_pos.append(i)
            if grad_pos:
                ret_mul = 1
                for j in range(len(node.args)):
                    if j not in grad_pos:
                        ret_mul *= node.args[j]
                return ret_mul
            return 0
        else:
            return 0

    def _find_grad_args(self, node):
        ret_val = 0
        if isinstance(node, Add):
            ret_val = filter(lambda x: x != 0, map(self._find_grad_args, node.args))
        elif isinstance(node, Mul):
            grads = filter(lambda n: isinstance(n, grad), node.args)
            if len(grads) == 1:
                ret_val = self._find_grad_args(grads[0])
            elif len(grads) > 1:
                raise RuntimeError("Don't know how to deal with grad(x)grad(y): given %s" % node)
        elif isinstance(node, grad):
            ret_val = node.args[0]
        return ret_val

    def _is_div_grad(self, node):
        is_div = isinstance(node, div)
        div_grad = False
        if is_div:
            for arg in preorder_traversal(node.args[0]):
                if isinstance(arg, grad):
                    div_grad = True
                    break
        return div_grad

    def _extract_diffusion(self, order_dict):
        second_order = order_dict.get(2, 0)
        div_grads = filter(self._is_div_grad, preorder_traversal(second_order))
        div_grad_args = map(lambda d: d.args[0], div_grads)
        grads = map(self._find_grad_coefficient, div_grad_args)
#        import pdb; pdb.set_trace()
        return Add(*list(grads))

    def _extract_hamiltonian(self, order_dict):
        raise NotImplementedError()
        return

    def _extract_potential(self, order_dict):
        second_order = order_dict.get(2, 0)
        div_grads = filter(self._is_div_grad, preorder_traversal(second_order))
        div_grad_args = map(lambda d: d.args[0], div_grads)
        potentials = flatten_list((map(self._find_grad_args, div_grad_args)))
        if len(potentials) == 1:
            potentials = potentials[0]
        return potentials

    def _extract_mass(self, order_dict):

        def _mass_visitor(node):
            if isinstance(node, Add):
                return Add(*map(lambda x:_mass_visitor(x, top), node.args))
            else:
                if self._find_obj_by_type(node, Dt):
                    return node
                return 0

        first_order = order_dict.get(1, 0)
        return _mass_visitor(first_order)

    def _extract_reaction(self, order_dict):
        return order_dict.get(0, 0)

    def extract_transport_coefficients(self):
        ret_dict = {}
        order_dict = self.separate_by_order()
        ret_dict["advection"] = self._extract_advection(order_dict)
        ret_dict["diffusion"] = self._extract_diffusion(order_dict)
        #ret_dict["hamiltonian"] =  self._extract_hamiltonian(order_dict)
        ret_dict["potential"] = self._extract_potential(order_dict)
        ret_dict["mass"] = self._extract_mass(order_dict)
        ret_dict["reaction"] = self._extract_reaction(order_dict)
        return ret_dict

class Variable(Symbol):
    """Represents an unknown quantity"""
    def __new__(cls, name, dim=1, space="L2"):
        obj = Symbol.__new__(cls, name)
        obj.dim = dim
        obj.space = space
        return obj

class Time(Variable):
    """Special Variable representing time"""
    def __new__(cls):
        obj = Variable.__new__(cls, 'time', space="L2")
        return obj

# Function space definition
class FunctionSpace(Symbol):
    """Represents a function space"""
    pass

# Domains
class Domain(Symbol):
    """Represents a domain"""
    pass

class Region(Domain):
    """Represents a region (or subset of a domain)"""
    pass

# Operators
class Operator(Expr):

    differential_order = 0

class div(Operator):

    differential_order = 1

    def _latex(self, *args, **kws):
        from sympy import latex
        return "\div " + latex(self.args[0])

class grad(Operator):

    differential_order = 1

class Dt(Operator):
    """Derivative with respect to time."""

    differential_order = 1

class Dx(Operator):
    """Deriviative with repect to space."""

    differential_order = 1

class Dn(Operator):
    """Deriviative along normal of boundary."""

    differential_order = 1

class curl(Operator):

    differential_order = 1

class NonLinearFunction(Operator):
    """A nonlinear function based on given variables.

    >>> u = Variable('u')
    >>> f_u = NonLinearFunction('u', order=3)
    """

    def __new__(cls, *args, **kws):
        diff_order = kws.pop("order", None)
        obj = Operator.__new__(cls, *args, **kws)
        obj._diff_order = diff_order
        return obj

    @property
    def order(self):
        return self._diff_order

# Coefficients
class Coefficient(Symbol):
    """Represents a coefficient that is evaluated at quadrature points.

    Expression that is not defined by unknowns from another SFL
    expr. Code linking to the generated code must define the named
    function for evaluation.
    """
    def __new__(cls, name, rank=None, dim=None):
        obj = Symbol.__new__(cls, name)
        obj.rank = rank
        return obj

class Constant(Coefficient):
    """Represents a constant coefficient that is evaluated at quadrature points.

    Expression that is not defined by unknowns from another SFL expr.
    """
    def __new__(cls, name, val=None, rank=None, dim=None):
        obj = Coefficient.__new__(cls, name, rank, dim)
        obj.val = np.array(val)
        return obj

    def _set(self, val):
        self.val = np.array(val)

class ChiConstant(Constant):
    """Represents a characteristic function"""
    pass

class RegionConstant(ChiConstant):
    """Represents a constant that is determined by the region of the domain"""
    pass

# Some utility functions
def _pluralize_obj_creation(obj):
    f = lambda name_str, *args, **kws: \
        map(lambda name: obj(name, *args, **kws), name_str.split(kws.get('sep', ' ')))
    f.__doc__ = "Calls %(obj_class)r on names in name_str.\n\n"\
                "See %(obj_class)r docstring for more details of args and kws" \
                % {"obj_class": obj}
    return f

Variables = _pluralize_obj_creation(Variable)
Constants = _pluralize_obj_creation(Constant)
Coefficients = _pluralize_obj_creation(Coefficient)
<commit_msg>Fixing advection extraction<commit_after>import numpy as np
from sympy import Add, Expr, Mul, Symbol, preorder_traversal

from ...utils import flatten_list

class StrongForm(object):

    def __init__(self, eqn):
        self.eqn = eqn

    def _find_obj(self, node, func, return_first=False):
        ret_set = set()
        if func(node):
            ret_set.add(node)
        if return_first and len(ret_set):
            return ret_set
        for arg in node.args:
            ret_set.update(self._find_obj(arg, func, return_first))
            if return_first and len(ret_set):
                return ret_set
        return ret_set

    def _find_obj_by_name(self, node, name):
        """Returns the first object with name"""
        func = lambda n: hasattr(n, "name") and n.name == name
        ret = self._find_obj(node, func, return_first=True)
        if len(ret):
            ret = ret.pop()
        else:
            ret = None
        return ret

    def _find_obj_by_type(self, node, type):
        """Returns all variables in node of given type"""
        func = lambda n: isinstance(n, type)
        return self._find_obj(node, func)

    def __getattr__(self, key):
        obj = self._find_obj_by_name(self.eqn, key)
        if obj is None:
            raise AttributeError("%s not found in 'StrongForm' or %s" % (key, self.eqn))
        return obj

    def __setattr__(self, key, val):
        if key != "eqn":
            try:
                obj = self._find_obj_by_name(self.eqn, key)
                if hasattr(obj, "_set"):
                    obj._set(val)
                    return
            except AttributeError:
                pass
        super(StrongForm, self).__setattr__(key, val)

    def variables(self):
        return self._find_obj_by_type(self.eqn, Variable)

    def separate_by_order(self):
        ret_dict = {}

        #TODO: Pretty gorpy, should probably use a dynamic programming solution
        def _order_visitor(node):
            if isinstance(node, Add):
                return max(map(_order_visitor, node.args))
            elif isinstance(node, Mul):
                return sum(map(_order_visitor, node.args))
            elif isinstance(node, Operator):
                return _order_visitor(node.args[0]) + node.differential_order
            else:
                return 0

        if isinstance(self.eqn, Add):
            for arg in self.eqn.args:
                order = _order_visitor(arg)
                ret_dict[order] = ret_dict.get(order, 0) + arg
        else:
            order = _order_visitor(arg)
            ret_dict[order] = self.eqn

        return ret_dict

    def _find_grad_coefficient(self, node):
        if isinstance(node, Add):
            return Add(*map(self._find_grad_coefficient, node.args))
        if isinstance(node, Mul):
            grad_pos = []
            for i in range(len(node.args)):
                if isinstance(node.args[i], grad):
                    grad_pos.append(i)
            if grad_pos:
                ret_mul = 1
                for j in range(len(node.args)):
                    if j not in grad_pos:
                        ret_mul *= node.args[j]
                return ret_mul
            return 0
        else:
            return 0

    def _find_grad_args(self, node):
        ret_val = 0
        if isinstance(node, Add):
            ret_val = filter(lambda x: x != 0, map(self._find_grad_args, node.args))
        elif isinstance(node, Mul):
            grads = filter(lambda n: isinstance(n, grad), node.args)
            if len(grads) == 1:
                ret_val = self._find_grad_args(grads[0])
            elif len(grads) > 1:
                raise RuntimeError("Don't know how to deal with grad(x)grad(y): given %s" % node)
        elif isinstance(node, grad):
            ret_val = node.args[0]
        return ret_val

    def _is_div_grad(self, node):
        """Returns true node is a div with a grad argument inside"""
        is_div = isinstance(node, div)
        div_grad = False
        if is_div:
            for arg in preorder_traversal(node.args[0]):
                if isinstance(arg, grad):
                    div_grad = True
                    break
        return div_grad

    def _has_grad(self, node):
        """Return true if node has a grad in it"""
        return len(filter(lambda n: isinstance(n, grad), preorder_traversal(node))) > 0

    def _extract_advection(self, node):
#        import pdb; pdb.set_trace()
        divs = filter(lambda x: isinstance(x, div), preorder_traversal(node))
        div_args = map(lambda d: d.args[0], divs)
        split_div_args = []
        for arg in div_args:
            if isinstance(arg, Add):
                split_div_args.extend(arg.args)
            else:
                split_div_args.append(arg)
        div_args = filter(lambda n: not self._has_grad(n), split_div_args)
        return Add(*div_args)

    def _extract_diffusion(self, order_dict):
        second_order = order_dict.get(2, 0)
        div_grads = filter(self._is_div_grad, preorder_traversal(second_order))
        div_grad_args = map(lambda d: d.args[0], div_grads)
        grads = map(self._find_grad_coefficient, div_grad_args)
#        import pdb; pdb.set_trace()
        return Add(*list(grads))

    def _extract_hamiltonian(self, order_dict):
        raise NotImplementedError()
        return

    def _extract_potential(self, order_dict):
        second_order = order_dict.get(2, 0)
        div_grads = filter(self._is_div_grad, preorder_traversal(second_order))
        div_grad_args = map(lambda d: d.args[0], div_grads)
        potentials = flatten_list((map(self._find_grad_args, div_grad_args)))
        if len(potentials) == 1:
            potentials = potentials[0]
        return potentials

    def _extract_mass(self, order_dict):

        def _mass_visitor(node):
            if isinstance(node, Add):
                return Add(*map(lambda x:_mass_visitor(x, top), node.args))
            else:
                if self._find_obj_by_type(node, Dt):
                    return node
                return 0

        first_order = order_dict.get(1, 0)
        return _mass_visitor(first_order)

    def _extract_reaction(self, order_dict):
        return order_dict.get(0, 0)

    def extract_transport_coefficients(self):
        ret_dict = {}
        order_dict = self.separate_by_order()
        ret_dict["advection"] = self._extract_advection(self.eqn)
        ret_dict["diffusion"] = self._extract_diffusion(order_dict)
        #ret_dict["hamiltonian"] =  self._extract_hamiltonian(order_dict)
        ret_dict["potential"] = self._extract_potential(order_dict)
        ret_dict["mass"] = self._extract_mass(order_dict)
        ret_dict["reaction"] = self._extract_reaction(order_dict)
        return ret_dict

class Variable(Symbol):
    """Represents an unknown quantity"""
    def __new__(cls, name, dim=1, space="L2"):
        obj = Symbol.__new__(cls, name)
        obj.dim = dim
        obj.space = space
        return obj

class Time(Variable):
    """Special Variable representing time"""
    def __new__(cls):
        obj = Variable.__new__(cls, 'time', space="L2")
        return obj

# Function space definition
class FunctionSpace(Symbol):
    """Represents a function space"""
    pass

# Domains
class Domain(Symbol):
    """Represents a domain"""
    pass

class Region(Domain):
    """Represents a region (or subset of a domain)"""
    pass

# Operators
class Operator(Expr):

    differential_order = 0

class div(Operator):

    differential_order = 1

    def _latex(self, *args, **kws):
        from sympy import latex
        return "\div " + latex(self.args[0])

class grad(Operator):

    differential_order = 1

class Dt(Operator):
    """Derivative with respect to time."""

    differential_order = 1

class Dx(Operator):
    """Deriviative with repect to space."""

    differential_order = 1

class Dn(Operator):
    """Deriviative along normal of boundary."""

    differential_order = 1

class curl(Operator):

    differential_order = 1

class NonLinearFunction(Operator):
    """A nonlinear function based on given variables.

    >>> u = Variable('u')
    >>> f_u = NonLinearFunction('u', order=3)
    """

    def __new__(cls, *args, **kws):
        diff_order = kws.pop("order", None)
        obj = Operator.__new__(cls, *args, **kws)
        obj._diff_order = diff_order
        return obj

    @property
    def order(self):
        return self._diff_order

# Coefficients
class Coefficient(Symbol):
    """Represents a coefficient that is evaluated at quadrature points.

    Expression that is not defined by unknowns from another SFL
    expr. Code linking to the generated code must define the named
    function for evaluation.
    """
    def __new__(cls, name, rank=None, dim=None):
        obj = Symbol.__new__(cls, name)
        obj.rank = rank
        return obj

class Constant(Coefficient):
    """Represents a constant coefficient that is evaluated at quadrature points.

    Expression that is not defined by unknowns from another SFL expr.
    """
    def __new__(cls, name, val=None, rank=None, dim=None):
        obj = Coefficient.__new__(cls, name, rank, dim)
        obj.val = np.array(val)
        return obj

    def _set(self, val):
        self.val = np.array(val)

class ChiConstant(Constant):
    """Represents a characteristic function"""
    pass

class RegionConstant(ChiConstant):
    """Represents a constant that is determined by the region of the domain"""
    pass

# Some utility functions
def _pluralize_obj_creation(obj):
    f = lambda name_str, *args, **kws: \
        map(lambda name: obj(name, *args, **kws), name_str.split(kws.get('sep', ' ')))
    f.__doc__ = "Calls %(obj_class)r on names in name_str.\n\n"\
                "See %(obj_class)r docstring for more details of args and kws" \
                % {"obj_class": obj}
    return f

Variables = _pluralize_obj_creation(Variable)
Constants = _pluralize_obj_creation(Constant)
Coefficients = _pluralize_obj_creation(Coefficient)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2008-2009, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#     * Neither the name of Image Engine Design nor the names of any
#       other contributors to this software may be used to endorse or
#       promote products derived from this software without specific prior
#       written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import maya.cmds as cmds
import maya.OpenMaya as OpenMaya
import unittest, MayaUnitTest
import os.path
import IECore
import IECoreMaya

class TestParameterisedHolder( unittest.TestCase ) :

	def testNode( self ):
		""" Test ParameterisedHolderNode """
		n = cmds.createNode( "ieParameterisedHolderNode" )
		h = IECoreMaya.FnParameterisedHolder( str(n) )
		self.assert_( h )

		p = IECore.ParticleMeshOp()

		h.setParameterised( p )

		p.parameters().filename = "testValue"
		h.setNodeValue( p.parameters().filename )
		pl = h.parameterPlug( p.parameters().filename )
		v = IECoreMaya.FromMayaPlugConverter.create( pl, IECore.TypeId.StringData ).convert()
		self.assertEqual( v.value, "testValue" )

		cmds.setAttr( pl.name(), "testValue2", typ="string" )
		h.setParameterisedValue( p.parameters().filename )
		self.assertEqual( p.parameters().filename.getValue().value, "testValue2" )

	def testParameterisedHolderSetReference( self ):
		""" Test multiple references to ieParameterisedHolderSet nodes """

		nodeType = "ieParameterisedHolderSet"

		nodeName = cmds.createNode( nodeType )

		cmds.file( rename = os.path.join( os.getcwd(), "test", "IECoreMaya", "reference.ma" ) )
		scene = cmds.file( force = True, type = "mayaAscii", save = True )

		cmds.file( new = True, force = True )
		cmds.file( scene, reference = True, namespace = "ns1" )
		cmds.file( scene, reference = True, namespace = "ns2" )

		cmds.file( rename = os.path.join( os.getcwd(), "test", "IECoreMaya", "referenceMaster.ma" ) )
		masterScene = cmds.file( force = True, type = "mayaAscii", save = True )

		cmds.file( masterScene, force = True, open = True )

		nodeName1 = "ns1:" + nodeName
		nodeName2 = "ns2:" + nodeName

		l = OpenMaya.MSelectionList()
		l.add( nodeName1 )
		l.add( nodeName2 )

		node1 = OpenMaya.MObject()
		l.getDependNode( 0, node1 )
		node2 = OpenMaya.MObject()
		l.getDependNode( 1, node2 )

		fn1 = OpenMaya.MFnDependencyNode( node1 )
		fn2 = OpenMaya.MFnDependencyNode( node2 )

		self.assert_( fn1.userNode() )
		self.assert_( fn2.userNode() ) # This failure is due to a Maya bug. When referencing the same scene twice, as an optimisation Maya will duplicate existing nodes instead of creating new ones. There is a bug in MPxObjectSet::copy() which gets exercised here. Setting the environment variable MAYA_FORCE_REF_READ to 1 will disable this optimisation, however.

	def testChangeDefault( self ) :
		""" Test that changing parameter defaults is correctly reflected in Maya attributes """

		def makeOp( defaultValue ) :

			class TestOp( IECore.Op ) :

				def __init__( self ) :

					IECore.Op.__init__( self, "TestOp", "Tests stuff",
						IECore.IntParameter(
							name = "result",
							description = "",
							defaultValue = 0
						)
					)

					self.parameters().addParameters(
						[
							IECore.Color3fParameter(
								name = "c",
								description = "",
								defaultValue = defaultValue
							),
						]
					)

			return TestOp()

		n = cmds.createNode( "ieParameterisedHolderNode" )
		h = IECoreMaya.FnParameterisedHolder( str(n) )
		self.assert_( h )

		p = makeOp( IECore.Color3f( 0, 0, 0 ) )
		h.setParameterised( p )
		dv = cmds.attributeQuery ( "parm_c", node = n, listDefault = True )
		self.assertEqual( dv, [ 0, 0, 0 ] )

		p = makeOp( IECore.Color3f( 1, 1, 1 ) )
		h.setParameterised( p )
		dv = cmds.attributeQuery ( "parm_c", node = n, listDefault = True )
		self.assertEqual( dv, [ 1, 1, 1 ] )

	def tearDown( self ) :

		for f in [ "test/IECoreMaya/reference.ma" , "test/IECoreMaya/referenceMaster.ma" ] :

			if os.path.exists( f ) :

				os.remove( f )

if __name__ == "__main__":
	MayaUnitTest.TestProgram()
<commit_msg>Fixing use of deprecated parameter access method.<commit_after>##########################################################################
#
#  Copyright (c) 2008-2009, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#     * Neither the name of Image Engine Design nor the names of any
#       other contributors to this software may be used to endorse or
#       promote products derived from this software without specific prior
#       written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import maya.cmds as cmds
import maya.OpenMaya as OpenMaya
import unittest, MayaUnitTest
import os.path
import IECore
import IECoreMaya

class TestParameterisedHolder( unittest.TestCase ) :

	def testNode( self ):
		""" Test ParameterisedHolderNode """
		n = cmds.createNode( "ieParameterisedHolderNode" )
		h = IECoreMaya.FnParameterisedHolder( str(n) )
		self.assert_( h )

		p = IECore.ParticleMeshOp()

		h.setParameterised( p )

		p.parameters()["filename"] = "testValue"
		h.setNodeValue( p.parameters()["filename"] )
		pl = h.parameterPlug( p.parameters()["filename"] )
		v = IECoreMaya.FromMayaPlugConverter.create( pl, IECore.TypeId.StringData ).convert()
		self.assertEqual( v.value, "testValue" )

		cmds.setAttr( pl.name(), "testValue2", typ="string" )
		h.setParameterisedValue( p.parameters()["filename"] )
		self.assertEqual( p.parameters()["filename"].getValue().value, "testValue2" )

	def testParameterisedHolderSetReference( self ):
		""" Test multiple references to ieParameterisedHolderSet nodes """

		nodeType = "ieParameterisedHolderSet"

		nodeName = cmds.createNode( nodeType )

		cmds.file( rename = os.path.join( os.getcwd(), "test", "IECoreMaya", "reference.ma" ) )
		scene = cmds.file( force = True, type = "mayaAscii", save = True )

		cmds.file( new = True, force = True )
		cmds.file( scene, reference = True, namespace = "ns1" )
		cmds.file( scene, reference = True, namespace = "ns2" )

		cmds.file( rename = os.path.join( os.getcwd(), "test", "IECoreMaya", "referenceMaster.ma" ) )
		masterScene = cmds.file( force = True, type = "mayaAscii", save = True )

		cmds.file( masterScene, force = True, open = True )

		nodeName1 = "ns1:" + nodeName
		nodeName2 = "ns2:" + nodeName

		l = OpenMaya.MSelectionList()
		l.add( nodeName1 )
		l.add( nodeName2 )

		node1 = OpenMaya.MObject()
		l.getDependNode( 0, node1 )
		node2 = OpenMaya.MObject()
		l.getDependNode( 1, node2 )

		fn1 = OpenMaya.MFnDependencyNode( node1 )
		fn2 = OpenMaya.MFnDependencyNode( node2 )

		self.assert_( fn1.userNode() )
		self.assert_( fn2.userNode() ) # This failure is due to a Maya bug. When referencing the same scene twice, as an optimisation Maya will duplicate existing nodes instead of creating new ones. There is a bug in MPxObjectSet::copy() which gets exercised here. Setting the environment variable MAYA_FORCE_REF_READ to 1 will disable this optimisation, however.

	def testChangeDefault( self ) :
		""" Test that changing parameter defaults is correctly reflected in Maya attributes """

		def makeOp( defaultValue ) :

			class TestOp( IECore.Op ) :

				def __init__( self ) :

					IECore.Op.__init__( self, "TestOp", "Tests stuff",
						IECore.IntParameter(
							name = "result",
							description = "",
							defaultValue = 0
						)
					)

					self.parameters().addParameters(
						[
							IECore.Color3fParameter(
								name = "c",
								description = "",
								defaultValue = defaultValue
							),
						]
					)

			return TestOp()

		n = cmds.createNode( "ieParameterisedHolderNode" )
		h = IECoreMaya.FnParameterisedHolder( str(n) )
		self.assert_( h )

		p = makeOp( IECore.Color3f( 0, 0, 0 ) )
		h.setParameterised( p )
		dv = cmds.attributeQuery ( "parm_c", node = n, listDefault = True )
		self.assertEqual( dv, [ 0, 0, 0 ] )

		p = makeOp( IECore.Color3f( 1, 1, 1 ) )
		h.setParameterised( p )
		dv = cmds.attributeQuery ( "parm_c", node = n, listDefault = True )
		self.assertEqual( dv, [ 1, 1, 1 ] )

	def tearDown( self ) :

		for f in [ "test/IECoreMaya/reference.ma" , "test/IECoreMaya/referenceMaster.ma" ] :

			if os.path.exists( f ) :

				os.remove( f )

if __name__ == "__main__":
	MayaUnitTest.TestProgram()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Build and test all of the Zephyr boards.

This is the entry point for the custom firmware builder workflow recipe.
"""

import argparse
import multiprocessing
import pathlib
import subprocess
import sys

from google.protobuf import json_format  # pylint: disable=import-error
import zmake.project

from chromite.api.gen_sdk.chromite.api import firmware_pb2

DEFAULT_BUNDLE_DIRECTORY = '/tmp/artifact_bundles'
DEFAULT_BUNDLE_METADATA_FILE = '/tmp/artifact_bundle_metadata'

def build(opts):
    """Builds all Zephyr firmware targets"""
    metric_list = firmware_pb2.FwBuildMetricList()

    cmd = ['zmake', '-D', 'build', '-a']
    if opts.code_coverage:
        cmd.append('--coverage')
    subprocess.run(cmd, cwd=pathlib.Path(__file__).parent, check=True)
    if not opts.code_coverage:
        zephyr_dir = pathlib.Path(__file__).parent
        platform_ec = zephyr_dir.resolve().parent
        for project in zmake.project.find_projects(zephyr_dir).values():
            if project.config.is_test:
                continue
            build_dir = (
                platform_ec / 'build' / 'zephyr' / project.config.project_name
            )
            metric = metric_list.value.add()
            metric.target_name = project.config.project_name
            metric.platform_name = project.config.zephyr_board
            for (variant, _) in project.iter_builds():
                build_log = build_dir / f'build-{variant}' / 'build.log'
                parse_buildlog(build_log, metric, variant.upper())
    with open(opts.metrics, 'w') as file:
        file.write(json_format.MessageToJson(metric_list))
    return 0

UNITS = {
    'B': 1,
    'KB': 1024,
    'MB': 1024 * 1024,
    'GB': 1024 * 1024 * 1024,
}

def parse_buildlog(filename, metric, variant):
    """Parse the build.log generated by zmake to find the size of the image."""
    with open(filename, 'r') as infile:
        # Skip over all lines until the memory report is found
        while True:
            line = infile.readline()
            if not line:
                return
            if line.startswith('Memory region'):
                break

        for line in infile.readlines():
            # Skip any lines that are not part of the report
            if not line.startswith(' '):
                continue
            parts = line.split()
            fw_section = metric.fw_section.add()
            fw_section.region = variant + '_' + parts[0][:-1]
            fw_section.used = int(parts[1]) * UNITS[parts[2]]
            fw_section.total = int(parts[3]) * UNITS[parts[4]]
            fw_section.track_on_gerrit = False

def bundle(opts):
    """Bundle the artifacts for either firmware or code coverage."""
    if opts.code_coverage:
        bundle_coverage(opts)
    else:
        bundle_firmware(opts)

def get_bundle_dir(opts):
    """Get the directory for the bundle from opts or use the default.

    Also create the directory if it doesn't exist.
    """
    if opts.output_dir:
        bundle_dir = opts.output_dir
    else:
        bundle_dir = DEFAULT_BUNDLE_DIRECTORY
    bundle_dir = pathlib.Path(bundle_dir)
    if not bundle_dir.is_dir():
        bundle_dir.mkdir()
    return bundle_dir

def write_metadata(opts, info):
    """Write the metadata about the bundle."""
    bundle_metadata_file = (
        opts.metadata if opts.metadata else DEFAULT_BUNDLE_METADATA_FILE
    )
    with open(bundle_metadata_file, 'w') as file:
        file.write(json_format.MessageToJson(info))

def bundle_coverage(opts):
    """Bundles the artifacts from code coverage into its own tarball."""
    info = firmware_pb2.FirmwareArtifactInfo()
    info.bcs_version_info.version_string = opts.bcs_version
    bundle_dir = get_bundle_dir(opts)
    zephyr_dir = pathlib.Path(__file__).parent
    platform_ec = zephyr_dir.resolve().parent
    build_dir = platform_ec / 'build' / 'zephyr'
    tarball_name = 'coverage.tbz2'
    tarball_path = bundle_dir / tarball_name
    cmd = ['tar', 'cvfj', tarball_path, 'lcov.info']
    subprocess.run(cmd, cwd=build_dir, check=True)
    meta = info.objects.add()
    meta.file_name = tarball_name
    meta.lcov_info.type = (
        firmware_pb2.FirmwareArtifactInfo.LcovTarballInfo.LcovType.LCOV
    )

    write_metadata(opts, info)

def bundle_firmware(opts):
    """Bundles the artifacts from each target into its own tarball."""
    info = firmware_pb2.FirmwareArtifactInfo()
    info.bcs_version_info.version_string = opts.bcs_version
    bundle_dir = get_bundle_dir(opts)
    zephyr_dir = pathlib.Path(__file__).parent
    platform_ec = zephyr_dir.resolve().parent
    for project in zmake.project.find_projects(zephyr_dir).values():
        if project.config.is_test:
            continue
        build_dir = (
            platform_ec / 'build' / 'zephyr' / project.config.project_name
        )
        artifacts_dir = build_dir / 'output'
        tarball_name = f'{project.config.project_name}.firmware.tbz2'
        tarball_path = bundle_dir.joinpath(tarball_name)
        cmd = ['tar', 'cvfj', tarball_path, '.']
        subprocess.run(cmd, cwd=artifacts_dir, check=True)
        meta = info.objects.add()
        meta.file_name = tarball_name
        meta.tarball_info.type = (
            firmware_pb2.FirmwareArtifactInfo.TarballInfo.FirmwareType.EC
        )
        # TODO(kmshelton): Populate the rest of metadata contents as it
        # gets defined in infra/proto/src/chromite/api/firmware.proto.

    write_metadata(opts, info)

def test(opts):
    """Runs all of the unit tests for Zephyr firmware"""
    # TODO(b/169178847): Add appropriate metric information
    metrics = firmware_pb2.FwTestMetricList()
    with open(opts.metrics, 'w') as file:
        file.write(json_format.MessageToJson(metrics))

    zephyr_dir = pathlib.Path(__file__).parent.resolve()

    # Run zmake tests to ensure we have a fully working zmake before
    # proceeding.
    subprocess.run([zephyr_dir / 'zmake' / 'run_tests.sh'], check=True)

    # Run formatting checks on all BUILD.py files.
    config_files = zephyr_dir.rglob('**/BUILD.py')
    subprocess.run(['black', '--diff', '--check', *config_files], check=True)

    cmd = ['zmake', '-D', 'test', '-a', '--no-rebuild']
    if opts.code_coverage:
        cmd.append('--coverage')
    ret = subprocess.run(cmd, check=True).returncode
    if ret:
        return ret
    if opts.code_coverage:
        platform_ec = zephyr_dir.parent
        build_dir = platform_ec / 'build' / 'zephyr'
        # Merge lcov files here because bundle failures are "infra" failures.
        cmd = [
            '/usr/bin/lcov',
            '-o',
            build_dir / 'fullpaths.info',
            '--rc',
            'lcov_branch_coverage=1',
            '-a',
            build_dir / 'all_tests.info',
            '-a',
            build_dir / 'all_builds.info',
        ]
        subprocess.run(cmd, cwd=pathlib.Path(__file__).parent, check=True)
        # Make filenames relative to platform/ec
        cmd = ['sed', '-e', 's|^SF:.*/platform/ec/|SF:|']
        with open(build_dir / 'fullpaths.info') as infile, open(
                build_dir / 'lcov.info', 'w'
        ) as outfile:
            subprocess.run(
                cmd,
                cwd=pathlib.Path(__file__).parent,
                stdin=infile,
                stdout=outfile,
                check=True,
            )

    return 0

def main(args):
    """Builds and tests all of the Zephyr targets and reports build metrics"""
    opts = parse_args(args)

    if not hasattr(opts, 'func'):
        print('Must select a valid sub command!')
        return -1

    # Run selected sub command function
    return opts.func(opts)

def parse_args(args):
    """Parse command line args."""
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument(
        '--cpus',
        default=multiprocessing.cpu_count(),
        help='The number of cores to use.',
    )

    parser.add_argument(
        '--metrics',
        dest='metrics',
        required=True,
        help='File to write the json-encoded MetricsList proto message.',
    )

    parser.add_argument(
        '--metadata',
        required=False,
        help=(
            'Full pathname for the file in which to write build artifact '
            'metadata.'
        ),
    )

    parser.add_argument(
        '--output-dir',
        required=False,
        help=(
            'Full pathname for the directory in which to bundle build '
            'artifacts.',
        )
    )

    parser.add_argument(
        '--code-coverage',
        required=False,
        action='store_true',
        help='Build host-based unit tests for code coverage.',
    )

    parser.add_argument(
        '--bcs-version',
        dest='bcs_version',
        default='',
        required=False,
        # TODO(b/180008931): make this required=True.
        help='BCS version to include in metadata.',
    )

    # Would make this required=True, but not available until 3.7
    sub_cmds = parser.add_subparsers()

    build_cmd = sub_cmds.add_parser('build', help='Builds all firmware targets')
    build_cmd.set_defaults(func=build)

    build_cmd = sub_cmds.add_parser(
        'bundle',
        help='Creates a tarball containing build '
        'artifacts from all firmware targets',
    )
    build_cmd.set_defaults(func=bundle)

    test_cmd = sub_cmds.add_parser('test', help='Runs all firmware unit tests')
    test_cmd.set_defaults(func=test)

    return parser.parse_args(args)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<commit_msg>zephyr: Export coverage metrics in CQ build<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Build and test all of the Zephyr boards.

This is the entry point for the custom firmware builder workflow recipe.
"""

import argparse
import multiprocessing
import pathlib
import re
import subprocess
import sys

from google.protobuf import json_format  # pylint: disable=import-error
import zmake.project

from chromite.api.gen_sdk.chromite.api import firmware_pb2

DEFAULT_BUNDLE_DIRECTORY = '/tmp/artifact_bundles'
DEFAULT_BUNDLE_METADATA_FILE = '/tmp/artifact_bundle_metadata'

def build(opts):
    """Builds all Zephyr firmware targets"""
    metric_list = firmware_pb2.FwBuildMetricList()

    cmd = ['zmake', '-D', 'build', '-a']
    if opts.code_coverage:
        cmd.append('--coverage')
    subprocess.run(cmd, cwd=pathlib.Path(__file__).parent, check=True)
    if not opts.code_coverage:
        zephyr_dir = pathlib.Path(__file__).parent
        platform_ec = zephyr_dir.resolve().parent
        for project in zmake.project.find_projects(zephyr_dir).values():
            if project.config.is_test:
                continue
            build_dir = (
                platform_ec / 'build' / 'zephyr' / project.config.project_name
            )
            metric = metric_list.value.add()
            metric.target_name = project.config.project_name
            metric.platform_name = project.config.zephyr_board
            for (variant, _) in project.iter_builds():
                build_log = build_dir / f'build-{variant}' / 'build.log'
                parse_buildlog(build_log, metric, variant.upper())
    with open(opts.metrics, 'w') as file:
        file.write(json_format.MessageToJson(metric_list))
    return 0

UNITS = {
    'B': 1,
    'KB': 1024,
    'MB': 1024 * 1024,
    'GB': 1024 * 1024 * 1024,
}

def parse_buildlog(filename, metric, variant):
    """Parse the build.log generated by zmake to find the size of the image."""
    with open(filename, 'r') as infile:
        # Skip over all lines until the memory report is found
        while True:
            line = infile.readline()
            if not line:
                return
            if line.startswith('Memory region'):
                break

        for line in infile.readlines():
            # Skip any lines that are not part of the report
            if not line.startswith(' '):
                continue
            parts = line.split()
            fw_section = metric.fw_section.add()
            fw_section.region = variant + '_' + parts[0][:-1]
            fw_section.used = int(parts[1]) * UNITS[parts[2]]
            fw_section.total = int(parts[3]) * UNITS[parts[4]]
            fw_section.track_on_gerrit = False

def bundle(opts):
    """Bundle the artifacts for either firmware or code coverage."""
    if opts.code_coverage:
        bundle_coverage(opts)
    else:
        bundle_firmware(opts)

def get_bundle_dir(opts):
    """Get the directory for the bundle from opts or use the default.

    Also create the directory if it doesn't exist.
    """
    if opts.output_dir:
        bundle_dir = opts.output_dir
    else:
        bundle_dir = DEFAULT_BUNDLE_DIRECTORY
    bundle_dir = pathlib.Path(bundle_dir)
    if not bundle_dir.is_dir():
        bundle_dir.mkdir()
    return bundle_dir

def write_metadata(opts, info):
    """Write the metadata about the bundle."""
    bundle_metadata_file = (
        opts.metadata if opts.metadata else DEFAULT_BUNDLE_METADATA_FILE
    )
    with open(bundle_metadata_file, 'w') as file:
        file.write(json_format.MessageToJson(info))

def bundle_coverage(opts):
    """Bundles the artifacts from code coverage into its own tarball."""
    info = firmware_pb2.FirmwareArtifactInfo()
    info.bcs_version_info.version_string = opts.bcs_version
    bundle_dir = get_bundle_dir(opts)
    zephyr_dir = pathlib.Path(__file__).parent
    platform_ec = zephyr_dir.resolve().parent
    build_dir = platform_ec / 'build' / 'zephyr'
    tarball_name = 'coverage.tbz2'
    tarball_path = bundle_dir / tarball_name
    cmd = ['tar', 'cvfj', tarball_path, 'lcov.info']
    subprocess.run(cmd, cwd=build_dir, check=True)
    meta = info.objects.add()
    meta.file_name = tarball_name
    meta.lcov_info.type = (
        firmware_pb2.FirmwareArtifactInfo.LcovTarballInfo.LcovType.LCOV
    )

    write_metadata(opts, info)

def bundle_firmware(opts):
    """Bundles the artifacts from each target into its own tarball."""
    info = firmware_pb2.FirmwareArtifactInfo()
    info.bcs_version_info.version_string = opts.bcs_version
    bundle_dir = get_bundle_dir(opts)
    zephyr_dir = pathlib.Path(__file__).parent
    platform_ec = zephyr_dir.resolve().parent
    for project in zmake.project.find_projects(zephyr_dir).values():
        if project.config.is_test:
            continue
        build_dir = (
            platform_ec / 'build' / 'zephyr' / project.config.project_name
        )
        artifacts_dir = build_dir / 'output'
        tarball_name = f'{project.config.project_name}.firmware.tbz2'
        tarball_path = bundle_dir.joinpath(tarball_name)
        cmd = ['tar', 'cvfj', tarball_path, '.']
        subprocess.run(cmd, cwd=artifacts_dir, check=True)
        meta = info.objects.add()
        meta.file_name = tarball_name
        meta.tarball_info.type = (
            firmware_pb2.FirmwareArtifactInfo.TarballInfo.FirmwareType.EC
        )
        # TODO(kmshelton): Populate the rest of metadata contents as it
        # gets defined in infra/proto/src/chromite/api/firmware.proto.

    write_metadata(opts, info)

def test(opts):
    """Runs all of the unit tests for Zephyr firmware"""
    metrics = firmware_pb2.FwTestMetricList()

    zephyr_dir = pathlib.Path(__file__).parent.resolve()

    # Run zmake tests to ensure we have a fully working zmake before
    # proceeding.
    subprocess.run([zephyr_dir / 'zmake' / 'run_tests.sh'], check=True)

    # Run formatting checks on all BUILD.py files.
    config_files = zephyr_dir.rglob('**/BUILD.py')
    subprocess.run(['black', '--diff', '--check', *config_files], check=True)

    cmd = ['zmake', '-D', 'test', '-a', '--no-rebuild']
    if opts.code_coverage:
        cmd.append('--coverage')
    ret = subprocess.run(cmd, check=True).returncode
    if ret:
        return ret
    if opts.code_coverage:
        platform_ec = zephyr_dir.parent
        build_dir = platform_ec / 'build' / 'zephyr'
        # Merge lcov files here because bundle failures are "infra" failures.
        cmd = [
            '/usr/bin/lcov',
            '-o',
            build_dir / 'fullpaths.info',
            '--rc',
            'lcov_branch_coverage=1',
            '-a',
            build_dir / 'all_tests.info',
            '-a',
            build_dir / 'all_builds.info',
        ]
        output = subprocess.run(
            cmd, cwd=pathlib.Path(__file__).parent, check=True,
            stdout=subprocess.PIPE, universal_newlines=True).stdout
        _extract_lcov_summary('EC_ZEPHYR_MERGED', metrics, output)

        output = subprocess.run(
            ['/usr/bin/lcov', '--summary', build_dir / 'all_tests.info'],
            cwd=pathlib.Path(__file__).parent, check=True,
            stdout=subprocess.PIPE, universal_newlines=True).stdout
        _extract_lcov_summary('EC_ZEPHYR_TESTS', metrics, output)
        # Make filenames relative to platform/ec
        cmd = ['sed', '-e', 's|^SF:.*/platform/ec/|SF:|']
        with open(build_dir / 'fullpaths.info') as infile, open(
                build_dir / 'lcov.info', 'w'
        ) as outfile:
            subprocess.run(
                cmd,
                cwd=pathlib.Path(__file__).parent,
                stdin=infile,
                stdout=outfile,
                check=True,
            )

    with open(opts.metrics, 'w') as file:
        file.write(json_format.MessageToJson(metrics))
    return 0

COVERAGE_RE = re.compile(r'lines\.*: *([0-9\.]+)% \(([0-9]+) of ([0-9]+) lines\)')
def _extract_lcov_summary(name, metrics, output):
    re_match = COVERAGE_RE.search(output)
    if re_match:
        metric = metrics.value.add()
        metric.name = name
        metric.coverage_percent = float(re_match.group(1))
        metric.covered_lines = int(re_match.group(2))
        metric.total_lines = int(re_match.group(3))

def main(args):
    """Builds and tests all of the Zephyr targets and reports build metrics"""
    opts = parse_args(args)

    if not hasattr(opts, 'func'):
        print('Must select a valid sub command!')
        return -1

    # Run selected sub command function
    return opts.func(opts)

def parse_args(args):
    """Parse command line args."""
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument(
        '--cpus',
        default=multiprocessing.cpu_count(),
        help='The number of cores to use.',
    )

    parser.add_argument(
        '--metrics',
        dest='metrics',
        required=True,
        help='File to write the json-encoded MetricsList proto message.',
    )

    parser.add_argument(
        '--metadata',
        required=False,
        help=(
            'Full pathname for the file in which to write build artifact '
            'metadata.'
        ),
    )

    parser.add_argument(
        '--output-dir',
        required=False,
        help=(
            'Full pathname for the directory in which to bundle build '
            'artifacts.'
        ),
    )

    parser.add_argument(
        '--code-coverage',
        required=False,
        action='store_true',
        help='Build host-based unit tests for code coverage.',
    )

    parser.add_argument(
        '--bcs-version',
        dest='bcs_version',
        default='',
        required=False,
        # TODO(b/180008931): make this required=True.
        help='BCS version to include in metadata.',
    )

    # Would make this required=True, but not available until 3.7
    sub_cmds = parser.add_subparsers()

    build_cmd = sub_cmds.add_parser('build', help='Builds all firmware targets')
    build_cmd.set_defaults(func=build)

    build_cmd = sub_cmds.add_parser(
        'bundle',
        help='Creates a tarball containing build '
        'artifacts from all firmware targets',
    )
    build_cmd.set_defaults(func=bundle)

    test_cmd = sub_cmds.add_parser('test', help='Runs all firmware unit tests')
    test_cmd.set_defaults(func=test)

    return parser.parse_args(args)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
<|endoftext|>