<commit_before>from __future__ import print_function, division
import pandas as pd
import numpy as np
import json
from datetime import datetime
from ..appliance import ApplianceID
from ..utils import find_nearest, container_to_string
from ..feature_detectors import cluster
from ..timeframe import merge_timeframes, list_of_timeframe_dicts, TimeFrame
from ..preprocessing import Clip

# Fix the seed for repeatability of experiments
SEED = 42
np.random.seed(SEED)

class CombinatorialOptimisation(object):

    """1 dimensional combinatorial optimisation NILM algorithm.

    Attributes
    ----------
    model : list of dicts
       Each dict has these keys:
           states : list of ints (the power (Watts) used in different states)
           training_metadata : ElecMeter or MeterGroup object used for training
               this set of states.  We need this information because we
               need the appliance type (and perhaps some other metadata)
               for each model.
    """

    def __init__(self):
        self.model = []

    def train(self, metergroup):
        """Train using 1D CO. Places the learnt model in the `model` attribute.

        Parameters
        ----------
        metergroup : a nilmtk.MeterGroup object

        Notes
        -----
        * only uses first chunk for each meter (TODO: handle all chunks).
        """

        if self.model:
            raise RuntimeError("This implementation of Combinatorial Optimisation"
                               " does not support multiple calls to `train`.")

        num_meters = len(metergroup.meters)
        if num_meters > 12:
            max_num_clusters = 2
        else:
            max_num_clusters = 3

        for i, meter in enumerate(metergroup.submeters().meters):
            print("Training model for submeter '{}'".format(meter))
            for chunk in meter.power_series(preprocessing=[Clip()]):
                states = cluster(chunk, max_num_clusters)
                self.model.append({
                    'states': states,
                    'training_metadata': meter})
                break  # TODO handle multiple chunks per appliance
        print("Done training!")

    def disaggregate(self, mains, output_datastore, **load_kwargs):
        '''Disaggregate mains according to the model learnt previously.

        Parameters
        ----------
        mains : nilmtk.ElecMeter or nilmtk.MeterGroup
        output_datastore : instance of nilmtk.DataStore subclass
            For storing power predictions from disaggregation algorithm.
        output_name : string, optional
            The `name` to use in the metadata for the `output_datastore`.
            e.g. some sort of name for this experiment.  Defaults to
            "NILMTK_CO_<date>"
        resample_seconds : number, optional
            The desired sample period in seconds.
        **load_kwargs : key word arguments
            Passed to `mains.power_series(**kwargs)`
        '''
        MIN_CHUNK_LENGTH = 100

        if not self.model:
            raise RuntimeError("The model needs to be instantiated before"
                               " calling `disaggregate`.  For example, the"
                               " model can be instantiated by running `train`.")

        # If we import sklearn at the top of the file then auto doc fails.
        from sklearn.utils.extmath import cartesian

        # sklearn produces lots of DepreciationWarnings with PyTables
        import warnings
        warnings.filterwarnings("ignore", category=DeprecationWarning)

        # Extract optional parameters from load_kwargs
        date_now = datetime.now().isoformat().split('.')[0]
        output_name = load_kwargs.pop('output_name', 'NILMTK_CO_' + date_now)
        resample_seconds = load_kwargs.pop('resample_seconds', 60)

        # Get centroids
        centroids = [model['states'] for model in self.model]
        state_combinations = cartesian(centroids)
        # state_combinations is a 2D array
        # each column is a chan
        # each row is a possible combination of power demand values e.g.
        # [[0, 0, 0, 0], [0, 0, 0, 100], [0, 0, 50, 0], [0, 0, 50, 100], ...]

        # Add vampire power to the model
        vampire_power = mains.vampire_power()
        print("vampire_power = {} watts".format(vampire_power))
        n_rows = state_combinations.shape[0]
        vampire_power_array = np.zeros((n_rows, 1)) + vampire_power
        state_combinations = np.hstack((state_combinations, vampire_power_array))

        summed_power_of_each_combination = np.sum(state_combinations, axis=1)
        # summed_power_of_each_combination is now an array where each
        # value is the total power demand for each combination of states.

        load_kwargs['sections'] = load_kwargs.pop('sections',
                                                  mains.good_sections())
        load_kwargs.setdefault('resample', True)
        load_kwargs.setdefault('sample_period', resample_seconds)
        timeframes = []
        building_path = '/building{}'.format(mains.building())
        mains_data_location = '{}/elec/meter1'.format(building_path)

        for chunk in mains.power_series(**load_kwargs):

            # Check that chunk is sensible size before resampling
            if len(chunk) < MIN_CHUNK_LENGTH:
                continue

            # Record metadata
            timeframes.append(chunk.timeframe)
            measurement = chunk.name

            # Start disaggregation
            indices_of_state_combinations, residual_power = find_nearest(
                summed_power_of_each_combination, chunk.values)

            for i, model in enumerate(self.model):
                print("Estimating power demand for '{}'".format(model['training_metadata']))
                predicted_power = state_combinations[
                    indices_of_state_combinations, i].flatten()
                cols = pd.MultiIndex.from_tuples([chunk.name])
                meter_instance = model['training_metadata'].instance()
                output_datastore.append('{}/elec/meter{}'
                                        .format(building_path, meter_instance),
                                        pd.DataFrame(predicted_power,
                                                     index=chunk.index,
                                                     columns=cols))

            # Copy mains data to disag output
            output_datastore.append(key=mains_data_location,
                                    value=pd.DataFrame(chunk, columns=cols))

        ##################################
        # Add metadata to output_datastore

        # TODO: `preprocessing_applied` for all meters
        # TODO: split this metadata code into a separate function
        # TODO: submeter measurement should probably be the mains
        #       measurement we used to train on, not the mains measurement.

        # DataSet and MeterDevice metadata:
        meter_devices = {
            'CO': {
                'model': 'CO',
                'sample_period': resample_seconds,
                'max_sample_period': resample_seconds,
                'measurements': [{
                    'physical_quantity': measurement[0],
                    'type': measurement[1]
                }]
            },
            'mains': {
                'model': 'mains',
                'sample_period': resample_seconds,
                'max_sample_period': resample_seconds,
                'measurements': [{
                    'physical_quantity': measurement[0],
                    'type': measurement[1]
                }]
            }
        }

        merged_timeframes = merge_timeframes(timeframes, gap=resample_seconds)
        total_timeframe = TimeFrame(merged_timeframes[0].start,
                                    merged_timeframes[-1].end)

        dataset_metadata = {'name': output_name, 'date': date_now,
                            'meter_devices': meter_devices,
                            'timeframe': total_timeframe.to_dict()}
        output_datastore.save_metadata('/', dataset_metadata)

        # Building metadata

        # Mains meter:
        elec_meters = {
            1: {
                'device_model': 'mains',
                'site_meter': True,
                'data_location': mains_data_location,
                'preprocessing_applied': {},  # TODO
                'statistics': {
                    'timeframe': total_timeframe.to_dict(),
                    'good_sections': list_of_timeframe_dicts(merged_timeframes)
                }
            }
        }

        # Appliances and submeters:
        appliances = []
        for model in self.model:
            meter = model['training_metadata']

            meter_instance = meter.instance()

            for app in meter.appliances:
                meters = app.metadata['meters']
                appliance = {
                    'meters': [meter_instance],
                    'type': app.identifier.type,
                    'instance': app.identifier.instance
                    # TODO this `instance` will only be correct when the
                    # model is trained on the same house as it is tested on.
                    # https://github.com/nilmtk/nilmtk/issues/194
                }
                appliances.append(appliance)

            elec_meters.update({
                meter_instance: {
                    'device_model': 'CO',
                    'submeter_of': 1,
                    'data_location': ('{}/elec/meter{}'
                                      .format(building_path, meter_instance)),
                    'preprocessing_applied': {},  # TODO
                    'statistics': {
                        'timeframe': total_timeframe.to_dict(),
                        'good_sections': list_of_timeframe_dicts(merged_timeframes)
                    }
                }
            })

            #Setting the name if it exists
            if meter.name:
                if len(meter.name)>0:
                    elec_meters[meter_instance]['name'] = meter.name

        building_metadata = {
            'instance': mains.building(),
            'elec_meters': elec_meters,
            'appliances': appliances
        }

        output_datastore.save_metadata(building_path, building_metadata)

    # TODO: fix export and import!
    # https://github.com/nilmtk/nilmtk/issues/193
    #
    # def export_model(self, filename):
    #     model_copy = {}
    #     for appliance, appliance_states in self.model.iteritems():
    #         model_copy[
    #             "{}_{}".format(appliance.name, appliance.instance)] = appliance_states
    #     j = json.dumps(model_copy)
    #     with open(filename, 'w+') as f:
    #         f.write(j)

    # def import_model(self, filename):
    #     with open(filename, 'r') as f:
    #         temp = json.loads(f.read())
    #     for appliance, centroids in temp.iteritems():
    #         appliance_name = appliance.split("_")[0].encode("ascii")
    #         appliance_instance = int(appliance.split("_")[1])
    #         appliance_name_instance = ApplianceID(
    #             appliance_name, appliance_instance)
    #         self.model[appliance_name_instance] = centroids
<commit_msg>Should fix "local variable 'measurement' reference before assignement" problem in CO. #365<commit_after>from __future__ import print_function, division
import pandas as pd
import numpy as np
import json
from datetime import datetime
from ..appliance import ApplianceID
from ..utils import find_nearest, container_to_string
from ..feature_detectors import cluster
from ..timeframe import merge_timeframes, list_of_timeframe_dicts, TimeFrame
from ..preprocessing import Clip

# Fix the seed for repeatability of experiments
SEED = 42
np.random.seed(SEED)

class CombinatorialOptimisation(object):

    """1 dimensional combinatorial optimisation NILM algorithm.

    Attributes
    ----------
    model : list of dicts
       Each dict has these keys:
           states : list of ints (the power (Watts) used in different states)
           training_metadata : ElecMeter or MeterGroup object used for training
               this set of states.  We need this information because we
               need the appliance type (and perhaps some other metadata)
               for each model.
    """

    def __init__(self):
        self.model = []

    def train(self, metergroup):
        """Train using 1D CO. Places the learnt model in the `model` attribute.

        Parameters
        ----------
        metergroup : a nilmtk.MeterGroup object

        Notes
        -----
        * only uses first chunk for each meter (TODO: handle all chunks).
        """

        if self.model:
            raise RuntimeError("This implementation of Combinatorial Optimisation"
                               " does not support multiple calls to `train`.")

        num_meters = len(metergroup.meters)
        if num_meters > 12:
            max_num_clusters = 2
        else:
            max_num_clusters = 3

        for i, meter in enumerate(metergroup.submeters().meters):
            print("Training model for submeter '{}'".format(meter))
            for chunk in meter.power_series(preprocessing=[Clip()]):
                states = cluster(chunk, max_num_clusters)
                self.model.append({
                    'states': states,
                    'training_metadata': meter})
                break  # TODO handle multiple chunks per appliance
        print("Done training!")

    def disaggregate(self, mains, output_datastore, **load_kwargs):
        '''Disaggregate mains according to the model learnt previously.

        Parameters
        ----------
        mains : nilmtk.ElecMeter or nilmtk.MeterGroup
        output_datastore : instance of nilmtk.DataStore subclass
            For storing power predictions from disaggregation algorithm.
        output_name : string, optional
            The `name` to use in the metadata for the `output_datastore`.
            e.g. some sort of name for this experiment.  Defaults to
            "NILMTK_CO_<date>"
        resample_seconds : number, optional
            The desired sample period in seconds.
        **load_kwargs : key word arguments
            Passed to `mains.power_series(**kwargs)`
        '''
        MIN_CHUNK_LENGTH = 100

        if not self.model:
            raise RuntimeError("The model needs to be instantiated before"
                               " calling `disaggregate`.  For example, the"
                               " model can be instantiated by running `train`.")

        # If we import sklearn at the top of the file then auto doc fails.
        from sklearn.utils.extmath import cartesian

        # sklearn produces lots of DepreciationWarnings with PyTables
        import warnings
        warnings.filterwarnings("ignore", category=DeprecationWarning)

        # Extract optional parameters from load_kwargs
        date_now = datetime.now().isoformat().split('.')[0]
        output_name = load_kwargs.pop('output_name', 'NILMTK_CO_' + date_now)
        resample_seconds = load_kwargs.pop('resample_seconds', 60)

        # Get centroids
        centroids = [model['states'] for model in self.model]
        state_combinations = cartesian(centroids)
        # state_combinations is a 2D array
        # each column is a chan
        # each row is a possible combination of power demand values e.g.
        # [[0, 0, 0, 0], [0, 0, 0, 100], [0, 0, 50, 0], [0, 0, 50, 100], ...]

        # Add vampire power to the model
        vampire_power = mains.vampire_power()
        print("vampire_power = {} watts".format(vampire_power))
        n_rows = state_combinations.shape[0]
        vampire_power_array = np.zeros((n_rows, 1)) + vampire_power
        state_combinations = np.hstack((state_combinations, vampire_power_array))

        summed_power_of_each_combination = np.sum(state_combinations, axis=1)
        # summed_power_of_each_combination is now an array where each
        # value is the total power demand for each combination of states.

        load_kwargs['sections'] = load_kwargs.pop('sections',
                                                  mains.good_sections())
        load_kwargs.setdefault('resample', True)
        load_kwargs.setdefault('sample_period', resample_seconds)
        timeframes = []
        building_path = '/building{}'.format(mains.building())
        mains_data_location = '{}/elec/meter1'.format(building_path)
        data_is_available = False

        for chunk in mains.power_series(**load_kwargs):

            # Check that chunk is sensible size before resampling
            if len(chunk) < MIN_CHUNK_LENGTH:
                continue

            # Record metadata
            timeframes.append(chunk.timeframe)
            measurement = chunk.name

            # Start disaggregation
            indices_of_state_combinations, residual_power = find_nearest(
                summed_power_of_each_combination, chunk.values)

            for i, model in enumerate(self.model):
                print("Estimating power demand for '{}'".format(model['training_metadata']))
                data_is_available = True
                predicted_power = state_combinations[
                    indices_of_state_combinations, i].flatten()
                cols = pd.MultiIndex.from_tuples([chunk.name])
                meter_instance = model['training_metadata'].instance()
                output_datastore.append('{}/elec/meter{}'
                                        .format(building_path, meter_instance),
                                        pd.DataFrame(predicted_power,
                                                     index=chunk.index,
                                                     columns=cols))

            # Copy mains data to disag output
            output_datastore.append(key=mains_data_location,
                                    value=pd.DataFrame(chunk, columns=cols))

        if not data_is_available:
            return

        ##################################
        # Add metadata to output_datastore

        # TODO: `preprocessing_applied` for all meters
        # TODO: split this metadata code into a separate function
        # TODO: submeter measurement should probably be the mains
        #       measurement we used to train on, not the mains measurement.

        # DataSet and MeterDevice metadata:
        meter_devices = {
            'CO': {
                'model': 'CO',
                'sample_period': resample_seconds,
                'max_sample_period': resample_seconds,
                'measurements': [{
                    'physical_quantity': measurement[0],
                    'type': measurement[1]
                }]
            },
            'mains': {
                'model': 'mains',
                'sample_period': resample_seconds,
                'max_sample_period': resample_seconds,
                'measurements': [{
                    'physical_quantity': measurement[0],
                    'type': measurement[1]
                }]
            }
        }

        merged_timeframes = merge_timeframes(timeframes, gap=resample_seconds)
        total_timeframe = TimeFrame(merged_timeframes[0].start,
                                    merged_timeframes[-1].end)

        dataset_metadata = {'name': output_name, 'date': date_now,
                            'meter_devices': meter_devices,
                            'timeframe': total_timeframe.to_dict()}
        output_datastore.save_metadata('/', dataset_metadata)

        # Building metadata

        # Mains meter:
        elec_meters = {
            1: {
                'device_model': 'mains',
                'site_meter': True,
                'data_location': mains_data_location,
                'preprocessing_applied': {},  # TODO
                'statistics': {
                    'timeframe': total_timeframe.to_dict(),
                    'good_sections': list_of_timeframe_dicts(merged_timeframes)
                }
            }
        }

        # Appliances and submeters:
        appliances = []
        for model in self.model:
            meter = model['training_metadata']

            meter_instance = meter.instance()

            for app in meter.appliances:
                meters = app.metadata['meters']
                appliance = {
                    'meters': [meter_instance],
                    'type': app.identifier.type,
                    'instance': app.identifier.instance
                    # TODO this `instance` will only be correct when the
                    # model is trained on the same house as it is tested on.
                    # https://github.com/nilmtk/nilmtk/issues/194
                }
                appliances.append(appliance)

            elec_meters.update({
                meter_instance: {
                    'device_model': 'CO',
                    'submeter_of': 1,
                    'data_location': ('{}/elec/meter{}'
                                      .format(building_path, meter_instance)),
                    'preprocessing_applied': {},  # TODO
                    'statistics': {
                        'timeframe': total_timeframe.to_dict(),
                        'good_sections': list_of_timeframe_dicts(merged_timeframes)
                    }
                }
            })

            #Setting the name if it exists
            if meter.name:
                if len(meter.name)>0:
                    elec_meters[meter_instance]['name'] = meter.name

        building_metadata = {
            'instance': mains.building(),
            'elec_meters': elec_meters,
            'appliances': appliances
        }

        output_datastore.save_metadata(building_path, building_metadata)

    # TODO: fix export and import!
    # https://github.com/nilmtk/nilmtk/issues/193
    #
    # def export_model(self, filename):
    #     model_copy = {}
    #     for appliance, appliance_states in self.model.iteritems():
    #         model_copy[
    #             "{}_{}".format(appliance.name, appliance.instance)] = appliance_states
    #     j = json.dumps(model_copy)
    #     with open(filename, 'w+') as f:
    #         f.write(j)

    # def import_model(self, filename):
    #     with open(filename, 'r') as f:
    #         temp = json.loads(f.read())
    #     for appliance, centroids in temp.iteritems():
    #         appliance_name = appliance.split("_")[0].encode("ascii")
    #         appliance_instance = int(appliance.split("_")[1])
    #         appliance_name_instance = ApplianceID(
    #             appliance_name, appliance_instance)
    #         self.model[appliance_name_instance] = centroids
<|endoftext|>


<commit_before>from .. import BaseModel, db
from syft import serialize, deserialize

from syft.proto.lib.torch.tensor_pb2 import TensorProto as TensorProto_PB

bin_to_proto = {TensorProto_PB.__name__: TensorProto_PB}

class BinObject(BaseModel):
    __tablename__ = "bin_object"

    id = db.Column(db.String(2056), primary_key=True)
    binary = db.Column(db.LargeBinary(4096))
    protobuf_name = db.Column(db.String(2056))

    @property
    def object(self):
        _proto_struct = bin_to_proto[self.protobuf_name]()
        _proto_struct.ParseFromString(self.binary)
        _obj = deserialize(blob=_proto_struct)
        return _obj

    @object.setter
    def object(self, value):
        serialized_value = serialize(value)
        self.binary = serialized_value.SerializeToString()
        self.protobuf_name = serialized_value.__class__.__name__

class ObjectMetadata(BaseModel):
    __tablename__ = "obj_metadata"

    id = db.Column(db.Integer(), primary_key=True, autoincrement=True)
    obj = db.Column(db.Integer, db.ForeignKey("bin_object.id"))
    tags = db.Column(db.JSON())
    description = db.Column(db.String())
    read_permissions = db.Column(db.JSON())
    search_permissions = db.Column(db.JSON())
<commit_msg>Update BinObject Strings<commit_after>from .. import BaseModel, db
from syft import serialize, deserialize

from syft.proto.lib.torch.tensor_pb2 import TensorProto as TensorProto_PB

bin_to_proto = {TensorProto_PB.__name__: TensorProto_PB}

class BinObject(BaseModel):
    __tablename__ = "bin_object"

    id = db.Column(db.String(3072), primary_key=True)
    binary = db.Column(db.LargeBinary(3072))
    protobuf_name = db.Column(db.String(3072))

    @property
    def object(self):
        _proto_struct = bin_to_proto[self.protobuf_name]()
        _proto_struct.ParseFromString(self.binary)
        _obj = deserialize(blob=_proto_struct)
        return _obj

    @object.setter
    def object(self, value):
        serialized_value = serialize(value)
        self.binary = serialized_value.SerializeToString()
        self.protobuf_name = serialized_value.__class__.__name__

class ObjectMetadata(BaseModel):
    __tablename__ = "obj_metadata"

    id = db.Column(db.Integer(), primary_key=True, autoincrement=True)
    obj = db.Column(db.Integer, db.ForeignKey("bin_object.id"))
    tags = db.Column(db.JSON())
    description = db.Column(db.String())
    read_permissions = db.Column(db.JSON())
    search_permissions = db.Column(db.JSON())
<|endoftext|>


<commit_before>g77_link_output = """
Driving: g77 -v empty.o -o empty -lfrtbegin -lg2c -lm -shared-libgcc
Reading specs from /usr/lib/gcc/i486-linux-gnu/3.4.6/specs
Configured with: ../src/configure -v --enable-languages=c,c++,f77,pascal --prefix=/usr --libexecdir=/usr/lib --with-gxx-include-dir=/usr/include/c++/3.4 --enable-shared --with-system-zlib --enable-nls --without-included-gettext --program-suffix=-3.4 --enable-__cxa_atexit --enable-clocale=gnu --enable-libstdcxx-debug --with-tune=pentium4 i486-linux-gnu
Thread model: posix
gcc version 3.4.6 (Ubuntu 3.4.6-6ubuntu1)
 /usr/lib/gcc/i486-linux-gnu/3.4.6/collect2 --eh-frame-hdr -m elf_i386 -dynamic-linker /lib/ld-linux.so.2 -o empty /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crt1.o /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crti.o /usr/lib/gcc/i486-linux-gnu/3.4.6/crtbegin.o -L/usr/lib/gcc/i486-linux-gnu/3.4.6 -L/usr/lib/gcc/i486-linux-gnu/3.4.6 -L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib -L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../.. -L/lib/../lib -L/usr/lib/../lib empty.o -lfrtbegin -lg2c -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/i486-linux-gnu/3.4.6/crtend.o /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crtn.o"""

g77_link_expected = ['-L/usr/lib/gcc/i486-linux-gnu/3.4.6',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../..', '-L/lib/../lib',
        '-L/usr/lib/../lib', '-lfrtbegin', '-lg2c', '-lm']

gfortran_link_output = """
Driving: gfortran -v -o hello hello.o -lgfortranbegin -lgfortran -lm -shared-libgcc
Using built-in specs.
Target: i486-linux-gnu
Configured with: ../src/configure -v --enable-languages=c,c++,fortran,objc,obj-c++,treelang --prefix=/usr --enable-shared --with-system-zlib --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --enable-nls --with-gxx-include-dir=/usr/include/c++/4.2 --program-suffix=-4.2 --enable-clocale=gnu --enable-libstdcxx-debug --enable-mpfr --enable-targets=all --enable-checking=release --build=i486-linux-gnu --host=i486-linux-gnu --target=i486-linux-gnu
Thread model: posix
gcc version 4.2.1 (Ubuntu 4.2.1-5ubuntu4)
 /usr/lib/gcc/i486-linux-gnu/4.2.1/collect2 --eh-frame-hdr -m elf_i386 --hash-style=both -dynamic-linker /lib/ld-linux.so.2 -o hello /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crt1.o /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crti.o /usr/lib/gcc/i486-linux-gnu/4.2.1/crtbegin.o -L/usr/lib/gcc/i486-linux-gnu/4.2.1 -L/usr/lib/gcc/i486-linux-gnu/4.2.1 -L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib -L/lib/../lib -L/usr/lib/../lib -L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../.. hello.o -lgfortranbegin -lgfortran -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/i486-linux-gnu/4.2.1/crtend.o /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crtn.o"""

gfortran_link_expected = ['-L/usr/lib/gcc/i486-linux-gnu/4.2.1',
        '-L/usr/lib/gcc/i486-linux-gnu/4.2.1',
        '-L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib', '-L/lib/../lib',
        '-L/usr/lib/../lib', '-L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../..',
        '-lgfortranbegin', '-lgfortran', '-lm']

sunfort_v12_link_output = """
NOTICE: Invoking /home/david/opt/sun/sunstudio12/bin/f90 -f77 -ftrap=%none -c empty.f
###     command line files and options (expanded):
    ### -f77=%all -ftrap=%none -v empty.o
    ### f90: Note: NLSPATH = /home/david/opt/sun/sunstudio12/prod/bin/../lib/locale/%L/LC_MESSAGES/%N.cat:/home/david/opt/sun/sunstudio12/prod/bin/../../lib/locale/%L/LC_MESSAGES/%N.cat
    ### f90: Note: LD_LIBRARY_PATH = /home/david/local/intel/cc/9.1.042/lib:/home/david/local/lib:
    ### f90: Note: LD_RUN_PATH     = (null)
    ### f90: Note: LD_OPTIONS = (null)
    /usr/bin/ld -m elf_i386 -dynamic-linker /lib/ld-linux.so.2 --enable-new-dtags -R/home/david/opt/sun/sunstudio12/lib:/opt/sun/sunstudio12/lib:/home/david/opt/sun/lib/rtlibs:/opt/sun/lib/rtlibs -o a.out /home/david/opt/sun/sunstudio12/prod/lib/crti.o /home/david/opt/sun/sunstudio12/prod/lib/crt1.o /home/david/opt/sun/sunstudio12/prod/lib/values-xi.o -Y P,/home/david/opt/sun/sunstudio12/lib:/home/david/opt/sun/sunstudio12/rtlibs:/home/david/opt/sun/sunstudio12/prod/lib:/lib:/usr/lib empty.o -lfui -lfai -lfsu -Bdynamic -lmtsk -lpthread -lm -lc /home/david/opt/sun/sunstudio12/prod/lib/libc_supp.a /home/david/opt/sun/sunstudio12/prod/lib/crtn.o
    empty.f:
     MAIN hello:
"""

sunfort_v12_link_expected = ['-R/home/david/opt/sun/sunstudio12/lib:'\
        '/opt/sun/sunstudio12/lib:/home/david/opt/sun/lib/rtlibs:'\
        '/opt/sun/lib/rtlibs', '-L/home/david/opt/sun/sunstudio12/lib',
        '-L/home/david/opt/sun/sunstudio12/rtlibs',
        '-L/home/david/opt/sun/sunstudio12/prod/lib', '-L/lib', '-L/usr/lib',
        '-lfui', '-lfai', '-lfsu', '-lmtsk', '-lpthread', '-lm']

ifort_v10_link_output = """
ld    /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crt1.o /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crti.o /usr/lib/gcc/i486-linux-gnu/4.1.3/crtbegin.o --eh-frame-hdr -dynamic-linker /lib/ld-linux.so.2 -m elf_i386 -o a.out /home/david/opt/intel/fc/10.0.023//lib/for_main.o empty.o -L/home/david/opt/intel/fc/10.0.023//lib -L/usr/lib/gcc/i486-linux-gnu/4.1.3/ -L/usr/lib/gcc/i486-linux-gnu/4.1.3/../../../ -Bstatic -lifport -lifcore -limf -Bdynamic -lm -Bstatic -lipgo -lirc -Bdynamic -lc -lgcc_s -lgcc -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/lib/gcc/i486-linux-gnu/4.1.3/crtend.o /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crtn.o
"""

ifort_v10_link_expected = ['-L/home/david/opt/intel/fc/10.0.023//lib',
        '-L/usr/lib/gcc/i486-linux-gnu/4.1.3/',
        '-L/usr/lib/gcc/i486-linux-gnu/4.1.3/../../../', '-lifport',
        '-lifcore', '-limf', '-lm', '-lipgo', '-lirc', '-lirc_s', '-ldl']

mingw_g77_link_output = """
['Driving: g77 -v -o build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0 build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.obj -lfrtbegin -lg2c', 'Reading specs from c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/specs', 'Configured with: ../gcc-3.4.5/configure --with-gcc --with-gnu-ld --with-gnu-as --host=mingw32 --target=mingw32 --prefix=/mingw --enable-threads --disable-nls --enable-languages=c,c++,f77,ada,objc,java --disable-win32-registry --disable-shared --enable-sjlj-exceptions --enable-libgcj --disable-java-awt --without-x --enable-java-gc=boehm --disable-libgcj-debug --enable-interpreter --enable-hash-synchronization --enable-libstdcxx-debug', 'Thread model: win32', 'gcc version 3.4.5 (mingw special)', ' c:/MINGW/bin/../libexec/gcc/mingw32/3.4.5/collect2.exe -Bdynamic -o build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.exe /mingw/lib/crt2.o c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/crtbegin.o -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5 -Lc:/MINGW/bin/../lib/gcc -L/mingw/lib/gcc/mingw32/3.4.5 -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5/../../../../mingw32/lib -L/mingw/lib/gcc/mingw32/3.4.5/../../../../mingw32/lib -L/mingw/lib -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5/../../.. -L/mingw/lib/gcc/mingw32/3.4.5/../../.. build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.obj -lfrtbegin -lg2c -lmingw32 -lgcc -lmoldname -lmingwex -lmsvcrt -luser32 -lkernel32 -ladvapi32 -lshell32 -lmingw32 -lgcc -lmoldname -lmingwex -lmsvcrt c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/crtend.o', '']"""

def generate_output(fcomp, vflag, fflag):
    from numscons.core.utils import popen_wrapper
    import sys

    if sys.platform == 'win32':
        objsuff = '.obj'
    else:
        objsuff = '.o'
    st, output = popen_wrapper('%s %s %s' % (fcomp, fflag, 'empty.f'))
    if st:
        raise RuntimeError("Compilation failed: status is %s. output is %s" %\
                           (st, output))
    st, output = popen_wrapper('%s %s %s' % (fcomp, vflag, 'empty%s' % objsuff))
    if st:
        raise RuntimeError("Link failed: status is %s. output is %s" %\
                           (st, output))
    return output

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option("-c", "--compile-flag", dest="fflag",
                      help="compiler flag (-c, etc...)")
    parser.add_option("-v", "--verbose-flag", dest="vflag",
                      help = "verbose flag (-v, etc...). This should be the "
                      "*linker* verbose flag, not the compiler one.")

    (options, args) = parser.parse_args()
    if options.fflag:
        fflag = options.fflag
    else:
        fflag = '-c'
    if options.vflag:
        vflag = options.vflag
    else:
        vflag = '-v'

    if len(args) < 1:
        raise "You should give the compiler as argument"
    fcomp = args[0]

    output = generate_output(fcomp, vflag, fflag)
    print output
<commit_msg>Use subprocess module to get reliable process spawning on win32<commit_after>g77_link_output = """
Driving: g77 -v empty.o -o empty -lfrtbegin -lg2c -lm -shared-libgcc
Reading specs from /usr/lib/gcc/i486-linux-gnu/3.4.6/specs
Configured with: ../src/configure -v --enable-languages=c,c++,f77,pascal --prefix=/usr --libexecdir=/usr/lib --with-gxx-include-dir=/usr/include/c++/3.4 --enable-shared --with-system-zlib --enable-nls --without-included-gettext --program-suffix=-3.4 --enable-__cxa_atexit --enable-clocale=gnu --enable-libstdcxx-debug --with-tune=pentium4 i486-linux-gnu
Thread model: posix
gcc version 3.4.6 (Ubuntu 3.4.6-6ubuntu1)
 /usr/lib/gcc/i486-linux-gnu/3.4.6/collect2 --eh-frame-hdr -m elf_i386 -dynamic-linker /lib/ld-linux.so.2 -o empty /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crt1.o /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crti.o /usr/lib/gcc/i486-linux-gnu/3.4.6/crtbegin.o -L/usr/lib/gcc/i486-linux-gnu/3.4.6 -L/usr/lib/gcc/i486-linux-gnu/3.4.6 -L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib -L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../.. -L/lib/../lib -L/usr/lib/../lib empty.o -lfrtbegin -lg2c -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/i486-linux-gnu/3.4.6/crtend.o /usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib/crtn.o"""

g77_link_expected = ['-L/usr/lib/gcc/i486-linux-gnu/3.4.6',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../../../lib',
        '-L/usr/lib/gcc/i486-linux-gnu/3.4.6/../../..', '-L/lib/../lib',
        '-L/usr/lib/../lib', '-lfrtbegin', '-lg2c', '-lm']

gfortran_link_output = """
Driving: gfortran -v -o hello hello.o -lgfortranbegin -lgfortran -lm -shared-libgcc
Using built-in specs.
Target: i486-linux-gnu
Configured with: ../src/configure -v --enable-languages=c,c++,fortran,objc,obj-c++,treelang --prefix=/usr --enable-shared --with-system-zlib --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --enable-nls --with-gxx-include-dir=/usr/include/c++/4.2 --program-suffix=-4.2 --enable-clocale=gnu --enable-libstdcxx-debug --enable-mpfr --enable-targets=all --enable-checking=release --build=i486-linux-gnu --host=i486-linux-gnu --target=i486-linux-gnu
Thread model: posix
gcc version 4.2.1 (Ubuntu 4.2.1-5ubuntu4)
 /usr/lib/gcc/i486-linux-gnu/4.2.1/collect2 --eh-frame-hdr -m elf_i386 --hash-style=both -dynamic-linker /lib/ld-linux.so.2 -o hello /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crt1.o /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crti.o /usr/lib/gcc/i486-linux-gnu/4.2.1/crtbegin.o -L/usr/lib/gcc/i486-linux-gnu/4.2.1 -L/usr/lib/gcc/i486-linux-gnu/4.2.1 -L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib -L/lib/../lib -L/usr/lib/../lib -L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../.. hello.o -lgfortranbegin -lgfortran -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /usr/lib/gcc/i486-linux-gnu/4.2.1/crtend.o /usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib/crtn.o"""

gfortran_link_expected = ['-L/usr/lib/gcc/i486-linux-gnu/4.2.1',
        '-L/usr/lib/gcc/i486-linux-gnu/4.2.1',
        '-L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../../../lib', '-L/lib/../lib',
        '-L/usr/lib/../lib', '-L/usr/lib/gcc/i486-linux-gnu/4.2.1/../../..',
        '-lgfortranbegin', '-lgfortran', '-lm']

sunfort_v12_link_output = """
NOTICE: Invoking /home/david/opt/sun/sunstudio12/bin/f90 -f77 -ftrap=%none -c empty.f
###     command line files and options (expanded):
    ### -f77=%all -ftrap=%none -v empty.o
    ### f90: Note: NLSPATH = /home/david/opt/sun/sunstudio12/prod/bin/../lib/locale/%L/LC_MESSAGES/%N.cat:/home/david/opt/sun/sunstudio12/prod/bin/../../lib/locale/%L/LC_MESSAGES/%N.cat
    ### f90: Note: LD_LIBRARY_PATH = /home/david/local/intel/cc/9.1.042/lib:/home/david/local/lib:
    ### f90: Note: LD_RUN_PATH     = (null)
    ### f90: Note: LD_OPTIONS = (null)
    /usr/bin/ld -m elf_i386 -dynamic-linker /lib/ld-linux.so.2 --enable-new-dtags -R/home/david/opt/sun/sunstudio12/lib:/opt/sun/sunstudio12/lib:/home/david/opt/sun/lib/rtlibs:/opt/sun/lib/rtlibs -o a.out /home/david/opt/sun/sunstudio12/prod/lib/crti.o /home/david/opt/sun/sunstudio12/prod/lib/crt1.o /home/david/opt/sun/sunstudio12/prod/lib/values-xi.o -Y P,/home/david/opt/sun/sunstudio12/lib:/home/david/opt/sun/sunstudio12/rtlibs:/home/david/opt/sun/sunstudio12/prod/lib:/lib:/usr/lib empty.o -lfui -lfai -lfsu -Bdynamic -lmtsk -lpthread -lm -lc /home/david/opt/sun/sunstudio12/prod/lib/libc_supp.a /home/david/opt/sun/sunstudio12/prod/lib/crtn.o
    empty.f:
     MAIN hello:
"""

sunfort_v12_link_expected = ['-R/home/david/opt/sun/sunstudio12/lib:'\
        '/opt/sun/sunstudio12/lib:/home/david/opt/sun/lib/rtlibs:'\
        '/opt/sun/lib/rtlibs', '-L/home/david/opt/sun/sunstudio12/lib',
        '-L/home/david/opt/sun/sunstudio12/rtlibs',
        '-L/home/david/opt/sun/sunstudio12/prod/lib', '-L/lib', '-L/usr/lib',
        '-lfui', '-lfai', '-lfsu', '-lmtsk', '-lpthread', '-lm']

ifort_v10_link_output = """
ld    /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crt1.o /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crti.o /usr/lib/gcc/i486-linux-gnu/4.1.3/crtbegin.o --eh-frame-hdr -dynamic-linker /lib/ld-linux.so.2 -m elf_i386 -o a.out /home/david/opt/intel/fc/10.0.023//lib/for_main.o empty.o -L/home/david/opt/intel/fc/10.0.023//lib -L/usr/lib/gcc/i486-linux-gnu/4.1.3/ -L/usr/lib/gcc/i486-linux-gnu/4.1.3/../../../ -Bstatic -lifport -lifcore -limf -Bdynamic -lm -Bstatic -lipgo -lirc -Bdynamic -lc -lgcc_s -lgcc -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/lib/gcc/i486-linux-gnu/4.1.3/crtend.o /usr/lib/gcc/i486-linux-gnu/4.1.3/../../../crtn.o
"""

ifort_v10_link_expected = ['-L/home/david/opt/intel/fc/10.0.023//lib',
        '-L/usr/lib/gcc/i486-linux-gnu/4.1.3/',
        '-L/usr/lib/gcc/i486-linux-gnu/4.1.3/../../../', '-lifport',
        '-lifcore', '-limf', '-lm', '-lipgo', '-lirc', '-lirc_s', '-ldl']

mingw_g77_link_output = """
['Driving: g77 -v -o build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0 build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.obj -lfrtbegin -lg2c', 'Reading specs from c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/specs', 'Configured with: ../gcc-3.4.5/configure --with-gcc --with-gnu-ld --with-gnu-as --host=mingw32 --target=mingw32 --prefix=/mingw --enable-threads --disable-nls --enable-languages=c,c++,f77,ada,objc,java --disable-win32-registry --disable-shared --enable-sjlj-exceptions --enable-libgcj --disable-java-awt --without-x --enable-java-gc=boehm --disable-libgcj-debug --enable-interpreter --enable-hash-synchronization --enable-libstdcxx-debug', 'Thread model: win32', 'gcc version 3.4.5 (mingw special)', ' c:/MINGW/bin/../libexec/gcc/mingw32/3.4.5/collect2.exe -Bdynamic -o build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.exe /mingw/lib/crt2.o c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/crtbegin.o -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5 -Lc:/MINGW/bin/../lib/gcc -L/mingw/lib/gcc/mingw32/3.4.5 -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5/../../../../mingw32/lib -L/mingw/lib/gcc/mingw32/3.4.5/../../../../mingw32/lib -L/mingw/lib -Lc:/MINGW/bin/../lib/gcc/mingw32/3.4.5/../../.. -L/mingw/lib/gcc/mingw32/3.4.5/../../.. build\\scons\\numpy\\scons_fake\\checkers\\.sconf\\conftest_0.obj -lfrtbegin -lg2c -lmingw32 -lgcc -lmoldname -lmingwex -lmsvcrt -luser32 -lkernel32 -ladvapi32 -lshell32 -lmingw32 -lgcc -lmoldname -lmingwex -lmsvcrt c:/MINGW/bin/../lib/gcc/mingw32/3.4.5/crtend.o', '']"""

def generate_output(fcomp, vflag, fflag, objsuff = '.o'):
    import sys
    from subprocess import Popen, PIPE, STDOUT

    cmd = '%s %s %s' % (fcomp, fflag, 'empty.f')
    p = Popen(cmd, shell = True)
    st = p.wait()
    if st:
        raise RuntimeError("Compilation failed: status is %s. output is %s" %\
                           (st, p.stdout.read()))
    cmd = '%s %s %s' % (fcomp, vflag, 'empty%s' % objsuff)
    p = Popen(cmd, stdout = PIPE, stderr = STDOUT, shell = True)
    st = p.wait()
    if st:
        raise RuntimeError("Link failed (cmd was %s): status is %s. output is %s" %\
                           (cmd, st, p.stdout.read()))
    output = p.stdout.read()
    return output

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option("-c", "--compile-flag", dest="fflag",
                      help="compiler flag (-c, etc...)")
    parser.add_option("-v", "--verbose-flag", dest="vflag",
                      help = "verbose flag (-v, etc...). This should be the "
                      "*linker* verbose flag, not the compiler one.")

    (options, args) = parser.parse_args()
    if options.fflag:
        fflag = options.fflag
    else:
        fflag = '-c'
    if options.vflag:
        vflag = options.vflag
    else:
        vflag = '-v'

    if len(args) < 1:
        raise "You should give the compiler as argument"
    fcomp = args[0]

    output = generate_output(fcomp, vflag, fflag)
    print output
<|endoftext|>


<commit_before>"""API to access local resources for synchronization."""

import unicodedata
from datetime import datetime
import hashlib
import os
import shutil
import re
import sys

from nxdrive.logging_config import get_logger
from nxdrive.client.common import safe_filename
from nxdrive.client.common import NotFound
from nxdrive.client.common import DEFAULT_IGNORED_PREFIXES
from nxdrive.client.common import DEFAULT_IGNORED_SUFFIXES
from nxdrive.utils import normalized_path
from nxdrive.utils import safe_long_path
from nxdrive.client.common import FILE_BUFFER_SIZE
from send2trash import send2trash

log = get_logger(__name__)

DEDUPED_BASENAME_PATTERN = ur'^(.*)__(\d{1,3})$'

# Data transfer objects

class FileInfo(object):
    """Data Transfer Object for file info on the Local FS"""

    def __init__(self, root, path, folderish, last_modification_time,
                 digest_func='md5', check_suspended=None):

        # Function to check during long-running processing like digest
        # computation if the synchronization thread needs to be suspended
        self.check_suspended = check_suspended

        root = unicodedata.normalize('NFC', root)
        path = unicodedata.normalize('NFC', path)
        self.root = root  # the sync root folder local path
        self.path = path  # the truncated path (under the root)
        self.folderish = folderish  # True if a Folder

        # Last OS modification date of the file
        self.last_modification_time = last_modification_time

        # Function to use
        self._digest_func = digest_func.lower()

        # Precompute base name once and for all are it's often useful in
        # practice
        self.name = os.path.basename(path)

        self.filepath = os.path.join(
            root, path[1:].replace(u'/', os.path.sep))

    def get_digest(self):
        """Lazy computation of the digest"""
        if self.folderish:
            return None
        digester = getattr(hashlib, self._digest_func, None)
        if digester is None:
            raise ValueError('Unknow digest method: ' + self.digest_func)

        h = digester()
        with open(safe_long_path(self.filepath), 'rb') as f:
            while True:
                # Check if synchronization thread was suspended
                if self.check_suspended is not None:
                    self.check_suspended('Digest computation: %s'
                                         % self.filepath)
                buffer_ = f.read(FILE_BUFFER_SIZE)
                if buffer_ == '':
                    break
                h.update(buffer_)
        return h.hexdigest()

class LocalClient(object):
    """Client API implementation for the local file system"""

    # TODO: initialize the prefixes and suffix with a dedicated Nuxeo
    # Automation operations fetched at controller init time.

    def __init__(self, base_folder, digest_func='md5', ignored_prefixes=None,
                 ignored_suffixes=None, check_suspended=None):
        self._case_sensitive = None
        # Function to check during long-running processing like digest
        # computation if the synchronization thread needs to be suspended
        self.check_suspended = check_suspended

        if ignored_prefixes is not None:
            self.ignored_prefixes = ignored_prefixes
        else:
            self.ignored_prefixes = DEFAULT_IGNORED_PREFIXES

        if ignored_suffixes is not None:
            self.ignored_suffixes = ignored_suffixes
        else:
            self.ignored_suffixes = DEFAULT_IGNORED_SUFFIXES

        while len(base_folder) > 1 and base_folder.endswith(os.path.sep):
            base_folder = base_folder[:-1]
        self.base_folder = base_folder
        self._digest_func = digest_func

    def is_case_sensitive(self):
        if self._case_sensitive is None:
            path = os.tempnam(self.base_folder, '.caseTest_')
            os.mkdir(path)
            if os.path.exists(path.upper()):
                self._case_sensitive = False
            else:
                self._case_sensitive = True
            os.rmdir(path)
        return self._case_sensitive

    # Getters
    def get_info(self, ref, raise_if_missing=True):
        os_path = self._abspath(ref)
        if not os.path.exists(os_path):
            if raise_if_missing:
                raise NotFound("Could not found file '%s' under '%s'" % (
                ref, self.base_folder))
            else:
                return None
        folderish = os.path.isdir(os_path)
        stat_info = os.stat(os_path)
        mtime = datetime.fromtimestamp(stat_info.st_mtime)
        path = u'/' + os_path[len(safe_long_path(self.base_folder)) + 1:]
        path = path.replace(os.path.sep, u'/')  # unix style path
        # On unix we could use the inode for file move detection but that won't
        # work on Windows. To reduce complexity of the code and the possibility
        # to have Windows specific bugs, let's not use the unix inode at all.
        # uid = str(stat_info.st_ino)
        return FileInfo(self.base_folder, path, folderish, mtime,
                        digest_func=self._digest_func,
                        check_suspended=self.check_suspended)

    def get_content(self, ref):
        return open(self._abspath(ref), "rb").read()

    def is_ignored(self, child_name):
        ignore = False
        for suffix in self.ignored_suffixes:
            if child_name.endswith(suffix):
                ignore = True
                break
        for prefix in self.ignored_prefixes:
            if child_name.startswith(prefix):
                ignore = True
                break
        return ignore

    def get_children_ref(self, parent_ref, name):
        if parent_ref == u'/':
            return parent_ref + name
        else:
            return parent_ref + u'/' + name

    def get_children_info(self, ref):
        os_path = self._abspath(ref)
        result = []
        children = os.listdir(os_path)
        children.sort()
        for child_name in children:

            if not self.is_ignored(child_name):
                child_ref = self.get_children_ref(ref, child_name)
                try:
                    result.append(self.get_info(child_ref))
                except (OSError, NotFound):
                    # the child file has been deleted in the mean time or while
                    # reading some of its attributes
                    pass

        return result

    def make_folder(self, parent, name):
        os_path, name = self._abspath_deduped(parent, name)
        os.mkdir(os_path)
        if parent == u"/":
            return u"/" + name
        return parent + u"/" + name

    def make_file(self, parent, name, content=None):
        os_path, name = self._abspath_deduped(parent, name)
        with open(os_path, "wb") as f:
            if content:
                f.write(content)
        if parent == u"/":
            return u"/" + name
        return parent + u"/" + name

    def get_new_file(self, parent, name):
        os_path, name = self._abspath_deduped(parent, name)
        if parent == u"/":
            path = u"/" + name
        else:
            path = parent + u"/" + name
        return path, os_path, name

    def update_content(self, ref, content):
        with open(self._abspath(ref), "wb") as f:
            f.write(content)

    def delete(self, ref):
        os_path = self._abspath(ref)
        # Remove the \\?\ for SHFileOperation on win
        if os_path[:4] == '\\\\?\\':
            # http://msdn.microsoft.com/en-us/library/cc249520.aspx
            # SHFileOperation don't handle \\?\ paths
            if len(os_path) > 260:
                # Rename to the drive root
                info = self.move(ref, '/')
                new_ref = info.path
                try:
                    send2trash(self._abspath(new_ref)[4:])
                except:
                    log.info('Cant use trash for ' + os_path
                                 + ', delete it')
                    self.delete_final(new_ref)
                return
            else:
                os_path = os_path[4:]
        log.info('Send ' + os_path + ' to trash')
        try:
            send2trash(os_path)
        except:
            log.info('Cant use trash for ' + os_path
                                 + ', delete it')
            self.delete_final(ref)

    def delete_final(self, ref):
        os_path = self._abspath(ref)
        if os.path.isfile(os_path):
            os.unlink(os_path)
        elif os.path.isdir(os_path):
            shutil.rmtree(os_path)

    def exists(self, ref):
        os_path = self._abspath(ref)
        return os.path.exists(os_path)

    def check_writable(self, ref):
        os_path = self._abspath(ref)
        return os.access(os_path, os.W_OK)

    def rename(self, ref, new_name):
        """Rename a local file or folder

        Return the actualized info object.

        """
        source_os_path = self._abspath(ref)
        parent = ref.rsplit(u'/', 1)[0]
        old_name = ref.rsplit(u'/', 1)[1]
        parent = u'/' if parent == '' else parent
        # Check if only case renaming
        if (old_name != new_name and old_name.lower() == new_name.lower()
            and not self.is_case_sensitive()):
            # Must use a temp rename as FS is not case sensitive
            temp_path = os.tempnam(self._abspath(parent),
                                   '.ren_' + old_name + '_')
            if sys.platform == 'win32':
                import ctypes
                ctypes.windll.kernel32.SetFileAttributesW(unicode(temp_path), 2)
            shutil.move(source_os_path, temp_path)
            source_os_path = temp_path
            # Try the os rename part
            target_os_path = self._abspath(os.path.join(parent, new_name))
        else:
            target_os_path, new_name = self._abspath_deduped(parent, new_name)
        shutil.move(source_os_path, target_os_path)
        if sys.platform == 'win32':
            import ctypes
            # See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
            ctypes.windll.kernel32.SetFileAttributesW(unicode(target_os_path), 128)
        new_ref = self.get_children_ref(parent, new_name)
        return self.get_info(new_ref)

    def move(self, ref, new_parent_ref):
        """Move a local file or folder into another folder

        Return the actualized info object.

        """
        if ref == u'/':
            raise ValueError("Cannot move the toplevel folder.")
        source_os_path = self._abspath(ref)
        name = ref.rsplit(u'/', 1)[1]
        target_os_path, new_name = self._abspath_deduped(new_parent_ref, name)
        shutil.move(source_os_path, target_os_path)
        new_ref = self.get_children_ref(new_parent_ref, new_name)
        return self.get_info(new_ref)

    def get_path(self, abspath):
        """Relative path to the local client from an absolute OS path"""
        path = abspath.split(self.base_folder, 1)[1]
        return path.replace(os.path.sep, '/')

    def _abspath(self, ref):
        """Absolute path on the operating system"""
        if not ref.startswith(u'/'):
            raise ValueError("LocalClient expects ref starting with '/'")
        path_suffix = ref[1:].replace('/', os.path.sep)
        path = normalized_path(os.path.join(self.base_folder, path_suffix))
        return safe_long_path(path)

    def _abspath_safe(self, parent, orig_name):
        """Absolute path on the operating system with deduplicated names"""
        # make name safe by removing invalid chars
        name = safe_filename(orig_name)

        # decompose the name into actionable components
        name, suffix = os.path.splitext(name)
        os_path = self._abspath(os.path.join(parent, name + suffix))
        return os_path

    def _abspath_deduped(self, parent, orig_name):
        """Absolute path on the operating system with deduplicated names"""
        # make name safe by removing invalid chars
        name = safe_filename(orig_name)

        # decompose the name into actionable components
        name, suffix = os.path.splitext(name)

        for _ in range(1000):
            os_path = self._abspath(os.path.join(parent, name + suffix))
            if not os.path.exists(os_path):
                return os_path, name + suffix

            # the is a duplicated file, try to come with a new name
            m = re.match(DEDUPED_BASENAME_PATTERN, name)
            if m:
                short_name, increment = m.groups()
                name = u"%s__%d" % (short_name, int(increment) + 1)
            else:
                name = name + u'__1'

        raise ValueError("Failed to de-duplicate '%s' under '%s'" % (
            orig_name, parent))
<commit_msg>NXP-14388: Add the parent ref to is_ignored<commit_after>"""API to access local resources for synchronization."""

import unicodedata
from datetime import datetime
import hashlib
import os
import shutil
import re
import sys

from nxdrive.logging_config import get_logger
from nxdrive.client.common import safe_filename
from nxdrive.client.common import NotFound
from nxdrive.client.common import DEFAULT_IGNORED_PREFIXES
from nxdrive.client.common import DEFAULT_IGNORED_SUFFIXES
from nxdrive.utils import normalized_path
from nxdrive.utils import safe_long_path
from nxdrive.client.common import FILE_BUFFER_SIZE
from send2trash import send2trash

log = get_logger(__name__)

DEDUPED_BASENAME_PATTERN = ur'^(.*)__(\d{1,3})$'

# Data transfer objects

class FileInfo(object):
    """Data Transfer Object for file info on the Local FS"""

    def __init__(self, root, path, folderish, last_modification_time,
                 digest_func='md5', check_suspended=None):

        # Function to check during long-running processing like digest
        # computation if the synchronization thread needs to be suspended
        self.check_suspended = check_suspended

        root = unicodedata.normalize('NFC', root)
        path = unicodedata.normalize('NFC', path)
        self.root = root  # the sync root folder local path
        self.path = path  # the truncated path (under the root)
        self.folderish = folderish  # True if a Folder

        # Last OS modification date of the file
        self.last_modification_time = last_modification_time

        # Function to use
        self._digest_func = digest_func.lower()

        # Precompute base name once and for all are it's often useful in
        # practice
        self.name = os.path.basename(path)

        self.filepath = os.path.join(
            root, path[1:].replace(u'/', os.path.sep))

    def get_digest(self):
        """Lazy computation of the digest"""
        if self.folderish:
            return None
        digester = getattr(hashlib, self._digest_func, None)
        if digester is None:
            raise ValueError('Unknow digest method: ' + self.digest_func)

        h = digester()
        with open(safe_long_path(self.filepath), 'rb') as f:
            while True:
                # Check if synchronization thread was suspended
                if self.check_suspended is not None:
                    self.check_suspended('Digest computation: %s'
                                         % self.filepath)
                buffer_ = f.read(FILE_BUFFER_SIZE)
                if buffer_ == '':
                    break
                h.update(buffer_)
        return h.hexdigest()

class LocalClient(object):
    """Client API implementation for the local file system"""

    # TODO: initialize the prefixes and suffix with a dedicated Nuxeo
    # Automation operations fetched at controller init time.

    def __init__(self, base_folder, digest_func='md5', ignored_prefixes=None,
                 ignored_suffixes=None, check_suspended=None):
        self._case_sensitive = None
        # Function to check during long-running processing like digest
        # computation if the synchronization thread needs to be suspended
        self.check_suspended = check_suspended

        if ignored_prefixes is not None:
            self.ignored_prefixes = ignored_prefixes
        else:
            self.ignored_prefixes = DEFAULT_IGNORED_PREFIXES

        if ignored_suffixes is not None:
            self.ignored_suffixes = ignored_suffixes
        else:
            self.ignored_suffixes = DEFAULT_IGNORED_SUFFIXES

        while len(base_folder) > 1 and base_folder.endswith(os.path.sep):
            base_folder = base_folder[:-1]
        self.base_folder = base_folder
        self._digest_func = digest_func

    def is_case_sensitive(self):
        if self._case_sensitive is None:
            path = os.tempnam(self.base_folder, '.caseTest_')
            os.mkdir(path)
            if os.path.exists(path.upper()):
                self._case_sensitive = False
            else:
                self._case_sensitive = True
            os.rmdir(path)
        return self._case_sensitive

    # Getters
    def get_info(self, ref, raise_if_missing=True):
        os_path = self._abspath(ref)
        if not os.path.exists(os_path):
            if raise_if_missing:
                raise NotFound("Could not found file '%s' under '%s'" % (
                ref, self.base_folder))
            else:
                return None
        folderish = os.path.isdir(os_path)
        stat_info = os.stat(os_path)
        mtime = datetime.fromtimestamp(stat_info.st_mtime)
        path = u'/' + os_path[len(safe_long_path(self.base_folder)) + 1:]
        path = path.replace(os.path.sep, u'/')  # unix style path
        # On unix we could use the inode for file move detection but that won't
        # work on Windows. To reduce complexity of the code and the possibility
        # to have Windows specific bugs, let's not use the unix inode at all.
        # uid = str(stat_info.st_ino)
        return FileInfo(self.base_folder, path, folderish, mtime,
                        digest_func=self._digest_func,
                        check_suspended=self.check_suspended)

    def get_content(self, ref):
        return open(self._abspath(ref), "rb").read()

    def is_ignored(self, ref, child_name):
        ignore = False
        for suffix in self.ignored_suffixes:
            if child_name.endswith(suffix):
                ignore = True
                break
        for prefix in self.ignored_prefixes:
            if child_name.startswith(prefix):
                ignore = True
                break
        return ignore

    def get_children_ref(self, parent_ref, name):
        if parent_ref == u'/':
            return parent_ref + name
        else:
            return parent_ref + u'/' + name

    def get_children_info(self, ref):
        os_path = self._abspath(ref)
        result = []
        children = os.listdir(os_path)
        children.sort()
        for child_name in children:

            if not self.is_ignored(ref, child_name):
                child_ref = self.get_children_ref(ref, child_name)
                try:
                    result.append(self.get_info(child_ref))
                except (OSError, NotFound):
                    # the child file has been deleted in the mean time or while
                    # reading some of its attributes
                    pass

        return result

    def make_folder(self, parent, name):
        os_path, name = self._abspath_deduped(parent, name)
        os.mkdir(os_path)
        if parent == u"/":
            return u"/" + name
        return parent + u"/" + name

    def make_file(self, parent, name, content=None):
        os_path, name = self._abspath_deduped(parent, name)
        with open(os_path, "wb") as f:
            if content:
                f.write(content)
        if parent == u"/":
            return u"/" + name
        return parent + u"/" + name

    def get_new_file(self, parent, name):
        os_path, name = self._abspath_deduped(parent, name)
        if parent == u"/":
            path = u"/" + name
        else:
            path = parent + u"/" + name
        return path, os_path, name

    def update_content(self, ref, content):
        with open(self._abspath(ref), "wb") as f:
            f.write(content)

    def delete(self, ref):
        os_path = self._abspath(ref)
        # Remove the \\?\ for SHFileOperation on win
        if os_path[:4] == '\\\\?\\':
            # http://msdn.microsoft.com/en-us/library/cc249520.aspx
            # SHFileOperation don't handle \\?\ paths
            if len(os_path) > 260:
                # Rename to the drive root
                info = self.move(ref, '/')
                new_ref = info.path
                try:
                    send2trash(self._abspath(new_ref)[4:])
                except:
                    log.info('Cant use trash for ' + os_path
                                 + ', delete it')
                    self.delete_final(new_ref)
                return
            else:
                os_path = os_path[4:]
        log.info('Send ' + os_path + ' to trash')
        try:
            send2trash(os_path)
        except:
            log.info('Cant use trash for ' + os_path
                                 + ', delete it')
            self.delete_final(ref)

    def delete_final(self, ref):
        os_path = self._abspath(ref)
        if os.path.isfile(os_path):
            os.unlink(os_path)
        elif os.path.isdir(os_path):
            shutil.rmtree(os_path)

    def exists(self, ref):
        os_path = self._abspath(ref)
        return os.path.exists(os_path)

    def check_writable(self, ref):
        os_path = self._abspath(ref)
        return os.access(os_path, os.W_OK)

    def rename(self, ref, new_name):
        """Rename a local file or folder

        Return the actualized info object.

        """
        source_os_path = self._abspath(ref)
        parent = ref.rsplit(u'/', 1)[0]
        old_name = ref.rsplit(u'/', 1)[1]
        parent = u'/' if parent == '' else parent
        # Check if only case renaming
        if (old_name != new_name and old_name.lower() == new_name.lower()
            and not self.is_case_sensitive()):
            # Must use a temp rename as FS is not case sensitive
            temp_path = os.tempnam(self._abspath(parent),
                                   '.ren_' + old_name + '_')
            if sys.platform == 'win32':
                import ctypes
                ctypes.windll.kernel32.SetFileAttributesW(unicode(temp_path), 2)
            shutil.move(source_os_path, temp_path)
            source_os_path = temp_path
            # Try the os rename part
            target_os_path = self._abspath(os.path.join(parent, new_name))
        else:
            target_os_path, new_name = self._abspath_deduped(parent, new_name)
        shutil.move(source_os_path, target_os_path)
        if sys.platform == 'win32':
            import ctypes
            # See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
            ctypes.windll.kernel32.SetFileAttributesW(unicode(target_os_path), 128)
        new_ref = self.get_children_ref(parent, new_name)
        return self.get_info(new_ref)

    def move(self, ref, new_parent_ref):
        """Move a local file or folder into another folder

        Return the actualized info object.

        """
        if ref == u'/':
            raise ValueError("Cannot move the toplevel folder.")
        source_os_path = self._abspath(ref)
        name = ref.rsplit(u'/', 1)[1]
        target_os_path, new_name = self._abspath_deduped(new_parent_ref, name)
        shutil.move(source_os_path, target_os_path)
        new_ref = self.get_children_ref(new_parent_ref, new_name)
        return self.get_info(new_ref)

    def get_path(self, abspath):
        """Relative path to the local client from an absolute OS path"""
        path = abspath.split(self.base_folder, 1)[1]
        return path.replace(os.path.sep, '/')

    def _abspath(self, ref):
        """Absolute path on the operating system"""
        if not ref.startswith(u'/'):
            raise ValueError("LocalClient expects ref starting with '/'")
        path_suffix = ref[1:].replace('/', os.path.sep)
        path = normalized_path(os.path.join(self.base_folder, path_suffix))
        return safe_long_path(path)

    def _abspath_safe(self, parent, orig_name):
        """Absolute path on the operating system with deduplicated names"""
        # make name safe by removing invalid chars
        name = safe_filename(orig_name)

        # decompose the name into actionable components
        name, suffix = os.path.splitext(name)
        os_path = self._abspath(os.path.join(parent, name + suffix))
        return os_path

    def _abspath_deduped(self, parent, orig_name):
        """Absolute path on the operating system with deduplicated names"""
        # make name safe by removing invalid chars
        name = safe_filename(orig_name)

        # decompose the name into actionable components
        name, suffix = os.path.splitext(name)

        for _ in range(1000):
            os_path = self._abspath(os.path.join(parent, name + suffix))
            if not os.path.exists(os_path):
                return os_path, name + suffix

            # the is a duplicated file, try to come with a new name
            m = re.match(DEDUPED_BASENAME_PATTERN, name)
            if m:
                short_name, increment = m.groups()
                name = u"%s__%d" % (short_name, int(increment) + 1)
            else:
                name = name + u'__1'

        raise ValueError("Failed to de-duplicate '%s' under '%s'" % (
            orig_name, parent))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Converter from Keras sequential NN to JSON
"""____________________________________________________________________
Variable specification file

In additon to the standard Keras architecture and weights files, you
must provide a "variable specification" json file.

Here `scale` and `offset` account for any scaling and shifting to the
input variables in preprocessing. The "default" value is optional.

If no file is provided, a template will be generated.
"""

import argparse
import json
import h5py
from collections import Counter
import sys, os
from keras_layer_converters import layer_converters, skip_layers

def _run():
    """Top level routine"""
    args = _get_args()
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)
    if not args.variables_file:
        _build_variables_file(args)
        sys.exit(0)
    with open(args.variables_file, 'r') as variables_file:
        variables = json.load(variables_file)

    _check_version(arch)

    with h5py.File(args.hdf5_file, 'r') as h5:
        layers, node_dict = _get_layers_and_nodes(arch, h5)
    input_layer_arch = arch['config']['input_layers']
    nodes = _build_output_node_list(node_dict, input_layer_arch)

    out_dict = {
        'layers': layers, 'nodes': nodes,
        'inputs': _parse_inputs(
            variables['inputs'],
            _get_vars_per_input(input_layer_arch, node_dict)),
        'outputs': _parse_outputs(
            variables['outputs'],
            arch['config']['output_layers'],
            node_dict),
    }
    print(json.dumps(out_dict, indent=2, sort_keys=True))

def _check_version(arch):
    if arch["class_name"] != "Model":
        sys.exit("this is not a graph, try using keras2json")
    if 'keras_version' not in arch:
        sys.stderr.write(
            'WARNING: no version number found for this archetecture!\n')
        return
    major, minor, *bugfix = arch['keras_version'].split('.')
    if major != '1' or minor < '2':
        warn_tmp = (
            "WARNNING: This converter was developed for Keras version 1.2. "
            "Your version (v{}.{}) may be incompatible.\n")
        sys.stderr.write(warn_tmp.format(major, minor))

def _get_args():
    parser = argparse.ArgumentParser(
        description="Converter from Keras saved NN to JSON",
        epilog=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('arch_file', help='architecture json file')
    parser.add_argument('hdf5_file', help='Keras weights file')
    parser.add_argument('variables_file', help='variable spec as json',
                        nargs='?')
    return parser.parse_args()

def _build_variables_file(args):
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)

    _check_version(arch)

    with h5py.File(args.hdf5_file, 'r') as h5:
        layers, node_dict = _get_layers_and_nodes(arch, h5)
    input_layer_arch = arch['config']['input_layers']
    inputs = []
    vars_per_input = _get_vars_per_input(input_layer_arch, node_dict)
    for nodenum, n_vars in sorted(vars_per_input.items()):
        def get_input(n):
            return {'name': 'variable_{}'.format(n), 'scale': 1, 'offset':0}
        the_input = {
            'name': 'node_{}'.format(nodenum),
            'variables': [get_input(n) for n in range(n_vars)]
        }
        inputs.append(the_input)
    outputs = []
    for kname, kid, ks in arch['config']['output_layers']:
        node = node_dict[(kname, kid)]
        output = {
            'name': '{}_{}'.format(kname, kid),
            'labels': ['out_{}'.format(x) for x in range(node.n_outputs)]
        }
        outputs.append(output)
    out_dict = {'inputs': inputs, 'outputs': outputs}
    print(json.dumps(out_dict, indent=2, sort_keys=True))

# __________________________________________________________________________
# master layer converter / inputs function

# output names
LAYERS = 'layers'
NODES = 'nodes'

_merge_layers = set(['concat'])

class Node:
    # TODO: make the node object a wrapper on the Keras layer dictionary
    def __init__(self, layer, idx):
        self.layer_type = layer['class_name'].lower()
        self.name = layer['name']
        self.idx = idx
        self.sources = []
        self.number = None
        self.layer_number = None
        self.n_outputs = None
        self.dims = None
        inbound = layer['inbound_nodes']
        if self.layer_type != "inputlayer":
            for sname, sidx, something in inbound[idx]:
                assert something == 0
                self.sources.append( (sname, sidx) )
        else:
            shape = layer['config']['batch_input_shape']
            self.n_outputs = shape[-1]
            self.dims = len(shape) - 1
        self.keras_layer = layer

    def __str__(self):
        parents = ', '.join(( str(x) for x in self.sources))
        tmp = '{} <- [{}]'
        if self.number is not None:
            tmp = '{}{{{n}}} <- [{}]'
        return tmp.format( (self.name, self.idx), parents, n=self.number)

    def get_key(self):
        return (self.name, self.idx)

    def __lt__(self, other):
        return self.get_key() < other.get_key()

# We want to build one node per _inbound_ node
def _build_node_dict(network):
    layers = {l['name']: l for l in network['config']['layers']}
    nodes = {}

    # first get the nodes that something points to
    for layer in layers.values():
        for sink in layer['inbound_nodes']:
            for kname, kid, something in sink:
                nodes[(kname, kid)] = Node(layers[kname], kid)

    # get the output nodes now
    for kname, kid, something in network['config']['output_layers']:
        id_tup = (kname, kid)
        if id_tup not in nodes:
            nodes[id_tup] = Node(layers[kname], kid)

    # now we collapse the node references
    for node in nodes.values():
        source_nodes = []
        for source in node.sources:
            source_nodes.append(nodes[source])
        node.sources = source_nodes
    return nodes

def _number_nodes(node_dict):
    for number, node in enumerate(sorted(node_dict.values())):
        node.number = number

def _build_layer(output_layers, node_key, h5, node_dict, layer_dict):
    node = node_dict[node_key]
    if node.n_outputs is not None:
        return

    for source in node.sources:
        _build_layer(output_layers, source.get_key(), h5,
                     node_dict, layer_dict)

    # special cases for merge layers
    if node.layer_type == 'merge':
        node.n_outputs = 0
        for source in node.sources:
            node.n_outputs += source.n_outputs
        return

    assert len(node.sources) == 1

    # if this layer is already defined, just add the node count and
    # continue
    if node.name in layer_dict:
        node.n_outputs = layer_dict[node.name]['n_outputs']
        node.layer_number = layer_dict[node.name]['pos']
        return

    layer_type = node.layer_type
    if layer_type in skip_layers:
        return
    convert = layer_converters[layer_type]

    # build the out layer
    n_inputs = sum(s.n_outputs for s in node.sources)
    layer_config = node.keras_layer['config']
    out_layer, node.n_outputs = convert(h5, layer_config, n_inputs)
    layer_number = len(output_layers)
    layer_dict[node.name] = {
        'n_outputs': node.n_outputs,
        'pos': layer_number}
    node.layer_number = layer_number
    output_layers.append(out_layer)

_node_type_map = {
    'merge': 'concatenate',
    'inputlayer': 'input',
    'dense': 'feed_forward',
    'lstm': 'sequence',
}

def _build_output_node_list(node_dict, input_layer_arch):
    """
    no effort is made to sort this list in any way, but the ordering
    is important because each node contains indices for other nodes
    """
    node_list = []
    input_map = {n[0]:i for i, n in enumerate(input_layer_arch)}
    for node in sorted(node_dict.values()):
        node_type = _node_type_map[node.layer_type]
        out_node = {'type': node_type}
        if node.sources:
            out_node['sources'] = [n.number for n in node.sources]

        if node_type == 'input':
            out_node['sources'] = [input_map[node.name]]
            out_node['size'] = node.n_outputs
            if node.dims > 1:
                out_node['type'] = 'input_sequence'
        elif node_type in ['feed_forward', 'sequence']:
            out_node['layer_index'] = node.layer_number
        node_list.append(out_node)
    return node_list

def _get_layers_and_nodes(network, h5):
    node_dict = _build_node_dict(network)
    _number_nodes(node_dict)

    output_layers = []
    layer_meta = {}
    for node_key in node_dict:
        _build_layer(output_layers, node_key, h5, node_dict, layer_meta)

    return output_layers, node_dict

def _get_vars_per_input(input_layer_arch, node_dict):
    vars_per_input = {}
    for nodenum, (lname, lidx, something) in enumerate(input_layer_arch):
        assert lidx == 0 and something == 0
        vars_per_input[nodenum] = node_dict[(lname, lidx)].n_outputs
    return vars_per_input

def _parse_inputs(input_list, vars_per_input):
    nodes = []
    for input_number, node in enumerate(input_list):
        inputs = []
        node_name = node['name']
        for val in node['variables']:
            var_info = {x: val[x] for x in ['offset', 'scale', 'name']}
            default = val.get("default")
            if default is not None:
                var_info['default'] = default
            inputs.append(var_info)

        assert vars_per_input[input_number] == len(inputs)

        nodes.append({'name': node_name, 'variables': inputs})
    return nodes

def _parse_outputs(user_outputs, output_layers, node_dict):
    outputs = {}
    assert len(user_outputs) == len(output_layers)
    for num, (usr, ker) in enumerate(zip(user_outputs, output_layers)):
        kname, kid, ks = ker
        assert ks == 0
        node = node_dict[(kname, kid)]
        assert node.n_outputs == len(usr['labels'])
        assert usr['name'] not in outputs
        output = {
            'node_index': node.number,
            'labels': usr['labels']
        }
        outputs[usr['name']] = output
    return outputs

def _parse_miscellaneous(variables):
    if 'miscellaneous' in variables:
        misc_dict = {}
        for key, val in variables['miscellaneous'].items():
            misc_dict[str(key)] = str(val)
        out['miscellaneous'] = misc_dict
    return out

if __name__ == '__main__':
    _run()
<commit_msg>Change JSON to store seq inputs in new object<commit_after>#!/usr/bin/env python3
#
# Converter from Keras sequential NN to JSON
"""____________________________________________________________________
Variable specification file

In additon to the standard Keras architecture and weights files, you
must provide a "variable specification" json file.

Here `scale` and `offset` account for any scaling and shifting to the
input variables in preprocessing. The "default" value is optional.

If no file is provided, a template will be generated.
"""

import argparse
import json
import h5py
from collections import Counter
import sys, os
from keras_layer_converters import layer_converters, skip_layers

def _run():
    """Top level routine"""
    args = _get_args()
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)
    if not args.variables_file:
        _build_variables_file(args)
        sys.exit(0)
    with open(args.variables_file, 'r') as variables_file:
        variables = json.load(variables_file)

    _check_version(arch)

    with h5py.File(args.hdf5_file, 'r') as h5:
        layers, node_dict = _get_layers_and_nodes(arch, h5)
    input_layer_arch = arch['config']['input_layers']
    nodes = _build_node_list(node_dict, input_layer_arch)

    vars_per_input = _get_vars_per_input(input_layer_arch, node_dict)
    out_dict = {
        'layers': layers, 'nodes': nodes,
        'inputs': _parse_inputs(
            variables['inputs'],
            vars_per_input[1]),
        'input_sequences': _parse_inputs(
            variables['input_sequences'],
            vars_per_input[2]),
        'outputs': _parse_outputs(
            variables['outputs'],
            arch['config']['output_layers'],
            node_dict),
    }
    print(json.dumps(out_dict, indent=2, sort_keys=True))

def _check_version(arch):
    if arch["class_name"] != "Model":
        sys.exit("this is not a graph, try using keras2json")
    if 'keras_version' not in arch:
        sys.stderr.write(
            'WARNING: no version number found for this archetecture!\n')
        return
    major, minor, *bugfix = arch['keras_version'].split('.')
    if major != '1' or minor < '2':
        warn_tmp = (
            "WARNNING: This converter was developed for Keras version 1.2. "
            "Your version (v{}.{}) may be incompatible.\n")
        sys.stderr.write(warn_tmp.format(major, minor))

def _get_args():
    parser = argparse.ArgumentParser(
        description="Converter from Keras saved NN to JSON",
        epilog=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('arch_file', help='architecture json file')
    parser.add_argument('hdf5_file', help='Keras weights file')
    parser.add_argument('variables_file', help='variable spec as json',
                        nargs='?')
    return parser.parse_args()

def _build_variables_file(args):
    with open(args.arch_file, 'r') as arch_file:
        arch = json.load(arch_file)

    _check_version(arch)

    with h5py.File(args.hdf5_file, 'r') as h5:
        layers, node_dict = _get_layers_and_nodes(arch, h5)
    input_layer_arch = arch['config']['input_layers']
    vars_per_input = _get_vars_per_input(input_layer_arch, node_dict)
    def get_input(n):
        return {'name': 'variable_{}'.format(n), 'scale': 1, 'offset':0}
    def build_inputs(input_items):
        inputs = []
        for nodenum, n_vars in input_items:
            the_input = {
                'name': 'node_{}'.format(nodenum),
                'variables': [get_input(n) for n in range(n_vars)]
            }
            inputs.append(the_input)
        return inputs
    inputs = build_inputs(sorted(vars_per_input[1].items()))
    seq_inputs = build_inputs(sorted(vars_per_input[2].items()))
    outputs = []
    for kname, kid, ks in arch['config']['output_layers']:
        node = node_dict[(kname, kid)]
        output = {
            'name': '{}_{}'.format(kname, kid),
            'labels': ['out_{}'.format(x) for x in range(node.n_outputs)]
        }
        outputs.append(output)
    out_dict = {
        'inputs': inputs, 'input_sequences': seq_inputs,
        'outputs': outputs}
    print(json.dumps(out_dict, indent=2, sort_keys=True))

# __________________________________________________________________________
# master layer converter / inputs function

# output names
LAYERS = 'layers'
NODES = 'nodes'

_merge_layers = set(['concat'])

class Node:
    # TODO: make the node object a wrapper on the Keras layer dictionary
    def __init__(self, layer, idx):
        self.layer_type = layer['class_name'].lower()
        self.name = layer['name']
        self.idx = idx
        self.sources = []
        self.number = None
        self.layer_number = None
        self.n_outputs = None
        self.dims = None
        inbound = layer['inbound_nodes']
        if self.layer_type != "inputlayer":
            for sname, sidx, something in inbound[idx]:
                assert something == 0
                self.sources.append( (sname, sidx) )
        else:
            shape = layer['config']['batch_input_shape']
            self.n_outputs = shape[-1]
            self.dims = len(shape) - 1
        self.keras_layer = layer

    def __str__(self):
        parents = ', '.join(( str(x) for x in self.sources))
        tmp = '{} <- [{}]'
        if self.number is not None:
            tmp = '{}{{{n}}} <- [{}]'
        return tmp.format( (self.name, self.idx), parents, n=self.number)

    def get_key(self):
        return (self.name, self.idx)

    def __lt__(self, other):
        return self.get_key() < other.get_key()

# We want to build one node per _inbound_ node
def _build_node_dict(network):
    layers = {l['name']: l for l in network['config']['layers']}
    nodes = {}

    # first get the nodes that something points to
    for layer in layers.values():
        for sink in layer['inbound_nodes']:
            for kname, kid, something in sink:
                nodes[(kname, kid)] = Node(layers[kname], kid)

    # get the output nodes now
    for kname, kid, something in network['config']['output_layers']:
        id_tup = (kname, kid)
        if id_tup not in nodes:
            nodes[id_tup] = Node(layers[kname], kid)

    # now we collapse the node references
    for node in nodes.values():
        source_nodes = []
        for source in node.sources:
            source_nodes.append(nodes[source])
        node.sources = source_nodes
    return nodes

def _number_nodes(node_dict):
    for number, node in enumerate(sorted(node_dict.values())):
        node.number = number

def _build_layer(output_layers, node_key, h5, node_dict, layer_dict):
    node = node_dict[node_key]
    if node.n_outputs is not None:
        return

    for source in node.sources:
        _build_layer(output_layers, source.get_key(), h5,
                     node_dict, layer_dict)

    # special cases for merge layers
    if node.layer_type == 'merge':
        node.n_outputs = 0
        for source in node.sources:
            node.n_outputs += source.n_outputs
        return

    assert len(node.sources) == 1

    # if this layer is already defined, just add the node count and
    # continue
    if node.name in layer_dict:
        node.n_outputs = layer_dict[node.name]['n_outputs']
        node.layer_number = layer_dict[node.name]['pos']
        return

    layer_type = node.layer_type
    if layer_type in skip_layers:
        return
    convert = layer_converters[layer_type]

    # build the out layer
    n_inputs = sum(s.n_outputs for s in node.sources)
    layer_config = node.keras_layer['config']
    out_layer, node.n_outputs = convert(h5, layer_config, n_inputs)
    layer_number = len(output_layers)
    layer_dict[node.name] = {
        'n_outputs': node.n_outputs,
        'pos': layer_number}
    node.layer_number = layer_number
    output_layers.append(out_layer)

_node_type_map = {
    'merge': 'concatenate',
    'inputlayer': 'input',
    'dense': 'feed_forward',
    'lstm': 'sequence',
}

def _build_node_list(node_dict, input_layer_arch):
    """
    no effort is made to sort this list in any way, but the ordering
    is important because each node contains indices for other nodes
    """
    node_list = []
    input_map = {}
    for kname, kid, ks in input_layer_arch:
        node = node_dict[(kname, kid)]
        submap = input_map.setdefault(node.dims, {})
        n_in = len(submap)
        submap[kname] = n_in

    for node in sorted(node_dict.values()):
        node_type = _node_type_map[node.layer_type]
        out_node = {'type': node_type}
        if node.sources:
            out_node['sources'] = [n.number for n in node.sources]

        if node_type == 'input':
            out_node['sources'] = [input_map[node.dims][node.name]]
            out_node['size'] = node.n_outputs
            if node.dims > 1:
                out_node['type'] = 'input_sequence'
        elif node_type in ['feed_forward', 'sequence']:
            out_node['layer_index'] = node.layer_number
        node_list.append(out_node)
    return node_list

def _get_layers_and_nodes(network, h5):
    node_dict = _build_node_dict(network)
    _number_nodes(node_dict)

    output_layers = []
    layer_meta = {}
    for node_key in node_dict:
        _build_layer(output_layers, node_key, h5, node_dict, layer_meta)

    return output_layers, node_dict

def _get_vars_per_input(input_layer_arch, node_dict):
    vars_per_input = {1: {}, 2: {}}
    for lname, lidx, something in input_layer_arch:
        assert lidx == 0 and something == 0
        node = node_dict[(lname, lidx)]
        nodenum = len(vars_per_input[node.dims])
        vars_per_input[node.dims][nodenum] = node.n_outputs
    return vars_per_input

def _parse_inputs(input_list, vars_per_input):
    nodes = []
    for input_number, node in enumerate(input_list):
        inputs = []
        node_name = node['name']
        for val in node['variables']:
            var_info = {x: val[x] for x in ['offset', 'scale', 'name']}
            default = val.get("default")
            if default is not None:
                var_info['default'] = default
            inputs.append(var_info)

        assert vars_per_input[input_number] == len(inputs)

        nodes.append({'name': node_name, 'variables': inputs})
    return nodes

def _parse_outputs(user_outputs, output_layers, node_dict):
    outputs = {}
    assert len(user_outputs) == len(output_layers)
    for num, (usr, ker) in enumerate(zip(user_outputs, output_layers)):
        kname, kid, ks = ker
        assert ks == 0
        node = node_dict[(kname, kid)]
        assert node.n_outputs == len(usr['labels'])
        assert usr['name'] not in outputs
        output = {
            'node_index': node.number,
            'labels': usr['labels']
        }
        outputs[usr['name']] = output
    return outputs

def _parse_miscellaneous(variables):
    if 'miscellaneous' in variables:
        misc_dict = {}
        for key, val in variables['miscellaneous'].items():
            misc_dict[str(key)] = str(val)
        out['miscellaneous'] = misc_dict
    return out

if __name__ == '__main__':
    _run()
<|endoftext|>