<commit_before>"""

Copyright (c) 2014 Alex Forencich

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

from myhdl import *

def words2list(f, N=1, WL=8):
    frame = []
    while len(f) > 0:
        data = 0
        keep = 0
        for i in range(N):
            data = data | (f.pop(0) << (i*WL))
            keep = keep | (1 << i)
            if len(f) == 0: break
        frame.append((data, keep))
    return frame

def list2words(f, N=1, WL=8):
    frame = []
    mask = 2**WL-1
    for data in f:
        keep = -1
        if type(data) is tuple:
            data, keep = data
        for i in range(N):
            if keep & (1 << i):
                frame.append((data >> (i*WL)) & mask)
    return frame

def AXIStreamSource(clk, rst,
                    tdata=None,
                    tkeep=None,
                    tvalid=Signal(bool(False)),
                    tready=Signal(bool(True)),
                    tlast=Signal(bool(False)),
                    fifo=None,
                    pause=0,
                    name=None):

    tready_int = Signal(bool(False))
    tvalid_int = Signal(bool(False))

    @always_comb
    def pause_logic():
        tready_int.next = tready and not pause
        tvalid.next = tvalid_int and not pause

    @instance
    def logic():
        frame = []
        N = len(tdata)
        M = 1
        b = False
        if tkeep is not None:
            M = len(tkeep)
        WL = (len(tdata)+M-1)/M
        if WL == 8:
            b = True

        while True:
            yield clk.posedge, rst.posedge

            if rst:
                tdata.next = 0
                if tkeep is not None:
                    tkeep.next = 0
                tvalid_int.next = False
                tlast.next = False
            else:
                if tready_int and tvalid:
                    if len(frame) > 0:
                        data, keep = frame.pop(0)
                        tdata.next = data
                        if tkeep is not None:
                            tkeep.next = keep
                        tvalid_int.next = True
                        tlast.next = len(frame) == 0
                    else:
                        tvalid_int.next = False
                        tlast.next = False
                if (tlast and tready_int and tvalid) or not tvalid_int:
                    if not fifo.empty():
                        frame = fifo.get()
                        if type(frame) is bytes:
                            frame = bytearray(frame)
                        if type(frame) is bytearray:
                            frame = words2list(frame, M, WL)
                        if name is not None:
                            print("[%s] Sending frame %s" % (name, repr(frame)))
                        data, keep = frame.pop(0)
                        tdata.next = data
                        if tkeep is not None:
                            tkeep.next = keep
                        tvalid_int.next = True
                        tlast.next = len(frame) == 0

    return logic, pause_logic

def AXIStreamSink(clk, rst,
                  tdata=None,
                  tkeep=None,
                  tvalid=Signal(bool(True)),
                  tready=Signal(bool(True)),
                  tlast=Signal(bool(True)),
                  fifo=None,
                  pause=0,
                  name=None):

    tready_int = Signal(bool(False))
    tvalid_int = Signal(bool(False))

    @always_comb
    def pause_logic():
        tready.next = tready_int and not pause
        tvalid_int.next = tvalid and not pause

    @instance
    def logic():
        frame = []
        N = len(tdata)
        M = 1
        b = False
        if tkeep is not None:
            M = len(tkeep)
        WL = (len(tdata)+M-1)/M
        if WL == 8:
            b = True

        while True:
            yield clk.posedge, rst.posedge

            if rst:
                tready_int.next = False
                frame = []
            else:
                tready_int.next = True

                if tvalid_int:
                    if tkeep is not None:
                        frame.append((int(tdata), int(tkeep)))
                    else:
                        frame.append(int(tdata))
                    if tlast:
                        if b:
                            frame = list2words(frame, M, WL)
                        if fifo is not None:
                            fifo.put(frame)
                        if name is not None:
                            print("[%s] Got frame %s" % (name, repr(frame)))
                        frame = []

    return logic, pause_logic

<commit_msg>Update AXI stream endpoint<commit_after>"""

Copyright (c) 2014 Alex Forencich

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

from myhdl import *

class AXIStreamFrame(object):
    def __init__(self, data=b'', keep=None, user=None):
        self.B = 0
        self.N = 8
        self.M = 1
        self.WL = 8
        self.data = b''
        self.keep = None
        self.user = None

        if type(data) is bytes or type(data) is bytearray:
            self.data = bytearray(data)
        elif type(data) is AXIStreamFrame:
            self.N = data.N
            self.WL = data.WL
            if type(data.data) is bytearray:
                self.data = bytearray(data.data)
            else:
                self.data = list(data.data)
            if data.keep is not None:
                self.keep = list(data.keep)
            if data.user is not None:
                if type(data.user) is int or type(data.user) is bool:
                    self.user = data.user
                else:
                    self.user = list(data.user)
        else:
            self.data = list(data)

    def build(self):
        if self.data is None:
            return

        f = list(self.data)
        tdata = []
        tkeep = []
        tuser = []
        i = 0

        assert_tuser = False
        if (type(self.user) is int or type(self.user) is bool) and self.user:
            assert_tuser = True
            self.user = None

        if self.B == 0:
            while len(f) > 0:
                data = 0
                keep = 0
                for j in range(self.M):
                    data = data | (f.pop(0) << (j*self.WL))
                    keep = keep | (1 << j)
                    if len(f) == 0: break
                tdata.append(data)
                if self.keep is None:
                    tkeep.append(keep)
                else:
                    tkeep.append(self.keep[i])
                if self.user is None:
                    tuser.append(0)
                else:
                    tuser.append(self.user[i])
                i += 1
        else:
            # multiple tdata signals
            while len(f) > 0:
                data = 0
                tdata.append(f.pop(0))
                tkeep.append(0)
                if self.user is None:
                    tuser.append(0)
                else:
                    tuser.append(self.user[i])
                i += 1

        if assert_tuser:
            tuser[-1] = 1
            self.user = 1

        return tdata, tkeep, tuser

    def parse(self, tdata, tkeep, tuser):
        if tdata is None or tkeep is None or tuser is None:
            return
        if len(tdata) != len(tkeep) or len(tdata) != len(tuser):
            raise Exception("Invalid data")

        self.data = []
        self.keep = []
        self.user = []

        if self.B == 0:
            mask = 2**self.WL-1

            for i in range(len(tdata)):
                for j in range(self.M):
                    if tkeep[i] & (1 << j):
                        self.data.append((tdata[i] >> (j*self.WL)) & mask)
                self.keep.append(tkeep[i])
                self.user.append(tuser[i])
        else:
            for i in range(len(tdata)):
                self.data.append(tdata[i])
                self.keep.append(tkeep[i])
                self.user.append(tuser[i])

        if self.WL == 8:
            self.data = bytearray(self.data)

    def __eq__(self, other):
        if type(other) is AXIStreamFrame:
            return self.data == other.data

    def __repr__(self):
        return 'AXIStreamFrame(data=%s, keep=%s, user=%s)' % (repr(self.data), repr(self.keep), repr(self.user))

    def __iter__(self):
        return self.data.__iter__()

def AXIStreamSource(clk, rst,
                    tdata=None,
                    tkeep=Signal(bool(True)),
                    tvalid=Signal(bool(False)),
                    tready=Signal(bool(True)),
                    tlast=Signal(bool(False)),
                    tuser=Signal(bool(False)),
                    fifo=None,
                    pause=0,
                    name=None):

    tready_int = Signal(bool(False))
    tvalid_int = Signal(bool(False))

    @always_comb
    def pause_logic():
        tready_int.next = tready and not pause
        tvalid.next = tvalid_int and not pause

    @instance
    def logic():
        frame = AXIStreamFrame()
        data = []
        keep = []
        user = []
        B = 0
        N = len(tdata)
        M = len(tkeep)
        WL = int((len(tdata)+M-1)/M)

        if type(tdata) is list or type(tdata) is tuple:
            # multiple tdata signals
            B = len(tdata)
            N = [len(b) for b in tdata]
            M = 1
            WL = [1]*B

        while True:
            yield clk.posedge, rst.posedge

            if rst:
                if B > 0:
                    for s in tdata:
                        s.next = 0
                else:
                    tdata.next = 0
                tkeep.next = 0
                tvalid_int.next = False
                tlast.next = False
            else:
                if tready_int and tvalid:
                    if len(data) > 0:
                        if B > 0:
                            l = data.pop(0)
                            for i in range(B):
                                tdata[i].next = l[i]
                        else:
                            tdata.next = data.pop(0)
                        tkeep.next = keep.pop(0)
                        tuser.next = user.pop(0)
                        tvalid_int.next = True
                        tlast.next = len(data) == 0
                    else:
                        tvalid_int.next = False
                        tlast.next = False
                if (tlast and tready_int and tvalid) or not tvalid_int:
                    if not fifo.empty():
                        frame = fifo.get()
                        frame = AXIStreamFrame(frame)
                        frame.B = B
                        frame.N = N
                        frame.M = M
                        frame.WL = WL
                        data, keep, user = frame.build()
                        if name is not None:
                            print("[%s] Sending frame %s" % (name, repr(frame)))
                        if B > 0:
                            l = data.pop(0)
                            for i in range(B):
                                tdata[i].next = l[i]
                        else:
                            tdata.next = data.pop(0)
                        tkeep.next = keep.pop(0)
                        tuser.next = user.pop(0)
                        tvalid_int.next = True
                        tlast.next = len(data) == 0

    return logic, pause_logic

def AXIStreamSink(clk, rst,
                  tdata=None,
                  tkeep=Signal(bool(True)),
                  tvalid=Signal(bool(True)),
                  tready=Signal(bool(True)),
                  tlast=Signal(bool(True)),
                  tuser=Signal(bool(False)),
                  fifo=None,
                  pause=0,
                  name=None):

    tready_int = Signal(bool(False))
    tvalid_int = Signal(bool(False))

    @always_comb
    def pause_logic():
        tready.next = tready_int and not pause
        tvalid_int.next = tvalid and not pause

    @instance
    def logic():
        frame = AXIStreamFrame()
        data = []
        keep = []
        user = []
        B = 0
        N = len(tdata)
        M = len(tkeep)
        WL = int((len(tdata)+M-1)/M)

        if type(tdata) is list or type(tdata) is tuple:
            # multiple tdata signals
            B = len(tdata)
            N = [len(b) for b in tdata]
            M = 1
            WL = [1]*B

        while True:
            yield clk.posedge, rst.posedge

            if rst:
                tready_int.next = False
                frame = AXIStreamFrame()
                data = []
                keep = []
                user = []
            else:
                tready_int.next = True

                if tvalid_int:
                    if B > 0:
                        l = []
                        for i in range(B):
                            l.append(int(tdata[i]))
                        data.append(l)
                    else:
                        data.append(int(tdata))
                    keep.append(int(tkeep))
                    user.append(int(tuser))
                    if tlast:
                        frame.B = B
                        frame.N = N
                        frame.M = M
                        frame.WL = WL
                        frame.parse(data, keep, user)
                        if fifo is not None:
                            fifo.put(frame)
                        if name is not None:
                            print("[%s] Got frame %s" % (name, repr(frame)))
                        frame = AXIStreamFrame()
                        data = []
                        keep = []
                        user = []

    return logic, pause_logic

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Utility functions for the neukrill-net classifier
"""
import glob
import io
import json
import os
import csv
import gzip
import numpy as np
import skimage
import random

import neukrill_net.image_processing as image_processing
import neukrill_net.constants as constants

class Settings:
    """
    A class to handle all the parsing of the settings.json as well as
    checking the values and providing defaults
    """
    def __init__(self, settings_file):
        """
        Initialise settings object
        """
        self.required_settings = ['data_dir']

        # read the provided settings json (or stringIO)
        parsed_settings = self.parse_settings(settings_file)

        # check all required options have been set
        self.user_input = self.check_mandatory_fields(parsed_settings)

        # check the data dir options and find the correct one
        self.data_dir = self.check_data_dir(self.user_input['data_dir'])

        # check user input for random seed and if absent set it
        self.random_seed = self.user_input.get('r_seed', 42)
        # don't shuffle by default
        self.shuffle_flag = False

        # check user defined classes if absent set it
        self.classes = self.user_input.get('classes',
                                            constants.classes)

        # a way to encode the superclasses if we need to but don't want to complete
        # just now as it is very monotonous to copy everything
        # self.super_classes = {'FISH': ('fish classes'),
        #                       'DIATOMS': ('diatom classes'),
        #                       'TRICHODESMIUM': ('you get the idea')
        #                       'PROTISTS': ('protists_classes')
        #                       'NO_SUPER_CLASS': ('unclassified')}

        self._image_fnames = {}

        self._class_priors = []

    def parse_settings(self, settings_file):
        """
        Parse json file or stringIO formatted settings file
        This function will likely be extended later to enable defaults and to
        check inputs
        input : settings_file - filename or stringIO obj
                required_settings - list of required fields
        output: settings - settings dict
        """
        if settings_file.__class__ is io.StringIO:
            settings = json.load(settings_file)

        else:
            if not os.path.exists(settings_file):
                raise ValueError('Settings file does not exist: {0}'.format(\
                                                                settings_file))
            with open(settings_file, 'r') as settings_fh:
                settings = json.load(settings_fh)

        return settings

    def check_mandatory_fields(self, parsed_user_input):
        """
        Ensure all the mandatory settings are present
        """
        for entry in self.required_settings:
            if entry not in parsed_user_input:
                raise ValueError('data_dir must be defined')

        return parsed_user_input

    def check_data_dir(self, data_dir_possibilities):
        """
        Make sure the data dirs exist and resolve the test and train dirs
        abspaths
        """
        for possible_dir in data_dir_possibilities:
            train_data_dir = os.path.join(possible_dir, 'train')
            test_data_dir = os.path.join(possible_dir, 'test')
            dirs_exist = os.path.exists(train_data_dir) and \
                                                os.path.exists(test_data_dir)

            if dirs_exist:
                return os.path.abspath(possible_dir)

        raise ValueError("Can't find data dir in options: {0}".format(\
                                                            data_dir_possibilities))

    @property
    def image_fnames(self):
        """
        Take in data dir and return dict of filenames
        input: data_directory - path as str
        output: image_fnames - dict {'test': (tuple of fnames abspaths),
                                     'train': {'class_1' : (tuple of fnames),
                                               'class_2  : (tuple of fnames),
                                               ...
                                               }
                                    }
        """

        if not self._image_fnames:
            test_fnames = tuple(sorted(glob.glob(os.path.join(self.data_dir,
                                                       'test',
                                                       '*.jpg'))))

            # check there are the correct number of images
            num_test_images = len(test_fnames)
            if num_test_images != 130400:
                raise ValueError('Wrong number of test images found: {0}'
                                 ' instead of 130400'.format(num_test_images))

            train_fnames = {}

            for name in sorted(glob.glob(os.path.join(self.data_dir,
                                               'train',
                                               '*',
                                               ''))):
                split_name = name.split(os.path.sep)
                class_name = split_name[-2]
                image_names = sorted(glob.glob(os.path.join(name, '*.jpg')))
                if self.shuffle_flag:
                    random.seed(self.random_seed)
                    # shuffle in place
                    random.shuffle(image_names)
                train_fnames.update({class_name: image_names})

            num_train_classes = len(train_fnames.keys())
            num_train_images = sum(map(len, train_fnames.values()))
            if num_train_classes != 121:
                raise ValueError('Incorrect num of training class directories '\
                        '121 expected: {0} found'.format(num_train_classes))

            if num_train_images != 30336:
                raise ValueError('Incorrect num of training images '\
                        ' 30336 expected: {0} found'.format(num_train_images))

            self._image_fnames = {'test': test_fnames,
                                  'train': train_fnames}

        return self._image_fnames

    def shuffle(self, seed=None):
        """
        Modifies processing of image_fnames to ensure that the paths inside
        each class will be shuffled. (Internally, wipes existing data
        structure and rewrites it with shuffling on next call.
        Input:
            seed: random seed to use for shuffling, default is 42.
        Output:
            None (sets internal flag)
        """
        self.shuffle_flag = True
        # if seed supplied, apply new value
        if seed:
            self.random_seed = seed
        # if image_fnames exists, get rid of it
        if self._image_fnames:
            self._image_fnames = None
        return None

    @property
    def class_priors(self):
        """
        Get the proportion of training data in each class
        """
        if self._class_priors == []:
            class_probs = np.zeros(len(self.classes))
            for class_index, class_name in enumerate(self.classes):
                # Tally up how many filenames are in this class
                # Set to zero if this class is not a field
                class_probs[class_index] = len(self.image_fnames['train'].get(class_name, 0))
            # Normalise the classes
            class_probs /= sum(class_probs)
            self._class_priors = class_probs

        return self._class_priors

def load_data(image_fname_dict, classes=None,
              processing=None, verbose=False):
    """
    Loads training or test data using image_processing.load_images func
    which applies the supplied processing function as required.

    This function is rapidly becoming a hack and will probably be replace with
    Scott's HighLevelFeatures idea.

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - image matrix
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image vectors
    """

    if not processing and verbose:
        print("Warning: no processing applied, it will \
        not be possible to stack these images due to \
        varying sizes.")

    # initialise lists
    #data = []
    # check augmentation factor
    dummy_images = processing(np.zeros((100,100)))
    augmentation_factor = len(dummy_images)

    # e.g. labelled training data
    if classes:
        labels = []

        image_fpaths = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            image_fpaths += image_fname_dict['train'][class_name]
            #data_subset = image_processing.load_images(image_fpaths,
            #                                              processing,
            #                                              verbose)
            #data.append(data_subset)
            #num_images = len(data_subset)
            # generate the class labels and add them to the list
            array_labels = augmentation_factor * [class_name]
            labels = labels + array_labels
        data = image_processing.load_images(image_fpaths, processing, verbose)
        return data, np.array(labels)

    # e.g. test data
    else:
        data = image_processing.load_images(image_fname_dict['test'],
                                            processing,
                                            verbose)
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return np.vstack(data), names

def load_rawdata(image_fname_dict, classes=None, verbose=False):
    """
    Loads training or test data without appyling any processing.

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - list of image matrices
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image matrices
    """

    # initialise lists
    data = []

    # e.g. labelled training data
    if classes:
        labels = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            fpaths = image_fname_dict['train'][class_name]

            # Load the data and add to list
            data += [skimage.io.imread(fpath) for fpath in fpaths]

            # generate the class labels and add them to the list
            labels += len(fpaths) * [class_name]

        return data, np.array(labels)

    # e.g. test data
    else:
        data = [skimage.io.imread(fpath) for fpath in image_fname_dict['test']]
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return data, names

def write_predictions(out_fname, p, names, classes):
    """
    Write probabilities to an output csv which is compressed with gzip.
    input:  out_fname - name of the output file.
                       Append .csv if you like. Do not append with .gz.
            p - probability matrix.
                dim-0 specifies the file predicitons are of
                dim-1 specifies which class the prediciton is for
            names - names of the things which are being classified
            classes - the classes predictions are placed into
    output: None
            writes a gzip compressed csv file to `out_fname`.gz on disk
    """

    # Write the probabilites as a CSV
    with open(out_fname, 'w') as csv_out:
        out_writer = csv.writer(csv_out, delimiter=',')
        out_writer.writerow(['image'] + list(classes))
        for index in range(len(names)):
            out_writer.writerow([names[index]] + list(p[index,]))

    # Compress with gzip
    with open(out_fname, 'rb') as f_in:
        f_out = gzip.open(out_fname + '.gz', 'wb')
        f_out.writelines(f_in)
        f_out.close()

    # Delete the uncompressed CSV
    os.unlink(out_fname)

def load_run_settings(run_settings_path, settings,
        settings_path="settings.json", verbose=False, force=False):
    """
    Loads the run settings and adds settings to dictionary, along
    with:
    * filename - the filename minus ".json" of the run settings file, for
        saving results, pickles, etc.
    * run_settings_path - abspath to the run settings file, is handed
        to the Dataset class for Pylearn2
    * settings_path - abspath to the settings file, assumed to be in the usual
        cwd
    * modeldir - directory in which to save models
    * pickle abspath - abspath where _this_ run will save its pickle file
    """

    with open(run_settings_path) as rf:
        run_settings = json.load(rf)
    # shoehorn run_settings filename into its own dictionary (for later)
    run_settings['filename'] = os.path.split(
                                        run_settings_path)[-1].split(".")[0]
    # and the full run settings path
    run_settings['run_settings_path'] = os.path.abspath(run_settings_path)
    # if shuffling is specified and it's 1
    if run_settings.get('shuffle',0):
        # shuffle the dataset and apply seed if we have one
        settings.shuffle(run_settings.get('random seed',None))
    # also put the settings in there
    run_settings['settings'] = settings
    # and the settings path, while we're at it
    run_settings['settings_path'] = os.path.abspath(settings_path)
    # add the models directory for this run
    # before saving model check there is somewhere for it to save to
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    run_settings['modeldir'] = modeldir

    # save the pickle name here, so it's less likely to get garbled between train and test
    run_settings['pickle abspath'] = os.path.join(modeldir,run_settings['filename']+".pkl")
    # check if the pickle already exists - and don't allow overwriting if so
    if os.path.exists(run_settings['pickle abspath']) and not force:
        # not sure what type of error this should be
        raise Exception("Run will overwrite model pickle file, delete or move"
                        " file to continue.")

    submissionsdir = os.path.join(settings.data_dir,"submissions")
    if not os.path.exists(submissionsdir):
        os.mkdir(submissionsdir)

    run_settings['submissions abspath'] = os.path.join(submissionsdir, run_settings['filename'] + ".csv")

    return run_settings
<commit_msg>fixed problem with class loading<commit_after>#!/usr/bin/env python
"""
Utility functions for the neukrill-net classifier
"""
import glob
import io
import json
import os
import csv
import gzip
import numpy as np
import skimage
import random

import neukrill_net.image_processing as image_processing
import neukrill_net.constants as constants

class Settings:
    """
    A class to handle all the parsing of the settings.json as well as
    checking the values and providing defaults
    """
    def __init__(self, settings_file):
        """
        Initialise settings object
        """
        self.required_settings = ['data_dir']

        # read the provided settings json (or stringIO)
        parsed_settings = self.parse_settings(settings_file)

        # check all required options have been set
        self.user_input = self.check_mandatory_fields(parsed_settings)

        # check the data dir options and find the correct one
        self.data_dir = self.check_data_dir(self.user_input['data_dir'])

        # check user input for random seed and if absent set it
        self.random_seed = self.user_input.get('r_seed', 42)
        # don't shuffle by default
        self.shuffle_flag = False

        # check user defined classes if absent set it
        self.classes = self.user_input.get('classes',
                                            constants.classes)

        # a way to encode the superclasses if we need to but don't want to complete
        # just now as it is very monotonous to copy everything
        # self.super_classes = {'FISH': ('fish classes'),
        #                       'DIATOMS': ('diatom classes'),
        #                       'TRICHODESMIUM': ('you get the idea')
        #                       'PROTISTS': ('protists_classes')
        #                       'NO_SUPER_CLASS': ('unclassified')}

        self._image_fnames = {}

        self._class_priors = []

    def parse_settings(self, settings_file):
        """
        Parse json file or stringIO formatted settings file
        This function will likely be extended later to enable defaults and to
        check inputs
        input : settings_file - filename or stringIO obj
                required_settings - list of required fields
        output: settings - settings dict
        """
        if settings_file.__class__ is io.StringIO:
            settings = json.load(settings_file)

        else:
            if not os.path.exists(settings_file):
                raise ValueError('Settings file does not exist: {0}'.format(\
                                                                settings_file))
            with open(settings_file, 'r') as settings_fh:
                settings = json.load(settings_fh)

        return settings

    def check_mandatory_fields(self, parsed_user_input):
        """
        Ensure all the mandatory settings are present
        """
        for entry in self.required_settings:
            if entry not in parsed_user_input:
                raise ValueError('data_dir must be defined')

        return parsed_user_input

    def check_data_dir(self, data_dir_possibilities):
        """
        Make sure the data dirs exist and resolve the test and train dirs
        abspaths
        """
        for possible_dir in data_dir_possibilities:
            train_data_dir = os.path.join(possible_dir, 'train')
            test_data_dir = os.path.join(possible_dir, 'test')
            dirs_exist = os.path.exists(train_data_dir) and \
                                                os.path.exists(test_data_dir)

            if dirs_exist:
                return os.path.abspath(possible_dir)

        raise ValueError("Can't find data dir in options: {0}".format(\
                                                            data_dir_possibilities))

    @property
    def image_fnames(self):
        """
        Take in data dir and return dict of filenames
        input: data_directory - path as str
        output: image_fnames - dict {'test': (tuple of fnames abspaths),
                                     'train': {'class_1' : (tuple of fnames),
                                               'class_2  : (tuple of fnames),
                                               ...
                                               }
                                    }
        """

        if not self._image_fnames:
            test_fnames = tuple(sorted(glob.glob(os.path.join(self.data_dir,
                                                       'test',
                                                       '*.jpg'))))

            # check there are the correct number of images
            num_test_images = len(test_fnames)
            if num_test_images != 130400:
                raise ValueError('Wrong number of test images found: {0}'
                                 ' instead of 130400'.format(num_test_images))

            train_fnames = {}

            for name in sorted(glob.glob(os.path.join(self.data_dir,
                                               'train',
                                               '*',
                                               ''))):
                split_name = name.split(os.path.sep)
                class_name = split_name[-2]
                image_names = sorted(glob.glob(os.path.join(name, '*.jpg')))
                if self.shuffle_flag:
                    random.seed(self.random_seed)
                    # shuffle in place
                    random.shuffle(image_names)
                train_fnames.update({class_name: image_names})

            num_train_classes = len(train_fnames.keys())
            num_train_images = sum(map(len, train_fnames.values()))
            if num_train_classes != 121:
                raise ValueError('Incorrect num of training class directories '\
                        '121 expected: {0} found'.format(num_train_classes))

            if num_train_images != 30336:
                raise ValueError('Incorrect num of training images '\
                        ' 30336 expected: {0} found'.format(num_train_images))

            self._image_fnames = {'test': test_fnames,
                                  'train': train_fnames}

        return self._image_fnames

    def shuffle(self, seed=None):
        """
        Modifies processing of image_fnames to ensure that the paths inside
        each class will be shuffled. (Internally, wipes existing data
        structure and rewrites it with shuffling on next call.
        Input:
            seed: random seed to use for shuffling, default is 42.
        Output:
            None (sets internal flag)
        """
        self.shuffle_flag = True
        # if seed supplied, apply new value
        if seed:
            self.random_seed = seed
        # if image_fnames exists, get rid of it
        if self._image_fnames:
            self._image_fnames = None
        return None

    @property
    def class_priors(self):
        """
        Get the proportion of training data in each class
        """
        if self._class_priors == []:
            class_probs = np.zeros(len(self.classes))
            for class_index, class_name in enumerate(self.classes):
                # Tally up how many filenames are in this class
                # Set to zero if this class is not a field
                class_probs[class_index] = len(self.image_fnames['train'].get(class_name, 0))
            # Normalise the classes
            class_probs /= sum(class_probs)
            self._class_priors = class_probs

        return self._class_priors

def load_data(image_fname_dict, classes=None,
              processing=None, verbose=False):
    """
    Loads training or test data using image_processing.load_images func
    which applies the supplied processing function as required.

    This function is rapidly becoming a hack and will probably be replace with
    Scott's HighLevelFeatures idea.

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - image matrix
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image vectors
    """

    if not processing and verbose:
        print("Warning: no processing applied, it will \
        not be possible to stack these images due to \
        varying sizes.")

    # initialise lists
    #data = []
    # check augmentation factor
    dummy_images = processing(np.zeros((100,100)))
    augmentation_factor = len(dummy_images)

    # e.g. labelled training data
    if classes:
        labels = []

        image_fpaths = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            class_fpaths = image_fname_dict['train'][class_name]
            array_labels = augmentation_factor * len(class_fpaths) * [class_name]
            labels = labels + array_labels
            image_fpaths += class_fpaths
            #data_subset = image_processing.load_images(image_fpaths,
            #                                              processing,
            #                                              verbose)
            #data.append(data_subset)
            #num_images = len(data_subset)
            # generate the class labels and add them to the list
        data = image_processing.load_images(image_fpaths, processing, verbose)
        return data, np.array(labels)

    # e.g. test data
    else:
        data = image_processing.load_images(image_fname_dict['test'],
                                            processing,
                                            verbose)
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return np.vstack(data), names

def load_rawdata(image_fname_dict, classes=None, verbose=False):
    """
    Loads training or test data without appyling any processing.

    If the classes kwarg is not none assumed to be loading labelled train
    data and returns two np objs:
        * data - list of image matrices
        * labels - vector of labels

    if classes kwarg is none, data will be loaded as test data and just return
        * data - list of image matrices
    """

    # initialise lists
    data = []

    # e.g. labelled training data
    if classes:
        labels = []

        for class_index, class_name in enumerate(classes):
            if verbose:
                print("class: {0} of 120: {1}".format(class_index, class_name))

            fpaths = image_fname_dict['train'][class_name]

            # Load the data and add to list
            data += [skimage.io.imread(fpath) for fpath in fpaths]

            # generate the class labels and add them to the list
            labels += len(fpaths) * [class_name]

        return data, np.array(labels)

    # e.g. test data
    else:
        data = [skimage.io.imread(fpath) for fpath in image_fname_dict['test']]
        names = [os.path.basename(fpath) for fpath in image_fname_dict['test']]
        return data, names

def write_predictions(out_fname, p, names, classes):
    """
    Write probabilities to an output csv which is compressed with gzip.
    input:  out_fname - name of the output file.
                       Append .csv if you like. Do not append with .gz.
            p - probability matrix.
                dim-0 specifies the file predicitons are of
                dim-1 specifies which class the prediciton is for
            names - names of the things which are being classified
            classes - the classes predictions are placed into
    output: None
            writes a gzip compressed csv file to `out_fname`.gz on disk
    """

    # Write the probabilites as a CSV
    with open(out_fname, 'w') as csv_out:
        out_writer = csv.writer(csv_out, delimiter=',')
        out_writer.writerow(['image'] + list(classes))
        for index in range(len(names)):
            out_writer.writerow([names[index]] + list(p[index,]))

    # Compress with gzip
    with open(out_fname, 'rb') as f_in:
        f_out = gzip.open(out_fname + '.gz', 'wb')
        f_out.writelines(f_in)
        f_out.close()

    # Delete the uncompressed CSV
    os.unlink(out_fname)

def load_run_settings(run_settings_path, settings,
        settings_path="settings.json", verbose=False, force=False):
    """
    Loads the run settings and adds settings to dictionary, along
    with:
    * filename - the filename minus ".json" of the run settings file, for
        saving results, pickles, etc.
    * run_settings_path - abspath to the run settings file, is handed
        to the Dataset class for Pylearn2
    * settings_path - abspath to the settings file, assumed to be in the usual
        cwd
    * modeldir - directory in which to save models
    * pickle abspath - abspath where _this_ run will save its pickle file
    """

    with open(run_settings_path) as rf:
        run_settings = json.load(rf)
    # shoehorn run_settings filename into its own dictionary (for later)
    run_settings['filename'] = os.path.split(
                                        run_settings_path)[-1].split(".")[0]
    # and the full run settings path
    run_settings['run_settings_path'] = os.path.abspath(run_settings_path)
    # if shuffling is specified and it's 1
    if run_settings.get('shuffle',0):
        # shuffle the dataset and apply seed if we have one
        settings.shuffle(run_settings.get('random seed',None))
    # also put the settings in there
    run_settings['settings'] = settings
    # and the settings path, while we're at it
    run_settings['settings_path'] = os.path.abspath(settings_path)
    # add the models directory for this run
    # before saving model check there is somewhere for it to save to
    modeldir = os.path.join(settings.data_dir,"models")
    if not os.path.exists(modeldir):
        os.mkdir(modeldir)
    run_settings['modeldir'] = modeldir

    # save the pickle name here, so it's less likely to get garbled between train and test
    run_settings['pickle abspath'] = os.path.join(modeldir,run_settings['filename']+".pkl")
    # check if the pickle already exists - and don't allow overwriting if so
    if os.path.exists(run_settings['pickle abspath']) and not force:
        # not sure what type of error this should be
        raise Exception("Run will overwrite model pickle file, delete or move"
                        " file to continue.")

    submissionsdir = os.path.join(settings.data_dir,"submissions")
    if not os.path.exists(submissionsdir):
        os.mkdir(submissionsdir)

    run_settings['submissions abspath'] = os.path.join(submissionsdir, run_settings['filename'] + ".csv")

    return run_settings

def save_run_settings(run_settings):
    """
    Takes a run_settings dictionary and saves it back where it was loaded from,
    using the path stored in its own dictionary. Return None.
    """
    # store the raw log loss results back in the run settings json
    with open(run_settings['run_settings_path'], 'w') as f:
        # have to remove the settings structure, can't serialise it
        del run_settings['settings']
        json.dump(run_settings, f, separators=(',',':'), indent=4,
                                                    sort_keys=True)
    return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import stationspinner.libs.fields
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('evemail', '0002_mail'),
        ('character', '0020_mailmessage_receivers')
    ]

    operations = [
        migrations.RunSQL(
            sql='''
            DROP VIEW evemail_mail;
            CREATE VIEW
                evemail_mail
            AS
                SELECT
                  mail.*,
                  status.read,
                  status.owner_id,
                  1.0 AS relevancy

                FROM
                  evemail_mailstatus status,
                  character_mailmessage mail
                WHERE status.message_id = mail."messageID";
            ''',
            reverse_sql='DROP VIEW evemail_mail'
        ),
    ]
<commit_msg>Wops, wrong migration<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import stationspinner.libs.fields
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('evemail', '0002_mail'),
        ('character', '0019_auto_20150114_2158')
    ]

    operations = [
        migrations.RunSQL(
            sql='''
            DROP VIEW evemail_mail;
            CREATE VIEW
                evemail_mail
            AS
                SELECT
                  mail.*,
                  status.read,
                  status.owner_id,
                  1.0 AS relevancy

                FROM
                  evemail_mailstatus status,
                  character_mailmessage mail
                WHERE status.message_id = mail."messageID";
            ''',
            reverse_sql='DROP VIEW evemail_mail'
        ),
    ]
<|endoftext|>


<commit_before>"""Training script for sequence to sequence learning."""

import argparse
import sys
import random
import os
import shlex
from shutil import copyfile
import subprocess
import numpy as np
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector

from neuralmonkey.checking import (CheckingException, check_dataset_and_coders,
                                   check_unused_initializers)
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop
from neuralmonkey.dataset import Dataset
from neuralmonkey.model.sequence import EmbeddedFactorSequence

def create_config() -> Configuration:
    config = Configuration()

    # training loop arguments
    config.add_argument("tf_manager")
    config.add_argument("epochs", cond=lambda x: x >= 0)
    config.add_argument("trainer")
    config.add_argument("batch_size", cond=lambda x: x > 0)
    config.add_argument("train_dataset")
    config.add_argument("val_dataset")
    config.add_argument("output")
    config.add_argument("evaluation")
    config.add_argument("runners")
    config.add_argument("test_datasets", required=False, default=[])
    config.add_argument("logging_period", required=False, default=20)
    config.add_argument("validation_period", required=False, default=500)
    config.add_argument("visualize_embeddings", required=False, default=None)
    config.add_argument("val_preview_input_series",
                        required=False, default=None)
    config.add_argument("val_preview_output_series",
                        required=False, default=None)
    config.add_argument("val_preview_num_examples",
                        required=False, default=15)
    config.add_argument("train_start_offset", required=False, default=0)
    config.add_argument("runners_batch_size", required=False, default=None)
    config.add_argument("postprocess")
    config.add_argument("name")
    config.add_argument("random_seed", required=False)
    config.add_argument("initial_variables", required=False, default=None)
    config.add_argument("overwrite_output_dir", required=False, default=False)

    return config

def save_git_info(repo_dir: str, git_commit_file: str, git_diff_file: str,
                  branch: str = "HEAD"):
    with open(git_commit_file, "wb") as file:
        subprocess.run(["git", "log", "-1", "--format=%H", branch],
                       cwd=repo_dir, stdout=file)

    with open(git_diff_file, "wb") as file:
        subprocess.run(["git", "--no-pager", "diff", "--color=always", branch],
                       cwd=repo_dir, stdout=file)

# pylint: disable=too-many-statements, too-many-locals, too-many-branches
def _main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("config", metavar="INI-FILE",
                        help="the configuration file for the experiment")
    parser.add_argument("-s", "--set", type=str, metavar="SETTING",
                        action="append", dest="config_changes", default=[],
                        help="override an option in the configuration; the "
                        "syntax is [section.]option=value")
    parser.add_argument("-v", "--var", type=str, metavar="VAR", default=[],
                        action="append", dest="config_vars",
                        help="set a variable in the configuration; the syntax "
                        "is var=value (shorthand for -s vars.var=value)")
    parser.add_argument("-i", "--init", dest="init_only", action="store_true",
                        help="initialize the experiment directory and exit "
                        "without building the model")
    parser.add_argument("-f", "--overwrite", action="store_true",
                        help="force overwriting the output directory; can be "
                        "used to start an experiment created with --init")
    args = parser.parse_args()

    # define valid parameters and defaults
    cfg = create_config()
    # load the params from the config file, getting also the simple arguments
    args.config_changes.extend("vars.{}".format(s) for s in args.config_vars)
    cfg.load_file(args.config, changes=args.config_changes)
    # various things like randseed or summarywriter should be set up here
    # so that graph building can be recorded
    # build all the objects specified in the config

    if cfg.args.random_seed is None:
        cfg.args.random_seed = 2574600
    random.seed(cfg.args.random_seed)
    np.random.seed(cfg.args.random_seed)
    tf.set_random_seed(cfg.args.random_seed)

    # pylint: disable=no-member
    if (os.path.isdir(cfg.args.output) and
            os.path.exists(os.path.join(cfg.args.output, "experiment.ini"))):
        if cfg.args.overwrite_output_dir or args.overwrite:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(cfg.args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(cfg.args.output), color="red")
            exit(1)

    # pylint: disable=broad-except
    if not os.path.isdir(cfg.args.output):
        try:
            os.mkdir(cfg.args.output)
        except Exception as exc:
            log("Failed to create experiment directory: {}. Exception: {}"
                .format(cfg.args.output, exc), color="red")
            exit(1)

    args_file = "{}/args".format(cfg.args.output)
    log_file = "{}/experiment.log".format(cfg.args.output)
    ini_file = "{}/experiment.ini".format(cfg.args.output)
    orig_ini_file = "{}/original.ini".format(cfg.args.output)
    git_commit_file = "{}/git_commit".format(cfg.args.output)
    git_diff_file = "{}/git_diff".format(cfg.args.output)
    variables_file_prefix = "{}/variables.data".format(cfg.args.output)

    cont_index = 0

    while (os.path.exists(log_file)
           or os.path.exists(ini_file)
           or os.path.exists(git_commit_file)
           or os.path.exists(git_diff_file)
           or os.path.exists(variables_file_prefix)
           or os.path.exists("{}.0".format(variables_file_prefix))):
        cont_index += 1

        args_file = "{}/args.cont-{}".format(
            cfg.args.output, cont_index)
        log_file = "{}/experiment.log.cont-{}".format(
            cfg.args.output, cont_index)
        ini_file = "{}/experiment.ini.cont-{}".format(
            cfg.args.output, cont_index)
        orig_ini_file = "{}/original.ini.cont-{}".format(
            cfg.args.output, cont_index)
        git_commit_file = "{}/git_commit.cont-{}".format(
            cfg.args.output, cont_index)
        git_diff_file = "{}/git_diff.cont-{}".format(
            cfg.args.output, cont_index)
        variables_file_prefix = "{}/variables.data.cont-{}".format(
            cfg.args.output, cont_index)

    with open(args_file, "w") as file:
        print(" ".join(shlex.quote(a) for a in sys.argv), file=file)

    cfg.save_file(ini_file)
    copyfile(args.config, orig_ini_file)

    if args.init_only:
        log("Experiment directory initialized.")

        cmd = [os.path.basename(sys.argv[0]), "-f", ini_file]
        log("To start experiment, run: {}".format(" ".join(shlex.quote(a)
                                                           for a in cmd)))
        exit(0)

    Logging.set_log_file(log_file)

    # this points inside the neuralmonkey/ dir inside the repo, but
    # it does not matter for git.
    repo_dir = os.path.dirname(os.path.realpath(__file__))
    save_git_info(repo_dir, git_commit_file, git_diff_file)

    cfg.build_model(warn_unused=True)

    cfg.model.tf_manager.init_saving(variables_file_prefix)

    try:
        check_dataset_and_coders(cfg.model.train_dataset,
                                 cfg.model.runners)
        if isinstance(cfg.model.val_dataset, Dataset):
            check_dataset_and_coders(cfg.model.val_dataset, cfg.model.runners)
        else:
            for val_dataset in cfg.model.val_dataset:
                check_dataset_and_coders(val_dataset, cfg.model.runners)

        check_unused_initializers()
    except CheckingException as exc:
        log(str(exc), color="red")
        exit(1)

    if cfg.model.visualize_embeddings:

        tb_projector = projector.ProjectorConfig()

        for sequence in cfg.model.visualize_embeddings:
            # TODO this check should be done when abstract class of embedded
            # sequences will be created, not only EmbeddedFactorSequence
            if not isinstance(sequence, EmbeddedFactorSequence):
                raise ValueError("Visualization must be embedded sequence.")
            sequence.tb_embedding_visualization(cfg.model.output, tb_projector)

        summary_writer = tf.summary.FileWriter(cfg.model.output)
        projector.visualize_embeddings(summary_writer, tb_projector)

    Logging.print_header(cfg.model.name, cfg.args.output)

    # runners_batch_size must be set to avoid problems on GPU
    if cfg.model.runners_batch_size is None:
        cfg.model.runners_batch_size = cfg.model.batch_size

    training_loop(
        tf_manager=cfg.model.tf_manager,
        epochs=cfg.model.epochs,
        trainer=cfg.model.trainer,
        batch_size=cfg.model.batch_size,
        log_directory=cfg.model.output,
        evaluators=cfg.model.evaluation,
        runners=cfg.model.runners,
        train_dataset=cfg.model.train_dataset,
        val_dataset=cfg.model.val_dataset,
        test_datasets=cfg.model.test_datasets,
        logging_period=cfg.model.logging_period,
        validation_period=cfg.model.validation_period,
        val_preview_input_series=cfg.model.val_preview_input_series,
        val_preview_output_series=cfg.model.val_preview_output_series,
        val_preview_num_examples=cfg.model.val_preview_num_examples,
        postprocess=cfg.model.postprocess,
        train_start_offset=cfg.model.train_start_offset,
        runners_batch_size=cfg.model.runners_batch_size,
        initial_variables=cfg.model.initial_variables)

def main() -> None:
    try:
        _main()
    except KeyboardInterrupt:
        log("Training interrupted by user.")
        exit(1)
<commit_msg>Update train.py<commit_after>"""Training script for sequence to sequence learning."""

import argparse
import sys
import random
import os
import shlex
from shutil import copyfile
import subprocess
import numpy as np
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector

from neuralmonkey.checking import (CheckingException, check_dataset_and_coders,
                                   check_unused_initializers)
from neuralmonkey.logging import Logging, log
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import training_loop
from neuralmonkey.dataset import Dataset
from neuralmonkey.model.sequence import EmbeddedFactorSequence

def create_config() -> Configuration:
    config = Configuration()

    # training loop arguments
    config.add_argument("tf_manager")
    config.add_argument("epochs", cond=lambda x: x >= 0)
    config.add_argument("trainer")
    config.add_argument("batch_size", cond=lambda x: x > 0)
    config.add_argument("train_dataset")
    config.add_argument("val_dataset")
    config.add_argument("output")
    config.add_argument("evaluation")
    config.add_argument("runners")
    config.add_argument("test_datasets", required=False, default=[])
    config.add_argument("logging_period", required=False, default=20)
    config.add_argument("validation_period", required=False, default=500)
    config.add_argument("visualize_embeddings", required=False, default=None)
    config.add_argument("val_preview_input_series",
                        required=False, default=None)
    config.add_argument("val_preview_output_series",
                        required=False, default=None)
    config.add_argument("val_preview_num_examples",
                        required=False, default=15)
    config.add_argument("train_start_offset", required=False, default=0)
    config.add_argument("runners_batch_size", required=False, default=None)
    config.add_argument("postprocess")
    config.add_argument("name", required=False, default="neural monkey experiment")
    config.add_argument("random_seed", required=False)
    config.add_argument("initial_variables", required=False, default=None)
    config.add_argument("overwrite_output_dir", required=False, default=False)

    return config

def save_git_info(repo_dir: str, git_commit_file: str, git_diff_file: str,
                  branch: str = "HEAD"):
    with open(git_commit_file, "wb") as file:
        subprocess.run(["git", "log", "-1", "--format=%H", branch],
                       cwd=repo_dir, stdout=file)

    with open(git_diff_file, "wb") as file:
        subprocess.run(["git", "--no-pager", "diff", "--color=always", branch],
                       cwd=repo_dir, stdout=file)

# pylint: disable=too-many-statements, too-many-locals, too-many-branches
def _main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("config", metavar="INI-FILE",
                        help="the configuration file for the experiment")
    parser.add_argument("-s", "--set", type=str, metavar="SETTING",
                        action="append", dest="config_changes", default=[],
                        help="override an option in the configuration; the "
                        "syntax is [section.]option=value")
    parser.add_argument("-v", "--var", type=str, metavar="VAR", default=[],
                        action="append", dest="config_vars",
                        help="set a variable in the configuration; the syntax "
                        "is var=value (shorthand for -s vars.var=value)")
    parser.add_argument("-i", "--init", dest="init_only", action="store_true",
                        help="initialize the experiment directory and exit "
                        "without building the model")
    parser.add_argument("-f", "--overwrite", action="store_true",
                        help="force overwriting the output directory; can be "
                        "used to start an experiment created with --init")
    args = parser.parse_args()

    # define valid parameters and defaults
    cfg = create_config()
    # load the params from the config file, getting also the simple arguments
    args.config_changes.extend("vars.{}".format(s) for s in args.config_vars)
    cfg.load_file(args.config, changes=args.config_changes)
    # various things like randseed or summarywriter should be set up here
    # so that graph building can be recorded
    # build all the objects specified in the config

    if cfg.args.random_seed is None:
        cfg.args.random_seed = 2574600
    random.seed(cfg.args.random_seed)
    np.random.seed(cfg.args.random_seed)
    tf.set_random_seed(cfg.args.random_seed)

    # pylint: disable=no-member
    if (os.path.isdir(cfg.args.output) and
            os.path.exists(os.path.join(cfg.args.output, "experiment.ini"))):
        if cfg.args.overwrite_output_dir or args.overwrite:
            # we do not want to delete the directory contents
            log("Directory with experiment.ini '{}' exists, "
                "overwriting enabled, proceeding."
                .format(cfg.args.output))
        else:
            log("Directory with experiment.ini '{}' exists, "
                "overwriting disabled."
                .format(cfg.args.output), color="red")
            exit(1)

    # pylint: disable=broad-except
    if not os.path.isdir(cfg.args.output):
        try:
            os.mkdir(cfg.args.output)
        except Exception as exc:
            log("Failed to create experiment directory: {}. Exception: {}"
                .format(cfg.args.output, exc), color="red")
            exit(1)

    args_file = "{}/args".format(cfg.args.output)
    log_file = "{}/experiment.log".format(cfg.args.output)
    ini_file = "{}/experiment.ini".format(cfg.args.output)
    orig_ini_file = "{}/original.ini".format(cfg.args.output)
    git_commit_file = "{}/git_commit".format(cfg.args.output)
    git_diff_file = "{}/git_diff".format(cfg.args.output)
    variables_file_prefix = "{}/variables.data".format(cfg.args.output)

    cont_index = 0

    while (os.path.exists(log_file)
           or os.path.exists(ini_file)
           or os.path.exists(git_commit_file)
           or os.path.exists(git_diff_file)
           or os.path.exists(variables_file_prefix)
           or os.path.exists("{}.0".format(variables_file_prefix))):
        cont_index += 1

        args_file = "{}/args.cont-{}".format(
            cfg.args.output, cont_index)
        log_file = "{}/experiment.log.cont-{}".format(
            cfg.args.output, cont_index)
        ini_file = "{}/experiment.ini.cont-{}".format(
            cfg.args.output, cont_index)
        orig_ini_file = "{}/original.ini.cont-{}".format(
            cfg.args.output, cont_index)
        git_commit_file = "{}/git_commit.cont-{}".format(
            cfg.args.output, cont_index)
        git_diff_file = "{}/git_diff.cont-{}".format(
            cfg.args.output, cont_index)
        variables_file_prefix = "{}/variables.data.cont-{}".format(
            cfg.args.output, cont_index)

    with open(args_file, "w") as file:
        print(" ".join(shlex.quote(a) for a in sys.argv), file=file)

    cfg.save_file(ini_file)
    copyfile(args.config, orig_ini_file)

    if args.init_only:
        log("Experiment directory initialized.")

        cmd = [os.path.basename(sys.argv[0]), "-f", ini_file]
        log("To start experiment, run: {}".format(" ".join(shlex.quote(a)
                                                           for a in cmd)))
        exit(0)

    Logging.set_log_file(log_file)

    # this points inside the neuralmonkey/ dir inside the repo, but
    # it does not matter for git.
    repo_dir = os.path.dirname(os.path.realpath(__file__))
    save_git_info(repo_dir, git_commit_file, git_diff_file)

    cfg.build_model(warn_unused=True)

    cfg.model.tf_manager.init_saving(variables_file_prefix)

    try:
        check_dataset_and_coders(cfg.model.train_dataset,
                                 cfg.model.runners)
        if isinstance(cfg.model.val_dataset, Dataset):
            check_dataset_and_coders(cfg.model.val_dataset, cfg.model.runners)
        else:
            for val_dataset in cfg.model.val_dataset:
                check_dataset_and_coders(val_dataset, cfg.model.runners)

        check_unused_initializers()
    except CheckingException as exc:
        log(str(exc), color="red")
        exit(1)

    if cfg.model.visualize_embeddings:

        tb_projector = projector.ProjectorConfig()

        for sequence in cfg.model.visualize_embeddings:
            # TODO this check should be done when abstract class of embedded
            # sequences will be created, not only EmbeddedFactorSequence
            if not isinstance(sequence, EmbeddedFactorSequence):
                raise ValueError("Visualization must be embedded sequence.")
            sequence.tb_embedding_visualization(cfg.model.output, tb_projector)

        summary_writer = tf.summary.FileWriter(cfg.model.output)
        projector.visualize_embeddings(summary_writer, tb_projector)

    Logging.print_header(cfg.model.name, cfg.args.output)

    # runners_batch_size must be set to avoid problems on GPU
    if cfg.model.runners_batch_size is None:
        cfg.model.runners_batch_size = cfg.model.batch_size

    training_loop(
        tf_manager=cfg.model.tf_manager,
        epochs=cfg.model.epochs,
        trainer=cfg.model.trainer,
        batch_size=cfg.model.batch_size,
        log_directory=cfg.model.output,
        evaluators=cfg.model.evaluation,
        runners=cfg.model.runners,
        train_dataset=cfg.model.train_dataset,
        val_dataset=cfg.model.val_dataset,
        test_datasets=cfg.model.test_datasets,
        logging_period=cfg.model.logging_period,
        validation_period=cfg.model.validation_period,
        val_preview_input_series=cfg.model.val_preview_input_series,
        val_preview_output_series=cfg.model.val_preview_output_series,
        val_preview_num_examples=cfg.model.val_preview_num_examples,
        postprocess=cfg.model.postprocess,
        train_start_offset=cfg.model.train_start_offset,
        runners_batch_size=cfg.model.runners_batch_size,
        initial_variables=cfg.model.initial_variables)

def main() -> None:
    try:
        _main()
    except KeyboardInterrupt:
        log("Training interrupted by user.")
        exit(1)
<|endoftext|>


<commit_before>import RPi.GPIO as GPIO
import time

def bin2dec(string_num):
    return str(int(string_num, 2))

data = []

GPIO.setmode(GPIO.BCM)

GPIO.setup(4,GPIO.OUT)
GPIO.output(4,GPIO.HIGH)
time.sleep(0.025)
GPIO.output(4,GPIO.LOW)
time.sleep(0.02)

GPIO.setup(4, GPIO.IN, pull_up_down=GPIO.PUD_UP)

for i in range(0,500):
    data.append(GPIO.input(4))

bit_count = 0
tmp = 0
count = 0
HumidityBit = ""
TemperatureBit = ""
crc = ""

try:
   while data[count] == 1:
      tmp = 1
      count = count + 1

   for i in range(0, 32):
      bit_count = 0

      while data[count] == 0:
         tmp = 1
         count = count + 1

      while data[count] == 1:
         bit_count = bit_count + 1
         count = count + 1

      if bit_count > 3:
         if i>=0 and i<8:
            HumidityBit = HumidityBit + "1"
         if i>=16 and i<24:
            TemperatureBit = TemperatureBit + "1"
      else:
         if i>=0 and i<8:
            HumidityBit = HumidityBit + "0"
         if i>=16 and i<24:
            TemperatureBit = TemperatureBit + "0"

except Exception as e:
   print "ERR_RANGE"
   print e
   exit(0)

try:
   for i in range(0, 8):
      bit_count = 0

      while data[count] == 0:
         tmp = 1
         count = count + 1

      while data[count] == 1:
         bit_count = bit_count + 1
         count = count + 1

      if bit_count > 3:
         crc = crc + "1"
      else:
         crc = crc + "0"
except:
   print "ERR_RANGE"
   exit(0)

Humidity = bin2dec(HumidityBit)
Temperature = bin2dec(TemperatureBit)

if int(Humidity) + int(Temperature) - int(bin2dec(crc)) == 0:
   print Humidity
   print Temperature
else:
   print "ERR_CRC"<commit_msg>See what we get in data<commit_after>import RPi.GPIO as GPIO
import time

def bin2dec(string_num):
    return str(int(string_num, 2))

data = []

GPIO.setmode(GPIO.BCM)

GPIO.setup(4,GPIO.OUT)
GPIO.output(4,GPIO.HIGH)
time.sleep(0.025)
GPIO.output(4,GPIO.LOW)
time.sleep(0.02)

GPIO.setup(4, GPIO.IN, pull_up_down=GPIO.PUD_UP)

for i in range(0,500):
    data.append(GPIO.input(4))

print "Input data is:\n{}".format(data)

bit_count = 0
tmp = 0
count = 0
HumidityBit = ""
TemperatureBit = ""
crc = ""

try:
   while data[count] == 1:
      tmp = 1
      count = count + 1

   for i in range(0, 32):
      bit_count = 0

      while data[count] == 0:
         tmp = 1
         count = count + 1

      while data[count] == 1:
         bit_count = bit_count + 1
         count = count + 1

      if bit_count > 3:
         if i>=0 and i<8:
            HumidityBit = HumidityBit + "1"
         if i>=16 and i<24:
            TemperatureBit = TemperatureBit + "1"
      else:
         if i>=0 and i<8:
            HumidityBit = HumidityBit + "0"
         if i>=16 and i<24:
            TemperatureBit = TemperatureBit + "0"

except Exception as e:
   print "ERR_RANGE"
   print e
   exit(0)

try:
   for i in range(0, 8):
      bit_count = 0

      while data[count] == 0:
         tmp = 1
         count = count + 1

      while data[count] == 1:
         bit_count = bit_count + 1
         count = count + 1

      if bit_count > 3:
         crc = crc + "1"
      else:
         crc = crc + "0"
except:
   print "ERR_RANGE"
   exit(0)

Humidity = bin2dec(HumidityBit)
Temperature = bin2dec(TemperatureBit)

if int(Humidity) + int(Temperature) - int(bin2dec(crc)) == 0:
   print Humidity
   print Temperature
else:
   print "ERR_CRC"<|endoftext|>


<commit_before>'''
Implementation of the algorithm presented in

> A structured matrix factorization framework for large scale calcium imaging
> data analysis.

by Pnevmatikakis et al. 2014

'''

import numpy as np

# GREEDY INITIALIZATION CODE
# -----------------------------------------------------------------------------

def center_data(data, method='mean'):
    if method == 'mean':
        return data - np.array(data.shape[0]*[data.mean(axis=0)])
    elif method == 'median':
        return data - np.array(data.shape[0]*[np.median(data,axis=0)])
    else:
        raise ValueError('Method "%s" not available for data centering.')

def gauss_kernel(size, var=None):
    if type(size) == int:
        size = (size, size)
    else:
        raise ValueError('We only provide symmetric kernels with sizes specified by an int.')
    if not np.mod(size[0],2) or not np.mod(size[1],2):
        raise ValueError('The size of the Kernel has to be odd.')
    s = (int(size[0]/2.), int(size[0]/2.))
    if var is None: v = s
    elif type(var) == int: v = (var, var)
    else: v = var
    x, y = np.mgrid[-s[0]:s[0]+1, -s[1]:s[1]+1]
    g = np.exp(-(x**2/(2*float(v[0]))+y**2/(2*float(v[1]))))
    return g / g.sum()

def gaussian_blur_matrix(imagesize, kernelsize, kernelvariance):
    dim = imagesize[0]*imagesize[1]
    D = np.zeros((dim, dim))
    for idx in range(dim):
        pos = (int(idx/imagesize[0]), np.mod(idx, imagesize[0]))
        D[:,idx] = gaussian_blur_image(
                imagesize, kernelsize, kernelvariance, pos
                ).flatten()
    return D

def gaussian_blur_image(imagesize, kernelsize, kernelvariance, position):
    pos = position
    ims = imagesize
    ks = kernelsize
    kernel = gauss_kernel(ks, kernelvariance)
    # generate a padded image
    im = np.zeros((ims[0]+ks-1, ims[1]+ks-1))
    im[pos[0]:pos[0]+ks, pos[1]:pos[1]+ks] = kernel
    # remove padding
    im = im[int(ks/2):-int(ks/2), int(ks/2):-int(ks/2)]
    return im

"""
def gaussian_blur_matrix_sparse(dim, kernelsize, kernelvariance):
    '''

    D is a symmetric matrix!

    tested with : plt.imshow(D.toarray()[:100,:100])
    '''
    from scipy import sparse as sps
    kernel = gauss_kernel(kernelsize, kernelvariance).flatten()
    kernel = kernel.flatten()
    s = int(len(kernel)/2)
    data = []
    offsets = []
    for i in range(len(kernel)):
        data.append(np.ones(dim)*kernel[i])
        offsets.append(s-i)
    D = sps.dia_matrix((np.array(data), np.array(offsets)), shape=(dim, dim))
    return D
"""
<commit_msg>smff first working version<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
=====================================================================================

Copyright (c) 2016-2017 Universite de Lorraine & Lulea tekniska universitet
Author: Luca Di Stasio <luca.distasio@gmail.com>
                       <luca.distasio@ingpec.eu>

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the distribution
Neither the name of the Université de Lorraine or Luleå tekniska universitet
nor the names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

=====================================================================================

DESCRIPTION

Tested with Abaqus Python 2.6 (64-bit) distribution in Windows 7.

'''

from os.path import isfile, join, exists
from os import makedirs,listdir
from datetime import datetime
from time import strftime
from platform import platform
import numpy as np
import visualization
import xyPlot
from odbAccess import *
from abaqusConstants import *
from odbMaterial import *
from odbSection import *
import re

def convertDatToCsv(wd,project,datfile,titleline):
    # define csv output folder
    csvfolder = join(wd,project,'csv')
    # define csv output folder
    datfolder = join(wd,project,'dat')
    datafile = join(datfolder,datfile)
    with open(datafile,'r') as dat:
        lines = dat.readlines()
    xy = []
    for line in lines:
        if len(line.replace('\n','').replace(' ',''))>0 and 'X' not in line:
            parts = line.replace('\n','').split(' ')
            temp = []
            for part in parts:
                if part!='':
                    temp.append(float(part))
            xy.append(temp)
    xy = np.array(xy)
    xy = xy[xy[:,0].argsort()]
    with open(join(csvfolder,datfile.split('.')[0]+'.csv'),'w') as csv:
        csv.write(titleline.replace('\n','') + '\n')
        for value in xy:
            line = ''
            for s,subvalue in enumerate(value):
                if s>0:
                    line += ','
                line += str(subvalue)
            csv.write(line + '\n')

def extractJintegralsFromODBoutputSet01(wd,project,nContour):
    print('Starting path extraction on project ' + project + '\n')
    # define database name
    odbname = project + '.odb'
    odbfullpath = join(wd,project,'abaqus',odbname)
    # define input file name
    inpname = project + '.inp'
    inpfullpath = join(wd,project,'abqinp',inpname)
    # define csv output folder
    csvfolder = join(wd,project,'csv')
    # define csv output folder
    datfolder = join(wd,project,'dat')
    print('...done.\n')
    #=======================================================================
    # read crack tips from csv
    #=======================================================================
    crackTips = []
    lines = []
    try:
        with open(join(csvfolder,'ENRRTs-Summary.csv'),'r') as csv:
            lines = csv.readlines()
    except Exception,e:
        sys.exc_clear()
    for line in lines[1:]:
        crackTips.append(int(line.replace('\n','').split(',')[0]))
    #=======================================================================
    # open database
    #=======================================================================
    odb = openOdb(path=odbfullpath)
    sessionOdb = session.openOdb(name=odbfullpath)
    session.viewports['Viewport: 1'].setValues(displayedObject=sessionOdb)
    for t,tip in enumerate(crackTips):
        for contourValue in range(1,nContour+1):
            if contourValue<10:
                contour = '0' + str(contourValue)
            else:
                contour = str(contourValue)
            try:
                J = xyPlot.XYDataFromHistory(odb=sessionOdb,
            outputVariableName='J-integral: J at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_J_Contour' + contour + '.dat'),xyData=J,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_J_Contour' + contour + '.dat','step time T,J')
            except Exception,e:
                sys.exc_clear()
            try:
                JKs = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='J-integral estimated from Ks: JKs at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_JKs_Contour' + contour + '.dat'),xyData=JKs,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_JKs_Contour' + contour + '.dat','step time T,JKs')
            except Exception,e:
                sys.exc_clear()
            try:
                K1 = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='Stress intensity factor K1: K1 at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_K1_Contour' + contour + '.dat'),xyData=K1,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_K1_Contour' + contour + '.dat','step time T,K1')
            except Exception,e:
                sys.exc_clear()
            try:
                K2 = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='Stress intensity factor K2: K2 at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_K2_Contour' + contour + '.dat'),xyData=K2,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_K2_Contour' + contour + '.dat','step time T,K2')
            except Exception,e:
                sys.exc_clear()
            try:
                T = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='T-stress: T at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_T_Contour' + contour + '.dat'),xyData=T,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_T_Contour' + contour + '.dat','step time T,T-stress')
            except Exception,e:
                sys.exc_clear()
    #=======================================================================
    # close database
    #=======================================================================
    print('\n')
    print('Close database...\n')
    odb.close()
    print('...done.\n')

def main(argv):

    wd = 'D:/01_Luca/07_Data/03_FEM'
    matdatafolder = 'D:/OneDrive/01_Luca/07_DocMASE/07_Data/02_Material-Properties'

    statusfile = '2017-03-01_AbaqusParametricRun_2017-03-03_12-56-32.sta'

    with open(join(wd,statusfile),'r') as sta:
        lines = sta.readlines()
    for i,line in enumerate(lines[1:]):
        words = line.split(',')
        project = words[0]
        if not exists(join(wd,project,'dat')):
            makedirs(join(wd,project,'dat'))
        extractJintegralsFromODBoutputSet01(wd,project,10)

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>fixes #113<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
=====================================================================================

Copyright (c) 2016-2017 Universite de Lorraine & Lulea tekniska universitet
Author: Luca Di Stasio <luca.distasio@gmail.com>
                       <luca.distasio@ingpec.eu>

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the distribution
Neither the name of the Université de Lorraine or Luleå tekniska universitet
nor the names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

=====================================================================================

DESCRIPTION

Tested with Abaqus Python 2.6 (64-bit) distribution in Windows 7.

'''

from os.path import isfile, join, exists
from os import makedirs,listdir
from datetime import datetime
from time import strftime
from platform import platform
import numpy as np
import visualization
import xyPlot
from odbAccess import *
from abaqusConstants import *
from odbMaterial import *
from odbSection import *
import re

def convertDatToCsv(wd,project,datfile,titleline):
    # define csv output folder
    csvfolder = join(wd,project,'csv')
    # define csv output folder
    datfolder = join(wd,project,'dat')
    datafile = join(datfolder,datfile)
    with open(datafile,'r') as dat:
        lines = dat.readlines()
    xy = []
    for line in lines:
        if len(line.replace('\n','').replace(' ',''))>0 and 'X' not in line:
            parts = line.replace('\n','').split(' ')
            temp = []
            for part in parts:
                if part!='':
                    temp.append(float(part))
            xy.append(temp)
    xy = np.array(xy)
    xy = xy[xy[:,0].argsort()]
    with open(join(csvfolder,datfile.split('.')[0]+'.csv'),'w') as csv:
        csv.write(titleline.replace('\n','') + '\n')
        for value in xy:
            line = ''
            for s,subvalue in enumerate(value):
                if s>0:
                    line += ','
                line += str(subvalue)
            csv.write(line + '\n')

def extractJintegralsFromODBoutputSet01(wd,project,nContour):
    print('Starting path extraction on project ' + project + '\n')
    # define database name
    odbname = project + '.odb'
    odbfullpath = join(wd,project,'abaqus',odbname)
    # define input file name
    inpname = project + '.inp'
    inpfullpath = join(wd,project,'abqinp',inpname)
    # define csv output folder
    csvfolder = join(wd,project,'csv')
    # define csv output folder
    datfolder = join(wd,project,'dat')
    print('...done.\n')
    #=======================================================================
    # read crack tips from csv
    #=======================================================================
    crackTips = []
    lines = []
    try:
        with open(join(csvfolder,'ENRRTs-Summary.csv'),'r') as csv:
            lines = csv.readlines()
    except Exception,e:
        sys.exc_clear()
    for line in lines[1:]:
        crackTips.append(int(line.replace('\n','').split(',')[0]))
    #=======================================================================
    # open database
    #=======================================================================
    odb = openOdb(path=odbfullpath)
    sessionOdb = session.openOdb(name=odbfullpath)
    session.viewports['Viewport: 1'].setValues(displayedObject=sessionOdb)
    for t,tip in enumerate(crackTips):
        for contourValue in range(1,nContour+1):
            if contourValue<10:
                contour = '0' + str(contourValue)
            else:
                contour = str(contourValue)
            try:
                J = xyPlot.XYDataFromHistory(odb=sessionOdb,
            outputVariableName='J-integral: J at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_J_Contour' + contour + '.dat'),xyData=J,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_J_Contour' + contour + '.dat','step time T,J')
            except Exception,e:
                sys.exc_clear()
            try:
                JKs = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='J-integral estimated from Ks: JKs at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_JKs_Contour' + contour + '.dat'),xyData=JKs,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_JKs_Contour' + contour + '.dat','step time T,JKs')
            except Exception,e:
                sys.exc_clear()
            try:
                K1 = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='Stress intensity factor K1: K1 at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_K1_Contour' + contour + '.dat'),xyData=K1,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_K1_Contour' + contour + '.dat','step time T,K1')
            except Exception,e:
                sys.exc_clear()
            try:
                K2 = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='Stress intensity factor K2: K2 at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_K2_Contour' + contour + '.dat'),xyData=K2,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_K2_Contour' + contour + '.dat','step time T,K2')
            except Exception,e:
                sys.exc_clear()
            try:
                T = xyPlot.XYDataFromHistory(odb=sessionOdb,outputVariableName='T-stress: T at INTERFACEATNODE-' + str(tip) + '_NODE-' + str(tip) + '_Contour_' + contour +' in ELSET  ALL ELEMENTS',steps=(odb.steps.keys()[-1], ), )
                session.writeXYReport(fileName=join(wd,project,'dat','CrackTip' + str(t+1) + '_T_Contour' + contour + '.dat'),xyData=T,appendMode=OFF)
                convertDatToCsv(wd,project,'CrackTip' + str(t+1) + '_T_Contour' + contour + '.dat','step time T,T-stress')
            except Exception,e:
                sys.exc_clear()
    #=======================================================================
    # close database
    #=======================================================================
    print('\n')
    print('Close database...\n')
    odb.close()
    print('...done.\n')

def main(argv):

    wd = 'D:/01_Luca/07_Data/03_FEM'
    matdatafolder = 'D:/OneDrive/01_Luca/07_DocMASE/07_Data/02_Material-Properties'

    statusfile = '2017-03-01_AbaqusParametricRun_2017-03-03_12-56-32.sta'

    with open(join(wd,statusfile),'r') as sta:
        lines = sta.readlines()
    for i,line in enumerate(lines[1:]):
        words = line.split(',')
        project = words[0]
        if not exists(join(wd,project,'dat')):
            makedirs(join(wd,project,'dat'))
        extractJintegralsFromODBoutputSet01(wd,project,10)

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>