<commit_before>"""Create the metadata files describing a WMT component."""

import os
import re
import fnmatch
import warnings
import collections
import json
import yaml
from wmtmetadata.utils import commonpath
from wmtmetadata import metadata_dir

class MetadataBase(object):

    _indent = 2

    def __init__(self, component):
        self.filename = None
        self.data = None
        self._api = component['api']
        self._files = component['files']
        self._info = component['info']
        self._parameters = component['parameters']
        self._provides = component['provides']
        self._uses = component['uses']
        self._component_config_file = os.path.join(metadata_dir,
                                                  self._api['class'],
                                                  'wmt.yaml')
        self._component_config = None

    def _load_component_config(self):
        with open(self._component_config_file, 'r') as fp:
            self._component_config = yaml.load(fp)

    def write(self):
        with open(self.filename, 'w') as fp:
            json.dump(self.data, fp, indent=self._indent)
        if self.filename == 'parameters.json':
            os.chmod(self.filename, 0666)

class Files(MetadataBase):

    def __init__(self, component):
        super(Files, self).__init__(component)
        self.filename = 'files.json'
        self.prefix = self._get_prefix()
        self.data = []
        for name in self._files:
            self.data.append(name[len(self.prefix):])

    def _get_prefix(self):
        if len(self._files) == 1:
            prefix = os.path.dirname(prefix)
        else:
            prefix = commonpath(self._files)
        prefix += os.path.sep
        return prefix

class Info(MetadataBase):

    def __init__(self, component):
        super(Info, self).__init__(component)
        self.filename = 'info.json'
        self.data = self._info.copy()
        for key in ['id', 'name', 'class']:
            self.data[key] = self._api['class']
        try:
            self.data['initialize_args'] = self._api['initialize_args']
        except KeyError:
            warnings.warn('missing initialize_args')
            self.data['initialize_args'] = ''

class Ports(MetadataBase):

    def __init__(self, component):
        super(Ports, self).__init__(component)
        self.data = []
        self._load_component_config()

    def load(self):
        port_type = type(self).__name__.lower()
        port = self._component_config.get(port_type, [])
        for name in port:
            names = []
            for pattern in port[name]['exchange_items']:
                p = re.compile(fnmatch.translate(pattern))
                names.extend(filter(p.match, getattr(self, '_' + port_type)))

            self.data.append({
                'id': name,
                'required': port[name]['required'],
                'exchange_items': names,
            })

class Uses(Ports):

    def __init__(self, component):
        super(Uses, self).__init__(component)
        self.filename = 'uses.json'
        self.load()

class Provides(Ports):

    def __init__(self, component):
        super(Provides, self).__init__(component)
        self.filename = 'provides.json'
        self.load()

def section_break(title):
    return {
        'key': 'separator',
        'name': title,
        'description': title,
        'value': {
            'type': 'string',
            'default': '',
        }
    }

def old_style_parameter(name, param):
    return {
        'key': name,
        'name': param['description'],
        'description': param['description'],
        'value': param['value'],
        'visible': param.get('visible', True),
        'global': param.get('global', False)
    }

class Parameters(MetadataBase):

    def __init__(self, component):
        super(Parameters, self).__init__(component)
        self.filename = 'parameters.json'
        self.data = []
        self._load_component_config()

        if len(self._provides) > 0:
            self.add_outputs()

        if self._component_config.has_key('extras'):
            self._parameters.update(self._component_config['extras'])

        if self._component_config.has_key('ignore'):
            self._hide_ignored()

        if self._component_config.has_key('globals'):
            driver = section_break('Globals')
            driver['global'] = True
            self.data.append(driver)
            for name in self._component_config.get('globals', []):
                if name in self._parameters.keys():
                    self._parameters[name]['global'] = True

        self.make_sections()
        self.make_groups()
        self.make_mappers()

    def _hide_ignored(self):
        ignored = []
        for pattern in self._component_config['ignore']:
            p = re.compile(fnmatch.translate(pattern))
            ignored.extend(filter(p.match, params))

        for name in ignored:
            if name in self._parameters:
                self._parameters[name]['visible'] = False
            else:
                if name not in self._parameters.keys():
                    msg = '{name}: pattern not in params'.format(name)
                    warnings.warn(msg)

    def make_sections(self):
        added = set()

        for name in self._parameters.keys():
            if self._parameters[name].has_key('global') and \
               self._parameters[name]['global'] is True:
                try:
                    self.data.append(
                        old_style_parameter(name, self._parameters[name]))
                except KeyError:
                    warnings.warn('{}: missing parameter'.format(name))
                else:
                    added.add(name)

        for section in self._component_config.get('sections', []):
            self.data.append(section_break(section['title']))
            for member in section['members']:
                try:
                    self.data.append(
                        old_style_parameter(member, self._parameters[member]))
                except KeyError:
                    warnings.warn('{}: missing parameter'.format(name))
                else:
                    added.add(member)

        missing = set(self._parameters.keys()) - added
        if len(missing) > 0:
            for name in missing:
                self._parameters[name]['visible'] = False
                self.data.append(
                    old_style_parameter(name, self._parameters[name]))

    def make_groups(self):
        groups = self._component_config.get('groups', {})
        for name, members in groups.items():
            for item in self.data:
                if item['key'] in members:
                    item['group'] = {
                        "name": name,
                        "leader": item['key'] == members[0],
                        "members": len(members),
                    }

    def _get_mapped_parameters(self, roles):
        mapped_params = [roles['selector']]
        for mapping in roles['selections']:
            param = mapping.values()[0]
            try:
                mapped_params.index(param)
            except ValueError:
                mapped_params.append(param)
        return mapped_params

    def _get_mappings(self, selections):
        mappings = collections.OrderedDict()
        for mapping in selections:
            for ptype, param in mapping.items():
                mappings[ptype] = param
        return mappings

    def make_mappers(self):
        mappers = self._component_config.get('parameter_mappers', {})
        for mapper, roles in mappers.items():
            mapped_params = self._get_mapped_parameters(roles)
            for item in self.data:
                if item['key'] in mapped_params:
                    item['selection'] = {
                        'name': mapper,
                        'selector': roles['selector'] == item['key'],
                        'members': len(mapped_params),
                        }
                    if item['selection']['selector'] is True:
                        item['selection']['mapping'] = \
                            self._get_mappings(roles['selections'])

    def add_outputs(self):
        print_section = {
            'title': 'Output files',
            'members': [],
        }

        for name in self._provides:
            self._parameters[name] = {
                'description': name,
                'value': {
                    'type': 'choice',
                    'default': 'off',
                    'choices': [
                        'off',
                        name
                    ]
                }
            }
            print_section['members'].append(name)

        self._component_config['sections'].append(print_section)
<commit_msg>Fix incorrect variable name<commit_after>"""Create the metadata files describing a WMT component."""

import os
import re
import fnmatch
import warnings
import collections
import json
import yaml
from wmtmetadata.utils import commonpath
from wmtmetadata import metadata_dir

class MetadataBase(object):

    _indent = 2

    def __init__(self, component):
        self.filename = None
        self.data = None
        self._api = component['api']
        self._files = component['files']
        self._info = component['info']
        self._parameters = component['parameters']
        self._provides = component['provides']
        self._uses = component['uses']
        self._component_config_file = os.path.join(metadata_dir,
                                                  self._api['class'],
                                                  'wmt.yaml')
        self._component_config = None

    def _load_component_config(self):
        with open(self._component_config_file, 'r') as fp:
            self._component_config = yaml.load(fp)

    def write(self):
        with open(self.filename, 'w') as fp:
            json.dump(self.data, fp, indent=self._indent)
        if self.filename == 'parameters.json':
            os.chmod(self.filename, 0666)

class Files(MetadataBase):

    def __init__(self, component):
        super(Files, self).__init__(component)
        self.filename = 'files.json'
        self.prefix = self._get_prefix()
        self.data = []
        for name in self._files:
            self.data.append(name[len(self.prefix):])

    def _get_prefix(self):
        if len(self._files) == 1:
            prefix = os.path.dirname(prefix)
        else:
            prefix = commonpath(self._files)
        prefix += os.path.sep
        return prefix

class Info(MetadataBase):

    def __init__(self, component):
        super(Info, self).__init__(component)
        self.filename = 'info.json'
        self.data = self._info.copy()
        for key in ['id', 'name', 'class']:
            self.data[key] = self._api['class']
        try:
            self.data['initialize_args'] = self._api['initialize_args']
        except KeyError:
            warnings.warn('missing initialize_args')
            self.data['initialize_args'] = ''

class Ports(MetadataBase):

    def __init__(self, component):
        super(Ports, self).__init__(component)
        self.data = []
        self._load_component_config()

    def load(self):
        port_type = type(self).__name__.lower()
        port = self._component_config.get(port_type, [])
        for name in port:
            names = []
            for pattern in port[name]['exchange_items']:
                p = re.compile(fnmatch.translate(pattern))
                names.extend(filter(p.match, getattr(self, '_' + port_type)))

            self.data.append({
                'id': name,
                'required': port[name]['required'],
                'exchange_items': names,
            })

class Uses(Ports):

    def __init__(self, component):
        super(Uses, self).__init__(component)
        self.filename = 'uses.json'
        self.load()

class Provides(Ports):

    def __init__(self, component):
        super(Provides, self).__init__(component)
        self.filename = 'provides.json'
        self.load()

def section_break(title):
    return {
        'key': 'separator',
        'name': title,
        'description': title,
        'value': {
            'type': 'string',
            'default': '',
        }
    }

def old_style_parameter(name, param):
    return {
        'key': name,
        'name': param['description'],
        'description': param['description'],
        'value': param['value'],
        'visible': param.get('visible', True),
        'global': param.get('global', False)
    }

class Parameters(MetadataBase):

    def __init__(self, component):
        super(Parameters, self).__init__(component)
        self.filename = 'parameters.json'
        self.data = []
        self._load_component_config()

        if len(self._provides) > 0:
            self.add_outputs()

        if self._component_config.has_key('extras'):
            self._parameters.update(self._component_config['extras'])

        if self._component_config.has_key('ignore'):
            self._hide_ignored()

        if self._component_config.has_key('globals'):
            driver = section_break('Globals')
            driver['global'] = True
            self.data.append(driver)
            for name in self._component_config.get('globals', []):
                if name in self._parameters.keys():
                    self._parameters[name]['global'] = True

        self.make_sections()
        self.make_groups()
        self.make_mappers()

    def _hide_ignored(self):
        ignored = []
        for pattern in self._component_config['ignore']:
            p = re.compile(fnmatch.translate(pattern))
            ignored.extend(filter(p.match, self._parameters))

        for name in ignored:
            if name in self._parameters:
                self._parameters[name]['visible'] = False
            else:
                if name not in self._parameters.keys():
                    msg = '{name}: pattern not in params'.format(name)
                    warnings.warn(msg)

    def make_sections(self):
        added = set()

        for name in self._parameters.keys():
            if self._parameters[name].has_key('global') and \
               self._parameters[name]['global'] is True:
                try:
                    self.data.append(
                        old_style_parameter(name, self._parameters[name]))
                except KeyError:
                    warnings.warn('{}: missing parameter'.format(name))
                else:
                    added.add(name)

        for section in self._component_config.get('sections', []):
            self.data.append(section_break(section['title']))
            for member in section['members']:
                try:
                    self.data.append(
                        old_style_parameter(member, self._parameters[member]))
                except KeyError:
                    warnings.warn('{}: missing parameter'.format(name))
                else:
                    added.add(member)

        missing = set(self._parameters.keys()) - added
        if len(missing) > 0:
            for name in missing:
                self._parameters[name]['visible'] = False
                self.data.append(
                    old_style_parameter(name, self._parameters[name]))

    def make_groups(self):
        groups = self._component_config.get('groups', {})
        for name, members in groups.items():
            for item in self.data:
                if item['key'] in members:
                    item['group'] = {
                        "name": name,
                        "leader": item['key'] == members[0],
                        "members": len(members),
                    }

    def _get_mapped_parameters(self, roles):
        mapped_params = [roles['selector']]
        for mapping in roles['selections']:
            param = mapping.values()[0]
            try:
                mapped_params.index(param)
            except ValueError:
                mapped_params.append(param)
        return mapped_params

    def _get_mappings(self, selections):
        mappings = collections.OrderedDict()
        for mapping in selections:
            for ptype, param in mapping.items():
                mappings[ptype] = param
        return mappings

    def make_mappers(self):
        mappers = self._component_config.get('parameter_mappers', {})
        for mapper, roles in mappers.items():
            mapped_params = self._get_mapped_parameters(roles)
            for item in self.data:
                if item['key'] in mapped_params:
                    item['selection'] = {
                        'name': mapper,
                        'selector': roles['selector'] == item['key'],
                        'members': len(mapped_params),
                        }
                    if item['selection']['selector'] is True:
                        item['selection']['mapping'] = \
                            self._get_mappings(roles['selections'])

    def add_outputs(self):
        print_section = {
            'title': 'Output files',
            'members': [],
        }

        for name in self._provides:
            self._parameters[name] = {
                'description': name,
                'value': {
                    'type': 'choice',
                    'default': 'off',
                    'choices': [
                        'off',
                        name
                    ]
                }
            }
            print_section['members'].append(name)

        self._component_config['sections'].append(print_section)
<|endoftext|>


<commit_before>import subprocess
import fileinput
import shutil
from skimage import io, util
import os
from PIL import Image
import tqdm
from keras.preprocessing.image import ImageDataGenerator, array_to_img, \
    img_to_array, load_img

'''
Simple python script to generate data from Finesse
First we run Finesse iterated over mode numbers, m, n
(Just pure modes, no superpositions)
Fix up the GNUplot raw image
Then use Keras preprocessing package to generate modified data
(rotated, skewed, zoomed)
Then we add noise (after Keras, otherwise wacky things happen)
Ready to feed into neural network
'''
FINESSEDIR = "../../finesse/./kat"
# saturation levels
colourRange = ['2', '2e-1', '2e-2', '2e-3', '2e-4']
colourName = ['0', '1', '2', '3', '4']
# max mode for n and m to generate
maxMode = 6
# open template kat file
originalKat = open('cavity.kat', 'r')

# Define data folders
dataFolder = 'rawData'
newDataFolder = 'newData'

# Image extension
ext = '.png'

# No. images to generate per raw image
imageNum = 20

# loop over m, then n, then colour range
for m in tqdm(range(maxMode)):
    for n in range(maxMode):
        for c in range(5):

                # generate names
                current = "cavity"+str(m)+str(n)+colourName[c]
                generateFileName = current+".kat"

                # make and/or change directory
                if not os.path.exists(dataFolder): os.mkdir(dataFolder)
                shutil.copy("cavity.kat", 'rawData/'+generateFileName)
                os.chdir('rawData')

                # generate new lines for .kat
                temLine = "tem laser "+str(m)+" "+str(n)+" 0.1 0.0" #replace lines
                colourLine = "set cbrange[0:"+colourRange[c]+"]"

                # replacing lines in .kat
                with fileinput.FileInput(generateFileName, inplace=True, backup='.bak') as file:
                        for line in file:
                            line = line.rstrip()
                            if "#replaceMode" in line:
                                line = line.replace("#replaceMode", temLine)
                            elif "#replaceColour" in line:
                                line = line.replace("#replaceColour", colourLine)
                            print(line)

                # run the new kat file
                subprocess.run([FINESSEDIR, generateFileName])

                # crop generated image
                # GNUplot generates a margin for some reason
                # assume square image
                image = Image.open(current+ext)
                width = image.size[0]
                height = width
                offset = height-1
                cropped = image.crop((width-offset, height-offset, width, height))
                cropped.save(current+ext)

                # cleaning files
                fileList = os.listdir()
                notImages = [x for x in fileList if not ext in x]
                for fileName in notImages: os.remove(fileName)

                #move back up directory
                os.chdir('..')

originalKat.close()

# Preprocessing
# Data generators, using Keras
trainDatagen = ImageDataGenerator(
        rotation_range=30,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        fill_mode='nearest')
testDatagen = ImageDataGenerator(
        rotation_range=25,
        width_shift_range=0.1,
        height_shift_range=0.1,
        shear_range=0.3,
        zoom_range=0.1,
        horizontal_flip=True,
        fill_mode='nearest')
#
os.chdir(dataFolder)
files = os.listdir()
imageList = [x for x in files if '.png' in x]
os.chdir('..')

possibleModes = [str(m)+str(n) for m in range(maxMode) for n in range(maxMode)]

# Create file structure if it does not exist
if not os.path.exists(newDataFolder): os.mkdir(newDataFolder)

# Generators which read pictures in subfolders of training and validation,
# indefinitely generate batches of augmented cavity images

print('Preprocessing raw data set')
for mode in tqdm(possibleModes):
       for image in [x for x in files if 'cavity'+mode in x]:
        loaded = load_img('rawData/'+image)
        array = img_to_array(loaded)
        array = array.reshape((1,)+array.shape)
        i = 0
        for batch in trainDatagen.flow(array, batch_size=1, save_to_dir=newDataFolder,
                                       save_prefix=mode, save_format=ext):
            i += 1
            if i > imageNum: break
        print('generated '+image+' set')

# Adding noise
def noisy(img):
    noisy = util.random_noise(loaded, mode='gaussian', clip=True)
    return noisy

for image in tqdm(newDataFolder):
    loaded = io.imread(newDataFolder+'/'+image)
    io.imsave(image, noisy(image))
<commit_msg>Added comments<commit_after># Running Finesse
import subprocess
# Editing kat file
import fileinput
# Copying files
import shutil
# Adding Gaussian noise
from skimage import io, util
# Navigating directories
import os
# Cropping raw images
from PIL import Image
# Nice loading bars for loops
import tqdm
# Keras preprocessing package for augmentation
from keras.preprocessing.image import ImageDataGenerator, array_to_img, \
    img_to_array, load_img

'''
Simple python script to generate data from Finesse
First we run Finesse iterated over mode numbers, m, n
(Just pure modes, no superpositions)
Fix up the GNUplot raw image
Then use Keras preprocessing package to generate modified data
(rotated, skewed, zoomed)
Then we add noise (after Keras, otherwise wacky things happen)
Ready to feed into neural network
'''

FINESSEDIR = "../../finesse/./kat"
# saturation levels
colourRange = ['2', '2e-1', '2e-2', '2e-3', '2e-4']
colourName = ['0', '1', '2', '3', '4']
# max mode for n and m to generate
maxMode = 6
# open template kat file
originalKat = open('cavity.kat', 'r')

# Define data folders
dataFolder = 'rawData'
newDataFolder = 'newData'

# Image extension
ext = '.png'

# No. images to generate per raw image
imageNum = 20

# loop over m, then n, then colour range
for m in tqdm(range(maxMode)):
    for n in range(maxMode):
        for c in range(5):

                # generate names
                current = "cavity"+str(m)+str(n)+colourName[c]
                generateFileName = current+".kat"

                # make and/or change directory
                if not os.path.exists(dataFolder): os.mkdir(dataFolder)
                shutil.copy("cavity.kat", 'rawData/'+generateFileName)
                os.chdir('rawData')

                # generate new lines for .kat
                temLine = "tem laser "+str(m)+" "+str(n)+" 0.1 0.0" #replace lines
                colourLine = "set cbrange[0:"+colourRange[c]+"]"

                # replacing lines in .kat
                with fileinput.FileInput(generateFileName, inplace=True, backup='.bak') as file:
                        for line in file:
                            line = line.rstrip()
                            if "#replaceMode" in line:
                                line = line.replace("#replaceMode", temLine)
                            elif "#replaceColour" in line:
                                line = line.replace("#replaceColour", colourLine)
                            print(line)

                # run the new kat file
                subprocess.run([FINESSEDIR, generateFileName])

                # crop generated image
                # GNUplot generates a margin for some reason
                # assume square image
                image = Image.open(current+ext)
                width = image.size[0]
                height = width
                offset = height-1
                cropped = image.crop((width-offset, height-offset, width, height))
                cropped.save(current+ext)

                # cleaning files
                fileList = os.listdir()
                notImages = [x for x in fileList if not ext in x]
                for fileName in notImages: os.remove(fileName)

                #move back up directory
                os.chdir('..')

originalKat.close()

# Preprocessing
# Data generators, using Keras
trainDatagen = ImageDataGenerator(
        rotation_range=30,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        fill_mode='nearest')
testDatagen = ImageDataGenerator(
        rotation_range=25,
        width_shift_range=0.1,
        height_shift_range=0.1,
        shear_range=0.3,
        zoom_range=0.1,
        horizontal_flip=True,
        fill_mode='nearest')
#
os.chdir(dataFolder)
files = os.listdir()
imageList = [x for x in files if '.png' in x]
os.chdir('..')

possibleModes = [str(m)+str(n) for m in range(maxMode) for n in range(maxMode)]

# Create file structure if it does not exist
if not os.path.exists(newDataFolder): os.mkdir(newDataFolder)

# Generators which read pictures in subfolders of training and validation,
# indefinitely generate batches of augmented cavity images

print('Preprocessing raw data set')
for mode in tqdm(possibleModes):
       for image in [x for x in files if 'cavity'+mode in x]:
        loaded = load_img('rawData/'+image)
        array = img_to_array(loaded)
        array = array.reshape((1,)+array.shape)
        i = 0
        for batch in trainDatagen.flow(array, batch_size=1, save_to_dir=newDataFolder,
                                       save_prefix=mode, save_format=ext):
            i += 1
            if i > imageNum: break
        print('generated '+image+' set')

# Adding noise
def noisy(img):
    noisy = util.random_noise(loaded, mode='gaussian', clip=True)
    return noisy

for image in tqdm(newDataFolder):
    loaded = io.imread(newDataFolder+'/'+image)
    io.imsave(image, noisy(image))
<|endoftext|>


<commit_before>import tensorflow as tf

from .cell import ln_lstm_cell as _DEFAULT_CELL
from ..util import funcname_scope, dimension_indices

@funcname_scope
def rnn(input_embeddings,
        *,
        output_embedding_size,
        dropout_prob,
        sequence_length=None):
  return _only_outputs(tf.nn.rnn(
      _DEFAULT_CELL(output_embedding_size, dropout_prob),
      _split_input_embeddings(input_embeddings),
      sequence_length=sequence_length,
      dtype=input_embeddings.dtype))

@funcname_scope
def bidirectional_rnn(input_embeddings,
                      *,
                      output_embedding_size,
                      dropout_prob,
                      sequence_length=None):
  assert output_embedding_size % 2 == 0
  cell = lambda: _DEFAULT_CELL(output_embedding_size // 2, dropout_prob)

  return _only_outputs(tf.nn.bidirectional_rnn(
      cell(),
      cell(),
      _split_input_embeddings(input_embeddings),
      sequence_length=sequence_length,
      dtype=input_embeddings.dtype))

def _only_outputs(rnn_return_values):
  return _pack_cell_outputs_in_batch_seq_embed_order(rnn_return_values[0])

@funcname_scope
def _pack_cell_outputs_in_batch_seq_embed_order(cell_outputs):
  return tf.transpose(tf.pack(cell_outputs), [1, 0, 2])

@funcname_scope
def _split_input_embeddings(input_embeddings):
  return tf.unpack(tf.transpose(
      input_embeddings,
      [1, 0] + dimension_indices(input_embeddings, 2)))
<commit_msg>Use FLAGS.dropout_prob<commit_after>import tensorflow as tf

from .cell import ln_lstm_cell as _DEFAULT_CELL
from ..util import funcname_scope, dimension_indices
from ..flags import FLAGS

@funcname_scope
def rnn(input_embeddings,
        *,
        output_embedding_size,
        dropout_prob,
        sequence_length=None):
  return _only_outputs(tf.nn.rnn(
      _DEFAULT_CELL(output_embedding_size, dropout_prob),
      _split_input_embeddings(input_embeddings),
      sequence_length=sequence_length,
      dtype=input_embeddings.dtype))

@funcname_scope
def bidirectional_rnn(input_embeddings,
                      *,
                      output_embedding_size,
                      dropout_prob=FLAGS.dropout_prob,
                      sequence_length=None):
  assert output_embedding_size % 2 == 0
  cell = lambda: _DEFAULT_CELL(output_embedding_size // 2, dropout_prob)

  return _only_outputs(tf.nn.bidirectional_rnn(
      cell(),
      cell(),
      _split_input_embeddings(input_embeddings),
      sequence_length=sequence_length,
      dtype=input_embeddings.dtype))

def _only_outputs(rnn_return_values):
  return _pack_cell_outputs_in_batch_seq_embed_order(rnn_return_values[0])

@funcname_scope
def _pack_cell_outputs_in_batch_seq_embed_order(cell_outputs):
  return tf.transpose(tf.pack(cell_outputs), [1, 0, 2])

@funcname_scope
def _split_input_embeddings(input_embeddings):
  return tf.unpack(tf.transpose(
      input_embeddings,
      [1, 0] + dimension_indices(input_embeddings, 2)))
<|endoftext|>


<commit_before>'''
Created on 26 Mar 2013

@author: michael
'''

def get_poll_percentages(poll_answers):
    total_vote_counts = 0.0
    vote_count_averages = []

    for poll_answer in poll_answers:
        total_vote_counts += poll_answer.vote_count

    for poll_answer in poll_answers:
        vote_count_averages.append(float('%0.2f' % ((poll_answer.vote_count / total_vote_counts) * 100)))

    return vote_count_averages<commit_msg>fixed issue on zerodivionerror for calculating vote averages<commit_after>'''
Created on 26 Mar 2013

@author: michael
'''

def get_poll_percentages(poll_answers):
    total_vote_counts = 0.0
    vote_count_averages = []

    for poll_answer in poll_answers:
        total_vote_counts += poll_answer.vote_count

    for poll_answer in poll_answers:
        try:
            vote_count_averages.append(float('%0.2f' % ((poll_answer.vote_count / total_vote_counts) * 100)))
        except ZeroDivisionError:
            vote_count_averages.append(0.0)

    return vote_count_averages<|endoftext|>


<commit_before>from interfaces import ISipPhone

from twisted.internet.protocol import DatagramProtocol
from twisted.protocols import sip as tpsip
from rtp import RTPProtocol

from twisted.python import log
import random

import prefs

def genCallId():
    return 400000000 + random.randint(0,2000000)

class Call(object):
    def __init__(self, to):
        self.cseq = random.randint(1000,5000)
        self.rtp = None
        self.url = tpsip.parseURL(to)
        self.setLocalIP((self.url.host, self.url.port or 5060))
        self.setCallID()
        self.state = 'NONE'

    def setCallID(self):
        self._callID = '%s@%s'%(genCallId(), self.getLocalSIPAddress()[0])

    def setState(self, state):
        self.state = state

    def getState(self):
        return self.state

    def setLocalIP(self, (host,port)):
        """ Try and determine the local IP address to use. We do a connect_ex
            in the (faint?) hope that on a machine with multiple interfaces,
            we'll get the right one
        """
        # XXX Allow over-riding
        import prefs
        if prefs.myip is not None:
            self._localIP = prefs.myip
        else:
            import socket
            rsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            rsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            rsock.connect_ex((host,port))
            self._localIP = rsock.getsockname()[0]
            rsock.close()
        log.msg("using local IP address %s"%(self._localIP))

    def getCSeq(self, incr=0):
        if incr:
            self.cseq += 1
        return self.cseq

    def getCallID(self):
        return self._callID

    def getLocalSIPAddress(self):
        return (self._localIP, 5060)

    def getRemoteSIPAddress(self):
        return (self.url.host, (self.url.port or 5060))

    def setupRTP(self):
        self.rtp = RTPProtocol()
        self._lrtpPort, self._lrtcpPort = self.rtp.createRTPSocket()

    def teardownRTP(self):
        self.rtp.stopSendingAndReceiving()

    def genInvite(self, toAddr):
        from multicast.SDP import SimpleSDP
        self.setLocalIP((self.url.host, self.url.port or 5060))
        invite = tpsip.Request('INVITE', str(self.url))
        invite.addHeader('via', 'SIP/2.0/UDP %s;rport'%(
                                    self.getLocalSIPAddress()[0]))
        invite.addHeader('cseq', '%s INVITE'%self.getCSeq(incr=1))
        invite.addHeader('to', str(self.url))
        invite.addHeader('content-type', 'application/sdp')
        invite.addHeader('from', '"%s" <sip:%s>;tag=%s'%(
                                    prefs.myname, prefs.myemail, prefs.mytag))
        invite.addHeader('call-id', self.getCallID())
        invite.addHeader('subject', 'sip%s'%(prefs.myemail))
        invite.addHeader('user-agent', 'Shtoom/0.0')
        invite.addHeader('contact', '"%s" <sip:%s;transport=udp>'%(
                                    prefs.myname, self.getLocalSIPAddress()[0]))
        s = SimpleSDP()
        s.setPacketSize(160)
        s.setServerIP(self.getLocalSIPAddress()[0])
        s.setLocalPort(self._lrtpPort)
        sdp = s.show()
        invite.addHeader('content-length', len(sdp))
        invite.bodyDataReceived(sdp)
        invite.creationFinished()
        return invite.toString()

    def genAck(self, okmessage, startRTP=0):
        from multicast.SDP import SDP
        sdp = SDP(okmessage.body)
        log.msg("RTP port is %s"%sdp.port)
        contact = okmessage.headers['contact']
        if type(contact) is list:
            contact = contact[0]
        if contact.startswith('<') and contact.endswith('>'):
            contact = contact[1:-1]
        self.contact = contact
        to = okmessage.headers['to']
        if type(to) is list:
            to = to[0]
        self.to = to

        url = tpsip.parseURL(to)
        ack = tpsip.Request('ACK', contact)
        ack.addHeader('via', 'SIP/2.0/UDP %s;rport'%self.getLocalSIPAddress()[0])
        ack.addHeader('cseq', '%s ACK'%self.getCSeq())
        ack.addHeader('to', to)
        ack.addHeader('from', '"%s" <sip:%s>;tag=%s'%(prefs.myname, prefs.myemail, prefs.mytag))
        ack.addHeader('call-id', self.getCallID())
        ack.addHeader('user-agent', 'Shtoom/0.1')
        ack.addHeader('content-length', 0)
        ack.creationFinished()
        self._sentAck = ack.toString(), (url.host, (url.port or 5060))
        if startRTP:
            self.rtp.startSendingAndReceiving((url.host,sdp.port))
        return self._sentAck

    def genBye(self):
        url = tpsip.parseURL(self.to)
        bye = tpsip.Request('BYE', self.contact)
        bye.addHeader('via', 'SIP/2.0/UDP %s;rport'%self.getLocalSIPAddress()[0])
        bye.addHeader('cseq', '%s BYE'%self.getCSeq(incr=1))
        bye.addHeader('to', self.to)
        bye.addHeader('from', '"%s" <sip:%s>;tag=%s'%(prefs.myname, prefs.myemail, prefs.mytag))
        bye.addHeader('call-id', self.getCallID())
        bye.addHeader('user-agent', 'Shtoom/0.1')
        bye.addHeader('content-length', 0)
        bye.creationFinished()
        self._sentBye = bye.toString(), (url.host, (url.port or 5060))
        return self._sentBye

class SipPhone(DatagramProtocol, object):
    """ A SIP phone
    """
    __implements__ = ISipPhone

    def __init__(self, ui, *args, **kwargs):
        self.ui = ui
        self._calls = {}
        super(SipPhone, self, *args, **kwargs)

    def _newCallObject(self, to):
        call = Call(to)
        self._calls[call.getCallID()] = call
        return call

    def _getCallObject(self, callid):
        if type(callid) is list:
            callid = callid[0]
        if callid.startswith('<') and callid.endswith('>'):
            callid = callid[1:-1]
        return self._calls[callid]

    def _delCallObject(self, callid):
        if type(callid) is list:
            callid = callid[0]
        if callid.startswith('<') and callid.endswith('>'):
            callid = callid[1:-1]
        del self._calls[callid]

    def placeCall(self, url):
        """ Place a call to 'url'

            Returns callid, a string
        """
        self.ui.debugMessage("placeCall starting")
        call = self._newCallObject(url)
        call.setupRTP()
        invite = call.genInvite(url)
        self.transport.write(invite, call.getRemoteSIPAddress())
        # Set up a callLater in case we don't get a response, for a retransmit
        call.setState('SENT_INVITE')
        return call.getCallID()

    def dropCall(self, callid):
        """ Drop call identified by 'callid'
        """

        call = self._getCallObject(callid)
        if not call:
            self.ui.debugMessage("no such call %s"%callid)
            return
        state = call.getState()
        if state == 'NONE':
            self.ui.debugMessage("no call to drop")
        elif state in ( 'SENT_ACK', 'SENT_INVITE'):
            bye, dest = call.genBye()
            self.transport.write(bye, dest)
            self.ui.debugMessage("sent a BYE message")
            call.setState('SENT_BYE')

    def startDTMF(self, digit):
        """ Start sending DTMF digit 'digit'
        """
        self.ui.debugMessage("startDTMF not implemented yet!")

    def stopDTMF(self):
        """ Stop sending DTMF digit 'digit'
        """
        self.ui.debugMessage("stopDTMF not implemented yet!")

    def datagramReceived(self, datagram, addr):
        self.ui.debugMessage("Got a SIP packet from %s:%s"%(addr))
        mp = tpsip.MessagesParser(self.sipMessageReceived)
        self.ui.debugMessage("SIP PACKET\n%s"%(datagram))
        mp.dataReceived(datagram)
        mp.dataDone()

    def sipMessageReceived(self, message):
        self.ui.debugMessage("got SIP response %s: %s"%(
                                        message.code, message.phrase))
        self.ui.statusMessage("%s: %s"%(message.code,message.phrase))
        self.ui.debugMessage("got SIP response\n %s"%( message.toString()))
        callid = message.headers['call-id']
        call = self._getCallObject(callid)
        if not call:
            self.ui.debugMessage("SIP response refers to unknown call %s"%(
                                                                    callid))
            return
        if message.code in ( 100, 180, 181, 182 ):
            return
        elif message.code == 200:
            state = call.getState()
            if state == 'SENT_INVITE':
                self.ui.debugMessage(message.body)
                ack, dest = call.genAck(message, startRTP=1)
                self.transport.write(ack, dest)
                call.setState('SENT_ACK')
            elif state == 'SENT_ACK':
                self.ui.debugMessage('Got duplicate OK to our ACK')
                ack, dest = call.genAck(message)
                self.transport.write(ack, dest)
            elif state == 'SENT_BYE':
                call.teardownRTP()
                self._delCallObject(callid)
                self.ui.debugMessage("Hung up on call %s"%callid)
                self.ui.statusMessage("idle")
            else:
                self.ui.debugMessage('Got OK in unexpected state %s'%state)
        else:
            self.ui.debugMessage(message.toString())

<commit_msg>Woo codecs.<commit_after>from interfaces import ISipPhone

from twisted.internet.protocol import DatagramProtocol
from twisted.protocols import sip as tpsip
from rtp import RTPProtocol

from twisted.python import log
import random

import prefs

def genCallId():
    return 400000000 + random.randint(0,2000000)

class Call(object):
    def __init__(self, to):
        self.cseq = random.randint(1000,5000)
        self.rtp = None
        self.url = tpsip.parseURL(to)
        self.setLocalIP((self.url.host, self.url.port or 5060))
        self.setCallID()
        self.state = 'NONE'

    def setCallID(self):
        self._callID = '%s@%s'%(genCallId(), self.getLocalSIPAddress()[0])

    def setState(self, state):
        self.state = state

    def getState(self):
        return self.state

    def setLocalIP(self, (host,port)):
        """ Try and determine the local IP address to use. We do a connect_ex
            in the (faint?) hope that on a machine with multiple interfaces,
            we'll get the right one
        """
        # XXX Allow over-riding
        import prefs
        if prefs.myip is not None:
            self._localIP = prefs.myip
        else:
            import socket
            rsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            rsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            rsock.connect_ex((host,port))
            self._localIP = rsock.getsockname()[0]
            rsock.close()
        log.msg("using local IP address %s"%(self._localIP))

    def getCSeq(self, incr=0):
        if incr:
            self.cseq += 1
        return self.cseq

    def getCallID(self):
        return self._callID

    def getLocalSIPAddress(self):
        return (self._localIP, 5060)

    def getRemoteSIPAddress(self):
        return (self.url.host, (self.url.port or 5060))

    def setupRTP(self):
        self.rtp = RTPProtocol()
        self._lrtpPort, self._lrtcpPort = self.rtp.createRTPSocket()

    def teardownRTP(self):
        self.rtp.stopSendingAndReceiving()

    def genInvite(self, toAddr):
        from multicast.SDP import SimpleSDP
        self.setLocalIP((self.url.host, self.url.port or 5060))
        invite = tpsip.Request('INVITE', str(self.url))
        invite.addHeader('via', 'SIP/2.0/UDP %s;rport'%(
                                    self.getLocalSIPAddress()[0]))
        invite.addHeader('cseq', '%s INVITE'%self.getCSeq(incr=1))
        invite.addHeader('to', str(self.url))
        invite.addHeader('content-type', 'application/sdp')
        invite.addHeader('from', '"%s" <sip:%s>;tag=%s'%(
                                    prefs.myname, prefs.myemail, prefs.mytag))
        invite.addHeader('call-id', self.getCallID())
        invite.addHeader('subject', 'sip%s'%(prefs.myemail))
        invite.addHeader('user-agent', 'Shtoom/0.0')
        invite.addHeader('contact', '"%s" <sip:%s;transport=udp>'%(
                                    prefs.myname, self.getLocalSIPAddress()[0]))
        s = SimpleSDP()
        s.setPacketSize(160)
        s.setServerIP(self.getLocalSIPAddress()[0])
        s.setLocalPort(self._lrtpPort)
        s.addRtpMap(0, 'PCMU', 8000, 1)
        #s.addRtpMap(3, 'GSM', 8000, 1)
        #s.addRtpMap(5, 'DVI4', 8000, 1)
        #s.addRtpMap(6, 'DVI4', 16000, 1)
        #s.addRtpMap(110, 'speex', 8000, 1)
        #s.addRtpMap(111, 'speex', 16000, 1)
        sdp = s.show()
        invite.addHeader('content-length', len(sdp))
        invite.bodyDataReceived(sdp)
        invite.creationFinished()
        return invite.toString()

    def genAck(self, okmessage, startRTP=0):
        from multicast.SDP import SDP
        sdp = SDP(okmessage.body)
        log.msg("RTP port is %s"%sdp.port)
        contact = okmessage.headers['contact']
        if type(contact) is list:
            contact = contact[0]
        if contact.startswith('<') and contact.endswith('>'):
            contact = contact[1:-1]
        self.contact = contact
        to = okmessage.headers['to']
        if type(to) is list:
            to = to[0]
        self.to = to

        url = tpsip.parseURL(to)
        ack = tpsip.Request('ACK', contact)
        ack.addHeader('via', 'SIP/2.0/UDP %s;rport'%self.getLocalSIPAddress()[0])
        ack.addHeader('cseq', '%s ACK'%self.getCSeq())
        ack.addHeader('to', to)
        ack.addHeader('from', '"%s" <sip:%s>;tag=%s'%(prefs.myname, prefs.myemail, prefs.mytag))
        ack.addHeader('call-id', self.getCallID())
        ack.addHeader('user-agent', 'Shtoom/0.1')
        ack.addHeader('content-length', 0)
        ack.creationFinished()
        self._sentAck = ack.toString(), (url.host, (url.port or 5060))
        if startRTP:
            self.rtp.startSendingAndReceiving((url.host,sdp.port))
        return self._sentAck

    def genBye(self):
        url = tpsip.parseURL(self.to)
        bye = tpsip.Request('BYE', self.contact)
        bye.addHeader('via', 'SIP/2.0/UDP %s;rport'%self.getLocalSIPAddress()[0])
        bye.addHeader('cseq', '%s BYE'%self.getCSeq(incr=1))
        bye.addHeader('to', self.to)
        bye.addHeader('from', '"%s" <sip:%s>;tag=%s'%(prefs.myname, prefs.myemail, prefs.mytag))
        bye.addHeader('call-id', self.getCallID())
        bye.addHeader('user-agent', 'Shtoom/0.1')
        bye.addHeader('content-length', 0)
        bye.creationFinished()
        self._sentBye = bye.toString(), (url.host, (url.port or 5060))
        return self._sentBye

class SipPhone(DatagramProtocol, object):
    """ A SIP phone
    """
    __implements__ = ISipPhone

    def __init__(self, ui, *args, **kwargs):
        self.ui = ui
        self._calls = {}
        super(SipPhone, self, *args, **kwargs)

    def _newCallObject(self, to):
        call = Call(to)
        self._calls[call.getCallID()] = call
        return call

    def _getCallObject(self, callid):
        if type(callid) is list:
            callid = callid[0]
        if callid.startswith('<') and callid.endswith('>'):
            callid = callid[1:-1]
        return self._calls[callid]

    def _delCallObject(self, callid):
        if type(callid) is list:
            callid = callid[0]
        if callid.startswith('<') and callid.endswith('>'):
            callid = callid[1:-1]
        del self._calls[callid]

    def placeCall(self, url):
        """ Place a call to 'url'

            Returns callid, a string
        """
        self.ui.debugMessage("placeCall starting")
        call = self._newCallObject(url)
        call.setupRTP()
        invite = call.genInvite(url)
        self.transport.write(invite, call.getRemoteSIPAddress())
        # Set up a callLater in case we don't get a response, for a retransmit
        call.setState('SENT_INVITE')
        return call.getCallID()

    def dropCall(self, callid):
        """ Drop call identified by 'callid'
        """

        call = self._getCallObject(callid)
        if not call:
            self.ui.debugMessage("no such call %s"%callid)
            return
        state = call.getState()
        if state == 'NONE':
            self.ui.debugMessage("no call to drop")
        elif state in ( 'SENT_ACK', 'SENT_INVITE'):
            bye, dest = call.genBye()
            self.transport.write(bye, dest)
            self.ui.debugMessage("sent a BYE message")
            call.setState('SENT_BYE')

    def startDTMF(self, digit):
        """ Start sending DTMF digit 'digit'
        """
        self.ui.debugMessage("startDTMF not implemented yet!")

    def stopDTMF(self):
        """ Stop sending DTMF digit 'digit'
        """
        self.ui.debugMessage("stopDTMF not implemented yet!")

    def datagramReceived(self, datagram, addr):
        self.ui.debugMessage("Got a SIP packet from %s:%s"%(addr))
        mp = tpsip.MessagesParser(self.sipMessageReceived)
        self.ui.debugMessage("SIP PACKET\n%s"%(datagram))
        mp.dataReceived(datagram)
        mp.dataDone()

    def sipMessageReceived(self, message):
        self.ui.debugMessage("got SIP response %s: %s"%(
                                        message.code, message.phrase))
        self.ui.statusMessage("%s: %s"%(message.code,message.phrase))
        self.ui.debugMessage("got SIP response\n %s"%( message.toString()))
        callid = message.headers['call-id']
        call = self._getCallObject(callid)
        if not call:
            self.ui.debugMessage("SIP response refers to unknown call %s"%(
                                                                    callid))
            return
        if message.code in ( 100, 180, 181, 182 ):
            return
        elif message.code == 200:
            state = call.getState()
            if state == 'SENT_INVITE':
                self.ui.debugMessage(message.body)
                ack, dest = call.genAck(message, startRTP=1)
                self.transport.write(ack, dest)
                call.setState('SENT_ACK')
            elif state == 'SENT_ACK':
                self.ui.debugMessage('Got duplicate OK to our ACK')
                ack, dest = call.genAck(message)
                self.transport.write(ack, dest)
            elif state == 'SENT_BYE':
                call.teardownRTP()
                self._delCallObject(callid)
                self.ui.debugMessage("Hung up on call %s"%callid)
                self.ui.statusMessage("idle")
            else:
                self.ui.debugMessage('Got OK in unexpected state %s'%state)
        else:
            self.ui.debugMessage(message.toString())

<|endoftext|>


<commit_before>class markov():
    def __init__(self):
        self.ngrams = dict()

    def __update_grams(self, key, word):
        n = self.ngrams[key]['_n']
        if word not in self.ngrams[key]:
            self.ngrams[key][word] = 0
        for x in self.ngrams[key]:
            tmp = self.ngrams[key][x]
            self.ngrams[key][x] = (tmp * n + (1 if x == word else 0)) / (n + 1)
        self.ngrams[key]['_n'] = n + 1

    def learn(self, s):
        sentence = s.split(' ')
        for i in range(len(sentence) - 1):
            if sentence[i] in self.ngrams:
                self.__update_grams(sentence[i], sentence[i + 1])
            else:
                self.ngrams[sentence[i]] = {sentence[i + 1]: 1, '_n': 1}

    def predict(self, w):
        res = ('', 0)
        if w not in self.ngrams:
            return ''
        for x in self.ngrams[w]:
            if self.ngrams[w][x] > res[1] and x != '_n':
                res = (x, self.ngrams[w][x])
        return res[0]

    def show(self):
        print(self.ngrams)

if __name__ == '__main__':
    IA = markov()
    while True:
        line = input()
        if line.strip() == 'exit':
            print('done')
            exit()
        if line.strip() == 'show':
            print(IA.show())
            continue
        IA.learn(line.strip())
        print(IA.predict(line.strip().split(' ')[-1]))
<commit_msg>[UPD] learn api route done<commit_after>from typing import Union, Dict, List, Any
from flask import Flask, request
import json
import sys
import re
import os

app = Flask(__name__)

def foreach(fn, elems: Any):
    for elem in elems:
        fn(elem)

def exists(item: Any):
    return item

Ngram = int #Dict[str, Union[str, float, Dict[str, Union[str, float]]]]

class Markov():
    """ markov is the class engine for word prediction.

    the ngrams are stored in hashmap in wich we can navigate in order
    to predict the next word. The two main purpose of this class is to finish
    the current word with the 3 highest likely words and predict the next word.

    Attribute:
        ngrams: tree of the ngrams
        deep: the depth of the tree
    """

    def __init__(self, deep = 0):
        """ Constructor """
        self.ngrams = { '_n': 0 }
        self.deep = deep

    def __update_gram(self, base: Ngram, word: str):
        """ Update the gram.

        Args:
            base: the reference to the subtree to update
            word: the word to learn
        """
        n = base['_n']
        if word not in base:
            base[word] = { '_p': 0, '_n': 0 }
        # __update change the probabilty for all the words with inc the good word then normalize
        def __update(x): base[x]['_p'] = (base[x]['_p'] * n + (1 if x == word else 0)) / (n + 1)
        foreach(__update, base.keys() - {'_p', '_n'})
        base['_n'] += 1

    def __clean(self, word: str) -> str:
        """ Clean a word from ponctuation.

        Args:
            word: the word to treat

        Returns:
            the cleanned word
        """
        ponctuations = {',', '.', ':', ';'}
        for ponctuation in ponctuations:
            if word.startswith(ponctuation):
                word = word[1:]
            if word.endswith(ponctuation):
                word = word[:-1]
        return word

    def __tokenize(self, text: str) -> List[str]:
        """ Tokenize the corpus.

        Args:
            text: the corpus to tokenize.

        Returns:
            tokenized corpus.
        """
        sentences = re.split('\. |: |; ', text.replace('\n', ' '))
        tokens = [ list(filter(exists, map(self.__clean, sentence.split(' ')))) for sentence in sentences ]
        return list(filter(exists, tokens))

    def __learn_sentence(self, tokens: List[str]):
        """ Learn a full token sentence.

        Args:
            tokens: list of token to train
        """
        tokens = tokens[:-self.deep]
        base = self.ngrams
        for token in tokens:
            self.__update_gram(base, token)
            base = base[token]

    def learn(self, text: str):
        """ Learn the text corpus.

        Args:
            text: the learning corpus
        """
        token_sentences = self.__tokenize(text)
        for token_sentence in token_sentences:
            self.__learn_sentence(token_sentence)
            print(token_sentence)

@app.route('/learn', methods = ['POST'])
def learn():
    IA = learn.IA
    text = request.form['body']
    IA.learn(text)
    return '', 200

if __name__ == '__main__':
    IA = Markov(3)
    learn.IA = IA
    app.run('0.0.0.0', 5000)
    exit(os.EX_OK)
    with open(sys.argv[1]) as f:
        IA.learn(f.read())
<|endoftext|>


<commit_before>################################################################################
# Copyright (c) 2021 ContinualAI.                                              #
# Copyrights licensed under the MIT License.                                   #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 12-10-2020                                                             #
# Author(s): Vincenzo Lomonaco                                                 #
# E-mail: contact@continualai.org                                              #
# Website: avalanche.continualai.org                                           #
################################################################################

"""
This is a simple example on how to use the Replay strategy.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import torch
from torch.nn import CrossEntropyLoss
from torchvision import transforms
from torchvision.transforms import ToTensor, RandomCrop
import torch.optim.lr_scheduler
import matplotlib.pyplot as plt
import numpy as np
from avalanche.benchmarks import SplitMNIST
from avalanche.models import VAE
from avalanche.training.supervised import VAETraining
from avalanche.training.plugins import GenerativeReplayPlugin
from avalanche.logging import InteractiveLogger

def main(args):
    # --- CONFIG
    device = torch.device(
        f"cuda:{args.cuda}"
        if torch.cuda.is_available() and args.cuda >= 0
        else "cpu"
    )
    n_batches = 5
    # ---------

    # --- TRANSFORMATIONS
    train_transform = transforms.Compose(
        [
            RandomCrop(28, padding=4),
            ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,)),
        ]
    )
    test_transform = transforms.Compose(
        [ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
    )
    # ---------

    # --- SCENARIO CREATION
    scenario = SplitMNIST(n_experiences=10, seed=1234)
    # ---------

    # MODEL CREATION
    model = VAE((1, 28, 28), nhid=2)

    # choose some metrics and evaluation method
    interactive_logger = InteractiveLogger()

    # CREATE THE STRATEGY INSTANCE (GenerativeReplay)
    cl_strategy = VAETraining(
        model,
        torch.optim.Adam(model.parameters(), lr=0.001),
        train_mb_size=100,
        train_epochs=4,
        device=device,
        plugins=[GenerativeReplayPlugin()]
    )

    # TRAINING LOOP
    print("Starting experiment...")
    for experience in scenario.train_stream:
        print("Start of experience ", experience.current_experience)
        cl_strategy.train(experience)
        print("Training completed")

        samples = model.generate(10)
        samples = samples.cpu().numpy()

        f, axarr = plt.subplots(1, 10)
        for j in range(10):
            axarr[j].imshow(samples[j, 0], cmap="gray")
        np.vectorize(lambda ax: ax.axis('off'))(axarr)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--cuda",
        type=int,
        default=0,
        help="Select zero-indexed cuda device. -1 to use CPU.",
    )
    args = parser.parse_args()
    main(args)
<commit_msg>detach() samples.<commit_after>################################################################################
# Copyright (c) 2021 ContinualAI.                                              #
# Copyrights licensed under the MIT License.                                   #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 12-10-2020                                                             #
# Author(s): Vincenzo Lomonaco                                                 #
# E-mail: contact@continualai.org                                              #
# Website: avalanche.continualai.org                                           #
################################################################################

"""
This is a simple example on how to use the Replay strategy.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import torch
from torch.nn import CrossEntropyLoss
from torchvision import transforms
from torchvision.transforms import ToTensor, RandomCrop
import torch.optim.lr_scheduler
import matplotlib.pyplot as plt
import numpy as np
from avalanche.benchmarks import SplitMNIST
from avalanche.models import VAE
from avalanche.training.supervised import VAETraining
from avalanche.training.plugins import GenerativeReplayPlugin
from avalanche.logging import InteractiveLogger

def main(args):
    # --- CONFIG
    device = torch.device(
        f"cuda:{args.cuda}"
        if torch.cuda.is_available() and args.cuda >= 0
        else "cpu"
    )
    n_batches = 5
    # ---------

    # --- TRANSFORMATIONS
    train_transform = transforms.Compose(
        [
            RandomCrop(28, padding=4),
            ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,)),
        ]
    )
    test_transform = transforms.Compose(
        [ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
    )
    # ---------

    # --- SCENARIO CREATION
    scenario = SplitMNIST(n_experiences=10, seed=1234)
    # ---------

    # MODEL CREATION
    model = VAE((1, 28, 28), nhid=2)

    # choose some metrics and evaluation method
    interactive_logger = InteractiveLogger()

    # CREATE THE STRATEGY INSTANCE (GenerativeReplay)
    cl_strategy = VAETraining(
        model,
        torch.optim.Adam(model.parameters(), lr=0.001),
        train_mb_size=100,
        train_epochs=4,
        device=device,
        plugins=[GenerativeReplayPlugin()]
    )

    # TRAINING LOOP
    print("Starting experiment...")
    for experience in scenario.train_stream:
        print("Start of experience ", experience.current_experience)
        cl_strategy.train(experience)
        print("Training completed")

        samples = model.generate(10)
        samples = samples.detach().cpu().numpy()

        f, axarr = plt.subplots(1, 10)
        for j in range(10):
            axarr[j].imshow(samples[j, 0], cmap="gray")
        np.vectorize(lambda ax: ax.axis('off'))(axarr)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--cuda",
        type=int,
        default=0,
        help="Select zero-indexed cuda device. -1 to use CPU.",
    )
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>import csv

# A Frame will one day be a pointer to an h2o cloud
# For now, see if we can make (small) 2-D associative arrays,
# and overload everything.
class Frame(object):
  def __init__(self, str=None, file=None, vecs=None):
    # Read a CSV file
    if file != None:
      with open(file, 'rb') as csvfile:
        self._vecs = []
        for name in csvfile.readline().split(','):
          self._vecs.append(Vec(str+name.rstrip(),[]))
        for row in csv.reader(csvfile):
          for i,data in enumerate(row): self._vecs[i].append(data)
    # Construct from an array of Vecs already passed in
    elif vecs!= None:
      self._vecs=vecs

  # Print [col, cols...]
  def __str__(self):  return self._vecs.__repr__()

  # Column selection via integer, string (name) returns a Vec
  # Column selection via slice returns a subset Frame
  def __getitem__(self,i):
    if isinstance(i,int):
      return self._vecs[i]
    if isinstance(i,str):
      for v in self._vecs:
        if i==v._name:
          return v
      raise ValueError("Name "+i+" not in Frame")
    # Slice; return a Frame not a Vec
    if isinstance(i,slice):
      return Frame(vecs=self._vecs[i])
    raise NotImplementedError

  # Number of columns
  def __len__(self): return len(self._vecs)

  # Addition
  def __add__(self,i):
    if len(self)==0:  return self
    if isinstance(i,Frame):
      if len(i) != len(self):
        raise ValueError("Frame len()="+len(self)+" cannot be broadcast across len(i)="+len(i))
      return Frame(vecs=[x+y for x,y in zip(self._vecs,i._vecs)])
    if isinstance(i,Vec):
      if len(i) != len(self._vecs[0]):
        raise ValueError("Vec len()="+len(self._vecs[0])+" cannot be broadcast across len(i)="+len(i))
      return Frame(vecs=[x+i for x in self._vecs])
    if isinstance(i,int):
      return Frame(vecs=[x+i for x in self._vecs])
    raise NotImplementedError

  def __radd__(self,i):  return self+i # Add is associative

########
# A single column of data, possibly lazily computed
class Vec(object):
  def __init__(self, name, data):
    self._name = name
    self._data = data
    self._len = len(data)

  # Force eager evaluation
  def eager(self):
    if isinstance(self._data,Expr):  self._data=self._data.compute()
    return self._data

  # Append a value during CSV read, convert to float
  def append(self,data):
    __x__ = data
    try:  __x__ = float(data)
    except ValueError:  pass
    self._data.append(__x__)

  # Print self; forces eager evaluation
  def __repr__(self): return self._name+" "+self.eager().__str__()

  # Basic indexed or sliced lookup
  def __getitem__(self,i):  return self._data[i]

  def __add__(self,i):
    if isinstance(i,Vec):       # Vec+Vec
      if len(i) != len(self):
        raise ValueError("Vec len()="+len(self)+" cannot be broadcast across len(i)="+len(i))
      return Vec(self._name+"+"+i._name,Expr("+",self,i))
    if isinstance(i,int):       # Vec+int
      if i==0:  return self     # Additive identity
      return Vec(self._name+"+"+str(i),Expr("+",self,i))
    raise NotImplementedError

  def __radd__(self,i):  return self+i # Add is associative

  # Number of rows
  def __len__(self):  return len(self._data)

  def __del__(self):
    if Expr==None or not isinstance(self._data,Expr):
      print "DEL of "+self._name

########
# A pending to-be-computed expression
class Expr(object):
  def __init__(self,op,left,rite):
    self._op = op
    self._left = left
    self._rite = rite

  def compute(self):
    print "WORK: ",self
    if self._op == "+":
      if isinstance(self._rite,Vec):
        return [x+y for x,y in zip(self._left.eager(),self._rite.eager())]
      if isinstance(self._rite,int):
        return [x+self._rite for x in self._left.eager()]
      raise NotImplementedError
    raise NotImplementedError

  def __len__(self):
    return len(self._left) if isinstance(self._left,Vec) else len(self._rite)

  def __repr__(self):
    return "("+str(self._left._name)+self._op+str(self._rite._name if isinstance(self._rite,Vec) else self._rite)+")"

######################################################
#
# Sample use-cases

a = Frame("a.",file="smalldata/iris/iris_wheader.csv")[0:4]

print a[0]._name    # Column header
print a[0][2]       # column 0, row 2 value
print a["a.sepal_len"][2] # Column 0, row 2 value
print "EVAL a[0]+2 is: "   ,a[0]+2        # Add 2 to every element; broadcast a constant
print "EVAL a[0]+a[1] is: ",a[0]+a[1]     # Add 2 columns; broadcast parallel add
print "EVAL mean(a) is: ",sum(a["a.sepal_len"])/len(a[0])

print "FAIL TEST"
try:
  print a["Sepal_len"] # Error, mispelt column name
except ValueError,e:
  pass              # Expected error

print "READ b"
b = Frame("b.",file="smalldata/iris/iris_wheader.csv")[0:4]
print "EVAL c=a+b"
c = a+b
print "EVAL d=c+c+sum(a)"
d = c+c+sum(a)
print "EVAL c+a+1"
e = c+(a+1)
print "EVAL e is: ",e
<commit_msg>more cleanup, prep for next go...<commit_after>import csv

# A Frame will one day be a pointer to an h2o cloud
# For now, see if we can make (small) 2-D associative arrays,
# and overload everything.
class Frame(object):
  def __init__(self, str=None, file=None, vecs=None):
    # Read a CSV file
    if file != None:
      with open(file, 'rb') as csvfile:
        self._vecs = []
        for name in csvfile.readline().split(','):
          self._vecs.append(Vec(str+name.rstrip(),[]))
        for row in csv.reader(csvfile):
          for i,data in enumerate(row): self._vecs[i].append(data)
      print "READ: +",len(self),file
    # Construct from an array of Vecs already passed in
    elif vecs!= None:
      self._vecs=vecs

  # Print [col, cols...]
  def __str__(self):  return self._vecs.__repr__()

  # Column selection via integer, string (name) returns a Vec
  # Column selection via slice returns a subset Frame
  def __getitem__(self,i):
    if isinstance(i,int):   return self._vecs[i]
    if isinstance(i,str):
      for v in self._vecs:
        if i==v._name:  return v
      raise ValueError("Name "+i+" not in Frame")
    # Slice; return a Frame not a Vec
    if isinstance(i,slice): return Frame(vecs=self._vecs[i])
    raise NotImplementedError

  # Number of columns
  def __len__(self): return len(self._vecs)

  # Addition
  def __add__(self,i):
    if len(self)==0:  return self
    if isinstance(i,Frame):
      if len(i) != len(self):
        raise ValueError("Frame len()="+len(self)+" cannot be broadcast across len(i)="+len(i))
      return Frame(vecs=[x+y for x,y in zip(self._vecs,i._vecs)])
    if isinstance(i,Vec):
      if len(i) != len(self._vecs[0]):
        raise ValueError("Vec len()="+len(self._vecs[0])+" cannot be broadcast across len(i)="+len(i))
      return Frame(vecs=[x+i for x in self._vecs])
    if isinstance(i,int):
      return Frame(vecs=[x+i for x in self._vecs])
    raise NotImplementedError

  def __radd__(self,i):  return self+i # Add is associative

########
# A single column of data, possibly lazily computed
class Vec(object):
  def __init__(self, name, data):
    self._name = name
    self._data = data
    self._len = len(data)

  # Force eager evaluation
  def eager(self):
    if isinstance(self._data,Expr):  self._data=self._data.compute()
    return self._data

  # Append a value during CSV read, convert to float
  def append(self,data):
    __x__ = data
    try:  __x__ = float(data)
    except ValueError:  pass
    self._data.append(__x__)

  # Print self; forces eager evaluation
  def __repr__(self): return self._name+" "+self.eager().__str__()

  # Basic indexed or sliced lookup
  def __getitem__(self,i):  return self._data[i]

  # Basic (broadening) addition
  def __add__(self,i):
    if isinstance(i,Vec):       # Vec+Vec
      if len(i) != len(self):
        raise ValueError("Vec len()="+len(self)+" cannot be broadcast across len(i)="+len(i))
      return Vec(self._name+"+"+i._name,Expr("+",self,i))
    if isinstance(i,int):       # Vec+int
      if i==0:  return self     # Additive identity
      return Vec(self._name+"+"+str(i),Expr("+",self,i))
    raise NotImplementedError

  def __radd__(self,i):  return self+i # Add is associative

  # Number of rows
  def __len__(self):  return len(self._data)

  def __del__(self):
    if Expr==None or not isinstance(self._data,Expr):
      print "DELE: -1",self._name

########
# A pending to-be-computed expression
class Expr(object):
  def __init__(self,op,left,rite):
    self._op = op
    self._left = left
    self._rite = rite

  def compute(self):
    print "WORK: +1",self
    if self._op == "+":
      if isinstance(self._rite,Vec):
        return [x+y for x,y in zip(self._left.eager(),self._rite.eager())]
      if isinstance(self._rite,int):
        return [x+self._rite for x in self._left.eager()]
      raise NotImplementedError
    raise NotImplementedError

  def __len__(self):
    return len(self._left) if isinstance(self._left,Vec) else len(self._rite)

  def __repr__(self):
    return "("+str(self._left._name)+self._op+str(self._rite._name if isinstance(self._rite,Vec) else self._rite)+")"

######################################################
#
# Sample use-cases

a = Frame("a.",file="smalldata/iris/iris_wheader.csv")[0:4]

print a[0]._name    # Column header
print a[0][2]       # column 0, row 2 value
print a["a.sepal_len"][2] # Column 0, row 2 value
print a[0]+2        # Add 2 to every element; broadcast a constant
print a[0]+a[1]     # Add 2 columns; broadcast parallel add
print sum(a["a.sepal_len"])/len(a[0])

try:   print a["Sepal_len"] # Error, mispelt column name
except ValueError,e:  pass  # Expected error

b = Frame("b.",file="smalldata/iris/iris_wheader.csv")[0:4]
c = a+b
d = c+c+sum(a)
e = c+(a+1)
print e
<|endoftext|>


<commit_before>"""
Convert between text notebook metadata and jupyter cell metadata.

See also https://ipython.org/ipython-doc/3/notebook/nbformat.html#cell-metadata

metadata.hide_input and metadata.hide_output are documented here:
http://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/runtools/readme.html

TODO: Update this if a standard gets defined at
https://github.com/jupyter/notebook/issues/3700

Note: Nteract uses "outputHidden" and "inputHidden". We may want to switch
to those.
"""

import ast
import json
import re

from .languages import _JUPYTER_LANGUAGES

try:
    unicode  # Python 2
except NameError:
    unicode = str  # Python 3

_BOOLEAN_OPTIONS_DICTIONARY = [('hide_input', 'echo', True),
                               ('hide_output', 'include', True)]
_JUPYTEXT_CELL_METADATA = [
    # Pre-jupytext metadata
    'skipline', 'noskipline',
    # Jupytext metadata
    'cell_marker', 'lines_to_next_cell', 'lines_to_end_of_cell_marker']
_IGNORE_CELL_METADATA = ','.join('-{}'.format(name) for name in [
    # Frequent cell metadata that should not enter the text representation
    # (these metadata are preserved in the paired Jupyter notebook).
    'autoscroll', 'collapsed', 'scrolled', 'trusted', 'ExecuteTime'] +
                                 _JUPYTEXT_CELL_METADATA)
_PERCENT_CELL = re.compile(
    r'(# |#)%%([^\{\[]*)(|\[raw\]|\[markdown\])([^\{\[]*)(|\{.*\})\s*$')

def _r_logical_values(pybool):
    return 'TRUE' if pybool else 'FALSE'

class RLogicalValueError(Exception):
    """Incorrect value for R boolean"""
    pass

class RMarkdownOptionParsingError(Exception):
    """Error when parsing Rmd cell options"""
    pass

def _py_logical_values(rbool):
    if rbool in ['TRUE', 'T']:
        return True
    if rbool in ['FALSE', 'F']:
        return False
    raise RLogicalValueError

def metadata_to_rmd_options(language, metadata):
    """
    Convert language and metadata information to their rmd representation
    :param language:
    :param metadata:
    :return:
    """
    options = (language or 'R').lower()
    if 'name' in metadata:
        options += ' ' + metadata['name'] + ','
        del metadata['name']
    for jupyter_option, rmd_option, rev in _BOOLEAN_OPTIONS_DICTIONARY:
        if jupyter_option in metadata:
            options += ' {}={},'.format(
                rmd_option, _r_logical_values(metadata[jupyter_option] != rev))
            del metadata[jupyter_option]
    for opt_name in metadata:
        opt_value = metadata[opt_name]
        opt_name = opt_name.strip()
        if opt_name == 'active':
            options += ' {}="{}",'.format(opt_name, str(opt_value))
        elif isinstance(opt_value, bool):
            options += ' {}={},'.format(
                opt_name, 'TRUE' if opt_value else 'FALSE')
        elif isinstance(opt_value, list):
            options += ' {}={},'.format(
                opt_name, 'c({})'.format(
                    ', '.join(['"{}"'.format(str(v)) for v in opt_value])))
        else:
            options += ' {}={},'.format(opt_name, str(opt_value))
    if not language:
        options = options[2:]
    return options.strip(',').strip()

def update_metadata_from_rmd_options(name, value, metadata):
    """
    Update metadata using the _BOOLEAN_OPTIONS_DICTIONARY mapping
    :param name: option name
    :param value: option value
    :param metadata:
    :return:
    """
    for jupyter_option, rmd_option, rev in _BOOLEAN_OPTIONS_DICTIONARY:
        if name == rmd_option:
            try:
                metadata[jupyter_option] = _py_logical_values(value) != rev
                return True
            except RLogicalValueError:
                pass
    return False

class ParsingContext:
    """
    Class for determining where to split rmd options
    """
    parenthesis_count = 0
    curly_bracket_count = 0
    square_bracket_count = 0
    in_single_quote = False
    in_double_quote = False

    def __init__(self, line):
        self.line = line

    def in_global_expression(self):
        """Currently inside an expression"""
        return (self.parenthesis_count == 0 and self.curly_bracket_count == 0
                and self.square_bracket_count == 0
                and not self.in_single_quote and not self.in_double_quote)

    def count_special_chars(self, char, prev_char):
        """Update parenthesis counters"""
        if char == '(':
            self.parenthesis_count += 1
        elif char == ')':
            self.parenthesis_count -= 1
            if self.parenthesis_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing parentheses'.format(self.line))
        elif char == '{':
            self.curly_bracket_count += 1
        elif char == '}':
            self.curly_bracket_count -= 1
            if self.curly_bracket_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing curly brackets'.format(self.line))
        elif char == '[':
            self.square_bracket_count += 1
        elif char == ']':
            self.square_bracket_count -= 1
            if self.square_bracket_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing square brackets'.format(self.line))
        elif char == "'" and prev_char != '\\':
            self.in_single_quote = not self.in_single_quote
        elif char == '"' and prev_char != '\\':
            self.in_double_quote = not self.in_double_quote

def parse_rmd_options(line):
    """
    Given a R markdown option line, returns a list of pairs name,value
    :param line:
    :return:
    """
    parsing_context = ParsingContext(line)

    result = []
    prev_char = ''

    name = ''
    value = ''

    for char in ',' + line + ',':
        if parsing_context.in_global_expression():
            if char == ',':
                if name != '' or value != '':
                    if result and name == '':
                        raise RMarkdownOptionParsingError(
                            'Option line "{}" has no name for '
                            'option value {}'.format(line, value))
                    result.append((name.strip(), value.strip()))
                    name = ''
                    value = ''
            elif char == '=':
                if name == '':
                    name = value
                    value = ''
                else:
                    value += char
            else:
                parsing_context.count_special_chars(char, prev_char)
                value += char
        else:
            parsing_context.count_special_chars(char, prev_char)
            value += char
        prev_char = char

    if not parsing_context.in_global_expression():
        raise RMarkdownOptionParsingError(
            'Option line "{}" is not properly terminated'.format(line))

    return result

def rmd_options_to_metadata(options):
    """
    Parse rmd options and return a metadata dictionary
    :param options:
    :return:
    """
    options = re.split(r'\s|,', options, 1)
    if len(options) == 1:
        language = options[0]
        chunk_options = []
    else:
        language, others = options
        language = language.rstrip(' ,')
        others = others.lstrip(' ,')
        chunk_options = parse_rmd_options(others)

    language = 'R' if language == 'r' else language
    metadata = {}
    for i, opt in enumerate(chunk_options):
        name, value = opt
        if i == 0 and name == '':
            metadata['name'] = value
            continue
        else:
            if update_metadata_from_rmd_options(name, value, metadata):
                continue
            try:
                metadata[name] = _py_logical_values(value)
                continue
            except RLogicalValueError:
                metadata[name] = value

    for name in metadata:
        try_eval_metadata(metadata, name)

    if ('active' in metadata or metadata.get('run_control', {}).get('frozen') is True) and 'eval' in metadata:
        del metadata['eval']

    return metadata.get('language') or language, metadata

def md_options_to_metadata(options):
    """Parse markdown options and return language and metadata (cell name)"""
    language = None
    name = None

    options = [opt for opt in options.split(' ') if opt != '']
    if len(options) >= 2:
        language, name = options[:2]
    elif options:
        language = options[0]

    if language:
        for lang in _JUPYTER_LANGUAGES + ['julia', 'scheme', 'c++']:
            if language.lower() == lang.lower():
                if name:
                    return lang, {'name': name}
                return lang, {}

        return None, {'name': language}

    return None, {}

def try_eval_metadata(metadata, name):
    """Evaluate given metadata to a python object, if possible"""
    value = metadata[name]
    if not isinstance(value, (str, unicode)):
        return
    if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
        if name in ['active', 'magic_args', 'language']:
            metadata[name] = value[1:-1]
        return
    if value.startswith('c(') and value.endswith(')'):
        value = '[' + value[2:-1] + ']'
    elif value.startswith('list(') and value.endswith(')'):
        value = '[' + value[5:-1] + ']'
    try:
        metadata[name] = ast.literal_eval(value)
    except (SyntaxError, ValueError):
        return

def json_options_to_metadata(options, add_brackets=True):
    """Read metadata from its json representation"""
    try:
        options = json.loads('{' + options + '}' if add_brackets else options)
        return options
    except ValueError:
        return {}

def metadata_to_json_options(metadata):
    """Represent metadata as json text"""
    for key in _JUPYTEXT_CELL_METADATA:
        metadata.pop(key, None)

    return json.dumps(metadata)

def is_active(ext, metadata):
    """Is the cell active for the given file extension?"""
    if metadata.get('run_control', {}).get('frozen') is True:
        return False
    if 'active' not in metadata:
        return True
    return ext.replace('.', '') in re.split('\\.|,', metadata['active'])

def double_percent_options_to_metadata(options):
    """Parse double percent options"""
    matches = _PERCENT_CELL.findall('# %%' + options)

    # Fail safe when regexp matching fails #116
    # (occurs e.g. if square brackets are found in the title)
    if not matches:
        return {'title': options.strip()}

    matches = matches[0]

    # Fifth match are JSON metadata
    if matches[4]:
        metadata = json_options_to_metadata(matches[4], add_brackets=False)
    else:
        metadata = {}

    # Third match is cell type
    cell_type = matches[2]
    if cell_type:
        metadata['cell_type'] = cell_type[1:-1]

    # Second and fourth match are description
    title = [matches[i].strip() for i in [1, 3]]
    title = [part for part in title if part]
    if title:
        title = ' '.join(title)
        cell_depth = 0
        while title.startswith('%'):
            cell_depth += 1
            title = title[1:]

        if cell_depth:
            metadata['cell_depth'] = cell_depth
        metadata['title'] = title.strip()

    return metadata

def metadata_to_double_percent_options(metadata):
    """Metadata to double percent lines"""
    options = []
    if 'cell_depth' in metadata:
        options.append('%' * metadata.pop('cell_depth'))
    if 'title' in metadata:
        options.append(metadata.pop('title'))
    if 'cell_type' in metadata:
        options.append('[{}]'.format(metadata.pop('cell_type')))
    metadata = metadata_to_json_options(metadata)
    if metadata != '{}':
        options.append(metadata)
    return ' '.join(options)
<commit_msg>Doc update<commit_after>"""
Convert between text notebook metadata and jupyter cell metadata.

Standard cell metadata are documented here:
See also https://ipython.org/ipython-doc/3/notebook/nbformat.html#cell-metadata
"""

import ast
import json
import re

from .languages import _JUPYTER_LANGUAGES

try:
    unicode  # Python 2
except NameError:
    unicode = str  # Python 3

# Map R Markdown's "echo" and "include" to "hide_input" and "hide_output", that are understood by the `runtools`
# extension for Jupyter notebook, and by nbconvert (use the `hide_input_output.tpl` template).
# See http://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/runtools/readme.html
_BOOLEAN_OPTIONS_DICTIONARY = [('hide_input', 'echo', True),
                               ('hide_output', 'include', True)]
_JUPYTEXT_CELL_METADATA = [
    # Pre-jupytext metadata
    'skipline', 'noskipline',
    # Jupytext metadata
    'cell_marker', 'lines_to_next_cell', 'lines_to_end_of_cell_marker']
_IGNORE_CELL_METADATA = ','.join('-{}'.format(name) for name in [
    # Frequent cell metadata that should not enter the text representation
    # (these metadata are preserved in the paired Jupyter notebook).
    'autoscroll', 'collapsed', 'scrolled', 'trusted', 'ExecuteTime'] +
                                 _JUPYTEXT_CELL_METADATA)
_PERCENT_CELL = re.compile(
    r'(# |#)%%([^\{\[]*)(|\[raw\]|\[markdown\])([^\{\[]*)(|\{.*\})\s*$')

def _r_logical_values(pybool):
    return 'TRUE' if pybool else 'FALSE'

class RLogicalValueError(Exception):
    """Incorrect value for R boolean"""
    pass

class RMarkdownOptionParsingError(Exception):
    """Error when parsing Rmd cell options"""
    pass

def _py_logical_values(rbool):
    if rbool in ['TRUE', 'T']:
        return True
    if rbool in ['FALSE', 'F']:
        return False
    raise RLogicalValueError

def metadata_to_rmd_options(language, metadata):
    """
    Convert language and metadata information to their rmd representation
    :param language:
    :param metadata:
    :return:
    """
    options = (language or 'R').lower()
    if 'name' in metadata:
        options += ' ' + metadata['name'] + ','
        del metadata['name']
    for jupyter_option, rmd_option, rev in _BOOLEAN_OPTIONS_DICTIONARY:
        if jupyter_option in metadata:
            options += ' {}={},'.format(
                rmd_option, _r_logical_values(metadata[jupyter_option] != rev))
            del metadata[jupyter_option]
    for opt_name in metadata:
        opt_value = metadata[opt_name]
        opt_name = opt_name.strip()
        if opt_name == 'active':
            options += ' {}="{}",'.format(opt_name, str(opt_value))
        elif isinstance(opt_value, bool):
            options += ' {}={},'.format(
                opt_name, 'TRUE' if opt_value else 'FALSE')
        elif isinstance(opt_value, list):
            options += ' {}={},'.format(
                opt_name, 'c({})'.format(
                    ', '.join(['"{}"'.format(str(v)) for v in opt_value])))
        else:
            options += ' {}={},'.format(opt_name, str(opt_value))
    if not language:
        options = options[2:]
    return options.strip(',').strip()

def update_metadata_from_rmd_options(name, value, metadata):
    """
    Update metadata using the _BOOLEAN_OPTIONS_DICTIONARY mapping
    :param name: option name
    :param value: option value
    :param metadata:
    :return:
    """
    for jupyter_option, rmd_option, rev in _BOOLEAN_OPTIONS_DICTIONARY:
        if name == rmd_option:
            try:
                metadata[jupyter_option] = _py_logical_values(value) != rev
                return True
            except RLogicalValueError:
                pass
    return False

class ParsingContext:
    """
    Class for determining where to split rmd options
    """
    parenthesis_count = 0
    curly_bracket_count = 0
    square_bracket_count = 0
    in_single_quote = False
    in_double_quote = False

    def __init__(self, line):
        self.line = line

    def in_global_expression(self):
        """Currently inside an expression"""
        return (self.parenthesis_count == 0 and self.curly_bracket_count == 0
                and self.square_bracket_count == 0
                and not self.in_single_quote and not self.in_double_quote)

    def count_special_chars(self, char, prev_char):
        """Update parenthesis counters"""
        if char == '(':
            self.parenthesis_count += 1
        elif char == ')':
            self.parenthesis_count -= 1
            if self.parenthesis_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing parentheses'.format(self.line))
        elif char == '{':
            self.curly_bracket_count += 1
        elif char == '}':
            self.curly_bracket_count -= 1
            if self.curly_bracket_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing curly brackets'.format(self.line))
        elif char == '[':
            self.square_bracket_count += 1
        elif char == ']':
            self.square_bracket_count -= 1
            if self.square_bracket_count < 0:
                raise RMarkdownOptionParsingError(
                    'Option line "{}" has too many '
                    'closing square brackets'.format(self.line))
        elif char == "'" and prev_char != '\\':
            self.in_single_quote = not self.in_single_quote
        elif char == '"' and prev_char != '\\':
            self.in_double_quote = not self.in_double_quote

def parse_rmd_options(line):
    """
    Given a R markdown option line, returns a list of pairs name,value
    :param line:
    :return:
    """
    parsing_context = ParsingContext(line)

    result = []
    prev_char = ''

    name = ''
    value = ''

    for char in ',' + line + ',':
        if parsing_context.in_global_expression():
            if char == ',':
                if name != '' or value != '':
                    if result and name == '':
                        raise RMarkdownOptionParsingError(
                            'Option line "{}" has no name for '
                            'option value {}'.format(line, value))
                    result.append((name.strip(), value.strip()))
                    name = ''
                    value = ''
            elif char == '=':
                if name == '':
                    name = value
                    value = ''
                else:
                    value += char
            else:
                parsing_context.count_special_chars(char, prev_char)
                value += char
        else:
            parsing_context.count_special_chars(char, prev_char)
            value += char
        prev_char = char

    if not parsing_context.in_global_expression():
        raise RMarkdownOptionParsingError(
            'Option line "{}" is not properly terminated'.format(line))

    return result

def rmd_options_to_metadata(options):
    """
    Parse rmd options and return a metadata dictionary
    :param options:
    :return:
    """
    options = re.split(r'\s|,', options, 1)
    if len(options) == 1:
        language = options[0]
        chunk_options = []
    else:
        language, others = options
        language = language.rstrip(' ,')
        others = others.lstrip(' ,')
        chunk_options = parse_rmd_options(others)

    language = 'R' if language == 'r' else language
    metadata = {}
    for i, opt in enumerate(chunk_options):
        name, value = opt
        if i == 0 and name == '':
            metadata['name'] = value
            continue
        else:
            if update_metadata_from_rmd_options(name, value, metadata):
                continue
            try:
                metadata[name] = _py_logical_values(value)
                continue
            except RLogicalValueError:
                metadata[name] = value

    for name in metadata:
        try_eval_metadata(metadata, name)

    if ('active' in metadata or metadata.get('run_control', {}).get('frozen') is True) and 'eval' in metadata:
        del metadata['eval']

    return metadata.get('language') or language, metadata

def md_options_to_metadata(options):
    """Parse markdown options and return language and metadata (cell name)"""
    language = None
    name = None

    options = [opt for opt in options.split(' ') if opt != '']
    if len(options) >= 2:
        language, name = options[:2]
    elif options:
        language = options[0]

    if language:
        for lang in _JUPYTER_LANGUAGES + ['julia', 'scheme', 'c++']:
            if language.lower() == lang.lower():
                if name:
                    return lang, {'name': name}
                return lang, {}

        return None, {'name': language}

    return None, {}

def try_eval_metadata(metadata, name):
    """Evaluate given metadata to a python object, if possible"""
    value = metadata[name]
    if not isinstance(value, (str, unicode)):
        return
    if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
        if name in ['active', 'magic_args', 'language']:
            metadata[name] = value[1:-1]
        return
    if value.startswith('c(') and value.endswith(')'):
        value = '[' + value[2:-1] + ']'
    elif value.startswith('list(') and value.endswith(')'):
        value = '[' + value[5:-1] + ']'
    try:
        metadata[name] = ast.literal_eval(value)
    except (SyntaxError, ValueError):
        return

def json_options_to_metadata(options, add_brackets=True):
    """Read metadata from its json representation"""
    try:
        options = json.loads('{' + options + '}' if add_brackets else options)
        return options
    except ValueError:
        return {}

def metadata_to_json_options(metadata):
    """Represent metadata as json text"""
    for key in _JUPYTEXT_CELL_METADATA:
        metadata.pop(key, None)

    return json.dumps(metadata)

def is_active(ext, metadata):
    """Is the cell active for the given file extension?"""
    if metadata.get('run_control', {}).get('frozen') is True:
        return False
    if 'active' not in metadata:
        return True
    return ext.replace('.', '') in re.split('\\.|,', metadata['active'])

def double_percent_options_to_metadata(options):
    """Parse double percent options"""
    matches = _PERCENT_CELL.findall('# %%' + options)

    # Fail safe when regexp matching fails #116
    # (occurs e.g. if square brackets are found in the title)
    if not matches:
        return {'title': options.strip()}

    matches = matches[0]

    # Fifth match are JSON metadata
    if matches[4]:
        metadata = json_options_to_metadata(matches[4], add_brackets=False)
    else:
        metadata = {}

    # Third match is cell type
    cell_type = matches[2]
    if cell_type:
        metadata['cell_type'] = cell_type[1:-1]

    # Second and fourth match are description
    title = [matches[i].strip() for i in [1, 3]]
    title = [part for part in title if part]
    if title:
        title = ' '.join(title)
        cell_depth = 0
        while title.startswith('%'):
            cell_depth += 1
            title = title[1:]

        if cell_depth:
            metadata['cell_depth'] = cell_depth
        metadata['title'] = title.strip()

    return metadata

def metadata_to_double_percent_options(metadata):
    """Metadata to double percent lines"""
    options = []
    if 'cell_depth' in metadata:
        options.append('%' * metadata.pop('cell_depth'))
    if 'title' in metadata:
        options.append(metadata.pop('title'))
    if 'cell_type' in metadata:
        options.append('[{}]'.format(metadata.pop('cell_type')))
    metadata = metadata_to_json_options(metadata)
    if metadata != '{}':
        options.append(metadata)
    return ' '.join(options)
<|endoftext|>