<commit_before># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utils for EfficientNet models for Keras.
Write weights from  ckpt file as in original repo
(https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet)
to h5 file for keras implementation of the models.

Usage:

# use checkpoint efficientnet-b0/model.ckpt (can be downloaded from
# https://storage.googleapis.com/cloud-tpu-checkpoints/
#     efficientnet/ckptsaug/efficientnet-b0.tar.gz)
# to update weight without top layers, saving to efficientnetb0_notop.h5
python efficientnet_weight_update_util.py --model b0 --notop \
    --ckpt efficientnet-b0/model.ckpt --o efficientnetb0_notop.h5

# use checkpoint noisy_student_efficientnet-b3/model.ckpt (providing
# improved result for b3, can be downloaded from
# https://storage.googleapis.com/cloud-tpu-checkpoints/
#     efficientnet/noisystudent/noisy_student_efficientnet-b3.tar.gz)
# to update weight with top layers, saving to efficientnetb3_new.h5
python efficientnet_weight_update_util.py --model b3 --notop \
    --ckpt noisy_student_efficientnet-b3/model.ckpt --o efficientnetb3_new.h5

"""

import argparse

import tensorflow as tf
from tensorflow.keras.applications import efficientnet

def write_ckpt_to_h5(path_h5, path_ckpt, keras_model, use_ema=True):
  """ Map the weights in checkpoint file (tf) to h5 file (keras)

  Args:
    path_h5: str, path to output hdf5 file to write weights loaded
      from ckpt files.
    path_ckpt: str, path to the ckpt files (e.g. 'efficientnet-b0/model.ckpt')
      that records efficientnet weights from original repo
      https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet
    keras_model: keras model, built from keras.applications efficientnet
      functions (e.g. EfficientNetB0)
    use_ema: Bool, whether to use ExponentialMovingAverage result or not
  """
  model_name_keras = keras_model.name
  model_name_tf = model_name_keras.replace('efficientnet', 'efficientnet-')

  keras_weight_names = [w.name for w in keras_model.weights]
  tf_weight_names = get_variable_names_from_ckpt(path_ckpt)

  keras_blocks = get_keras_blocks(keras_weight_names)
  tf_blocks = get_tf_blocks(tf_weight_names)

  print('check variables match in each block')
  for keras_block, tf_block in zip(keras_blocks, tf_blocks):
    check_match(keras_block,
                tf_block,
                keras_weight_names,
                tf_weight_names,
                model_name_tf)
    print(f'{keras_block} and {tf_block} match.')

  block_mapping = {x[0]: x[1] for x in zip(keras_blocks, tf_blocks)}

  changed_weights = 0
  for w in keras_model.weights:
    if 'block' in w.name:
      # example: 'block1a_dwconv/depthwise_kernel:0' -> 'block1a'
      keras_block = w.name.split('/')[0].split('_')[0]
      tf_block = block_mapping[keras_block]
      tf_name = keras_name_to_tf_name_block(w.name,
                                            keras_block=keras_block,
                                            tf_block=tf_block,
                                            use_ema=use_ema,
                                            model_name_tf=model_name_tf)
    elif any([x in w.name for x in ['stem', 'top', 'predictions', 'probs']]):
      tf_name = keras_name_to_tf_name_stem_top(w.name,
                                               use_ema=use_ema,
                                               model_name_tf=model_name_tf)
    elif 'normalization' in w.name:
      print(f'skipping variable {w.name}: normalization is a layer'
            'in keras implementation, but preprocessing in TF implementation.')
      continue
    else:
      raise ValueError(f'{w.name} failed to parse.')

    try:
      w_tf = tf.train.load_variable(path_ckpt, tf_name)
      if (w.value().numpy() != w_tf).any():
        w.assign(w_tf)
        changed_weights += 1
    except ValueError as e:
      if any([x in w.name for x in ['top', 'predictions', 'probs']]):
        warnings.warn(f'Fail to load top layer variable {w.name}'
                      f'from {tf_name} because of {e}.')
      else:
        raise ValueError(f'Fail to load {w.name} from {tf_name}')
  print(f'{changed_weights}/{len(keras_model.weights)} weights updated')
  keras_model.save_weights(path_h5)

def get_variable_names_from_ckpt(path_ckpt, use_ema=True):
  """Get list of tensor names from checkpoint

  Args:
    path_ckpt: str, path to the ckpt files
    use_ema: Bool, whether to use ExponentialMovingAverage result or not
  """
  v_all = tf.train.list_variables(path_ckpt)

  # keep name only
  v_name_all = [x[0] for x in v_all]

  if use_ema:
    v_name_all = [x for x in v_name_all if 'ExponentialMovingAverage' in x]
  else:
    v_name_all = [x for x in v_name_all if 'ExponentialMovingAverage' not in x]

  # remove util variables used for RMSprop
  v_name_all = [x for x in v_name_all if 'RMS' not in x]
  return v_name_all

def get_tf_blocks(tf_weight_names):
  """Extract the block names from list of full weight names"""
  # Example: 'efficientnet-b0/blocks_0/conv2d/kernel' -> 'blocks_0'
  tf_blocks = {x.split('/')[1] for x in tf_weight_names if 'block' in x}
  # sort by number
  tf_blocks = sorted(tf_blocks, key=lambda x: int(x.split('_')[1]))
  return tf_blocks

def get_keras_blocks(keras_weight_names):
  """Extract the block names from list of full weight names"""
  # example: 'block1a_dwconv/depthwise_kernel:0' -> 'block1a'
  keras_blocks = {x.split('_')[0] for x in keras_weight_names if 'block' in x}
  return sorted(keras_blocks)

def keras_name_to_tf_name_stem_top(keras_name,
                                   use_ema=True,
                                   model_name_tf='efficientnet-b0'):
  """ map name in h5 to ckpt that is in stem or top (head)

  we map name keras_name that points to a weight in h5 file
  to a name of weight in ckpt file.

  Args:
    keras_name: str, the name of weight in the h5 file of keras implementation
    use_ema: Bool, use the ExponentialMovingAverage resuolt in ckpt or not
    model_name_tf: str, the name of model in ckpt.

  Returns:
    String for the name of weight as in ckpt file.

  Raises:
    KeyError if we cannot parse the keras_name
  """
  if use_ema:
    ema = '/ExponentialMovingAverage'
  else:
    ema = ''

  stem_top_dict = {
      'probs/bias:0': f'{model_name_tf}/head/dense/bias{ema}',
      'probs/kernel:0': f'{model_name_tf}/head/dense/kernel{ema}',
      'predictions/bias:0': f'{model_name_tf}/head/dense/bias{ema}',
      'predictions/kernel:0': f'{model_name_tf}/head/dense/kernel{ema}',
      'stem_conv/kernel:0': f'{model_name_tf}/stem/conv2d/kernel{ema}',
      'top_conv/kernel:0': f'{model_name_tf}/head/conv2d/kernel{ema}',
  }

  # stem batch normalization
  for bn_weights in ['beta', 'gamma', 'moving_mean', 'moving_variance']:
    f_string = '{}/stem/tpu_batch_normalization/{}{}'
    stem_top_dict[f'stem_bn/{bn_weights}:0'] = f_string.format(model_name_tf,
                                                               bn_weights,
                                                               ema)
  # top / head batch normalization
  for bn_weights in ['beta', 'gamma', 'moving_mean', 'moving_variance']:
    f_string = '{}/head/tpu_batch_normalization/{}{}'
    stem_top_dict[f'top_bn/{bn_weights}:0'] = f_string.format(model_name_tf,
                                                              bn_weights,
                                                              ema)

  if keras_name in stem_top_dict:
    return stem_top_dict[keras_name]
  raise KeyError(f'{keras_name} from h5 file cannot be parsed')

def keras_name_to_tf_name_block(keras_name,
                                keras_block='block1a',
                                tf_block='blocks_0',
                                use_ema=True,
                                model_name_tf='efficientnet-b0'):
  """ map name in h5 to ckpt that belongs to a block

  we map name keras_name that points to a weight in h5 file
  to a name of weight in ckpt file.

  Args:
    keras_name: str, the name of weight in the h5 file of keras implementation
    keras_block: str, the block name for keras implementation (e.g. 'block1a')
    tf_block: str, the block name for tf implementation (e.g. 'blocks_0')
    use_ema: Bool, use the ExponentialMovingAverage resuolt in ckpt or not
    model_name_tf: str, the name of model in ckpt.

  Returns:
    String for the name of weight as in ckpt file.

  Raises:
    ValueError if keras_block does not show up in keras_name
  """

  if f'{keras_block}' not in keras_name:
    raise ValueError(f'block name {keras_block} not found in {keras_name}')

  # all blocks in the first group will not have expand conv and bn
  is_first_blocks = (keras_block[5] == '1')

  tf_name = [model_name_tf, tf_block]

  # depthwide conv
  if 'dwconv' in keras_name:
    tf_name.append('depthwise_conv2d')
    tf_name.append('depthwise_kernel')

  # conv layers
  if is_first_blocks:
    # first blocks only have one conv2d
    if 'project_conv' in keras_name:
      tf_name.append('conv2d')
      tf_name.append('kernel')
  else:
    if 'project_conv' in keras_name:
      tf_name.append('conv2d_1')
      tf_name.append('kernel')
    elif 'expand_conv' in keras_name:
      tf_name.append('conv2d')
      tf_name.append('kernel')

  # squeeze expansion layers
  if '_se_' in keras_name:
    if 'reduce' in keras_name:
      tf_name.append('se/conv2d')
    elif 'expand' in keras_name:
      tf_name.append('se/conv2d_1')

    if 'kernel' in keras_name:
      tf_name.append('kernel')
    elif 'bias' in keras_name:
      tf_name.append('bias')

  # batch normalization layers
  if 'bn' in keras_name:
    if is_first_blocks:
      if 'project' in keras_name:
        tf_name.append('tpu_batch_normalization_1')
      else:
        tf_name.append('tpu_batch_normalization')
    else:
      if 'project' in keras_name:
        tf_name.append('tpu_batch_normalization_2')
      elif 'expand' in keras_name:
        tf_name.append('tpu_batch_normalization')
      else:
        tf_name.append('tpu_batch_normalization_1')

    for x in ['moving_mean', 'moving_variance', 'beta', 'gamma']:
      if x in keras_name:
        tf_name.append(x)
  if use_ema:
    tf_name.append('ExponentialMovingAverage')
  return '/'.join(tf_name)

def check_match(keras_block,
                tf_block,
                keras_weight_names,
                tf_weight_names,
                model_name_tf):
  """ Check if the weights in h5 and ckpt match

  we match each name from keras_weight_names that is in keras_block
  and check if there is 1-1 correspondence to names from tf_weight_names
  that is in tf_block

  Args:
    keras_block: str, the block name for keras implementation (e.g. 'block1a')
    tf_block: str, the block name for tf implementation (e.g. 'blocks_0')
    keras_weight_names: list of str, weight names in keras implementation
    tf_weight_names: list of str, weight names in tf implementation
  """
  names_from_keras = set()
  for x in keras_weight_names:
    if keras_block in x:
      y = keras_name_to_tf_name_block(x,
                                      keras_block=keras_block,
                                      tf_block=tf_block,
                                      model_name_tf=model_name_tf)
      names_from_keras.add(y)

  names_from_tf = set()
  for x in tf_weight_names:
    if tf_block in x and x.split('/')[1].endswith(tf_block):
      names_from_tf.add(x)

  names_missing = names_from_keras - names_from_tf
  if len(names_missing) > 0:
    raise ValueError(f'{len(names_missing)} variables not found'
                     f'in checkpoint file: {names_missing}')

  names_unused = names_from_tf - names_from_keras
  if len(names_unused) > 0:
    warnings.warn(f'{len(names_unused)} variables from checkpoint file'
                  f'are not used: {names_unused}')

if __name__ == '__main__':
  arg_to_model = {
      'b0': efficientnet.EfficientNetB0,
      'b1': efficientnet.EfficientNetB1,
      'b2': efficientnet.EfficientNetB2,
      'b3': efficientnet.EfficientNetB3,
      'b4': efficientnet.EfficientNetB4,
      'b5': efficientnet.EfficientNetB5,
      'b6': efficientnet.EfficientNetB6,
      'b7': efficientnet.EfficientNetB7
  }

  p = argparse.ArgumentParser(description="write weights from checkpoint to h5")
  p.add_argument('--model',
                 required=True,
                 type=str,
                 help="name of efficient model",
                 choices=arg_to_model.keys())
  p.add_argument('--notop',
                 action="store_true",
                 help="do not include top layers",
                 default=False)
  p.add_argument('--ckpt',
                 required=True,
                 type=str,
                 help="checkpoint path")
  p.add_argument('--output',
                 '-o',
                 required=True,
                 type=str,
                 help="output (h5) file path")
  args = p.parse_args()

  include_top = not args.notop

  model = arg_to_model[args.model](include_top=include_top)
  write_ckpt_to_h5(args.output, args.ckpt, keras_model=model)
<commit_msg>add future import<commit_after># Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utils for EfficientNet models for Keras.
Write weights from  ckpt file as in original repo
(https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet)
to h5 file for keras implementation of the models.

Usage:

# use checkpoint efficientnet-b0/model.ckpt (can be downloaded from
# https://storage.googleapis.com/cloud-tpu-checkpoints/
#     efficientnet/ckptsaug/efficientnet-b0.tar.gz)
# to update weight without top layers, saving to efficientnetb0_notop.h5
python efficientnet_weight_update_util.py --model b0 --notop \
    --ckpt efficientnet-b0/model.ckpt --o efficientnetb0_notop.h5

# use checkpoint noisy_student_efficientnet-b3/model.ckpt (providing
# improved result for b3, can be downloaded from
# https://storage.googleapis.com/cloud-tpu-checkpoints/
#     efficientnet/noisystudent/noisy_student_efficientnet-b3.tar.gz)
# to update weight with top layers, saving to efficientnetb3_new.h5
python efficientnet_weight_update_util.py --model b3 --notop \
    --ckpt noisy_student_efficientnet-b3/model.ckpt --o efficientnetb3_new.h5

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse

import tensorflow as tf
from tensorflow.keras.applications import efficientnet

def write_ckpt_to_h5(path_h5, path_ckpt, keras_model, use_ema=True):
  """ Map the weights in checkpoint file (tf) to h5 file (keras)

  Args:
    path_h5: str, path to output hdf5 file to write weights loaded
      from ckpt files.
    path_ckpt: str, path to the ckpt files (e.g. 'efficientnet-b0/model.ckpt')
      that records efficientnet weights from original repo
      https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet
    keras_model: keras model, built from keras.applications efficientnet
      functions (e.g. EfficientNetB0)
    use_ema: Bool, whether to use ExponentialMovingAverage result or not
  """
  model_name_keras = keras_model.name
  model_name_tf = model_name_keras.replace('efficientnet', 'efficientnet-')

  keras_weight_names = [w.name for w in keras_model.weights]
  tf_weight_names = get_variable_names_from_ckpt(path_ckpt)

  keras_blocks = get_keras_blocks(keras_weight_names)
  tf_blocks = get_tf_blocks(tf_weight_names)

  print('check variables match in each block')
  for keras_block, tf_block in zip(keras_blocks, tf_blocks):
    check_match(keras_block,
                tf_block,
                keras_weight_names,
                tf_weight_names,
                model_name_tf)
    print(f'{keras_block} and {tf_block} match.')

  block_mapping = {x[0]: x[1] for x in zip(keras_blocks, tf_blocks)}

  changed_weights = 0
  for w in keras_model.weights:
    if 'block' in w.name:
      # example: 'block1a_dwconv/depthwise_kernel:0' -> 'block1a'
      keras_block = w.name.split('/')[0].split('_')[0]
      tf_block = block_mapping[keras_block]
      tf_name = keras_name_to_tf_name_block(w.name,
                                            keras_block=keras_block,
                                            tf_block=tf_block,
                                            use_ema=use_ema,
                                            model_name_tf=model_name_tf)
    elif any([x in w.name for x in ['stem', 'top', 'predictions', 'probs']]):
      tf_name = keras_name_to_tf_name_stem_top(w.name,
                                               use_ema=use_ema,
                                               model_name_tf=model_name_tf)
    elif 'normalization' in w.name:
      print(f'skipping variable {w.name}: normalization is a layer'
            'in keras implementation, but preprocessing in TF implementation.')
      continue
    else:
      raise ValueError(f'{w.name} failed to parse.')

    try:
      w_tf = tf.train.load_variable(path_ckpt, tf_name)
      if (w.value().numpy() != w_tf).any():
        w.assign(w_tf)
        changed_weights += 1
    except ValueError as e:
      if any([x in w.name for x in ['top', 'predictions', 'probs']]):
        warnings.warn(f'Fail to load top layer variable {w.name}'
                      f'from {tf_name} because of {e}.')
      else:
        raise ValueError(f'Fail to load {w.name} from {tf_name}')
  print(f'{changed_weights}/{len(keras_model.weights)} weights updated')
  keras_model.save_weights(path_h5)

def get_variable_names_from_ckpt(path_ckpt, use_ema=True):
  """Get list of tensor names from checkpoint

  Args:
    path_ckpt: str, path to the ckpt files
    use_ema: Bool, whether to use ExponentialMovingAverage result or not
  """
  v_all = tf.train.list_variables(path_ckpt)

  # keep name only
  v_name_all = [x[0] for x in v_all]

  if use_ema:
    v_name_all = [x for x in v_name_all if 'ExponentialMovingAverage' in x]
  else:
    v_name_all = [x for x in v_name_all if 'ExponentialMovingAverage' not in x]

  # remove util variables used for RMSprop
  v_name_all = [x for x in v_name_all if 'RMS' not in x]
  return v_name_all

def get_tf_blocks(tf_weight_names):
  """Extract the block names from list of full weight names"""
  # Example: 'efficientnet-b0/blocks_0/conv2d/kernel' -> 'blocks_0'
  tf_blocks = {x.split('/')[1] for x in tf_weight_names if 'block' in x}
  # sort by number
  tf_blocks = sorted(tf_blocks, key=lambda x: int(x.split('_')[1]))
  return tf_blocks

def get_keras_blocks(keras_weight_names):
  """Extract the block names from list of full weight names"""
  # example: 'block1a_dwconv/depthwise_kernel:0' -> 'block1a'
  keras_blocks = {x.split('_')[0] for x in keras_weight_names if 'block' in x}
  return sorted(keras_blocks)

def keras_name_to_tf_name_stem_top(keras_name,
                                   use_ema=True,
                                   model_name_tf='efficientnet-b0'):
  """ map name in h5 to ckpt that is in stem or top (head)

  we map name keras_name that points to a weight in h5 file
  to a name of weight in ckpt file.

  Args:
    keras_name: str, the name of weight in the h5 file of keras implementation
    use_ema: Bool, use the ExponentialMovingAverage resuolt in ckpt or not
    model_name_tf: str, the name of model in ckpt.

  Returns:
    String for the name of weight as in ckpt file.

  Raises:
    KeyError if we cannot parse the keras_name
  """
  if use_ema:
    ema = '/ExponentialMovingAverage'
  else:
    ema = ''

  stem_top_dict = {
      'probs/bias:0': f'{model_name_tf}/head/dense/bias{ema}',
      'probs/kernel:0': f'{model_name_tf}/head/dense/kernel{ema}',
      'predictions/bias:0': f'{model_name_tf}/head/dense/bias{ema}',
      'predictions/kernel:0': f'{model_name_tf}/head/dense/kernel{ema}',
      'stem_conv/kernel:0': f'{model_name_tf}/stem/conv2d/kernel{ema}',
      'top_conv/kernel:0': f'{model_name_tf}/head/conv2d/kernel{ema}',
  }

  # stem batch normalization
  for bn_weights in ['beta', 'gamma', 'moving_mean', 'moving_variance']:
    f_string = '{}/stem/tpu_batch_normalization/{}{}'
    stem_top_dict[f'stem_bn/{bn_weights}:0'] = f_string.format(model_name_tf,
                                                               bn_weights,
                                                               ema)
  # top / head batch normalization
  for bn_weights in ['beta', 'gamma', 'moving_mean', 'moving_variance']:
    f_string = '{}/head/tpu_batch_normalization/{}{}'
    stem_top_dict[f'top_bn/{bn_weights}:0'] = f_string.format(model_name_tf,
                                                              bn_weights,
                                                              ema)

  if keras_name in stem_top_dict:
    return stem_top_dict[keras_name]
  raise KeyError(f'{keras_name} from h5 file cannot be parsed')

def keras_name_to_tf_name_block(keras_name,
                                keras_block='block1a',
                                tf_block='blocks_0',
                                use_ema=True,
                                model_name_tf='efficientnet-b0'):
  """ map name in h5 to ckpt that belongs to a block

  we map name keras_name that points to a weight in h5 file
  to a name of weight in ckpt file.

  Args:
    keras_name: str, the name of weight in the h5 file of keras implementation
    keras_block: str, the block name for keras implementation (e.g. 'block1a')
    tf_block: str, the block name for tf implementation (e.g. 'blocks_0')
    use_ema: Bool, use the ExponentialMovingAverage resuolt in ckpt or not
    model_name_tf: str, the name of model in ckpt.

  Returns:
    String for the name of weight as in ckpt file.

  Raises:
    ValueError if keras_block does not show up in keras_name
  """

  if f'{keras_block}' not in keras_name:
    raise ValueError(f'block name {keras_block} not found in {keras_name}')

  # all blocks in the first group will not have expand conv and bn
  is_first_blocks = (keras_block[5] == '1')

  tf_name = [model_name_tf, tf_block]

  # depthwide conv
  if 'dwconv' in keras_name:
    tf_name.append('depthwise_conv2d')
    tf_name.append('depthwise_kernel')

  # conv layers
  if is_first_blocks:
    # first blocks only have one conv2d
    if 'project_conv' in keras_name:
      tf_name.append('conv2d')
      tf_name.append('kernel')
  else:
    if 'project_conv' in keras_name:
      tf_name.append('conv2d_1')
      tf_name.append('kernel')
    elif 'expand_conv' in keras_name:
      tf_name.append('conv2d')
      tf_name.append('kernel')

  # squeeze expansion layers
  if '_se_' in keras_name:
    if 'reduce' in keras_name:
      tf_name.append('se/conv2d')
    elif 'expand' in keras_name:
      tf_name.append('se/conv2d_1')

    if 'kernel' in keras_name:
      tf_name.append('kernel')
    elif 'bias' in keras_name:
      tf_name.append('bias')

  # batch normalization layers
  if 'bn' in keras_name:
    if is_first_blocks:
      if 'project' in keras_name:
        tf_name.append('tpu_batch_normalization_1')
      else:
        tf_name.append('tpu_batch_normalization')
    else:
      if 'project' in keras_name:
        tf_name.append('tpu_batch_normalization_2')
      elif 'expand' in keras_name:
        tf_name.append('tpu_batch_normalization')
      else:
        tf_name.append('tpu_batch_normalization_1')

    for x in ['moving_mean', 'moving_variance', 'beta', 'gamma']:
      if x in keras_name:
        tf_name.append(x)
  if use_ema:
    tf_name.append('ExponentialMovingAverage')
  return '/'.join(tf_name)

def check_match(keras_block,
                tf_block,
                keras_weight_names,
                tf_weight_names,
                model_name_tf):
  """ Check if the weights in h5 and ckpt match

  we match each name from keras_weight_names that is in keras_block
  and check if there is 1-1 correspondence to names from tf_weight_names
  that is in tf_block

  Args:
    keras_block: str, the block name for keras implementation (e.g. 'block1a')
    tf_block: str, the block name for tf implementation (e.g. 'blocks_0')
    keras_weight_names: list of str, weight names in keras implementation
    tf_weight_names: list of str, weight names in tf implementation
  """
  names_from_keras = set()
  for x in keras_weight_names:
    if keras_block in x:
      y = keras_name_to_tf_name_block(x,
                                      keras_block=keras_block,
                                      tf_block=tf_block,
                                      model_name_tf=model_name_tf)
      names_from_keras.add(y)

  names_from_tf = set()
  for x in tf_weight_names:
    if tf_block in x and x.split('/')[1].endswith(tf_block):
      names_from_tf.add(x)

  names_missing = names_from_keras - names_from_tf
  if len(names_missing) > 0:
    raise ValueError(f'{len(names_missing)} variables not found'
                     f'in checkpoint file: {names_missing}')

  names_unused = names_from_tf - names_from_keras
  if len(names_unused) > 0:
    warnings.warn(f'{len(names_unused)} variables from checkpoint file'
                  f'are not used: {names_unused}')

if __name__ == '__main__':
  arg_to_model = {
      'b0': efficientnet.EfficientNetB0,
      'b1': efficientnet.EfficientNetB1,
      'b2': efficientnet.EfficientNetB2,
      'b3': efficientnet.EfficientNetB3,
      'b4': efficientnet.EfficientNetB4,
      'b5': efficientnet.EfficientNetB5,
      'b6': efficientnet.EfficientNetB6,
      'b7': efficientnet.EfficientNetB7
  }

  p = argparse.ArgumentParser(description="write weights from checkpoint to h5")
  p.add_argument('--model',
                 required=True,
                 type=str,
                 help="name of efficient model",
                 choices=arg_to_model.keys())
  p.add_argument('--notop',
                 action="store_true",
                 help="do not include top layers",
                 default=False)
  p.add_argument('--ckpt',
                 required=True,
                 type=str,
                 help="checkpoint path")
  p.add_argument('--output',
                 '-o',
                 required=True,
                 type=str,
                 help="output (h5) file path")
  args = p.parse_args()

  include_top = not args.notop

  model = arg_to_model[args.model](include_top=include_top)
  write_ckpt_to_h5(args.output, args.ckpt, keras_model=model)
<|endoftext|>


<commit_before># *********************************************************************************************************************
# locality-clean.py
# *********************************************************************************************************************
#
# Takes the already processed locality_boundaries from the gnaf-loader (see https://github.com/minus34/gnaf-loader) and
# prepares them for presentation and visualisation, by doing the following:
#  1. Trims the localities to the coastline;
#  2. Cleans the overlaps and gaps along each state border;
#  3. Thins the polygons to for faster display in both desktop GIS and in browsers; and
#  4. Exports the end result to Shapefile and GeoJSON (for use in Elasticsearch)
#
# Organisation: IAG
# Author: Hugh Saalmans, Location Engineering Director
# GitHub: iag-geo
#
# Copyright:
#  - Code is copyright IAG - licensed under an Apache License, version 2.0
#  - Data is copyright PSMA - licensed under a Creative Commons (By Attribution) license
#
# Pre-requisites
#  - Either: run the gnaf-loader Python script (30-60 mins); or load the gnaf-loader admin-bdys schema into Postgres
#      (see https://github.com/minus34/gnaf-loader)
#  - Postgres 9.x (tested on 9.3, 9.4 & 9.5 on Windows and 9.5 & 9.6 on macOS)
#  - PostGIS 2.2+
#  - Python 2.7 or 3.5 with Psycopg2 2.6.x
#
# TO DO:
#  - Create postcode boundaries by aggregating the final localities by their postcode (derived from raw GNAF)
#
# *********************************************************************************************************************

import argparse
import json
import logging.config
import os
import platform
import psma
import psycopg2

from datetime import datetime

def main():
    full_start_time = datetime.now()

    # set command line arguments
    args = set_arguments()
    print(args)
    # get settings from arguments
    settings = get_settings(args)
    print(settings)
    # connect to Postgres
    try:
        pg_conn = psycopg2.connect(settings['pg_connect_string'])
    except psycopg2.Error:
        logger.fatal("Unable to connect to database\nACTION: Check your Postgres parameters and/or database security")
        return False

    pg_conn.autocommit = True
    pg_cur = pg_conn.cursor()

    # log postgres/postgis versions being used
    psma.check_postgis_version(pg_cur, settings, logger)

    # add Postgres functions to clean out non-polygon geometries from GeometryCollections
    pg_cur.execute(psma.open_sql_file("create-polygon-intersection-function.sql", settings))
    pg_cur.execute(psma.open_sql_file("create-multi-linestring-split-function.sql", settings))

    # let's build some clean localities!
    logger.info("")
    create_states_and_prep_localities(settings)
    get_split_localities(pg_cur, settings)
    verify_locality_polygons(pg_cur, settings)
    get_locality_state_border_gaps(pg_cur, settings)
    finalise_display_localities(pg_cur, settings)
    export_display_localities(pg_cur, settings)
    qa_display_localities(pg_cur, settings)

    pg_cur.close()
    pg_conn.close()

    logger.info("Total time : {0}".format(datetime.now() - full_start_time))

    return True

# set the command line arguments for the script
def set_arguments():
    parser = argparse.ArgumentParser(
        description='A quick way to load the complete GNAF and PSMA Admin Boundaries into Postgres, '
                    'simplified and ready to use as reference data for geocoding, analysis and visualisation.')

    parser.add_argument(
        '--max-processes', type=int, default=3,
        help='Maximum number of parallel processes to use for the data load. (Set it to the number of cores on the '
             'Postgres server minus 2, limit to 12 if 16+ cores - there is minimal benefit beyond 12). Defaults to 6.')

    # PG Options
    parser.add_argument(
        '--pghost',
        help='Host name for Postgres server. Defaults to PGHOST environment variable if set, otherwise localhost.')
    parser.add_argument(
        '--pgport', type=int,
        help='Port number for Postgres server. Defaults to PGPORT environment variable if set, otherwise 5432.')
    parser.add_argument(
        '--pgdb',
        help='Database name for Postgres server. Defaults to PGDATABASE environment variable if set, '
             'otherwise psma.')
    parser.add_argument(
        '--pguser',
        help='Username for Postgres server. Defaults to PGUSER environment variable if set, otherwise postgres.')
    parser.add_argument(
        '--pgpassword',
        help='Password for Postgres server. Defaults to PGPASSWORD environment variable if set, '
             'otherwise \'password\'.')

    # schema names for the raw gnaf, flattened reference and admin boundary tables
    psma_version = psma.get_psma_version(datetime.today())

    parser.add_argument(
        '--psma-version', default=psma_version,
        help='PSMA Version number as YYYYMM. Defaults to last release year and month \'' + psma_version + '\'.')
    parser.add_argument(
        '--gnaf-schema', default='gnaf_' + psma_version,
        help='Destination schema name to store final GNAF tables in. Defaults to \'gnaf_' + psma_version + '\'.')
    parser.add_argument(
        '--admin-schema', default='admin_bdys_' + psma_version,
        help='Destination schema name to store final admin boundary tables in. Defaults to \'admin_bdys_'
             + psma_version + '\'.')
    parser.add_argument(
        '--sa4-boundary-table', default='abs_2016_sa4',
        help='SA4 table name used to create state boundaries. Defaults to \'abs_2016_sa4\'. Other options are: \'abs_2011_sa4\'')
    # output directory
    parser.add_argument(
        '--output-path', required=True,
        help='Local path where the Shapefile and GeoJSON files will be output.')

    return parser.parse_args()

# create the dictionary of settings
def get_settings(args):
    settings = dict()

    settings['max_concurrent_processes'] = args.max_processes
    settings['psma_version'] = args.psma_version
    settings['gnaf_schema'] = args.gnaf_schema
    settings['admin_bdys_schema'] = args.admin_schema
    settings['sa4_boundary_table'] = args.sa4_boundary_table
    settings['output_path'] = args.output_path

    # create postgres connect string
    settings['pg_host'] = args.pghost or os.getenv("PGHOST", "localhost")
    settings['pg_port'] = args.pgport or os.getenv("PGPORT", 5432)
    settings['pg_db'] = args.pgdb or os.getenv("PGDATABASE", "psma")
    settings['pg_user'] = args.pguser or os.getenv("PGUSER", "postgres")
    settings['pg_password'] = args.pgpassword or os.getenv("PGPASSWORD", "password")

    settings['pg_connect_string'] = "dbname='{0}' host='{1}' port='{2}' user='{3}' password='{4}'".format(
        settings['pg_db'], settings['pg_host'], settings['pg_port'], settings['pg_user'], settings['pg_password'])

    # set postgres script directory
    settings['sql_dir'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), "postgres-scripts")

    # full path and file name to export the resulting Shapefile to
    settings['shapefile_export_path'] = os.path.join(settings['output_path'], "locality-bdys-display-{0}.shp"
                                                     .format(settings['psma_version']))
    settings['geojson_export_path'] = os.path.join(settings['output_path'], "locality-bdys-display-{0}.geojson"
                                                   .format(settings['psma_version']))

    # left over issue with the psma.py module - don't edit this
    settings['raw_gnaf_schema'] = None
    settings['raw_admin_bdys_schema'] = None

    return settings

def create_states_and_prep_localities(settings):
    start_time = datetime.now()
    sql_list = [psma.open_sql_file("01a-create-states-from-sa4s.sql", settings),
                psma.open_sql_file("01b-prep-locality-boundaries.sql", settings)]
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 1 of 7 : state table created & localities prepped : {0}".format(datetime.now() - start_time))

# split locality bdys by state bdys, using multiprocessing
def get_split_localities(pg_cur, settings):
    start_time = datetime.now()
    sql = psma.open_sql_file("02-split-localities-by-state-borders.sql", settings)
    sql_list = psma.split_sql_into_list(pg_cur, sql, settings['admin_bdys_schema'], "temp_localities", "loc", "gid",
                                        settings, logger)
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 2 of 7 : localities split by state : {0}".format(datetime.now() - start_time))

def verify_locality_polygons(pg_cur, settings):
    start_time = datetime.now()
    pg_cur.execute(psma.open_sql_file("03a-verify-split-polygons.sql", settings))
    pg_cur.execute(psma.open_sql_file("03b-load-messy-centroids.sql", settings))
    logger.info("\t- Step 3 of 7 : messy locality polygons verified : {0}".format(datetime.now() - start_time))

# get holes in the localities along the state borders, using multiprocessing (doesn't help much - too few states!)
def get_locality_state_border_gaps(pg_cur, settings):
    start_time = datetime.now()
    sql = psma.open_sql_file("04-create-holes-along-borders.sql", settings)
    sql_list = psma.split_sql_into_list(pg_cur, sql, settings['admin_bdys_schema'],
                                        "temp_state_border_buffers_subdivided", "ste", "new_gid", settings, logger)
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 4 of 7 : locality holes created : {0}".format(datetime.now() - start_time))

def finalise_display_localities(pg_cur, settings):
    start_time = datetime.now()
    pg_cur.execute(psma.open_sql_file("05-finalise-display-localities.sql", settings))
    logger.info("\t- Step 5 of 7 : display localities finalised : {0}".format(datetime.now() - start_time))

def export_display_localities(pg_cur, settings):
    start_time = datetime.now()

    sql = psma.open_sql_file("06-export-display-localities.sql", settings)

    if platform.system() == "Windows":
        password_str = "SET"
    else:
        password_str = "export"

    password_str += " PGPASSWORD={0}&&".format(settings['pg_password'])

    cmd = password_str + "pgsql2shp -f \"{0}\" -u {1} -h {2} -p {3} {4} \"{5}\""\
        .format(settings['shapefile_export_path'], settings['pg_user'], settings['pg_host'],
                settings['pg_port'], settings['pg_db'], sql)

    # logger.info(cmd
    psma.run_command_line(cmd)

    logger.info("\t- Step 6 of 7 : display localities exported to SHP : {0}".format(datetime.now() - start_time))
    logger.warning("\t\t- If this step took < 1 second - it may have failed silently. "
                   "Check your output directory!")

    start_time = datetime.now()

    # Export as GeoJSON FeatureCollection
    sql = psma.prep_sql("SELECT gid, locality_pid, locality_name, COALESCE(postcode, '') AS postcode, state, "
                        "locality_class, address_count, street_count, ST_AsGeoJSON(geom, 5, 0) AS geom "
                        "FROM {0}.locality_bdys_display".format(settings['admin_bdys_schema']), settings)
    pg_cur.execute(sql)

    # Create the GeoJSON output with an array of dictionaries containing the field names and values

    # get column names from cursor
    column_names = [desc[0] for desc in pg_cur.description]

    json_dicts = []
    row = pg_cur.fetchone()

    if row is not None:
        while row is not None:
            rec = {}
            props = {}
            i = 0
            rec["type"] = "Feature"

            for column in column_names:
                if column == "geometry" or column == "geom":
                    rec["geometry"] = row[i]
                else:
                    props[column] = row[i]

                i += 1

            rec["properties"] = props
            json_dicts.append(rec)
            row = pg_cur.fetchone()

    gj = json.dumps(json_dicts).replace("\\", "").replace('"{', '{').replace('}"', '}')

    geojson = ''.join(['{"type":"FeatureCollection","features":', gj, '}'])

    text_file = open(settings['geojson_export_path'], "w")
    text_file.write(geojson)
    text_file.close()

    logger.info("\t- Step 6 of 7 : display localities exported to GeoJSON : {0}".format(datetime.now() - start_time))

def qa_display_localities(pg_cur, settings):
    logger.info("\t- Step 7 of 7 : Start QA")
    start_time = datetime.now()

    pg_cur.execute(psma.prep_sql("SELECT locality_pid, Locality_name, postcode, state, address_count, street_count "
                                 "FROM admin_bdys.locality_bdys_display WHERE NOT ST_IsValid(geom);", settings))
    display_qa_results("Invalid Geometries", pg_cur)

    pg_cur.execute(psma.prep_sql("SELECT locality_pid, Locality_name, postcode, state, address_count, street_count "
                                 "FROM admin_bdys.locality_bdys_display WHERE ST_IsEmpty(geom);", settings))
    display_qa_results("Empty Geometries", pg_cur)

    pg_cur.execute(psma.open_sql_file("07-qa-display-localities.sql", settings))
    display_qa_results("Dropped Localities", pg_cur)

    logger.info("\t- Step 7 of 7 : display localities qa'd : {0}".format(datetime.now() - start_time))

def display_qa_results(purpose, pg_cur):
    logger.info("\t\t" + purpose)
    logger.info("\t\t----------------------------------------")

    results = pg_cur.fetchall()

    if results is not None:
        # print the column names returned
        logger.info("\t\t" + ",".join([desc[0] for desc in pg_cur.description]))

        for result in results:
            logger.info("\t\t" + ",".join(map(str, result)))
    else:
        logger.info("\t\t" + "No records")

    logger.info("\t\t----------------------------------------")

if __name__ == '__main__':
    logger = logging.getLogger()

    # set logger
    log_file = os.path.abspath(__file__).replace(".py", ".log")
    logging.basicConfig(filename=log_file, level=logging.DEBUG, format="%(asctime)s %(message)s",
                        datefmt="%m/%d/%Y %I:%M:%S %p")

    # setup logger to write to screen as well as writing to log file
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    logger.info("")
    logger.info("Start locality-clean")
    psma.check_python_version(logger)

    if main():
        logger.info("Finished successfully!")
    else:
        logger.fatal("Something bad happened!")

    logger.info("")
    logger.info("-------------------------------------------------------------------------------")
<commit_msg>removed debug<commit_after># *********************************************************************************************************************
# locality-clean.py
# *********************************************************************************************************************
#
# Takes the already processed locality_boundaries from the gnaf-loader (see https://github.com/minus34/gnaf-loader) and
# prepares them for presentation and visualisation, by doing the following:
#  1. Trims the localities to the coastline;
#  2. Cleans the overlaps and gaps along each state border;
#  3. Thins the polygons to for faster display in both desktop GIS and in browsers; and
#  4. Exports the end result to Shapefile and GeoJSON (for use in Elasticsearch)
#
# Organisation: IAG
# Author: Hugh Saalmans, Location Engineering Director
# GitHub: iag-geo
#
# Copyright:
#  - Code is copyright IAG - licensed under an Apache License, version 2.0
#  - Data is copyright PSMA - licensed under a Creative Commons (By Attribution) license
#
# Pre-requisites
#  - Either: run the gnaf-loader Python script (30-60 mins); or load the gnaf-loader admin-bdys schema into Postgres
#      (see https://github.com/minus34/gnaf-loader)
#  - Postgres 9.x (tested on 9.3, 9.4 & 9.5 on Windows and 9.5 & 9.6 on macOS)
#  - PostGIS 2.2+
#  - Python 2.7 or 3.5 with Psycopg2 2.6.x
#
# TO DO:
#  - Create postcode boundaries by aggregating the final localities by their postcode (derived from raw GNAF)
#
# *********************************************************************************************************************

import argparse
import json
import logging.config
import os
import platform
import psma
import psycopg2

from datetime import datetime

def main():
    full_start_time = datetime.now()

    # set command line arguments
    args = set_arguments()
    # get settings from arguments
    settings = get_settings(args)
    # connect to Postgres
    try:
        pg_conn = psycopg2.connect(settings['pg_connect_string'])
    except psycopg2.Error:
        logger.fatal("Unable to connect to database\nACTION: Check your Postgres parameters and/or database security")
        return False

    pg_conn.autocommit = True
    pg_cur = pg_conn.cursor()

    # log postgres/postgis versions being used
    psma.check_postgis_version(pg_cur, settings, logger)

    # add Postgres functions to clean out non-polygon geometries from GeometryCollections
    pg_cur.execute(psma.open_sql_file("create-polygon-intersection-function.sql", settings))
    pg_cur.execute(psma.open_sql_file("create-multi-linestring-split-function.sql", settings))

    # let's build some clean localities!
    logger.info("")
    create_states_and_prep_localities(settings)
    get_split_localities(pg_cur, settings)
    verify_locality_polygons(pg_cur, settings)
    get_locality_state_border_gaps(pg_cur, settings)
    finalise_display_localities(pg_cur, settings)
    export_display_localities(pg_cur, settings)
    qa_display_localities(pg_cur, settings)

    pg_cur.close()
    pg_conn.close()

    logger.info("Total time : {0}".format(datetime.now() - full_start_time))

    return True

# set the command line arguments for the script
def set_arguments():
    parser = argparse.ArgumentParser(
        description='A quick way to load the complete GNAF and PSMA Admin Boundaries into Postgres, '
                    'simplified and ready to use as reference data for geocoding, analysis and visualisation.')

    parser.add_argument(
        '--max-processes', type=int, default=3,
        help='Maximum number of parallel processes to use for the data load. (Set it to the number of cores on the '
             'Postgres server minus 2, limit to 12 if 16+ cores - there is minimal benefit beyond 12). Defaults to 6.')

    # PG Options
    parser.add_argument(
        '--pghost',
        help='Host name for Postgres server. Defaults to PGHOST environment variable if set, otherwise localhost.')
    parser.add_argument(
        '--pgport', type=int,
        help='Port number for Postgres server. Defaults to PGPORT environment variable if set, otherwise 5432.')
    parser.add_argument(
        '--pgdb',
        help='Database name for Postgres server. Defaults to PGDATABASE environment variable if set, '
             'otherwise psma.')
    parser.add_argument(
        '--pguser',
        help='Username for Postgres server. Defaults to PGUSER environment variable if set, otherwise postgres.')
    parser.add_argument(
        '--pgpassword',
        help='Password for Postgres server. Defaults to PGPASSWORD environment variable if set, '
             'otherwise \'password\'.')

    # schema names for the raw gnaf, flattened reference and admin boundary tables
    psma_version = psma.get_psma_version(datetime.today())

    parser.add_argument(
        '--psma-version', default=psma_version,
        help='PSMA Version number as YYYYMM. Defaults to last release year and month \'' + psma_version + '\'.')
    parser.add_argument(
        '--gnaf-schema', default='gnaf_' + psma_version,
        help='Destination schema name to store final GNAF tables in. Defaults to \'gnaf_' + psma_version + '\'.')
    parser.add_argument(
        '--admin-schema', default='admin_bdys_' + psma_version,
        help='Destination schema name to store final admin boundary tables in. Defaults to \'admin_bdys_'
             + psma_version + '\'.')
    parser.add_argument(
        '--sa4-boundary-table', default='abs_2016_sa4',
        help='SA4 table name used to create state boundaries. Defaults to \'abs_2016_sa4\'. Other options are: \'abs_2011_sa4\'')
    # output directory
    parser.add_argument(
        '--output-path', required=True,
        help='Local path where the Shapefile and GeoJSON files will be output.')

    return parser.parse_args()

# create the dictionary of settings
def get_settings(args):
    settings = dict()

    settings['max_concurrent_processes'] = args.max_processes
    settings['psma_version'] = args.psma_version
    settings['gnaf_schema'] = args.gnaf_schema
    settings['admin_bdys_schema'] = args.admin_schema
    settings['sa4_boundary_table'] = args.sa4_boundary_table
    settings['output_path'] = args.output_path

    # create postgres connect string
    settings['pg_host'] = args.pghost or os.getenv("PGHOST", "localhost")
    settings['pg_port'] = args.pgport or os.getenv("PGPORT", 5432)
    settings['pg_db'] = args.pgdb or os.getenv("PGDATABASE", "psma")
    settings['pg_user'] = args.pguser or os.getenv("PGUSER", "postgres")
    settings['pg_password'] = args.pgpassword or os.getenv("PGPASSWORD", "password")

    settings['pg_connect_string'] = "dbname='{0}' host='{1}' port='{2}' user='{3}' password='{4}'".format(
        settings['pg_db'], settings['pg_host'], settings['pg_port'], settings['pg_user'], settings['pg_password'])

    # set postgres script directory
    settings['sql_dir'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), "postgres-scripts")

    # full path and file name to export the resulting Shapefile to
    settings['shapefile_export_path'] = os.path.join(settings['output_path'], "locality-bdys-display-{0}.shp"
                                                     .format(settings['psma_version']))
    settings['geojson_export_path'] = os.path.join(settings['output_path'], "locality-bdys-display-{0}.geojson"
                                                   .format(settings['psma_version']))

    # left over issue with the psma.py module - don't edit this
    settings['raw_gnaf_schema'] = None
    settings['raw_admin_bdys_schema'] = None

    return settings

def create_states_and_prep_localities(settings):
    start_time = datetime.now()
    sql_list = [psma.open_sql_file("01a-create-states-from-sa4s.sql", settings),
                psma.open_sql_file("01b-prep-locality-boundaries.sql", settings)]
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 1 of 7 : state table created & localities prepped : {0}".format(datetime.now() - start_time))

# split locality bdys by state bdys, using multiprocessing
def get_split_localities(pg_cur, settings):
    start_time = datetime.now()
    sql = psma.open_sql_file("02-split-localities-by-state-borders.sql", settings)
    sql_list = psma.split_sql_into_list(pg_cur, sql, settings['admin_bdys_schema'], "temp_localities", "loc", "gid",
                                        settings, logger)
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 2 of 7 : localities split by state : {0}".format(datetime.now() - start_time))

def verify_locality_polygons(pg_cur, settings):
    start_time = datetime.now()
    pg_cur.execute(psma.open_sql_file("03a-verify-split-polygons.sql", settings))
    pg_cur.execute(psma.open_sql_file("03b-load-messy-centroids.sql", settings))
    logger.info("\t- Step 3 of 7 : messy locality polygons verified : {0}".format(datetime.now() - start_time))

# get holes in the localities along the state borders, using multiprocessing (doesn't help much - too few states!)
def get_locality_state_border_gaps(pg_cur, settings):
    start_time = datetime.now()
    sql = psma.open_sql_file("04-create-holes-along-borders.sql", settings)
    sql_list = psma.split_sql_into_list(pg_cur, sql, settings['admin_bdys_schema'],
                                        "temp_state_border_buffers_subdivided", "ste", "new_gid", settings, logger)
    psma.multiprocess_list("sql", sql_list, settings, logger)
    logger.info("\t- Step 4 of 7 : locality holes created : {0}".format(datetime.now() - start_time))

def finalise_display_localities(pg_cur, settings):
    start_time = datetime.now()
    pg_cur.execute(psma.open_sql_file("05-finalise-display-localities.sql", settings))
    logger.info("\t- Step 5 of 7 : display localities finalised : {0}".format(datetime.now() - start_time))

def export_display_localities(pg_cur, settings):
    start_time = datetime.now()

    sql = psma.open_sql_file("06-export-display-localities.sql", settings)

    if platform.system() == "Windows":
        password_str = "SET"
    else:
        password_str = "export"

    password_str += " PGPASSWORD={0}&&".format(settings['pg_password'])

    cmd = password_str + "pgsql2shp -f \"{0}\" -u {1} -h {2} -p {3} {4} \"{5}\""\
        .format(settings['shapefile_export_path'], settings['pg_user'], settings['pg_host'],
                settings['pg_port'], settings['pg_db'], sql)

    # logger.info(cmd
    psma.run_command_line(cmd)

    logger.info("\t- Step 6 of 7 : display localities exported to SHP : {0}".format(datetime.now() - start_time))
    logger.warning("\t\t- If this step took < 1 second - it may have failed silently. "
                   "Check your output directory!")

    start_time = datetime.now()

    # Export as GeoJSON FeatureCollection
    sql = psma.prep_sql("SELECT gid, locality_pid, locality_name, COALESCE(postcode, '') AS postcode, state, "
                        "locality_class, address_count, street_count, ST_AsGeoJSON(geom, 5, 0) AS geom "
                        "FROM {0}.locality_bdys_display".format(settings['admin_bdys_schema']), settings)
    pg_cur.execute(sql)

    # Create the GeoJSON output with an array of dictionaries containing the field names and values

    # get column names from cursor
    column_names = [desc[0] for desc in pg_cur.description]

    json_dicts = []
    row = pg_cur.fetchone()

    if row is not None:
        while row is not None:
            rec = {}
            props = {}
            i = 0
            rec["type"] = "Feature"

            for column in column_names:
                if column == "geometry" or column == "geom":
                    rec["geometry"] = row[i]
                else:
                    props[column] = row[i]

                i += 1

            rec["properties"] = props
            json_dicts.append(rec)
            row = pg_cur.fetchone()

    gj = json.dumps(json_dicts).replace("\\", "").replace('"{', '{').replace('}"', '}')

    geojson = ''.join(['{"type":"FeatureCollection","features":', gj, '}'])

    text_file = open(settings['geojson_export_path'], "w")
    text_file.write(geojson)
    text_file.close()

    logger.info("\t- Step 6 of 7 : display localities exported to GeoJSON : {0}".format(datetime.now() - start_time))

def qa_display_localities(pg_cur, settings):
    logger.info("\t- Step 7 of 7 : Start QA")
    start_time = datetime.now()

    pg_cur.execute(psma.prep_sql("SELECT locality_pid, Locality_name, postcode, state, address_count, street_count "
                                 "FROM admin_bdys.locality_bdys_display WHERE NOT ST_IsValid(geom);", settings))
    display_qa_results("Invalid Geometries", pg_cur)

    pg_cur.execute(psma.prep_sql("SELECT locality_pid, Locality_name, postcode, state, address_count, street_count "
                                 "FROM admin_bdys.locality_bdys_display WHERE ST_IsEmpty(geom);", settings))
    display_qa_results("Empty Geometries", pg_cur)

    pg_cur.execute(psma.open_sql_file("07-qa-display-localities.sql", settings))
    display_qa_results("Dropped Localities", pg_cur)

    logger.info("\t- Step 7 of 7 : display localities qa'd : {0}".format(datetime.now() - start_time))

def display_qa_results(purpose, pg_cur):
    logger.info("\t\t" + purpose)
    logger.info("\t\t----------------------------------------")

    results = pg_cur.fetchall()

    if results is not None:
        # print the column names returned
        logger.info("\t\t" + ",".join([desc[0] for desc in pg_cur.description]))

        for result in results:
            logger.info("\t\t" + ",".join(map(str, result)))
    else:
        logger.info("\t\t" + "No records")

    logger.info("\t\t----------------------------------------")

if __name__ == '__main__':
    logger = logging.getLogger()

    # set logger
    log_file = os.path.abspath(__file__).replace(".py", ".log")
    logging.basicConfig(filename=log_file, level=logging.DEBUG, format="%(asctime)s %(message)s",
                        datefmt="%m/%d/%Y %I:%M:%S %p")

    # setup logger to write to screen as well as writing to log file
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger('').addHandler(console)

    logger.info("")
    logger.info("Start locality-clean")
    psma.check_python_version(logger)

    if main():
        logger.info("Finished successfully!")
    else:
        logger.fatal("Something bad happened!")

    logger.info("")
    logger.info("-------------------------------------------------------------------------------")
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Template file used by the OPF Experiment Generator to generate the actual
description.py file by replacing $XXXXXXXX tokens with desired values.

This description.py file was generated by:
'/Users/ronmarianetti/nta/eng/lib/python2.6/site-packages/nupic/frameworks/opf/expGenerator/ExpGenerator.py'
"""

from nupic.frameworks.opf.expdescriptionapi import ExperimentDescriptionAPI

from nupic.frameworks.opf.expdescriptionhelpers import (
  updateConfigFromSubConfig,
  applyValueGettersToContainer,
  DeferredDictLookup)

from nupic.frameworks.opf.clamodelcallbacks import *
from nupic.frameworks.opf.metrics import MetricSpec
from nupic.frameworks.opf.opfutils import (InferenceType,
                                           InferenceElement)
from nupic.support import aggregationDivide

from nupic.frameworks.opf.opftaskdriver import (
                                            IterationPhaseSpecLearnOnly,
                                            IterationPhaseSpecInferOnly,
                                            IterationPhaseSpecLearnAndInfer)

# Model Configuration Dictionary:
#
# Define the model parameters and adjust for any modifications if imported
# from a sub-experiment.
#
# These fields might be modified by a sub-experiment; this dict is passed
# between the sub-experiment and base experiment
#
#
# NOTE: Use of DEFERRED VALUE-GETTERs: dictionary fields and list elements
#   within the config dictionary may be assigned futures derived from the
#   ValueGetterBase class, such as DeferredDictLookup.
#   This facility is particularly handy for enabling substitution of values in
#   the config dictionary from other values in the config dictionary, which is
#   needed by permutation.py-based experiments. These values will be resolved
#   during the call to applyValueGettersToContainer(),
#   which we call after the base experiment's config dictionary is updated from
#   the sub-experiment. See ValueGetterBase and
#   DeferredDictLookup for more details about value-getters.
#
#   For each custom encoder parameter to be exposed to the sub-experiment/
#   permutation overrides, define a variable in this section, using key names
#   beginning with a single underscore character to avoid collisions with
#   pre-defined keys (e.g., _dsEncoderFieldName2_N).
#
#   Example:
#      config = dict(
#        _dsEncoderFieldName2_N = 70,
#        _dsEncoderFieldName2_W = 5,
#        dsEncoderSchema = [
#          base=dict(
#            fieldname='Name2', type='ScalarEncoder',
#            name='Name2', minval=0, maxval=270, clipInput=True,
#            n=DeferredDictLookup('_dsEncoderFieldName2_N'),
#            w=DeferredDictLookup('_dsEncoderFieldName2_W')),
#        ],
#      )
#      updateConfigFromSubConfig(config)
#      applyValueGettersToContainer(config)
config = {
    # Type of model that the rest of these parameters apply to.
    'model': "CLA",

    # Version that specifies the format of the config.
    'version': 1,

    # Intermediate variables used to compute fields in modelParams and also
    # referenced from the control section.
    'aggregationInfo': {   'days': 0,
        'fields': [   (u'timestamp', 'first'),
                      (u'gym', 'first'),
                      (u'consumption', 'mean'),
                      (u'address', 'first')],
        'hours': 0,
        'microseconds': 0,
        'milliseconds': 0,
        'minutes': 0,
        'months': 0,
        'seconds': 0,
        'weeks': 0,
        'years': 0},

    'predictAheadTime': None,

    # Model parameter dictionary.
    'modelParams': {
        # The type of inference that this model will perform
        'inferenceType': 'TemporalNextStep',

        'sensorParams': {
            # Sensor diagnostic output verbosity control;
            # if > 0: sensor region will print out on screen what it's sensing
            # at each step 0: silent; >=1: some info; >=2: more info;
            # >=3: even more info (see compute() in py/regions/RecordSensor.py)
            'verbosity' : 0,

            # Example:
            #     dsEncoderSchema = [
            #       DeferredDictLookup('__field_name_encoder'),
            #     ],
            #
            # (value generated from DS_ENCODER_SCHEMA)
            'encoders': {
                'address': {   'fieldname': u'address',
                               'n': 300,
                               'name': u'address',
                               'type': 'SDRCategoryEncoder',
                               'w': 21},
                'consumption': {   'clipInput': True,
                                   'fieldname': u'consumption',
                                   'maxval': 200,
                                   'minval': 0,
                                   'n': 1500,
                                   'name': u'consumption',
                                   'type': 'ScalarEncoder',
                                   'w': 21},
                'gym': {   'fieldname': u'gym',
                           'n': 300,
                           'name': u'gym',
                           'type': 'SDRCategoryEncoder',
                           'w': 7},
                'timestamp_dayOfWeek': {   'dayOfWeek': (7, 3),
                                           'fieldname': u'timestamp',
                                           'name': u'timestamp_dayOfWeek',
                                           'type': 'DateEncoder'},
                'timestamp_timeOfDay': {   'fieldname': u'timestamp',
                                           'name': u'timestamp_timeOfDay',
                                           'timeOfDay': (7, 8),
                                           'type': 'DateEncoder'}},

            # A dictionary specifying the period for automatically-generated
            # resets from a RecordSensor;
            #
            # None = disable automatically-generated resets (also disabled if
            # all of the specified values evaluate to 0).
            # Valid keys is the desired combination of the following:
            #   days, hours, minutes, seconds, milliseconds, microseconds, weeks
            #
            # Example for 1.5 days: sensorAutoReset = dict(days=1,hours=12),
            #
            # (value generated from SENSOR_AUTO_RESET)
            'sensorAutoReset' : None,
        },

        'spEnable': True,

        'spParams': {
            # SP diagnostic output verbosity control;
            # 0: silent; >=1: some info; >=2: more info;
            'spVerbosity' : 0,

            'globalInhibition': 1,

            # Number of cell columns in the cortical region (same number for
            # SP and TP)
            # (see also tpNCellsPerCol)
            'columnCount': 2048,

            'inputWidth': 0,

            # SP inhibition control (absolute value);
            # Maximum number of active columns in the SP region's output (when
            # there are more, the weaker ones are suppressed)
            'numActivePerInhArea': 40,

            'seed': 1956,

            # coincInputPoolPct
            # What percent of the columns's receptive field is available
            # for potential synapses. At initialization time, we will
            # choose coincInputPoolPct * (2*coincInputRadius+1)^2
            'coincInputPoolPct': 0.5,

            # The default connected threshold. Any synapse whose
            # permanence value is above the connected threshold is
            # a "connected synapse", meaning it can contribute to the
            # cell's firing. Typical value is 0.10. Cells whose activity
            # level before inhibition falls below minDutyCycleBeforeInh
            # will have their own internal synPermConnectedCell
            # threshold set below this default value.
            # (This concept applies to both SP and TP and so 'cells'
            # is correct here as opposed to 'columns')
            'synPermConnected': 0.1,

            'synPermActiveInc': 0.1,

            'synPermInactiveDec': 0.01,
        },

        # Controls whether TP is enabled or disabled;
        # TP is necessary for making temporal predictions, such as predicting
        # the next inputs.  Without TP, the model is only capable of
        # reconstructing missing sensor inputs (via SP).
        'tpEnable' : True,

        'tpParams': {
            # TP diagnostic output verbosity control;
            # 0: silent; [1..6]: increasing levels of verbosity
            # (see verbosity in nta/trunk/py/nupic/research/TP.py and TP10X*.py)
            'verbosity': 0,

            # Number of cell columns in the cortical region (same number for
            # SP and TP)
            # (see also tpNCellsPerCol)
            'columnCount': 2048,

            # The number of cells (i.e., states), allocated per column.
            'cellsPerColumn': 32,

            'inputWidth': 2048,

            'seed': 1960,

            # Temporal Pooler implementation selector (see _getTPClass in
            # CLARegion.py).
            'temporalImp': 'cpp',

            # New Synapse formation count
            # NOTE: If None, use spNumActivePerInhArea
            #
            # TODO: need better explanation
            'newSynapseCount': 15,

            # Maximum number of synapses per segment
            #  > 0 for fixed-size CLA
            # -1 for non-fixed-size CLA
            #
            # TODO: for Ron: once the appropriate value is placed in TP
            # constructor, see if we should eliminate this parameter from
            # description.py.
            'maxSynapsesPerSegment': 32,

            # Maximum number of segments per cell
            #  > 0 for fixed-size CLA
            # -1 for non-fixed-size CLA
            #
            # TODO: for Ron: once the appropriate value is placed in TP
            # constructor, see if we should eliminate this parameter from
            # description.py.
            'maxSegmentsPerCell': 128,

            # Initial Permanence
            # TODO: need better explanation
            'initialPerm': 0.21,

            # Permanence Increment
            'permanenceInc': 0.1,

            # Permanence Decrement
            # If set to None, will automatically default to tpPermanenceInc
            # value.
            'permanenceDec' : 0.1,

            'globalDecay': 0.0,

            'maxAge': 0,

            # Minimum number of active synapses for a segment to be considered
            # during search for the best-matching segments.
            # None=use default
            # Replaces: tpMinThreshold
            'minThreshold': 12,

            # Segment activation threshold.
            # A segment is active if it has >= tpSegmentActivationThreshold
            # connected synapses that are active due to infActiveState
            # None=use default
            # Replaces: tpActivationThreshold
            'activationThreshold': 16,

            'outputType': 'normal',

            # "Pay Attention Mode" length. This tells the TP how many new
            # elements to append to the end of a learned sequence at a time.
            # Smaller values are better for datasets with short sequences,
            # higher values are better for datasets with long sequences.
            'pamLength': 1,
        },

        'clParams': {
            'regionName' : 'CLAClassifierRegion',

            # Classifier diagnostic output verbosity control;
            # 0: silent; [1..6]: increasing levels of verbosity
            'clVerbosity' : 0,

            # This controls how fast the classifier learns/forgets. Higher values
            # make it adapt faster and forget older patterns faster.
            'alpha': 0.001,

            # This is set after the call to updateConfigFromSubConfig and is
            # computed from the aggregationInfo and predictAheadTime.
            'steps': '1',

        },

        'trainSPNetOnlyIfRequested': False,
    },

}
# end of config dictionary

# Adjust base config dictionary for any modifications if imported from a
# sub-experiment
updateConfigFromSubConfig(config)

# Compute predictionSteps based on the predictAheadTime and the aggregation
# period, which may be permuted over.
if config['predictAheadTime'] is not None:
  predictionSteps = int(round(aggregationDivide(
      config['predictAheadTime'], config['aggregationInfo'])))
  assert (predictionSteps >= 1)
  config['modelParams']['clParams']['steps'] = str(predictionSteps)

# Adjust config by applying ValueGetterBase-derived
# futures. NOTE: this MUST be called after updateConfigFromSubConfig() in order
# to support value-getter-based substitutions from the sub-experiment (if any)
applyValueGettersToContainer(config)

################################################################################
control = {
  # The environment that the current model is being run in
  "environment": 'grok',

  # Input stream specification per py/grokengine/cluster/database/StreamDef.json.
  #
  'dataset' :   {u'info': u'test_NoProviders',
      u'streams': [   {   u'columns': [u'*'],
                          u'info': "test data",
                          u'source': "file://test_data.csv"}],
      u'version': 1},

  # Iteration count: maximum number of iterations.  Each iteration corresponds
  # to one record from the (possibly aggregated) dataset.  The task is
  # terminated when either number of iterations reaches iterationCount or
  # all records in the (possibly aggregated) database have been processed,
  # whichever occurs first.
  #
  # iterationCount of -1 = iterate over the entire dataset
  #'iterationCount' : ITERATION_COUNT,

  # Metrics: A list of MetricSpecs that instantiate the metrics that are
  # computed for this experiment
  'metrics':[
    MetricSpec(field=u'consumption',inferenceElement=InferenceElement.prediction,
               metric='rmse'),
  ],

  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
}

################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
<commit_msg>fix swarming unittest (w=21)<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Template file used by the OPF Experiment Generator to generate the actual
description.py file by replacing $XXXXXXXX tokens with desired values.

This description.py file was generated by:
'/Users/ronmarianetti/nta/eng/lib/python2.6/site-packages/nupic/frameworks/opf/expGenerator/ExpGenerator.py'
"""

from nupic.frameworks.opf.expdescriptionapi import ExperimentDescriptionAPI

from nupic.frameworks.opf.expdescriptionhelpers import (
  updateConfigFromSubConfig,
  applyValueGettersToContainer,
  DeferredDictLookup)

from nupic.frameworks.opf.clamodelcallbacks import *
from nupic.frameworks.opf.metrics import MetricSpec
from nupic.frameworks.opf.opfutils import (InferenceType,
                                           InferenceElement)
from nupic.support import aggregationDivide

from nupic.frameworks.opf.opftaskdriver import (
                                            IterationPhaseSpecLearnOnly,
                                            IterationPhaseSpecInferOnly,
                                            IterationPhaseSpecLearnAndInfer)

# Model Configuration Dictionary:
#
# Define the model parameters and adjust for any modifications if imported
# from a sub-experiment.
#
# These fields might be modified by a sub-experiment; this dict is passed
# between the sub-experiment and base experiment
#
#
# NOTE: Use of DEFERRED VALUE-GETTERs: dictionary fields and list elements
#   within the config dictionary may be assigned futures derived from the
#   ValueGetterBase class, such as DeferredDictLookup.
#   This facility is particularly handy for enabling substitution of values in
#   the config dictionary from other values in the config dictionary, which is
#   needed by permutation.py-based experiments. These values will be resolved
#   during the call to applyValueGettersToContainer(),
#   which we call after the base experiment's config dictionary is updated from
#   the sub-experiment. See ValueGetterBase and
#   DeferredDictLookup for more details about value-getters.
#
#   For each custom encoder parameter to be exposed to the sub-experiment/
#   permutation overrides, define a variable in this section, using key names
#   beginning with a single underscore character to avoid collisions with
#   pre-defined keys (e.g., _dsEncoderFieldName2_N).
#
#   Example:
#      config = dict(
#        _dsEncoderFieldName2_N = 70,
#        _dsEncoderFieldName2_W = 5,
#        dsEncoderSchema = [
#          base=dict(
#            fieldname='Name2', type='ScalarEncoder',
#            name='Name2', minval=0, maxval=270, clipInput=True,
#            n=DeferredDictLookup('_dsEncoderFieldName2_N'),
#            w=DeferredDictLookup('_dsEncoderFieldName2_W')),
#        ],
#      )
#      updateConfigFromSubConfig(config)
#      applyValueGettersToContainer(config)
config = {
    # Type of model that the rest of these parameters apply to.
    'model': "CLA",

    # Version that specifies the format of the config.
    'version': 1,

    # Intermediate variables used to compute fields in modelParams and also
    # referenced from the control section.
    'aggregationInfo': {   'days': 0,
        'fields': [   (u'timestamp', 'first'),
                      (u'gym', 'first'),
                      (u'consumption', 'mean'),
                      (u'address', 'first')],
        'hours': 0,
        'microseconds': 0,
        'milliseconds': 0,
        'minutes': 0,
        'months': 0,
        'seconds': 0,
        'weeks': 0,
        'years': 0},

    'predictAheadTime': None,

    # Model parameter dictionary.
    'modelParams': {
        # The type of inference that this model will perform
        'inferenceType': 'TemporalNextStep',

        'sensorParams': {
            # Sensor diagnostic output verbosity control;
            # if > 0: sensor region will print out on screen what it's sensing
            # at each step 0: silent; >=1: some info; >=2: more info;
            # >=3: even more info (see compute() in py/regions/RecordSensor.py)
            'verbosity' : 0,

            # Example:
            #     dsEncoderSchema = [
            #       DeferredDictLookup('__field_name_encoder'),
            #     ],
            #
            # (value generated from DS_ENCODER_SCHEMA)
            'encoders': {
                'address': {   'fieldname': u'address',
                               'n': 300,
                               'name': u'address',
                               'type': 'SDRCategoryEncoder',
                               'w': 21},
                'consumption': {   'clipInput': True,
                                   'fieldname': u'consumption',
                                   'maxval': 200,
                                   'minval': 0,
                                   'n': 1500,
                                   'name': u'consumption',
                                   'type': 'ScalarEncoder',
                                   'w': 21},
                'gym': {   'fieldname': u'gym',
                           'n': 600,
                           'name': u'gym',
                           'type': 'SDRCategoryEncoder',
                           'w': 21},
                'timestamp_dayOfWeek': {   'dayOfWeek': (7, 3),
                                           'fieldname': u'timestamp',
                                           'name': u'timestamp_dayOfWeek',
                                           'type': 'DateEncoder'},
                'timestamp_timeOfDay': {   'fieldname': u'timestamp',
                                           'name': u'timestamp_timeOfDay',
                                           'timeOfDay': (7, 8),
                                           'type': 'DateEncoder'}},

            # A dictionary specifying the period for automatically-generated
            # resets from a RecordSensor;
            #
            # None = disable automatically-generated resets (also disabled if
            # all of the specified values evaluate to 0).
            # Valid keys is the desired combination of the following:
            #   days, hours, minutes, seconds, milliseconds, microseconds, weeks
            #
            # Example for 1.5 days: sensorAutoReset = dict(days=1,hours=12),
            #
            # (value generated from SENSOR_AUTO_RESET)
            'sensorAutoReset' : None,
        },

        'spEnable': True,

        'spParams': {
            # SP diagnostic output verbosity control;
            # 0: silent; >=1: some info; >=2: more info;
            'spVerbosity' : 0,

            'globalInhibition': 1,

            # Number of cell columns in the cortical region (same number for
            # SP and TP)
            # (see also tpNCellsPerCol)
            'columnCount': 2048,

            'inputWidth': 0,

            # SP inhibition control (absolute value);
            # Maximum number of active columns in the SP region's output (when
            # there are more, the weaker ones are suppressed)
            'numActivePerInhArea': 40,

            'seed': 1956,

            # coincInputPoolPct
            # What percent of the columns's receptive field is available
            # for potential synapses. At initialization time, we will
            # choose coincInputPoolPct * (2*coincInputRadius+1)^2
            'coincInputPoolPct': 0.5,

            # The default connected threshold. Any synapse whose
            # permanence value is above the connected threshold is
            # a "connected synapse", meaning it can contribute to the
            # cell's firing. Typical value is 0.10. Cells whose activity
            # level before inhibition falls below minDutyCycleBeforeInh
            # will have their own internal synPermConnectedCell
            # threshold set below this default value.
            # (This concept applies to both SP and TP and so 'cells'
            # is correct here as opposed to 'columns')
            'synPermConnected': 0.1,

            'synPermActiveInc': 0.1,

            'synPermInactiveDec': 0.01,
        },

        # Controls whether TP is enabled or disabled;
        # TP is necessary for making temporal predictions, such as predicting
        # the next inputs.  Without TP, the model is only capable of
        # reconstructing missing sensor inputs (via SP).
        'tpEnable' : True,

        'tpParams': {
            # TP diagnostic output verbosity control;
            # 0: silent; [1..6]: increasing levels of verbosity
            # (see verbosity in nta/trunk/py/nupic/research/TP.py and TP10X*.py)
            'verbosity': 0,

            # Number of cell columns in the cortical region (same number for
            # SP and TP)
            # (see also tpNCellsPerCol)
            'columnCount': 2048,

            # The number of cells (i.e., states), allocated per column.
            'cellsPerColumn': 32,

            'inputWidth': 2048,

            'seed': 1960,

            # Temporal Pooler implementation selector (see _getTPClass in
            # CLARegion.py).
            'temporalImp': 'cpp',

            # New Synapse formation count
            # NOTE: If None, use spNumActivePerInhArea
            #
            # TODO: need better explanation
            'newSynapseCount': 15,

            # Maximum number of synapses per segment
            #  > 0 for fixed-size CLA
            # -1 for non-fixed-size CLA
            #
            # TODO: for Ron: once the appropriate value is placed in TP
            # constructor, see if we should eliminate this parameter from
            # description.py.
            'maxSynapsesPerSegment': 32,

            # Maximum number of segments per cell
            #  > 0 for fixed-size CLA
            # -1 for non-fixed-size CLA
            #
            # TODO: for Ron: once the appropriate value is placed in TP
            # constructor, see if we should eliminate this parameter from
            # description.py.
            'maxSegmentsPerCell': 128,

            # Initial Permanence
            # TODO: need better explanation
            'initialPerm': 0.21,

            # Permanence Increment
            'permanenceInc': 0.1,

            # Permanence Decrement
            # If set to None, will automatically default to tpPermanenceInc
            # value.
            'permanenceDec' : 0.1,

            'globalDecay': 0.0,

            'maxAge': 0,

            # Minimum number of active synapses for a segment to be considered
            # during search for the best-matching segments.
            # None=use default
            # Replaces: tpMinThreshold
            'minThreshold': 12,

            # Segment activation threshold.
            # A segment is active if it has >= tpSegmentActivationThreshold
            # connected synapses that are active due to infActiveState
            # None=use default
            # Replaces: tpActivationThreshold
            'activationThreshold': 16,

            'outputType': 'normal',

            # "Pay Attention Mode" length. This tells the TP how many new
            # elements to append to the end of a learned sequence at a time.
            # Smaller values are better for datasets with short sequences,
            # higher values are better for datasets with long sequences.
            'pamLength': 1,
        },

        'clParams': {
            'regionName' : 'CLAClassifierRegion',

            # Classifier diagnostic output verbosity control;
            # 0: silent; [1..6]: increasing levels of verbosity
            'clVerbosity' : 0,

            # This controls how fast the classifier learns/forgets. Higher values
            # make it adapt faster and forget older patterns faster.
            'alpha': 0.001,

            # This is set after the call to updateConfigFromSubConfig and is
            # computed from the aggregationInfo and predictAheadTime.
            'steps': '1',

        },

        'trainSPNetOnlyIfRequested': False,
    },

}
# end of config dictionary

# Adjust base config dictionary for any modifications if imported from a
# sub-experiment
updateConfigFromSubConfig(config)

# Compute predictionSteps based on the predictAheadTime and the aggregation
# period, which may be permuted over.
if config['predictAheadTime'] is not None:
  predictionSteps = int(round(aggregationDivide(
      config['predictAheadTime'], config['aggregationInfo'])))
  assert (predictionSteps >= 1)
  config['modelParams']['clParams']['steps'] = str(predictionSteps)

# Adjust config by applying ValueGetterBase-derived
# futures. NOTE: this MUST be called after updateConfigFromSubConfig() in order
# to support value-getter-based substitutions from the sub-experiment (if any)
applyValueGettersToContainer(config)

################################################################################
control = {
  # The environment that the current model is being run in
  "environment": 'grok',

  # Input stream specification per py/grokengine/cluster/database/StreamDef.json.
  #
  'dataset' :   {u'info': u'test_NoProviders',
      u'streams': [   {   u'columns': [u'*'],
                          u'info': "test data",
                          u'source': "file://test_data.csv"}],
      u'version': 1},

  # Iteration count: maximum number of iterations.  Each iteration corresponds
  # to one record from the (possibly aggregated) dataset.  The task is
  # terminated when either number of iterations reaches iterationCount or
  # all records in the (possibly aggregated) database have been processed,
  # whichever occurs first.
  #
  # iterationCount of -1 = iterate over the entire dataset
  #'iterationCount' : ITERATION_COUNT,

  # Metrics: A list of MetricSpecs that instantiate the metrics that are
  # computed for this experiment
  'metrics':[
    MetricSpec(field=u'consumption',inferenceElement=InferenceElement.prediction,
               metric='rmse'),
  ],

  # Logged Metrics: A sequence of regular expressions that specify which of
  # the metrics from the Inference Specifications section MUST be logged for
  # every prediction. The regex's correspond to the automatically generated
  # metric labels. This is similar to the way the optimization metric is
  # specified in permutations.py.
}

################################################################################
################################################################################
descriptionInterface = ExperimentDescriptionAPI(modelConfig=config,
                                                control=control)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import networkx as nx
import itertools
from db_connect import db_

g_ = nx.DiGraph( )
venues_ = set( )
slots_ = set( )
courses_ = set( )
entries_ = [ ]
max_weight_ = 0

def init( ):
    global g_
    global entries_
    global max_weight_
    cur = db_.cursor( dictionary = True )

    g_.add_node( 'SOURCE' )
    g_.add_node( 'SINK' )

    cur.execute( "SELECT * FROM upcoming_course_schedule WHERE status='VALID'" )
    for row in cur:
        entries_.append( row )
        slots_.add( row[ 'slot' ] )
        venues_.add( row[ 'venue' ] )
        courses_.add( row[ 'course_id' ] )
        if int( row[ 'weight'] ) > max_weight_:
            max_weight_ = int( row[ 'weight' ]) + 1

    # SOURCE to slot.
    for slot, venue in itertools.product( slots_, venues_ ):
        print( 'Slot %s and venue %s' % (slot, venue ) )
        key = slot + ':' + venue
        g_.add_node( key )
        g_.add_edge( 'SOURCE', key, capacity = 1 )

    #  To sink
    for c in courses_:
        print( '[INFO] Course is %s' % c )
        g_.add_node( c )
        g_.add_edge( c, 'SINK', capacity = 1 )

    # Now SLOT to COURSE
    for e in entries_:
        venue, slot = e['venue'], e['slot']
        key = slot + ':' + venue
        g_.add_edge( key, e[ 'course_id' ], capacity = 100
                , weight = max_weight_ - int( e['weight'] )
                )

    print( '[INFO] Max Weight is %d' % max_weight_ )
    assignments = { }
    solution = nx.min_cost_flow( g_, 'SOURCE', 'SINK' )
    for k in solution:
        if k in [ 'SOURCE', 'SINK' ]:
            continue
        vs = solution[ k ]
        for v in vs:
            if v == 'SINK':
                continue
            assignments[ v ] = k.split( ':' )

    # Remove all previous assignments.
    cur.execute( """
        UPDATE upcoming_course_schedule
        SET alloted_slot=NULL AND alloted_venue=NULL
        """ )
    db_.commit( )

    for k in assignments:
        slot, venue =  assignments[k]
        query = """
                UPDATE upcoming_course_schedule
            SET alloted_slot='{s}',alloted_venue='{v}'
                WHERE
            course_id='{c}' AND slot='{s}'
                AND venue='{v}'""".format(s=slot, v=venue, c=k)
        cur.execute( query )
    db_.commit( )

def main( ):
    init( )

if __name__ == '__main__':
    main()
<commit_msg>python2.7.<commit_after>#!/usr/bin/env python2.7

import networkx as nx
import itertools
from db_connect import db_

g_ = nx.DiGraph( )
venues_ = set( )
slots_ = set( )
courses_ = set( )
entries_ = [ ]
max_weight_ = 0

def init( ):
    global g_
    global entries_
    global max_weight_
    cur = db_.cursor( dictionary = True )

    g_.add_node( 'SOURCE' )
    g_.add_node( 'SINK' )

    cur.execute( "SELECT * FROM upcoming_course_schedule WHERE status='VALID'" )
    for row in cur:
        entries_.append( row )
        slots_.add( row[ 'slot' ] )
        venues_.add( row[ 'venue' ] )
        courses_.add( row[ 'course_id' ] )
        if int( row[ 'weight'] ) > max_weight_:
            max_weight_ = int( row[ 'weight' ]) + 1

    # SOURCE to slot.
    for slot, venue in itertools.product( slots_, venues_ ):
        print( 'Slot %s and venue %s' % (slot, venue ) )
        key = slot + ':' + venue
        g_.add_node( key )
        g_.add_edge( 'SOURCE', key, capacity = 1 )

    #  To sink
    for c in courses_:
        print( '[INFO] Course is %s' % c )
        g_.add_node( c )
        g_.add_edge( c, 'SINK', capacity = 1 )

    # Now SLOT to COURSE
    for e in entries_:
        venue, slot = e['venue'], e['slot']
        key = slot + ':' + venue
        g_.add_edge( key, e[ 'course_id' ], capacity = 100
                , weight = max_weight_ - int( e['weight'] )
                )

    print( '[INFO] Max Weight is %d' % max_weight_ )
    assignments = { }
    solution = nx.min_cost_flow( g_, 'SOURCE', 'SINK' )
    for k in solution:
        if k in [ 'SOURCE', 'SINK' ]:
            continue
        vs = solution[ k ]
        for v in vs:
            if v == 'SINK':
                continue
            assignments[ v ] = k.split( ':' )

    # Remove all previous assignments.
    cur.execute( """
        UPDATE upcoming_course_schedule
        SET alloted_slot=NULL AND alloted_venue=NULL
        """ )
    db_.commit( )

    for k in assignments:
        slot, venue =  assignments[k]
        query = """
                UPDATE upcoming_course_schedule
            SET alloted_slot='{s}',alloted_venue='{v}'
                WHERE
            course_id='{c}' AND slot='{s}'
                AND venue='{v}'""".format(s=slot, v=venue, c=k)
        cur.execute( query )
    db_.commit( )

def main( ):
    init( )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
    Imports our modified version of coverage.py which has support for the COVERAGE_DIR
    directive.  If the COVERAGE_DIR environment variable is set, it will automatically
    start running coverage.  Forks will continue to run coverage, to a different pid.
"""
import atexit
import imp
import os
import signal
import sys

from conary.lib import util

def install():
    """
        Starts the coverage tool if the coverage dir directive is set.
    """
    if os.environ.get('COVERAGE_DIR', None):
        _install()

def saveState(signal, f):
    sys.modules['coverage'].the_coverage.save()
    sys.exit(0)

def _install():
    coverageLoc = os.environ.get('COVERAGE_TOOL', None)
    if not coverageLoc:
        raise RuntimeError, 'cannot find coverage.py!'

    coverageDir = os.environ.get('COVERAGE_DIR', None)
    if not coverageDir:
        raise RuntimeError, 'COVERAGE_DIR must be set to a path for cache file'
    util.mkdirChain(coverageDir)

    coverage = imp.load_source('coverage', coverageLoc)
    the_coverage = coverage.the_coverage
    if hasattr(the_coverage, 'pid') and the_coverage.pid == os.getpid():
        return
    elif hasattr(the_coverage, 'pid'):
        _reset(coverage)

    _installOsWrapper()
    _run(coverage)
    return

def _run(coverage):
    signal.signal(signal.SIGQUIT, saveState)
    atexit.register(coverage.the_coverage.save)
    coverage.the_coverage.get_ready()
    sys.settrace(coverage.t)

origOsFork = os.fork
origOsExit = os._exit
def _installOsWrapper():
    """
        wrap fork to automatically start a new coverage
        file with the forked pid.
    """
    global origOsFork
    global origOsExit
    def fork_wrapper():
        pid = origOsFork()
        if pid:
            return pid
        else:
            _reset(sys.modules['coverage'])
            _run(sys.modules['coverage'])
            return 0

    def exit_wrapper(*args):
        sys.modules['coverage'].the_coverage.save()
        os._exit(*args)

    if os.fork is origOsFork:
        os.fork = fork_wrapper

    if os._exit is origOsExit:
        os._exit = exit_wrapper

def _reset(coverage):
    sys.settrace(None)
    coverage.c.clear()
    coverage.the_coverage = None
    coverage.the_coverage = coverage.coverage()

install()
<commit_msg>don't use the wrapper in the wrapper!<commit_after># Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
    Imports our modified version of coverage.py which has support for the COVERAGE_DIR
    directive.  If the COVERAGE_DIR environment variable is set, it will automatically
    start running coverage.  Forks will continue to run coverage, to a different pid.
"""
import atexit
import imp
import os
import signal
import sys

from conary.lib import util

def install():
    """
        Starts the coverage tool if the coverage dir directive is set.
    """
    if os.environ.get('COVERAGE_DIR', None):
        _install()

def saveState(signal, f):
    sys.modules['coverage'].the_coverage.save()
    sys.exit(0)

def _install():
    coverageLoc = os.environ.get('COVERAGE_TOOL', None)
    if not coverageLoc:
        raise RuntimeError, 'cannot find coverage.py!'

    coverageDir = os.environ.get('COVERAGE_DIR', None)
    if not coverageDir:
        raise RuntimeError, 'COVERAGE_DIR must be set to a path for cache file'
    util.mkdirChain(coverageDir)

    coverage = imp.load_source('coverage', coverageLoc)
    the_coverage = coverage.the_coverage
    if hasattr(the_coverage, 'pid') and the_coverage.pid == os.getpid():
        return
    elif hasattr(the_coverage, 'pid'):
        _reset(coverage)

    _installOsWrapper()
    _run(coverage)
    return

def _run(coverage):
    signal.signal(signal.SIGQUIT, saveState)
    atexit.register(coverage.the_coverage.save)
    coverage.the_coverage.get_ready()
    sys.settrace(coverage.t)

origOsFork = os.fork
origOsExit = os._exit
def _installOsWrapper():
    """
        wrap fork to automatically start a new coverage
        file with the forked pid.
    """
    global origOsFork
    global origOsExit
    def fork_wrapper():
        pid = origOsFork()
        if pid:
            return pid
        else:
            _reset(sys.modules['coverage'])
            _run(sys.modules['coverage'])
            return 0

    def exit_wrapper(*args):
        sys.modules['coverage'].the_coverage.save()
        origOsExit(*args)

    if os.fork is origOsFork:
        os.fork = fork_wrapper

    if os._exit is origOsExit:
        os._exit = exit_wrapper

def _reset(coverage):
    sys.settrace(None)
    coverage.c.clear()
    coverage.the_coverage = None
    coverage.the_coverage = coverage.coverage()

install()
<|endoftext|>


<commit_before><commit_msg>Undefined name: import sys for line 91<commit_after><|endoftext|>