<commit_before># Copyright 2022 The KerasCV Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Title: Training a KerasCV model for Imagenet Classification
Author: [ianjjohnson](https://github.com/ianjjohnson)
Date created: 2022/07/25
Last modified: 2022/07/25
Description: Use KerasCV to train an image classifier using modern best practices
"""

import sys

import tensorflow as tf
from absl import flags
from tensorflow.keras import callbacks
from tensorflow.keras import losses
from tensorflow.keras import metrics
from tensorflow.keras import optimizers

import keras_cv
from keras_cv import models
from keras_cv.datasets import imagenet

"""
## Overview
KerasCV makes training state-of-the-art classification models easy by providing implementations of modern models, preprocessing techniques, and layers.
In this tutorial, we walk through training a model against the Imagenet dataset using Keras and KerasCV.
This tutorial requires you to have KerasCV installed:
```shell
pip install keras-cv
```
"""

"""
## Setup and flags

"""

flags.DEFINE_string(
    "model_name", None, "The name of the model in KerasCV.models to use."
)
flags.DEFINE_string("imagenet_path", None, "Directory from which to load Imagenet.")
flags.DEFINE_string(
    "backup_path", None, "Directory which will be used for training backups."
)
flags.DEFINE_string(
    "weights_path", None, "Directory which will be used to store weight checkpoints."
)
flags.DEFINE_string(
    "tensorboard_path", None, "Directory which will be used to store tensorboard logs."
)
flags.DEFINE_integer("batch_size", 256, "Batch size for training and evaluation.")
flags.DEFINE_boolean(
    "use_xla", True, "Whether or not to use XLA (jit_compile) for training."
)
flags.DEFINE_float(
    "initial_learning_rate",
    0.1,
    "Initial learning rate which will reduce on plateau.",
)
flags.DEFINE_string(
    "model_kwargs",
    "{}",
    "Keyword argument dictionary to pass to the constructor of the model being trained",
)

FLAGS = flags.FLAGS
FLAGS(sys.argv)

if FLAGS.model_name not in models.__dict__:
    raise ValueError(f"Invalid model name: {FLAGS.model_name}")

CLASSES = 1000
IMAGE_SIZE = (224, 224)
EPOCHS = 250

"""
## Data loading
This guide uses the
[Imagenet dataset](https://www.tensorflow.org/datasets/catalog/imagenet2012).
Note that this requires manual download and preprocessing. You can find more
information about preparing this dataset at keras_cv/datasets/imagenet/README.md
"""

train_ds = imagenet.load(
    split="train",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=FLAGS.batch_size,
    img_size=IMAGE_SIZE,
)
test_ds = imagenet.load(
    split="validation",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=FLAGS.batch_size,
    img_size=IMAGE_SIZE,
)

"""
Next, we augment our dataset.
We define a set of augmentation layers and then apply them to our input dataset.
"""

AUGMENT_LAYERS = [
    keras_cv.layers.RandomFlip(),
    keras_cv.layers.RandAugment(value_range=(0, 255), magnitude=0.3),
    keras_cv.layers.CutMix(),
]

@tf.function
def augment(img, label):
    inputs = {"images": img, "labels": label}
    for layer in AUGMENT_LAYERS:
        inputs = layer(inputs)
    return inputs["images"], inputs["labels"]

train_ds = train_ds.map(augment, num_parallel_calls=tf.data.AUTOTUNE).prefetch(
    tf.data.AUTOTUNE
)
test_ds = test_ds.prefetch(tf.data.AUTOTUNE)

"""
Now we can begin training our model. We begin by loading a model from KerasCV.
Note that we also specify a distribution strategy while creating the model.
Different distribution strategies may be used for different training hardware, as indicated below.
"""

# For TPU training, use tf.distribute.TPUStrategy()
# MirroredStrategy is best for a single machine with multiple GPUs
strategy = tf.distribute.MirroredStrategy()

with strategy.scope():
    model = models.__dict__[FLAGS.model_name]
    model = model(
        include_rescaling=True,
        include_top=True,
        classes=CLASSES,
        input_shape=IMAGE_SIZE + (3,),
        **eval(FLAGS.model_kwargs),
    )

"""
Next, we pick an optimizer. Here we use Adam with a constant learning rate.
Note that learning rate will decrease over time due to the ReduceLROnPlateau callback.
"""

optimizer = optimizers.SGD(learning_rate=FLAGS.initial_learning_rate, momentum=0.9)

"""
Next, we pick a loss function. We use CategoricalCrossentropy with label smoothing.
"""

loss_fn = losses.CategoricalCrossentropy(label_smoothing=0.1)

"""
Next, we specify the metrics that we want to track. For this example, we track accuracy.
"""

with strategy.scope():
    training_metrics = [metrics.CategoricalAccuracy()]

"""
As a last piece of configuration, we configure callbacks for the method.
We use EarlyStopping, BackupAndRestore, and a model checkpointing callback.
"""

callbacks = [
    callbacks.ReduceLROnPlateau(
        monitor="val_loss", factor=0.1, patience=10, min_delta=0.001, min_lr=0.0001
    ),
    callbacks.EarlyStopping(patience=20),
    callbacks.BackupAndRestore(FLAGS.backup_path),
    callbacks.ModelCheckpoint(FLAGS.weights_path, save_weights_only=True),
    callbacks.TensorBoard(log_dir=FLAGS.tensorboard_path),
]

"""
We can now compile the model and fit it to the training dataset.
"""

model.compile(
    optimizer=optimizer,
    loss=loss_fn,
    metrics=training_metrics,
    jit_compile=FLAGS.use_xla,
)

model.fit(
    train_ds,
    batch_size=FLAGS.batch_size,
    epochs=EPOCHS,
    callbacks=callbacks,
    validation_data=test_ds,
)
<commit_msg>Write steps_per_second in basic_training (#758)<commit_after># Copyright 2022 The KerasCV Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Title: Training a KerasCV model for Imagenet Classification
Author: [ianjjohnson](https://github.com/ianjjohnson)
Date created: 2022/07/25
Last modified: 2022/07/25
Description: Use KerasCV to train an image classifier using modern best practices
"""

import sys

import tensorflow as tf
from absl import flags
from tensorflow.keras import callbacks
from tensorflow.keras import losses
from tensorflow.keras import metrics
from tensorflow.keras import optimizers

import keras_cv
from keras_cv import models
from keras_cv.datasets import imagenet

"""
## Overview
KerasCV makes training state-of-the-art classification models easy by providing implementations of modern models, preprocessing techniques, and layers.
In this tutorial, we walk through training a model against the Imagenet dataset using Keras and KerasCV.
This tutorial requires you to have KerasCV installed:
```shell
pip install keras-cv
```
"""

"""
## Setup and flags

"""

flags.DEFINE_string(
    "model_name", None, "The name of the model in KerasCV.models to use."
)
flags.DEFINE_string("imagenet_path", None, "Directory from which to load Imagenet.")
flags.DEFINE_string(
    "backup_path", None, "Directory which will be used for training backups."
)
flags.DEFINE_string(
    "weights_path", None, "Directory which will be used to store weight checkpoints."
)
flags.DEFINE_string(
    "tensorboard_path", None, "Directory which will be used to store tensorboard logs."
)
flags.DEFINE_integer("batch_size", 256, "Batch size for training and evaluation.")
flags.DEFINE_boolean(
    "use_xla", True, "Whether or not to use XLA (jit_compile) for training."
)
flags.DEFINE_float(
    "initial_learning_rate",
    0.1,
    "Initial learning rate which will reduce on plateau.",
)
flags.DEFINE_string(
    "model_kwargs",
    "{}",
    "Keyword argument dictionary to pass to the constructor of the model being trained",
)

FLAGS = flags.FLAGS
FLAGS(sys.argv)

if FLAGS.model_name not in models.__dict__:
    raise ValueError(f"Invalid model name: {FLAGS.model_name}")

CLASSES = 1000
IMAGE_SIZE = (224, 224)
EPOCHS = 250

"""
## Data loading
This guide uses the
[Imagenet dataset](https://www.tensorflow.org/datasets/catalog/imagenet2012).
Note that this requires manual download and preprocessing. You can find more
information about preparing this dataset at keras_cv/datasets/imagenet/README.md
"""

train_ds = imagenet.load(
    split="train",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=FLAGS.batch_size,
    img_size=IMAGE_SIZE,
)
test_ds = imagenet.load(
    split="validation",
    tfrecord_path=FLAGS.imagenet_path,
    batch_size=FLAGS.batch_size,
    img_size=IMAGE_SIZE,
)

"""
Next, we augment our dataset.
We define a set of augmentation layers and then apply them to our input dataset.
"""

AUGMENT_LAYERS = [
    keras_cv.layers.RandomFlip(),
    keras_cv.layers.RandAugment(value_range=(0, 255), magnitude=0.3),
    keras_cv.layers.CutMix(),
]

@tf.function
def augment(img, label):
    inputs = {"images": img, "labels": label}
    for layer in AUGMENT_LAYERS:
        inputs = layer(inputs)
    return inputs["images"], inputs["labels"]

train_ds = train_ds.map(augment, num_parallel_calls=tf.data.AUTOTUNE).prefetch(
    tf.data.AUTOTUNE
)
test_ds = test_ds.prefetch(tf.data.AUTOTUNE)

"""
Now we can begin training our model. We begin by loading a model from KerasCV.
Note that we also specify a distribution strategy while creating the model.
Different distribution strategies may be used for different training hardware, as indicated below.
"""

# For TPU training, use tf.distribute.TPUStrategy()
# MirroredStrategy is best for a single machine with multiple GPUs
strategy = tf.distribute.MirroredStrategy()

with strategy.scope():
    model = models.__dict__[FLAGS.model_name]
    model = model(
        include_rescaling=True,
        include_top=True,
        classes=CLASSES,
        input_shape=IMAGE_SIZE + (3,),
        **eval(FLAGS.model_kwargs),
    )

"""
Next, we pick an optimizer. Here we use Adam with a constant learning rate.
Note that learning rate will decrease over time due to the ReduceLROnPlateau callback.
"""

optimizer = optimizers.SGD(learning_rate=FLAGS.initial_learning_rate, momentum=0.9)

"""
Next, we pick a loss function. We use CategoricalCrossentropy with label smoothing.
"""

loss_fn = losses.CategoricalCrossentropy(label_smoothing=0.1)

"""
Next, we specify the metrics that we want to track. For this example, we track accuracy.
"""

with strategy.scope():
    training_metrics = [metrics.CategoricalAccuracy()]

"""
As a last piece of configuration, we configure callbacks for the method.
We use EarlyStopping, BackupAndRestore, and a model checkpointing callback.
"""

callbacks = [
    callbacks.ReduceLROnPlateau(
        monitor="val_loss", factor=0.1, patience=10, min_delta=0.001, min_lr=0.0001
    ),
    callbacks.EarlyStopping(patience=20),
    callbacks.BackupAndRestore(FLAGS.backup_path),
    callbacks.ModelCheckpoint(FLAGS.weights_path, save_weights_only=True),
    callbacks.TensorBoard(log_dir=FLAGS.tensorboard_path, write_steps_per_second=True),
]

"""
We can now compile the model and fit it to the training dataset.
"""

model.compile(
    optimizer=optimizer,
    loss=loss_fn,
    metrics=training_metrics,
    jit_compile=FLAGS.use_xla,
)

model.fit(
    train_ds,
    batch_size=FLAGS.batch_size,
    epochs=EPOCHS,
    callbacks=callbacks,
    validation_data=test_ds,
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf.urls import patterns, url, include

urlpatterns = patterns('',
    url(r'^logout/$', 'bakery.views.logout', name='logout'),
    url(r'^login-error/$', 'bakery.views.login_error', name='login_error'),

    url(r'^profile/(?P<username>.*?)/$', 'bakery.views.profile', name='profile'),
)
<commit_msg>fix auth.urls<commit_after># -*- coding: utf-8 -*-
from django.conf.urls import patterns, url, include

urlpatterns = patterns('bakery.auth.views',
    url(r'^logout/$', 'logout', name='logout'),
    url(r'^login-error/$', 'login_error', name='login_error'),
    url(r'^profile/(?P<username>.*?)/$', 'profile', name='profile'),
)
<|endoftext|>


<commit_before>from dimagi.utils.logging import notify_exception

class SyncCouchToSQLMixin(object):
    """
    A mixin to be used for migrating a Couch model
    to a SQL model (see also SyncSQLToCouchMixin).

    Steps to use it:

    1. Create a Django ORM model which inherits from SyncSQLToCouchMixin
    and is identical to the Couch model with one additional field named
    couch_id, for the _id of the Couch doc. Make sure you index couch_id
    and allow it to be null since it can be initially null. Do not include
    this field in _migration_get_fields().
    You can override '_migration_couch_id_name' if `couch_id` won't work.

    2. Make the Couch model inherit from SyncCouchToSQLMixin.

    3. Implement the unimplemented methods from the mixins on both models.

    4. Make sure neither of the models overrides the save() or delete()
    methods. If either does, you will have to manually insert the sync
    code from the corresponding mixin. Also make sure the mixins are the
    left-most classes being inherited from.

    After that, any saves to the Couch model will sync the SQL model, and
    any saves to the SQL model will sync the Couch model. After migrating
    each doc, you can incrementally change the code to use to SQL model
    instead of the Couch model, and eventually remove the Couch model entirely.

    If you have a custom sync process, just override _migration_sync_to_sql.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_sql_model_class(cls):
        """
        Should return the class of the SQL model.
        """
        raise NotImplementedError()

    def _migration_automatically_handle_dups(self):
        """
        Ideally, you should use a CriticalSection to lock out your get and save
        functionality so that each sync will automatically be atomic. However,
        even after doing so, during periods of high latency the locks may time
        out and you still might end up with a race condition in the sync that
        creates more than one SQL model from this couch model.

        Make this method return True if you want subsequent syncs to delete
        the duplicate records and resync. Otherwise, the default behavior is
        that a notify exception email will go out each time duplicates are found.
        """
        return False

    def _migration_get_sql_object(self):
        cls = self._migration_get_sql_model_class()
        try:
            return cls.objects.get(**{cls._migration_couch_id_name: self._id})
        except cls.DoesNotExist:
            return None
        except cls.MultipleObjectsReturned:
            if not self._migration_automatically_handle_dups():
                raise
            cls.objects.filter(**{cls._migration_couch_id_name: self._id}).delete()
            return None

    def _migration_get_or_create_sql_object(self):
        cls = self._migration_get_sql_model_class()
        obj = self._migration_get_sql_object()
        if obj is None:
            obj = cls(**{cls._migration_couch_id_name: self._id})
        return obj

    def _migration_sync_to_sql(self, sql_object):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(sql_object, field_name, value)
        sql_object.save(sync_to_couch=False)

    def _migration_do_sync(self):
        sql_object = self._migration_get_or_create_sql_object()
        self._migration_sync_to_sql(sql_object)

    def save(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        super(SyncCouchToSQLMixin, self).save(*args, **kwargs)
        if sync_to_sql:
            try:
                self._migration_do_sync()
            except:
                sql_class_name = self._migration_get_sql_model_class().__name__
                couch_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s SQL object from %s %s' % (sql_class_name,
                        couch_class_name, self._id))

    def delete(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        if sync_to_sql:
            sql_object = self._migration_get_sql_object()
            if sql_object is not None:
                sql_object.delete(sync_to_couch=False)
        super(SyncCouchToSQLMixin, self).delete(*args, **kwargs)

class SyncSQLToCouchMixin(object):
    """
    See SyncCouchToSQLMixin.

    If you have a custom sync process, just override _migration_sync_to_couch.
    """
    _migration_couch_id_name = "couch_id"

    @property
    def _migration_couch_id(self):
        return getattr(self, self._migration_couch_id_name)

    @_migration_couch_id.setter
    def _migration_couch_id(self, value):
        setattr(self, self._migration_couch_id_name, value)

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_couch_model_class(cls):
        """
        Should return the class of the Couch model.
        """
        raise NotImplementedError()

    def _migration_get_couch_object(self):
        if not self._migration_couch_id:
            return None
        cls = self._migration_get_couch_model_class()
        return cls.get(self._migration_couch_id)

    def _migration_get_or_create_couch_object(self):
        cls = self._migration_get_couch_model_class()
        obj = self._migration_get_couch_object()
        if obj is None:
            obj = cls()
            obj.save(sync_to_sql=False)
            self._migration_couch_id = obj._id
            self.save(sync_to_couch=False)
        return obj

    def _migration_sync_to_couch(self, couch_obj):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(couch_obj, field_name, value)
        couch_obj.save(sync_to_sql=False)

    def _migration_do_sync(self):
        couch_obj = self._migration_get_or_create_couch_object()
        self._migration_sync_to_couch(couch_obj)

    def save(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        super(SyncSQLToCouchMixin, self).save(*args, **kwargs)
        if sync_to_couch:
            try:
                self._migration_do_sync()
            except:
                couch_class_name = self._migration_get_couch_model_class().__name__
                sql_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s Couch object from %s %s' % (couch_class_name,
                        sql_class_name, self.pk))

    def delete(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        if sync_to_couch:
            couch_object = self._migration_get_couch_object()
            if couch_object is not None:
                couch_object.delete(sync_to_sql=False)
        super(SyncSQLToCouchMixin, self).delete(*args, **kwargs)
<commit_msg>Added support for SchemaListProperty attributes to couch to sql sync mixins<commit_after>from collections import namedtuple

from dimagi.utils.logging import notify_exception

SubModelSpec = namedtuple('SubModelSpec', [
    'sql_attr',
    'sql_class',
    'sql_fields',
    'couch_attr',
    'couch_class',
    'couch_fields',
])

class SyncCouchToSQLMixin(object):
    """
    A mixin to be used for migrating a Couch model
    to a SQL model (see also SyncSQLToCouchMixin).

    Steps to use it:

    1. Create a Django ORM model which inherits from SyncSQLToCouchMixin
    and is identical to the Couch model with one additional field named
    couch_id, for the _id of the Couch doc. Make sure you index couch_id
    and allow it to be null since it can be initially null. Do not include
    this field in _migration_get_fields().
    You can override '_migration_couch_id_name' if `couch_id` won't work.

    2. Make the Couch model inherit from SyncCouchToSQLMixin.

    3. Implement the unimplemented methods from the mixins on both models.

    4. Make sure neither of the models overrides the save() or delete()
    methods. If either does, you will have to manually insert the sync
    code from the corresponding mixin. Also make sure the mixins are the
    left-most classes being inherited from.

    After that, any saves to the Couch model will sync the SQL model, and
    any saves to the SQL model will sync the Couch model. After migrating
    each doc, you can incrementally change the code to use to SQL model
    instead of the Couch model, and eventually remove the Couch model entirely.

    If you have a custom sync process, just override _migration_sync_to_sql.
    """

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_submodels(cls):
        """
        Should return a list of SubModelSpec tuples, one for each SchemaListProperty
        in the couch class. Should be identical in the couch and sql mixins.
        """
        return []

    @classmethod
    def _migration_get_sql_model_class(cls):
        """
        Should return the class of the SQL model.
        """
        raise NotImplementedError()

    def _migration_automatically_handle_dups(self):
        """
        Ideally, you should use a CriticalSection to lock out your get and save
        functionality so that each sync will automatically be atomic. However,
        even after doing so, during periods of high latency the locks may time
        out and you still might end up with a race condition in the sync that
        creates more than one SQL model from this couch model.

        Make this method return True if you want subsequent syncs to delete
        the duplicate records and resync. Otherwise, the default behavior is
        that a notify exception email will go out each time duplicates are found.
        """
        return False

    def _migration_get_sql_object(self):
        cls = self._migration_get_sql_model_class()
        try:
            return cls.objects.get(**{cls._migration_couch_id_name: self._id})
        except cls.DoesNotExist:
            return None
        except cls.MultipleObjectsReturned:
            if not self._migration_automatically_handle_dups():
                raise
            cls.objects.filter(**{cls._migration_couch_id_name: self._id}).delete()
            return None

    def _migration_get_or_create_sql_object(self):
        cls = self._migration_get_sql_model_class()
        obj = self._migration_get_sql_object()
        if obj is None:
            obj = cls(**{cls._migration_couch_id_name: self._id})
        return obj

    def _migration_sync_to_sql(self, sql_object):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(sql_object, field_name, value)
        for spec in self._migration_get_submodels():
            sql_submodels = []
            for couch_submodel in getattr(self, spec.couch_attr):
                sql_fields = {
                    sql_field: getattr(couch_submodel, couch_field)
                    for couch_field, sql_field in zip(spec.couch_fields, spec.sql_fields)
                }
                sql_submodels.append(spec.sql_class(**sql_fields))
            sql_attr = getattr(sql_object, spec.sql_attr)
            sql_attr.all().delete()
            sql_attr.set(sql_submodels, bulk=False)
        sql_object.save(sync_to_couch=False)

    def _migration_do_sync(self):
        sql_object = self._migration_get_or_create_sql_object()
        self._migration_sync_to_sql(sql_object)

    def save(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        super(SyncCouchToSQLMixin, self).save(*args, **kwargs)
        if sync_to_sql:
            try:
                self._migration_do_sync()
            except:
                sql_class_name = self._migration_get_sql_model_class().__name__
                couch_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s SQL object from %s %s' % (sql_class_name,
                        couch_class_name, self._id))

    def delete(self, *args, **kwargs):
        sync_to_sql = kwargs.pop('sync_to_sql', True)
        if sync_to_sql:
            sql_object = self._migration_get_sql_object()
            if sql_object is not None:
                sql_object.delete(sync_to_couch=False)
        super(SyncCouchToSQLMixin, self).delete(*args, **kwargs)

class SyncSQLToCouchMixin(object):
    """
    See SyncCouchToSQLMixin.

    If you have a custom sync process, just override _migration_sync_to_couch.
    """
    _migration_couch_id_name = "couch_id"

    @property
    def _migration_couch_id(self):
        return getattr(self, self._migration_couch_id_name)

    @_migration_couch_id.setter
    def _migration_couch_id(self, value):
        setattr(self, self._migration_couch_id_name, value)

    @classmethod
    def _migration_get_fields(cls):
        """
        Should return a list of field names to sync. These field names should
        be identical between the Couch and SQL models.
        """
        raise NotImplementedError()

    @classmethod
    def _migration_get_submodels(cls):
        """
        Should return a list of SubModelSpec tuples, one for each SchemaListProperty
        in the couch class. Should be identical in the couch and sql mixins.
        """
        return []

    @classmethod
    def _migration_get_couch_model_class(cls):
        """
        Should return the class of the Couch model.
        """
        raise NotImplementedError()

    def _migration_get_couch_object(self):
        if not self._migration_couch_id:
            return None
        cls = self._migration_get_couch_model_class()
        return cls.get(self._migration_couch_id)

    def _migration_get_or_create_couch_object(self):
        cls = self._migration_get_couch_model_class()
        obj = self._migration_get_couch_object()
        if obj is None:
            obj = cls()
            obj.save(sync_to_sql=False)
            self._migration_couch_id = obj._id
            self.save(sync_to_couch=False)
        return obj

    def _migration_sync_to_couch(self, couch_object):
        for field_name in self._migration_get_fields():
            value = getattr(self, field_name)
            setattr(couch_object, field_name, value)
        for spec in self._migration_get_submodels():
            couch_submodels = []
            for sql_submodel in getattr(self, spec.sql_attr).all():
                couch_fields = {
                    couch_field: getattr(sql_submodel, sql_field)
                    for couch_field, sql_field in zip(spec.couch_fields, spec.sql_fields)
                }
                couch_submodels.append(spec.couch_class(**couch_fields))
            setattr(couch_object, spec.couch_attr, couch_submodels)
        couch_object.save(sync_to_sql=False)

    def _migration_do_sync(self):
        couch_object = self._migration_get_or_create_couch_object()
        self._migration_sync_to_couch(couch_object)

    def save(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        super(SyncSQLToCouchMixin, self).save(*args, **kwargs)
        if sync_to_couch:
            try:
                self._migration_do_sync()
            except:
                couch_class_name = self._migration_get_couch_model_class().__name__
                sql_class_name = self.__class__.__name__
                notify_exception(None,
                    message='Could not sync %s Couch object from %s %s' % (couch_class_name,
                        sql_class_name, self.pk))

    def delete(self, *args, **kwargs):
        sync_to_couch = kwargs.pop('sync_to_couch', True)
        if sync_to_couch:
            couch_object = self._migration_get_couch_object()
            if couch_object is not None:
                couch_object.delete(sync_to_sql=False)
        super(SyncSQLToCouchMixin, self).delete(*args, **kwargs)
<|endoftext|>


<commit_before>import os
import CoolProp.CoolProp as CP
import pint
from itertools import combinations
from functools import wraps
from collections import OrderedDict

__all__ = ['State', 'fluid_list', 'ureg', 'Q_', 'convert_to_base_units']

# define pint unit registry
new_units = os.path.join(os.path.dirname(__file__), 'new_units.txt')
ureg = pint.UnitRegistry()
ureg.load_definitions(new_units)
Q_ = ureg.Quantity

CP.set_config_string(CP.ALTERNATIVE_REFPROP_PATH, '/home/raphael/REFPROP-cmake/build/')

mixture = ['CarbonDioxide', 'Nitrogen', 'R134a', 'Oxygen']
for mix1, mix2 in combinations(mixture, 2):
    cas1 = CP.get_fluid_param_string(mix1, 'CAS')
    cas2 = CP.get_fluid_param_string(mix2, 'CAS')
    CP.apply_simple_mixing_rule(cas1, cas2, 'linear')

# list of available fluids
fluid_list = CP.get_global_param_string('fluids_list').split(',')

def normalize_mix(molar_fractions):
    """
    Normalize the molar fractions so that the sum is 1.

    Parameters
    ----------
    molar_fractions : list
        Molar fractions of the components.

    Returns
    -------
    molar_fractions: list
        Molar fractions list will be modified in place.
    """
    total = sum(molar_fractions)
    for i, comp in enumerate(molar_fractions):
        molar_fractions[i] = comp / total

def convert_to_base_units(func):
    """Convert units.

    Decorator used to convert units based on **kwargs.
    """
    # get units from kwargs. Set default if not provided.
    @wraps(func)
    def inner(*args, **kwargs):
        p_units = kwargs.get('p_units', ureg.Pa)
        T_units = kwargs.get('T_units', ureg.degK)
        speed_units = kwargs.get('speed_units', ureg.rad / ureg.s)
        flow_m_units = kwargs.get('flow_m_units', ureg.kg / ureg.s)
        flow_v_units = kwargs.get('flow_v_units', ureg.m**3 / ureg.s)
        power_units = kwargs.get('power_units', ureg.W)

        for arg_name, value in kwargs.items():
            if arg_name == 'p':
                p_ = Q_(value, p_units)
                p_.ito_base_units()
                kwargs[arg_name] = p_.magnitude
            elif arg_name is 'T':
                T_ = Q_(value, T_units)
                T_.ito_base_units()
                kwargs[arg_name] = T_.magnitude
            elif arg_name is 'speed':
                speed_ = Q_(value, speed_units)
                speed_.ito_base_units()
                kwargs[arg_name] = speed_.magnitude
            elif arg_name is 'flow_m':
                flow_m_ = Q_(value, flow_m_units)
                flow_m_.ito_base_units()
                kwargs[arg_name] = flow_m_.magnitude
            elif arg_name is 'flow_v':
                flow_v_ = Q_(value, flow_v_units)
                flow_v_.ito_base_units()
                kwargs[arg_name] = flow_v_.magnitude
            elif arg_name is 'power':
                power_ = Q_(value, power_units)
                power_.ito_base_units()
                kwargs[arg_name] = power_.magnitude

        return func(*args, **kwargs)

    return inner

class State(CP.AbstractState):
    """State class.

    This class is inherited from CP.AbstractState.
    Some extra functionality has been added.
    """
    # new class to add methods to AbstractState
    # no call to super(). see :
    # http://stackoverflow.com/questions/18260095/
    def __init__(self, EOS, fluid):
        self.EOS = EOS
        self.fluid = fluid

    def fluid_dict(self):
        # preserve the dictionary from define method
        fluid_dict = {}
        for k, v in zip(self.fluid_names(), self.get_mole_fractions()):
            fluid_dict[k] = v
        return fluid_dict

    def __repr__(self):
        return (
            'State: \n{}'.format(self.fluid_dict())
            + '\n Temperature: {:10.5} K'.format(self.T())
            + '\n Pressure:    {:10.8} Pa'.format(self.p())
            + '\n Enthalpy:    {:10.5} J/kg'.format(self.hmass())
        )

        # TODO add more properties

    @classmethod
    @convert_to_base_units
    def define(
            cls,
            p=None, T=None, h=None, s=None, d=None,
            fluid=None,
            EOS='REFPROP',
            **kwargs):
        """Constructor for state.

        Creates a state from fluid composition and two properties.
        Properties should be in SI units, **kwargs can be passed
        to change units.

        Parameters
        ----------
        p : float
            State's pressure
        T : float
            State's temperature
        h : float
            State's enthalpy
        s : float
            State's entropy
        d : float

        fluid : dict
            Dictionary with constituent and composition
            (e.g.: ({'Oxygen': 0.2096, 'Nitrogen': 0.7812, 'Argon': 0.0092})
        EOS : string
            String with HEOS or REFPROP

        Returns
        -------
        state : State object

        Examples:
        ---------
        >>> fluid = {'Oxygen': 0.2096, 'Nitrogen': 0.7812, 'Argon': 0.0092}
        >>> s = State.define(fluid=fluid, p=101008, T=273, EOS='HEOS')
        >>> s.rhomass()
        1.2893965217814896
        """
        # define constituents and molar fractions to create and update state

        # fluid = kwargs.get('fluid')
        # EOS = kwargs.get('EOS', 'REFPROP')

        constituents = []
        molar_fractions = []

        for k, v in fluid.items():
            if EOS == 'REFPROP':
                k = CP.get_REFPROPname(k)
            constituents.append(k)
            molar_fractions.append(v)

        # create an adequate fluid string to cp.AbstractState
        _fluid = '&'.join(constituents)

        state = cls(EOS, _fluid)
        normalize_mix(molar_fractions)
        state.set_mole_fractions(molar_fractions)
        if p is not None:
            if T is not None:
                state.update(CP.PT_INPUTS, p, T)
            if s is not None:
                state.update(CP.PSmass_INPUTS, p, s)
        if h and s is not None:
            state.update(CP.HmassSmass_INPUTS, h, s)
        if d and s is not None:
            state.update(CP.DmassSmass_INPUTS, d, s)

        return state

    def __copy__(self):
        return self.define(p=self.p(), T=self.T(), fluid=self.fluid_dict(), EOS=self.EOS)
<commit_msg>Changing repr.<commit_after>import os
import CoolProp.CoolProp as CP
import pint
from itertools import combinations
from functools import wraps
from collections import OrderedDict

__all__ = ['State', 'fluid_list', 'ureg', 'Q_', 'convert_to_base_units']

# define pint unit registry
new_units = os.path.join(os.path.dirname(__file__), 'new_units.txt')
ureg = pint.UnitRegistry()
ureg.load_definitions(new_units)
Q_ = ureg.Quantity

CP.set_config_string(CP.ALTERNATIVE_REFPROP_PATH, '/home/raphael/REFPROP-cmake/build/')

mixture = ['CarbonDioxide', 'Nitrogen', 'R134a', 'Oxygen']
for mix1, mix2 in combinations(mixture, 2):
    cas1 = CP.get_fluid_param_string(mix1, 'CAS')
    cas2 = CP.get_fluid_param_string(mix2, 'CAS')
    CP.apply_simple_mixing_rule(cas1, cas2, 'linear')

# list of available fluids
fluid_list = CP.get_global_param_string('fluids_list').split(',')

def normalize_mix(molar_fractions):
    """
    Normalize the molar fractions so that the sum is 1.

    Parameters
    ----------
    molar_fractions : list
        Molar fractions of the components.

    Returns
    -------
    molar_fractions: list
        Molar fractions list will be modified in place.
    """
    total = sum(molar_fractions)
    for i, comp in enumerate(molar_fractions):
        molar_fractions[i] = comp / total

def convert_to_base_units(func):
    """Convert units.

    Decorator used to convert units based on **kwargs.
    """
    # get units from kwargs. Set default if not provided.
    @wraps(func)
    def inner(*args, **kwargs):
        p_units = kwargs.get('p_units', ureg.Pa)
        T_units = kwargs.get('T_units', ureg.degK)
        speed_units = kwargs.get('speed_units', ureg.rad / ureg.s)
        flow_m_units = kwargs.get('flow_m_units', ureg.kg / ureg.s)
        flow_v_units = kwargs.get('flow_v_units', ureg.m**3 / ureg.s)
        power_units = kwargs.get('power_units', ureg.W)

        for arg_name, value in kwargs.items():
            if arg_name == 'p':
                p_ = Q_(value, p_units)
                p_.ito_base_units()
                kwargs[arg_name] = p_.magnitude
            elif arg_name is 'T':
                T_ = Q_(value, T_units)
                T_.ito_base_units()
                kwargs[arg_name] = T_.magnitude
            elif arg_name is 'speed':
                speed_ = Q_(value, speed_units)
                speed_.ito_base_units()
                kwargs[arg_name] = speed_.magnitude
            elif arg_name is 'flow_m':
                flow_m_ = Q_(value, flow_m_units)
                flow_m_.ito_base_units()
                kwargs[arg_name] = flow_m_.magnitude
            elif arg_name is 'flow_v':
                flow_v_ = Q_(value, flow_v_units)
                flow_v_.ito_base_units()
                kwargs[arg_name] = flow_v_.magnitude
            elif arg_name is 'power':
                power_ = Q_(value, power_units)
                power_.ito_base_units()
                kwargs[arg_name] = power_.magnitude

        return func(*args, **kwargs)

    return inner

class State(CP.AbstractState):
    """State class.

    This class is inherited from CP.AbstractState.
    Some extra functionality has been added.
    """
    # new class to add methods to AbstractState
    # no call to super(). see :
    # http://stackoverflow.com/questions/18260095/
    def __init__(self, EOS, fluid):
        self.EOS = EOS
        self.fluid = fluid

    def fluid_dict(self):
        # preserve the dictionary from define method
        fluid_dict = {}
        for k, v in zip(self.fluid_names(), self.get_mole_fractions()):
            fluid_dict[k] = v
        return fluid_dict

    def __repr__(self):
        composition = ''
        for k, v in self.fluid_dict().items():
            composition += '\n {:15}: {:.2f}%'.format(k, v * 100)
        return (
            'State: '
            + composition
            + '\n' + 35*'-'
            + '\n Temperature: {:10.5} K'.format(self.T())
            + '\n Pressure   : {:10.5} Pa'.format(self.p())
            + '\n Density    : {:10.5} kg/m^3'.format(self.rhomass())
            + '\n Enthalpy   : {:10.5} J/kg'.format(self.hmass())
            + '\n Entropy    : {:10.5} kg.J/K'.format(self.smass())
        )

        # TODO add more properties

    @classmethod
    @convert_to_base_units
    def define(
            cls,
            p=None, T=None, h=None, s=None, d=None,
            fluid=None,
            EOS='REFPROP',
            **kwargs):
        """Constructor for state.

        Creates a state from fluid composition and two properties.
        Properties should be in SI units, **kwargs can be passed
        to change units.

        Parameters
        ----------
        p : float
            State's pressure
        T : float
            State's temperature
        h : float
            State's enthalpy
        s : float
            State's entropy
        d : float

        fluid : dict
            Dictionary with constituent and composition
            (e.g.: ({'Oxygen': 0.2096, 'Nitrogen': 0.7812, 'Argon': 0.0092})
        EOS : string
            String with HEOS or REFPROP

        Returns
        -------
        state : State object

        Examples:
        ---------
        >>> fluid = {'Oxygen': 0.2096, 'Nitrogen': 0.7812, 'Argon': 0.0092}
        >>> s = State.define(fluid=fluid, p=101008, T=273, EOS='HEOS')
        >>> s.rhomass()
        1.2893965217814896
        """
        # define constituents and molar fractions to create and update state

        # fluid = kwargs.get('fluid')
        # EOS = kwargs.get('EOS', 'REFPROP')

        constituents = []
        molar_fractions = []

        for k, v in fluid.items():
            if EOS == 'REFPROP':
                k = CP.get_REFPROPname(k)
            constituents.append(k)
            molar_fractions.append(v)

        # create an adequate fluid string to cp.AbstractState
        _fluid = '&'.join(constituents)

        state = cls(EOS, _fluid)
        normalize_mix(molar_fractions)
        state.set_mole_fractions(molar_fractions)
        if p is not None:
            if T is not None:
                state.update(CP.PT_INPUTS, p, T)
            if s is not None:
                state.update(CP.PSmass_INPUTS, p, s)
        if h and s is not None:
            state.update(CP.HmassSmass_INPUTS, h, s)
        if d and s is not None:
            state.update(CP.DmassSmass_INPUTS, d, s)

        return state

    def __copy__(self):
        return self.define(p=self.p(), T=self.T(), fluid=self.fluid_dict(), EOS=self.EOS)
<|endoftext|>


<commit_before># Copyright (c) 2009-2021 The Regents of the University of Michigan
# This file is part of the HOOMD-blue project, released under the BSD 3-Clause
# License.

"""Implement parameter dictionaries."""

from collections.abc import MutableMapping
from itertools import product, combinations_with_replacement
from copy import copy

from hoomd.util import _to_camel_case, _is_iterable
from hoomd.data.typeconverter import (to_type_converter, RequiredArg,
                                      TypeConverterMapping, OnlyIf, Either)
from hoomd.data.smart_default import (_to_base_defaults, _to_default,
                                      _SmartDefault, _NoDefault)
from hoomd.error import TypeConversionError
import numpy as np

def has_str_elems(obj):
    """Returns True if all elements of iterable are str."""
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    """Returns True if object is iterable with respect to types."""
    return _is_iterable(obj) and has_str_elems(obj)

def proper_type_return(val):
    """Expects and requires a dictionary with type keys."""
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        _defaults = kwargs.pop('_defaults', _NoDefault)
        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")

        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            default_arg = kwargs
        else:
            default_arg = args[0]
        self._type_converter = to_type_converter(default_arg)
        self._default = _to_default(default_arg, _defaults)

    def _validate_values(self, val):
        val = self._type_converter(val)
        if isinstance(val, dict):
            if isinstance(self._type_converter, TypeConverterMapping):
                dft_keys = set(self._type_converter.keys())
            elif isinstance(self._type_converter.converter, OnlyIf):
                dft_keys = set(self._type_converter.converter.cond.keys())
            elif isinstance(self._type_converter.converter, Either):
                mapping = next(
                    filter(lambda x: isinstance(x, TypeConverterMapping),
                           self._type_converter.converter.specs))
                dft_keys = set(mapping.keys())
            else:
                raise ValueError
            bad_keys = set(val.keys()) - dft_keys
            if len(bad_keys) != 0:
                raise ValueError("Keys must be a subset of available keys. "
                                 "Bad keys are {}".format(bad_keys))
        if isinstance(self._default, _SmartDefault):
            val = self._default(val)
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        """Validate key given regardless of key length."""
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        """Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        """
        if isinstance(key, str):
            return [key]
        elif _is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        """Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        """
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([
                    not is_good_iterable(v) and not isinstance(v, str)
                    for v in key
            ]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif _is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        """Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        """
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    def __eq__(self, other):
        if not isinstance(other, _ValidatedDefaultDict):
            return NotImplemented
        return (self.default == other.default
                and set(self.keys()) == set(other.keys())
                and all(self[key] == other[key] for key in self.keys()))

    @property
    def default(self):
        if isinstance(self._default, _SmartDefault):
            return self._default.to_base()
        else:
            return copy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._type_converter(new_default)
        if isinstance(self._default, _SmartDefault):
            new_default = self._default(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
        self._default = _to_default(new_default)

class TypeParameterDict(_ValidatedDefaultDict):
    """Type parameter dictionary."""

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        """Access parameter by key."""
        vals = dict()
        for key in self._yield_keys(key):
            try:
                vals[key] = self._dict[key]
            except KeyError:
                vals[key] = self.default
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        """Set parameter by key."""
        keys = self._yield_keys(key)
        try:
            val = self._validate_values(val)
        except TypeConversionError as err:
            raise TypeConversionError("For types {}, error {}.".format(
                list(keys), str(err)))
        for key in keys:
            self._dict[key] = val

    def keys(self):
        """Get the keys in the dictionary."""
        if self._len_keys == 1:
            yield from self._dict.keys()
        else:
            for key in self._dict.keys():
                yield tuple(sorted(list(key)))

    def to_dict(self):
        """Convert to a `dict`."""
        return self._dict

class AttachedTypeParameterDict(_ValidatedDefaultDict):
    """Parameter dictionary synchronized with a C++ class."""

    def __init__(self, cpp_obj, param_name, type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict._default
        self._type_converter = type_param_dict._type_converter
        # add all types to c++
        for key in self.keys():
            self[key] = type_param_dict[key]

    def to_detached(self):
        """Convert to a detached parameter dict."""
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        type_param_dict._type_converter = self._type_converter
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        """Access parameter by key."""
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        """Set parameter by key."""
        keys = self._yield_keys(key)
        try:
            val = self._validate_values(val)
        except TypeConversionError as err:
            raise TypeConversionError("For types {}, error {}.".format(
                list(keys), str(err)))

        for key in keys:
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        """Includes key check for existing simulation keys."""
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    def _validate_values(self, val):
        val = super()._validate_values(val)
        if isinstance(val, dict):
            not_set_keys = []
            for k, v in val.items():
                if v is RequiredArg:
                    not_set_keys.append(k)
            if not_set_keys != []:
                raise ValueError("{} were not set.".format(not_set_keys))
        return val

    @property
    def _setter(self):
        return 'set' + _to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + _to_camel_case(self._param_name)

    def keys(self):
        """Iterate through the keys."""
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 1:
            yield from single_keys
        else:
            for key in combinations_with_replacement(single_keys,
                                                     self._len_keys):
                yield tuple(sorted(list(key)))

    def to_dict(self):
        """Convert to a `dict`."""
        rtn_dict = dict()
        for key in self.keys():
            rtn_dict[key] = getattr(self._cpp_obj, self._getter)(key)
        return rtn_dict

class ParameterDict(MutableMapping):
    """Parameter dictionary."""

    def __init__(self, _defaults=_NoDefault, **kwargs):
        self._type_converter = to_type_converter(kwargs)
        self._dict = {**_to_base_defaults(kwargs, _defaults)}

    def __setitem__(self, key, value):
        """Set parameter by key."""
        if key not in self._type_converter.keys():
            self._dict[key] = value
            self._type_converter[key] = to_type_converter(value)
        else:
            self._dict[key] = self._type_converter[key](value)

    def __getitem__(self, key):
        """Access parameter by key."""
        return self._dict[key]

    def __delitem__(self, key):
        """Remove parameter by key."""
        del self._dict[key]
        del self._type_converter[key]

    def __iter__(self):
        """Iterate over keys."""
        yield from self._dict

    def __len__(self):
        """int: The number of keys."""
        return len(self._dict)

    def __eq__(self, other):
        """Equality betwen ParameterDict objects"""
        if not isinstance(other, ParameterDict):
            return NotImplemented
        return (set(self.keys()) == set(other.keys()) and np.all(
            np.all(self[key] == other[key]) for key in self.keys()))

    def update(self, other):
        """Add keys and values to the dictionary."""
        if isinstance(other, ParameterDict):
            for key, value in other.items():
                self._type_converter[key] = other._type_converter[key]
                self._dict[key] = value
        else:
            for key, value in other.items():
                self[key] = value
<commit_msg>fix typo<commit_after># Copyright (c) 2009-2021 The Regents of the University of Michigan
# This file is part of the HOOMD-blue project, released under the BSD 3-Clause
# License.

"""Implement parameter dictionaries."""

from collections.abc import MutableMapping
from itertools import product, combinations_with_replacement
from copy import copy

from hoomd.util import _to_camel_case, _is_iterable
from hoomd.data.typeconverter import (to_type_converter, RequiredArg,
                                      TypeConverterMapping, OnlyIf, Either)
from hoomd.data.smart_default import (_to_base_defaults, _to_default,
                                      _SmartDefault, _NoDefault)
from hoomd.error import TypeConversionError
import numpy as np

def has_str_elems(obj):
    """Returns True if all elements of iterable are str."""
    return all([isinstance(elem, str) for elem in obj])

def is_good_iterable(obj):
    """Returns True if object is iterable with respect to types."""
    return _is_iterable(obj) and has_str_elems(obj)

def proper_type_return(val):
    """Expects and requires a dictionary with type keys."""
    if len(val) == 0:
        return None
    elif len(val) == 1:
        return list(val.values())[0]
    else:
        return val

class _ValidatedDefaultDict:

    def __init__(self, *args, **kwargs):
        _defaults = kwargs.pop('_defaults', _NoDefault)
        if len(kwargs) != 0 and len(args) != 0:
            raise ValueError("An unnamed argument and keyword arguments "
                             "cannot both be specified.")

        if len(kwargs) == 0 and len(args) == 0:
            raise ValueError("Either an unnamed argument or keyword "
                             "arguments must be specified.")
        if len(args) > 1:
            raise ValueError("Only one unnamed argument allowed.")
        if len(kwargs) > 0:
            default_arg = kwargs
        else:
            default_arg = args[0]
        self._type_converter = to_type_converter(default_arg)
        self._default = _to_default(default_arg, _defaults)

    def _validate_values(self, val):
        val = self._type_converter(val)
        if isinstance(val, dict):
            if isinstance(self._type_converter, TypeConverterMapping):
                dft_keys = set(self._type_converter.keys())
            elif isinstance(self._type_converter.converter, OnlyIf):
                dft_keys = set(self._type_converter.converter.cond.keys())
            elif isinstance(self._type_converter.converter, Either):
                mapping = next(
                    filter(lambda x: isinstance(x, TypeConverterMapping),
                           self._type_converter.converter.specs))
                dft_keys = set(mapping.keys())
            else:
                raise ValueError
            bad_keys = set(val.keys()) - dft_keys
            if len(bad_keys) != 0:
                raise ValueError("Keys must be a subset of available keys. "
                                 "Bad keys are {}".format(bad_keys))
        if isinstance(self._default, _SmartDefault):
            val = self._default(val)
        return val

    # Add function to validate dictionary keys' value types as well
    # Could follow current model on the args based type checking

    def _validate_and_split_key(self, key):
        """Validate key given regardless of key length."""
        if self._len_keys == 1:
            return self._validate_and_split_len_one(key)
        else:
            return self._validate_and_split_len(key)

    def _validate_and_split_len_one(self, key):
        """Validate single type keys.

        Accepted input is a type string, and arbitrarily nested interators that
        culminate in str types.
        """
        if isinstance(key, str):
            return [key]
        elif _is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len_one(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _validate_and_split_len(self, key):
        """Validate all key lengths greater than one, N.

        Valid input is an arbitrarily deep series of iterables that culminate
        in N length tuples, this includes an iterable depth of zero. The N
        length tuples can contain for each member either a type string or an
        iterable of type strings.
        """
        if isinstance(key, tuple) and len(key) == self._len_keys:
            fst, snd = key
            if any([
                    not is_good_iterable(v) and not isinstance(v, str)
                    for v in key
            ]):
                raise KeyError("The key {} is not valid.".format(key))
            key = list(key)
            for ind in range(len(key)):
                if isinstance(key[ind], str):
                    key[ind] = [key[ind]]
            return list(product(*key))
        elif _is_iterable(key):
            keys = []
            for k in key:
                keys.extend(self._validate_and_split_len(k))
            return keys
        else:
            raise KeyError("The key {} is not valid.".format(key))

    def _yield_keys(self, key):
        """Returns the generated keys in proper sorted order.

        The order is necessary so ('A', 'B') is equivalent to ('B', A').
        """
        if self._len_keys > 1:
            keys = self._validate_and_split_key(key)
            for key in keys:
                yield tuple(sorted(list(key)))
        else:
            yield from self._validate_and_split_key(key)

    def __eq__(self, other):
        if not isinstance(other, _ValidatedDefaultDict):
            return NotImplemented
        return (self.default == other.default
                and set(self.keys()) == set(other.keys())
                and all(self[key] == other[key] for key in self.keys()))

    @property
    def default(self):
        if isinstance(self._default, _SmartDefault):
            return self._default.to_base()
        else:
            return copy(self._default)

    @default.setter
    def default(self, new_default):
        new_default = self._type_converter(new_default)
        if isinstance(self._default, _SmartDefault):
            new_default = self._default(new_default)
        if isinstance(new_default, dict):
            keys = set(self._default.keys())
            provided_keys = set(new_default.keys())
            if keys.intersection(provided_keys) != provided_keys:
                raise ValueError("New default must a subset of current keys.")
        self._default = _to_default(new_default)

class TypeParameterDict(_ValidatedDefaultDict):
    """Type parameter dictionary."""

    def __init__(self, *args, len_keys, **kwargs):

        # Validate proper key constraint
        if len_keys < 1 or len_keys != int(len_keys):
            raise ValueError("len_keys must be a positive integer.")
        self._len_keys = len_keys
        super().__init__(*args, **kwargs)
        self._dict = dict()

    def __getitem__(self, key):
        """Access parameter by key."""
        vals = dict()
        for key in self._yield_keys(key):
            try:
                vals[key] = self._dict[key]
            except KeyError:
                vals[key] = self.default
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        """Set parameter by key."""
        keys = self._yield_keys(key)
        try:
            val = self._validate_values(val)
        except TypeConversionError as err:
            raise TypeConversionError("For types {}, error {}.".format(
                list(keys), str(err)))
        for key in keys:
            self._dict[key] = val

    def keys(self):
        """Get the keys in the dictionary."""
        if self._len_keys == 1:
            yield from self._dict.keys()
        else:
            for key in self._dict.keys():
                yield tuple(sorted(list(key)))

    def to_dict(self):
        """Convert to a `dict`."""
        return self._dict

class AttachedTypeParameterDict(_ValidatedDefaultDict):
    """Parameter dictionary synchronized with a C++ class."""

    def __init__(self, cpp_obj, param_name, type_kind, type_param_dict, sim):
        # store info to communicate with c++
        self._cpp_obj = cpp_obj
        self._param_name = param_name
        self._sim = sim
        self._type_kind = type_kind
        self._len_keys = type_param_dict._len_keys
        # Get default data
        self._default = type_param_dict._default
        self._type_converter = type_param_dict._type_converter
        # add all types to c++
        for key in self.keys():
            self[key] = type_param_dict[key]

    def to_detached(self):
        """Convert to a detached parameter dict."""
        if isinstance(self.default, dict):
            type_param_dict = TypeParameterDict(**self.default,
                                                len_keys=self._len_keys)
        else:
            type_param_dict = TypeParameterDict(self.default,
                                                len_keys=self._len_keys)
        type_param_dict._type_converter = self._type_converter
        for key in self.keys():
            type_param_dict[key] = self[key]
        return type_param_dict

    def __getitem__(self, key):
        """Access parameter by key."""
        vals = dict()
        for key in self._yield_keys(key):
            vals[key] = getattr(self._cpp_obj, self._getter)(key)
        return proper_type_return(vals)

    def __setitem__(self, key, val):
        """Set parameter by key."""
        keys = self._yield_keys(key)
        try:
            val = self._validate_values(val)
        except TypeConversionError as err:
            raise TypeConversionError("For types {}, error {}.".format(
                list(keys), str(err)))

        for key in keys:
            getattr(self._cpp_obj, self._setter)(key, val)

    def _yield_keys(self, key):
        """Includes key check for existing simulation keys."""
        curr_keys = self.keys()
        for key in super()._yield_keys(key):
            if key not in curr_keys:
                raise KeyError("Type {} does not exist in the "
                               "system.".format(key))
            else:
                yield key

    def _validate_values(self, val):
        val = super()._validate_values(val)
        if isinstance(val, dict):
            not_set_keys = []
            for k, v in val.items():
                if v is RequiredArg:
                    not_set_keys.append(k)
            if not_set_keys != []:
                raise ValueError("{} were not set.".format(not_set_keys))
        return val

    @property
    def _setter(self):
        return 'set' + _to_camel_case(self._param_name)

    @property
    def _getter(self):
        return 'get' + _to_camel_case(self._param_name)

    def keys(self):
        """Iterate through the keys."""
        single_keys = getattr(self._sim.state, self._type_kind)
        if self._len_keys == 1:
            yield from single_keys
        else:
            for key in combinations_with_replacement(single_keys,
                                                     self._len_keys):
                yield tuple(sorted(list(key)))

    def to_dict(self):
        """Convert to a `dict`."""
        rtn_dict = dict()
        for key in self.keys():
            rtn_dict[key] = getattr(self._cpp_obj, self._getter)(key)
        return rtn_dict

class ParameterDict(MutableMapping):
    """Parameter dictionary."""

    def __init__(self, _defaults=_NoDefault, **kwargs):
        self._type_converter = to_type_converter(kwargs)
        self._dict = {**_to_base_defaults(kwargs, _defaults)}

    def __setitem__(self, key, value):
        """Set parameter by key."""
        if key not in self._type_converter.keys():
            self._dict[key] = value
            self._type_converter[key] = to_type_converter(value)
        else:
            self._dict[key] = self._type_converter[key](value)

    def __getitem__(self, key):
        """Access parameter by key."""
        return self._dict[key]

    def __delitem__(self, key):
        """Remove parameter by key."""
        del self._dict[key]
        del self._type_converter[key]

    def __iter__(self):
        """Iterate over keys."""
        yield from self._dict

    def __len__(self):
        """int: The number of keys."""
        return len(self._dict)

    def __eq__(self, other):
        """Equality between ParameterDict objects"""
        if not isinstance(other, ParameterDict):
            return NotImplemented
        return (set(self.keys()) == set(other.keys()) and np.all(
            np.all(self[key] == other[key]) for key in self.keys()))

    def update(self, other):
        """Add keys and values to the dictionary."""
        if isinstance(other, ParameterDict):
            for key, value in other.items():
                self._type_converter[key] = other._type_converter[key]
                self._dict[key] = value
        else:
            for key, value in other.items():
                self[key] = value
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import os

class ParkingMeter:
    categories = ["Get Back to Nature", "Within 2km", "Give Back to Community", "Today only", "Weekly"]

    def send_request(self, category, latitude, longitude):
        # just print instead of sending get request for now
        print "Sending request with parameters {{ category: {0}, latitude: {1}, longitude: {2} }}".format(category, latitude, longitude)
        # await response from server
        poi_list = ["Cathedral Square", "Cantebury Museum", "Botanic Gardens"] # mock response for now
        return poi_list

    def display(self, text):
        # for now just print to console
        # print text
        # try echo to console
        os.system("echo '" + text + "'")
        # try echo to ttyACM0
        os.system("echo '" + text + "' > test.text")
        os.system("echo '" + text + "' > /dev/ttyACM0")

    def get_choice(self):
        return input("Your choice: ") # change this to receive input from buttons on parking machine

    def print_ticket(self, poi):
        print "Printing ticket for " + poi

    def make_poi_selection(self, poi_list):
        for index, item in enumerate(poi_list, start = 1):
            self.display("{0}) {1}".format(index, item))
        sys.stdout.flush()
        self.display("Please choose an option: ")
        choice = self.get_choice()
        if choice > 0 and choice <= len(poi_list):
            self.print_ticket(poi_list[choice-1])
        else:
            self.display("Sorry, invalid choice")

    def start(self):
        self.display("Welcome to Open City! Please choose a category and hit enter:")
        for index, item in enumerate(self.categories, start = 1):
            self.display("{0}) {1}".format(index, item))
        sys.stdout.flush()
        choice = self.get_choice()
        if choice > 0 and choice <= len(self.categories):
            print "You chose " + self.categories[choice-1] # for debugging
            poi_list = self.send_request(self.categories[choice-1], 0.00, 0.00)
            self.make_poi_selection(poi_list)
        else:
            self.display("Sorry, you chose an invalid category")

instance = ParkingMeter()
instance.start()<commit_msg>uses serial as well as print<commit_after>#!/usr/bin/python
import sys
import subprocess
import serial

class ParkingMeter:
    categories = ["Get Back to Nature", "Within 2km", "Give Back to Community", "Today only", "Weekly"]
    ser = ""

    def send_request(self, category, latitude, longitude):
        # just print instead of sending get request for now
        print "Sending request with parameters {{ category: {0}, latitude: {1}, longitude: {2} }}".format(category, latitude, longitude)
        # await response from server
        poi_list = ["Cathedral Square", "Cantebury Museum", "Botanic Gardens"] # mock response for now
        return poi_list

    def display(self, text):
        # for now just print to console
        print text
        # try send to serial
        self.ser.write(text + "\n")

    def get_choice(self):
        return input("Your choice: ") # change this to receive input from buttons on parking machine

    def print_ticket(self, poi):
        print "Printing ticket for " + poi

    def make_poi_selection(self, poi_list):
        for index, item in enumerate(poi_list, start = 1):
            self.display("{0}) {1}".format(index, item))
        sys.stdout.flush()
        self.display("Please choose an option: ")
        choice = self.get_choice()
        if choice > 0 and choice <= len(poi_list):
            self.print_ticket(poi_list[choice-1])
        else:
            self.display("Sorry, invalid choice")

    def start(self):
        self.ser = serial.Serial('/dev/ttyACM0', 115200)
        self.ser.write("\xFE\x42")
        self.display("Welcome to Open City! Please choose a category and hit enter:")
        for index, item in enumerate(self.categories, start = 1):
            self.display("{0}) {1}".format(index, item))
        sys.stdout.flush()
        choice = self.get_choice()
        if choice > 0 and choice <= len(self.categories):
            print "You chose " + self.categories[choice-1] # for debugging
            poi_list = self.send_request(self.categories[choice-1], 0.00, 0.00)
            self.make_poi_selection(poi_list)
        else:
            self.display("Sorry, you chose an invalid category")
        self.ser.write("\xFE\x46")
        self.ser.close()

instance = ParkingMeter()
instance.start()<|endoftext|>


<commit_before>#!/usr/bin/env python

r"""
This module provides valuable argument processing functions like gen_get_options and sprint_args.
"""

import sys
try:
    import __builtin__
except ImportError:
    import builtins as __builtin__
import atexit
import signal
import argparse

import gen_print as gp
import gen_valid as gv

default_string = '  The default value is "%(default)s".'

def gen_get_options(parser,
                    stock_list=[]):
    r"""
    Parse the command line arguments using the parser object passed and return True/False (i.e. pass/fail).
    However, if gv.exit_on_error is set, simply exit the program on failure.  Also set the following built in
    values:

    __builtin__.quiet      This value is used by the qprint functions.
    __builtin__.test_mode  This value is used by command processing functions.
    __builtin__.debug      This value is used by the dprint functions.
    __builtin__.arg_obj    This value is used by print_program_header, etc.
    __builtin__.parser     This value is used by print_program_header, etc.

    Description of arguments:
    parser                          A parser object.  See argparse module documentation for details.
    stock_list                      The caller can use this parameter to request certain stock parameters
                                    offered by this function.  For example, this function will define a
                                    "quiet" option upon request.  This includes stop help text and parm
                                    checking.  The stock_list is a list of tuples each of which consists of
                                    an arg_name and a default value.  Example: stock_list = [("test_mode",
                                    0), ("quiet", 1), ("debug", 0)]
    """

    # This is a list of stock parms that we support.
    master_stock_list = ["quiet", "test_mode", "debug", "loglevel"]

    # Process stock_list.
    for ix in range(0, len(stock_list)):
        if len(stock_list[ix]) < 1:
            error_message = "Programmer error - stock_list[" + str(ix) +\
                            "] is supposed to be a tuple containing at" +\
                            " least one element which is the name of" +\
                            " the desired stock parameter:\n" +\
                            gp.sprint_var(stock_list)
            return gv.process_error_message(error_message)
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None

        if arg_name not in master_stock_list:
            error_message = "Programmer error - arg_name \"" + arg_name +\
                            "\" not found found in stock list:\n" +\
                            gp.sprint_var(master_stock_list)
            return gv.process_error_message(error_message)

        if arg_name == "quiet":
            if default is None:
                default = 0
            parser.add_argument(
                '--quiet',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s'
                     + ' will print only essential information, i.e. it will'
                     + ' not echo parameters, echo commands, print the total'
                     + ' run time, etc.' + default_string)
        elif arg_name == "test_mode":
            if default is None:
                default = 0
            parser.add_argument(
                '--test_mode',
                default=default,
                type=int,
                choices=[1, 0],
                help='This means that %(prog)s should go through all the'
                     + ' motions but not actually do anything substantial.'
                     + '  This is mainly to be used by the developer of'
                     + ' %(prog)s.' + default_string)
        elif arg_name == "debug":
            if default is None:
                default = 0
            parser.add_argument(
                '--debug',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)
        elif arg_name == "loglevel":
            if default is None:
                default = "info"
            parser.add_argument(
                '--loglevel',
                default=default,
                type=str,
                choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL',
                         'debug', 'info', 'warning', 'error', 'critical'],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)

    arg_obj = parser.parse_args()

    __builtin__.quiet = 0
    __builtin__.test_mode = 0
    __builtin__.debug = 0
    __builtin__.loglevel = 'WARNING'
    for ix in range(0, len(stock_list)):
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None
        if arg_name == "quiet":
            __builtin__.quiet = arg_obj.quiet
        elif arg_name == "test_mode":
            __builtin__.test_mode = arg_obj.test_mode
        elif arg_name == "debug":
            __builtin__.debug = arg_obj.debug
        elif arg_name == "loglevel":
            __builtin__.loglevel = arg_obj.loglevel

    __builtin__.arg_obj = arg_obj
    __builtin__.parser = parser

    # For each command line parameter, create a corresponding global variable and assign it the appropriate
    # value.  For example, if the command line contained "--last_name='Smith', we'll create a global variable
    # named "last_name" with the value "Smith".
    module = sys.modules['__main__']
    for key in arg_obj.__dict__:
        setattr(module, key, getattr(__builtin__.arg_obj, key))

    return True

def set_pgm_arg(var_value,
                var_name=None):
    r"""
    Set the value of the arg_obj.__dict__ entry named in var_name with the var_value provided.  Also, set
    corresponding global variable.

    Description of arguments:
    var_value                       The value to set in the variable.
    var_name                        The name of the variable to set.  This defaults to the name of the
                                    variable used for var_value when calling this function.
    """

    if var_name is None:
        var_name = gp.get_arg_name(None, 1, 2)

    arg_obj.__dict__[var_name] = var_value
    module = sys.modules['__main__']
    setattr(module, var_name, var_value)
    if var_name == "quiet":
        __builtin__.quiet = var_value
    elif var_name == "debug":
        __builtin__.debug = var_value
    elif var_name == "test_mode":
        __builtin__.test_mode = var_value

def sprint_args(arg_obj,
                indent=0):
    r"""
    sprint_var all of the arguments found in arg_obj and return the result as a string.

    Description of arguments:
    arg_obj                         An argument object such as is returned by the argparse parse_args()
                                    method.
    indent                          The number of spaces to indent each line of output.
    """

    col1_width = gp.dft_col1_width + indent

    buffer = ""
    for key in arg_obj.__dict__:
        buffer += gp.sprint_varx(key, getattr(arg_obj, key), 0, indent,
                                 col1_width)
    return buffer

module = sys.modules["__main__"]

def gen_exit_function(signal_number=0,
                      frame=None):
    r"""
    Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT).
    """

    gp.dprint_executing()
    gp.dprint_var(signal_number)

    # Call the main module's exit_function if it is defined.
    exit_function = getattr(module, "exit_function", None)
    if exit_function:
        exit_function(signal_number, frame)

    gp.qprint_pgm_footer()

def gen_signal_handler(signal_number,
                       frame):
    r"""
    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the program would terminate immediately
    with return code 143 and without calling the exit_function.
    """

    # The convention is to set up exit_function with atexit.register() so there is no need to explicitly
    # call exit_function from here.

    gp.dprint_executing()

    # Calling exit prevents control from returning to the code that was running when the signal was received.
    exit(0)

def gen_post_validation(exit_function=None,
                        signal_handler=None):
    r"""
    Do generic post-validation processing.  By "post", we mean that this is to be called from a validation
    function after the caller has done any validation desired.  If the calling program passes exit_function
    and signal_handler parms, this function will register them.  In other words, it will make the
    signal_handler functions get called for SIGINT and SIGTERM and will make the exit_function function run
    prior to the termination of the program.

    Description of arguments:
    exit_function                   A function object pointing to the caller's exit function.  This defaults
                                    to this module's gen_exit_function.
    signal_handler                  A function object pointing to the caller's signal_handler function.  This
                                    defaults to this module's gen_signal_handler.
    """

    # Get defaults.
    exit_function = exit_function or gen_exit_function
    signal_handler = signal_handler or gen_signal_handler

    atexit.register(exit_function)
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

def gen_setup():
    r"""
    Do general setup for a program.
    """

    # Set exit_on_error for gen_valid functions.
    gv.set_exit_on_error(True)

    # Get main module variable values.
    parser = getattr(module, "parser")
    stock_list = getattr(module, "stock_list")
    validate_parms = getattr(module, "validate_parms", None)

    gen_get_options(parser, stock_list)

    if validate_parms:
        validate_parms()
    gen_post_validation()

    gp.qprint_pgm_header()
<commit_msg>gen_exit_function: set ignore_err to 1<commit_after>#!/usr/bin/env python

r"""
This module provides valuable argument processing functions like gen_get_options and sprint_args.
"""

import sys
try:
    import __builtin__
except ImportError:
    import builtins as __builtin__
import atexit
import signal
import argparse

import gen_print as gp
import gen_valid as gv

default_string = '  The default value is "%(default)s".'

def gen_get_options(parser,
                    stock_list=[]):
    r"""
    Parse the command line arguments using the parser object passed and return True/False (i.e. pass/fail).
    However, if gv.exit_on_error is set, simply exit the program on failure.  Also set the following built in
    values:

    __builtin__.quiet      This value is used by the qprint functions.
    __builtin__.test_mode  This value is used by command processing functions.
    __builtin__.debug      This value is used by the dprint functions.
    __builtin__.arg_obj    This value is used by print_program_header, etc.
    __builtin__.parser     This value is used by print_program_header, etc.

    Description of arguments:
    parser                          A parser object.  See argparse module documentation for details.
    stock_list                      The caller can use this parameter to request certain stock parameters
                                    offered by this function.  For example, this function will define a
                                    "quiet" option upon request.  This includes stop help text and parm
                                    checking.  The stock_list is a list of tuples each of which consists of
                                    an arg_name and a default value.  Example: stock_list = [("test_mode",
                                    0), ("quiet", 1), ("debug", 0)]
    """

    # This is a list of stock parms that we support.
    master_stock_list = ["quiet", "test_mode", "debug", "loglevel"]

    # Process stock_list.
    for ix in range(0, len(stock_list)):
        if len(stock_list[ix]) < 1:
            error_message = "Programmer error - stock_list[" + str(ix) +\
                            "] is supposed to be a tuple containing at" +\
                            " least one element which is the name of" +\
                            " the desired stock parameter:\n" +\
                            gp.sprint_var(stock_list)
            return gv.process_error_message(error_message)
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None

        if arg_name not in master_stock_list:
            error_message = "Programmer error - arg_name \"" + arg_name +\
                            "\" not found found in stock list:\n" +\
                            gp.sprint_var(master_stock_list)
            return gv.process_error_message(error_message)

        if arg_name == "quiet":
            if default is None:
                default = 0
            parser.add_argument(
                '--quiet',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s'
                     + ' will print only essential information, i.e. it will'
                     + ' not echo parameters, echo commands, print the total'
                     + ' run time, etc.' + default_string)
        elif arg_name == "test_mode":
            if default is None:
                default = 0
            parser.add_argument(
                '--test_mode',
                default=default,
                type=int,
                choices=[1, 0],
                help='This means that %(prog)s should go through all the'
                     + ' motions but not actually do anything substantial.'
                     + '  This is mainly to be used by the developer of'
                     + ' %(prog)s.' + default_string)
        elif arg_name == "debug":
            if default is None:
                default = 0
            parser.add_argument(
                '--debug',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)
        elif arg_name == "loglevel":
            if default is None:
                default = "info"
            parser.add_argument(
                '--loglevel',
                default=default,
                type=str,
                choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL',
                         'debug', 'info', 'warning', 'error', 'critical'],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)

    arg_obj = parser.parse_args()

    __builtin__.quiet = 0
    __builtin__.test_mode = 0
    __builtin__.debug = 0
    __builtin__.loglevel = 'WARNING'
    for ix in range(0, len(stock_list)):
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None
        if arg_name == "quiet":
            __builtin__.quiet = arg_obj.quiet
        elif arg_name == "test_mode":
            __builtin__.test_mode = arg_obj.test_mode
        elif arg_name == "debug":
            __builtin__.debug = arg_obj.debug
        elif arg_name == "loglevel":
            __builtin__.loglevel = arg_obj.loglevel

    __builtin__.arg_obj = arg_obj
    __builtin__.parser = parser

    # For each command line parameter, create a corresponding global variable and assign it the appropriate
    # value.  For example, if the command line contained "--last_name='Smith', we'll create a global variable
    # named "last_name" with the value "Smith".
    module = sys.modules['__main__']
    for key in arg_obj.__dict__:
        setattr(module, key, getattr(__builtin__.arg_obj, key))

    return True

def set_pgm_arg(var_value,
                var_name=None):
    r"""
    Set the value of the arg_obj.__dict__ entry named in var_name with the var_value provided.  Also, set
    corresponding global variable.

    Description of arguments:
    var_value                       The value to set in the variable.
    var_name                        The name of the variable to set.  This defaults to the name of the
                                    variable used for var_value when calling this function.
    """

    if var_name is None:
        var_name = gp.get_arg_name(None, 1, 2)

    arg_obj.__dict__[var_name] = var_value
    module = sys.modules['__main__']
    setattr(module, var_name, var_value)
    if var_name == "quiet":
        __builtin__.quiet = var_value
    elif var_name == "debug":
        __builtin__.debug = var_value
    elif var_name == "test_mode":
        __builtin__.test_mode = var_value

def sprint_args(arg_obj,
                indent=0):
    r"""
    sprint_var all of the arguments found in arg_obj and return the result as a string.

    Description of arguments:
    arg_obj                         An argument object such as is returned by the argparse parse_args()
                                    method.
    indent                          The number of spaces to indent each line of output.
    """

    col1_width = gp.dft_col1_width + indent

    buffer = ""
    for key in arg_obj.__dict__:
        buffer += gp.sprint_varx(key, getattr(arg_obj, key), 0, indent,
                                 col1_width)
    return buffer

module = sys.modules["__main__"]

def gen_exit_function(signal_number=0,
                      frame=None):
    r"""
    Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT).
    """

    gp.dprint_executing()
    gp.dprint_var(signal_number)

    # ignore_err influences the way shell_cmd processes errors.  Since we're doing exit processing, we don't
    # want to stop the program due to a shell_cmd failure.
    ignore_err = 1

    # Call the main module's exit_function if it is defined.
    exit_function = getattr(module, "exit_function", None)
    if exit_function:
        exit_function(signal_number, frame)

    gp.qprint_pgm_footer()

def gen_signal_handler(signal_number,
                       frame):
    r"""
    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the program would terminate immediately
    with return code 143 and without calling the exit_function.
    """

    # The convention is to set up exit_function with atexit.register() so there is no need to explicitly
    # call exit_function from here.

    gp.dprint_executing()

    # Calling exit prevents control from returning to the code that was running when the signal was received.
    exit(0)

def gen_post_validation(exit_function=None,
                        signal_handler=None):
    r"""
    Do generic post-validation processing.  By "post", we mean that this is to be called from a validation
    function after the caller has done any validation desired.  If the calling program passes exit_function
    and signal_handler parms, this function will register them.  In other words, it will make the
    signal_handler functions get called for SIGINT and SIGTERM and will make the exit_function function run
    prior to the termination of the program.

    Description of arguments:
    exit_function                   A function object pointing to the caller's exit function.  This defaults
                                    to this module's gen_exit_function.
    signal_handler                  A function object pointing to the caller's signal_handler function.  This
                                    defaults to this module's gen_signal_handler.
    """

    # Get defaults.
    exit_function = exit_function or gen_exit_function
    signal_handler = signal_handler or gen_signal_handler

    atexit.register(exit_function)
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

def gen_setup():
    r"""
    Do general setup for a program.
    """

    # Set exit_on_error for gen_valid functions.
    gv.set_exit_on_error(True)

    # Get main module variable values.
    parser = getattr(module, "parser")
    stock_list = getattr(module, "stock_list")
    validate_parms = getattr(module, "validate_parms", None)

    gen_get_options(parser, stock_list)

    if validate_parms:
        validate_parms()
    gen_post_validation()

    gp.qprint_pgm_header()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import urllib2
import re
import collections

page = 10

url = 'http://movie.douban.com/top250?start=%d&filter=&format='

#title_re = re.compile(r'>\n                            (.+?)')
#country_re = re.compile(r' / (.+?) / ')
#year_re = re.compile(r'                            (.+?) / .+? / ')
#rank_re = re.compile(r'rating.+?em>(.+?)')
#info_re = re.compile(r'title">(.+?).+?title.+?bd.+?br.+?                            (.+?) / (.+?) / .+?rating.+?(.+?)', re.DOTALL)
info_re = re.compile(r'<img alt="(.+?)".+?bd.+?br.+?                            (.+?) / (.+?) / .+?rating.+?(.+?)', re.DOTALL)
movies = []
for i in range(0, page):
    page_url = url %(i*25)
    html = urllib2.urlopen(page_url).read()
    #movie = []
    for x in info_re.findall(html):
        tmp = []
        tmp.append(x[0])
        tmp.append(x[1])
        tmp.append(x[2])
        tmp.append(x[3])
        movies.append(tmp)
        #break
    #for x in title_re.findall(html):
        #print x
    #for y in country_re.findall(html):
        #print y
  #  for z in year_re.findall(html):
      #  index += 1
       # movie1.append(z)
        #print z
    #for w in rank_re.findall(html):
        #print w
for item in movies:
    print '%s\t\t\t%s\t\t\t%s\t\t\t%s' %(item[0].decode('utf-8'), item[1].decode('utf-8'), item[2].decode('utf-8'), item[3].decode('utf-8'))

trend = {}
for item in movies:
    if item[1] in trend:
        trend[item[1]] += 1
    else:
        trend[item[1]] = 1

#collections.OrderedDict(sorted(trend.items(), key=lambda t:t[1]))

#for name, address in trend.items():
    #print 'year %s has %d great moives' %(name, address)
year = 1895
while year < 2014:
    if str(year) in trend:
        print 'year %d has %d great moives' %(year, trend[str(year)])
    year = year + 1
print

<commit_msg>delete useless lines<commit_after># -*- coding: utf-8 -*-

import urllib2
import re
import collections

page = 10

url = 'http://movie.douban.com/top250?start=%d&filter=&format='

info_re = re.compile(r'<img alt="(.+?)".+?bd.+?br.+?                            (.+?) / (.+?) / .+?rating.+?(.+?)', re.DOTALL)
movies = []
for i in range(0, page):
    page_url = url %(i*25)
    html = urllib2.urlopen(page_url).read()
    for x in info_re.findall(html):
        tmp = []
        tmp.append(x[0])
        tmp.append(x[1])
        tmp.append(x[2])
        tmp.append(x[3])
        movies.append(tmp)

for item in movies:
    print '%s\t\t\t%s\t\t\t%s\t\t\t%s' %(item[0].decode('utf-8'), item[1].decode('utf-8'), item[2].decode('utf-8'), item[3].decode('utf-8'))

trend = {}
for item in movies:
    if item[1] in trend:
        trend[item[1]] += 1
    else:
        trend[item[1]] = 1

year = 1895
while year < 2014:
    if str(year) in trend:
        print 'year %d has %d great moives' %(year, trend[str(year)])
    year = year + 1
print

<|endoftext|>