<commit_before>"""
@brief      test tree node (time=10s)
"""

import sys
import os
import unittest
import re
import shutil
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper import fLOG, get_temp_folder
from src.pyquickhelper.pycode.build_helper import get_build_script, get_script_command, get_extra_script_command, _default_nofolder
from src.pyquickhelper.pycode.setup_helper import write_pyproj

class TestBuildScript(unittest.TestCase):

    def test_build_script(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.platform.startswith("win") and sys.version_info[0] != 2:
            sc = get_build_script("pyquickhelper")
            # fLOG(sc)
            ver = "%d%s" % sys.version_info[:2]
            assert "c:\\Python{0}_x64vir%virtual_env_suffix%\\install".format(
                ver) in sc
            lines = sc.split("\n")
            for line in lines:
                if "__" in line and _default_nofolder not in line:
                    raise Exception("issue with __ in:\n" + line)

            scc = get_script_command(
                "unittest", "pyquickhelper", requirements=[])
            assert "setup.py" in scc
            assert "__" not in scc

            sccc = get_extra_script_command(
                "local_pypi", "pyquickhelper", port=8067, requirements=[])
            assert "python" in sccc
            if "__" in sccc:
                raise Exception(sccc)
        else:
            # not yet implemented for this platform
            return

    def test_build_script_all(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        project_var_name = "pyquickhelper"
        requirements = None
        port = 8067

        if sys.platform.startswith("win"):

            for c in {"build_script", "clean_space",
                      "write_version", "clean_pyd",
                      "build_sphinx", "unittests",
                      "unittests_LONG", "unittests_SKIP",
                      "copy27", "test_local_pypi",
                      "setup_hook"}:
                sc = get_script_command(
                    c, project_var_name, requirements=requirements, port=port)
                assert len(sc) > 0
                assert "__" not in sc

            unit_test_folder = os.path.abspath(
                os.path.join(os.path.dirname(__file__), ".."))
            for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
                      "build27", "setupdep", "copy_dist",
                      "any_setup_command", "build_dist"}:
                sc = get_extra_script_command(
                    c, project_var_name, requirements=requirements, port=port,
                    unit_test_folder=unit_test_folder)
                assert len(sc) > 0
                if "__" in sc:
                    raise Exception(sc)
                if c == "run27":
                    if "nosetest" not in sc:
                        raise Exception(sc)
        else:
            # not yet implemented for this platform
            return

    def test_build_pyproj(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_pyproj")
        root = os.path.normpath(os.path.join(temp, "..", "..", ".."))

        if sys.version_info[0] == 2:
            # not available
            return

        write_pyproj(root, temp)

        with open(os.path.join(temp, "ptvs_project.pyproj"), "r", encoding="utf8") as f:
            content = f.read()
        assert "build\\" not in content
        assert "setup_helper.py" in content

if __name__ == "__main__":
    unittest.main()
<commit_msg>better exception in unit test<commit_after>"""
@brief      test tree node (time=10s)
"""

import sys
import os
import unittest
import re
import shutil
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper import fLOG, get_temp_folder
from src.pyquickhelper.pycode.build_helper import get_build_script, get_script_command, get_extra_script_command, _default_nofolder
from src.pyquickhelper.pycode.setup_helper import write_pyproj

class TestBuildScript(unittest.TestCase):

    def test_build_script(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.platform.startswith("win") and sys.version_info[0] != 2:
            sc = get_build_script("pyquickhelper")
            # fLOG(sc)
            ver = "%d%s" % sys.version_info[:2]
            if "c:\\Python{0}_x64vir%virtual_env_suffix%\\install".format(
                ver) not in sc:
                raise Exception("c:\\Python{0}_x64vir%virtual_env_suffix%\\install".format(ver))
            lines = sc.split("\n")
            for line in lines:
                if "__" in line and _default_nofolder not in line:
                    raise Exception("issue with __ in:\n" + line)

            scc = get_script_command(
                "unittest", "pyquickhelper", requirements=[])
            assert "setup.py" in scc
            assert "__" not in scc

            sccc = get_extra_script_command(
                "local_pypi", "pyquickhelper", port=8067, requirements=[])
            assert "python" in sccc
            if "__" in sccc:
                raise Exception(sccc)
        else:
            # not yet implemented for this platform
            return

    def test_build_script_all(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        project_var_name = "pyquickhelper"
        requirements = None
        port = 8067

        if sys.platform.startswith("win"):

            for c in {"build_script", "clean_space",
                      "write_version", "clean_pyd",
                      "build_sphinx", "unittests",
                      "unittests_LONG", "unittests_SKIP",
                      "copy27", "test_local_pypi",
                      "setup_hook"}:
                sc = get_script_command(
                    c, project_var_name, requirements=requirements, port=port)
                assert len(sc) > 0
                if "__" in sc:
                    raise Exception(sc)

            unit_test_folder = os.path.abspath(
                os.path.join(os.path.dirname(__file__), ".."))
            for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
                      "build27", "setupdep", "copy_dist",
                      "any_setup_command", "build_dist"}:
                sc = get_extra_script_command(
                    c, project_var_name, requirements=requirements, port=port,
                    unit_test_folder=unit_test_folder)
                assert len(sc) > 0
                if "__" in sc:
                    raise Exception(sc)
                if c == "run27":
                    if "nosetest" not in sc:
                        raise Exception(sc)
        else:
            # not yet implemented for this platform
            return

    def test_build_pyproj(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_pyproj")
        root = os.path.normpath(os.path.join(temp, "..", "..", ".."))

        if sys.version_info[0] == 2:
            # not available
            return

        write_pyproj(root, temp)

        with open(os.path.join(temp, "ptvs_project.pyproj"), "r", encoding="utf8") as f:
            content = f.read()
        if "build\\" in content:
            raise Exception(content)
        if "setup_helper.py" not in content:
            raise Exception(content)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import boto
from dataactvalidator.filestreaming.csvAbstractReader import CsvAbstractReader

class CsvS3Reader(CsvAbstractReader):
    """
    Reads data from S3 CSV file
    """

    def openFile(self, region, bucket, filename, csvSchema, bucketName, errorFilename):
        """ Opens file and prepares to read each record, mapping entries to specified column names
        Args:
            bucket : the S3 Bucket
            filename: The file path for the CSV file in S3
        Returns:
        """
        s3connection = boto.s3.connect_to_region(region)
        s3Bucket = s3connection.lookup(bucket)
        self.s3File = s3Bucket.lookup(filename)
        self.isLocal = False
        if not self.s3File:
            raise ValueError("Filename {} not found on S3".format(filename))

        super(CsvS3Reader, self).openFile(
            region, bucket, filename, csvSchema, bucketName, errorFilename)

    def downloadFile(self, region, bucket, filename, targetLocation):
        """ After opening a file, download entire file to filename """
        s3connection = boto.s3.connect_to_region(region)
        s3Bucket = s3connection.lookup(bucket)
        if not s3Bucket:
            raise ValueError("Bucket {} not found in region {}".format(
                bucket, region))
        self.s3File = s3Bucket.get_key(filename)
        if not self.s3File:
            raise ValueError("Filename {} not found in bucket {}".format(
                filename, bucket))
        self.s3File.get_contents_to_filename(targetLocation)

    def close(self):
        """ Don't need to close file when streaming from S3 """
        pass

    def _getFileSize(self):
        """
        Gets the size of the file
        """
        return self.s3File.size

    def _getNextPacket(self):
        """
        Gets the next packet from the file returns true if successful
        """
        offsetCheck = self.packetCounter * CsvAbstractReader.BUFFER_SIZE
        header = {'Range': 'bytes={}-{}'.format(
            offsetCheck, offsetCheck + CsvAbstractReader.BUFFER_SIZE - 1)}
        try:
            packet = self.s3File.get_contents_as_string(headers=header)
            return True, packet
        except:
            return False, ""
<commit_msg>better None checking<commit_after>import boto
from dataactvalidator.filestreaming.csvAbstractReader import CsvAbstractReader

class CsvS3Reader(CsvAbstractReader):
    """
    Reads data from S3 CSV file
    """

    def openFile(self, region, bucket, filename, csvSchema, bucketName, errorFilename):
        """ Opens file and prepares to read each record, mapping entries to specified column names
        Args:
            bucket : the S3 Bucket
            filename: The file path for the CSV file in S3
        Returns:
        """
        s3connection = boto.s3.connect_to_region(region)
        s3Bucket = s3connection.lookup(bucket)
        self.s3File = s3Bucket.lookup(filename)
        self.isLocal = False
        if self.s3File is None:
            raise ValueError("Filename {} not found on S3".format(filename))

        super(CsvS3Reader, self).openFile(
            region, bucket, filename, csvSchema, bucketName, errorFilename)

    def downloadFile(self, region, bucket, filename, targetLocation):
        """ After opening a file, download entire file to filename """
        s3connection = boto.s3.connect_to_region(region)
        s3Bucket = s3connection.lookup(bucket)
        if not s3Bucket:
            raise ValueError("Bucket {} not found in region {}".format(
                bucket, region))
        self.s3File = s3Bucket.get_key(filename)
        if not self.s3File:
            raise ValueError("Filename {} not found in bucket {}".format(
                filename, bucket))
        self.s3File.get_contents_to_filename(targetLocation)

    def close(self):
        """ Don't need to close file when streaming from S3 """
        pass

    def _getFileSize(self):
        """
        Gets the size of the file
        """
        return self.s3File.size

    def _getNextPacket(self):
        """
        Gets the next packet from the file returns true if successful
        """
        offsetCheck = self.packetCounter * CsvAbstractReader.BUFFER_SIZE
        header = {'Range': 'bytes={}-{}'.format(
            offsetCheck, offsetCheck + CsvAbstractReader.BUFFER_SIZE - 1)}
        try:
            packet = self.s3File.get_contents_as_string(headers=header)
            return True, packet
        except:
            return False, ""
<|endoftext|>


<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Runs a ResNet model on the ImageNet dataset using custom training loops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app as absl_app
from absl import flags
from absl import logging
import tensorflow as tf

from official.resnet.ctl import ctl_common
from official.vision.image_classification import imagenet_preprocessing
from official.vision.image_classification import common
from official.vision.image_classification import resnet_model
from official.utils.flags import core as flags_core
from official.utils.logs import logger
from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.utils.misc import model_helpers

def build_stats(train_result, eval_result, time_callback):
  """Normalizes and returns dictionary of stats.

  Args:
    train_result: The final loss at training time.
    eval_result: Output of the eval step. Assumes first value is eval_loss and
      second value is accuracy_top_1.
    time_callback: Time tracking callback instance.

  Returns:
    Dictionary of normalized results.
  """
  stats = {}

  if eval_result:
    stats['eval_loss'] = eval_result[0]
    stats['eval_acc'] = eval_result[1]

    stats['train_loss'] = train_result[0]
    stats['train_acc'] = train_result[1]

  if time_callback:
    timestamp_log = time_callback.timestamp_log
    stats['step_timestamp_log'] = timestamp_log
    stats['train_finish_time'] = time_callback.train_finish_time
    if len(timestamp_log) > 1:
      stats['avg_exp_per_second'] = (
          time_callback.batch_size * time_callback.log_steps *
          (len(time_callback.timestamp_log) - 1) /
          (timestamp_log[-1].timestamp - timestamp_log[0].timestamp))

  return stats

def get_input_dataset(flags_obj, strategy):
  """Returns the test and train input datasets."""
  dtype = flags_core.get_tf_dtype(flags_obj)
  if flags_obj.use_synthetic_data:
    input_fn = common.get_synth_input_fn(
        height=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
        width=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
        num_channels=imagenet_preprocessing.NUM_CHANNELS,
        num_classes=imagenet_preprocessing.NUM_CLASSES,
        dtype=dtype,
        drop_remainder=True)
  else:
    input_fn = imagenet_preprocessing.input_fn

  train_ds = input_fn(
      is_training=True,
      data_dir=flags_obj.data_dir,
      batch_size=flags_obj.batch_size,
      parse_record_fn=imagenet_preprocessing.parse_record,
      datasets_num_private_threads=flags_obj.datasets_num_private_threads,
      dtype=dtype)

  if strategy:
    train_ds = strategy.experimental_distribute_dataset(train_ds)

  test_ds = None
  if not flags_obj.skip_eval:
    test_ds = input_fn(
        is_training=False,
        data_dir=flags_obj.data_dir,
        batch_size=flags_obj.batch_size,
        parse_record_fn=imagenet_preprocessing.parse_record,
        dtype=dtype)

    if strategy:
      test_ds = strategy.experimental_distribute_dataset(test_ds)

  return train_ds, test_ds

def get_num_train_iterations(flags_obj):
  """Returns the number of training stesps, train and test epochs."""
  train_steps = (
      imagenet_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size)
  train_epochs = flags_obj.train_epochs

  if flags_obj.train_steps:
    train_steps = min(flags_obj.train_steps, train_steps)
    train_epochs = 1

  eval_steps = (
      imagenet_preprocessing.NUM_IMAGES['validation'] // flags_obj.batch_size)

  return train_steps, train_epochs, eval_steps

def run(flags_obj):
  """Run ResNet ImageNet training and eval loop using custom training loops.

  Args:
    flags_obj: An object containing parsed flag values.

  Raises:
    ValueError: If fp16 is passed as it is not currently supported.

  Returns:
    Dictionary of training and eval stats.
  """
  keras_utils.set_session_config(
      enable_eager=flags_obj.enable_eager,
      enable_xla=flags_obj.enable_xla)

  # TODO(anj-s): Set data_format without using Keras.
  data_format = flags_obj.data_format
  if data_format is None:
    data_format = ('channels_first'
                   if tf.test.is_built_with_cuda() else 'channels_last')
  tf.keras.backend.set_image_data_format(data_format)

  strategy = distribution_utils.get_distribution_strategy(
      distribution_strategy=flags_obj.distribution_strategy,
      num_gpus=flags_obj.num_gpus,
      num_workers=distribution_utils.configure_cluster(),
      all_reduce_alg=flags_obj.all_reduce_alg,
      num_packs=flags_obj.num_packs)

  train_ds, test_ds = get_input_dataset(flags_obj, strategy)
  train_steps, train_epochs, eval_steps = get_num_train_iterations(flags_obj)

  time_callback = keras_utils.TimeHistory(flags_obj.batch_size,
                                          flags_obj.log_steps)

  strategy_scope = distribution_utils.get_strategy_scope(strategy)
  with strategy_scope:
    model = resnet_model.resnet50(
        num_classes=imagenet_preprocessing.NUM_CLASSES,
        batch_size=flags_obj.batch_size,
        use_l2_regularizer=not flags_obj.single_l2_loss_op)

    optimizer = tf.keras.optimizers.SGD(
        learning_rate=common.BASE_LEARNING_RATE, momentum=0.9,
        nesterov=True)

    if flags_obj.fp16_implementation == "graph_rewrite":
      if not flags_obj.use_tf_function:
        raise ValueError("--fp16_implementation=graph_rewrite requires "
                         "--use_tf_function to be true")
      loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
      optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite(
                      optimizer, loss_scale)

    training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
        'training_accuracy', dtype=tf.float32)
    test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)
    test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
        'test_accuracy', dtype=tf.float32)

    trainable_variables = model.trainable_variables

    def train_step(train_ds_inputs):
      """Training StepFn."""
      def step_fn(inputs):
        """Per-Replica StepFn."""
        images, labels = inputs
        with tf.GradientTape() as tape:
          logits = model(images, training=True)

          prediction_loss = tf.keras.losses.sparse_categorical_crossentropy(
              labels, logits)
          loss = tf.reduce_sum(prediction_loss) * (1.0/ flags_obj.batch_size)
          num_replicas = tf.distribute.get_strategy().num_replicas_in_sync

          if flags_obj.single_l2_loss_op:
            filtered_variables = [
                tf.reshape(v, (-1,))
                for v in trainable_variables
                if 'bn' not in v.name
            ]
            l2_loss = resnet_model.L2_WEIGHT_DECAY * 2 * tf.nn.l2_loss(
                tf.concat(filtered_variables, axis=0))
            loss += (l2_loss / num_replicas)
          else:
            loss += (tf.reduce_sum(model.losses) / num_replicas)

          # Scale the loss
          if flags_obj.dtype == "fp16":
            loss = optimizer.get_scaled_loss(loss)

        grads = tape.gradient(loss, trainable_variables)

        # Unscale the grads
        if flags_obj.dtype == "fp16":
          grads = optimizer.get_unscaled_gradients(grads)

        optimizer.apply_gradients(zip(grads, trainable_variables))

        training_accuracy.update_state(labels, logits)
        return loss

      if strategy:
        per_replica_losses = strategy.experimental_run_v2(
            step_fn, args=(train_ds_inputs,))
        return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,
                               axis=None)
      else:
        return step_fn(train_ds_inputs)

    def test_step(test_ds_inputs):
      """Evaluation StepFn."""
      def step_fn(inputs):
        images, labels = inputs
        logits = model(images, training=False)
        loss = tf.keras.losses.sparse_categorical_crossentropy(labels,
                                                               logits)
        loss = tf.reduce_sum(loss) * (1.0/ flags_obj.batch_size)
        test_loss.update_state(loss)
        test_accuracy.update_state(labels, logits)

      if strategy:
        strategy.experimental_run_v2(step_fn, args=(test_ds_inputs,))
      else:
        step_fn(test_ds_inputs)

    if flags_obj.use_tf_function:
      train_step = tf.function(train_step)
      test_step = tf.function(test_step)

    time_callback.on_train_begin()
    for epoch in range(train_epochs):

      train_iter = iter(train_ds)
      total_loss = 0.0
      training_accuracy.reset_states()

      for step in range(train_steps):
        optimizer.lr = common.learning_rate_schedule(
            epoch, step, train_steps, flags_obj.batch_size)

        time_callback.on_batch_begin(step+epoch*train_steps)
        total_loss += train_step(next(train_iter))
        time_callback.on_batch_end(step+epoch*train_steps)

      train_loss = total_loss / train_steps
      logging.info('Training loss: %s, accuracy: %s%% at epoch: %d',
                   train_loss.numpy(),
                   training_accuracy.result().numpy(),
                   epoch)

      if (not flags_obj.skip_eval and
          (epoch + 1) % flags_obj.epochs_between_evals == 0):
        test_loss.reset_states()
        test_accuracy.reset_states()

        test_iter = iter(test_ds)
        for _ in range(eval_steps):
          test_step(next(test_iter))

        logging.info('Test loss: %s, accuracy: %s%% at epoch: %d',
                     test_loss.result().numpy(),
                     test_accuracy.result().numpy(),
                     epoch)

    time_callback.on_train_end()

    eval_result = None
    train_result = None
    if not flags_obj.skip_eval:
      eval_result = [test_loss.result().numpy(),
                     test_accuracy.result().numpy()]
      train_result = [train_loss.numpy(),
                      training_accuracy.result().numpy()]

    stats = build_stats(train_result, eval_result, time_callback)
    return stats

def main(_):
  model_helpers.apply_clean(flags.FLAGS)
  with logger.benchmark_context(flags.FLAGS):
    return run(flags.FLAGS)

if __name__ == '__main__':
  logging.set_verbosity(logging.INFO)
  common.define_keras_flags()
  ctl_common.define_ctl_flags()
  flags.adopt_module_key_flags(ctl_common)
  absl_app.run(main)
<commit_msg>fixed a bad indent<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Runs a ResNet model on the ImageNet dataset using custom training loops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import app as absl_app
from absl import flags
from absl import logging
import tensorflow as tf

from official.resnet.ctl import ctl_common
from official.vision.image_classification import imagenet_preprocessing
from official.vision.image_classification import common
from official.vision.image_classification import resnet_model
from official.utils.flags import core as flags_core
from official.utils.logs import logger
from official.utils.misc import distribution_utils
from official.utils.misc import keras_utils
from official.utils.misc import model_helpers

def build_stats(train_result, eval_result, time_callback):
  """Normalizes and returns dictionary of stats.

  Args:
    train_result: The final loss at training time.
    eval_result: Output of the eval step. Assumes first value is eval_loss and
      second value is accuracy_top_1.
    time_callback: Time tracking callback instance.

  Returns:
    Dictionary of normalized results.
  """
  stats = {}

  if eval_result:
    stats['eval_loss'] = eval_result[0]
    stats['eval_acc'] = eval_result[1]

    stats['train_loss'] = train_result[0]
    stats['train_acc'] = train_result[1]

  if time_callback:
    timestamp_log = time_callback.timestamp_log
    stats['step_timestamp_log'] = timestamp_log
    stats['train_finish_time'] = time_callback.train_finish_time
    if len(timestamp_log) > 1:
      stats['avg_exp_per_second'] = (
          time_callback.batch_size * time_callback.log_steps *
          (len(time_callback.timestamp_log) - 1) /
          (timestamp_log[-1].timestamp - timestamp_log[0].timestamp))

  return stats

def get_input_dataset(flags_obj, strategy):
  """Returns the test and train input datasets."""
  dtype = flags_core.get_tf_dtype(flags_obj)
  if flags_obj.use_synthetic_data:
    input_fn = common.get_synth_input_fn(
        height=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
        width=imagenet_preprocessing.DEFAULT_IMAGE_SIZE,
        num_channels=imagenet_preprocessing.NUM_CHANNELS,
        num_classes=imagenet_preprocessing.NUM_CLASSES,
        dtype=dtype,
        drop_remainder=True)
  else:
    input_fn = imagenet_preprocessing.input_fn

  train_ds = input_fn(
      is_training=True,
      data_dir=flags_obj.data_dir,
      batch_size=flags_obj.batch_size,
      parse_record_fn=imagenet_preprocessing.parse_record,
      datasets_num_private_threads=flags_obj.datasets_num_private_threads,
      dtype=dtype)

  if strategy:
    train_ds = strategy.experimental_distribute_dataset(train_ds)

  test_ds = None
  if not flags_obj.skip_eval:
    test_ds = input_fn(
        is_training=False,
        data_dir=flags_obj.data_dir,
        batch_size=flags_obj.batch_size,
        parse_record_fn=imagenet_preprocessing.parse_record,
        dtype=dtype)

    if strategy:
      test_ds = strategy.experimental_distribute_dataset(test_ds)

  return train_ds, test_ds

def get_num_train_iterations(flags_obj):
  """Returns the number of training stesps, train and test epochs."""
  train_steps = (
      imagenet_preprocessing.NUM_IMAGES['train'] // flags_obj.batch_size)
  train_epochs = flags_obj.train_epochs

  if flags_obj.train_steps:
    train_steps = min(flags_obj.train_steps, train_steps)
    train_epochs = 1

  eval_steps = (
      imagenet_preprocessing.NUM_IMAGES['validation'] // flags_obj.batch_size)

  return train_steps, train_epochs, eval_steps

def run(flags_obj):
  """Run ResNet ImageNet training and eval loop using custom training loops.

  Args:
    flags_obj: An object containing parsed flag values.

  Raises:
    ValueError: If fp16 is passed as it is not currently supported.

  Returns:
    Dictionary of training and eval stats.
  """
  keras_utils.set_session_config(
      enable_eager=flags_obj.enable_eager,
      enable_xla=flags_obj.enable_xla)

  # TODO(anj-s): Set data_format without using Keras.
  data_format = flags_obj.data_format
  if data_format is None:
    data_format = ('channels_first'
                   if tf.test.is_built_with_cuda() else 'channels_last')
  tf.keras.backend.set_image_data_format(data_format)

  strategy = distribution_utils.get_distribution_strategy(
      distribution_strategy=flags_obj.distribution_strategy,
      num_gpus=flags_obj.num_gpus,
      num_workers=distribution_utils.configure_cluster(),
      all_reduce_alg=flags_obj.all_reduce_alg,
      num_packs=flags_obj.num_packs)

  train_ds, test_ds = get_input_dataset(flags_obj, strategy)
  train_steps, train_epochs, eval_steps = get_num_train_iterations(flags_obj)

  time_callback = keras_utils.TimeHistory(flags_obj.batch_size,
                                          flags_obj.log_steps)

  strategy_scope = distribution_utils.get_strategy_scope(strategy)
  with strategy_scope:
    model = resnet_model.resnet50(
        num_classes=imagenet_preprocessing.NUM_CLASSES,
        batch_size=flags_obj.batch_size,
        use_l2_regularizer=not flags_obj.single_l2_loss_op)

    optimizer = tf.keras.optimizers.SGD(
        learning_rate=common.BASE_LEARNING_RATE, momentum=0.9,
        nesterov=True)

    if flags_obj.fp16_implementation == "graph_rewrite":
      if not flags_obj.use_tf_function:
        raise ValueError("--fp16_implementation=graph_rewrite requires "
                         "--use_tf_function to be true")
      loss_scale = flags_core.get_loss_scale(flags_obj, default_for_fp16=128)
      optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite(
          optimizer, loss_scale)

    training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
        'training_accuracy', dtype=tf.float32)
    test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)
    test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
        'test_accuracy', dtype=tf.float32)

    trainable_variables = model.trainable_variables

    def train_step(train_ds_inputs):
      """Training StepFn."""
      def step_fn(inputs):
        """Per-Replica StepFn."""
        images, labels = inputs
        with tf.GradientTape() as tape:
          logits = model(images, training=True)

          prediction_loss = tf.keras.losses.sparse_categorical_crossentropy(
              labels, logits)
          loss = tf.reduce_sum(prediction_loss) * (1.0/ flags_obj.batch_size)
          num_replicas = tf.distribute.get_strategy().num_replicas_in_sync

          if flags_obj.single_l2_loss_op:
            filtered_variables = [
                tf.reshape(v, (-1,))
                for v in trainable_variables
                if 'bn' not in v.name
            ]
            l2_loss = resnet_model.L2_WEIGHT_DECAY * 2 * tf.nn.l2_loss(
                tf.concat(filtered_variables, axis=0))
            loss += (l2_loss / num_replicas)
          else:
            loss += (tf.reduce_sum(model.losses) / num_replicas)

          # Scale the loss
          if flags_obj.dtype == "fp16":
            loss = optimizer.get_scaled_loss(loss)

        grads = tape.gradient(loss, trainable_variables)

        # Unscale the grads
        if flags_obj.dtype == "fp16":
          grads = optimizer.get_unscaled_gradients(grads)

        optimizer.apply_gradients(zip(grads, trainable_variables))

        training_accuracy.update_state(labels, logits)
        return loss

      if strategy:
        per_replica_losses = strategy.experimental_run_v2(
            step_fn, args=(train_ds_inputs,))
        return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,
                               axis=None)
      else:
        return step_fn(train_ds_inputs)

    def test_step(test_ds_inputs):
      """Evaluation StepFn."""
      def step_fn(inputs):
        images, labels = inputs
        logits = model(images, training=False)
        loss = tf.keras.losses.sparse_categorical_crossentropy(labels,
                                                               logits)
        loss = tf.reduce_sum(loss) * (1.0/ flags_obj.batch_size)
        test_loss.update_state(loss)
        test_accuracy.update_state(labels, logits)

      if strategy:
        strategy.experimental_run_v2(step_fn, args=(test_ds_inputs,))
      else:
        step_fn(test_ds_inputs)

    if flags_obj.use_tf_function:
      train_step = tf.function(train_step)
      test_step = tf.function(test_step)

    time_callback.on_train_begin()
    for epoch in range(train_epochs):

      train_iter = iter(train_ds)
      total_loss = 0.0
      training_accuracy.reset_states()

      for step in range(train_steps):
        optimizer.lr = common.learning_rate_schedule(
            epoch, step, train_steps, flags_obj.batch_size)

        time_callback.on_batch_begin(step+epoch*train_steps)
        total_loss += train_step(next(train_iter))
        time_callback.on_batch_end(step+epoch*train_steps)

      train_loss = total_loss / train_steps
      logging.info('Training loss: %s, accuracy: %s%% at epoch: %d',
                   train_loss.numpy(),
                   training_accuracy.result().numpy(),
                   epoch)

      if (not flags_obj.skip_eval and
          (epoch + 1) % flags_obj.epochs_between_evals == 0):
        test_loss.reset_states()
        test_accuracy.reset_states()

        test_iter = iter(test_ds)
        for _ in range(eval_steps):
          test_step(next(test_iter))

        logging.info('Test loss: %s, accuracy: %s%% at epoch: %d',
                     test_loss.result().numpy(),
                     test_accuracy.result().numpy(),
                     epoch)

    time_callback.on_train_end()

    eval_result = None
    train_result = None
    if not flags_obj.skip_eval:
      eval_result = [test_loss.result().numpy(),
                     test_accuracy.result().numpy()]
      train_result = [train_loss.numpy(),
                      training_accuracy.result().numpy()]

    stats = build_stats(train_result, eval_result, time_callback)
    return stats

def main(_):
  model_helpers.apply_clean(flags.FLAGS)
  with logger.benchmark_context(flags.FLAGS):
    return run(flags.FLAGS)

if __name__ == '__main__':
  logging.set_verbosity(logging.INFO)
  common.define_keras_flags()
  ctl_common.define_ctl_flags()
  flags.adopt_module_key_flags(ctl_common)
  absl_app.run(main)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'Submission.retired'
        db.add_column('queue_submission', 'retired',
                      self.gf('django.db.models.fields.BooleanField')(default=False),
                      keep_default=False)

    def backwards(self, orm):
        # Deleting field 'Submission.retired'
        db.delete_column('queue_submission', 'retired')

    models = {
        'queue.submission': {
            'Meta': {'object_name': 'Submission'},
            'arrival_time': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'grader_id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'grader_reply': ('django.db.models.fields.TextField', [], {}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'lms_ack': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'lms_callback_url': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'num_failures': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'pull_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            'pullkey': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'push_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            'queue_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'requester_id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'retired': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'return_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            's3_keys': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
            's3_urls': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
            'xqueue_body': ('django.db.models.fields.TextField', [], {}),
            'xqueue_header': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
        }
    }

    complete_apps = ['queue']<commit_msg>queue_submission.retired should default to True for all old rows in the table.<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'Submission.retired'
        db.add_column('queue_submission', 'retired',
                      self.gf('django.db.models.fields.BooleanField')(default=True),
                      keep_default=False)

    def backwards(self, orm):
        # Deleting field 'Submission.retired'
        db.delete_column('queue_submission', 'retired')

    models = {
        'queue.submission': {
            'Meta': {'object_name': 'Submission'},
            'arrival_time': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'grader_id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'grader_reply': ('django.db.models.fields.TextField', [], {}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'lms_ack': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'lms_callback_url': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'num_failures': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'pull_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            'pullkey': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'push_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            'queue_name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'requester_id': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'retired': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'return_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
            's3_keys': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
            's3_urls': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
            'xqueue_body': ('django.db.models.fields.TextField', [], {}),
            'xqueue_header': ('django.db.models.fields.CharField', [], {'max_length': '1024'})
        }
    }

    complete_apps = ['queue']
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Example for sending batch information to InfluxDB via UDP."""

"""
INFO: In order to use UDP, one should enable the UDP service from the
`influxdb.conf` under section
    [[udp]]
        enabled = true
        bind-address = ":8089" # port number for sending data via UDP
        database = "udp1" # name of database to be stored
    [[udp]]
        enabled = true
        bind-address = ":8090"
        database = "udp2"
"""

import argparse

from influxdb import InfluxDBClient

def main(uport):
    """Instantiate connection to the InfluxDB."""
    # NOTE: structure of the UDP packet is different than that of information
    #       sent via HTTP
    json_body = {
        "tags": {
            "host": "server01",
            "region": "us-west"
        },
        "time": "2009-11-10T23:00:00Z",
        "points": [{
            "measurement": "cpu_load_short",
            "fields": {
                "value": 0.64
            }
        },
            {
                "measurement": "cpu_load_short",
                "fields": {
                    "value": 0.67
                }
        }]
    }

    # make `use_udp` True and  add `udp_port` number from `influxdb.conf` file
    # no need to mention the database name since it is already configured
    client = InfluxDBClient(use_udp=True, udp_port=uport)

    # Instead of `write_points` use `send_packet`
    client.send_packet(json_body)

def parse_args():
    """Parse the args."""
    parser = argparse.ArgumentParser(
        description='example code to play with InfluxDB along with UDP Port')
    parser.add_argument('--uport', type=int, required=True,
                        help=' UDP port of InfluxDB')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()
    main(uport=args.uport)
<commit_msg>Refactor `tutorial_udp` example for timestamps (#808)<commit_after># -*- coding: utf-8 -*-
"""Example for sending batch information to InfluxDB via UDP."""

"""
INFO: In order to use UDP, one should enable the UDP service from the
`influxdb.conf` under section
    [[udp]]
        enabled = true
        bind-address = ":8089" # port number for sending data via UDP
        database = "udp1" # name of database to be stored
    [[udp]]
        enabled = true
        bind-address = ":8090"
        database = "udp2"
"""

import argparse

from influxdb import InfluxDBClient

def main(uport):
    """Instantiate connection to the InfluxDB."""
    # NOTE: structure of the UDP packet is different than that of information
    #       sent via HTTP
    json_body = {
        "tags": {
            "host": "server01",
            "region": "us-west"
        },
        "points": [{
            "measurement": "cpu_load_short",
            "fields": {
                "value": 0.64
            },
            "time": "2009-11-10T23:00:00Z",
        },
        {
            "measurement": "cpu_load_short",
            "fields": {
                "value": 0.67
            },
            "time": "2009-11-10T23:05:00Z"
        }]
    }

    # make `use_udp` True and  add `udp_port` number from `influxdb.conf` file
    # no need to mention the database name since it is already configured
    client = InfluxDBClient(use_udp=True, udp_port=uport)

    # Instead of `write_points` use `send_packet`
    client.send_packet(json_body)

def parse_args():
    """Parse the args."""
    parser = argparse.ArgumentParser(
        description='example code to play with InfluxDB along with UDP Port')
    parser.add_argument('--uport', type=int, required=True,
                        help=' UDP port of InfluxDB')
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()
    main(uport=args.uport)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Modelica AST definitions
"""
from __future__ import print_function, absolute_import, division, print_function, unicode_literals

import copy
import json
import sys

VALIDATE_AST = True

"""
AST Node Type Hierarchy

File
    Class
        Equation
            ComponentRef
            Expression
            Primary
        ConnectClause
            ComponentRef
        Symbol
"""

def to_json(var):
    if isinstance(var, list):
        res = [to_json(item) for item in var]
    elif isinstance(var, dict):
        res = {key: to_json(var[key]) for key in var.keys()}
    elif isinstance(var, Node):
        res = {key: to_json(var.__dict__[key]) for key in var.__dict__.keys()}
    else:
        res = var
    return res

class Field(object):
    def __init__(self, types, default=None):
        if default is None:
            if types == dict:
                default = {}
            elif types == list:
                default = []
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val, throw=True):
        if not type(val) in self.types:
            if throw:
                raise IOError('{:s}.{:s} requires types ({:s}), but got {:s}'.format(
                    name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
            else:
                return False
        else:
            return True

class List(list):
    def __init__(self, types, default=[]):
        super(List, self).__init__()
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val_list, throw=True):
        if type(val_list) != list:
            if throw:
                raise IOError('{:s}.{:s} requires types ({:s}), but got {:s}'.format(
                name, key, 'list', type(val).__name__))
            else:
                return False

        for val in val_list:
            if not type(val) in self.types:
                if throw:
                    raise IOError('{:s}.{:s} requires list items of type ({:s}), but got {:s}'.format(
                        name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
                else:
                    return False
            else:
                return True

    def __add__(self, other):
        if other in self.types:
            super(List, self).__add__(other)
        else:
            raise IOError('List requires elements of type', self.types)

class Dict(dict):
    def __init__(self, types, default=[]):
        super(Dict, self).__init__()
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val_dict, throw=True):
        if type(val_dict) != dict:
            if throw:
                raise IOError('{:s}.{:s} requires types ({:s}), but got {:s}'.format(
                name, key, 'dict', type(val).__name__))
            else:
                return False

        for val in val_dict.values():
            print('dict val', val)
            if not type(val) in self.types:
                if throw:
                    raise IOError('{:s}.{:s} requires list items of type ({:s}), but got {:s}'.format(
                        name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
                else:
                    return False
            else:
                return True

class Node(object):
    ast_spec = {}

    def __init__(self, **kwargs):
        for key in self.ast_spec.keys():
            # make sure we don't share ast_spec default data by using deep copy
            self.__dict__[key] = copy.deepcopy(self.ast_spec[key].default)
        for key in kwargs.keys():
            # types = self.ast_spec[key].types
            val = kwargs[key]
            self.set_field(key, val)

    def set_field(self, key, value):
        if VALIDATE_AST:
            name = self.__class__.__name__
            if key not in self.ast_spec.keys():
                raise IOError('{:s} not a child of {:s}'.format(key, name))
            self.ast_spec[key].validate(name, key, value)
        self.__dict__[key] = value

    def __setattr__(self, key, value):
        self.set_field(key, value)

    def __repr__(self):
        return json.dumps(to_json(self), indent=2, sort_keys=True)

    __str__ = __repr__

class Primary(Node):

    def __init__(self, **kwargs):
        super(Primary, self).__init__(**kwargs)

class ComponentRef(Node):

    def __init__(self, **kwargs):
        super(ComponentRef, self).__init__(**kwargs)

class Expression(Node):

    def __init__(self, **kwargs):
        super(Expression, self).__init__(**kwargs)

class Equation(Node):

    def __init__(self, **kwargs):
        super(Equation, self).__init__(**kwargs)

class ConnectClause(Node):

    def __init__(self, **kwargs):
        super(ConnectClause, self).__init__(**kwargs)

class Symbol(Node):
    def __init__(self, **kwargs):
        super(Symbol, self).__init__(**kwargs)

class ComponentClause(Node):
    def __init__(self, **kwargs):
        super(ComponentClause, self).__init__(**kwargs)

class EquationSection(Node):
    def __init__(self, **kwargs):
        super(EquationSection, self).__init__(**kwargs)

class Class(Node):
    def __init__(self, **kwargs):
        super(Class, self).__init__(**kwargs)

class File(Node):
    def __init__(self, **kwargs):
        super(File, self).__init__(**kwargs)

# Here we define the AST specifications for all nodes
# these are static variables shared between class instances
# and are defined here to allow a class to list itself in
# the allowed field types, this self referencing is not
# possible when initially declaring a class in python

Primary.ast_spec = {
    'value': Field((bool, float, int, str))
}

ComponentRef.ast_spec = {
    'name': Field(str),
}

Expression.ast_spec = {
    'operator': Field(str),
    'operands': List((Expression, ComponentRef)),
}

Equation.ast_spec = {
    'left': Field((Expression, Primary, ComponentRef)),
    'right': Field((Expression, Primary, ComponentRef)),
    'comment': Field(str),
}

ConnectClause.ast_spec = {
    'left': Field(ComponentRef),
    'right': Field(ComponentRef),
    'comment': Field(str),
}

Symbol.ast_spec = {
    'name': Field(str, ''),
    'type': Field(str, ''),
    'prefixes': List((str), []),
    'redeclare': Field(bool, False),
    'final': Field(bool, False),
    'inner': Field(bool, False),
    'outer': Field(bool, False),
    'dimensions': Field(list, [1]),
    'comment': Field(str, ''),
    'start': Field(Primary),
    'id': Field(int),
    'order': Field(int),
}

ComponentClause.ast_spec = {
    'prefixes': Field(list, []),
    'type': Field(str, ''),
    'dimensions': List((int), [1]),
    'comment': List((str), []),
    'symbol_list': List((Symbol), []),
}

EquationSection.ast_spec = {
    'initial': Field(bool, False),
    'equations': List((Equation, ConnectClause), []),
}

Class.ast_spec = {
    'name': Field(str),
    'encapsulated': Field(bool, False),
    'partial': Field(bool, False),
    'final': Field(bool, False),
    'type': Field(str, ''),
    'comment': Field(str, ''),
    'symbols': Dict((ComponentClause), {}),
    'initial_equations': List((Equation), []),
    'equations': List((Equation, ConnectClause), []),
}

File.ast_spec = {
    'within': Field(str),
    'classes': Dict((Class), {}),
}

<commit_msg>Added list AST validation.<commit_after>#!/usr/bin/env python
"""
Modelica AST definitions
"""
from __future__ import print_function, absolute_import, division, print_function, unicode_literals

import copy
import json
import sys

VALIDATE_AST = True

"""
AST Node Type Hierarchy

File
    Class
        Equation
            ComponentRef
            Expression
            Primary
        ConnectClause
            ComponentRef
        Symbol
"""

def to_json(var):
    if isinstance(var, list):
        res = [to_json(item) for item in var]
    elif isinstance(var, dict):
        res = {key: to_json(var[key]) for key in var.keys()}
    elif isinstance(var, Node):
        res = {key: to_json(var.__dict__[key]) for key in var.__dict__.keys()}
    else:
        res = var
    return res

class Field(object):
    def __init__(self, types, default=None):
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val, throw=True):
        if not type(val) in self.types:
            if throw:
                raise IOError('{:s}.{:s} requires types ({:s}), but got {:s}'.format(
                    name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
            else:
                return False
        else:
            return True

class FieldList(list):
    def __init__(self, types, default=[]):
        super(FieldList, self).__init__()
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val_list, throw=True):
        if type(val_list) != list:
            if throw:
                raise IOError('{:s}.{:s} requires types {:s}, but got {:s}'.format(
                name, key, 'list', type(val_list).__name__))
            else:
                return False

        for val in val_list:
            if not type(val) in self.types:
                if throw:
                    raise IOError('{:s}.{:s} requires list items of type ({:s}), but got {:s}'.format(
                        name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
                else:
                    return False
            else:
                return True

    def __add__(self, other):
        if other in self.types:
            super(FieldList, self).__add__(other)
        else:
            raise IOError('List requires elements of type', self.types)

# TODO get dict validation working
class FieldDict(dict):
    def __init__(self, types, default):
        super(FieldDict, self).__init__()
        if type(types) is type or not hasattr(types, '__iter__'):
            types = [types]
        types = list(types)
        if sys.version_info < (3,):
            if str in types:
                types += [unicode]
        self.types = types
        self.default = default

    def validate(self, name, key, val_dict, throw=True):
        if type(val_dict) != dict:
            if throw:
                raise IOError('{:s}.{:s} requires types ({:s}), but got {:s}'.format(
                name, key, 'dict', type(val).__name__))
            else:
                return False

        for val in val_dict.values():
            if not type(val) in self.types:
                if throw:
                    raise IOError('{:s}.{:s} requires list items of type ({:s}), but got {:s}'.format(
                        name, key, ','.join([t.__name__ for t in self.types]), type(val).__name__))
                else:
                    return False
            else:
                return True

    def __setitem__(self, key, value):
        if VALIDATE_AST:
            if not type(value) in self.types:
                print('type', type(value))
                raise IOError('{:s} requires dict values of type ({:s}), but got {:s}'.format(
                    key, ','.join([t.__name__ for t in self.types]), type(value).__name__))
        super(FieldDict, self).__setitem__(key, value)

class Node(object):
    ast_spec = {}

    def __init__(self, **kwargs):
        for key in self.ast_spec.keys():
            # make sure we don't share ast_spec default data by using deep copy
            field_type = type(self.ast_spec[key])
            default = self.ast_spec[key].default
            self.__dict__[key] = copy.deepcopy(default)
        for key in kwargs.keys():
            # types = self.ast_spec[key].types
            val = kwargs[key]
            self.set_field(key, val)

    def set_field(self, key, value):
        if VALIDATE_AST:
            name = self.__class__.__name__
            if key not in self.ast_spec.keys():
                raise IOError('{:s} not a child of {:s}'.format(key, name))
            self.ast_spec[key].validate(name, key, value)
        self.__dict__[key] = value

    def __setattr__(self, key, value):
        self.set_field(key, value)

    def __repr__(self):
        return json.dumps(to_json(self), indent=2, sort_keys=True)

    __str__ = __repr__

class Primary(Node):
    def __init__(self, **kwargs):
        super(Primary, self).__init__(**kwargs)

class ComponentRef(Node):
    def __init__(self, **kwargs):
        super(ComponentRef, self).__init__(**kwargs)

class Expression(Node):
    def __init__(self, **kwargs):
        super(Expression, self).__init__(**kwargs)

class Equation(Node):
    def __init__(self, **kwargs):
        super(Equation, self).__init__(**kwargs)

class ConnectClause(Node):
    def __init__(self, **kwargs):
        super(ConnectClause, self).__init__(**kwargs)

class Symbol(Node):
    def __init__(self, **kwargs):
        super(Symbol, self).__init__(**kwargs)

class ComponentClause(Node):
    def __init__(self, **kwargs):
        super(ComponentClause, self).__init__(**kwargs)

class EquationSection(Node):
    def __init__(self, **kwargs):
        super(EquationSection, self).__init__(**kwargs)

class Class(Node):
    def __init__(self, **kwargs):
        super(Class, self).__init__(**kwargs)

class File(Node):
    def __init__(self, **kwargs):
        super(File, self).__init__(**kwargs)

# Here we define the AST specifications for all nodes
# these are static variables shared between class instances
# and are defined here to allow a class to list itself in
# the allowed field types, this self referencing is not
# possible when initially declaring a class in python

Primary.ast_spec = {
    'value': Field([bool, float, int, str])
}

ComponentRef.ast_spec = {
    'name': Field([str]),
}

Expression.ast_spec = {
    'operator': Field([str]),
    'operands': FieldList([Expression, ComponentRef]),
}

Equation.ast_spec = {
    'left': Field([Expression, Primary, ComponentRef]),
    'right': Field([Expression, Primary, ComponentRef]),
    'comment': Field([str]),
}

ConnectClause.ast_spec = {
    'left': Field([ComponentRef]),
    'right': Field([ComponentRef]),
    'comment': Field([str]),
}

Symbol.ast_spec = {
    'name': Field([str], ''),
    'type': Field([str], ''),
    'prefixes': FieldList([str], []),
    'redeclare': Field([bool], False),
    'final': Field([bool], False),
    'inner': Field([bool], False),
    'outer': Field([bool], False),
    'dimensions': FieldList([int], [1]),
    'comment': Field([str], ''),
    'start': Field([Primary], Primary(value=0)),
    'id': Field([int], 0),
    'order': Field([int], 0),
}

ComponentClause.ast_spec = {
    'prefixes': FieldList([str], []),
    'type': Field([str], ''),
    'dimensions': FieldList([int], [1]),
    'comment': FieldList([str], []),
    'symbol_list': FieldList([Symbol], []),
}

EquationSection.ast_spec = {
    'initial': Field([bool], False),
    'equations': FieldList([Equation, ConnectClause], []),
}

Class.ast_spec = {
    'name': Field(str),
    'encapsulated': Field([bool], False),
    'partial': Field([bool], False),
    'final': Field([bool], False),
    'type': Field([str], ''),
    'comment': Field(str, ''),
    'symbols': FieldDict([Symbol], {}),
    'initial_equations': FieldList([Equation], []),
    'equations': FieldList([Equation, ConnectClause], []),
}

File.ast_spec = {
    'within': Field([str], ''),
    'classes': FieldDict([Class], {}),
}

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from datetime import datetime, timedelta
import uuid

from odoo import fields, models, api, _
from odoo.addons.base.models.res_partner import _tz_get
from odoo.exceptions import UserError
from odoo.tools.misc import _format_time_ago
from odoo.http import request
from odoo.osv import expression

class WebsiteTrack(models.Model):
    _name = 'website.track'
    _description = 'Visited Pages'
    _order = 'visit_datetime DESC'
    _log_access = False

    visitor_id = fields.Many2one('website.visitor', ondelete="cascade", index=True, required=True, readonly=True)
    page_id = fields.Many2one('website.page', index=True, ondelete='cascade', readonly=True)
    url = fields.Text('Url', index=True)
    visit_datetime = fields.Datetime('Visit Date', default=fields.Datetime.now, required=True, readonly=True)

class VisitorLastConnection(models.Model):
    _name = 'website.visitor.lastconnection'
    _description = 'temporary table for Visitor'

    visitor_id = fields.Many2one('website.visitor', ondelete="cascade", index=True, required=True, readonly=True)
    connection_datetime = fields.Datetime(default=fields.Datetime.now, required=True, readonly=True)

class WebsiteVisitor(models.Model):
    _name = 'website.visitor'
    _description = 'Website Visitor'
    _order = 'create_date DESC'

    name = fields.Char('Name', default=lambda self: _('Website Visitor'))
    access_token = fields.Char(required=True, default=lambda x: uuid.uuid4().hex, index=True, copy=False, groups='base.group_website_publisher')
    active = fields.Boolean('Active', default=True)
    website_id = fields.Many2one('website', "Website", readonly=True)
    user_partner_id = fields.Many2one('res.partner', string="Linked Partner", help="Partner of the last logged in user.")
    partner_image = fields.Binary(related='user_partner_id.image_1920')

    # localisation and info
    country_id = fields.Many2one('res.country', 'Country', readonly=True)
    country_flag = fields.Binary(related="country_id.image", string="Country Flag")
    lang_id = fields.Many2one('res.lang', string='Language', help="Language from the website when visitor has been created")
    timezone = fields.Selection(_tz_get, string='Timezone')
    email = fields.Char(string='Email', compute='_compute_email_phone')
    mobile = fields.Char(string='Mobile Phone', compute='_compute_email_phone')

    # Visit fields
    visit_count = fields.Integer('Number of visits', default=1, readonly=True, help="A new visit is considered if last connection was more than 24 hours ago.")
    website_track_ids = fields.One2many('website.track', 'visitor_id', string='Visited Pages History', readonly=True)
    visitor_page_count = fields.Integer('Page Views', compute="_compute_page_statistics", help="Total number of visits on tracked pages")
    page_ids = fields.Many2many('website.page', string="Visited Pages", compute="_compute_page_statistics")
    page_count = fields.Integer('# Visited Pages', compute="_compute_page_statistics", help="Total number of tracked page visited")
    last_visited_page_id = fields.Many2one('website.page', string="Last Visited Page", compute="_compute_last_visited_page_id")

    # Time fields
    create_date = fields.Datetime('First connection date', readonly=True)
    last_connection_datetime = fields.Datetime('Last Connection', compute="_compute_time_statistics", help="Last page view date", search='_search_last_connection', readonly=True)
    last_connections_ids = fields.One2many('website.visitor.lastconnection', 'visitor_id', readonly=True)
    time_since_last_action = fields.Char('Last action', compute="_compute_time_statistics", help='Time since last page view. E.g.: 2 minutes ago')
    is_connected = fields.Boolean('Is connected ?', compute='_compute_time_statistics', help='A visitor is considered as connected if his last page view was within the last 5 minutes.')

    @api.depends('name')
    def name_get(self):
        return [(
            record.id,
            '%s #%d' % (record.name, record.id) if record.name == _('Website Visitor') else record.name
        ) for record in self]

    @api.model
    def _search_last_connection(self, operator, value):
        assert operator in expression.TERM_OPERATORS
        self.env['website.visitor.lastconnection'].flush(['visitor_id'])
        query = """
            SELECT v.visitor_id
            FROM website_visitor_lastconnection v
            WHERE v.connection_datetime %s %%s
            GROUP BY v.visitor_id, v.connection_datetime
            ORDER BY v.connection_datetime
        """ % (operator,)
        return [('id', 'inselect', (query, [value]))]

    @api.depends('user_partner_id.email_normalized', 'user_partner_id.mobile')
    def _compute_email_phone(self):
        results = self.env['res.partner'].search_read(
            [('id', 'in', self.user_partner_id.ids)],
            ['id', 'email_normalized', 'mobile'],
        )
        mapped_data = {
            result['id']: {
                'email_normalized': result['email_normalized'],
                'mobile': result['mobile']
            } for result in results
        }

        for visitor in self:
            visitor.email = mapped_data.get(visitor.user_partner_id.id, {}).get('email_normalized')
            visitor.mobile = mapped_data.get(visitor.user_partner_id.id, {}).get('mobile')

    @api.depends('website_track_ids')
    def _compute_page_statistics(self):
        results = self.env['website.track'].read_group(
            [('visitor_id', 'in', self.ids), ('url', '!=', False)], ['visitor_id', 'page_id', 'url'], ['visitor_id', 'page_id', 'url'], lazy=False)
        mapped_data = {}
        for result in results:
            visitor_info = mapped_data.get(result['visitor_id'][0], {'page_count': 0, 'visitor_page_count': 0, 'page_ids': set()})
            visitor_info['visitor_page_count'] += result['__count']
            visitor_info['page_count'] += 1
            if result['page_id']:
                visitor_info['page_ids'].add(result['page_id'][0])
            mapped_data[result['visitor_id'][0]] = visitor_info

        for visitor in self:
            visitor_info = mapped_data.get(visitor.id, {'page_ids': [], 'page_count': 0})
            visitor.page_ids = [(6, 0, visitor_info['page_ids'])]
            visitor.visitor_page_count = visitor_info['visitor_page_count']
            visitor.page_count = visitor_info['page_count']

    @api.depends('website_track_ids.page_id')
    def _compute_last_visited_page_id(self):
        results = self.env['website.track'].read_group([('visitor_id', 'in', self.ids)],
                                                                        ['visitor_id', 'page_id', 'visit_datetime:max'],
                                                                        ['visitor_id', 'page_id'], lazy=False)
        mapped_data = {result['visitor_id'][0]: result['page_id'][0] for result in results if result['page_id']}
        for visitor in self:
            visitor.last_visited_page_id = mapped_data.get(visitor.id, False)

    @api.depends('last_connections_ids')
    def _compute_time_statistics(self):
        results = self.env['website.visitor.lastconnection'].read_group([('visitor_id', 'in', self.ids)], ['visitor_id', 'connection_datetime:max'], ['visitor_id'])
        mapped_data = {result['visitor_id'][0]: result['connection_datetime'] for result in results}
        for visitor in self:
            last_connection_datetime = mapped_data.get(visitor.id, False)
            if last_connection_datetime:
                visitor.last_connection_datetime = last_connection_datetime
                visitor.time_since_last_action = _format_time_ago(self.env, (datetime.now() - last_connection_datetime))
                visitor.is_connected = (datetime.now() - last_connection_datetime) < timedelta(minutes=5)

    def _prepare_visitor_send_mail_values(self):
        if self.user_partner_id.email:
            return {
                'res_model': 'res.partner',
                'res_id': self.user_partner_id.id,
                'partner_ids': [self.user_partner_id.id],
            }
        return {}

    def action_send_mail(self):
        self.ensure_one()
        visitor_mail_values = self._prepare_visitor_send_mail_values()
        if not visitor_mail_values:
            raise UserError(_("There is no email linked this visitor."))
        compose_form = self.env.ref('mail.email_compose_message_wizard_form', False)
        ctx = dict(
            default_model=visitor_mail_values.get('res_model'),
            default_res_id=visitor_mail_values.get('res_id'),
            default_use_template=False,
            default_partner_ids=[(6, 0, visitor_mail_values.get('partner_ids'))],
            default_composition_mode='comment',
            default_reply_to=self.env.user.partner_id.email,
        )
        return {
            'name': _('Compose Email'),
            'type': 'ir.actions.act_window',
            'view_mode': 'form',
            'res_model': 'mail.compose.message',
            'views': [(compose_form.id, 'form')],
            'view_id': compose_form.id,
            'target': 'new',
            'context': ctx,
        }

    def _get_visitor_from_request(self, with_previous_visitors=False):
        if not request:
            return None
        if with_previous_visitors and not request.env.user._is_public():
            partner_id = request.env.user.partner_id
            # Retrieve all the previous partner's visitors to have full history of his last products viewed.
            return request.env['website.visitor'].sudo().with_context(active_test=False).search([('user_partner_id', '=', partner_id.id)])
        else:
            visitor = self.env['website.visitor']
            access_token = request.httprequest.cookies.get('visitor_id')
            if access_token:
                visitor = visitor.sudo().with_context(active_test=False).search([('access_token', '=', access_token)])
            return visitor

    def _handle_webpage_dispatch(self, response, website_page):
        # get visitor. Done here to avoid having to do it multiple times in case of override.
        visitor_sudo = self._get_visitor_from_request()
        self._handle_website_page_visit(response, website_page, visitor_sudo)

    def _handle_website_page_visit(self, response, website_page, visitor_sudo):
        """ Called on dispatch. This will create a website.visitor if the http request object
        is a tracked website page or a tracked view. Only on tracked elements to avoid having
        too much operations done on every page or other http requests.
        Note: The side effect is that the last_connection_datetime is updated ONLY on tracked elements."""
        url = request.httprequest.url
        website_track_values = {
            'url': url,
            'visit_datetime': datetime.now(),
        }
        if website_page:
            website_track_values['page_id'] = website_page.id
            domain = [('page_id', '=', website_page.id)]
        else:
            domain = [('url', '=', url)]
        if not visitor_sudo:
            visitor_sudo = self._create_visitor(website_track_values)
            expiration_date = datetime.now() + timedelta(days=365)
            response.set_cookie('visitor_id', visitor_sudo.access_token, expires=expiration_date)
        else:
            visitor_sudo._add_tracking(domain, website_track_values)
            if visitor_sudo.lang_id.id != request.lang.id:
                visitor_sudo.write({'lang_id': request.lang.id})

    def _add_tracking(self, domain, website_track_values):
        """ Update the visitor when a website_track is added"""
        domain = expression.AND([domain, [('visitor_id', '=', self.id)]])
        last_view = self.env['website.track'].sudo().search(domain, limit=1)
        if not last_view or last_view.visit_datetime < datetime.now() - timedelta(minutes=30):
            website_track_values['visitor_id'] = self.id
            self.env['website.track'].create(website_track_values)
        self.env['website.visitor.lastconnection'].create({
            'visitor_id': self.id,
            'connection_datetime': website_track_values['visit_datetime']
        })

    def _create_visitor(self, website_track_values=None):
        """ Create a visitor and add a track to it if website_track_values is set."""
        country_code = request.session.get('geoip', {}).get('country_code', False)
        country_id = request.env['res.country'].sudo().search([('code', '=', country_code)], limit=1).id if country_code else False
        vals = {
            'lang_id': request.lang.id,
            'country_id': country_id,
            'website_id': request.website.id,
            'last_connections_ids': [(0, 0, {'connection_datetime': website_track_values['visit_datetime'] or datetime.now()})],
        }
        if not self.env.user._is_public():
            vals['user_partner_id'] = self.env.user.partner_id.id
            vals['name'] = self.env.user.partner_id.name
        if website_track_values:
            vals['website_track_ids'] = [(0, 0, website_track_values)]
        return self.sudo().create(vals)

    def _cron_archive_visitors(self):
        self.flush(['visit_count'])
        yesterday = datetime.now() - timedelta(days=1)
        query = """
            UPDATE website_visitor
            SET visit_count = visit_count + 1
            WHERE id in (
                SELECT visitor_id
                FROM website_visitor_lastconnection
                GROUP BY visitor_id
                HAVING COUNT(*) > 1)
        """
        self.env.cr.execute(query, [yesterday])

        query = """
            DELETE FROM website_visitor_lastconnection
            WHERE (visitor_id, connection_datetime) not in (
               SELECT v.visitor_id, max(connection_datetime)
               FROM website_visitor_lastconnection v
               GROUP BY v.visitor_id)
        """
        self.env.cr.execute(query, [])

        one_week_ago = datetime.now() - timedelta(days=7)
        visitors_to_archive = self.env['website.visitor'].sudo().search([('last_connection_datetime', '<', one_week_ago)])
        visitors_to_archive.write({'active': False})
<commit_msg>[REV] website: revert commit 5b9a81a2<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from datetime import datetime, timedelta
import uuid

from odoo import fields, models, api, _
from odoo.addons.base.models.res_partner import _tz_get
from odoo.exceptions import UserError
from odoo.tools.misc import _format_time_ago
from odoo.http import request
from odoo.osv import expression

class WebsiteTrack(models.Model):
    _name = 'website.track'
    _description = 'Visited Pages'
    _order = 'visit_datetime DESC'
    _log_access = False

    visitor_id = fields.Many2one('website.visitor', ondelete="cascade", index=True, required=True, readonly=True)
    page_id = fields.Many2one('website.page', index=True, ondelete='cascade', readonly=True)
    url = fields.Text('Url', index=True)
    visit_datetime = fields.Datetime('Visit Date', default=fields.Datetime.now, required=True, readonly=True)

class VisitorLastConnection(models.Model):
    _name = 'website.visitor.lastconnection'
    _description = 'temporary table for Visitor'

    visitor_id = fields.Many2one('website.visitor', ondelete="cascade", index=True, required=True, readonly=True)
    connection_datetime = fields.Datetime(default=fields.Datetime.now, required=True, readonly=True)

class WebsiteVisitor(models.Model):
    _name = 'website.visitor'
    _description = 'Website Visitor'
    _order = 'create_date DESC'

    name = fields.Char('Name', default=_('Website Visitor'))
    access_token = fields.Char(required=True, default=lambda x: uuid.uuid4().hex, index=True, copy=False, groups='base.group_website_publisher')
    active = fields.Boolean('Active', default=True)
    website_id = fields.Many2one('website', "Website", readonly=True)
    user_partner_id = fields.Many2one('res.partner', string="Linked Partner", help="Partner of the last logged in user.")
    partner_image = fields.Binary(related='user_partner_id.image_1920')

    # localisation and info
    country_id = fields.Many2one('res.country', 'Country', readonly=True)
    country_flag = fields.Binary(related="country_id.image", string="Country Flag")
    lang_id = fields.Many2one('res.lang', string='Language', help="Language from the website when visitor has been created")
    timezone = fields.Selection(_tz_get, string='Timezone')
    email = fields.Char(string='Email', compute='_compute_email_phone')
    mobile = fields.Char(string='Mobile Phone', compute='_compute_email_phone')

    # Visit fields
    visit_count = fields.Integer('Number of visits', default=1, readonly=True, help="A new visit is considered if last connection was more than 24 hours ago.")
    website_track_ids = fields.One2many('website.track', 'visitor_id', string='Visited Pages History', readonly=True)
    visitor_page_count = fields.Integer('Page Views', compute="_compute_page_statistics", help="Total number of visits on tracked pages")
    page_ids = fields.Many2many('website.page', string="Visited Pages", compute="_compute_page_statistics")
    page_count = fields.Integer('# Visited Pages', compute="_compute_page_statistics", help="Total number of tracked page visited")
    last_visited_page_id = fields.Many2one('website.page', string="Last Visited Page", compute="_compute_last_visited_page_id")

    # Time fields
    create_date = fields.Datetime('First connection date', readonly=True)
    last_connection_datetime = fields.Datetime('Last Connection', compute="_compute_time_statistics", help="Last page view date", search='_search_last_connection', readonly=True)
    last_connections_ids = fields.One2many('website.visitor.lastconnection', 'visitor_id', readonly=True)
    time_since_last_action = fields.Char('Last action', compute="_compute_time_statistics", help='Time since last page view. E.g.: 2 minutes ago')
    is_connected = fields.Boolean('Is connected ?', compute='_compute_time_statistics', help='A visitor is considered as connected if his last page view was within the last 5 minutes.')

    @api.depends('name')
    def name_get(self):
        ret_list = []
        for record in self:
            name = '%s #%d' % (record.name, record.id)
            ret_list.append((record.id, name))
        return ret_list

    @api.model
    def _search_last_connection(self, operator, value):
        assert operator in expression.TERM_OPERATORS
        self.env['website.visitor.lastconnection'].flush(['visitor_id'])
        query = """
            SELECT v.visitor_id
            FROM website_visitor_lastconnection v
            WHERE v.connection_datetime %s %%s
            GROUP BY v.visitor_id, v.connection_datetime
            ORDER BY v.connection_datetime
        """ % (operator,)
        return [('id', 'inselect', (query, [value]))]

    @api.depends('user_partner_id.email_normalized', 'user_partner_id.mobile')
    def _compute_email_phone(self):
        results = self.env['res.partner'].search_read(
            [('id', 'in', self.user_partner_id.ids)],
            ['id', 'email_normalized', 'mobile'],
        )
        mapped_data = {
            result['id']: {
                'email_normalized': result['email_normalized'],
                'mobile': result['mobile']
            } for result in results
        }

        for visitor in self:
            visitor.email = mapped_data.get(visitor.user_partner_id.id, {}).get('email_normalized')
            visitor.mobile = mapped_data.get(visitor.user_partner_id.id, {}).get('mobile')

    @api.depends('website_track_ids')
    def _compute_page_statistics(self):
        results = self.env['website.track'].read_group(
            [('visitor_id', 'in', self.ids), ('url', '!=', False)], ['visitor_id', 'page_id', 'url'], ['visitor_id', 'page_id', 'url'], lazy=False)
        mapped_data = {}
        for result in results:
            visitor_info = mapped_data.get(result['visitor_id'][0], {'page_count': 0, 'visitor_page_count': 0, 'page_ids': set()})
            visitor_info['visitor_page_count'] += result['__count']
            visitor_info['page_count'] += 1
            if result['page_id']:
                visitor_info['page_ids'].add(result['page_id'][0])
            mapped_data[result['visitor_id'][0]] = visitor_info

        for visitor in self:
            visitor_info = mapped_data.get(visitor.id, {'page_ids': [], 'page_count': 0})
            visitor.page_ids = [(6, 0, visitor_info['page_ids'])]
            visitor.visitor_page_count = visitor_info['visitor_page_count']
            visitor.page_count = visitor_info['page_count']

    @api.depends('website_track_ids.page_id')
    def _compute_last_visited_page_id(self):
        results = self.env['website.track'].read_group([('visitor_id', 'in', self.ids)],
                                                                        ['visitor_id', 'page_id', 'visit_datetime:max'],
                                                                        ['visitor_id', 'page_id'], lazy=False)
        mapped_data = {result['visitor_id'][0]: result['page_id'][0] for result in results if result['page_id']}
        for visitor in self:
            visitor.last_visited_page_id = mapped_data.get(visitor.id, False)

    @api.depends('last_connections_ids')
    def _compute_time_statistics(self):
        results = self.env['website.visitor.lastconnection'].read_group([('visitor_id', 'in', self.ids)], ['visitor_id', 'connection_datetime:max'], ['visitor_id'])
        mapped_data = {result['visitor_id'][0]: result['connection_datetime'] for result in results}
        for visitor in self:
            last_connection_datetime = mapped_data.get(visitor.id, False)
            if last_connection_datetime:
                visitor.last_connection_datetime = last_connection_datetime
                visitor.time_since_last_action = _format_time_ago(self.env, (datetime.now() - last_connection_datetime))
                visitor.is_connected = (datetime.now() - last_connection_datetime) < timedelta(minutes=5)

    def _prepare_visitor_send_mail_values(self):
        if self.user_partner_id.email:
            return {
                'res_model': 'res.partner',
                'res_id': self.user_partner_id.id,
                'partner_ids': [self.user_partner_id.id],
            }
        return {}

    def action_send_mail(self):
        self.ensure_one()
        visitor_mail_values = self._prepare_visitor_send_mail_values()
        if not visitor_mail_values:
            raise UserError(_("There is no email linked this visitor."))
        compose_form = self.env.ref('mail.email_compose_message_wizard_form', False)
        ctx = dict(
            default_model=visitor_mail_values.get('res_model'),
            default_res_id=visitor_mail_values.get('res_id'),
            default_use_template=False,
            default_partner_ids=[(6, 0, visitor_mail_values.get('partner_ids'))],
            default_composition_mode='comment',
            default_reply_to=self.env.user.partner_id.email,
        )
        return {
            'name': _('Compose Email'),
            'type': 'ir.actions.act_window',
            'view_mode': 'form',
            'res_model': 'mail.compose.message',
            'views': [(compose_form.id, 'form')],
            'view_id': compose_form.id,
            'target': 'new',
            'context': ctx,
        }

    def _get_visitor_from_request(self, with_previous_visitors=False):
        if not request:
            return None
        if with_previous_visitors and not request.env.user._is_public():
            partner_id = request.env.user.partner_id
            # Retrieve all the previous partner's visitors to have full history of his last products viewed.
            return request.env['website.visitor'].sudo().with_context(active_test=False).search([('user_partner_id', '=', partner_id.id)])
        else:
            visitor = self.env['website.visitor']
            access_token = request.httprequest.cookies.get('visitor_id')
            if access_token:
                visitor = visitor.sudo().with_context(active_test=False).search([('access_token', '=', access_token)])
            return visitor

    def _handle_webpage_dispatch(self, response, website_page):
        # get visitor. Done here to avoid having to do it multiple times in case of override.
        visitor_sudo = self._get_visitor_from_request()
        self._handle_website_page_visit(response, website_page, visitor_sudo)

    def _handle_website_page_visit(self, response, website_page, visitor_sudo):
        """ Called on dispatch. This will create a website.visitor if the http request object
        is a tracked website page or a tracked view. Only on tracked elements to avoid having
        too much operations done on every page or other http requests.
        Note: The side effect is that the last_connection_datetime is updated ONLY on tracked elements."""
        url = request.httprequest.url
        website_track_values = {
            'url': url,
            'visit_datetime': datetime.now(),
        }
        if website_page:
            website_track_values['page_id'] = website_page.id
            domain = [('page_id', '=', website_page.id)]
        else:
            domain = [('url', '=', url)]
        if not visitor_sudo:
            visitor_sudo = self._create_visitor(website_track_values)
            expiration_date = datetime.now() + timedelta(days=365)
            response.set_cookie('visitor_id', visitor_sudo.access_token, expires=expiration_date)
        else:
            visitor_sudo._add_tracking(domain, website_track_values)
            if visitor_sudo.lang_id.id != request.lang.id:
                visitor_sudo.write({'lang_id': request.lang.id})

    def _add_tracking(self, domain, website_track_values):
        """ Update the visitor when a website_track is added"""
        domain = expression.AND([domain, [('visitor_id', '=', self.id)]])
        last_view = self.env['website.track'].sudo().search(domain, limit=1)
        if not last_view or last_view.visit_datetime < datetime.now() - timedelta(minutes=30):
            website_track_values['visitor_id'] = self.id
            self.env['website.track'].create(website_track_values)
        self.env['website.visitor.lastconnection'].create({
            'visitor_id': self.id,
            'connection_datetime': website_track_values['visit_datetime']
        })

    def _create_visitor(self, website_track_values=None):
        """ Create a visitor and add a track to it if website_track_values is set."""
        country_code = request.session.get('geoip', {}).get('country_code', False)
        country_id = request.env['res.country'].sudo().search([('code', '=', country_code)], limit=1).id if country_code else False
        vals = {
            'lang_id': request.lang.id,
            'country_id': country_id,
            'website_id': request.website.id,
            'last_connections_ids': [(0, 0, {'connection_datetime': website_track_values['visit_datetime'] or datetime.now()})],
        }
        if not self.env.user._is_public():
            vals['user_partner_id'] = self.env.user.partner_id.id
            vals['name'] = self.env.user.partner_id.name
        if website_track_values:
            vals['website_track_ids'] = [(0, 0, website_track_values)]
        return self.sudo().create(vals)

    def _cron_archive_visitors(self):
        self.flush(['visit_count'])
        yesterday = datetime.now() - timedelta(days=1)
        query = """
            UPDATE website_visitor
            SET visit_count = visit_count + 1
            WHERE id in (
                SELECT visitor_id
                FROM website_visitor_lastconnection
                GROUP BY visitor_id
                HAVING COUNT(*) > 1)
        """
        self.env.cr.execute(query, [yesterday])

        query = """
            DELETE FROM website_visitor_lastconnection
            WHERE (visitor_id, connection_datetime) not in (
               SELECT v.visitor_id, max(connection_datetime)
               FROM website_visitor_lastconnection v
               GROUP BY v.visitor_id)
        """
        self.env.cr.execute(query, [])

        one_week_ago = datetime.now() - timedelta(days=7)
        visitors_to_archive = self.env['website.visitor'].sudo().search([('last_connection_datetime', '<', one_week_ago)])
        visitors_to_archive.write({'active': False})
<|endoftext|>