<commit_before># Open your $EDITOR to compose a message in weechat
#
# Usage:
# /edit
#
# Optional settings:
# /set plugins.var.python.edit.editor "vim -f"
# /set plugins.var.python.edit.terminal "xterm"
# /set plugins.var.python.edit.run_externally "false"
#
# History:
# 10-18-2015
# Version 1.0.2: Add the ability to run the editor in a external terminal
# Version 1.0.1: Add configurable editor key
# Version 1.0.0: initial release

import os
import os.path
import shlex
import subprocess
import weechat

def xdg_cache_dir():
    return os.path.expanduser(os.environ.get("XDG_CACHE_HOME", "~/.cache/"))

def weechat_cache_dir():
    cache_dir = os.path.join(xdg_cache_dir(), "weechat")
    if os.path.exists(cache_dir):
        return cache_dir
    return os.path.expanduser(os.environ.get("WEECHAT_HOME", "~/.weechat/"))

PATH = os.path.join(weechat_cache_dir(), "message.txt")

def editor_process_cb(data, command, return_code, out, err):
    buf = data

    if return_code != 0:
        cleanup(PATH, buf)
        weechat.prnt("", "{}: {}".format(
            err.strip(),
            return_code
        ))
        return weechat.WEECHAT_RC_ERROR

    if return_code == 0:
        read_file(PATH, buf)
        cleanup(PATH, buf)

    return weechat.WEECHAT_RC_OK

def cleanup(path, buf):
    try:
        os.remove(path)
    except (OSError, IOError):
        pass

    weechat.command(buf, "/window refresh")

def read_file(path, buf):
    try:
        with open(PATH) as f:
            text = f.read()

        weechat.buffer_set(buf, "input", text)
        weechat.buffer_set(buf, "input_pos", str(len(text)))

    except (OSError, IOError):
        pass

    weechat.command(buf, "/window refresh")

def hook_editor_process(terminal, editor, path, buf):
    term_cmd = "{} -e".format(terminal)
    editor_cmd = "{} {}".format(editor, path)
    weechat.hook_process("{} \"{}\"".format(
        term_cmd,
        editor_cmd
    ), 0, "editor_process_cb", buf)

def run_blocking(editor, path, buf):
    cmd = shlex.split(editor) + [path]
    code = subprocess.Popen(cmd).wait()

    if code != 0:
        cleanup(path,  buf)

    read_file(path, buf)

def edit(data, buf, args):
    editor = (weechat.config_get_plugin("editor")
              or os.environ.get("EDITOR", "vim -f"))

    terminal = (weechat.config_get_plugin("terminal")
                or os.getenv("TERMCMD"))

    terminal = terminal or "xterm"

    run_externally = weechat.config_string_to_boolean(
        weechat.config_get_plugin("run_externally")
    )
    run_externally = bool(run_externally)

    with open(PATH, "w+") as f:
        f.write(weechat.buffer_get_string(buf, "input"))

    if run_externally:
        hook_editor_process(terminal, editor, PATH, buf)
    else:
        run_blocking(editor, PATH, buf)

    return weechat.WEECHAT_RC_OK

def main():
    if not weechat.register("edit", "Keith Smiley", "1.0.0", "MIT",
                            "Open your $EDITOR to compose a message", "", ""):
        return weechat.WEECHAT_RC_ERROR

    weechat.hook_command("edit", "Open your $EDITOR to compose a message", "",
                         "", "", "edit", "")

if __name__ == "__main__":
    main()
<commit_msg>Remove trailing newline character if exists (#13)<commit_after># Open your $EDITOR to compose a message in weechat
#
# Usage:
# /edit
#
# Optional settings:
# /set plugins.var.python.edit.editor "vim -f"
# /set plugins.var.python.edit.terminal "xterm"
# /set plugins.var.python.edit.run_externally "false"
#
# History:
# 10-18-2015
# Version 1.0.2: Add the ability to run the editor in a external terminal
# Version 1.0.1: Add configurable editor key
# Version 1.0.0: initial release

import os
import os.path
import shlex
import subprocess
import weechat

def xdg_cache_dir():
    return os.path.expanduser(os.environ.get("XDG_CACHE_HOME", "~/.cache/"))

def weechat_cache_dir():
    cache_dir = os.path.join(xdg_cache_dir(), "weechat")
    if os.path.exists(cache_dir):
        return cache_dir
    return os.path.expanduser(os.environ.get("WEECHAT_HOME", "~/.weechat/"))

PATH = os.path.join(weechat_cache_dir(), "message.txt")

def editor_process_cb(data, command, return_code, out, err):
    buf = data

    if return_code != 0:
        cleanup(PATH, buf)
        weechat.prnt("", "{}: {}".format(
            err.strip(),
            return_code
        ))
        return weechat.WEECHAT_RC_ERROR

    if return_code == 0:
        read_file(PATH, buf)
        cleanup(PATH, buf)

    return weechat.WEECHAT_RC_OK

def cleanup(path, buf):
    try:
        os.remove(path)
    except (OSError, IOError):
        pass

    weechat.command(buf, "/window refresh")

def read_file(path, buf):
    try:
        with open(PATH) as f:
            text = f.read().strip()
        weechat.buffer_set(buf, "input", text)
        weechat.buffer_set(buf, "input_pos", str(len(text)))

    except (OSError, IOError):
        pass

    weechat.command(buf, "/window refresh")

def hook_editor_process(terminal, editor, path, buf):
    term_cmd = "{} -e".format(terminal)
    editor_cmd = "{} {}".format(editor, path)
    weechat.hook_process("{} \"{}\"".format(
        term_cmd,
        editor_cmd
    ), 0, "editor_process_cb", buf)

def run_blocking(editor, path, buf):
    cmd = shlex.split(editor) + [path]
    code = subprocess.Popen(cmd).wait()

    if code != 0:
        cleanup(path,  buf)

    read_file(path, buf)

def edit(data, buf, args):
    editor = (weechat.config_get_plugin("editor")
              or os.environ.get("EDITOR", "vim -f"))

    terminal = (weechat.config_get_plugin("terminal")
                or os.getenv("TERMCMD"))

    terminal = terminal or "xterm"

    run_externally = weechat.config_string_to_boolean(
        weechat.config_get_plugin("run_externally")
    )
    run_externally = bool(run_externally)

    with open(PATH, "w+") as f:
        f.write(weechat.buffer_get_string(buf, "input"))

    if run_externally:
        hook_editor_process(terminal, editor, PATH, buf)
    else:
        run_blocking(editor, PATH, buf)

    return weechat.WEECHAT_RC_OK

def main():
    if not weechat.register("edit", "Keith Smiley", "1.0.0", "MIT",
                            "Open your $EDITOR to compose a message", "", ""):
        return weechat.WEECHAT_RC_ERROR

    weechat.hook_command("edit", "Open your $EDITOR to compose a message", "",
                         "", "", "edit", "")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""

import glob
import json
import os
import time

from absl import logging
import eval_util
import frame_level_models
import losses
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow.python.lib.io import file_io
import utils
import video_level_models

FLAGS = flags.FLAGS

if __name__ == "__main__":
  # Dataset flags.
  flags.DEFINE_string(
      "train_dir", "/tmp/yt8m_model/",
      "The directory to load the model files from. "
      "The tensorboard metrics files are also saved to this "
      "directory.")
  flags.DEFINE_string(
      "eval_data_pattern", "",
      "File glob defining the evaluation dataset in tensorflow.SequenceExample "
      "format. The SequenceExamples are expected to have an 'rgb' byte array "
      "sequence feature as well as a 'labels' int64 context feature.")
  flags.DEFINE_bool(
      "segment_labels", False,
      "If set, then --train_data_pattern must be frame-level features (but with"
      " segment_labels). Otherwise, --train_data_pattern must be aggregated "
      "video-level features. The model must also be set appropriately (i.e. to "
      "read 3D batches VS 4D batches.")

  # Other flags.
  flags.DEFINE_integer("batch_size", 1024,
                       "How many examples to process per batch.")
  flags.DEFINE_integer("num_readers", 8,
                       "How many threads to use for reading input files.")
  flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
  flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")

def find_class_by_name(name, modules):
  """Searches the provided modules for the named class and returns it."""
  modules = [getattr(module, name, None) for module in modules]
  return next(a for a in modules if a)

def get_input_evaluation_tensors(reader,
                                 data_pattern,
                                 batch_size=1024,
                                 num_readers=1):
  """Creates the section of the graph which reads the evaluation data.

  Args:
    reader: A class which parses the training data.
    data_pattern: A 'glob' style path to the data files.
    batch_size: How many examples to process at a time.
    num_readers: How many I/O threads to use.

  Returns:
    A tuple containing the features tensor, labels tensor, and optionally a
    tensor containing the number of frames per video. The exact dimensions
    depend on the reader being used.

  Raises:
    IOError: If no files matching the given pattern were found.
  """
  logging.info("Using batch size of %d for evaluation.", batch_size)
  with tf.name_scope("eval_input"):
    files = gfile.Glob(data_pattern)
    if not files:
      raise IOError("Unable to find the evaluation files.")
    logging.info("number of evaluation files: %d", len(files))
    filename_queue = tf.train.string_input_producer(
        files, shuffle=False, num_epochs=1)
    eval_data = [
        reader.prepare_reader(filename_queue) for _ in range(num_readers)
    ]
    return tf.train.batch_join(
        eval_data,
        batch_size=batch_size,
        capacity=3 * batch_size,
        allow_smaller_final_batch=True,
        enqueue_many=True)

def build_graph(reader,
                model,
                eval_data_pattern,
                label_loss_fn,
                batch_size=1024,
                num_readers=1):
  """Creates the Tensorflow graph for evaluation.

  Args:
    reader: The data file reader. It should inherit from BaseReader.
    model: The core model (e.g. logistic or neural net). It should inherit from
      BaseModel.
    eval_data_pattern: glob path to the evaluation data files.
    label_loss_fn: What kind of loss to apply to the model. It should inherit
      from BaseLoss.
    batch_size: How many examples to process at a time.
    num_readers: How many threads to use for I/O operations.
  """

  global_step = tf.Variable(0, trainable=False, name="global_step")
  input_data_dict = get_input_evaluation_tensors(
      reader, eval_data_pattern, batch_size=batch_size, num_readers=num_readers)
  video_id_batch = input_data_dict["video_ids"]
  model_input_raw = input_data_dict["video_matrix"]
  labels_batch = input_data_dict["labels"]
  num_frames = input_data_dict["num_frames"]
  tf.summary.histogram("model_input_raw", model_input_raw)

  feature_dim = len(model_input_raw.get_shape()) - 1

  # Normalize input features.
  model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)

  with tf.variable_scope("tower"):
    result = model.create_model(
        model_input,
        num_frames=num_frames,
        vocab_size=reader.num_classes,
        labels=labels_batch,
        is_training=False)
    predictions = result["predictions"]
    tf.summary.histogram("model_activations", predictions)
    if "loss" in result.keys():
      label_loss = result["loss"]
    else:
      label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)

  tf.add_to_collection("global_step", global_step)
  tf.add_to_collection("loss", label_loss)
  tf.add_to_collection("predictions", predictions)
  tf.add_to_collection("input_batch", model_input)
  tf.add_to_collection("input_batch_raw", model_input_raw)
  tf.add_to_collection("video_id_batch", video_id_batch)
  tf.add_to_collection("num_frames", num_frames)
  tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
  if FLAGS.segment_labels:
    tf.add_to_collection("label_weights", input_data_dict["label_weights"])
  tf.add_to_collection("summary_op", tf.summary.merge_all())

def evaluation_loop(fetches, saver, summary_writer, evl_metrics,
                    last_global_step_val):
  """Run the evaluation loop once.

  Args:
    fetches: a dict of tensors to be run within Session.
    saver: a tensorflow saver to restore the model.
    summary_writer: a tensorflow summary_writer
    evl_metrics: an EvaluationMetrics object.
    last_global_step_val: the global step used in the previous evaluation.

  Returns:
    The global_step used in the latest model.
  """

  global_step_val = -1
  with tf.Session() as sess:
    latest_checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
    if latest_checkpoint:
      logging.info("Loading checkpoint for eval: %s", latest_checkpoint)
      # Restores from checkpoint
      saver.restore(sess, latest_checkpoint)
      # Assuming model_checkpoint_path looks something like:
      # /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
      global_step_val = os.path.basename(latest_checkpoint).split("-")[-1]

      # Save model
      saver.save(
          sess,
          os.path.join(FLAGS.train_dir, "inference_model", "inference_model"))
    else:
      logging.info("No checkpoint file found.")
      return global_step_val

    if global_step_val == last_global_step_val:
      logging.info(
          "skip this checkpoint global_step_val=%s "
          "(same as the previous one).", global_step_val)
      return global_step_val

    sess.run([tf.local_variables_initializer()])

    # Start the queue runners.
    coord = tf.train.Coordinator()
    try:
      threads = []
      for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
        threads.extend(
            qr.create_threads(sess, coord=coord, daemon=True, start=True))
      logging.info("enter eval_once loop global_step_val = %s. ",
                   global_step_val)

      evl_metrics.clear()

      examples_processed = 0
      while not coord.should_stop():
        batch_start_time = time.time()
        output_data_dict = sess.run(fetches)
        seconds_per_batch = time.time() - batch_start_time
        labels_val = output_data_dict["labels"]
        summary_val = output_data_dict["summary"]
        example_per_second = labels_val.shape[0] / seconds_per_batch
        examples_processed += labels_val.shape[0]

        predictions = output_data_dict["predictions"]
        if FLAGS.segment_labels:
          # This is a workaround to ignore the unrated labels.
          predictions *= output_data_dict["label_weights"]
        iteration_info_dict = evl_metrics.accumulate(
            predictions, labels_val,
            output_data_dict["loss"])
        iteration_info_dict["examples_per_second"] = example_per_second

        iterinfo = utils.AddGlobalStepSummary(
            summary_writer,
            global_step_val,
            iteration_info_dict,
            summary_scope="Eval")
        logging.info("examples_processed: %d | %s", examples_processed,
                     iterinfo)

    except tf.errors.OutOfRangeError as e:
      logging.info(
          "Done with batched inference. Now calculating global performance "
          "metrics.")
      # calculate the metrics for the entire epoch
      epoch_info_dict = evl_metrics.get()
      epoch_info_dict["epoch_id"] = global_step_val

      summary_writer.add_summary(summary_val, global_step_val)
      epochinfo = utils.AddEpochSummary(
          summary_writer,
          global_step_val,
          epoch_info_dict,
          summary_scope="Eval")
      logging.info(epochinfo)
      evl_metrics.clear()
    except Exception as e:  # pylint: disable=broad-except
      logging.info("Unexpected exception: %s", str(e))
      coord.request_stop(e)

    coord.request_stop()
    coord.join(threads, stop_grace_period_secs=10)
    logging.info("Total: examples_processed: %d", examples_processed)

    return global_step_val

def evaluate():
  """Starts main evaluation loop."""
  tf.set_random_seed(0)  # for reproducibility

  # Write json of flags
  model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
  if not file_io.file_exists(model_flags_path):
    raise IOError(("Cannot find file %s. Did you run train.py on the same "
                   "--train_dir?") % model_flags_path)
  flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())

  with tf.Graph().as_default():
    # convert feature_names and feature_sizes to lists of values
    feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
        flags_dict["feature_names"], flags_dict["feature_sizes"])

    if flags_dict["frame_features"]:
      reader = readers.YT8MFrameFeatureReader(
          feature_names=feature_names,
          feature_sizes=feature_sizes,
          segment_labels=FLAGS.segment_labels)
    else:
      reader = readers.YT8MAggregatedFeatureReader(
          feature_names=feature_names, feature_sizes=feature_sizes)

    model = find_class_by_name(flags_dict["model"],
                               [frame_level_models, video_level_models])()
    label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])()

    if not FLAGS.eval_data_pattern:
      raise IOError("'eval_data_pattern' was not specified. Nothing to "
                    "evaluate.")

    build_graph(
        reader=reader,
        model=model,
        eval_data_pattern=FLAGS.eval_data_pattern,
        label_loss_fn=label_loss_fn,
        num_readers=FLAGS.num_readers,
        batch_size=FLAGS.batch_size)
    logging.info("built evaluation graph")

    # A dict of tensors to be run in Session.
    fetches = {
        "video_id": tf.get_collection("video_id_batch")[0],
        "predictions": tf.get_collection("predictions")[0],
        "labels": tf.get_collection("labels")[0],
        "loss": tf.get_collection("loss")[0],
        "summary": tf.get_collection("summary_op")[0]
    }
    if FLAGS.segment_labels:
      fetches["label_weights"] = tf.get_collection("label_weights")[0]

    saver = tf.train.Saver(tf.global_variables())
    summary_writer = tf.summary.FileWriter(
        os.path.join(FLAGS.train_dir, "eval"), graph=tf.get_default_graph())

    evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)

    last_global_step_val = -1
    while True:
      last_global_step_val = evaluation_loop(fetches, saver, summary_writer,
                                             evl_metrics, last_global_step_val)
      if FLAGS.run_once:
        break

def main(unused_argv):
  logging.set_verbosity(tf.logging.INFO)
  logging.info("tensorflow version: %s", tf.__version__)
  evaluate()

if __name__ == "__main__":
  app.run()
<commit_msg>Lint fix.<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Binary for evaluating Tensorflow models on the YouTube-8M dataset."""

import json
import os
import time

from absl import logging
import eval_util
import frame_level_models
import losses
import readers
import tensorflow as tf
from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow.python.lib.io import file_io
import utils
import video_level_models

FLAGS = flags.FLAGS

if __name__ == "__main__":
  # Dataset flags.
  flags.DEFINE_string(
      "train_dir", "/tmp/yt8m_model/",
      "The directory to load the model files from. "
      "The tensorboard metrics files are also saved to this "
      "directory.")
  flags.DEFINE_string(
      "eval_data_pattern", "",
      "File glob defining the evaluation dataset in tensorflow.SequenceExample "
      "format. The SequenceExamples are expected to have an 'rgb' byte array "
      "sequence feature as well as a 'labels' int64 context feature.")
  flags.DEFINE_bool(
      "segment_labels", False,
      "If set, then --train_data_pattern must be frame-level features (but with"
      " segment_labels). Otherwise, --train_data_pattern must be aggregated "
      "video-level features. The model must also be set appropriately (i.e. to "
      "read 3D batches VS 4D batches.")

  # Other flags.
  flags.DEFINE_integer("batch_size", 1024,
                       "How many examples to process per batch.")
  flags.DEFINE_integer("num_readers", 8,
                       "How many threads to use for reading input files.")
  flags.DEFINE_boolean("run_once", False, "Whether to run eval only once.")
  flags.DEFINE_integer("top_k", 20, "How many predictions to output per video.")

def find_class_by_name(name, modules):
  """Searches the provided modules for the named class and returns it."""
  modules = [getattr(module, name, None) for module in modules]
  return next(a for a in modules if a)

def get_input_evaluation_tensors(reader,
                                 data_pattern,
                                 batch_size=1024,
                                 num_readers=1):
  """Creates the section of the graph which reads the evaluation data.

  Args:
    reader: A class which parses the training data.
    data_pattern: A 'glob' style path to the data files.
    batch_size: How many examples to process at a time.
    num_readers: How many I/O threads to use.

  Returns:
    A tuple containing the features tensor, labels tensor, and optionally a
    tensor containing the number of frames per video. The exact dimensions
    depend on the reader being used.

  Raises:
    IOError: If no files matching the given pattern were found.
  """
  logging.info("Using batch size of %d for evaluation.", batch_size)
  with tf.name_scope("eval_input"):
    files = gfile.Glob(data_pattern)
    if not files:
      raise IOError("Unable to find the evaluation files.")
    logging.info("number of evaluation files: %d", len(files))
    filename_queue = tf.train.string_input_producer(
        files, shuffle=False, num_epochs=1)
    eval_data = [
        reader.prepare_reader(filename_queue) for _ in range(num_readers)
    ]
    return tf.train.batch_join(
        eval_data,
        batch_size=batch_size,
        capacity=3 * batch_size,
        allow_smaller_final_batch=True,
        enqueue_many=True)

def build_graph(reader,
                model,
                eval_data_pattern,
                label_loss_fn,
                batch_size=1024,
                num_readers=1):
  """Creates the Tensorflow graph for evaluation.

  Args:
    reader: The data file reader. It should inherit from BaseReader.
    model: The core model (e.g. logistic or neural net). It should inherit from
      BaseModel.
    eval_data_pattern: glob path to the evaluation data files.
    label_loss_fn: What kind of loss to apply to the model. It should inherit
      from BaseLoss.
    batch_size: How many examples to process at a time.
    num_readers: How many threads to use for I/O operations.
  """

  global_step = tf.Variable(0, trainable=False, name="global_step")
  input_data_dict = get_input_evaluation_tensors(
      reader, eval_data_pattern, batch_size=batch_size, num_readers=num_readers)
  video_id_batch = input_data_dict["video_ids"]
  model_input_raw = input_data_dict["video_matrix"]
  labels_batch = input_data_dict["labels"]
  num_frames = input_data_dict["num_frames"]
  tf.summary.histogram("model_input_raw", model_input_raw)

  feature_dim = len(model_input_raw.get_shape()) - 1

  # Normalize input features.
  model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)

  with tf.variable_scope("tower"):
    result = model.create_model(
        model_input,
        num_frames=num_frames,
        vocab_size=reader.num_classes,
        labels=labels_batch,
        is_training=False)
    predictions = result["predictions"]
    tf.summary.histogram("model_activations", predictions)
    if "loss" in result.keys():
      label_loss = result["loss"]
    else:
      label_loss = label_loss_fn.calculate_loss(predictions, labels_batch)

  tf.add_to_collection("global_step", global_step)
  tf.add_to_collection("loss", label_loss)
  tf.add_to_collection("predictions", predictions)
  tf.add_to_collection("input_batch", model_input)
  tf.add_to_collection("input_batch_raw", model_input_raw)
  tf.add_to_collection("video_id_batch", video_id_batch)
  tf.add_to_collection("num_frames", num_frames)
  tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
  if FLAGS.segment_labels:
    tf.add_to_collection("label_weights", input_data_dict["label_weights"])
  tf.add_to_collection("summary_op", tf.summary.merge_all())

def evaluation_loop(fetches, saver, summary_writer, evl_metrics,
                    last_global_step_val):
  """Run the evaluation loop once.

  Args:
    fetches: a dict of tensors to be run within Session.
    saver: a tensorflow saver to restore the model.
    summary_writer: a tensorflow summary_writer
    evl_metrics: an EvaluationMetrics object.
    last_global_step_val: the global step used in the previous evaluation.

  Returns:
    The global_step used in the latest model.
  """

  global_step_val = -1
  with tf.Session() as sess:
    latest_checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
    if latest_checkpoint:
      logging.info("Loading checkpoint for eval: %s", latest_checkpoint)
      # Restores from checkpoint
      saver.restore(sess, latest_checkpoint)
      # Assuming model_checkpoint_path looks something like:
      # /my-favorite-path/yt8m_train/model.ckpt-0, extract global_step from it.
      global_step_val = os.path.basename(latest_checkpoint).split("-")[-1]

      # Save model
      saver.save(
          sess,
          os.path.join(FLAGS.train_dir, "inference_model", "inference_model"))
    else:
      logging.info("No checkpoint file found.")
      return global_step_val

    if global_step_val == last_global_step_val:
      logging.info(
          "skip this checkpoint global_step_val=%s "
          "(same as the previous one).", global_step_val)
      return global_step_val

    sess.run([tf.local_variables_initializer()])

    # Start the queue runners.
    coord = tf.train.Coordinator()
    try:
      threads = []
      for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
        threads.extend(
            qr.create_threads(sess, coord=coord, daemon=True, start=True))
      logging.info("enter eval_once loop global_step_val = %s. ",
                   global_step_val)

      evl_metrics.clear()

      examples_processed = 0
      while not coord.should_stop():
        batch_start_time = time.time()
        output_data_dict = sess.run(fetches)
        seconds_per_batch = time.time() - batch_start_time
        labels_val = output_data_dict["labels"]
        summary_val = output_data_dict["summary"]
        example_per_second = labels_val.shape[0] / seconds_per_batch
        examples_processed += labels_val.shape[0]

        predictions = output_data_dict["predictions"]
        if FLAGS.segment_labels:
          # This is a workaround to ignore the unrated labels.
          predictions *= output_data_dict["label_weights"]
        iteration_info_dict = evl_metrics.accumulate(
            predictions, labels_val,
            output_data_dict["loss"])
        iteration_info_dict["examples_per_second"] = example_per_second

        iterinfo = utils.AddGlobalStepSummary(
            summary_writer,
            global_step_val,
            iteration_info_dict,
            summary_scope="Eval")
        logging.info("examples_processed: %d | %s", examples_processed,
                     iterinfo)

    except tf.errors.OutOfRangeError as e:
      logging.info(
          "Done with batched inference. Now calculating global performance "
          "metrics.")
      # calculate the metrics for the entire epoch
      epoch_info_dict = evl_metrics.get()
      epoch_info_dict["epoch_id"] = global_step_val

      summary_writer.add_summary(summary_val, global_step_val)
      epochinfo = utils.AddEpochSummary(
          summary_writer,
          global_step_val,
          epoch_info_dict,
          summary_scope="Eval")
      logging.info(epochinfo)
      evl_metrics.clear()
    except Exception as e:  # pylint: disable=broad-except
      logging.info("Unexpected exception: %s", str(e))
      coord.request_stop(e)

    coord.request_stop()
    coord.join(threads, stop_grace_period_secs=10)
    logging.info("Total: examples_processed: %d", examples_processed)

    return global_step_val

def evaluate():
  """Starts main evaluation loop."""
  tf.set_random_seed(0)  # for reproducibility

  # Write json of flags
  model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
  if not file_io.file_exists(model_flags_path):
    raise IOError(("Cannot find file %s. Did you run train.py on the same "
                   "--train_dir?") % model_flags_path)
  flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())

  with tf.Graph().as_default():
    # convert feature_names and feature_sizes to lists of values
    feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
        flags_dict["feature_names"], flags_dict["feature_sizes"])

    if flags_dict["frame_features"]:
      reader = readers.YT8MFrameFeatureReader(
          feature_names=feature_names,
          feature_sizes=feature_sizes,
          segment_labels=FLAGS.segment_labels)
    else:
      reader = readers.YT8MAggregatedFeatureReader(
          feature_names=feature_names, feature_sizes=feature_sizes)

    model = find_class_by_name(flags_dict["model"],
                               [frame_level_models, video_level_models])()
    label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])()

    if not FLAGS.eval_data_pattern:
      raise IOError("'eval_data_pattern' was not specified. Nothing to "
                    "evaluate.")

    build_graph(
        reader=reader,
        model=model,
        eval_data_pattern=FLAGS.eval_data_pattern,
        label_loss_fn=label_loss_fn,
        num_readers=FLAGS.num_readers,
        batch_size=FLAGS.batch_size)
    logging.info("built evaluation graph")

    # A dict of tensors to be run in Session.
    fetches = {
        "video_id": tf.get_collection("video_id_batch")[0],
        "predictions": tf.get_collection("predictions")[0],
        "labels": tf.get_collection("labels")[0],
        "loss": tf.get_collection("loss")[0],
        "summary": tf.get_collection("summary_op")[0]
    }
    if FLAGS.segment_labels:
      fetches["label_weights"] = tf.get_collection("label_weights")[0]

    saver = tf.train.Saver(tf.global_variables())
    summary_writer = tf.summary.FileWriter(
        os.path.join(FLAGS.train_dir, "eval"), graph=tf.get_default_graph())

    evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)

    last_global_step_val = -1
    while True:
      last_global_step_val = evaluation_loop(fetches, saver, summary_writer,
                                             evl_metrics, last_global_step_val)
      if FLAGS.run_once:
        break

def main(unused_argv):
  logging.set_verbosity(tf.logging.INFO)
  logging.info("tensorflow version: %s", tf.__version__)
  evaluate()

if __name__ == "__main__":
  app.run()
<|endoftext|>


<commit_before>"""abd automates the creation and landing of reviews from branches."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdi_processrepo
#
# Public Functions:
#   is_based_on
#   create_review
#   verify_review_branch_base
#   create_differential_review
#   update_review
#   update_in_review
#   land
#   create_failed_review
#   try_create_review
#   process_updated_branch
#   process_abandoned_branch
#   process_branches
#
# Public Assignments:
#   MAX_DIFF_SIZE
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================

import abdcmnt_commenter
import abdt_conduitgit
import abdt_exception

# TODO: split into appropriate modules

_DEFAULT_TEST_PLAN = "I DIDNT TEST"
MAX_DIFF_SIZE = 1.5 * 1024 * 1024

def is_based_on(name, base):
    # TODO: actually do this
    return True

def create_review(conduit, branch, plugin_manager):
    plugin_manager.hook(
        "before_create_review",
        {"conduit": conduit, "branch": branch})

    branch.verify_review_branch_base()

    # TODO: we should also cc other users on the branch
    # TODO: if there are emails that don't match up to users then we should
    #       note that on the review and perhaps use the mailer to notify them
    name, email, user = abdt_conduitgit.getPrimaryNameEmailAndUserFromBranch(
        conduit, branch)

    print "- author: " + user

    used_default_test_plan = False

    parsed = abdt_conduitgit.getFieldsFromBranch(conduit, branch)
    if parsed.errors:
        used_default_test_plan = True
        parsed = abdt_conduitgit.getFieldsFromBranch(
            conduit, branch, _DEFAULT_TEST_PLAN)
        if parsed.errors:
            print parsed
            raise abdt_exception.CommitMessageParseException(
                errors=parsed.errors,
                fields=parsed.fields,
                digest=branch.make_message_digest())

    rawDiff = branch.make_raw_diff()

    revisionid = create_differential_review(
        conduit, user, parsed, branch, rawDiff)

    commenter = abdcmnt_commenter.Commenter(conduit, revisionid)

    if used_default_test_plan:
        commenter = abdcmnt_commenter.Commenter(conduit, revisionid)
        commenter.usedDefaultTestPlan(
            branch.review_branch_name(), _DEFAULT_TEST_PLAN)

    plugin_manager.hook(
        "after_create_review",
        {"parsed": parsed, "conduit": conduit, "branch": branch,
            "rawDiff": rawDiff, "commenter": commenter}
    )

def verify_review_branch_base(git_context, review_branch):
    if review_branch.base not in git_context.branches:
        raise abdt_exception.MissingBaseException(
            review_branch.branch, review_branch.base)
    if not is_based_on(review_branch.branch, review_branch.base):
        raise abdt_exception.AbdUserException(
            "'" + review_branch.branch +
            "' is not based on '" + review_branch.base + "'")

def create_differential_review(conduit, user, parsed, branch, raw_diff):
    print "- creating revision"
    revision_id = conduit.create_revision_as_user(
        raw_diff, parsed.fields, user)
    print "- created " + str(revision_id)
    branch.mark_ok_new_review(revision_id)

    print "- commenting on " + str(revision_id)
    commenter = abdcmnt_commenter.Commenter(conduit, revision_id)
    commenter.createdReview(
        branch.review_branch_name(),
        branch.base_branch_name())

    return revision_id

def update_review(conduit, branch):
    if branch.has_new_commits():
        print "changes on branch"
        branch.verify_review_branch_base()
        update_in_review(conduit, branch)
    elif branch.is_status_bad() and not branch.is_status_bad_land():
        try:
            print "try updating bad branch"
            branch.verify_review_branch_base()
            update_in_review(conduit, branch)
        except abdt_exception.AbdUserException:
            print "still bad"

    if not branch.is_status_bad():
        if conduit.is_review_accepted(branch.review_id_or_none()):
            branch.verify_review_branch_base()
            land(conduit, branch)
            # TODO: we probably want to do a better job of cleaning up locally
        else:
            print "do nothing"

def update_in_review(conduit, branch):
    print "update_in_review"

    print "- creating diff"
    rawDiff = branch.make_raw_diff()
    review_id = branch.review_id_or_none()
    review_id_str = str(review_id)

    print "- updating revision " + review_id_str
    conduit.update_revision(review_id, rawDiff, "update")

    branch.mark_ok_in_review()

    print "- commenting on revision " + review_id_str
    commenter = abdcmnt_commenter.Commenter(conduit, review_id)
    commenter.updatedReview(branch.review_branch_name())

def land(conduit, branch):
    print "landing " + branch.review_branch_name()
    name, email, user = abdt_conduitgit.getPrimaryNameEmailAndUserFromBranch(
        conduit, branch)

    review_id = branch.review_id_or_none()

    # compose the commit message
    message = conduit.get_commit_message(review_id)

    review_branch_name = branch.review_branch_name()
    base_branch_name = branch.base_branch_name()
    land_message = branch.land(name, email, message)

    print "- commenting on revision " + str(review_id)
    commenter = abdcmnt_commenter.Commenter(conduit, review_id)
    commenter.landedReview(
        review_branch_name,
        base_branch_name,
        land_message)

    conduit.close_revision(review_id)

def create_failed_review(conduit, branch, exception):
    user = abdt_conduitgit.getAnyUserFromBranch(conduit, branch)
    reviewid = conduit.create_empty_revision_as_user(user)
    commenter = abdcmnt_commenter.Commenter(conduit, reviewid)
    commenter.failedCreateReview(branch.review_branch_name(), exception)
    branch.mark_new_bad_in_review(reviewid)

def try_create_review(mailer, conduit, branch, plugin_manager, mail_on_fail):
    try:
        create_review(conduit, branch, plugin_manager)
    except abdt_exception.AbdUserException as e:
        print "failed to create:"
        print e
        try:
            create_failed_review(conduit, branch, e)
        except abdt_exception.NoUsersOnBranchException as e:
            print "failed to create failed review:"
            print e
            branch.mark_bad_pre_review()
            if mail_on_fail:
                mailer.noUsersOnBranch(
                    e.review_branch_name, e.base_name, e.emails)

def process_updated_branch(mailer, conduit, branch, plugin_manager):
    abdte = abdt_exception
    review_branch_name = branch.review_branch_name()
    if branch.is_new():
        print "create review for " + review_branch_name
        try_create_review(
            mailer,
            conduit,
            branch,
            plugin_manager,
            mail_on_fail=True)
    else:
        review_id = branch.review_id_or_none()
        commenter = abdcmnt_commenter.Commenter(conduit, review_id)
        if branch.is_status_bad_pre_review():
            print "try again to create review for " + review_branch_name
            has_new_commits = branch.has_new_commits()
            branch.clear_mark()
            try_create_review(
                mailer,
                conduit,
                branch,
                plugin_manager,
                mail_on_fail=has_new_commits)
        else:
            print "update review for " + review_branch_name
            try:
                update_review(conduit, branch)
            except abdte.LandingException as e:
                print "landing exception"
                branch.mark_bad_land()
                commenter.exception(e)
                conduit.set_requires_revision(review_id)
            except abdte.AbdUserException as e:
                print "user exception"
                branch.mark_bad_in_review()
                commenter.exception(e)

def process_abandoned_branch(conduit, branch):
    print "untracking abandoned branch: " + branch.review_branch_name()
    review_id = branch.review_id_or_none()
    if review_id is not None:
        commenter = abdcmnt_commenter.Commenter(conduit, review_id)
        commenter.abandonedBranch(branch.review_branch_name())
        # TODO: abandon the associated revision if not already
    branch.abandon()

def process_branches(branches, conduit, mailer, plugin_manager):
    for branch in branches:
        if branch.is_abandoned():
            process_abandoned_branch(conduit, branch)
        elif branch.is_null():
            pass  # TODO: should handle these
        else:
            print "pending:", branch.review_branch_name()
            process_updated_branch(mailer, conduit, branch, plugin_manager)

#------------------------------------------------------------------------------
# Copyright (C) 2012 Bloomberg L.P.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#------------------------------- END-OF-FILE ----------------------------------
<commit_msg>Tidy abdi_processrepo: remove unused functions<commit_after>"""abd automates the creation and landing of reviews from branches."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdi_processrepo
#
# Public Functions:
#   create_review
#   create_differential_review
#   update_review
#   update_in_review
#   land
#   create_failed_review
#   try_create_review
#   process_updated_branch
#   process_abandoned_branch
#   process_branches
#
# Public Assignments:
#   MAX_DIFF_SIZE
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================

import abdcmnt_commenter
import abdt_conduitgit
import abdt_exception

# TODO: split into appropriate modules

_DEFAULT_TEST_PLAN = "I DIDNT TEST"
MAX_DIFF_SIZE = 1.5 * 1024 * 1024

def create_review(conduit, branch, plugin_manager):
    plugin_manager.hook(
        "before_create_review",
        {"conduit": conduit, "branch": branch})

    branch.verify_review_branch_base()

    # TODO: we should also cc other users on the branch
    # TODO: if there are emails that don't match up to users then we should
    #       note that on the review and perhaps use the mailer to notify them
    name, email, user = abdt_conduitgit.getPrimaryNameEmailAndUserFromBranch(
        conduit, branch)

    print "- author: " + user

    used_default_test_plan = False

    parsed = abdt_conduitgit.getFieldsFromBranch(conduit, branch)
    if parsed.errors:
        used_default_test_plan = True
        parsed = abdt_conduitgit.getFieldsFromBranch(
            conduit, branch, _DEFAULT_TEST_PLAN)
        if parsed.errors:
            print parsed
            raise abdt_exception.CommitMessageParseException(
                errors=parsed.errors,
                fields=parsed.fields,
                digest=branch.make_message_digest())

    rawDiff = branch.make_raw_diff()

    revisionid = create_differential_review(
        conduit, user, parsed, branch, rawDiff)

    commenter = abdcmnt_commenter.Commenter(conduit, revisionid)

    if used_default_test_plan:
        commenter = abdcmnt_commenter.Commenter(conduit, revisionid)
        commenter.usedDefaultTestPlan(
            branch.review_branch_name(), _DEFAULT_TEST_PLAN)

    plugin_manager.hook(
        "after_create_review",
        {"parsed": parsed, "conduit": conduit, "branch": branch,
            "rawDiff": rawDiff, "commenter": commenter}
    )

def create_differential_review(conduit, user, parsed, branch, raw_diff):
    print "- creating revision"
    revision_id = conduit.create_revision_as_user(
        raw_diff, parsed.fields, user)
    print "- created " + str(revision_id)
    branch.mark_ok_new_review(revision_id)

    print "- commenting on " + str(revision_id)
    commenter = abdcmnt_commenter.Commenter(conduit, revision_id)
    commenter.createdReview(
        branch.review_branch_name(),
        branch.base_branch_name())

    return revision_id

def update_review(conduit, branch):
    if branch.has_new_commits():
        print "changes on branch"
        branch.verify_review_branch_base()
        update_in_review(conduit, branch)
    elif branch.is_status_bad() and not branch.is_status_bad_land():
        try:
            print "try updating bad branch"
            branch.verify_review_branch_base()
            update_in_review(conduit, branch)
        except abdt_exception.AbdUserException:
            print "still bad"

    if not branch.is_status_bad():
        if conduit.is_review_accepted(branch.review_id_or_none()):
            branch.verify_review_branch_base()
            land(conduit, branch)
            # TODO: we probably want to do a better job of cleaning up locally
        else:
            print "do nothing"

def update_in_review(conduit, branch):
    print "update_in_review"

    print "- creating diff"
    rawDiff = branch.make_raw_diff()
    review_id = branch.review_id_or_none()
    review_id_str = str(review_id)

    print "- updating revision " + review_id_str
    conduit.update_revision(review_id, rawDiff, "update")

    branch.mark_ok_in_review()

    print "- commenting on revision " + review_id_str
    commenter = abdcmnt_commenter.Commenter(conduit, review_id)
    commenter.updatedReview(branch.review_branch_name())

def land(conduit, branch):
    print "landing " + branch.review_branch_name()
    name, email, user = abdt_conduitgit.getPrimaryNameEmailAndUserFromBranch(
        conduit, branch)

    review_id = branch.review_id_or_none()

    # compose the commit message
    message = conduit.get_commit_message(review_id)

    review_branch_name = branch.review_branch_name()
    base_branch_name = branch.base_branch_name()
    land_message = branch.land(name, email, message)

    print "- commenting on revision " + str(review_id)
    commenter = abdcmnt_commenter.Commenter(conduit, review_id)
    commenter.landedReview(
        review_branch_name,
        base_branch_name,
        land_message)

    conduit.close_revision(review_id)

def create_failed_review(conduit, branch, exception):
    user = abdt_conduitgit.getAnyUserFromBranch(conduit, branch)
    reviewid = conduit.create_empty_revision_as_user(user)
    commenter = abdcmnt_commenter.Commenter(conduit, reviewid)
    commenter.failedCreateReview(branch.review_branch_name(), exception)
    branch.mark_new_bad_in_review(reviewid)

def try_create_review(mailer, conduit, branch, plugin_manager, mail_on_fail):
    try:
        create_review(conduit, branch, plugin_manager)
    except abdt_exception.AbdUserException as e:
        print "failed to create:"
        print e
        try:
            create_failed_review(conduit, branch, e)
        except abdt_exception.NoUsersOnBranchException as e:
            print "failed to create failed review:"
            print e
            branch.mark_bad_pre_review()
            if mail_on_fail:
                mailer.noUsersOnBranch(
                    e.review_branch_name, e.base_name, e.emails)

def process_updated_branch(mailer, conduit, branch, plugin_manager):
    abdte = abdt_exception
    review_branch_name = branch.review_branch_name()
    if branch.is_new():
        print "create review for " + review_branch_name
        try_create_review(
            mailer,
            conduit,
            branch,
            plugin_manager,
            mail_on_fail=True)
    else:
        review_id = branch.review_id_or_none()
        commenter = abdcmnt_commenter.Commenter(conduit, review_id)
        if branch.is_status_bad_pre_review():
            print "try again to create review for " + review_branch_name
            has_new_commits = branch.has_new_commits()
            branch.clear_mark()
            try_create_review(
                mailer,
                conduit,
                branch,
                plugin_manager,
                mail_on_fail=has_new_commits)
        else:
            print "update review for " + review_branch_name
            try:
                update_review(conduit, branch)
            except abdte.LandingException as e:
                print "landing exception"
                branch.mark_bad_land()
                commenter.exception(e)
                conduit.set_requires_revision(review_id)
            except abdte.AbdUserException as e:
                print "user exception"
                branch.mark_bad_in_review()
                commenter.exception(e)

def process_abandoned_branch(conduit, branch):
    print "untracking abandoned branch: " + branch.review_branch_name()
    review_id = branch.review_id_or_none()
    if review_id is not None:
        commenter = abdcmnt_commenter.Commenter(conduit, review_id)
        commenter.abandonedBranch(branch.review_branch_name())
        # TODO: abandon the associated revision if not already
    branch.abandon()

def process_branches(branches, conduit, mailer, plugin_manager):
    for branch in branches:
        if branch.is_abandoned():
            process_abandoned_branch(conduit, branch)
        elif branch.is_null():
            pass  # TODO: should handle these
        else:
            print "pending:", branch.review_branch_name()
            process_updated_branch(mailer, conduit, branch, plugin_manager)

#------------------------------------------------------------------------------
# Copyright (C) 2012 Bloomberg L.P.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#------------------------------- END-OF-FILE ----------------------------------
<|endoftext|>


<commit_before>'''
Function to provide rapidly a model of a preprocessed image
in order to have a pixel variance estimation that is
almost not correlated with pixel value.
'''

import os
import numpy as np
import copy

import numba
import scipy.interpolate

from desispec.image import Image
from desiutil.log import get_logger
from desispec.io.xytraceset import read_xytraceset
from desispec.qproc.qextract import qproc_boxcar_extraction
from desispec.qproc.qsky import qproc_sky_subtraction
from desispec.qproc.qfiberflat import qproc_apply_fiberflat

@numba.jit
def numba_proj(image,x,sigma,flux) :
    '''
    Add a spectrum to a model of the pixel values in a CCD image assuming a Gaussian cross-dispersion profile.

    Inputs:
       image: 2D numpy array , this array will be modified
       x: 1D numpy array of size image.shape[0], coordinate of center of cross-dispersion profile of spectral trace, in pixel units, for each CCD row
       sigma : 1D numpy array of size image.shape[0], sigma of cross-dispersion profile of spectral trace, in pixel units, for each CCD row
       flux : 1D numpy array of size image.shape[0], quantity to project (for design usage: electrons per pixel) for each CCD row
    '''
    n0=image.shape[0]
    n1=image.shape[1]
    hw=3
    prof=np.zeros(2*hw+1)
    for j in range(n0) :
        sumprof=0.
        b=max(0,int(x[j]-hw))
        e=min(n1,int(x[j]+hw+1))
        for i in range(b,e) :
            prof[i-b]=np.exp(-(i-x[j])**2/2./sigma[j]**2)
            sumprof += prof[i-b]
        for i in range(b,e) :
            image[j,i] += flux[j]*prof[i-b]/sumprof

def compute_image_model(image,xytraceset,fiberflat=None,fibermap=None,with_spectral_smoothing=True,with_sky_model=True,\
                        spectral_smoothing_sigma_length=10.,spectral_smoothing_nsig=4.,psf=None):
    '''
    Returns a model of the input image, using a fast extraction, a processing of
    spectra with a common sky model and a smoothing, followed by a reprojection
    on the CCD image.

    Inputs:
       image: a preprocessed image in the form of a desispec.image.Image object
       xytraceset: a desispec.xytraceset.XYTraceSet object with trace coordinates

    Optional:
       fiberflat: a desispec.fiberflat.FiberFlat object
       with_spectral_smoothing: try and smooth the spectra to reduce noise (and eventualy reduce variance correlation)
       with_sky_model: use a sky model as part of the spectral modeling to reduce the noise (requires a fiberflat)
       spectral_smoothing_sigma_length: sigma of Gaussian smoothing along wavelength in A
       spectral_smoothing_nsig: number of sigma rejection threshold to fall back to the original extracted spectrum instead of the smooth one
       psf: specter.psf.GaussHermitePSF object to be used for the 1D projection (slow, by default=None, in which case a Gaussian profile is used)
    returns:
       a 2D np.array of same shape as image.pix
    '''

    log=get_logger()

    # first perform a fast boxcar extraction
    log.info("extract spectra")
    image.mask = None
    qframe = qproc_boxcar_extraction(xytraceset,image)
    fqframe = None
    sqframe = None
    ratio = None
    if fiberflat is not None :
       log.info("fiberflat")
       fqframe = copy.deepcopy(qframe)
       flat=qproc_apply_fiberflat(fqframe,fiberflat=fiberflat,return_flat=True)
    if with_sky_model :
        if fiberflat is None :
            log.warning("cannot compute and use a sky model without a fiberflat")
        else :
            # crude model of the sky, accounting for fiber throughput variation
            log.info("sky")
            sqframe = copy.deepcopy(fqframe)
            sky = qproc_sky_subtraction(sqframe,return_skymodel=True)

    if with_spectral_smoothing :

        log.info("spectral smoothing")
        sigma = spectral_smoothing_sigma_length/np.mean(np.gradient(qframe.wave[qframe.nspec//2]))
        log.debug("smoothing sigma in flux bin units = {}".format(sigma))
        hw=int(3*sigma)
        u=(np.arange(2*hw+1)-hw)
        kernel=np.exp(-u**2/sigma**2/2.)
        kernel/=np.sum(kernel)
        nsig=5

        y=np.arange(qframe.flux.shape[1])
        for s in range(qframe.nspec) :
            if sqframe is not None :
                fflux=sqframe.flux[s]
                fivar=sqframe.ivar[s]
            elif fqframe is not None :
                fflux=fqframe.flux[s]
                fivar=fqframe.ivar[s]
            else :
                fflux=qframe.flux[s]
                fivar=qframe.ivar[s]
            sfflux=scipy.signal.fftconvolve(fflux,kernel,"same")
            good=((fivar*(fflux-sfflux)**2)<(spectral_smoothing_nsig**2))
            out=~good
            nout=np.sum(out)
            if nout>0 :
                # recompute sflux while masking outliers
                tflux = fflux.copy()
                tflux[out] = np.interp(y[out],y[good],fflux[good])
                sfflux=scipy.signal.fftconvolve(tflux,kernel,"same")
                # and replace the 'out' region by the original data
                # because we want to keep it to have a fair variance
                sfflux[out] = fflux[out]

            # replace by smooth version (+ possibly average sky)
            if sqframe is not None :
                qframe.flux[s]=(sky[s]+sfflux)*flat[s]
            elif fqframe is not None :
                qframe.flux[s]=sfflux*flat[s]
            else :
                qframe.flux[s]=sfflux

    log.info("project back spectra on image")
    y=np.arange(image.pix.shape[0])

    # cross dispersion profile
    xsig=1.*np.ones((qframe.nspec,image.pix.shape[0]))
    if xytraceset.xsig_vs_wave_traceset :
        log.info("use traceset in PSF for xsig")
        for s in range(xytraceset.nspec) :
            wave = xytraceset.wave_vs_y(s,y)
            xsig[s] = xytraceset.xsig_vs_wave(s,wave)
    else :
        log.info("use default xsig={}".format(np.mean(xsig)))
    # keep only positive flux
    qframe.flux *= (qframe.flux>0.)

    # convert counts per A to counts per pixel
    dwave = np.gradient(qframe.wave,axis=1)
    qframe.flux *= dwave

    # because true profile is not Gaussian and we do not integrate the
    # profile in pixels, we have to apply an adjustment
    empirical_scale = 1.1
    log.info("empirical adjustment of xsig by {:4.2f}".format(empirical_scale))
    xsig *= empirical_scale

    model=np.zeros(image.pix.shape)

    if psf is None : # use simple Gaussian
        log.info("use Gaussian sigma of average = {:4.2f}".format(np.mean(xsig)))
        for s in range(qframe.nspec) :
            x=xytraceset.x_vs_y(s,y)
            numba_proj(model,x,xsig[s],qframe.flux[s])
    else : # this takes a lot of time
        log.debug("Use PSF for projection, but in 1D")
        psf._polyparams['HSIZEY']=0
        psf._polyparams['GHDEGY']=0
        model = psf.project(qframe.wave, qframe.flux, xyrange=(0,image.pix.shape[1],0,image.pix.shape[0]))
        log.debug("done projecting")

    log.info("done")
    return model
<commit_msg>fix indentation<commit_after>'''
Function to provide rapidly a model of a preprocessed image
in order to have a pixel variance estimation that is
almost not correlated with pixel value.
'''

import os
import numpy as np
import copy

import numba
import scipy.interpolate

from desispec.image import Image
from desiutil.log import get_logger
from desispec.io.xytraceset import read_xytraceset
from desispec.qproc.qextract import qproc_boxcar_extraction
from desispec.qproc.qsky import qproc_sky_subtraction
from desispec.qproc.qfiberflat import qproc_apply_fiberflat

@numba.jit
def numba_proj(image,x,sigma,flux) :
    '''
    Add a spectrum to a model of the pixel values in a CCD image assuming a Gaussian cross-dispersion profile.

    Inputs:
       image: 2D numpy array , this array will be modified
       x: 1D numpy array of size image.shape[0], coordinate of center of cross-dispersion profile of spectral trace, in pixel units, for each CCD row
       sigma : 1D numpy array of size image.shape[0], sigma of cross-dispersion profile of spectral trace, in pixel units, for each CCD row
       flux : 1D numpy array of size image.shape[0], quantity to project (for design usage: electrons per pixel) for each CCD row
    '''
    n0=image.shape[0]
    n1=image.shape[1]
    hw=3
    prof=np.zeros(2*hw+1)
    for j in range(n0) :
        sumprof=0.
        b=max(0,int(x[j]-hw))
        e=min(n1,int(x[j]+hw+1))
        for i in range(b,e) :
            prof[i-b]=np.exp(-(i-x[j])**2/2./sigma[j]**2)
            sumprof += prof[i-b]
        for i in range(b,e) :
            image[j,i] += flux[j]*prof[i-b]/sumprof

def compute_image_model(image,xytraceset,fiberflat=None,fibermap=None,with_spectral_smoothing=True,with_sky_model=True,\
                        spectral_smoothing_sigma_length=10.,spectral_smoothing_nsig=4.,psf=None):
    '''
    Returns a model of the input image, using a fast extraction, a processing of
    spectra with a common sky model and a smoothing, followed by a reprojection
    on the CCD image.

    Inputs:
       image: a preprocessed image in the form of a desispec.image.Image object
       xytraceset: a desispec.xytraceset.XYTraceSet object with trace coordinates

    Optional:
       fiberflat: a desispec.fiberflat.FiberFlat object
       with_spectral_smoothing: try and smooth the spectra to reduce noise (and eventualy reduce variance correlation)
       with_sky_model: use a sky model as part of the spectral modeling to reduce the noise (requires a fiberflat)
       spectral_smoothing_sigma_length: sigma of Gaussian smoothing along wavelength in A
       spectral_smoothing_nsig: number of sigma rejection threshold to fall back to the original extracted spectrum instead of the smooth one
       psf: specter.psf.GaussHermitePSF object to be used for the 1D projection (slow, by default=None, in which case a Gaussian profile is used)
    returns:
       a 2D np.array of same shape as image.pix
    '''

    log=get_logger()

    # first perform a fast boxcar extraction
    log.info("extract spectra")
    image.mask = None
    qframe = qproc_boxcar_extraction(xytraceset,image)
    fqframe = None
    sqframe = None
    ratio = None
    if fiberflat is not None :
        log.info("fiberflat")
        fqframe = copy.deepcopy(qframe)
        flat=qproc_apply_fiberflat(fqframe,fiberflat=fiberflat,return_flat=True)
    if with_sky_model :
        if fiberflat is None :
            log.warning("cannot compute and use a sky model without a fiberflat")
        else :
            # crude model of the sky, accounting for fiber throughput variation
            log.info("sky")
            sqframe = copy.deepcopy(fqframe)
            sky = qproc_sky_subtraction(sqframe,return_skymodel=True)

    if with_spectral_smoothing :

        log.info("spectral smoothing")
        sigma = spectral_smoothing_sigma_length/np.mean(np.gradient(qframe.wave[qframe.nspec//2]))
        log.debug("smoothing sigma in flux bin units = {}".format(sigma))
        hw=int(3*sigma)
        u=(np.arange(2*hw+1)-hw)
        kernel=np.exp(-u**2/sigma**2/2.)
        kernel/=np.sum(kernel)
        nsig=5

        y=np.arange(qframe.flux.shape[1])
        for s in range(qframe.nspec) :
            if sqframe is not None :
                fflux=sqframe.flux[s]
                fivar=sqframe.ivar[s]
            elif fqframe is not None :
                fflux=fqframe.flux[s]
                fivar=fqframe.ivar[s]
            else :
                fflux=qframe.flux[s]
                fivar=qframe.ivar[s]
            sfflux=scipy.signal.fftconvolve(fflux,kernel,"same")
            good=((fivar*(fflux-sfflux)**2)<(spectral_smoothing_nsig**2))
            out=~good
            nout=np.sum(out)
            if nout>0 :
                # recompute sflux while masking outliers
                tflux = fflux.copy()
                tflux[out] = np.interp(y[out],y[good],fflux[good])
                sfflux=scipy.signal.fftconvolve(tflux,kernel,"same")
                # and replace the 'out' region by the original data
                # because we want to keep it to have a fair variance
                sfflux[out] = fflux[out]

            # replace by smooth version (+ possibly average sky)
            if sqframe is not None :
                qframe.flux[s]=(sky[s]+sfflux)*flat[s]
            elif fqframe is not None :
                qframe.flux[s]=sfflux*flat[s]
            else :
                qframe.flux[s]=sfflux

    log.info("project back spectra on image")
    y=np.arange(image.pix.shape[0])

    # cross dispersion profile
    xsig=1.*np.ones((qframe.nspec,image.pix.shape[0]))
    if xytraceset.xsig_vs_wave_traceset :
        log.info("use traceset in PSF for xsig")
        for s in range(xytraceset.nspec) :
            wave = xytraceset.wave_vs_y(s,y)
            xsig[s] = xytraceset.xsig_vs_wave(s,wave)
    else :
        log.info("use default xsig={}".format(np.mean(xsig)))
    # keep only positive flux
    qframe.flux *= (qframe.flux>0.)

    # convert counts per A to counts per pixel
    dwave = np.gradient(qframe.wave,axis=1)
    qframe.flux *= dwave

    # because true profile is not Gaussian and we do not integrate the
    # profile in pixels, we have to apply an adjustment
    empirical_scale = 1.1
    log.info("empirical adjustment of xsig by {:4.2f}".format(empirical_scale))
    xsig *= empirical_scale

    model=np.zeros(image.pix.shape)

    if psf is None : # use simple Gaussian
        log.info("use Gaussian sigma of average = {:4.2f}".format(np.mean(xsig)))
        for s in range(qframe.nspec) :
            x=xytraceset.x_vs_y(s,y)
            numba_proj(model,x,xsig[s],qframe.flux[s])
    else : # this takes a lot of time
        log.debug("Use PSF for projection, but in 1D")
        psf._polyparams['HSIZEY']=0
        psf._polyparams['GHDEGY']=0
        model = psf.project(qframe.wave, qframe.flux, xyrange=(0,image.pix.shape[1],0,image.pix.shape[0]))
        log.debug("done projecting")

    log.info("done")
    return model
<|endoftext|>


<commit_before>#- Provided by Data Systems to be called by DOS

import math
import time
from PyAstronomy import pyasl
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, FK5, AltAz, EarthLocation
from astropy.coordinates import Angle, Latitude, Longitude
import astropy.units as u

def get_next_field(dateobs, skylevel, seeing, transparency, previoustiles,
    programname=None):
    """
    Returns structure with information about next field to observe.

    Args:
        dateobs (float): start time of observation in UTC (TAI).
            Could be past, present, or future.
        skylevel: current sky level [counts/s/cm^2/arcsec^2]
        seeing: current astmospheric seeing PSF FWHM [arcsec]
        transparency: current atmospheric transparency
        previoustiles: list of tile IDs previously observed.
        programname (string, optional): if given, the output result will be for
            that program.  Otherwise, next_field_selector() chooses the
            program based upon the current conditions.

    Returns dictionary with keys
        tileid: tile ID [integer]
            --> DOS should just add this to the raw data header
        programname: DESI (or other) program name, e.g. "Dark Time Survey",
            "Bright Galaxy Survey", etc.
            --> DOS should just add this to the raw data header
        telera, teledec: telescope central pointing RA, dec [J2000 degrees]
        exptime: expected exposure time [seconds]
        maxtime: maximum allowable exposure time [seconds]
        fibers: dictionary with the following keys, each of which contains
            a list of 5000 values for each of the positioners
            - ra: RA for each fiber [J2000 degrees]
            - dec: dec for each fiber [J2000 degrees]
            - lambdaref: wavelength to optimize each positioner [Angstrom]
        gfa: dictionary with the following keys, each of which contains
            a list of values for objects detectecable by the GFAs, including
            border regions in RA,dec to assist with acquisition
            - id : ID of GFA for this object
            - ra, dec : RA and dec for each object [J2000 degrees]
            - objtype : 'point', 'extended', 'sky'
                --> point sources with okguide=True can be used for guiding;
                    knowledge of the existence of extended sources may help
                    with acquisition; sky locations are large enough to be
                    used for estimating sky backgrounds.
            - okguide : True if good for guiding
            - mag : magnitude [SDSS r-band AB magnitude]
                --> or a flux instead?

        Additional keys may be present and should be ignored

    e.g. result['fibers']['ra'] gives the 5000 RA locations for the fibers

    Notes:
      * get_next_field() will calculate the LST and moon phase/location
        based upon the input datetime.
      * skylevel, seeing, and transparency are in the filter of the guider.
      * The contents of the returned dictionary should be *everything* needed
        as input to point the telescope and take an exposure.  If that isn't
        true, we need to add more.  An ancillary/test/commissioning program
        that defines all of these quantities (e.g. in a JSON file)
        should be sufficient to take observations.
      * previoustiles is a required input rather than having get_next_field()
        query ObsDB to get the history.  Two reasons:
        - Easier to test without requiring live database
        - Decouples code dependencies
      * result['fibers'] will be pre-calculated by fiber assignment;
        DOS shouldn't care as long as get_next_field is fast (<1 sec).
      * Current expectation is the ObsDB/DOS only tracks the past, i.e. what
        observations were taken, but not which observations we would like to
        take in the future.  As such, get_next_field() will need to look up
        the DESI tiling (currently in desimodel/data/footprint/desi-tiles.*)
        and a list of overrides for tiles that were observed by deemed bad
        and need to be redone (details TBD).
        DOS shouldn't care about those details.

    TBD:
      * Error handling: if the request is impossible (e.g. the sun is up),
        should this raise an exception?  Or return a default zenith answer
        with some calib programname?  Or?
    """

    """
        Below is an algorithm for calculating the local apparent sidereal time.
        astropy should be able to this (I think), but I keep getting an error when I try
        to use those functions (and I'm pretty sure I'm importing all the necessary
        components of astropy). A bunch of lines have been commented out which were
        used for testing purposes.
    """

    #Find the Julian date of the previous midnight
    if (dateobs-math.floor(dateobs) >= 0.5):
        jd_0 = math.floor(dateobs)+0.5
    elif (dateobs-math.floor(dateobs) < 0.5):
        jd_0 = math.floor(dateobs)-0.5

    d_0 = jd_0-2451545.0 #Difference between last Julian midnight and J2000
    d = dateobs-2451545.0 #Difference between observation date and J2000
    t = d_0/36525 #Fraction of Julian century that's past since J2000

    #Calculate the sideral time in Greenwich for the last Julian midnight
    gmst_0 = 100.4606184+36000.77005361*t+0.00038793*t**2-2.6E-08*t**3

    #Add correction for the number of hours that have past since midnight
    gmst = gmst_0+0.25068447733746215*(dateobs-jd_0)*24*60

    #Calculate the equation of equinoxes
    l = 280.47+0.98565*d
    omega = 125.04-0.052954*d
    del_psi = -0.000319*math.sin(omega*math.pi/180)-0.00024*math.sin(2*l*math.pi/180)
    e = 23.4393-0.0000004*d

    eqeq = del_psi*math.cos(e*math.pi/180)

    #Correct with the equation of equinoxes to get the current apparent sidereal time
    gast = gmst+eqeq

    #Add the longitude of the observatory to get the local sidereal time
    last = gast-111.5984796

    #Shift the local sidereal time into the range of 0 to 360 degrees
    if last >= 360:
        n = math.floor(last/360)
        last = last-360*n

    #hour = math.floor(LAST/15)
    #minute = math.floor((LAST/15-hour)*60)
    #second = ((LAST/15-hour)*60-minute)*60

    #print( str(hour) + "h " + str(minute) + "m " + str(second) +"s")

    #print("dateobs = " + str(dateobs))
    #print("JD_0 = " + str(JD_0))
    #print("D_0 = " + str(D_0))
    #print("D = " + str(D))
    #print("T = " + str(T))
    #print("H = " + str(H))
    #print("GMST = " + str(GMST))
    #print("LAST = " + str(LAST))

    #Use PyAstronomy to calculate the position of the Sun.
    pos_sun = pyasl.sunpos(dateobs,full_output=True)

    #print("ra_sun = " + str(float(pos_sun[1])) + " dec_sun = " + str(float(pos_sun[2])))

    #Check to see if the Sun is up.
    ra_sun = float(pos_sun[1])
    dec_sun = float(pos_sun[2])
    ha_sun = last - ra_sun
    if (ha_sun < 0):
        ha_sun = ha_sun+360
    if (ha_sun > 360):
        ha_sun = ha_sun-360
    #print("ha_sun = " + str(ha_sun))

    #Calculate the altitude of the Sun to determine if it is up
    alt_sun = (math.asin(math.sin(dec_sun*math.pi/180)
                         *math.sin(31.9614929*math.pi/180)
                         +math.cos(dec_sun*math.pi/180)
                         *math.cos(31.9614929*math.pi/180)
                         *math.cos(ha_sun*math.pi/180)))*(180/math.pi)

    #print("alt_sun = " + str(alt_sun))

    #Print warning if the Sun is up. We may decide this should do more than just warn
    if (alt_sun >= 0):
        print("WARNING: The sun is currently up!")

    #Find the position of the Moon using PyAstronomy
    pos_moon = pyasl.moonpos(dateobs)

    ra_moon = float(pos_moon[0])
    dec_moon = float(pos_moon[1])

    print("ra_moon = " + str(ra_moon) + " dec_moon = " + str(dec_moon))

    #Find the phase of the moon using PyAstronomy
    moon_phase = pyasl.moonphase(dateobs)
    phase = float(moon_phase)

    print(phase)

    #Opens a file containing information about all of the DESI tiles. Not sure if
    #this is the best way to get that information?
    tiles_file = open("desi-tiles-full.par.txt","r")

    #Declare some variables to store data from the file
    tiles = []
    idnum = []
    ra = []
    dec = []
    passnum = []
    in_desi = []
    ebv_med = []
    airmass = []
    exposefac = []
    i=0
    j=0
    possibletiles = 0
    impossibletiles = 0
    mindec = 100.0
    nextfield = 0
    #Read the data from the file and find the next field
    for line in tiles_file:
        tiles.append(line)
        c = tiles[j].split(" ")
        if (c[0] == "STRUCT1"):
            idnum.append(int(c[1]))
            ra.append(float(c[2]))
            dec.append(float(c[3]))
            passnum.append(int(c[4]))
            in_desi.append(int(c[5]))
            ebv_med.append(float(c[6]))
            airmass.append(float(c[7]))
            exposefac.append(float(c[8]))
            ha = last-ra[i]
            if ha < 0:
                ha = ha + 360
            if ha > 360:
                ha = ha - 360
            alt = (math.asin(math.sin(dec[i]*math.pi/180)
                             *math.sin(31.9614929*math.pi/180)
                             +math.cos(dec[i]*math.pi/180)
                             *math.cos(31.9614929*math.pi/180)
                             *math.cos(ha*math.pi/180)))*(180/math.pi)
            if (alt >= 0):
                if (dec[i] < mindec and ra[i] >= last-1 and ra[i] <= last+1):
                    mindec = dec[i]
                    nextfield = idnum[i]
            i = i + 1
        j = j + 1

    tiles_file.close()

    #raise NotImplementedError

    #Create dictionary with information that is needed to point the telescope
    next_field = {'tileid':idnum[nextfield-1], 'programname':'DESI', 'telera':ra[nextfield-1], 'teledec':dec[nextfield-1], 'exptime':1800, 'maxtime':2000, 'fibers':{}, 'gfa':{}}

    #Return the dictionary
    return next_field

""" The lines below allow the function to be tested by itself with the user
inputting a Julian date of observation. They also calculate the execution time for
purposes of optimizing."""

dateobs = float(raw_input('Enter the date of observation: '))
skylevel = 0
transparency = 0
previoustiles = []
programname = 'DESI'
start_time = time.time()
next_field = get_next_field(dateobs, skylevel, transparency, previoustiles, programname)

print("Total execution time: %s seconds" % (time.time()-start_time))
print next_field
<commit_msg>Switched from PyAstronomy to astropy for calculating the Sun's position at the time of observation<commit_after>#- Provided by Data Systems to be called by DOS

import math
import time
from PyAstronomy import pyasl
from astropy import coordinates
from astropy.time import Time
from astropy.coordinates import SkyCoord
from astropy.coordinates import ICRS, FK5, AltAz, EarthLocation
from astropy.coordinates import Angle, Latitude, Longitude
import astropy.units as u

def get_next_field(dateobs, skylevel, seeing, transparency, previoustiles,
    programname=None):
    """
    Returns structure with information about next field to observe.

    Args:
        dateobs (float): start time of observation in UTC (TAI).
            Could be past, present, or future.
        skylevel: current sky level [counts/s/cm^2/arcsec^2]
        seeing: current astmospheric seeing PSF FWHM [arcsec]
        transparency: current atmospheric transparency
        previoustiles: list of tile IDs previously observed.
        programname (string, optional): if given, the output result will be for
            that program.  Otherwise, next_field_selector() chooses the
            program based upon the current conditions.

    Returns dictionary with keys
        tileid: tile ID [integer]
            --> DOS should just add this to the raw data header
        programname: DESI (or other) program name, e.g. "Dark Time Survey",
            "Bright Galaxy Survey", etc.
            --> DOS should just add this to the raw data header
        telera, teledec: telescope central pointing RA, dec [J2000 degrees]
        exptime: expected exposure time [seconds]
        maxtime: maximum allowable exposure time [seconds]
        fibers: dictionary with the following keys, each of which contains
            a list of 5000 values for each of the positioners
            - ra: RA for each fiber [J2000 degrees]
            - dec: dec for each fiber [J2000 degrees]
            - lambdaref: wavelength to optimize each positioner [Angstrom]
        gfa: dictionary with the following keys, each of which contains
            a list of values for objects detectecable by the GFAs, including
            border regions in RA,dec to assist with acquisition
            - id : ID of GFA for this object
            - ra, dec : RA and dec for each object [J2000 degrees]
            - objtype : 'point', 'extended', 'sky'
                --> point sources with okguide=True can be used for guiding;
                    knowledge of the existence of extended sources may help
                    with acquisition; sky locations are large enough to be
                    used for estimating sky backgrounds.
            - okguide : True if good for guiding
            - mag : magnitude [SDSS r-band AB magnitude]
                --> or a flux instead?

        Additional keys may be present and should be ignored

    e.g. result['fibers']['ra'] gives the 5000 RA locations for the fibers

    Notes:
      * get_next_field() will calculate the LST and moon phase/location
        based upon the input datetime.
      * skylevel, seeing, and transparency are in the filter of the guider.
      * The contents of the returned dictionary should be *everything* needed
        as input to point the telescope and take an exposure.  If that isn't
        true, we need to add more.  An ancillary/test/commissioning program
        that defines all of these quantities (e.g. in a JSON file)
        should be sufficient to take observations.
      * previoustiles is a required input rather than having get_next_field()
        query ObsDB to get the history.  Two reasons:
        - Easier to test without requiring live database
        - Decouples code dependencies
      * result['fibers'] will be pre-calculated by fiber assignment;
        DOS shouldn't care as long as get_next_field is fast (<1 sec).
      * Current expectation is the ObsDB/DOS only tracks the past, i.e. what
        observations were taken, but not which observations we would like to
        take in the future.  As such, get_next_field() will need to look up
        the DESI tiling (currently in desimodel/data/footprint/desi-tiles.*)
        and a list of overrides for tiles that were observed by deemed bad
        and need to be redone (details TBD).
        DOS shouldn't care about those details.

    TBD:
      * Error handling: if the request is impossible (e.g. the sun is up),
        should this raise an exception?  Or return a default zenith answer
        with some calib programname?  Or?
    """

    """
        Below is an algorithm for calculating the local apparent sidereal time.
        astropy should be able to this (I think), but I keep getting an error when I try
        to use those functions (and I'm pretty sure I'm importing all the necessary
        components of astropy). A bunch of lines have been commented out which were
        used for testing purposes.
    """

    tobs = Time(dateobs, format='jd', scale='utc')

    #Find the Julian date of the previous midnight
    if (dateobs-math.floor(dateobs) >= 0.5):
        jd_0 = math.floor(dateobs)+0.5
    elif (dateobs-math.floor(dateobs) < 0.5):
        jd_0 = math.floor(dateobs)-0.5

    d_0 = jd_0-2451545.0 #Difference between last Julian midnight and J2000
    d = dateobs-2451545.0 #Difference between observation date and J2000
    t = d_0/36525 #Fraction of Julian century that's past since J2000

    #Calculate the sideral time in Greenwich for the last Julian midnight
    gmst_0 = 100.4606184+36000.77005361*t+0.00038793*t**2-2.6E-08*t**3

    #Add correction for the number of hours that have past since midnight
    gmst = gmst_0+0.25068447733746215*(dateobs-jd_0)*24*60

    #Calculate the equation of equinoxes
    l = 280.47+0.98565*d
    omega = 125.04-0.052954*d
    del_psi = -0.000319*math.sin(omega*math.pi/180)-0.00024*math.sin(2*l*math.pi/180)
    e = 23.4393-0.0000004*d

    eqeq = del_psi*math.cos(e*math.pi/180)

    #Correct with the equation of equinoxes to get the current apparent sidereal time
    gast = gmst+eqeq

    #Add the longitude of the observatory to get the local sidereal time
    last = gast-111.5984796

    #Shift the local sidereal time into the range of 0 to 360 degrees
    if last >= 360:
        n = math.floor(last/360)
        last = last-360*n

    #hour = math.floor(LAST/15)
    #minute = math.floor((LAST/15-hour)*60)
    #second = ((LAST/15-hour)*60-minute)*60

    #print( str(hour) + "h " + str(minute) + "m " + str(second) +"s")

    #print("dateobs = " + str(dateobs))
    #print("JD_0 = " + str(JD_0))
    #print("D_0 = " + str(D_0))
    #print("D = " + str(D))
    #print("T = " + str(T))
    #print("H = " + str(H))
    #print("GMST = " + str(GMST))
    #print("LAST = " + str(LAST))

    #Use PyAstronomy to calculate the position of the Sun.
    #pos_sun = pyasl.sunpos(dateobs,full_output=True)
    pos_sun = coordinates.get_sun(tobs)

    print pos_sun

    print pos_sun.ra.value
    print pos_sun.dec.value

    #print("ra_sun = " + str(float(pos_sun[1])) + " dec_sun = " + str(float(pos_sun[2])))

    #Check to see if the Sun is up.
    ra_sun = pos_sun.ra.value
    dec_sun = pos_sun.dec.value

    print ra_sun
    print dec_sun

    ha_sun = last - ra_sun
    if (ha_sun < 0):
        ha_sun = ha_sun+360
    if (ha_sun > 360):
        ha_sun = ha_sun-360
    #print("ha_sun = " + str(ha_sun))

    #Calculate the altitude of the Sun to determine if it is up
    alt_sun = (math.asin(math.sin(dec_sun*math.pi/180)
                         *math.sin(31.9614929*math.pi/180)
                         +math.cos(dec_sun*math.pi/180)
                         *math.cos(31.9614929*math.pi/180)
                         *math.cos(ha_sun*math.pi/180)))*(180/math.pi)

    #print("alt_sun = " + str(alt_sun))

    #Print warning if the Sun is up. We may decide this should do more than just warn
    if (alt_sun >= 0):
        print("WARNING: The sun is currently up!")

    #Find the position of the Moon using PyAstronomy
    pos_moon = pyasl.moonpos(dateobs)

    ra_moon = float(pos_moon[0])
    dec_moon = float(pos_moon[1])

    print("ra_moon = " + str(ra_moon) + " dec_moon = " + str(dec_moon))

    #Find the phase of the moon using PyAstronomy
    moon_phase = pyasl.moonphase(dateobs)
    phase = float(moon_phase)

    print(phase)

    #Opens a file containing information about all of the DESI tiles. Not sure if
    #this is the best way to get that information?
    tiles_file = open("desi-tiles-full.par.txt","r")

    #Declare some variables to store data from the file
    tiles = []
    idnum = []
    ra = []
    dec = []
    passnum = []
    in_desi = []
    ebv_med = []
    airmass = []
    exposefac = []
    i = 0
    j = 0
    possibletiles = 0
    impossibletiles = 0
    mindec = 100.0
    nextfield = 0
    #Read the data from the file and find the next field
    for line in tiles_file:
        tiles.append(line)
        c = tiles[j].split(" ")
        if (c[0] == "STRUCT1"):
            idnum.append(int(c[1]))
            ra.append(float(c[2]))
            dec.append(float(c[3]))
            passnum.append(int(c[4]))
            in_desi.append(int(c[5]))
            ebv_med.append(float(c[6]))
            airmass.append(float(c[7]))
            exposefac.append(float(c[8]))
            ha = last-ra[i]
            if ha < 0:
                ha = ha + 360
            if ha > 360:
                ha = ha - 360
            alt = (math.asin(math.sin(dec[i]*math.pi/180)
                             *math.sin(31.9614929*math.pi/180)
                             +math.cos(dec[i]*math.pi/180)
                             *math.cos(31.9614929*math.pi/180)
                             *math.cos(ha*math.pi/180)))*(180/math.pi)
            if (alt >= 0):
                if (dec[i] < mindec and ra[i] >= last-1 and ra[i] <= last+1):
                    mindec = dec[i]
                    nextfield = idnum[i]
            i = i + 1
        j = j + 1

    tiles_file.close()

    #raise NotImplementedError

    #Create dictionary with information that is needed to point the telescope
    next_field = {'tileid':idnum[nextfield-1], 'programname':'DESI', 'telera':ra[nextfield-1], 'teledec':dec[nextfield-1], 'exptime':1800, 'maxtime':2000, 'fibers':{}, 'gfa':{}}

    #Return the dictionary
    return next_field

""" The lines below allow the function to be tested by itself with the user
inputting a Julian date of observation. They also calculate the execution time for
purposes of optimizing."""

dateobs = float(raw_input('Enter the date of observation: '))
skylevel = 0
transparency = 0
previoustiles = []
programname = 'DESI'
start_time = time.time()
next_field = get_next_field(dateobs, skylevel, transparency, previoustiles, programname)

print("Total execution time: %s seconds" % (time.time()-start_time))
print next_field
<|endoftext|>