<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 9, 18)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<commit_msg>compat: Update forward compatibility horizon to 2020-09-19<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 9, 19)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 5, 28)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<commit_msg>compat: Update forward compatibility horizon to 2020-05-29<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 5, 29)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 4, 26)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<commit_msg>compat: Update forward compatibility horizon to 2020-04-27<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 4, 27)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<|endoftext|>


<commit_before># Copyright ClusterHQ Inc.  See LICENSE file for details.

"""
An HTTP API implementing the Docker Volumes Plugin API.

See https://github.com/docker/docker/tree/master/docs/extend for details.
"""

from functools import wraps
from uuid import UUID

import yaml

from bitmath import GiB

from twisted.python.filepath import FilePath
from twisted.internet.task import deferLater

from klein import Klein

from ..restapi import structured
from ..control._config import dataset_id_from_name
from ..apiclient import DatasetAlreadyExists

SCHEMA_BASE = FilePath(__file__).sibling(b'schema')
SCHEMAS = {
    b'/types.json': yaml.safe_load(
        SCHEMA_BASE.child(b'types.yml').getContent()),
    b'/endpoints.json': yaml.safe_load(
        SCHEMA_BASE.child(b'endpoints.yml').getContent()),
    }

# The default size of a created volume. Pick a number that isn't the same
# as devicemapper loopback size (100GiB) so we don't trigger
# https://clusterhq.atlassian.net/browse/FLOC-2889
DEFAULT_SIZE = int(GiB(48).to_Byte().value)

def _endpoint(name, ignore_body=False):
    """
    Decorator factory for API endpoints, adding appropriate JSON in/out
    encoding.

    :param unicode name: The name of the endpoint in the schema.
    :param ignore_body: If true, ignore the contents of the body for all
        HTTP methods, including ``POST``. By default the body is only
        ignored for ``GET`` and ``HEAD``.

    :return: Decorator for a method.
    """
    def decorator(f):
        @wraps(f)
        @structured(
            inputSchema={},
            outputSchema={u"$ref": u"/endpoints.json#/definitions/" + name},
            schema_store=SCHEMAS,
            ignore_body=ignore_body)
        def wrapped(*args, **kwargs):
            return f(*args, **kwargs)
        return wrapped
    return decorator

class VolumePlugin(object):
    """
    An implementation of the Docker Volumes Plugin API.

    We don't validate inputs with a schema since this is pre-existing code
    maintained by someone else, and lacking a schema provided by Docker we
    can't be sure they won't change things in minor ways. We do validate
    outputs to ensure we output the documented requirements.
    """
    _POLL_INTERVAL = 0.05

    app = Klein()

    def __init__(self, reactor, flocker_client, node_id):
        """
        :param IReactorTime reactor: Reactor time interface implementation.
        :param IFlockerAPIV1Client flocker_client: Client that allows
            communication with Flocker.
        :param UUID node_id: The identity of the local node this plugin is
            running on.
        """
        self._reactor = reactor
        self._flocker_client = flocker_client
        self._node_id = node_id

    @app.route("/Plugin.Activate", methods=["POST"])
    @_endpoint(u"PluginActivate", ignore_body=True)
    def plugin_activate(self):
        """
        Return which Docker plugin APIs this object supports.
        """
        return {u"Implements": [u"VolumeDriver"]}

    @app.route("/VolumeDriver.Remove", methods=["POST"])
    @_endpoint(u"Remove")
    def volumedriver_remove(self, Name):
        """
        Remove a Docker volume.

        :param unicode Name: The name of the volume.

        In practice we don't actually delete anything. As a multi-node
        volume driver we want to keep volumes around beyond lifetime of a
        specific container, or even a single node, so we don't delete
        datasets based on information from Docker.

        :return: Result indicating success.
        """
        return {u"Err": None}

    @app.route("/VolumeDriver.Unmount", methods=["POST"])
    @_endpoint(u"Unmount")
    def volumedriver_unmount(self, Name):
        """
        The Docker container is no longer using the given volume.

        :param unicode Name: The name of the volume.

        For now this does nothing. In FLOC-2755 this will release the
        lease acquired for the dataset by the ``VolumeDriver.Mount``
        handler.

        :return: Result indicating success.
        """
        return {u"Err": None}

    @app.route("/VolumeDriver.Create", methods=["POST"])
    @_endpoint(u"Create")
    def volumedriver_create(self, Name):
        """
        Create a volume with the given name.

        We hash the name to give a consistent dataset. This ensures that
        if due to race condition we attempt to create two volumes with
        same name only one will be created.

        We also check for existence of matching dataset based on
        ``"name"`` field in metadata, in case we're talking to cluster
        that has datasets that weren't created with this hashing
        mechanism.

        If there is a duplicate we don't return an error, but rather
        success: we will likely get unneeded creates from Docker since it
        doesn't necessarily know about existing persistent volumes.

        :param unicode Name: The name of the volume.

        :return: Result indicating success.
        """
        listing = self._flocker_client.list_datasets_configuration()

        def got_configured(configured):
            for dataset in configured:
                if dataset.metadata.get(u"name") == Name:
                    raise DatasetAlreadyExists
        listing.addCallback(got_configured)

        creating = listing.addCallback(
            lambda _: self._flocker_client.create_dataset(
                self._node_id, DEFAULT_SIZE, metadata={u"name": Name},
                dataset_id=UUID(dataset_id_from_name(Name))))
        creating.addErrback(lambda reason: reason.trap(DatasetAlreadyExists))
        creating.addCallback(lambda _: {u"Err": None})
        return creating

    def _path_response(self, name):
        """
        Wait until a dataset has been mounted and then return its path.

        :param unicode name: The name of the volume.

        :return: Result that includes the mountpoint.
        """
        # If we ever get rid of dataset_id hashing hack we'll need to
        # lookup the dataset by its metadata, not its id.
        def get_state():
            dataset_id = UUID(dataset_id_from_name(name))
            d = self._flocker_client.list_datasets_state()

            def got_state(datasets):
                datasets = [dataset for dataset in datasets
                            if dataset.dataset_id == dataset_id]
                if datasets and datasets[0].primary == self._node_id:
                    return datasets[0].path
                return deferLater(
                    self._reactor, self._POLL_INTERVAL, get_state)
            d.addCallback(got_state)
            return d

        result = get_state()
        result.addCallback(lambda path: {u"Err": None,
                                         u"Mountpoint": path.path})
        return result

    @app.route("/VolumeDriver.Mount", methods=["POST"])
    @_endpoint(u"Mount")
    def volumedriver_mount(self, Name):
        """
        Move a volume with the given name to the current node and mount it.

        Since we need to return the filesystem path we wait until the
        dataset is mounted locally.

        :param unicode Name: The name of the volume.

        :return: Result that includes the mountpoint.
        """
        dataset_id = UUID(dataset_id_from_name(Name))
        d = self._flocker_client.move_dataset(self._node_id, dataset_id)
        d.addCallback(lambda _: self._path_response(Name))
        return d

    @app.route("/VolumeDriver.Path", methods=["POST"])
    @_endpoint(u"Path")
    def volumedriver_path(self, Name):
        """
        Wait until the dataset is mounted locally, then return its path.

        :param unicode Name: The name of the volume.

        :return: Result indicating success.
        """
        return self._path_response(Name)
<commit_msg>Increase default size to something that won't break Rackspace.<commit_after># Copyright ClusterHQ Inc.  See LICENSE file for details.

"""
An HTTP API implementing the Docker Volumes Plugin API.

See https://github.com/docker/docker/tree/master/docs/extend for details.
"""

from functools import wraps
from uuid import UUID

import yaml

from bitmath import GiB

from twisted.python.filepath import FilePath
from twisted.internet.task import deferLater

from klein import Klein

from ..restapi import structured
from ..control._config import dataset_id_from_name
from ..apiclient import DatasetAlreadyExists

SCHEMA_BASE = FilePath(__file__).sibling(b'schema')
SCHEMAS = {
    b'/types.json': yaml.safe_load(
        SCHEMA_BASE.child(b'types.yml').getContent()),
    b'/endpoints.json': yaml.safe_load(
        SCHEMA_BASE.child(b'endpoints.yml').getContent()),
    }

# The default size of a created volume. Pick a number that isn't the same
# as devicemapper loopback size (100GiB) so we don't trigger
# https://clusterhq.atlassian.net/browse/FLOC-2889 and that is large
# enough to hit Rackspace minimums. This is, obviously, not ideal.
DEFAULT_SIZE = int(GiB(75).to_Byte().value)

def _endpoint(name, ignore_body=False):
    """
    Decorator factory for API endpoints, adding appropriate JSON in/out
    encoding.

    :param unicode name: The name of the endpoint in the schema.
    :param ignore_body: If true, ignore the contents of the body for all
        HTTP methods, including ``POST``. By default the body is only
        ignored for ``GET`` and ``HEAD``.

    :return: Decorator for a method.
    """
    def decorator(f):
        @wraps(f)
        @structured(
            inputSchema={},
            outputSchema={u"$ref": u"/endpoints.json#/definitions/" + name},
            schema_store=SCHEMAS,
            ignore_body=ignore_body)
        def wrapped(*args, **kwargs):
            return f(*args, **kwargs)
        return wrapped
    return decorator

class VolumePlugin(object):
    """
    An implementation of the Docker Volumes Plugin API.

    We don't validate inputs with a schema since this is pre-existing code
    maintained by someone else, and lacking a schema provided by Docker we
    can't be sure they won't change things in minor ways. We do validate
    outputs to ensure we output the documented requirements.
    """
    _POLL_INTERVAL = 0.05

    app = Klein()

    def __init__(self, reactor, flocker_client, node_id):
        """
        :param IReactorTime reactor: Reactor time interface implementation.
        :param IFlockerAPIV1Client flocker_client: Client that allows
            communication with Flocker.
        :param UUID node_id: The identity of the local node this plugin is
            running on.
        """
        self._reactor = reactor
        self._flocker_client = flocker_client
        self._node_id = node_id

    @app.route("/Plugin.Activate", methods=["POST"])
    @_endpoint(u"PluginActivate", ignore_body=True)
    def plugin_activate(self):
        """
        Return which Docker plugin APIs this object supports.
        """
        return {u"Implements": [u"VolumeDriver"]}

    @app.route("/VolumeDriver.Remove", methods=["POST"])
    @_endpoint(u"Remove")
    def volumedriver_remove(self, Name):
        """
        Remove a Docker volume.

        :param unicode Name: The name of the volume.

        In practice we don't actually delete anything. As a multi-node
        volume driver we want to keep volumes around beyond lifetime of a
        specific container, or even a single node, so we don't delete
        datasets based on information from Docker.

        :return: Result indicating success.
        """
        return {u"Err": None}

    @app.route("/VolumeDriver.Unmount", methods=["POST"])
    @_endpoint(u"Unmount")
    def volumedriver_unmount(self, Name):
        """
        The Docker container is no longer using the given volume.

        :param unicode Name: The name of the volume.

        For now this does nothing. In FLOC-2755 this will release the
        lease acquired for the dataset by the ``VolumeDriver.Mount``
        handler.

        :return: Result indicating success.
        """
        return {u"Err": None}

    @app.route("/VolumeDriver.Create", methods=["POST"])
    @_endpoint(u"Create")
    def volumedriver_create(self, Name):
        """
        Create a volume with the given name.

        We hash the name to give a consistent dataset. This ensures that
        if due to race condition we attempt to create two volumes with
        same name only one will be created.

        We also check for existence of matching dataset based on
        ``"name"`` field in metadata, in case we're talking to cluster
        that has datasets that weren't created with this hashing
        mechanism.

        If there is a duplicate we don't return an error, but rather
        success: we will likely get unneeded creates from Docker since it
        doesn't necessarily know about existing persistent volumes.

        :param unicode Name: The name of the volume.

        :return: Result indicating success.
        """
        listing = self._flocker_client.list_datasets_configuration()

        def got_configured(configured):
            for dataset in configured:
                if dataset.metadata.get(u"name") == Name:
                    raise DatasetAlreadyExists
        listing.addCallback(got_configured)

        creating = listing.addCallback(
            lambda _: self._flocker_client.create_dataset(
                self._node_id, DEFAULT_SIZE, metadata={u"name": Name},
                dataset_id=UUID(dataset_id_from_name(Name))))
        creating.addErrback(lambda reason: reason.trap(DatasetAlreadyExists))
        creating.addCallback(lambda _: {u"Err": None})
        return creating

    def _path_response(self, name):
        """
        Wait until a dataset has been mounted and then return its path.

        :param unicode name: The name of the volume.

        :return: Result that includes the mountpoint.
        """
        # If we ever get rid of dataset_id hashing hack we'll need to
        # lookup the dataset by its metadata, not its id.
        def get_state():
            dataset_id = UUID(dataset_id_from_name(name))
            d = self._flocker_client.list_datasets_state()

            def got_state(datasets):
                datasets = [dataset for dataset in datasets
                            if dataset.dataset_id == dataset_id]
                if datasets and datasets[0].primary == self._node_id:
                    return datasets[0].path
                return deferLater(
                    self._reactor, self._POLL_INTERVAL, get_state)
            d.addCallback(got_state)
            return d

        result = get_state()
        result.addCallback(lambda path: {u"Err": None,
                                         u"Mountpoint": path.path})
        return result

    @app.route("/VolumeDriver.Mount", methods=["POST"])
    @_endpoint(u"Mount")
    def volumedriver_mount(self, Name):
        """
        Move a volume with the given name to the current node and mount it.

        Since we need to return the filesystem path we wait until the
        dataset is mounted locally.

        :param unicode Name: The name of the volume.

        :return: Result that includes the mountpoint.
        """
        dataset_id = UUID(dataset_id_from_name(Name))
        d = self._flocker_client.move_dataset(self._node_id, dataset_id)
        d.addCallback(lambda _: self._path_response(Name))
        return d

    @app.route("/VolumeDriver.Path", methods=["POST"])
    @_endpoint(u"Path")
    def volumedriver_path(self, Name):
        """
        Wait until the dataset is mounted locally, then return its path.

        :param unicode Name: The name of the volume.

        :return: Result indicating success.
        """
        return self._path_response(Name)
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Example Connection Command

Make sure you can authenticate before running this command.

For example:
    python -m examples.connection
"""

import sys

import os_client_config

from examples import common
from openstack import connection

def make_connection(opts):
    occ = os_client_config.OpenStackConfig()
    cloud = occ.get_one_cloud(opts.cloud, opts)
    auth = cloud.config['auth']
    conn = connection.Connection(preference=opts.user_preferences, **auth)
    return conn

def run_connection(opts):
    conn = make_connection(opts)
    print("Connection: %s" % conn)
    for flavor in conn.compute.list_flavors():
        print(flavor.id + " " + flavor.name)
    return

if __name__ == "__main__":
    opts = common.setup()
    sys.exit(common.main(opts, run_connection))
<commit_msg>Rename list_flavors flavors in example<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""
Example Connection Command

Make sure you can authenticate before running this command.

For example:
    python -m examples.connection
"""

import sys

import os_client_config

from examples import common
from openstack import connection

def make_connection(opts):
    occ = os_client_config.OpenStackConfig()
    cloud = occ.get_one_cloud(opts.cloud, opts)
    auth = cloud.config['auth']
    conn = connection.Connection(preference=opts.user_preferences, **auth)
    return conn

def run_connection(opts):
    conn = make_connection(opts)
    print("Connection: %s" % conn)
    for flavor in conn.compute.flavors():
        print(flavor.id + " " + flavor.name)
    return

if __name__ == "__main__":
    opts = common.setup()
    sys.exit(common.main(opts, run_connection))
<|endoftext|>


<commit_before>"""
Serializers used in various InvenTree apps
"""

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from rest_framework import serializers
from rest_framework.utils import model_meta

import os

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError as DjangoValidationError

from rest_framework import serializers
from rest_framework.fields import empty
from rest_framework.exceptions import ValidationError

class UserSerializer(serializers.ModelSerializer):
    """ Serializer for User - provides all fields """

    class Meta:
        model = User
        fields = 'all'

class UserSerializerBrief(serializers.ModelSerializer):
    """ Serializer for User - provides limited information """

    class Meta:
        model = User
        fields = [
            'pk',
            'username',
        ]

class InvenTreeModelSerializer(serializers.ModelSerializer):
    """
    Inherits the standard Django ModelSerializer class,
    but also ensures that the underlying model class data are checked on validation.
    """

    def get_initial(self):
        """
        Construct initial data for the serializer.
        Use the 'default' values specified by the django model definition
        """

        initials = super().get_initial()

        # Are we creating a new instance?
        if self.instance is None:
            ModelClass = self.Meta.model

            fields = model_meta.get_field_info(ModelClass)

            for field_name, field in fields.fields.items():

                if field.has_default():
                    initials[field_name] = field.default

        return initials

    def run_validation(self, data=empty):
        """ Perform serializer validation.
        In addition to running validators on the serializer fields,
        this class ensures that the underlying model is also validated.
        """

        # Run any native validation checks first (may raise a ValidationError)
        data = super().run_validation(data)

        # Now ensure the underlying model is correct

        if not hasattr(self, 'instance') or self.instance is None:
            # No instance exists (we are creating a new one)
            instance = self.Meta.model(**data)
        else:
            # Instance already exists (we are updating!)
            instance = self.instance

            # Update instance fields
            for attr, value in data.items():
                setattr(instance, attr, value)

        # Run a 'full_clean' on the model.
        # Note that by default, DRF does *not* perform full model validation!
        try:
            instance.full_clean()
        except (ValidationError, DjangoValidationError) as exc:
            raise ValidationError(detail=serializers.as_serializer_error(exc))

        return data

class InvenTreeAttachmentSerializerField(serializers.FileField):
    """
    Override the DRF native FileField serializer,
    to remove the leading server path.

    For example, the FileField might supply something like:

    http://127.0.0.1:8000/media/foo/bar.jpg

    Whereas we wish to return:

    /media/foo/bar.jpg

    Why? You can't handle the why!

    Actually, if the server process is serving the data at 127.0.0.1,
    but a proxy service (e.g. nginx) is then providing DNS lookup to the outside world,
    then an attachment which prefixes the "address" of the internal server
    will not be accessible from the outside world.
    """

    def to_representation(self, value):

        if not value:
            return None

        return os.path.join(str(settings.MEDIA_URL), str(value))

class InvenTreeImageSerializerField(serializers.ImageField):
    """
    Custom image serializer.
    On upload, validate that the file is a valid image file
    """

    def to_representation(self, value):

        if not value:
            return None

        return os.path.join(str(settings.MEDIA_URL), str(value))
<commit_msg>PEP style fixes<commit_after>"""
Serializers used in various InvenTree apps
"""

# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError as DjangoValidationError

from rest_framework import serializers
from rest_framework.utils import model_meta
from rest_framework.fields import empty
from rest_framework.exceptions import ValidationError

class UserSerializer(serializers.ModelSerializer):
    """ Serializer for User - provides all fields """

    class Meta:
        model = User
        fields = 'all'

class UserSerializerBrief(serializers.ModelSerializer):
    """ Serializer for User - provides limited information """

    class Meta:
        model = User
        fields = [
            'pk',
            'username',
        ]

class InvenTreeModelSerializer(serializers.ModelSerializer):
    """
    Inherits the standard Django ModelSerializer class,
    but also ensures that the underlying model class data are checked on validation.
    """

    def get_initial(self):
        """
        Construct initial data for the serializer.
        Use the 'default' values specified by the django model definition
        """

        initials = super().get_initial()

        # Are we creating a new instance?
        if self.instance is None:
            ModelClass = self.Meta.model

            fields = model_meta.get_field_info(ModelClass)

            for field_name, field in fields.fields.items():

                if field.has_default():
                    initials[field_name] = field.default

        return initials

    def run_validation(self, data=empty):
        """ Perform serializer validation.
        In addition to running validators on the serializer fields,
        this class ensures that the underlying model is also validated.
        """

        # Run any native validation checks first (may raise a ValidationError)
        data = super().run_validation(data)

        # Now ensure the underlying model is correct

        if not hasattr(self, 'instance') or self.instance is None:
            # No instance exists (we are creating a new one)
            instance = self.Meta.model(**data)
        else:
            # Instance already exists (we are updating!)
            instance = self.instance

            # Update instance fields
            for attr, value in data.items():
                setattr(instance, attr, value)

        # Run a 'full_clean' on the model.
        # Note that by default, DRF does *not* perform full model validation!
        try:
            instance.full_clean()
        except (ValidationError, DjangoValidationError) as exc:
            raise ValidationError(detail=serializers.as_serializer_error(exc))

        return data

class InvenTreeAttachmentSerializerField(serializers.FileField):
    """
    Override the DRF native FileField serializer,
    to remove the leading server path.

    For example, the FileField might supply something like:

    http://127.0.0.1:8000/media/foo/bar.jpg

    Whereas we wish to return:

    /media/foo/bar.jpg

    Why? You can't handle the why!

    Actually, if the server process is serving the data at 127.0.0.1,
    but a proxy service (e.g. nginx) is then providing DNS lookup to the outside world,
    then an attachment which prefixes the "address" of the internal server
    will not be accessible from the outside world.
    """

    def to_representation(self, value):

        if not value:
            return None

        return os.path.join(str(settings.MEDIA_URL), str(value))

class InvenTreeImageSerializerField(serializers.ImageField):
    """
    Custom image serializer.
    On upload, validate that the file is a valid image file
    """

    def to_representation(self, value):

        if not value:
            return None

        return os.path.join(str(settings.MEDIA_URL), str(value))
<|endoftext|>


<commit_before># Copyright 2016 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from rosidl_cmake import convert_camel_case_to_lower_case_underscore
from rosidl_cmake import expand_template
from rosidl_cmake import extract_message_types
from rosidl_cmake import get_newest_modification_time
from rosidl_parser import parse_message_file
# from rosidl_parser import parse_service_file
from rosidl_parser import validate_field_types

def generate_typesupport_opensplice_c(args):
    template_dir = args['template_dir']
    mapping_msgs = {
        # os.path.join(template_dir, 'msg__type_support.hpp.template'): '%s__type_support.hpp',
        os.path.join(template_dir, 'msg__type_support_c.cpp.template'): '%s__type_support_c.cpp',
    }

    # mapping_srvs = {
    #     os.path.join(template_dir, 'srv__type_support_c.cpp.template'): '%s__type_support_c.cpp',
    # }

    for template_file in mapping_msgs.keys():
        assert os.path.exists(template_file), 'Could not find template: ' + template_file

    # for template_file in mapping_srvs.keys():
    #     assert os.path.exists(template_file), 'Could not find template: ' + template_file

    pkg_name = args['package_name']
    known_msg_types = extract_message_types(
        pkg_name, args['ros_interface_files'], args.get('ros_interface_dependencies', []))

    functions = {
        'get_header_filename_from_msg_name': convert_camel_case_to_lower_case_underscore,
    }
    # generate_dds_opensplice_cpp() and therefore the make target depend on the additional files
    # therefore they must be listed here even if the generated type support files are independent
    latest_target_timestamp = get_newest_modification_time(
        args['target_dependencies'] + args.get('additional_files', []))

    for idl_file in args['ros_interface_files']:
        extension = os.path.splitext(idl_file)[1]
        if extension == '.msg':
            spec = parse_message_file(pkg_name, idl_file)
            validate_field_types(spec, known_msg_types)
            subfolder = os.path.basename(os.path.dirname(idl_file))
            for template_file, generated_filename in mapping_msgs.items():
                generated_file = os.path.join(
                    args['output_dir'], subfolder, 'dds_opensplice_c', generated_filename %
                    convert_camel_case_to_lower_case_underscore(spec.base_type.type))

                data = {
                    'spec': spec,
                    'pkg': spec.base_type.pkg_name,
                    'msg': spec.msg_name,
                    'type': spec.base_type.type,
                    'subfolder': subfolder,
                }
                data.update(functions)
                try:
                    expand_template(
                        template_file, data, generated_file,
                        minimum_timestamp=latest_target_timestamp)
                except Exception:
                    print("\nException when expanding '{0}' into '{1}':\n"
                          .format(template_file, generated_file))
                    raise

        # elif extension == '.srv':
        #     spec = parse_service_file(pkg_name, idl_file)
        #     validate_field_types(spec, known_msg_types)
        #     for template_file, generated_filename in mapping_srvs.items():
        #         generated_file = os.path.join(
        #             args['output_dir'], 'srv', 'dds_opensplice_c', generated_filename %
        #             convert_camel_case_to_lower_case_underscore(spec.srv_name))

        #         data = {'spec': spec}
        #         data.update(functions)
        #         expand_template(
        #             template_file, data, generated_file,
        #             minimum_timestamp=latest_target_timestamp)

    return 0
<commit_msg>remove unused line<commit_after># Copyright 2016 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from rosidl_cmake import convert_camel_case_to_lower_case_underscore
from rosidl_cmake import expand_template
from rosidl_cmake import extract_message_types
from rosidl_cmake import get_newest_modification_time
from rosidl_parser import parse_message_file
# from rosidl_parser import parse_service_file
from rosidl_parser import validate_field_types

def generate_typesupport_opensplice_c(args):
    template_dir = args['template_dir']
    mapping_msgs = {
        os.path.join(template_dir, 'msg__type_support_c.cpp.template'): '%s__type_support_c.cpp',
    }

    # mapping_srvs = {
    #     os.path.join(template_dir, 'srv__type_support_c.cpp.template'): '%s__type_support_c.cpp',
    # }

    for template_file in mapping_msgs.keys():
        assert os.path.exists(template_file), 'Could not find template: ' + template_file

    # for template_file in mapping_srvs.keys():
    #     assert os.path.exists(template_file), 'Could not find template: ' + template_file

    pkg_name = args['package_name']
    known_msg_types = extract_message_types(
        pkg_name, args['ros_interface_files'], args.get('ros_interface_dependencies', []))

    functions = {
        'get_header_filename_from_msg_name': convert_camel_case_to_lower_case_underscore,
    }
    # generate_dds_opensplice_cpp() and therefore the make target depend on the additional files
    # therefore they must be listed here even if the generated type support files are independent
    latest_target_timestamp = get_newest_modification_time(
        args['target_dependencies'] + args.get('additional_files', []))

    for idl_file in args['ros_interface_files']:
        extension = os.path.splitext(idl_file)[1]
        if extension == '.msg':
            spec = parse_message_file(pkg_name, idl_file)
            validate_field_types(spec, known_msg_types)
            subfolder = os.path.basename(os.path.dirname(idl_file))
            for template_file, generated_filename in mapping_msgs.items():
                generated_file = os.path.join(
                    args['output_dir'], subfolder, 'dds_opensplice_c', generated_filename %
                    convert_camel_case_to_lower_case_underscore(spec.base_type.type))

                data = {
                    'spec': spec,
                    'pkg': spec.base_type.pkg_name,
                    'msg': spec.msg_name,
                    'type': spec.base_type.type,
                    'subfolder': subfolder,
                }
                data.update(functions)
                try:
                    expand_template(
                        template_file, data, generated_file,
                        minimum_timestamp=latest_target_timestamp)
                except Exception:
                    print("\nException when expanding '{0}' into '{1}':\n"
                          .format(template_file, generated_file))
                    raise

        # elif extension == '.srv':
        #     spec = parse_service_file(pkg_name, idl_file)
        #     validate_field_types(spec, known_msg_types)
        #     for template_file, generated_filename in mapping_srvs.items():
        #         generated_file = os.path.join(
        #             args['output_dir'], 'srv', 'dds_opensplice_c', generated_filename %
        #             convert_camel_case_to_lower_case_underscore(spec.srv_name))

        #         data = {'spec': spec}
        #         data.update(functions)
        #         expand_template(
        #             template_file, data, generated_file,
        #             minimum_timestamp=latest_target_timestamp)

    return 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy
import math
import time

from geometry_msgs.msg import Twist
from sensor_msgs.msg import LaserScan

class WallAvoid(object):
    def __init__(self, timeout=None):
        rospy.init_node("WallAvoid")

        self.turnCoef = [(x ** 2 - 8100) / 10000000.0 for x in range(-90, 0)] + [(-x ** 2 + 8100) / 10000000.0 for x in range(0, 91)]
        self.speedCoef = [(-x ** 2 + 8100) / 10000000.0 for x in range(-90,91)]

        rospy.Subscriber("/scan", LaserScan, self._latestScan)
        self.pub = rospy.Publisher("/cmd_vel", Twist, queue_size=10)

        rospy.loginfo("Ready to get out there and avoid some walls!")
        rospy.logdebug(self.turnCoef)
        rospy.logdebug(self.speedCoef)

        self.timeout = None
        if timeout:
            self.timeout = time.time() + timeout

        rospy.spin()

    def isTimedout(self):
        return self.timeout <= time.time()

    def _latestScan(self, data):
        if self.timeout and self.timeout <= time.time():
            rospy.signal_shutdown("Execution timer expired")

        turnVal = 0
        speedVal = 0
        right_zone = data.ranges[0:65]
        front_zone = data.ranges[65:115]
        left_zone = data.ranges[115:180]

        front_zone_avg = sum(front_zone) / len(front_zone)
        right_zone_avg = sum(right_zone) / len(right_zone)
        left_zone_avg = sum(left_zone) / len(left_zone)

        # If average is really REALLY close, might want to back up instead
        if front_zone_avg < 1.5 or min(front_zone) < 0.8:
            speedVal = -0.1
            if left_zone_avg > right_zone_avg and min(left_zone) > min(right_zone):
                rospy.loginfo("Backing up to the left...")
                turnVal = 0.5
            else:
                rospy.loginfo("Backing up to the right...")
                turnVal = -0.3

        else:
            rospy.loginfo("Normal hallway")
            for p in range(0, 181):
                # Inf range return means its over 10m from the LIDAR
                if math.isinf(data.ranges[p]) or math.isnan(data.ranges[p]):
                    speedVal = speedVal + (self.speedCoef[p] * 10)
                    # Don't account long ranges into turn calcs
                else:
                    speedVal = speedVal + (self.speedCoef[p] * data.ranges[p])

                    # Turn away from walls
                    turnVal = turnVal + (self.turnCoef[p] * data.ranges[p])

            speedVal = min(speedVal * 1.2, 0.4) # sets max speed
            turnVal = turnVal * 1.4

            if front_zone_avg < 2.0:
                turnVal = turnVal * 2.0
                speedVal = speedVal * 1.1

        cmd = Twist()
        cmd.linear.x = speedVal
        cmd.angular.z = turnVal

        rospy.loginfo(cmd)

        self.pub.publish(cmd)

# standard ros boilerplate
if __name__ == "__main__":
    try:
        run = WallAvoid() # seconds
    except rospy.ROSInterruptException:
        pass
<commit_msg>Changing turning in a backup to follow frmo before.<commit_after>#!/usr/bin/env python
import rospy
import math
import time

from geometry_msgs.msg import Twist
from sensor_msgs.msg import LaserScan

class WallAvoid(object):
    def __init__(self, timeout=None):
        rospy.init_node("WallAvoid")

        self.turnCoef = [(x ** 2 - 8100) / 10000000.0 for x in range(-90, 0)] + [(-x ** 2 + 8100) / 10000000.0 for x in range(0, 91)]
        self.speedCoef = [(-x ** 2 + 8100) / 10000000.0 for x in range(-90,91)]

        self.last_speed = 0
        self.last_turn = 0

        rospy.Subscriber("/scan", LaserScan, self._latestScan)
        self.pub = rospy.Publisher("/cmd_vel", Twist, queue_size=10)

        rospy.loginfo("Ready to get out there and avoid some walls!")
        rospy.logdebug(self.turnCoef)
        rospy.logdebug(self.speedCoef)

        self.timeout = None
        if timeout:
            self.timeout = time.time() + timeout

        rospy.spin()

    def isTimedout(self):
        return self.timeout <= time.time()

    def _latestScan(self, data):
        if self.timeout and self.timeout <= time.time():
            rospy.signal_shutdown("Execution timer expired")

        turnVal = 0
        speedVal = 0
        right_zone = data.ranges[0:65]
        front_zone = data.ranges[65:115]
        left_zone = data.ranges[115:180]

        front_zone_avg = sum(front_zone) / len(front_zone)
        right_zone_avg = sum(right_zone) / len(right_zone)
        left_zone_avg = sum(left_zone) / len(left_zone)

        # If average is really REALLY close, might want to back up instead
        if front_zone_avg < 1.5 or min(front_zone) < 0.8:
            speedVal = -0.1
            if last_turn > 0
                rospy.loginfo("Backing up to the left...")
                turnVal = 0.5
            else:
                rospy.loginfo("Backing up to the right...")
                turnVal = -0.3

        else:
            rospy.loginfo("Normal hallway")
            for p in range(0, 181):
                # Inf range return means its over 10m from the LIDAR
                if math.isinf(data.ranges[p]) or math.isnan(data.ranges[p]):
                    speedVal = speedVal + (self.speedCoef[p] * 10)
                    # Don't account long ranges into turn calcs
                else:
                    speedVal = speedVal + (self.speedCoef[p] * data.ranges[p])

                    # Turn away from walls
                    turnVal = turnVal + (self.turnCoef[p] * data.ranges[p])

            speedVal = min(speedVal * 1.2, 0.4) # sets max speed
            turnVal = turnVal * 1.4

            if front_zone_avg < 2.0:
                turnVal = turnVal * 2.0
                speedVal = speedVal * 1.1

        cmd = Twist()
        cmd.linear.x = speedVal
        cmd.angular.z = turnVal

        self.last_speed = speedVal
        self.last_turn = turnVal

        rospy.loginfo(cmd)

        self.pub.publish(cmd)

# standard ros boilerplate
if __name__ == "__main__":
    try:
        run = WallAvoid() # seconds
    except rospy.ROSInterruptException:
        pass
<|endoftext|>


<commit_before>from malleefowl import wpslogging as logging
logger = logging.getLogger(__name__)

from os.path import join, abspath, dirname, getsize

DIR_SHP = join(abspath(dirname(__file__)), 'processes', 'shapefiles')

def call(resource=[], variable=None, dimension_map=None, calc=None,
  calc_grouping= None, conform_units_to=None, memory_limit=None,  prefix=None,
  geom=None, output_format_options=True, select_ugid=None, time_region=None, time_range=None,
  dir_output=None, output_format='nc'):
  '''
  ocgis operation call

  :param resource:
  :param variable: variable in the input file to be picked
  :param dimension_map: dimension map in case of unconventional starage of data
  :param calc: ocgis calc syntax for calcultion opartion
  :param calc_grouping: time aggregate grouping
  :param conform_units_to:
  :param memory_limit: limit the amout of data to be loaded into the memory at once if None(default) free memory is detected by birdhouse
  :param prefix:
  :param geom: name of shapefile stored in birdhouse shape cabinet
  :param output_format_options: output options for netCDF e.g compression level()
  :param select_ugid: ugid for appropriate poligons
  :param time_region:
  :param time_range: sequence of two datetime.datetime objects to mark start and end point
  :param dir_output:
  :param output_format:
  :return: output file path
  '''

  logger.info('Start ocgis module call function')
  from ocgis import OcgOperations, RequestDataset , env
  from ocgis.util.large_array import compute

  # prepare the environment
  env.DIR_SHPCABINET = DIR_SHP
  env.OVERWRITE = True
  env.DIR_OUTPUT = dir_output
  env.PREFIX = prefix

  if output_format_options == False:
    output_format_options = None
  elif output_format_options == True:
    output_format_options={'data_model': 'NETCDF4', # NETCDF4_CLASSIC
                         'variable_kwargs': {'zlib': True, 'complevel': 9}}
  else:
    logger.info('output_format_options are set to %s ' % ( output_format_options ))

  if type(resource) != list:
    resource = list([resource])

  # execute ocgis
  logger.info('Execute ocgis module call function')

  try:
    rd = RequestDataset(resource, variable=variable,
      dimension_map=dimension_map, conform_units_to=conform_units_to,
      time_region=time_region)

    ops = OcgOperations(dataset=rd,
        output_format_options=output_format_options,
        #options=options,
        calc=calc,
        calc_grouping=calc_grouping,
        output_format=output_format, # 'nc' is necessary for chunked execution
        select_ugid=select_ugid,
        geom=geom,
        add_auxiliary_files=False)
    logger.info('OcgOperations set')
  except Exception as e:
    logger.error('failed to setup OcgOperations: %s' % (e))

  # check memory load
  from numpy import sqrt
  from flyingpigeon.utils import FreeMemory

  if memory_limit == None:
    f = FreeMemory()
    mem_kb = f.user_free
    mem_mb = mem_kb / 1024.
    mem_limit = mem_mb / 2. # set limit to half of the free memory
  else:
    mem_limit = memory_limit

  if mem_limit >= 1024. * 4:
    mem_limit = 1024. * 4
    # 475.0 MB for openDAP

  data_kb = ops.get_base_request_size()['total']
  data_mb = data_kb / 1024.

  if variable == None:
    variable = rd.variable
    logger.info('%s as variable dedected' % (variable))

  #data_kb = size['total']/reduce(lambda x,y: x*y,size['variables'][variable]['value']['shape'])
  logger.info('data_mb  = %s ; memory_limit = %s ' % (data_mb  , mem_limit ))
  if data_mb <= mem_limit :  # input is smaler than the half of free memory size
    logger.info('ocgis module call as ops.execute()')
    try:
      geom_file = ops.execute()
    except Exception as e:
      logger.error('failed to execute ocgis operation: %s' % e)
  else:
    size = ops.get_base_request_size()
    nb_time_coordinates_rd = size['variables'][variable]['temporal']['shape'][0]
    element_in_kb = size['total']/reduce(lambda x,y: x*y,size['variables'][variable]['value']['shape'])
    element_in_mb = element_in_kb / 1024.

    tile_dim = sqrt(mem_limit/(element_in_mb*nb_time_coordinates_rd)) # maximum chunk size
    # calcultion of chunk size
    try:
      logger.info('tile_dim = %s; calc = %s ' % (tile_dim, calc))
      if calc == None:
        calc = '%s=%s*1' % (variable, variable)
        logger.info('calc set to = %s ' %  calc)
        ops = OcgOperations(dataset=rd,
          output_format_options=output_format_options,
          calc=calc,
          output_format=output_format, # 'nc' is necessary for chunked execution
          select_ugid=select_ugid,
          geom=geom,
          add_auxiliary_files=False)
      geom_file = compute(ops, tile_dimension=int(tile_dim) , verbose=True)
    except Exception as e:
      logger.error('failed to compute ocgis operation: %s' % e)

  logger.info('Succeeded with ocgis module call function')
  return geom_file<commit_msg>ocgis_module netCDF transformation False as defaolt<commit_after>from malleefowl import wpslogging as logging
logger = logging.getLogger(__name__)

from os.path import join, abspath, dirname, getsize

DIR_SHP = join(abspath(dirname(__file__)), 'processes', 'shapefiles')

def call(resource=[], variable=None, dimension_map=None, calc=None,
  calc_grouping= None, conform_units_to=None, memory_limit=None,  prefix=None,
  geom=None, output_format_options=False, select_ugid=None, time_region=None, time_range=None,
  dir_output=None, output_format='nc'):
  '''
  ocgis operation call

  :param resource:
  :param variable: variable in the input file to be picked
  :param dimension_map: dimension map in case of unconventional starage of data
  :param calc: ocgis calc syntax for calcultion opartion
  :param calc_grouping: time aggregate grouping
  :param conform_units_to:
  :param memory_limit: limit the amout of data to be loaded into the memory at once if None(default) free memory is detected by birdhouse
  :param prefix:
  :param geom: name of shapefile stored in birdhouse shape cabinet
  :param output_format_options: output options for netCDF e.g compression level()
  :param select_ugid: ugid for appropriate poligons
  :param time_region:
  :param time_range: sequence of two datetime.datetime objects to mark start and end point
  :param dir_output:
  :param output_format:
  :return: output file path
  '''

  logger.info('Start ocgis module call function')
  from ocgis import OcgOperations, RequestDataset , env
  from ocgis.util.large_array import compute

  # prepare the environment
  env.DIR_SHPCABINET = DIR_SHP
  env.OVERWRITE = True
  env.DIR_OUTPUT = dir_output
  env.PREFIX = prefix

  if output_format_options == False:
    output_format_options = None
  elif output_format_options == True:
    output_format_options={'data_model': 'NETCDF4', # NETCDF4_CLASSIC
                         'variable_kwargs': {'zlib': True, 'complevel': 9}}
  else:
    logger.info('output_format_options are set to %s ' % ( output_format_options ))

  if type(resource) != list:
    resource = list([resource])

  # execute ocgis
  logger.info('Execute ocgis module call function')

  try:
    rd = RequestDataset(resource, variable=variable,
      dimension_map=dimension_map, conform_units_to=conform_units_to,
      time_region=time_region)

    ops = OcgOperations(dataset=rd,
        output_format_options=output_format_options,
        #options=options,
        calc=calc,
        calc_grouping=calc_grouping,
        output_format=output_format, # 'nc' is necessary for chunked execution
        select_ugid=select_ugid,
        geom=geom,
        add_auxiliary_files=False)
    logger.info('OcgOperations set')
  except Exception as e:
    logger.error('failed to setup OcgOperations: %s' % (e))

  # check memory load
  from numpy import sqrt
  from flyingpigeon.utils import FreeMemory

  if memory_limit == None:
    f = FreeMemory()
    mem_kb = f.user_free
    mem_mb = mem_kb / 1024.
    mem_limit = mem_mb / 2. # set limit to half of the free memory
  else:
    mem_limit = memory_limit

  if mem_limit >= 1024. * 4:
    mem_limit = 1024. * 4
    # 475.0 MB for openDAP

  data_kb = ops.get_base_request_size()['total']
  data_mb = data_kb / 1024.

  if variable == None:
    variable = rd.variable
    logger.info('%s as variable dedected' % (variable))

  #data_kb = size['total']/reduce(lambda x,y: x*y,size['variables'][variable]['value']['shape'])
  logger.info('data_mb  = %s ; memory_limit = %s ' % (data_mb  , mem_limit ))
  if data_mb <= mem_limit :  # input is smaler than the half of free memory size
    logger.info('ocgis module call as ops.execute()')
    try:
      geom_file = ops.execute()
    except Exception as e:
      logger.error('failed to execute ocgis operation: %s' % e)
  else:
    size = ops.get_base_request_size()
    nb_time_coordinates_rd = size['variables'][variable]['temporal']['shape'][0]
    element_in_kb = size['total']/reduce(lambda x,y: x*y,size['variables'][variable]['value']['shape'])
    element_in_mb = element_in_kb / 1024.

    tile_dim = sqrt(mem_limit/(element_in_mb*nb_time_coordinates_rd)) # maximum chunk size
    # calcultion of chunk size
    try:
      logger.info('tile_dim = %s; calc = %s ' % (tile_dim, calc))
      if calc == None:
        calc = '%s=%s*1' % (variable, variable)
        logger.info('calc set to = %s ' %  calc)
        ops = OcgOperations(dataset=rd,
          output_format_options=output_format_options,
          calc=calc,
          output_format=output_format, # 'nc' is necessary for chunked execution
          select_ugid=select_ugid,
          geom=geom,
          add_auxiliary_files=False)
      geom_file = compute(ops, tile_dimension=int(tile_dim) , verbose=True)
    except Exception as e:
      logger.error('failed to compute ocgis operation: %s' % e)

  logger.info('Succeeded with ocgis module call function')
  return geom_file<|endoftext|>