<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2021, 2, 25)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<commit_msg>compat: Update forward compatibility horizon to 2021-02-26<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2021, 2, 26)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  if date < _FORWARD_COMPATIBILITY_HORIZON:
    logging.warning("Trying to set the forward compatibility date to the past"
                    " date %s. This will be ignored by TensorFlow." % (date))
    return
  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 3, 21)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<commit_msg>compat: Update forward compatibility horizon to 2020-03-22<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for API compatibility between TensorFlow release versions.

See [Version
Compatibility](https://tensorflow.org/guide/version_compat#backward_forward)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import datetime
import os

from tensorflow.python.util import tf_contextlib
from tensorflow.python.util.tf_export import tf_export

# This value changes every day with an automatic CL. It can be modified in code
# via `forward_compatibility_horizon()` or with the environment variable
# TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date.
_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 3, 22)
_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS"
_FORWARD_COMPATIBILITY_DATE_NUMBER = None

def _date_to_date_number(year, month, day):
  return (year << 9) | (month << 5) | day

def _update_forward_compatibility_date_number(date_to_override=None):
  """Update the base date to compare in forward_compatible function."""

  global _FORWARD_COMPATIBILITY_DATE_NUMBER

  if date_to_override:
    date = date_to_override
  else:
    date = _FORWARD_COMPATIBILITY_HORIZON
    delta_days = os.getenv(_FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME)
    if delta_days:
      date += datetime.timedelta(days=int(delta_days))

  _FORWARD_COMPATIBILITY_DATE_NUMBER = _date_to_date_number(
      date.year, date.month, date.day)

_update_forward_compatibility_date_number()

@tf_export("compat.forward_compatible")
def forward_compatible(year, month, day):
  """Return true if the forward compatibility window has expired.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  Forward-compatibility refers to scenarios where the producer of a TensorFlow
  model (a GraphDef or SavedModel) is compiled against a version of the
  TensorFlow library newer than what the consumer was compiled against. The
  "producer" is typically a Python program that constructs and trains a model
  while the "consumer" is typically another program that loads and serves the
  model.

  TensorFlow has been supporting a 3 week forward-compatibility window for
  programs compiled from source at HEAD.

  For example, consider the case where a new operation `MyNewAwesomeAdd` is
  created with the intent of replacing the implementation of an existing Python
  wrapper - `tf.add`.  The Python wrapper implementation should change from
  something like:

  ```python
  def add(inputs, name=None):
    return gen_math_ops.add(inputs, name)
  ```

  to:

  ```python
  from tensorflow.python.compat import compat

  def add(inputs, name=None):
    if compat.forward_compatible(year, month, day):
      # Can use the awesome new implementation.
      return gen_math_ops.my_new_awesome_add(inputs, name)
    # To maintain forward compatibility, use the old implementation.
    return gen_math_ops.add(inputs, name)
  ```

  Where `year`, `month`, and `day` specify the date beyond which binaries
  that consume a model are expected to have been updated to include the
  new operations. This date is typically at least 3 weeks beyond the date
  the code that adds the new operation is committed.

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Returns:
    True if the caller can expect that serialized TensorFlow graphs produced
    can be consumed by programs that are compiled with the TensorFlow library
    source code after (year, month, day).
  """
  return _FORWARD_COMPATIBILITY_DATE_NUMBER > _date_to_date_number(
      year, month, day)

@tf_export("compat.forward_compatibility_horizon")
@tf_contextlib.contextmanager
def forward_compatibility_horizon(year, month, day):
  """Context manager for testing forward compatibility of generated graphs.

  See [Version
  compatibility](https://tensorflow.org/guide/version_compat#backward_forward).

  To ensure forward compatibility of generated graphs (see `forward_compatible`)
  with older binaries, new features can be gated with:

  ```python
  if compat.forward_compatible(year=2018, month=08, date=01):
    generate_graph_with_new_features()
  else:
    generate_graph_so_older_binaries_can_consume_it()
  ```

  However, when adding new features, one may want to unittest it before
  the forward compatibility window expires. This context manager enables
  such tests. For example:

  ```python
  from tensorflow.python.compat import compat

  def testMyNewFeature(self):
    with compat.forward_compatibility_horizon(2018, 08, 02):
       # Test that generate_graph_with_new_features() has an effect
  ```

  Args:
    year:  A year (e.g., 2018). Must be an `int`.
    month: A month (1 <= month <= 12) in year. Must be an `int`.
    day:   A day (1 <= day <= 31, or 30, or 29, or 28) in month. Must be an
      `int`.

  Yields:
    Nothing.
  """
  try:
    _update_forward_compatibility_date_number(datetime.date(year, month, day))
    yield
  finally:
    _update_forward_compatibility_date_number()
<|endoftext|>


<commit_before># encoding=utf-8
import logging
import re
import requests
import time

from terroroftinytown.client import alphabet
from terroroftinytown.client.errors import (UnhandledStatusCode,
    UnexpectedNoResult, ScraperError, PleaseRetry)

_logger = logging.getLogger(__name__)

class Scraper(object):
    '''URL shortner scraper.

    Args:
        shortener_params (dict): The mapping has the keys:

            * url_template (str)
            * alphabet (str)
            * redirect_codes (list)
            * no_redirect_codes (list)
            * unavailable_codes (list)
            * banned_codes (list)

        todo_list (list): A list of integers.

    '''

    def __init__(self, shortener_params, todo_list):
        self.params = shortener_params
        self.todo_list = todo_list
        self.current_shortcode = None
        self.results = {}

    def run(self):
        while self.todo_list:
            self.scrape_one()
            sleep_time = self.params['request_delay']
            time.sleep(sleep_time)

        return self.results

    def scrape_one(self):
        sequence_number = self.todo_list.pop()
        self.current_shortcode = shortcode = alphabet.int_to_str(
            sequence_number, self.params['alphabet']
        )
        url = self.params['url_template'].format(shortcode=shortcode)

        _logger.info('Requesting %s', url)

        response = self.fetch_url(url)
        result_url = self.process_response(response)

        if result_url is not None:
            _logger.info('Got a result.')
            _logger.debug('%s %s', result_url, response.encoding)

            self.results[shortcode] = {
                'url': result_url,
                'encoding': response.encoding
            }

    def fetch_url(self, url):
        if self.params['method'] == 'get':
            response = requests.get(url, allow_redirects=False)
        else:
            response = requests.head(url, allow_redirects=False)

        return response

    def process_response(self, response):
        status_code = response.status_code

        if status_code in self.params['redirect_codes']:
            return self.process_redirect(response)
        elif status_code in self.params['no_redirect_codes']:
            return self.process_no_redirect(response)
        elif status_code in self.params['unavailable_codes']:
            return self.process_unavailable(response)
        elif status_code in self.params['banned_codes']:
            return self.process_banned(response)
        else:
            return self.process_unknown_code(response)

    def process_redirect(self, response):
        if 'Location' in response.headers:
            result_url = response.headers['Location']
            return result_url
        else:
            return self.process_redirect_body(response)

    def process_redirect_body(self, response):
        pattern = self.params['body_regex']
        match = re.search(pattern, response.text)

        if match:
            return match.group(1)
        else:
            raise UnexpectedNoResult()

    def process_no_redirect(self, response):
        return None

    def process_unavailable(self, response):
        raise ScraperError('Not implemented.')

    def process_banned(self, response):
        raise PleaseRetry()

    def process_unknown_code(self, response):
        raise UnhandledStatusCode(
            'Unknown status code {0}'.format(response.status_code)
        )
<commit_msg>client.scraper: Assume latin-1 if no encoding detected<commit_after># encoding=utf-8
import logging
import re
import requests
import time

from terroroftinytown.client import alphabet
from terroroftinytown.client.errors import (UnhandledStatusCode,
    UnexpectedNoResult, ScraperError, PleaseRetry)

_logger = logging.getLogger(__name__)

class Scraper(object):
    '''URL shortner scraper.

    Args:
        shortener_params (dict): The mapping has the keys:

            * url_template (str)
            * alphabet (str)
            * redirect_codes (list)
            * no_redirect_codes (list)
            * unavailable_codes (list)
            * banned_codes (list)

        todo_list (list): A list of integers.

    '''

    def __init__(self, shortener_params, todo_list):
        self.params = shortener_params
        self.todo_list = todo_list
        self.current_shortcode = None
        self.results = {}

    def run(self):
        while self.todo_list:
            self.scrape_one()
            sleep_time = self.params['request_delay']
            time.sleep(sleep_time)

        return self.results

    def scrape_one(self):
        sequence_number = self.todo_list.pop()
        self.current_shortcode = shortcode = alphabet.int_to_str(
            sequence_number, self.params['alphabet']
        )
        url = self.params['url_template'].format(shortcode=shortcode)

        _logger.info('Requesting %s', url)

        response = self.fetch_url(url)
        result_url = self.process_response(response)

        if result_url is not None:
            _logger.info('Got a result.')
            _logger.debug('%s %s', result_url, response.encoding)

            self.results[shortcode] = {
                'url': result_url,
                'encoding': response.encoding or 'latin-1'
            }

    def fetch_url(self, url):
        if self.params['method'] == 'get':
            response = requests.get(url, allow_redirects=False)
        else:
            response = requests.head(url, allow_redirects=False)

        return response

    def process_response(self, response):
        status_code = response.status_code

        if status_code in self.params['redirect_codes']:
            return self.process_redirect(response)
        elif status_code in self.params['no_redirect_codes']:
            return self.process_no_redirect(response)
        elif status_code in self.params['unavailable_codes']:
            return self.process_unavailable(response)
        elif status_code in self.params['banned_codes']:
            return self.process_banned(response)
        else:
            return self.process_unknown_code(response)

    def process_redirect(self, response):
        if 'Location' in response.headers:
            result_url = response.headers['Location']
            return result_url
        else:
            return self.process_redirect_body(response)

    def process_redirect_body(self, response):
        pattern = self.params['body_regex']
        match = re.search(pattern, response.text)

        if match:
            return match.group(1)
        else:
            raise UnexpectedNoResult()

    def process_no_redirect(self, response):
        return None

    def process_unavailable(self, response):
        raise ScraperError('Not implemented.')

    def process_banned(self, response):
        raise PleaseRetry()

    def process_unknown_code(self, response):
        raise UnhandledStatusCode(
            'Unknown status code {0}'.format(response.status_code)
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
from django.core.management import execute_manager
try:
    import settings # Assumed to be in the same directory.
except ImportError:
    import sys
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

if __name__ == "__main__":
    execute_manager(settings)
<commit_msg>Need to add pubsubpull to the path.<commit_after>#!/usr/bin/env python
from django.core.management import execute_manager
import sys
try:
    import settings # Assumed to be in the same directory.
except ImportError:
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

sys.path.append('../..')

if __name__ == "__main__":
    execute_manager(settings)
<|endoftext|>


<commit_before>#!/usr/bin/python

file = open("story.txt", "r+")

#words = 0
#unique = 0
#paragraphs = 0
#sentences = 0

#def get_words():
#for word in file.read().split():
words = file.read().split()
number_of_words = len(words)
print "Words: {words}".format(words=number_of_words)

#def get_sentences():
sentences = file.read().split('. ')
number_of_sentences = len(sentences)
print "Sentences: {sentences}".format(sentences=number_of_sentences)

#def get_paragraphs():
paragraphs = file.read().split("\n\n")
number_of_paragraphs = len(paragraphs)
print "Paragraphs: {paragraphs}".format(paragraphs=number_of_paragraphs)

print "Unique words and their counts: \n"

#def get_unique_count():
wordcount = {}
for word in file.read().split():
    if word not in wordcount:
        wordcount[word] = 1
    else:
        wordcount[word] += 1
for k, v in wordcount.items():
    print k, v
<commit_msg>updated to utilize functions, fixed unique word count<commit_after>#!/usr/bin/python

file = open("story.txt", "r")
file_contents = file.read()

#Count number of words using split()
def get_words(story):
    words = len(story.split())
    return words

#Count number of sentences by counting terminal punctuation(?,!,.)
def get_sentences(story):
    x = story.count('.')
    y = story.count('?')
    z = story.count('!')
    sentences = x + y + z
    return sentences

def get_paragraphs(story):
    paragraphs = story.split("\n\n")
    number_of_paragraphs = len(paragraphs)

print "Unique words and their counts: \n"

#def get_unique_count():
wordcount = {}
for word in file.read().split():
    if word not in wordcount:
        wordcount[word] = 1
    else:
        wordcount[word] += 1
for k, v in wordcount.items():
    print k, v

print "Total Unique Words: {}".format(get_unique_count(file_contents))
print "Total Word Count: {}".format(get_words(file_contents))
print "Paragraphs: {}".format(get_paragraphs(file_contents))
print "Sentences: {}".format(get_sentences(file_contents))
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_age_limit,
)

class ScreenJunkiesIE(InfoExtractor):
    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
    _TESTS = [{
        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
        'md5': '5c2b686bec3d43de42bde9ec047536b0',
        'info_dict': {
            'id': '2841915',
            'display_id': 'best-quentin-tarantino-movie',
            'ext': 'mp4',
            'title': 'Best Quentin Tarantino Movie',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3671,
            'age_limit': 13,
            'tags': list,
        },
    }, {
        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
        'info_dict': {
            'id': '2348808',
            'display_id': 'honest-trailers-the-dark-knight',
            'ext': 'mp4',
            'title': "Honest Trailers: 'The Dark Knight'",
            'thumbnail': 're:^https?://.*\.jpg',
            'age_limit': 10,
            'tags': list,
        },
    }, {
        # requires subscription but worked around
        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
        'info_dict': {
            'id': '3003285',
            'display_id': 'knocking-dead-ep-1-the-show-so-far',
            'ext': 'mp4',
            'title': 'Knocking Dead Ep 1: State of The Dead Recap',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3307,
            'age_limit': 13,
            'tags': list,
        },
    }]

    _DEFAULT_BITRATES = (48, 150, 496, 864, 2240)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        if not video_id:
            webpage = self._download_webpage(url, display_id)
            video_id = self._search_regex(
                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
                webpage, 'video id')

        webpage = self._download_webpage(
            'http://www.screenjunkies.com/embed/%s' % video_id,
            display_id, 'Downloading video embed page')
        embed_vars = self._parse_json(
            self._search_regex(
                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
            display_id)

        title = embed_vars['contentName']

        formats = []
        bitrates = []
        for f in embed_vars.get('media', []):
            if not f.get('uri') or f.get('mediaPurpose') != 'play':
                continue
            bitrate = int_or_none(f.get('bitRate'))
            if bitrate:
                bitrates.append(bitrate)
            formats.append({
                'url': f['uri'],
                'format_id': 'http-%d' % bitrate if bitrate else 'http',
                'width': int_or_none(f.get('width')),
                'height': int_or_none(f.get('height')),
                'tbr': bitrate,
                'format': 'mp4',
            })

        if not bitrates:
            # When subscriptionLevel > 0, i.e. plus subscription is required
            # media list will be empty. However, hds and hls uri are still
            # available. We can grabbing them assuming bitrates to be default.
            bitrates = self._DEFAULT_BITRATES

        auth_token = embed_vars.get('AuthToken')

        def construct_manifest_url(base_url, ext):
            pieces = [base_url]
            pieces.extend([compat_str(b) for b in bitrates])
            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
            return ','.join(pieces)

        if bitrates and auth_token:
            hds_url = embed_vars.get('hdsUri')
            if hds_url:
                f4m_formats = self._extract_f4m_formats(
                    construct_manifest_url(hds_url, 'f4m'),
                    display_id, f4m_id='hds', fatal=False)
                if len(f4m_formats) == len(bitrates):
                    for f, bitrate in zip(f4m_formats, bitrates):
                        if not f.get('tbr'):
                            f['format_id'] = 'hds-%d' % bitrate
                            f['tbr'] = bitrate
                # TODO: fix f4m downloader to handle manifests without bitrates if possible
                # formats.extend(f4m_formats)

            hls_url = embed_vars.get('hlsUri')
            if hls_url:
                formats.extend(self._extract_m3u8_formats(
                    construct_manifest_url(hls_url, 'm3u8'),
                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': embed_vars.get('thumbUri'),
            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
            'tags': embed_vars.get('tags', '').split(','),
            'formats': formats,
        }
<commit_msg>[screenjunkies] Fix spelling<commit_after>from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    parse_age_limit,
)

class ScreenJunkiesIE(InfoExtractor):
    _VALID_URL = r'http://www.screenjunkies.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
    _TESTS = [{
        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
        'md5': '5c2b686bec3d43de42bde9ec047536b0',
        'info_dict': {
            'id': '2841915',
            'display_id': 'best-quentin-tarantino-movie',
            'ext': 'mp4',
            'title': 'Best Quentin Tarantino Movie',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3671,
            'age_limit': 13,
            'tags': list,
        },
    }, {
        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
        'info_dict': {
            'id': '2348808',
            'display_id': 'honest-trailers-the-dark-knight',
            'ext': 'mp4',
            'title': "Honest Trailers: 'The Dark Knight'",
            'thumbnail': 're:^https?://.*\.jpg',
            'age_limit': 10,
            'tags': list,
        },
    }, {
        # requires subscription but worked around
        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
        'info_dict': {
            'id': '3003285',
            'display_id': 'knocking-dead-ep-1-the-show-so-far',
            'ext': 'mp4',
            'title': 'Knocking Dead Ep 1: State of The Dead Recap',
            'thumbnail': 're:^https?://.*\.jpg',
            'duration': 3307,
            'age_limit': 13,
            'tags': list,
        },
    }]

    _DEFAULT_BITRATES = (48, 150, 496, 864, 2240)

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')
        display_id = mobj.group('display_id')

        if not video_id:
            webpage = self._download_webpage(url, display_id)
            video_id = self._search_regex(
                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
                webpage, 'video id')

        webpage = self._download_webpage(
            'http://www.screenjunkies.com/embed/%s' % video_id,
            display_id, 'Downloading video embed page')
        embed_vars = self._parse_json(
            self._search_regex(
                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
            display_id)

        title = embed_vars['contentName']

        formats = []
        bitrates = []
        for f in embed_vars.get('media', []):
            if not f.get('uri') or f.get('mediaPurpose') != 'play':
                continue
            bitrate = int_or_none(f.get('bitRate'))
            if bitrate:
                bitrates.append(bitrate)
            formats.append({
                'url': f['uri'],
                'format_id': 'http-%d' % bitrate if bitrate else 'http',
                'width': int_or_none(f.get('width')),
                'height': int_or_none(f.get('height')),
                'tbr': bitrate,
                'format': 'mp4',
            })

        if not bitrates:
            # When subscriptionLevel > 0, i.e. plus subscription is required
            # media list will be empty. However, hds and hls uris are still
            # available. We can grab them assuming bitrates to be default.
            bitrates = self._DEFAULT_BITRATES

        auth_token = embed_vars.get('AuthToken')

        def construct_manifest_url(base_url, ext):
            pieces = [base_url]
            pieces.extend([compat_str(b) for b in bitrates])
            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
            return ','.join(pieces)

        if bitrates and auth_token:
            hds_url = embed_vars.get('hdsUri')
            if hds_url:
                f4m_formats = self._extract_f4m_formats(
                    construct_manifest_url(hds_url, 'f4m'),
                    display_id, f4m_id='hds', fatal=False)
                if len(f4m_formats) == len(bitrates):
                    for f, bitrate in zip(f4m_formats, bitrates):
                        if not f.get('tbr'):
                            f['format_id'] = 'hds-%d' % bitrate
                            f['tbr'] = bitrate
                # TODO: fix f4m downloader to handle manifests without bitrates if possible
                # formats.extend(f4m_formats)

            hls_url = embed_vars.get('hlsUri')
            if hls_url:
                formats.extend(self._extract_m3u8_formats(
                    construct_manifest_url(hls_url, 'm3u8'),
                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
        self._sort_formats(formats)

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'thumbnail': embed_vars.get('thumbUri'),
            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
            'tags': embed_vars.get('tags', '').split(','),
            'formats': formats,
        }
<|endoftext|>


<commit_before>import os
from slugify import slugify

from shutil import copyfile
from zipfile import ZipFile

from flask import current_app
from flask_mail import Message

from zou.app import config
from zou.app.stores import file_store

from zou.app.models.build_job import BuildJob
from zou.app.models.playlist import Playlist
from zou.app.models.preview_file import PreviewFile
from zou.app.models.task import Task
from zou.app.models.task_type import TaskType

from zou.app.utils import fields, movie_utils, events

from zou.app.services import (
    base_service,
    files_service,
    projects_service,
    shots_service,
    tasks_service,
    names_service
)

from zou.app.services.exception import (
    BuildJobNotFoundException,
    PlaylistNotFoundException
)

def all_playlists_for_project(project_id):
    """
    Return all playlists created for given project.
    """
    return fields.serialize_value(Playlist.get_all_by(project_id=project_id))

def all_playlists_for_episode(episode_id):
    """
    Return all playlists created for given project.
    """
    return fields.serialize_value(Playlist.get_all_by(episode_id=episode_id))

def get_playlist_with_preview_file_revisions(playlist_id):
    """
    Return given playlist. Shot list is augmented with all previews available
    for a given shot.
    """
    playlist = Playlist.get(playlist_id)

    if playlist is None:
        raise PlaylistNotFoundException()

    playlist_dict = playlist.serialize()

    playlist_dict["build_jobs"] = []
    for build_job in reversed(playlist.build_jobs):
        playlist_dict["build_jobs"].append(fields.serialize_dict({
          "id": build_job.id,
          "status": build_job.status,
          "created_at": build_job.created_at
        }))

    if playlist_dict["shots"] is None:
        playlist_dict["shots"] = []

    (
        playlist_dict,
        preview_file_map
    ) = set_preview_files_for_shots(playlist_dict)

    for shot in playlist_dict["shots"]:
        try:
            preview_file = preview_file_map.get(shot["preview_file_id"], None)
            if preview_file is not None:
                shot["preview_file_id"] = str(preview_file.id)
                shot["extension"] = preview_file.extension
                shot["annotations"] = preview_file.annotations
                shot["task_id"] = fields.serialize_value(preview_file.task_id)
            else:
                del shot["preview_file_id"]
        except Exception as e:
            print(e)

    return playlist_dict

def set_preview_files_for_shots(playlist_dict):
    """
    """
    shot_ids = [
        shot["shot_id"] for shot in playlist_dict["shots"]
    ]
    previews = {}
    preview_file_map = {}

    preview_files = PreviewFile.query \
        .filter_by(extension="mp4") \
        .join(Task) \
        .join(TaskType) \
        .filter(Task.entity_id.in_(shot_ids)) \
        .order_by(TaskType.priority.desc()) \
        .order_by(TaskType.name) \
        .order_by(PreviewFile.revision.desc()) \
        .add_column(Task.task_type_id) \
        .add_column(Task.entity_id) \
        .all()

    for (
        preview_file,
        task_type_id,
        shot_id
    ) in preview_files:
        shot_id = str(shot_id)
        task_type_id = str(task_type_id)
        if shot_id not in previews:
            previews[shot_id] = {}

        if task_type_id not in previews[shot_id]:
            previews[shot_id][task_type_id] = []

        task_id = str(preview_file.task_id)
        preview_file_id = str(preview_file.id)

        previews[shot_id][task_type_id].append({
            "id": preview_file_id,
            "revision": preview_file.revision,
            "extension": preview_file.extension,
            "annotations": preview_file.annotations,
            "created_at": fields.serialize_value(
                preview_file.created_at
            ),
            "task_id": task_id
        })  # Do not add too much field to avoid building too big responses

        preview_file_map[preview_file_id] = preview_file

    for shot in playlist_dict["shots"]:
        if str(shot["shot_id"]) in previews:
            shot["preview_files"] = previews[str(shot["shot_id"])]
        else:
            shot["preview_files"] = []

    return (
        fields.serialize_value(playlist_dict),
        preview_file_map
    )

def get_preview_files_for_shot(shot_id):
    """
    Get all preview files available for given shot.
    """
    tasks = tasks_service.get_tasks_for_shot(shot_id)
    previews = {}

    for task in tasks:
        preview_files = PreviewFile.query \
            .filter_by(task_id=task["id"]) \
            .filter_by(extension="mp4") \
            .join(Task) \
            .join(TaskType) \
            .order_by(TaskType.priority.desc()) \
            .order_by(TaskType.name) \
            .order_by(PreviewFile.revision.desc()) \
            .all()
        task_type_id = task["task_type_id"]

        if len(preview_files) > 0:
            previews[task_type_id] = [
                {
                    "id": str(preview_file.id),
                    "revision": preview_file.revision,
                    "extension": preview_file.extension,
                    "annotations": preview_file.annotations,
                    "created_at": fields.serialize_value(
                        preview_file.created_at
                    ),
                    "task_id": str(preview_file.task_id)
                }
                for preview_file in preview_files
            ]  # Do not add too much field to avoid building too big responses

    return previews

def get_preview_files_for_entity(entity_id):
    """
    Get all preview files available for given entity.
    """
    tasks = tasks_service.get_task_dicts_for_entity(entity_id)
    previews = {}

    for task in tasks:
        preview_files = PreviewFile.query \
            .filter_by(task_id=task["id"]) \
            .filter_by(extension="mp4") \
            .join(Task) \
            .join(TaskType) \
            .order_by(TaskType.priority.desc()) \
            .order_by(TaskType.name) \
            .order_by(PreviewFile.revision.desc()) \
            .all()
        task_type_id = task["task_type_id"]

        if len(preview_files) > 0:
            previews[task_type_id] = [
                {
                    "id": str(preview_file.id),
                    "revision": preview_file.revision,
                    "extension": preview_file.extension,
                    "task_id": str(preview_file.task_id)
                }
                for preview_file in preview_files
            ]  # Do not add too much field to avoid building too big responses

    return previews

def get_playlist_raw(playlist_id):
    """
    Return given playlist as active record.
    """
    return base_service.get_instance(
        Playlist,
        playlist_id,
        PlaylistNotFoundException
    )

def get_playlist(playlist_id):
    """
    Return given playlist as a dict.
    """
    return get_playlist_raw(playlist_id).serialize()

def retrieve_playlist_tmp_files(playlist):
    """
    Retrieve all files for a given playlist into the temporary folder.
    """
    preview_file_ids = []
    for shot in playlist["shots"]:
        if "preview_file_id" in shot \
           and shot["preview_file_id"] is not None \
           and len(shot["preview_file_id"]) > 0:
            preview_file = \
                files_service.get_preview_file(shot["preview_file_id"])
            if preview_file is not None and preview_file.extension == "mp4":
                preview_file_ids.append(preview_file["id"])

    file_paths = []
    for preview_file_id in preview_file_ids:
        if config.FS_BACKEND == "local":
            file_path = file_store.get_local_movie_path(
                "previews",
                preview_file_id
            )
        else:
            file_path = os.path.join(
                config.TMP_DIR,
                "cache-previews-%s.%s" % (preview_file_id, "mp4")
            )
            if not os.path.exists(file_path) \
               or os.path.getsize(file_path) == 0:
                with open(file_path, 'wb') as tmp_file:
                    for chunk in file_store.open_movie(
                        "previews",
                        preview_file_id
                    ):
                        tmp_file.write(chunk)

        file_name = names_service.get_preview_file_name(preview_file_id)
        tmp_file_path = os.path.join(
            config.TMP_DIR,
            file_name
        )
        copyfile(file_path, tmp_file_path)
        file_paths.append((tmp_file_path, file_name))
    return file_paths

def build_playlist_zip_file(playlist):
    """
    Build a zip for all files for a given playlist into the temporary folder.
    """
    tmp_file_paths = retrieve_playlist_tmp_files(playlist)
    zip_file_path = get_playlist_zip_file_path(playlist)
    if os.path.exists(zip_file_path):
        os.remove(zip_file_path)
    with ZipFile(zip_file_path, 'w') as zip:
        for file_path, file_name in tmp_file_paths:
            zip.write(file_path, file_name)
    return zip_file_path

def build_playlist_movie_file(playlist):
    """
    Build a movie for all files for a given playlist into the temporary folder.
    """
    job = start_build_job(playlist)
    project = projects_service.get_project(playlist["project_id"])
    tmp_file_paths = retrieve_playlist_tmp_files(playlist)
    movie_file_path = get_playlist_movie_file_path(playlist, job)
    (width, height) = shots_service.get_preview_dimensions(project)
    fps = shots_service.get_preview_fps(project)

    result = movie_utils.build_playlist_movie(
        tmp_file_paths,
        movie_file_path,
        width,
        height,
        fps
    )
    file_store.add_movie("playlists", job["id"], movie_file_path)
    end_build_job(playlist, job, result)
    return job

def start_build_job(playlist):
    """
    clients that a new job is running.
    Register in database that a new build is running. Emits an event to notify
    """
    job = BuildJob.create(
        status="running",
        job_type="movie",
        playlist_id=playlist["id"]
    )
    events.emit("build-job:new", {
        "build_job_id": str(job.id),
        "playlist_id": playlist["id"],
        "created_at": fields.serialize_value(job.created_at)
    })
    return job.serialize()

def end_build_job(playlist, job, result):
    """
    Register in database that a build is finished. Emits an event to notify
    clients that the build is done.
    """
    build_job = BuildJob.get(job["id"])
    build_job.end()
    status = "succeeded"
    if not result["success"]:
        status = "failed"
    events.emit("build-job:update", {
        "build_job_id": str(build_job.id),
        "playlist_id": playlist["id"],
        "status": status
    })
    return build_job.serialize()

def build_playlist_job(playlist, email):
    """
    Build playlist file (concatenate all mnovie previews). This function is
    aimed at being runned as a job in a job queue.
    """
    from zou.app import app, mail
    with app.app_context():
        job = None
        try:
            job = build_playlist_movie_file(playlist)
        except:
            if job is not None:
                end_build_job(playlist, job)
            raise

        message_text = """
Your playlist %s is available at:
https://%s/api/data/playlists/%s/jobs/%s/build/mp4
""" % (playlist["name"], config.DOMAIN_NAME, playlist["id"], job["id"])
        message = Message(
            body=message_text,
            subject="CGWire playlist download",
            recipients=[email]
        )
        mail.send(message)

def get_playlist_file_name(playlist):
    """
    Build file name for the movie file matching given playlist.
    """
    project = projects_service.get_project(playlist["project_id"])
    attachment_filename = "%s_%s" % (
        slugify(project["name"]),
        slugify(playlist["name"]),
    )
    return slugify(attachment_filename)

def get_playlist_movie_file_path(playlist, build_job):
    """
    Build file path for the movie file matching given playlist.
    """
    movie_file_name = "cache-playlists-%s.mp4" % build_job["id"]
    return os.path.join(config.TMP_DIR, movie_file_name)

def get_playlist_zip_file_path(playlist):
    """
    Build file path for the archive file matching given playlist.
    """
    zip_file_name = "%s.zip" % playlist["id"]
    return os.path.join(config.TMP_DIR, zip_file_name)

def get_build_job_raw(build_job_id):
    """
    Return given build job as active record.
    """
    return base_service.get_instance(
        BuildJob,
        build_job_id,
        BuildJobNotFoundException
    )

def get_build_job(build_job_id):
    """
    Return given build job as a dict.
    """
    return get_build_job_raw(build_job_id).serialize()

def remove_playlist(playlist_id):
    """
    Remove given playlist from database (and delete related build jobs).
    """
    playlist = get_playlist_raw(playlist_id)
    playlist_dict = playlist.serialize()
    query = BuildJob.query.filter_by(playlist_id=playlist_id)
    for job in query.all():
        remove_build_job(playlist_dict, job.id)
    playlist.delete()
    events.emit("playlist:delete", {"playlist_id": playlist_dict["id"]})
    return playlist_dict

def remove_build_job(playlist, build_job_id):
    """
    Remove build job from database and remove related temporary file from
    hard drive.
    """
    job = BuildJob.get(build_job_id)
    movie_file_path = get_playlist_movie_file_path(playlist, job.serialize())
    if os.path.exists(movie_file_path):
        os.remove(movie_file_path)
    try:
        file_store.remove_movie("playlists", build_job_id)
    except:
        current_app.logger.error(
            "Playlist file can't be deleted: %s" % build_job_id
        )
    job.delete()
    events.emit("build-job:delete", {
        "build_job_id": build_job_id,
        "playlist_id": playlist["id"]
    })
    return movie_file_path

def get_build_jobs_for_project(project_id):
    """
    Return all build_jobs for given project.
    """
    build_jobs = BuildJob.query \
        .join(Playlist) \
        .filter(Playlist.project_id == project_id)
    return fields.serialize_list(build_jobs)
<commit_msg>Fix playlist building (zip and mov)<commit_after>import os
from slugify import slugify

from shutil import copyfile
from zipfile import ZipFile

from flask import current_app
from flask_mail import Message

from zou.app import config
from zou.app.stores import file_store

from zou.app.models.build_job import BuildJob
from zou.app.models.playlist import Playlist
from zou.app.models.preview_file import PreviewFile
from zou.app.models.task import Task
from zou.app.models.task_type import TaskType

from zou.app.utils import fields, movie_utils, events

from zou.app.services import (
    base_service,
    files_service,
    projects_service,
    shots_service,
    tasks_service,
    names_service
)

from zou.app.services.exception import (
    BuildJobNotFoundException,
    PlaylistNotFoundException
)

def all_playlists_for_project(project_id):
    """
    Return all playlists created for given project.
    """
    return fields.serialize_value(Playlist.get_all_by(project_id=project_id))

def all_playlists_for_episode(episode_id):
    """
    Return all playlists created for given project.
    """
    return fields.serialize_value(Playlist.get_all_by(episode_id=episode_id))

def get_playlist_with_preview_file_revisions(playlist_id):
    """
    Return given playlist. Shot list is augmented with all previews available
    for a given shot.
    """
    playlist = Playlist.get(playlist_id)

    if playlist is None:
        raise PlaylistNotFoundException()

    playlist_dict = playlist.serialize()

    playlist_dict["build_jobs"] = []
    for build_job in reversed(playlist.build_jobs):
        playlist_dict["build_jobs"].append(fields.serialize_dict({
          "id": build_job.id,
          "status": build_job.status,
          "created_at": build_job.created_at
        }))

    if playlist_dict["shots"] is None:
        playlist_dict["shots"] = []

    (
        playlist_dict,
        preview_file_map
    ) = set_preview_files_for_shots(playlist_dict)

    for shot in playlist_dict["shots"]:
        try:
            preview_file = preview_file_map.get(shot["preview_file_id"], None)
            if preview_file is not None:
                shot["preview_file_id"] = str(preview_file.id)
                shot["extension"] = preview_file.extension
                shot["annotations"] = preview_file.annotations
                shot["task_id"] = fields.serialize_value(preview_file.task_id)
            else:
                del shot["preview_file_id"]
        except Exception as e:
            print(e)

    return playlist_dict

def set_preview_files_for_shots(playlist_dict):
    """
    """
    shot_ids = [
        shot["shot_id"] for shot in playlist_dict["shots"]
    ]
    previews = {}
    preview_file_map = {}

    preview_files = PreviewFile.query \
        .filter_by(extension="mp4") \
        .join(Task) \
        .join(TaskType) \
        .filter(Task.entity_id.in_(shot_ids)) \
        .order_by(TaskType.priority.desc()) \
        .order_by(TaskType.name) \
        .order_by(PreviewFile.revision.desc()) \
        .add_column(Task.task_type_id) \
        .add_column(Task.entity_id) \
        .all()

    for (
        preview_file,
        task_type_id,
        shot_id
    ) in preview_files:
        shot_id = str(shot_id)
        task_type_id = str(task_type_id)
        if shot_id not in previews:
            previews[shot_id] = {}

        if task_type_id not in previews[shot_id]:
            previews[shot_id][task_type_id] = []

        task_id = str(preview_file.task_id)
        preview_file_id = str(preview_file.id)

        previews[shot_id][task_type_id].append({
            "id": preview_file_id,
            "revision": preview_file.revision,
            "extension": preview_file.extension,
            "annotations": preview_file.annotations,
            "created_at": fields.serialize_value(
                preview_file.created_at
            ),
            "task_id": task_id
        })  # Do not add too much field to avoid building too big responses

        preview_file_map[preview_file_id] = preview_file

    for shot in playlist_dict["shots"]:
        if str(shot["shot_id"]) in previews:
            shot["preview_files"] = previews[str(shot["shot_id"])]
        else:
            shot["preview_files"] = []

    return (
        fields.serialize_value(playlist_dict),
        preview_file_map
    )

def get_preview_files_for_shot(shot_id):
    """
    Get all preview files available for given shot.
    """
    tasks = tasks_service.get_tasks_for_shot(shot_id)
    previews = {}

    for task in tasks:
        preview_files = PreviewFile.query \
            .filter_by(task_id=task["id"]) \
            .filter_by(extension="mp4") \
            .join(Task) \
            .join(TaskType) \
            .order_by(TaskType.priority.desc()) \
            .order_by(TaskType.name) \
            .order_by(PreviewFile.revision.desc()) \
            .all()
        task_type_id = task["task_type_id"]

        if len(preview_files) > 0:
            previews[task_type_id] = [
                {
                    "id": str(preview_file.id),
                    "revision": preview_file.revision,
                    "extension": preview_file.extension,
                    "annotations": preview_file.annotations,
                    "created_at": fields.serialize_value(
                        preview_file.created_at
                    ),
                    "task_id": str(preview_file.task_id)
                }
                for preview_file in preview_files
            ]  # Do not add too much field to avoid building too big responses

    return previews

def get_preview_files_for_entity(entity_id):
    """
    Get all preview files available for given entity.
    """
    tasks = tasks_service.get_task_dicts_for_entity(entity_id)
    previews = {}

    for task in tasks:
        preview_files = PreviewFile.query \
            .filter_by(task_id=task["id"]) \
            .filter_by(extension="mp4") \
            .join(Task) \
            .join(TaskType) \
            .order_by(TaskType.priority.desc()) \
            .order_by(TaskType.name) \
            .order_by(PreviewFile.revision.desc()) \
            .all()
        task_type_id = task["task_type_id"]

        if len(preview_files) > 0:
            previews[task_type_id] = [
                {
                    "id": str(preview_file.id),
                    "revision": preview_file.revision,
                    "extension": preview_file.extension,
                    "task_id": str(preview_file.task_id)
                }
                for preview_file in preview_files
            ]  # Do not add too much field to avoid building too big responses

    return previews

def get_playlist_raw(playlist_id):
    """
    Return given playlist as active record.
    """
    return base_service.get_instance(
        Playlist,
        playlist_id,
        PlaylistNotFoundException
    )

def get_playlist(playlist_id):
    """
    Return given playlist as a dict.
    """
    return get_playlist_raw(playlist_id).serialize()

def retrieve_playlist_tmp_files(playlist):
    """
    Retrieve all files for a given playlist into the temporary folder.
    """
    preview_file_ids = []
    for shot in playlist["shots"]:
        if "preview_file_id" in shot \
           and shot["preview_file_id"] is not None \
           and len(shot["preview_file_id"]) > 0:
            preview_file = \
                files_service.get_preview_file(shot["preview_file_id"])
            if preview_file is not None and preview_file["extension"] == "mp4":
                preview_file_ids.append(preview_file["id"])

    file_paths = []
    for preview_file_id in preview_file_ids:
        if config.FS_BACKEND == "local":
            file_path = file_store.get_local_movie_path(
                "previews",
                preview_file_id
            )
        else:
            file_path = os.path.join(
                config.TMP_DIR,
                "cache-previews-%s.%s" % (preview_file_id, "mp4")
            )
            if not os.path.exists(file_path) \
               or os.path.getsize(file_path) == 0:
                with open(file_path, 'wb') as tmp_file:
                    for chunk in file_store.open_movie(
                        "previews",
                        preview_file_id
                    ):
                        tmp_file.write(chunk)

        file_name = names_service.get_preview_file_name(preview_file_id)
        tmp_file_path = os.path.join(
            config.TMP_DIR,
            file_name
        )
        copyfile(file_path, tmp_file_path)
        file_paths.append((tmp_file_path, file_name))
    return file_paths

def build_playlist_zip_file(playlist):
    """
    Build a zip for all files for a given playlist into the temporary folder.
    """
    tmp_file_paths = retrieve_playlist_tmp_files(playlist)
    zip_file_path = get_playlist_zip_file_path(playlist)
    if os.path.exists(zip_file_path):
        os.remove(zip_file_path)
    with ZipFile(zip_file_path, 'w') as zip:
        for file_path, file_name in tmp_file_paths:
            zip.write(file_path, file_name)
    return zip_file_path

def build_playlist_movie_file(playlist):
    """
    Build a movie for all files for a given playlist into the temporary folder.
    """
    job = start_build_job(playlist)
    project = projects_service.get_project(playlist["project_id"])
    tmp_file_paths = retrieve_playlist_tmp_files(playlist)
    movie_file_path = get_playlist_movie_file_path(playlist, job)
    (width, height) = shots_service.get_preview_dimensions(project)
    fps = shots_service.get_preview_fps(project)

    result = movie_utils.build_playlist_movie(
        tmp_file_paths,
        movie_file_path,
        width,
        height,
        fps
    )
    file_store.add_movie("playlists", job["id"], movie_file_path)
    end_build_job(playlist, job, result)
    return job

def start_build_job(playlist):
    """
    clients that a new job is running.
    Register in database that a new build is running. Emits an event to notify
    """
    job = BuildJob.create(
        status="running",
        job_type="movie",
        playlist_id=playlist["id"]
    )
    events.emit("build-job:new", {
        "build_job_id": str(job.id),
        "playlist_id": playlist["id"],
        "created_at": fields.serialize_value(job.created_at)
    })
    return job.serialize()

def end_build_job(playlist, job, result):
    """
    Register in database that a build is finished. Emits an event to notify
    clients that the build is done.
    """
    build_job = BuildJob.get(job["id"])
    build_job.end()
    status = "succeeded"
    if not result["success"]:
        status = "failed"
    events.emit("build-job:update", {
        "build_job_id": str(build_job.id),
        "playlist_id": playlist["id"],
        "status": status
    })
    return build_job.serialize()

def build_playlist_job(playlist, email):
    """
    Build playlist file (concatenate all mnovie previews). This function is
    aimed at being runned as a job in a job queue.
    """
    from zou.app import app, mail
    with app.app_context():
        job = None
        try:
            job = build_playlist_movie_file(playlist)
        except:
            if job is not None:
                end_build_job(playlist, job)
            raise

        message_text = """
Your playlist %s is available at:
https://%s/api/data/playlists/%s/jobs/%s/build/mp4
""" % (playlist["name"], config.DOMAIN_NAME, playlist["id"], job["id"])
        message = Message(
            body=message_text,
            subject="CGWire playlist download",
            recipients=[email]
        )
        mail.send(message)

def get_playlist_file_name(playlist):
    """
    Build file name for the movie file matching given playlist.
    """
    project = projects_service.get_project(playlist["project_id"])
    attachment_filename = "%s_%s" % (
        slugify(project["name"]),
        slugify(playlist["name"]),
    )
    return slugify(attachment_filename)

def get_playlist_movie_file_path(playlist, build_job):
    """
    Build file path for the movie file matching given playlist.
    """
    movie_file_name = "cache-playlists-%s.mp4" % build_job["id"]
    return os.path.join(config.TMP_DIR, movie_file_name)

def get_playlist_zip_file_path(playlist):
    """
    Build file path for the archive file matching given playlist.
    """
    zip_file_name = "%s.zip" % playlist["id"]
    return os.path.join(config.TMP_DIR, zip_file_name)

def get_build_job_raw(build_job_id):
    """
    Return given build job as active record.
    """
    return base_service.get_instance(
        BuildJob,
        build_job_id,
        BuildJobNotFoundException
    )

def get_build_job(build_job_id):
    """
    Return given build job as a dict.
    """
    return get_build_job_raw(build_job_id).serialize()

def remove_playlist(playlist_id):
    """
    Remove given playlist from database (and delete related build jobs).
    """
    playlist = get_playlist_raw(playlist_id)
    playlist_dict = playlist.serialize()
    query = BuildJob.query.filter_by(playlist_id=playlist_id)
    for job in query.all():
        remove_build_job(playlist_dict, job.id)
    playlist.delete()
    events.emit("playlist:delete", {"playlist_id": playlist_dict["id"]})
    return playlist_dict

def remove_build_job(playlist, build_job_id):
    """
    Remove build job from database and remove related temporary file from
    hard drive.
    """
    job = BuildJob.get(build_job_id)
    movie_file_path = get_playlist_movie_file_path(playlist, job.serialize())
    if os.path.exists(movie_file_path):
        os.remove(movie_file_path)
    try:
        file_store.remove_movie("playlists", build_job_id)
    except:
        current_app.logger.error(
            "Playlist file can't be deleted: %s" % build_job_id
        )
    job.delete()
    events.emit("build-job:delete", {
        "build_job_id": build_job_id,
        "playlist_id": playlist["id"]
    })
    return movie_file_path

def get_build_jobs_for_project(project_id):
    """
    Return all build_jobs for given project.
    """
    build_jobs = BuildJob.query \
        .join(Playlist) \
        .filter(Playlist.project_id == project_id)
    return fields.serialize_list(build_jobs)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Measures your typing speed in words per minute (WPM).

This file is part of the wpm software.
Copyright 2017 Christian Stigen Larsen

Distributed under the GNU GPL v3 or later. See the file LICENSE.txt for the
full license text. This software makes use of open source software.
"""

__author__ = "Christian Stigen Larsen"
__copyright__ = "Copyright 2017 Christian Stigen Larsen"
__license__ = "GNU GPL v3 or later"
__version__ = "1.30"
<commit_msg>Bumps to v1.31<commit_after>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Measures your typing speed in words per minute (WPM).

This file is part of the wpm software.
Copyright 2017 Christian Stigen Larsen

Distributed under the GNU GPL v3 or later. See the file LICENSE.txt for the
full license text. This software makes use of open source software.
"""

__author__ = "Christian Stigen Larsen"
__copyright__ = "Copyright 2017 Christian Stigen Larsen"
__license__ = "GNU GPL v3 or later"
__version__ = "1.31"
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
# Name:        qual
# Purpose:
#
# Authors:     Maik Heistermann, Stephan Jacobi and Thomas Pfaff
#
# Created:     26.10.2011
# Copyright:   (c) Maik Heistermann, Stephan Jacobi and Thomas Pfaff 2011
# Licence:     The MIT License
#-------------------------------------------------------------------------------
#!/usr/bin/env python

"""
Data Quality
^^^^^^^^^^^^

This module will serve two purposes:

#. provide routines to create simple radar data quality related fields.
#. provide routines to decide which radar pixel to choose based on the
   competing information in different quality fields.

Data is supposed to be stored in 'aligned' arrays. Aligned here means that
all fields are structured such that in each field the data for a certain index
is representative for the same physical target.

Therefore no assumptions are made on the dimensions or shape of the input
fields except that they exhibit the numpy ndarray interface.

.. autosummary::
   :nosignatures:
   :toctree: generated/

    beam_height_ft
    bin_volume

"""

import numpy as np

def beam_height_ft(ranges, elevations, degrees=True, re=6371000):
    """Calculates the height of a radar beam above the antenna according to \
    the 4/3 (four-thirds -> ft) effective Earth radius model.

    Parameters
    ----------
    ranges : array
        the distances of each bin from the radar [m]
    elevations : array
        the elevation angles of each bin from the radar [degrees or radians]
    degrees : bool
        if True (the default) elevation angles are given in degrees and will
        be converted to radians before calculation. If False no transformation
        will be done and elevations has to be given in radians.
    re : float
        earth radius [m]

    Returns
    -------
    output : height of the beam [m]

    Notes
    -----
    The shape of `elevations` and `ranges` may differ in which case numpy's
    broadcasting rules will apply and the shape of `output` will be that of
    the broadcast arrays. See the numpy documentation on how broadcasting works.

    """
    if degrees:
        elev = np.deg2rad(elevations)
    else:
        elev = elevations

    return ((ranges**2*np.cos(elev)**2)/(2*(4./3.)*re))+ranges*np.sin(elev)

def pulse_volume(ranges, h, theta):
    """Calculates the sampling volume of the radar beam per bin depending on \
    range and aperture.

    We assume a cone frustum which has the volume V=(pi/3)*h*(R**2 + R*r + r**2).
    R and r are the radii of the two frustum surface circles. Assuming that the
    pulse width is small compared to the range, we get R=r=tan(theta*pi/180)*range.
    Thus, the pulse volume simpy becomes a the volume of a cylinder with
    V=pi * h * range**2 * tan(theta*pi/180)**2

    Parameters
    ----------
    ranges : array
        the distances of each bin from the radar [m]
    h : float
        puls width (which corresponds to the range resolution [m])
    theta : float
        the aperture angle (beam width) of the radar beam [degree]

    Returns
    -------
    output : volume of radar bins at each range in `ranges` (qubic metres)

    """
    return np.pi * h * (ranges**2) * (np.tan( np.radians(theta) ))**2

if __name__ == '__main__':
    print 'wradlib: Calling module <qual> as main...'
<commit_msg>small change in module docstring<commit_after>#-------------------------------------------------------------------------------
# Name:        qual
# Purpose:
#
# Authors:     Maik Heistermann, Stephan Jacobi and Thomas Pfaff
#
# Created:     26.10.2011
# Copyright:   (c) Maik Heistermann, Stephan Jacobi and Thomas Pfaff 2011
# Licence:     The MIT License
#-------------------------------------------------------------------------------
#!/usr/bin/env python

"""
Data Quality
^^^^^^^^^^^^

This module will serve two purposes:

#. provide routines to create simple radar data quality related fields.
#. provide routines to decide which radar pixel to choose based on the
   competing information in different quality fields.

Data is supposed to be stored in 'aligned' arrays. Aligned here means that
all fields are structured such that in each field the data for a certain index
is representative for the same physical target.

Therefore no assumptions are made on the dimensions or shape of the input
fields except that they exhibit the numpy ndarray interface.

.. autosummary::
   :nosignatures:
   :toctree: generated/

    beam_height_ft
    pulse_volume

"""

import numpy as np

def beam_height_ft(ranges, elevations, degrees=True, re=6371000):
    """Calculates the height of a radar beam above the antenna according to \
    the 4/3 (four-thirds -> ft) effective Earth radius model.

    Parameters
    ----------
    ranges : array
        the distances of each bin from the radar [m]
    elevations : array
        the elevation angles of each bin from the radar [degrees or radians]
    degrees : bool
        if True (the default) elevation angles are given in degrees and will
        be converted to radians before calculation. If False no transformation
        will be done and elevations has to be given in radians.
    re : float
        earth radius [m]

    Returns
    -------
    output : height of the beam [m]

    Notes
    -----
    The shape of `elevations` and `ranges` may differ in which case numpy's
    broadcasting rules will apply and the shape of `output` will be that of
    the broadcast arrays. See the numpy documentation on how broadcasting works.

    """
    if degrees:
        elev = np.deg2rad(elevations)
    else:
        elev = elevations

    return ((ranges**2*np.cos(elev)**2)/(2*(4./3.)*re))+ranges*np.sin(elev)

def pulse_volume(ranges, h, theta):
    """Calculates the sampling volume of the radar beam per bin depending on \
    range and aperture.

    We assume a cone frustum which has the volume V=(pi/3)*h*(R**2 + R*r + r**2).
    R and r are the radii of the two frustum surface circles. Assuming that the
    pulse width is small compared to the range, we get R=r=tan(theta*pi/180)*range.
    Thus, the pulse volume simpy becomes a the volume of a cylinder with
    V=pi * h * range**2 * tan(theta*pi/180)**2

    Parameters
    ----------
    ranges : array
        the distances of each bin from the radar [m]
    h : float
        puls width (which corresponds to the range resolution [m])
    theta : float
        the aperture angle (beam width) of the radar beam [degree]

    Returns
    -------
    output : volume of radar bins at each range in `ranges` (qubic metres)

    """
    return np.pi * h * (ranges**2) * (np.tan( np.radians(theta) ))**2

if __name__ == '__main__':
    print 'wradlib: Calling module <qual> as main...'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#
#    Author: Yannick Vaucher
#    Copyright 2015 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
from openerp import models, fields, api

_logger = logging.getLogger(__name__)

class StockMove(models.Model):
    _inherit = 'stock.move'

    departure_shipment_id = fields.Many2one(
        'shipment.plan', 'Departure shipment',
        readonly=True,
    )
    arrival_shipment_id = fields.Many2one(
        'shipment.plan', 'Arrival shipment',
        readonly=True,
    )

    product_volume = fields.Float(
        related='product_id.volume',
        readonly=True,
        store=True
    )

    ship_partner_id = fields.Many2one(
        comodel_name='res.partner',
        compute='_get_ship_partner_id',
        string='Partner',
        readonly=True,
        store=True,
    )
    ship_carrier_id = fields.Many2one(
        related='departure_shipment_id.carrier_id',
        readonly=True,
        store=True
    )
    ship_carrier_tracking_ref = fields.Char(
        related='departure_shipment_id.carrier_tracking_ref',
        string='Tracking Ref.',
        readonly=True
    )
    ship_from_address_id = fields.Many2one(
        related='picking_id.origin_address_id',
        string='From Address',
        readonly=True,
        store=True
    )
    ship_to_address_id = fields.Many2one(
        related='picking_id.delivery_address_id',
        string='To Address',
        store=True
    )
    ship_consignee_id = fields.Many2one(
        related='picking_id.consignee_id',
        string='Consignee',
        readonly=True,
        store=True
    )
    ship_transport_mode_id = fields.Many2one(
        related='departure_shipment_id.transport_mode_id',
        string='Transport by',
        readonly=True,
    )
    ship_state = fields.Selection(
        related='departure_shipment_id.state',
        store=True,
    )
    ship_etd = fields.Datetime(
        related='date_expected',
        string='ETD',
        store=True
    )
    ship_eta = fields.Datetime(
        related='move_dest_id.date_expected',
        string='ETA',
        store=True
    )

    @api.one
    @api.depends('picking_id.sale_id',
                 'purchase_line_id.order_id')
    def _get_ship_partner_id(self):
        partner = False
        if self.picking_id.sale_id:
            partner = self.picking_id.sale_id.partner_id
        elif self.purchase_line_id.order_id.partner_id:
            partner = self.purchase_line_id.order_id.partner_id
        self.ship_partner_id = partner

    @api.multi
    def write(self, values):
        res = super(StockMove, self).write(values)
        if values.get('state', '') in ('done', 'cancel'):
            for ship in self.mapped('arrival_shipment_id'):
                if ship.state == 'in_transit':
                    ship.signal_workflow('transit_end')
        return res

    @api.multi
    def _picking_assign(self, procurement_group, location_from, location_to):
        """ When an arrival transit move is split, we create a backorder

        """
        res = super(StockMove, self)._picking_assign(
            procurement_group, location_from, location_to)
        if self.env.context.get('split_transit_arrival') == 'arrival':
            context = {'do_only_split': True}
            Picking = self.env['stock.picking'].with_context(**context)
            Picking._create_backorder(self.picking_id, backorder_moves=self)
        return res

    @api.model
    def split(self, move, qty,
              restrict_lot_id=False, restrict_partner_id=False):
        """ Propagate the splitting of a departure transit move to
        its arrival transit move

        Here, we remove picking_id to force arrival move to trigger
        _picking_assign for new move

        """
        context = {}
        if move.departure_shipment_id:
            context['split_transit_arrival'] = 'departure'
        # On propagate split
        if self.env.context.get('split_transit_arrival') == 'departure':
            context['split_transit_arrival'] = 'arrival'

        _self = self.with_context(**context)
        res = super(StockMove, _self).split(
            move, qty,
            restrict_lot_id=restrict_lot_id,
            restrict_partner_id=restrict_partner_id)
        return res

    @api.multi
    def copy(self, default=None):
        """ Ensure an arrival move from Transit is not assigned to same picking
        when source move is split

        """
        if default is None:
            default = {}
        if self.env.context.get('split_transit_arrival') == 'arrival':
            default['picking_id'] = False
        return super(StockMove, self).copy(default=default)

    @api.multi
    def action_remove_from_shipment(self):
        self.move_dest_id.arrival_shipment_id = False
        self.departure_shipment_id = False
        return True
<commit_msg>Shipment - Prevent cancellation of arrival move if departure is not canceled<commit_after># -*- coding: utf-8 -*-
#
#
#    Author: Yannick Vaucher
#    Copyright 2015 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
from openerp import models, fields, api, _, exceptions

_logger = logging.getLogger(__name__)

class StockMove(models.Model):
    _inherit = 'stock.move'

    departure_shipment_id = fields.Many2one(
        'shipment.plan', 'Departure shipment',
        readonly=True,
    )
    arrival_shipment_id = fields.Many2one(
        'shipment.plan', 'Arrival shipment',
        readonly=True,
    )

    product_volume = fields.Float(
        related='product_id.volume',
        readonly=True,
        store=True
    )

    ship_partner_id = fields.Many2one(
        comodel_name='res.partner',
        compute='_get_ship_partner_id',
        string='Partner',
        readonly=True,
        store=True,
    )
    ship_carrier_id = fields.Many2one(
        related='departure_shipment_id.carrier_id',
        readonly=True,
        store=True
    )
    ship_carrier_tracking_ref = fields.Char(
        related='departure_shipment_id.carrier_tracking_ref',
        string='Tracking Ref.',
        readonly=True
    )
    ship_from_address_id = fields.Many2one(
        related='picking_id.origin_address_id',
        string='From Address',
        readonly=True,
        store=True
    )
    ship_to_address_id = fields.Many2one(
        related='picking_id.delivery_address_id',
        string='To Address',
        store=True
    )
    ship_consignee_id = fields.Many2one(
        related='picking_id.consignee_id',
        string='Consignee',
        readonly=True,
        store=True
    )
    ship_transport_mode_id = fields.Many2one(
        related='departure_shipment_id.transport_mode_id',
        string='Transport by',
        readonly=True,
    )
    ship_state = fields.Selection(
        related='departure_shipment_id.state',
        store=True,
    )
    ship_etd = fields.Datetime(
        related='date_expected',
        string='ETD',
        store=True
    )
    ship_eta = fields.Datetime(
        related='move_dest_id.date_expected',
        string='ETA',
        store=True
    )

    @api.one
    @api.depends('picking_id.sale_id',
                 'purchase_line_id.order_id')
    def _get_ship_partner_id(self):
        partner = False
        if self.picking_id.sale_id:
            partner = self.picking_id.sale_id.partner_id
        elif self.purchase_line_id.order_id.partner_id:
            partner = self.purchase_line_id.order_id.partner_id
        self.ship_partner_id = partner

    @api.multi
    def action_cancel(self):
        """ Forbid to cancel a shipment arrival move if it's departure move
        is not canceled.

        As dest_move are canceled before origin moves, we check that

        """
        cancel_list = self.env.context.get('cancel_list', [])
        for move in self:
            if (
                move.arrival_shipment_id and
                not (move.move_orig_ids.state == 'cancel' or
                     set(move.move_orig_ids.ids).issubset(set(cancel_list))
                     )
            ):
                raise exceptions.Warning(
                    _('You cannot cancel a shipment arrival stock move that '
                      'depends on an uncancelled shipment departure stock '
                      'move.'))
            cancel_list.append(move.id)
        return super(StockMove, self.with_context(cancel_list=cancel_list)
                     ).action_cancel()

    @api.multi
    def write(self, values):
        res = super(StockMove, self).write(values)
        if values.get('state', '') in ('done', 'cancel'):
            for ship in self.mapped('arrival_shipment_id'):
                if ship.state == 'in_transit':
                    ship.signal_workflow('transit_end')
        return res

    @api.multi
    def _picking_assign(self, procurement_group, location_from, location_to):
        """ When an arrival transit move is split, we create a backorder

        """
        res = super(StockMove, self)._picking_assign(
            procurement_group, location_from, location_to)
        if self.env.context.get('split_transit_arrival') == 'arrival':
            context = {'do_only_split': True}
            Picking = self.env['stock.picking'].with_context(**context)
            Picking._create_backorder(self.picking_id, backorder_moves=self)
        return res

    @api.model
    def split(self, move, qty,
              restrict_lot_id=False, restrict_partner_id=False):
        """ Propagate the splitting of a departure transit move to
        its arrival transit move

        Here, we remove picking_id to force arrival move to trigger
        _picking_assign for new move

        """
        context = {}
        if move.departure_shipment_id:
            context['split_transit_arrival'] = 'departure'
        # On propagate split
        if self.env.context.get('split_transit_arrival') == 'departure':
            context['split_transit_arrival'] = 'arrival'

        _self = self.with_context(**context)
        res = super(StockMove, _self).split(
            move, qty,
            restrict_lot_id=restrict_lot_id,
            restrict_partner_id=restrict_partner_id)
        return res

    @api.multi
    def copy(self, default=None):
        """ Ensure an arrival move from Transit is not assigned to same picking
        when source move is split

        """
        if default is None:
            default = {}
        if self.env.context.get('split_transit_arrival') == 'arrival':
            default['picking_id'] = False
        return super(StockMove, self).copy(default=default)

    @api.multi
    def action_remove_from_shipment(self):
        self.move_dest_id.arrival_shipment_id = False
        self.departure_shipment_id = False
        return True
<|endoftext|>