<commit_before># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

"""son-analyze fetch operation"""

import os
from enum import Enum
import logging
import collections
from urllib.parse import ParseResult, urljoin
from typing import Dict, Any, Tuple
import requests

_LOGGER = logging.getLogger(__name__)

class FetchError(Exception):
    """Base class for exceptions in this module."""
    pass

class InvalidResourceReferenceError(FetchError):
    """Raised when a resource (nfd) points to a missing vnfd."""

    # pylint: disable=unsubscriptable-object
    def __init__(self, nsd: Dict[str, Any], missing_vnf_id: str) -> None:
        super().__init__()
        self.nsd = nsd
        self.missing_vnf_id = missing_vnf_id
        self.message = ('The nsd ("{}","{}","{}") has a vnf_id field "{}" '
                        'pointing to a non-existing vnfd').format(
                            nsd['vendor'], nsd['name'], nsd['version'],
                            missing_vnf_id)

def _get_workspace_token() -> str:
    """Retrieve the authentification token from the workspace"""
    home = os.path.expanduser('~')
    path = os.path.join(home, '.son-workspace', 'platforms', 'token.txt')
    with open(path) as tkn:
        return tkn.read()

class _Kind(Enum):
    nsd = 1
    vnfd = 2

# pylint: disable=unsubscriptable-object
def _fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult, path: str,
                            uuid: str) -> Dict[str, Any]:
    """Fetch a resource by its uuid. Return `None` if nothing is found.
    Raise a RuntimeError exception when a error is detected within the
    gatekeeper's API."""
    url = urljoin(gatekeeper_endpoint.geturl(), os.path.join(path, uuid))
    _LOGGER.info('Fetching a resource by uuid at %s', url)
    auth = 'Bearer ' + _get_workspace_token()
    res_resp = requests.get(url, headers={'content-type': 'application/json',
                                          'Authorization': auth})
    try:
        res_resp.raise_for_status()
    except requests.exceptions.HTTPError as exc_notfound:
        _LOGGER.exception('Failed to retrieve a resource at %s '
                          '(status code = %d)', res_resp.url,
                          res_resp.status_code)
        if exc_notfound.response.status_code == 404:
            return None
        else:
            raise
    tmp = res_resp.json()
    if not isinstance(tmp, dict) or len(tmp) <= 0:
        exc = RuntimeError('The returned json is malformed:  {}'.format(tmp))
        _LOGGER.error(exc)
        raise exc
    _LOGGER.info('Succeed to retrieve the resource %s (status code = %d)',
                 res_resp.url, res_resp.status_code)
    return tmp

# pylint: disable=unsubscriptable-object
def _fetch_resource(gatekeeper_endpoint: ParseResult, kind: _Kind, path: str,
                    vendor: str, name: str, version: str) -> Dict[str, Any]:
    """Fetch a resource and return the Json as a dictionary. Return `None` if
     nothing is found. It raise a RuntimeError exception when a gatekeeper API
     is dectected"""
    url = urljoin(gatekeeper_endpoint.geturl(), path)
    _LOGGER.info('Fetching a %s resource by name at %s', kind, url)
    query_params_raw = {'vendor': vendor,  # Dict[Str, Str]
                        'name': name,
                        'version': version}
    # We force the order of the query's parameters to lower the impact on tests
    # when a key is added or removed
    query_params = collections.OrderedDict(sorted(query_params_raw.items()))
    auth = 'Bearer ' + _get_workspace_token()
    res_resp = requests.get(url, params=query_params,
                            headers={'content-type': 'application/json',
                                     'Authorization': auth})
    try:
        res_resp.raise_for_status()
    except requests.exceptions.HTTPError:
        _LOGGER.exception('Failed to retrieve a resource at %s '
                          '(status code = %d)', res_resp.url,
                          res_resp.status_code)
        # REMARK: if nothing is found, then the API return an empty [] and
        # not 404
        raise
    tmp = res_resp.json()
    if not isinstance(tmp, list):
        exc = RuntimeError('The returned json is not boxed by a list')
        _LOGGER.error(exc)
        raise exc
    _LOGGER.info('Succeed to retrieve the resource %s (status code = %d): %s',
                 res_resp.url, res_resp.status_code, tmp[:20])
    for elt in tmp:
        if kind.name in elt:  # the resource is boxed
            elt = elt[kind.name]
        if all([elt['vendor'] == vendor, elt['name'] == name,
                elt['version'] == version]):
            return elt
    return None

# pylint: disable=unsubscriptable-object
def fetch_vnfd(gatekeeper_endpoint: ParseResult, vendor: str, name: str,
               version: str) -> Dict[str, Any]:
    """Fetch a Vnfd. Return `None` if nothing is found."""
    return _fetch_resource(gatekeeper_endpoint, _Kind.vnfd, 'functions',
                           vendor, name, version)

# pylint: disable=unsubscriptable-object
def _complete_nsd_with_vnfds(gatekeeper_endpoint: ParseResult,
                             nsd: Dict[str, Any]) -> Tuple[Dict[str, Any],
                                                           Dict[str, Any]]:
    """Retrieve the vnfds mentioned in a nsd. Raise a
    InvalidResourceReferenceError exception if a vnfd is missing."""
    _LOGGER.info('Fetching the inner vnfds of the %s nsd', nsd['name'])
    acc = {}  # Dict[str, Any]
    for fun_desc in nsd['network_functions']:
        vnfd = fetch_vnfd(gatekeeper_endpoint, fun_desc['vnf_vendor'],
                          fun_desc['vnf_name'], fun_desc['vnf_version'])
        if not vnfd:
            exc = InvalidResourceReferenceError(nsd, fun_desc['vnf_id'])
            _LOGGER.error(exc)
            raise exc
        acc[fun_desc['vnf_id']] = vnfd
    return (nsd, acc)

# pylint: disable=unsubscriptable-object
def fetch_nsd(gatekeeper_endpoint: ParseResult, vendor: str, name: str,
              version: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """Fetch a Nsd with its related Vnfd. Return `None` if nothing is found.
    Raise a FileNotFoundError if  """
    nsd = _fetch_resource(gatekeeper_endpoint, _Kind.nsd, 'services', vendor,
                          name, version)
    if nsd:
        return _complete_nsd_with_vnfds(gatekeeper_endpoint, nsd)
    return nsd, {}

# pylint: disable=unsubscriptable-object
def fetch_vnfd_by_uuid(gatekeeper_endpoint: ParseResult,
                       uuid: str) -> Dict[str, Any]:
    """Fetch a vnfd by its uuid"""
    return _fetch_resource_by_uuid(gatekeeper_endpoint, 'functions', uuid)

# pylint: disable=unsubscriptable-object
def fetch_nsd_by_uuid(gatekeeper_endpoint: ParseResult,
                      uuid: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """Fetch a nsd by its uuid. Return `None` if not found. Raise a
    InvalidResourceReferenceError exception is nothing is found."""
    nsd = _fetch_resource_by_uuid(gatekeeper_endpoint, 'services', uuid)
    if nsd:
        return _complete_nsd_with_vnfds(gatekeeper_endpoint, nsd)
    return None
<commit_msg>Create a function to fetch vnfr by uuid<commit_after># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

"""son-analyze fetch operation"""

import os
from enum import Enum
import logging
import collections
from urllib.parse import ParseResult, urljoin
from typing import Dict, Any, Tuple
import requests

_LOGGER = logging.getLogger(__name__)

class FetchError(Exception):
    """Base class for exceptions in this module."""
    pass

class InvalidResourceReferenceError(FetchError):
    """Raised when a resource (nfd) points to a missing vnfd."""

    # pylint: disable=unsubscriptable-object
    def __init__(self, nsd: Dict[str, Any], missing_vnf_id: str) -> None:
        super().__init__()
        self.nsd = nsd
        self.missing_vnf_id = missing_vnf_id
        self.message = ('The nsd ("{}","{}","{}") has a vnf_id field "{}" '
                        'pointing to a non-existing vnfd').format(
                            nsd['vendor'], nsd['name'], nsd['version'],
                            missing_vnf_id)

def _get_workspace_token() -> str:
    """Retrieve the authentification token from the workspace"""
    home = os.path.expanduser('~')
    path = os.path.join(home, '.son-workspace', 'platforms', 'token.txt')
    with open(path) as tkn:
        return tkn.read()

class _Kind(Enum):
    nsd = 1
    vnfd = 2
    vnfr = 3

# pylint: disable=unsubscriptable-object
def _fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult, path: str,
                            uuid: str) -> Dict[str, Any]:
    """Fetch a resource by its uuid. Return `None` if nothing is found.
    Raise a RuntimeError exception when a error is detected within the
    gatekeeper's API."""
    url = urljoin(gatekeeper_endpoint.geturl(), os.path.join(path, uuid))
    _LOGGER.info('Fetching a resource by uuid at %s', url)
    auth = 'Bearer ' + _get_workspace_token()
    res_resp = requests.get(url, headers={'content-type': 'application/json',
                                          'Authorization': auth})
    try:
        res_resp.raise_for_status()
    except requests.exceptions.HTTPError as exc_notfound:
        _LOGGER.exception('Failed to retrieve a resource at %s '
                          '(status code = %d)', res_resp.url,
                          res_resp.status_code)
        if exc_notfound.response.status_code == 404:
            return None
        else:
            raise
    tmp = res_resp.json()
    if not isinstance(tmp, dict) or len(tmp) <= 0:
        exc = RuntimeError('The returned json is malformed:  {}'.format(tmp))
        _LOGGER.error(exc)
        raise exc
    _LOGGER.info('Succeed to retrieve the resource %s (status code = %d)',
                 res_resp.url, res_resp.status_code)
    return tmp

# pylint: disable=unsubscriptable-object
def _fetch_resource(gatekeeper_endpoint: ParseResult, kind: _Kind, path: str,
                    vendor: str, name: str, version: str) -> Dict[str, Any]:
    """Fetch a resource and return the Json as a dictionary. Return `None` if
     nothing is found. It raise a RuntimeError exception when a gatekeeper API
     is dectected"""
    url = urljoin(gatekeeper_endpoint.geturl(), path)
    _LOGGER.info('Fetching a %s resource by name at %s', kind, url)
    query_params_raw = {'vendor': vendor,  # Dict[Str, Str]
                        'name': name,
                        'version': version}
    # We force the order of the query's parameters to lower the impact on tests
    # when a key is added or removed
    query_params = collections.OrderedDict(sorted(query_params_raw.items()))
    auth = 'Bearer ' + _get_workspace_token()
    res_resp = requests.get(url, params=query_params,
                            headers={'content-type': 'application/json',
                                     'Authorization': auth})
    try:
        res_resp.raise_for_status()
    except requests.exceptions.HTTPError:
        _LOGGER.exception('Failed to retrieve a resource at %s '
                          '(status code = %d)', res_resp.url,
                          res_resp.status_code)
        # REMARK: if nothing is found, then the API return an empty [] and
        # not 404
        raise
    tmp = res_resp.json()
    if not isinstance(tmp, list):
        exc = RuntimeError('The returned json is not boxed by a list')
        _LOGGER.error(exc)
        raise exc
    _LOGGER.info('Succeed to retrieve the resource %s (status code = %d): %s',
                 res_resp.url, res_resp.status_code, tmp[:20])
    for elt in tmp:
        if kind.name in elt:  # the resource is boxed
            elt = elt[kind.name]
        if all([elt['vendor'] == vendor, elt['name'] == name,
                elt['version'] == version]):
            return elt
    return None

# pylint: disable=unsubscriptable-object
def fetch_vnfd(gatekeeper_endpoint: ParseResult, vendor: str, name: str,
               version: str) -> Dict[str, Any]:
    """Fetch a Vnfd. Return `None` if nothing is found."""
    return _fetch_resource(gatekeeper_endpoint, _Kind.vnfd, 'functions',
                           vendor, name, version)

# pylint: disable=unsubscriptable-object
def _complete_nsd_with_vnfds(gatekeeper_endpoint: ParseResult,
                             nsd: Dict[str, Any]) -> Tuple[Dict[str, Any],
                                                           Dict[str, Any]]:
    """Retrieve the vnfds mentioned in a nsd. Raise a
    InvalidResourceReferenceError exception if a vnfd is missing."""
    _LOGGER.info('Fetching the inner vnfds of the %s nsd', nsd['name'])
    acc = {}  # Dict[str, Any]
    for fun_desc in nsd['network_functions']:
        vnfd = fetch_vnfd(gatekeeper_endpoint, fun_desc['vnf_vendor'],
                          fun_desc['vnf_name'], fun_desc['vnf_version'])
        if not vnfd:
            exc = InvalidResourceReferenceError(nsd, fun_desc['vnf_id'])
            _LOGGER.error(exc)
            raise exc
        acc[fun_desc['vnf_id']] = vnfd
    return (nsd, acc)

# pylint: disable=unsubscriptable-object
def fetch_nsd(gatekeeper_endpoint: ParseResult, vendor: str, name: str,
              version: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """Fetch a Nsd with its related Vnfd. Return `None` if nothing is found.
    Raise a FileNotFoundError if  """
    nsd = _fetch_resource(gatekeeper_endpoint, _Kind.nsd, 'services', vendor,
                          name, version)
    if nsd:
        return _complete_nsd_with_vnfds(gatekeeper_endpoint, nsd)
    return nsd, {}

# pylint: disable=unsubscriptable-object
def fetch_vnfd_by_uuid(gatekeeper_endpoint: ParseResult,
                       uuid: str) -> Dict[str, Any]:
    """Fetch a vnfd by its uuid"""
    return _fetch_resource_by_uuid(gatekeeper_endpoint, 'functions', uuid)

# pylint: disable=unsubscriptable-object
def fetch_nsd_by_uuid(gatekeeper_endpoint: ParseResult,
                      uuid: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
    """Fetch a nsd by its uuid. Return `None` if not found. Raise a
    InvalidResourceReferenceError exception is nothing is found."""
    nsd = _fetch_resource_by_uuid(gatekeeper_endpoint, 'services', uuid)
    if nsd:
        return _complete_nsd_with_vnfds(gatekeeper_endpoint, nsd)
    return None

# pylint: disable=unsubscriptable-object
def fetch_vnfr_by_uuid(gatekeeper_endpoint: ParseResult,
                       uuid: str) -> Dict[str, Any]:
    """Fetch a vnfr by its uuid"""
    return _fetch_resource_by_uuid(gatekeeper_endpoint, 'records/functions',
                                   uuid)
<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests the load_projects_buckets_acls_pipeline."""

from google.apputils import basetest
import mock

# pylint: disable=line-too-long
from google.cloud.security.common.data_access import bucket_dao
from google.cloud.security.common.data_access import errors as data_access_errors
from google.cloud.security.common.gcp_api import errors as api_errors
from google.cloud.security.common.gcp_api import storage
from google.cloud.security.common.util import log_util
from google.cloud.security.inventory import errors as inventory_errors
from google.cloud.security.inventory.pipelines import load_projects_buckets_acls_pipeline
from tests.inventory.pipelines.test_data import fake_buckets
from tests.inventory.pipelines.test_data import fake_configs
# pylint: enable=line-too-long

class LoadProjectsBucketsAclsPipelineTest(basetest.TestCase):
	"""Tests for the load_projects_buckets_acls_pipeline."""

	FAKE_PROJECT_NUMBERS = ['11111']
	FAKE_BUCKETS = ['fakebucket1']

	def setUp(self):
		"""Set up."""

		self.cycle_timestamp = '20001225T120000Z'
		self.configs = fake_configs.FAKE_CONFIGS
		self.mock_gcs_acl = mock.create_autospec(storage.StorageClient)
		self.mock_dao = mock.create_autospec(bucket_dao.BucketDao)
		self.maxDiff = None
		self.pipeline = (
			load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline(
				self.cycle_timestamp,
				self.configs,
				self.mock_gcs_acl,
				self.mock_dao))

	def test_can_transform_bucket_acls(self):
		"""Test that bucket acls can be tranformed."""

		loadable_buckets = list(self.pipeline._transform(
			fake_buckets.FAKE_BUCKET_ACL_MAP))
		self.assertEquals(
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS,
			loadable_buckets)

	def test_api_is_called_to_retrieve_bucket_acls(self):
		"""Test that api is called to retrive bucket acls."""

		self.pipeline.dao.project_numbers_dao_stub.return_value = (
			self.FAKE_PROJECT_NUMBERS)
		self.pipeline.dao.get_buckets_by_project_number.return_value = (
			self.FAKE_BUCKETS)
		self.pipeline._retrieve()

		self.pipeline.dao.project_numbers_dao_stub.assert_called_once_with(
			self.pipeline.PROJECTS_RESOURCE_NAME,
			self.pipeline.cycle_timestamp)

		self.pipeline.dao.get_buckets_by_project_number.assert_called_once_with(
			self.pipeline.RESOURCE_NAME,
			self.pipeline.cycle_timestamp,
			self.FAKE_PROJECT_NUMBERS[0])

		self.pipeline.api_client.get_bucket_acls.assert_called_once_with(
			self.FAKE_BUCKETS[0])

		self.assertEquals(
			1, self.pipeline.api_client.get_bucket_acls.call_count)

	def test_api_error_is_handled_when_retrieving(self):
		"""Test that exceptions are handled when retrieving.

		We don't want to fail the pipeline when any one project's bucket acls
        can not be retrieved.  We just want to log the error, and continue
        with the other projects.
		"""
		load_projects_buckets_acls_pipeline.LOGGER = (
			mock.create_autospec(log_util).get_logger('foo'))
		self.pipeline.dao.project_numbers_dao_stub.return_value = (
			self.FAKE_PROJECT_NUMBERS)
		self.pipeline.dao.get_buckets_by_project_number.return_value = (
			self.FAKE_BUCKETS)
		self.pipeline.api_client.get_bucket_acls.side_effect = (
			api_errors.ApiExecutionError('error error', mock.MagicMock()))

		self.pipeline._retrieve()

		self.assertEquals(
			1,
			load_projects_buckets_acls_pipeline.LOGGER.error.call_count)

	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_get_loaded_count')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_load')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_transform')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_retrieve')
	def test_subroutines_are_called_by_run(self, mock_retrieve, mock_transform,mock_load, mock_get_loaded_count):
		"""Test that the subroutines are called by run."""
		mock_retrieve.return_value = (
			fake_buckets.FAKE_BUCKET_ACL_MAP)
		mock_transform.return_value = (
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS)
		self.pipeline.run()

		mock_retrieve.assert_called_once_with()

		mock_transform.assert_called_once_with(
			fake_buckets.FAKE_BUCKET_ACL_MAP)

		self.assertEquals(1, mock_load.call_count)

		# The regular data is loaded.
		called_args, called_kwargs = mock_load.call_args_list[0]
		expected_args = (
			self.pipeline.RESOURCE_NAME,
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS)
		self.assertEquals(expected_args, called_args)

		mock_get_loaded_count.assert_called_once
<commit_msg>Fix for tests after the improvement for inheritance<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests the load_projects_buckets_acls_pipeline."""

from google.apputils import basetest
import mock

# pylint: disable=line-too-long
from google.cloud.security.common.data_access import bucket_dao
from google.cloud.security.common.data_access import errors as data_access_errors
from google.cloud.security.common.gcp_api import errors as api_errors
from google.cloud.security.common.gcp_api import storage
from google.cloud.security.common.util import log_util
from google.cloud.security.inventory import errors as inventory_errors
from google.cloud.security.inventory.pipelines import load_projects_buckets_acls_pipeline
from tests.inventory.pipelines.test_data import fake_buckets
from tests.inventory.pipelines.test_data import fake_configs
# pylint: enable=line-too-long

class LoadProjectsBucketsAclsPipelineTest(basetest.TestCase):
	"""Tests for the load_projects_buckets_acls_pipeline."""

	FAKE_PROJECT_NUMBERS = ['11111']
	FAKE_BUCKETS = ['fakebucket1']

	def setUp(self):
		"""Set up."""

		self.cycle_timestamp = '20001225T120000Z'
		self.configs = fake_configs.FAKE_CONFIGS
		self.mock_gcs_acl = mock.create_autospec(storage.StorageClient)
		self.mock_dao = mock.create_autospec(bucket_dao.BucketDao)
		self.maxDiff = None
		self.pipeline = (
			load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline(
				self.cycle_timestamp,
				self.configs,
				self.mock_gcs_acl,
				self.mock_dao))

	def test_can_transform_bucket_acls(self):
		"""Test that bucket acls can be tranformed."""

		loadable_buckets = list(self.pipeline._transform(
			fake_buckets.FAKE_BUCKET_ACL_MAP))
		self.assertEquals(
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS,
			loadable_buckets)

	def test_api_is_called_to_retrieve_bucket_acls(self):
		"""Test that api is called to retrive bucket acls."""

		self.pipeline.dao.get_project_numbers.return_value = (
			self.FAKE_PROJECT_NUMBERS)
		self.pipeline.dao.get_buckets_by_project_number.return_value = (
			self.FAKE_BUCKETS)
		self.pipeline._retrieve()

		self.pipeline.dao.get_project_numbers.assert_called_once_with(
			self.pipeline.PROJECTS_RESOURCE_NAME,
			self.pipeline.cycle_timestamp)

		self.pipeline.dao.get_buckets_by_project_number.assert_called_once_with(
			self.pipeline.RESOURCE_NAME,
			self.pipeline.cycle_timestamp,
			self.FAKE_PROJECT_NUMBERS[0])

		self.pipeline.api_client.get_bucket_acls.assert_called_once_with(
			self.FAKE_BUCKETS[0])

		self.assertEquals(
			1, self.pipeline.api_client.get_bucket_acls.call_count)

	def test_api_error_is_handled_when_retrieving(self):
		"""Test that exceptions are handled when retrieving.

		We don't want to fail the pipeline when any one project's bucket acls
        can not be retrieved.  We just want to log the error, and continue
        with the other projects.
		"""
		load_projects_buckets_acls_pipeline.LOGGER = (
			mock.create_autospec(log_util).get_logger('foo'))
		self.pipeline.dao.get_project_numbers.return_value = (
			self.FAKE_PROJECT_NUMBERS)
		self.pipeline.dao.get_buckets_by_project_number.return_value = (
			self.FAKE_BUCKETS)
		self.pipeline.api_client.get_bucket_acls.side_effect = (
			api_errors.ApiExecutionError('error error', mock.MagicMock()))

		self.pipeline._retrieve()

		self.assertEquals(
			1,
			load_projects_buckets_acls_pipeline.LOGGER.error.call_count)

	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_get_loaded_count')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_load')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_transform')
	@mock.patch.object(
		load_projects_buckets_acls_pipeline.LoadProjectsBucketsAclsPipeline,
		'_retrieve')
	def test_subroutines_are_called_by_run(self, mock_retrieve, mock_transform,mock_load, mock_get_loaded_count):
		"""Test that the subroutines are called by run."""
		mock_retrieve.return_value = (
			fake_buckets.FAKE_BUCKET_ACL_MAP)
		mock_transform.return_value = (
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS)
		self.pipeline.run()

		mock_retrieve.assert_called_once_with()

		mock_transform.assert_called_once_with(
			fake_buckets.FAKE_BUCKET_ACL_MAP)

		self.assertEquals(1, mock_load.call_count)

		# The regular data is loaded.
		called_args, called_kwargs = mock_load.call_args_list[0]
		expected_args = (
			self.pipeline.RESOURCE_NAME,
			fake_buckets.EXPECTED_LOADABLE_BUCKET_ACLS)
		self.assertEquals(expected_args, called_args)

		mock_get_loaded_count.assert_called_once
<|endoftext|>


<commit_before>import string
import random
import inspect
import visitor
import astnode as ast
from codegen import calvin_astgen, calvin_components, query, PortlistRewrite
from parser import calvin_parse
from calvin.actorstore.store import DocumentationStore

_docstore = DocumentationStore()

def _refname(name):
   return name.replace(':', '_')

def _lookup_definition(actor_type, root):
    if '.' in actor_type:
        doc = _docstore.metadata(actor_type)
        if not doc['is_known']:
            return ([], [], '')
        return (doc['inputs'], doc['outputs'], doc['type'])
    comps = query(root, kind=ast.Component, attributes={'name':actor_type}, maxdepth=2)
    if not comps:
        return ([], [], '')
    return (comps[0].inports, comps[0].outports, 'component')

class BaseRenderer(object):
    """docstring for BaseRenderer"""
    def __init__(self, debug=False):
        super(BaseRenderer, self).__init__()
        self.debug = debug

    def _default(self, node, order):
        if self.debug:
            print "Not handling: {} ({})".format(node.__class__.__name__, order)
        return ''

    def _add(self, stmt):
        if stmt:
            self.statements.append(stmt)

    def render(self, node, order='preorder'):
        handler = getattr(self, node.__class__.__name__, self._default)
        args, _, _, _ = inspect.getargspec(handler)
        if len(args) == 3:
            self._add(handler(node, order))
        elif order == 'preorder':
            self._add(handler(node))

    def begin(self):
        self.statements = []
        self._add(self.preamble())

    def end(self):
        self._add(self.postamble())

    def result(self):
        return ''.join(self.statements)

    def preamble(self):
        return ''

    def postamble(self):
        return ''

# FIXME: PortList with TransformedPort breaks graphviz
class DotRenderer(BaseRenderer):

    def __init__(self, show_args=False, debug=False):
        super(DotRenderer, self).__init__()
        self.debug = debug
        self.show_args = show_args
        self._namespace_stack = []

    def push_namespace(self, ns):
        self._namespace_stack.append(ns)

    def pop_namespace(self):
        if self._namespace_stack:
            self._namespace_stack.pop()

    @property
    def namespace(self):
        return "{}_".format(self._namespace_stack[-1]) if self._namespace_stack else ''

    def _random_id(self):
        return ''.join(random.choice(string.ascii_uppercase) for x in range(8))

    def _truncate_label(self, label, maxlen=16):
        if len(label) > maxlen:
            label = label[:maxlen/2-2] + " ... " + label[-maxlen/2-2:]
        return label

    def preamble(self):
        return 'digraph structs { node [shape=plaintext]; rankdir=LR splines=spline;\n'

    def postamble(self):
        return '}\n'

    def Constant(self, node, order):
        if order == 'preorder':
            return '/* define '
        if order == 'inorder':
            return ' = '
        if order == 'postorder':
            return ' */\n'

    def Assignment(self, node, order):
        # ident, actor_type, args
        root = node
        while root.parent:
            root = root.parent
        _inports, _outports, _type = _lookup_definition(node.actor_type, root)
        inlen = len(_inports)
        outlen = len(_outports)
        portrows = max(inlen, outlen)
        inports = _inports + ['']*(portrows - inlen)
        outports = _outports + ['']*(portrows - outlen)
        hdrcolor = {
            'actor': 'lightblue',
            'component': 'lightyellow'
        }.get(_type, 'tomato')
        show_args = self.show_args and node.children

        if order == 'preorder':
            lines = []
            lines.append('{}{} [label=<'.format(self.namespace, _refname(node.ident)))
            lines.append('
')
            # Name
            lines.append('
{0}
'.format(node.ident, hdrcolor))
            # Class
            lines.append('
{}
'.format(node.actor_type))
            # Skipping arguments arriving inorder by commenting them out for now
            lines.append('
' if show_args else '/* ')
            return '\n'.join(lines)

        if order == 'inorder':
            return '
\n
' if show_args else ', '

        if order == 'postorder':
            # Close comment
            lines = ['
'if show_args else ' */']
            is_first=True
            for inport, outport in zip(inports, outports):
                inref = ' bgcolor="lightgrey" PORT="{}_in"'.format(inport) if inport else ''
                outref = ' bgcolor="lightgrey" PORT="{}_out"'.format(outport) if outport else ''
                if is_first:
                    is_first = False
                    middle = '    	'.format(portrows)
                else:
                    middle = ''
                lines.append('
{1}	{4}{3}
'.format(inref, inport, outref, outport, middle))
            lines.append('
>];\n')

            return '\n'.join(lines)

    def OutPort(self, node):
        return "{}{}:{}_out:e".format(self.namespace, _refname(node.actor), node.port)

    def InPort(self, node):
        return "{}{}:{}_in:w".format(self.namespace, _refname(node.actor), node.port)

    def InternalOutPort(self, node):
        return "{}{}_out:e".format(self.namespace, node.port)

    def InternalInPort(self, node):
        return "{}{}_in:w".format(self.namespace, node.port)

    def ImplicitPort(self, node, order):
        if order == 'preorder':
            return '{{{} [shape="box" style="filled" fillcolor="white" label="'.format(self._random_id())
        if order == 'postorder':
            return '" ]}'

    def TransformedPort(self, node):
        # N.B. port and value are properties of node, not children
        self.render(node.port)
        return ' [label=<

/{}/

>]'.format(self.Value(node.value))

    def Void(self, node):
        return '{{{} [shape="point" width=0.15]}}'.format(self._random_id())

    def Value(self, node):
        fmt = "'{}'" if type(node.value) is str else '{}'
        return self._truncate_label(fmt.format(node.value))

    def NamedArg(self, node, order):
        if order == 'inorder':
            return " = "

    def Id(self, node):
        return "{}".format(node.ident)

    def Link(self, node, order):
        if order == 'inorder' : return ' -> '
        if order == 'postorder' : return ';\n'

    def Component(self, node, order):
        if order == 'preorder':
            self.push_namespace(node.name)
            lines = ['subgraph cluster_{0} {{ style="filled"; color="lightyellow"; label="Component: {0}";'.format(node.name)]
            for p in node.inports:
                lines.append('{{/* rank="source" */ {1}{0}_out [shape="cds" style="filled" fillcolor="lightgrey" label="{0}"]}};'.format(p, self.namespace))
            for p in node.outports:
                lines.append('{{/* rank="sink" */ {1}{0}_in [shape="cds" style="filled" fillcolor="lightgrey" label="{0}"]}};'.format(p, self.namespace))
            return '\n'.join(lines)

        if order == 'postorder':
            self.pop_namespace()
            return '}\n'

class Visualize(object):
   """docstring for VizPrinter"""
   def __init__(self, renderer=None):
       super(Visualize, self).__init__()
       self.renderer = renderer or DotRenderer()

   def process(self, root):
       self.renderer.begin()
       self.visit(root)
       self.renderer.end()
       return self.renderer.result()

   @visitor.on('node')
   def visit(self, node):
       pass

   @visitor.when(ast.Node)
   def visit(self, node):
       self.renderer.render(node, order='preorder')
       if not node.children:
           self.renderer.render(node, order='postorder')
       for n in node.children or []:
           self.visit(n)
           self.renderer.render(node, order='postorder' if n is node.children[-1] else 'inorder')

def visualize_script(source_text, show_args=False):
    """Process script and return graphviz (dot) source representing application."""
    # Here we need the unprocessed tree ...
    ir, issuetracker = calvin_parse(source_text)
    # ... but expand portlists to simplify rendering
    rw = PortlistRewrite(issuetracker)
    rw.visit(ir)
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ir)
    return dot_source, issuetracker

def visualize_deployment(source_text, show_args=False):
    ast_root, issuetracker = calvin_astgen(source_text, 'visualizer')
    # Here we need the processed tree
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ast_root)
    return dot_source, issuetracker

def visualize_component(source_text, name, show_args=False):
    # STUB
    ir_list, issuetracker = calvin_components(source_text, names=[name])
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ir_list[0])
    return dot_source, issuetracker

if __name__ == '__main__':
    source_text = """
    component FilterUnchanged() string -> string {
      iip : std.Init(data="nothing")
      cmp : std.Compare(op="=")
      sel : std.Select()

      .string > cmp.a
      iip.out > cmp.b
      cmp.result > sel.select
      .string > sel.data
      sel.case_true > voidport
      sel.case_false > iip.in
      sel.case_false > .string
    }

    /* Actors */
    src : std.CountTimer()
    snk : io.Print()
    /* Connections */
    src.integer > snk.token
    """

    dot_src, it =  visualize_script(source_text)
    print dot_src

    dot_src, it =  visualize_deployment(source_text)
    print dot_src

    dot_src, it =  visualize_component(source_text, "FilterUnchanged")
    print dot_src

<commit_msg>Handle the fact that a value is somtimes an identifier.<commit_after>import string
import random
import inspect
import visitor
import astnode as ast
from codegen import calvin_astgen, calvin_components, query, PortlistRewrite
from parser import calvin_parse
from calvin.actorstore.store import DocumentationStore

_docstore = DocumentationStore()

def _refname(name):
   return name.replace(':', '_')

def _lookup_definition(actor_type, root):
    if '.' in actor_type:
        doc = _docstore.metadata(actor_type)
        if not doc['is_known']:
            return ([], [], '')
        return (doc['inputs'], doc['outputs'], doc['type'])
    comps = query(root, kind=ast.Component, attributes={'name':actor_type}, maxdepth=2)
    if not comps:
        return ([], [], '')
    return (comps[0].inports, comps[0].outports, 'component')

class BaseRenderer(object):
    """docstring for BaseRenderer"""
    def __init__(self, debug=False):
        super(BaseRenderer, self).__init__()
        self.debug = debug

    def _default(self, node, order):
        if self.debug:
            print "Not handling: {} ({})".format(node.__class__.__name__, order)
        return ''

    def _add(self, stmt):
        if stmt:
            self.statements.append(stmt)

    def render(self, node, order='preorder'):
        handler = getattr(self, node.__class__.__name__, self._default)
        args, _, _, _ = inspect.getargspec(handler)
        if len(args) == 3:
            self._add(handler(node, order))
        elif order == 'preorder':
            self._add(handler(node))

    def begin(self):
        self.statements = []
        self._add(self.preamble())

    def end(self):
        self._add(self.postamble())

    def result(self):
        return ''.join(self.statements)

    def preamble(self):
        return ''

    def postamble(self):
        return ''

# FIXME: PortList with TransformedPort breaks graphviz
class DotRenderer(BaseRenderer):

    def __init__(self, show_args=False, debug=False):
        super(DotRenderer, self).__init__()
        self.debug = debug
        self.show_args = show_args
        self._namespace_stack = []

    def push_namespace(self, ns):
        self._namespace_stack.append(ns)

    def pop_namespace(self):
        if self._namespace_stack:
            self._namespace_stack.pop()

    @property
    def namespace(self):
        return "{}_".format(self._namespace_stack[-1]) if self._namespace_stack else ''

    def _random_id(self):
        return ''.join(random.choice(string.ascii_uppercase) for x in range(8))

    def _truncate_label(self, label, maxlen=16):
        if len(label) > maxlen:
            label = label[:maxlen/2-2] + " ... " + label[-maxlen/2-2:]
        return label

    def preamble(self):
        return 'digraph structs { node [shape=plaintext]; rankdir=LR splines=spline;\n'

    def postamble(self):
        return '}\n'

    def Constant(self, node, order):
        if order == 'preorder':
            return '/* define '
        if order == 'inorder':
            return ' = '
        if order == 'postorder':
            return ' */\n'

    def Assignment(self, node, order):
        # ident, actor_type, args
        root = node
        while root.parent:
            root = root.parent
        _inports, _outports, _type = _lookup_definition(node.actor_type, root)
        inlen = len(_inports)
        outlen = len(_outports)
        portrows = max(inlen, outlen)
        inports = _inports + ['']*(portrows - inlen)
        outports = _outports + ['']*(portrows - outlen)
        hdrcolor = {
            'actor': 'lightblue',
            'component': 'lightyellow'
        }.get(_type, 'tomato')
        show_args = self.show_args and node.children

        if order == 'preorder':
            lines = []
            lines.append('{}{} [label=<'.format(self.namespace, _refname(node.ident)))
            lines.append('
')
            # Name
            lines.append('
{0}
'.format(node.ident, hdrcolor))
            # Class
            lines.append('
{}
'.format(node.actor_type))
            # Skipping arguments arriving inorder by commenting them out for now
            lines.append('
' if show_args else '/* ')
            return '\n'.join(lines)

        if order == 'inorder':
            return '
\n
' if show_args else ', '

        if order == 'postorder':
            # Close comment
            lines = ['
'if show_args else ' */']
            is_first=True
            for inport, outport in zip(inports, outports):
                inref = ' bgcolor="lightgrey" PORT="{}_in"'.format(inport) if inport else ''
                outref = ' bgcolor="lightgrey" PORT="{}_out"'.format(outport) if outport else ''
                if is_first:
                    is_first = False
                    middle = '    	'.format(portrows)
                else:
                    middle = ''
                lines.append('
{1}	{4}{3}
'.format(inref, inport, outref, outport, middle))
            lines.append('
>];\n')

            return '\n'.join(lines)

    def OutPort(self, node):
        return "{}{}:{}_out:e".format(self.namespace, _refname(node.actor), node.port)

    def InPort(self, node):
        return "{}{}:{}_in:w".format(self.namespace, _refname(node.actor), node.port)

    def InternalOutPort(self, node):
        return "{}{}_out:e".format(self.namespace, node.port)

    def InternalInPort(self, node):
        return "{}{}_in:w".format(self.namespace, node.port)

    def ImplicitPort(self, node, order):
        if order == 'preorder':
            return '{{{} [shape="box" style="filled" fillcolor="white" label="'.format(self._random_id())
        if order == 'postorder':
            return '" ]}'

    def TransformedPort(self, node):
        # N.B. port and value are properties of node, not children
        self.render(node.port)
        val = self.Value(node.value) if type(node.value) is ast.Value else self.Id(node.value)
        return ' [label=<

/{}/

>]'.format(val)

    def Void(self, node):
        return '{{{} [shape="point" width=0.15]}}'.format(self._random_id())

    def Value(self, node):
        fmt = "'{}'" if type(node.value) is str else '{}'
        return self._truncate_label(fmt.format(node.value))

    def NamedArg(self, node, order):
        if order == 'inorder':
            return " = "

    def Id(self, node):
        return "{}".format(node.ident)

    def Link(self, node, order):
        if order == 'inorder' : return ' -> '
        if order == 'postorder' : return ';\n'

    def Component(self, node, order):
        if order == 'preorder':
            self.push_namespace(node.name)
            lines = ['subgraph cluster_{0} {{ style="filled"; color="lightyellow"; label="Component: {0}";'.format(node.name)]
            for p in node.inports:
                lines.append('{{/* rank="source" */ {1}{0}_out [shape="cds" style="filled" fillcolor="lightgrey" label="{0}"]}};'.format(p, self.namespace))
            for p in node.outports:
                lines.append('{{/* rank="sink" */ {1}{0}_in [shape="cds" style="filled" fillcolor="lightgrey" label="{0}"]}};'.format(p, self.namespace))
            return '\n'.join(lines)

        if order == 'postorder':
            self.pop_namespace()
            return '}\n'

class Visualize(object):
   """docstring for VizPrinter"""
   def __init__(self, renderer=None):
       super(Visualize, self).__init__()
       self.renderer = renderer or DotRenderer()

   def process(self, root):
       self.renderer.begin()
       self.visit(root)
       self.renderer.end()
       return self.renderer.result()

   @visitor.on('node')
   def visit(self, node):
       pass

   @visitor.when(ast.Node)
   def visit(self, node):
       self.renderer.render(node, order='preorder')
       if not node.children:
           self.renderer.render(node, order='postorder')
       for n in node.children or []:
           self.visit(n)
           self.renderer.render(node, order='postorder' if n is node.children[-1] else 'inorder')

def visualize_script(source_text, show_args=False):
    """Process script and return graphviz (dot) source representing application."""
    # Here we need the unprocessed tree ...
    ir, issuetracker = calvin_parse(source_text)
    # ... but expand portlists to simplify rendering
    rw = PortlistRewrite(issuetracker)
    rw.visit(ir)
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ir)
    return dot_source, issuetracker

def visualize_deployment(source_text, show_args=False):
    ast_root, issuetracker = calvin_astgen(source_text, 'visualizer')
    # Here we need the processed tree
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ast_root)
    return dot_source, issuetracker

def visualize_component(source_text, name, show_args=False):
    # STUB
    ir_list, issuetracker = calvin_components(source_text, names=[name])
    r = DotRenderer(show_args, debug=False)
    v = Visualize(renderer = r)
    dot_source = v.process(ir_list[0])
    return dot_source, issuetracker

if __name__ == '__main__':
    source_text = """
    component FilterUnchanged() string -> string {
      iip : std.Init(data="nothing")
      cmp : std.Compare(op="=")
      sel : std.Select()

      .string > cmp.a
      iip.out > cmp.b
      cmp.result > sel.select
      .string > sel.data
      sel.case_true > voidport
      sel.case_false > iip.in
      sel.case_false > .string
    }

    /* Actors */
    src : std.CountTimer()
    snk : io.Print()
    /* Connections */
    src.integer > snk.token
    """

    dot_src, it =  visualize_script(source_text)
    print dot_src

    dot_src, it =  visualize_deployment(source_text)
    print dot_src

    dot_src, it =  visualize_component(source_text, "FilterUnchanged")
    print dot_src

<|endoftext|>


<commit_before># Copyright 2018 elego Software Solutions GmbH - Yu Weng
# Copyright 2018 initOS GmbH - Nikolina Todorova
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from datetime import datetime, timedelta
from dateutil import easter

from odoo import _, api, fields, models
from . import hr_holidays_public_generator
from odoo.exceptions import UserError

hr_holidays_public_generator.COUNTRY_GENERATORS.append("DE")

class HrHolidaysPublicGenerator(models.TransientModel):
    _inherit = 'hr.holidays.public.generator'

    @api.model
    def calculate_easter_sunday(self, year):
        return easter.easter(year)

    @api.model
    def calculate_new_good_friday(self, easter):
        good_friday = easter - timedelta(days=2)
        return fields.Date.to_string(good_friday)

    @api.model
    def calculate_easter_monday(self, easter):
        easter_monday = easter + timedelta(days=1)
        return fields.Date.to_string(easter_monday)

    @api.model
    def calculate_ascension_day(self, easter):
        ascension_day = easter + timedelta(days=39)
        return fields.Date.to_string(ascension_day)

    @api.model
    def calculate_whit_monday(self, easter):
        whit_monday = easter + timedelta(days=50)
        return fields.Date.to_string(whit_monday)

    @api.model
    def calculate_corpus_christi(self, easter):
        corpus_christi = easter + timedelta(days=60)
        return fields.Date.to_string(corpus_christi)

    @api.model
    def calculate_floating_holidays(self, existing_holidays):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        easter = self.calculate_easter_sunday(self.year)

        public_holiday_line_obj.create({'name': _('Good Friday'),
                                        'date':
                                            self.calculate_new_good_friday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})
        public_holiday_line_obj.create({'name': _('Easter Sunday'),
                                        'date': fields.Date.to_string(easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

        public_holiday_line_obj.create({'name': _('Easter Monday'),
                                        'date':
                                            self.calculate_easter_monday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})
        public_holiday_line_obj.create({'name': _('Ascension Day'),
                                        'date':
                                            self.calculate_ascension_day(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

        public_holiday_line_obj.create({'name': _('Whit Monday'),
                                        'date':
                                            self.calculate_whit_monday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

    @api.model
    def get_state_ids(self, codes=[]):
        """
        :codes: list of states' codes
        Returns states ids in :codes:
        """
        country = self.env.ref('base.de')
        state_ids = self.env['res.country.state'].search([
            ('code', 'in', codes), ('country_id', '=', country.id)
        ])
        return state_ids.ids

    @api.model
    def calculate_state_floating_holidays(self,
                                          existing_holidays,
                                          state=None):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        easter = self.calculate_easter_sunday(self.year)
        # Baden-Württemberg, Bavaria, Hesse, North
        # Rhine-Westphalia, Rhineland-Palatinate, Saarland
        codes = ['BW', 'BY', 'HE', 'NW', 'RP', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Corpus Christi"),
                'date': self.calculate_corpus_christi(easter),
                'variable_date': True,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })

    @api.model
    def calculate_fixed_holidays(self, existing_holidays):
        public_holiday_line_obj = self.env['hr.holidays.public.line']

        public_holiday_line_obj.create({
            'name': _("New Years's Day"),
            'date': "%s-01-01" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("International Workers' Day"),
            'date': "%s-05-01" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Day of German Unity"),
            'date': "%s-10-03" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Christmas Day"),
            'date': "%s-12-25" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Boxing Day"),
            'date': "%s-12-26" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })

    @api.model
    def calculate_state_fixed_holidays(self, existing_holidays, state=None):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        # Baden-Württemberg, Bavaria, Saxony-Anhalt
        codes = ['BW', 'BY', 'ST']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Three Kings Day"),
                'date': "%s-01-06" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Bavaria, Saarland
        codes = ['BY', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Assumption Day"),
                'date': "%s-08-15" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # BB, HB, HH, MVP, NDS, SH, SN, ST, TH
        # (depends on
        # https://www.timeanddate.com/holidays/germany/reformation-day)
        codes = ['BB', 'HB', 'HH', 'MV', 'NI', 'SH', 'SN', 'ST', 'TH']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Day of Reformation"),
                'date': "%s-10-31" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Baden-Württemberg, Bavaria,
        # North Rhine-Westphalia, Rhineland-Palatinate, Saarland
        codes = ['BW', 'BY', 'NW', 'RP', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("All Saints' Day"),
                'date': "%s-11-01" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Sachsen
        codes = ['SN']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Repentance Day"),
                'date': "%s-11-23" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })

    @api.multi
    def action_delete_holidays(self, existing_holidays):
        self.ensure_one()
        if existing_holidays:
            for holiday_line in existing_holidays.line_ids:
                holiday_line.unlink()
        return existing_holidays

    @api.multi
    def action_generate_de_holidays(self):
        public_holiday_obj = self.env['hr.holidays.public']

        for wizard in self:
            existing_holidays = \
                public_holiday_obj.search(
                    [('year', '=', self.year),
                     ('country_id', '=', wizard.country_id.id)]
                )
            if not existing_holidays:
                existing_holidays = \
                    public_holiday_obj.create({
                        'year': wizard.year,
                        'country_id': wizard.country_id.id})
            wizard.action_delete_holidays(existing_holidays)
            wizard.calculate_floating_holidays(existing_holidays)
            wizard.calculate_fixed_holidays(existing_holidays)
            wizard.calculate_state_floating_holidays(
                existing_holidays, state=wizard.state_id
            )
            wizard.calculate_state_fixed_holidays(
                existing_holidays, state=wizard.state_id
            )

        return {
            'type': 'ir.actions.act_window_close',
        }

    @api.multi
    def action_copy_de_holidays(self):
        public_holiday_obj = self.env['hr.holidays.public']
        public_holiday_line_obj = self.env['hr.holidays.public.line']

        for wizard in self:
            if wizard.year == wizard.template_id.year:
                raise UserError(
                    _('You cannot copy the holidays to the same year.')
                )

            # unlink all currently existing holiday lines in target year
            # before deleting target year
            existing_holidays_year_to = \
                public_holiday_obj.search([
                    ('year', '=', wizard.year),
                    ('country_id', '=', wizard.country_id.id)
                ])
            wizard.action_delete_holidays(existing_holidays_year_to)

            # create new year for holidays
            if not existing_holidays_year_to:
                new_holiday_year = \
                    public_holiday_obj.create({
                        'year': wizard.year,
                        'country_id': wizard.country_id.id
                    })
            else:
                new_holiday_year = existing_holidays_year_to[0]
            # copy fixed holidays from source year replacing the year
            for holiday in wizard.template_id.line_ids:
                if holiday.variable_date:
                    continue
                holiday_date = datetime.strptime(holiday.date, '%Y-%m-%d')
                new_holiday_date = \
                    "%s-%s-%s" % (wizard.year,
                                  holiday_date.month, holiday_date.day)
                public_holiday_line_obj.create({
                    'name': holiday.name,
                    'date': new_holiday_date,
                    'variable_date': False,
                    'state_ids': [(6, 0, [s.id for s in holiday.state_ids])],
                    'year_id': new_holiday_year.id
                })
            wizard.calculate_floating_holidays(new_holiday_year)
            wizard.calculate_state_floating_holidays(
                new_holiday_year,
                state=wizard.state_id
            )

        return {
            'type': 'ir.actions.act_window_close',
        }
<commit_msg>[FIX] TypeError: strptime() argument 1 must be str, not datetime.date<commit_after># Copyright 2018 elego Software Solutions GmbH - Yu Weng
# Copyright 2018 initOS GmbH - Nikolina Todorova
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from datetime import datetime, timedelta
from dateutil import easter

from odoo import _, api, fields, models
from . import hr_holidays_public_generator
from odoo.exceptions import UserError

hr_holidays_public_generator.COUNTRY_GENERATORS.append("DE")

class HrHolidaysPublicGenerator(models.TransientModel):
    _inherit = 'hr.holidays.public.generator'

    @api.model
    def calculate_easter_sunday(self, year):
        return easter.easter(year)

    @api.model
    def calculate_new_good_friday(self, easter):
        good_friday = easter - timedelta(days=2)
        return fields.Date.to_string(good_friday)

    @api.model
    def calculate_easter_monday(self, easter):
        easter_monday = easter + timedelta(days=1)
        return fields.Date.to_string(easter_monday)

    @api.model
    def calculate_ascension_day(self, easter):
        ascension_day = easter + timedelta(days=39)
        return fields.Date.to_string(ascension_day)

    @api.model
    def calculate_whit_monday(self, easter):
        whit_monday = easter + timedelta(days=50)
        return fields.Date.to_string(whit_monday)

    @api.model
    def calculate_corpus_christi(self, easter):
        corpus_christi = easter + timedelta(days=60)
        return fields.Date.to_string(corpus_christi)

    @api.model
    def calculate_floating_holidays(self, existing_holidays):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        easter = self.calculate_easter_sunday(self.year)

        public_holiday_line_obj.create({'name': _('Good Friday'),
                                        'date':
                                            self.calculate_new_good_friday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})
        public_holiday_line_obj.create({'name': _('Easter Sunday'),
                                        'date': fields.Date.to_string(easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

        public_holiday_line_obj.create({'name': _('Easter Monday'),
                                        'date':
                                            self.calculate_easter_monday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})
        public_holiday_line_obj.create({'name': _('Ascension Day'),
                                        'date':
                                            self.calculate_ascension_day(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

        public_holiday_line_obj.create({'name': _('Whit Monday'),
                                        'date':
                                            self.calculate_whit_monday(
                                                easter),
                                        'variable_date': True,
                                        'year_id': existing_holidays.id})

    @api.model
    def get_state_ids(self, codes=[]):
        """
        :codes: list of states' codes
        Returns states ids in :codes:
        """
        country = self.env.ref('base.de')
        state_ids = self.env['res.country.state'].search([
            ('code', 'in', codes), ('country_id', '=', country.id)
        ])
        return state_ids.ids

    @api.model
    def calculate_state_floating_holidays(self,
                                          existing_holidays,
                                          state=None):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        easter = self.calculate_easter_sunday(self.year)
        # Baden-Württemberg, Bavaria, Hesse, North
        # Rhine-Westphalia, Rhineland-Palatinate, Saarland
        codes = ['BW', 'BY', 'HE', 'NW', 'RP', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Corpus Christi"),
                'date': self.calculate_corpus_christi(easter),
                'variable_date': True,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })

    @api.model
    def calculate_fixed_holidays(self, existing_holidays):
        public_holiday_line_obj = self.env['hr.holidays.public.line']

        public_holiday_line_obj.create({
            'name': _("New Years's Day"),
            'date': "%s-01-01" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("International Workers' Day"),
            'date': "%s-05-01" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Day of German Unity"),
            'date': "%s-10-03" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Christmas Day"),
            'date': "%s-12-25" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })
        public_holiday_line_obj.create({
            'name': _("Boxing Day"),
            'date': "%s-12-26" % existing_holidays.year,
            'variable_date': False,
            'year_id': existing_holidays.id
        })

    @api.model
    def calculate_state_fixed_holidays(self, existing_holidays, state=None):
        public_holiday_line_obj = self.env['hr.holidays.public.line']
        # Baden-Württemberg, Bavaria, Saxony-Anhalt
        codes = ['BW', 'BY', 'ST']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Three Kings Day"),
                'date': "%s-01-06" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Bavaria, Saarland
        codes = ['BY', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Assumption Day"),
                'date': "%s-08-15" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # BB, HB, HH, MVP, NDS, SH, SN, ST, TH
        # (depends on
        # https://www.timeanddate.com/holidays/germany/reformation-day)
        codes = ['BB', 'HB', 'HH', 'MV', 'NI', 'SH', 'SN', 'ST', 'TH']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Day of Reformation"),
                'date': "%s-10-31" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Baden-Württemberg, Bavaria,
        # North Rhine-Westphalia, Rhineland-Palatinate, Saarland
        codes = ['BW', 'BY', 'NW', 'RP', 'SL']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("All Saints' Day"),
                'date': "%s-11-01" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })
        # Sachsen
        codes = ['SN']
        state_ids = self.get_state_ids(codes)

        if not state or state.id in state_ids:
            public_holiday_line_obj.create({
                'name': _("Repentance Day"),
                'date': "%s-11-23" % existing_holidays.year,
                'variable_date': False,
                'state_ids': [(6, 0, state_ids)],
                'year_id': existing_holidays.id
            })

    @api.multi
    def action_delete_holidays(self, existing_holidays):
        self.ensure_one()
        if existing_holidays:
            for holiday_line in existing_holidays.line_ids:
                holiday_line.unlink()
        return existing_holidays

    @api.multi
    def action_generate_de_holidays(self):
        public_holiday_obj = self.env['hr.holidays.public']

        for wizard in self:
            existing_holidays = \
                public_holiday_obj.search(
                    [('year', '=', self.year),
                     ('country_id', '=', wizard.country_id.id)]
                )
            if not existing_holidays:
                existing_holidays = \
                    public_holiday_obj.create({
                        'year': wizard.year,
                        'country_id': wizard.country_id.id})
            wizard.action_delete_holidays(existing_holidays)
            wizard.calculate_floating_holidays(existing_holidays)
            wizard.calculate_fixed_holidays(existing_holidays)
            wizard.calculate_state_floating_holidays(
                existing_holidays, state=wizard.state_id
            )
            wizard.calculate_state_fixed_holidays(
                existing_holidays, state=wizard.state_id
            )

        return {
            'type': 'ir.actions.act_window_close',
        }

    @api.multi
    def action_copy_de_holidays(self):
        public_holiday_obj = self.env['hr.holidays.public']
        public_holiday_line_obj = self.env['hr.holidays.public.line']

        for wizard in self:
            if wizard.year == wizard.template_id.year:
                raise UserError(
                    _('You cannot copy the holidays to the same year.')
                )

            # unlink all currently existing holiday lines in target year
            # before deleting target year
            existing_holidays_year_to = \
                public_holiday_obj.search([
                    ('year', '=', wizard.year),
                    ('country_id', '=', wizard.country_id.id)
                ])
            wizard.action_delete_holidays(existing_holidays_year_to)

            # create new year for holidays
            if not existing_holidays_year_to:
                new_holiday_year = \
                    public_holiday_obj.create({
                        'year': wizard.year,
                        'country_id': wizard.country_id.id
                    })
            else:
                new_holiday_year = existing_holidays_year_to[0]
            # copy fixed holidays from source year replacing the year
            for holiday in wizard.template_id.line_ids:
                if holiday.variable_date:
                    continue
                holiday_date = datetime.strptime(str(holiday.date), '%Y-%m-%d')
                new_holiday_date = \
                    "%s-%s-%s" % (wizard.year,
                                  holiday_date.month, holiday_date.day)
                public_holiday_line_obj.create({
                    'name': holiday.name,
                    'date': new_holiday_date,
                    'variable_date': False,
                    'state_ids': [(6, 0, [s.id for s in holiday.state_ids])],
                    'year_id': new_holiday_year.id
                })
            wizard.calculate_floating_holidays(new_holiday_year)
            wizard.calculate_state_floating_holidays(
                new_holiday_year,
                state=wizard.state_id
            )

        return {
            'type': 'ir.actions.act_window_close',
        }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from plumbing.models import *
from django.test import TestCase
from datetime import datetime
import pytz

class TestProcesses(TestCase):
	fixtures = [ 'initial_data.yaml', '*']

	def setUp(self):
		self.process = Process.objects.get_or_create(name='year.Mmonth')[0]
		self.other_process = Process.objects.get(pk=6)
		self.stream = Stream(root_path="/var/service/data/GVAR_IMG/argentina/")

	def test_serialization(self):
		# check if the __str__ method is defined to return the class name with the name parameter.
		self.assertEquals(str(self.process.type_cast()), 'CollectTimed [year.Mmonth]')
		self.assertEquals(str(self.other_process.type_cast()), "FilterSolarElevation [Filter night's images]")

	def test_mark_with_tags(self):
		# check if the mark_with_tags method in the Process class don't
		# append a new tag into the stream.
		self.process.mark_with_tags(self.stream)
		self.assertTrue(self.stream.empty())
		self.other_process.mark_with_tags(self.stream)
		self.assertTrue(self.stream.empty())<commit_msg>plumbing: Try to pass the build.<commit_after># -*- coding: utf-8 -*-
from plumbing.models import *
from django.test import TestCase
from datetime import datetime
import pytz

class TestProcesses(TestCase):
	fixtures = [ 'initial_data.yaml', '*']

	def setUp(self):
		self.process = Process.objects.get(name='year.Mmonth')
		self.other_process = Process.objects.get(pk=6)
		self.stream = Stream(root_path="/var/service/data/GVAR_IMG/argentina/")

	def test_serialization(self):
		# check if the __str__ method is defined to return the class name with the name parameter.
		self.assertEquals(str(self.process.type_cast()), 'CollectTimed [year.Mmonth]')
		self.assertEquals(str(self.other_process.type_cast()), "FilterSolarElevation [Filter night's images]")

	def test_mark_with_tags(self):
		# check if the mark_with_tags method in the Process class don't
		# append a new tag into the stream.
		self.process.mark_with_tags(self.stream)
		self.assertTrue(self.stream.empty())
		self.other_process.mark_with_tags(self.stream)
		self.assertTrue(self.stream.empty())<|endoftext|>


<commit_before>#!/usr/bin/python
# -!- coding: utf-8 -!-
#
# Copyright 2016 Hector Martin <marcan@marcan.st>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import urllib2, hashlib, sys, os, os.path, struct, lz4, sqlite3, random, logging, errno

def unlz4(path):
    fd = open(path)
    magic, uncomp, comp, unk = struct.unpack("<IIII", fd.read(16))
    d = lz4.loads(struct.pack("<I", uncomp) + fd.read())
    assert len(d) == uncomp
    return d

class ResourceError(Exception):
    pass

class ResourceManager(object):
    URLBASE = "http://asset-starlight-stage.akamaized.net/"

    def __init__(self, res_ver, cache_dir, logger):
        self.res_ver = res_ver
        self.cache_dir = cache_dir
        self.logger = logger
        self.platform = "Android"
        self.alvl = "High"
        self.slvl = "High"

    def _makedirs(self, path):
        dir, name = os.path.split(path)
        if not os.path.exists(dir):
            try:
                os.makedirs(dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise

    def _writefile(self, dest, data):
        self._makedirs(dest)
        tmp = dest + ".%08x" % random.randrange(2**64)
        with open(tmp, "w") as fd:
            fd.write(data)
        os.rename(tmp, dest)

    def fetch(self, path, md5=None):
        dest = self.cache_dir + "/storage/" + path
        if os.path.exists(dest):
            return dest

        url = self.URLBASE + path
        self.logger.info("Fetch: %s -> %s", url, dest)
        req = urllib2.Request(url)
        req.add_header("X-Unity-Version", "2017.4.2f2")
        response = urllib2.urlopen(req)
        data = response.read()
        if md5 is not None:
            if hashlib.md5(data).hexdigest() != md5:
                raise ResourceError("MD5 digest mismatch for %s" % path)
        self._writefile(dest, data)
        return dest

    def fetch_lz4(self, path, md5=None):
        dest = self.cache_dir + "/unlz4/" + path
        if os.path.exists(dest):
            return dest

        src = self.fetch(path, md5)
        self.logger.info("unLZ4: %s -> %s", src, dest)

        data = unlz4(src)
        self._writefile(dest, data)
        return dest

    def load_manifest(self):
        base = "dl/%s/" % self.res_ver

        for line in open(self.fetch(base + "manifests/all_dbmanifest")):
            manifest_name, md5, plat, alvl, slvl = line.replace("\n","").split(",")
            if plat == self.platform and alvl == self.alvl and slvl == self.slvl:
                break
        else:
            raise ResourceError("Could not find suitable manifest\n")

        manifest_path = self.fetch_lz4(base + "manifests/%s" % manifest_name)
        con = sqlite3.connect(manifest_path)
        con.row_factory = sqlite3.Row
        return con

    def get_asset_dl_path(self, manifest_entry):
        name = manifest_entry["name"]
        md5 = manifest_entry["hash"]

        if name.endswith(".unity3d"):
            path = "dl/resources/AssetBundles/%s/%s" % (md5[:2], md5)
        elif name.endswith(".mp4") or name.endswith(".ogg"):
            # FIXME: movie quality (unused thus far?)
            path = "dl/resources/Movie/%s/%s" % (md5[:2], md5)
        elif name.endswith(".acb") or name.endswith(".awb"):
            n_dir, _ = os.path.split(name)
            path = "dl/resources/Sound/Common/%s/%s" % (md5[:2], md5)
        elif name.endswith(".mdb") or name.endswith(".bdb"):
            path = "dl/resources/Generic/%s/%s" % (md5[:2], md5)
        else:
            raise ResourceException("Unknown asset type: %s" % name)

        return path

    def get(self, name):
        con = self.load_manifest()
        cur = con.cursor()

        cur.execute("SELECT * FROM manifests WHERE name = ?", (name,))
        row = cur.fetchone()
        if row is None:
            raise ResourceError("Resource %s not found in manifest", name)

        unlz4 = bool(row["attr"] & 1)
        if row["attr"] & ~1:
            raise ResourceError("Unknown attributes: 0x%x" % row["attr"])

        path = self.get_asset_dl_path(row)

        if unlz4:
            return self.fetch_lz4(path)
        else:
            return self.fetch(path)

if __name__ == "__main__":
    log = logging.getLogger("resource")
    mgr = ResourceManager(sys.argv[1], ".", log)
    print mgr.get(sys.argv[2])
<commit_msg>resource_mgr: make compatible with new lz4 module<commit_after>#!/usr/bin/python
# -!- coding: utf-8 -!-
#
# Copyright 2016 Hector Martin <marcan@marcan.st>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import urllib2, hashlib, sys, os, os.path, struct, sqlite3, random, logging, errno

try:
    import lz4.block
    lz4_decompress = lz4.block.decompress
except ImportError:
    import lz4
    lz4_decompress = lz4.loads

def unlz4(path):
    fd = open(path)
    magic, uncomp, comp, unk = struct.unpack("<IIII", fd.read(16))
    d = lz4_decompress(struct.pack("<I", uncomp) + fd.read())
    assert len(d) == uncomp
    return d

class ResourceError(Exception):
    pass

class ResourceManager(object):
    URLBASE = "http://asset-starlight-stage.akamaized.net/"

    def __init__(self, res_ver, cache_dir, logger):
        self.res_ver = res_ver
        self.cache_dir = cache_dir
        self.logger = logger
        self.platform = "Android"
        self.alvl = "High"
        self.slvl = "High"

    def _makedirs(self, path):
        dir, name = os.path.split(path)
        if not os.path.exists(dir):
            try:
                os.makedirs(dir)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise

    def _writefile(self, dest, data):
        self._makedirs(dest)
        tmp = dest + ".%08x" % random.randrange(2**64)
        with open(tmp, "w") as fd:
            fd.write(data)
        os.rename(tmp, dest)

    def fetch(self, path, md5=None):
        dest = self.cache_dir + "/storage/" + path
        if os.path.exists(dest):
            return dest

        url = self.URLBASE + path
        self.logger.info("Fetch: %s -> %s", url, dest)
        req = urllib2.Request(url)
        req.add_header("X-Unity-Version", "2017.4.2f2")
        response = urllib2.urlopen(req)
        data = response.read()
        if md5 is not None:
            if hashlib.md5(data).hexdigest() != md5:
                raise ResourceError("MD5 digest mismatch for %s" % path)
        self._writefile(dest, data)
        return dest

    def fetch_lz4(self, path, md5=None):
        dest = self.cache_dir + "/unlz4/" + path
        if os.path.exists(dest):
            return dest

        src = self.fetch(path, md5)
        self.logger.info("unLZ4: %s -> %s", src, dest)

        data = unlz4(src)
        self._writefile(dest, data)
        return dest

    def load_manifest(self):
        base = "dl/%s/" % self.res_ver

        for line in open(self.fetch(base + "manifests/all_dbmanifest")):
            manifest_name, md5, plat, alvl, slvl = line.replace("\n","").split(",")
            if plat == self.platform and alvl == self.alvl and slvl == self.slvl:
                break
        else:
            raise ResourceError("Could not find suitable manifest\n")

        manifest_path = self.fetch_lz4(base + "manifests/%s" % manifest_name)
        con = sqlite3.connect(manifest_path)
        con.row_factory = sqlite3.Row
        return con

    def get_asset_dl_path(self, manifest_entry):
        name = manifest_entry["name"]
        md5 = manifest_entry["hash"]

        if name.endswith(".unity3d"):
            path = "dl/resources/AssetBundles/%s/%s" % (md5[:2], md5)
        elif name.endswith(".mp4") or name.endswith(".ogg"):
            # FIXME: movie quality (unused thus far?)
            path = "dl/resources/Movie/%s/%s" % (md5[:2], md5)
        elif name.endswith(".acb") or name.endswith(".awb"):
            n_dir, _ = os.path.split(name)
            path = "dl/resources/Sound/Common/%s/%s" % (md5[:2], md5)
        elif name.endswith(".mdb") or name.endswith(".bdb"):
            path = "dl/resources/Generic/%s/%s" % (md5[:2], md5)
        else:
            raise ResourceException("Unknown asset type: %s" % name)

        return path

    def get(self, name):
        con = self.load_manifest()
        cur = con.cursor()

        cur.execute("SELECT * FROM manifests WHERE name = ?", (name,))
        row = cur.fetchone()
        if row is None:
            raise ResourceError("Resource %s not found in manifest", name)

        unlz4 = bool(row["attr"] & 1)
        if row["attr"] & ~1:
            raise ResourceError("Unknown attributes: 0x%x" % row["attr"])

        path = self.get_asset_dl_path(row)

        if unlz4:
            return self.fetch_lz4(path)
        else:
            return self.fetch(path)

if __name__ == "__main__":
    log = logging.getLogger("resource")
    mgr = ResourceManager(sys.argv[1], ".", log)
    print mgr.get(sys.argv[2])
<|endoftext|>


<commit_before>import json
from datetime import datetime

import requests

API_URL = 'https://convore.com/api/'

#^groups/discover/friend.json
#^groups/discover/explore/(?P<angle>popular|recent|alphabetical).json
#^groups/discover/category.json
#^groups/discover/category/(?P<category>[\w-]+).json
#^groups/discover/search.json

# =======
# Helpers
# =======

def _safe_response(r):
    try:
        r.raise_for_status()
        return r
    except requests.HTTPError:
        if r.status_code == 401:
            raise LoginFailed
        else:
#            raise APIError
            return r

def get(*path):
    """
    api.get('groups')
    api.get('groups', 'id')
    api.get('accounts', 'verify')
    """
    url =  '%s%s%s' % (API_URL, '/'.join(map(str, path)), '.json')

    r = requests.get(url)
    print r.url

    return _safe_response(r)

def post(params, *path):

    url =  '%s%s%s' % (API_URL, '/'.join(map(str, path)), '.json')
    r = requests.post(url, params=params)
    return _safe_response(r)

# ==========
# Exceptions
# ==========

class LoginFailed(RuntimeError):
    """Login falied!"""

class APIError(RuntimeError):
    """There was a problem properly accessing the Convore API."""

def login(username, password):
    auth = requests.AuthObject(username, password)
    requests.add_autoauth(API_URL, auth)

# ======
# Models
# ======

class User(object):
    """Convore User object."""

    def __init__(self):
        self.username = None
        self.url = None
        self.id = None
        self.img = None

    def import_from_api(self, d):
        """Constructs User from Deserialized API Response."""

        self.username = d.get('username', None)
        self.url = d.get('url', None)
        self.id = d.get('id', None)
        self.img = d.get('img', None)

    def __repr__(self):
        return '<user @%s>' % (self.username)

class Group(object):
    """Convore Group object."""

    def __init__(self):
        self.kind = None
        self.members_count = None
        self.name = None
        self.creator = None
        self.url = None
        self.slug = None
        self.date_latest_message = None
        self.date_created = None
        self.topics_count = None
        self.friends = None
        self.unread = None
        self.id = None
        self.joined = False

    def import_from_api(self, d):
        """Constructs Group from Deserialized API Response."""

        self.creator = User()

        self.kind = d.get('kind', None)
        self.members_count = d.get('members_count', None)
        self.name = d.get('name', None)
        self.creator.import_from_api(d.get('creator', None))
        self.url = d.get('url', None)
        self.slug = d.get('slug', None)
        self.date_latest_message = datetime.utcfromtimestamp(
                d.get('date_latest_message', None)
        )
        self.date_created = datetime.utcfromtimestamp(
                d.get('date_created', None)
        )
        self.topics_count = d.get('topics_count', None)
        self.unread = d.get('unread', None)
        self.id = d.get('id', None)

        self.friends = []

        if 'friend_list' in d:
            for friend in d.get('friend_list', None):
                _user = User()
                _user.import_from_api(friend)
                self.friends.append(_user)

    def __repr__(self):
        return '<group %s>' % (self.slug)

# ==========
# End Points
# ==========

class Groups(object):

    def __init__(self):
        self.groups = []
        self.sync()

    def joined(self):
        """Returns list of Joined groups."""

        return [g for g in self.groups if g.joined]

    def discover_friend(self):

        _groups = []

        r = get('groups', 'discover', 'friend')

        for group in json.loads(r.content)['groups']:
            _group = Group()
            _group.import_from_api(group)
            _groups.append(_group)

        return _groups

    def __repr__(self):
        return str(self.groups)

    def __getitem__(self, key):

        for group in self.groups:

            if str(key) in [group.id, group.slug]:
               return group

        r = get('groups', key)

        _group = Group()
        _group.import_from_api(json.loads(r.content)['group'])
        self.groups.append(_group)

        return _group

    def __iter__(self):
        for group in self.groups:
            yield group

    def __contains__(self, key):

        if isinstance(key, int):
            key = unicode(key)

        for group in self.groups:
            if key in [group.id, group.slug]:
               return True

        return False

    def sync(self):

        self.groups = []

        r = requests.get(API_URL + 'groups.json')
        for _group in json.loads(r.content)['groups']:

            group = Group()
            group.import_from_api(_group)
            group.joined = True
            self.groups.append(group)
<commit_msg>added GroupsDiscover endpoint.<commit_after>import json
from datetime import datetime

import requests

API_URL = 'https://convore.com/api/'

#^groups/discover/friend.json
#^groups/discover/explore/(?P<angle>popular|recent|alphabetical).json
#^groups/discover/category.json
#^groups/discover/category/(?P<category>[\w-]+).json
#^groups/discover/search.json

# =======
# Helpers
# =======

def _safe_response(r):
    try:
        r.raise_for_status()
        return r
    except requests.HTTPError:
        if r.status_code == 401:
            raise LoginFailed
        else:
#            raise APIError
            return r

def get(*path):
    """
    api.get('groups')
    api.get('groups', 'id')
    api.get('accounts', 'verify')
    """
    url =  '%s%s%s' % (API_URL, '/'.join(map(str, path)), '.json')

    r = requests.get(url)
    print r.url

    return _safe_response(r)

def post(params, *path):

    url =  '%s%s%s' % (API_URL, '/'.join(map(str, path)), '.json')
    r = requests.post(url, params=params)
    return _safe_response(r)

# ==========
# Exceptions
# ==========

class LoginFailed(RuntimeError):
    """Login falied!"""

class APIError(RuntimeError):
    """There was a problem properly accessing the Convore API."""

def login(username, password):
    auth = requests.AuthObject(username, password)
    requests.add_autoauth(API_URL, auth)

# ======
# Models
# ======

class User(object):
    """Convore User object."""

    def __init__(self):
        self.username = None
        self.url = None
        self.id = None
        self.img = None

    def import_from_api(self, d):
        """Constructs User from Deserialized API Response."""

        self.username = d.get('username', None)
        self.url = d.get('url', None)
        self.id = d.get('id', None)
        self.img = d.get('img', None)

    def __repr__(self):
        return '<user @%s>' % (self.username)

class Group(object):
    """Convore Group object."""

    def __init__(self):
        self.kind = None
        self.members_count = None
        self.name = None
        self.creator = None
        self.url = None
        self.slug = None
        self.date_latest_message = None
        self.date_created = None
        self.topics_count = None
        self.friends = None
        self.unread = None
        self.id = None
        self.joined = False

    def import_from_api(self, d):
        """Constructs Group from Deserialized API Response."""

        self.creator = User()

        self.kind = d.get('kind', None)
        self.members_count = d.get('members_count', None)
        self.name = d.get('name', None)
        self.creator.import_from_api(d.get('creator', None))
        self.url = d.get('url', None)
        self.slug = d.get('slug', None)
        self.date_latest_message = datetime.utcfromtimestamp(
                d.get('date_latest_message', None)
        )
        self.date_created = datetime.utcfromtimestamp(
                d.get('date_created', None)
        )
        self.topics_count = d.get('topics_count', None)
        self.unread = d.get('unread', None)
        self.id = d.get('id', None)

        self.friends = []

        if 'friend_list' in d:
            for friend in d.get('friend_list', None):
                _user = User()
                _user.import_from_api(friend)
                self.friends.append(_user)

    def __repr__(self):
        return '<group %s>' % (self.slug)

# ==========
# End Points
# ==========

class Groups(object):

    def __init__(self):
        self.groups = []
        self.sync()
        self.discover = GroupsDiscover()

    def joined(self):
        """Returns list of Joined groups."""

        return [g for g in self.groups if g.joined]

    def __repr__(self):
        return str(self.groups)

    def __getitem__(self, key):

        for group in self.groups:

            if str(key) in [group.id, group.slug]:
               return group

        r = get('groups', key)

        _group = Group()
        _group.import_from_api(json.loads(r.content)['group'])
        self.groups.append(_group)

        return _group

    def __iter__(self):
        for group in self.groups:
            yield group

    def __contains__(self, key):

        if isinstance(key, int):
            key = unicode(key)

        for group in self.groups:
            if key in [group.id, group.slug]:
               return True

        return False

    def sync(self):

        self.groups = []

        r = requests.get(API_URL + 'groups.json')
        for _group in json.loads(r.content)['groups']:

            group = Group()
            group.import_from_api(_group)
            group.joined = True
            self.groups.append(group)

class GroupsDiscover(object):
    def __init__(self):
        pass

    def friend(self):
        _groups = []

        r = get('groups', 'discover', 'friend')

        for group in json.loads(r.content)['groups']:
            _group = Group()
            _group.import_from_api(group)
            _groups.append(_group)

        return _groups
<|endoftext|>