<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import apache_beam as beam
import logging

from typing import Dict, Any, List

from uploaders import google_ads_utils as ads_utils
from uploaders import utils as utils
from utils.execution import AccountConfig, DestinationType
from utils.oauth_credentials import OAuthCredentials

class GoogleAdsCustomerMatchAbstractUploaderDoFn(beam.DoFn):

  def __init__(self, oauth_credentials: OAuthCredentials, developer_token: str):
    super().__init__()
    self.oauth_credentials = oauth_credentials
    self.developer_token = developer_token
    self.active = True
    if self.developer_token is None:
      self.active = False
    self._user_list_id_cache = {}

  def start_bundle(self):
    pass

  def _create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    if self._user_list_id_cache.get(list_name) is None:
      self._user_list_id_cache[list_name] = \
        self._do_create_list_if_it_does_not_exist(
            user_list_service, list_name, list_definition)

    return self._user_list_id_cache[list_name]

  def _do_create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    response = user_list_service.get([{
      'fields': ['Id', 'Name'],
      'predicates': [{
        'field': 'Name',
        'operator': 'EQUALS',
        'values': [list_name]
      }]
    }])
    if (len(response.entries) == 0):
      logging.getLogger().info('%s list does not exist, creating...' % list_name)
      result = user_list_service.mutate([{
        'operator': 'ADD',
        **list_definition
      }])
      id = result['value'][0]['id']
      logging.getLogger().info('List %s created with id: %d' % (list_name, id))
    else:
      id = response.entries[0]['id']
      logging.getLogger().info('List found %s with id: %d' % (list_name, id))
    return id

  # just to facilitate mocking
  def _get_user_list_service(self, customer_id):
    return ads_utils.get_ads_service(
      'AdwordsUserListService', 'v201809', self.oauth_credentials, self.developer_token.get(), customer_id)

  def _assert_execution_is_valid(self, elements, any_execution) -> None:
    ads_utils.assert_elements_have_same_execution(elements)
    destination= any_execution.destination.destination_metadata

    ## TODO: the number of parameters vary by upload. This test could be parameterized
    if not destination[0]:
      raise ValueError(
          'Missing destination information. Received {}'.format(str(destination)))

  @utils.safe_process(logger=logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader"))
  def process(self, elements, **kwargs) -> None:
    """
    Args:
       elements: List of dict with two elements: 'execution' and 'row'. All executions must be equal.
    """
    if not self.active:
      logging.getLogger().warning('Skipping upload to ads, parameters not configured.')
      return

    any_execution=elements[0]['execution']
    if any_execution.destination.destination_type is self.get_action_type():
      self._assert_execution_is_valid(elements, any_execution)
      user_list_service=self._get_user_list_service(any_execution.account_config.google_ads_account_id)
      list_id=self._create_list_if_it_does_not_exist(user_list_service,
                                                            any_execution.destination.destination_metadata[0],
                                                            self.get_list_definition(any_execution.account_config, any_execution.destination.destination_metadata[0]))

      rows=self.get_filtered_rows(
          utils.extract_rows(elements), self.get_row_keys())
      logging.getLogger().warning('Uploading {} rows to Google Ads'.format(len(rows)))
      mutate_members_operation={
        'operand': {
          'userListId': list_id,
          'membersList': rows
        },
        'operator': any_execution.destination.destination_metadata[1]
      }
      user_list_service.mutateMembers([mutate_members_operation])
    yield elements

  def get_filtered_rows(self, rows:List[Any], keys: List[str]) -> List[Dict[str, Any]]:
      return [{key: row.get(key) for key in keys} for row in rows]

  def get_list_definition(self, account_config:AccountConfig, list_name:str) -> Dict[str, Any]:
    pass

  def get_row_keys(self) -> List[str]:
    pass

  def get_action_type(self) -> DestinationType:
    pass<commit_msg>Fix logger in customer match abstract uploader<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import apache_beam as beam
import logging

from typing import Dict, Any, List

from uploaders import google_ads_utils as ads_utils
from uploaders import utils as utils
from utils.execution import AccountConfig, DestinationType
from utils.oauth_credentials import OAuthCredentials

class GoogleAdsCustomerMatchAbstractUploaderDoFn(beam.DoFn):

  def __init__(self, oauth_credentials: OAuthCredentials, developer_token: str):
    super().__init__()
    self.oauth_credentials = oauth_credentials
    self.developer_token = developer_token
    self.active = True
    if self.developer_token is None:
      self.active = False
    self._user_list_id_cache = {}

  def start_bundle(self):
    pass

  def _create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    if self._user_list_id_cache.get(list_name) is None:
      self._user_list_id_cache[list_name] = \
        self._do_create_list_if_it_does_not_exist(
            user_list_service, list_name, list_definition)

    return self._user_list_id_cache[list_name]

  def _do_create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    response = user_list_service.get([{
      'fields': ['Id', 'Name'],
      'predicates': [{
        'field': 'Name',
        'operator': 'EQUALS',
        'values': [list_name]
      }]
    }])
    if (len(response.entries) == 0):
      logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader").info('%s list does not exist, creating...' % list_name)
      result = user_list_service.mutate([{
        'operator': 'ADD',
        **list_definition
      }])
      id = result['value'][0]['id']
      logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader").info('List %s created with id: %d' % (list_name, id))
    else:
      id = response.entries[0]['id']
      logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader").info('List found %s with id: %d' % (list_name, id))
    return id

  # just to facilitate mocking
  def _get_user_list_service(self, customer_id):
    return ads_utils.get_ads_service(
      'AdwordsUserListService', 'v201809', self.oauth_credentials, self.developer_token.get(), customer_id)

  def _assert_execution_is_valid(self, elements, any_execution) -> None:
    ads_utils.assert_elements_have_same_execution(elements)
    destination= any_execution.destination.destination_metadata

    ## TODO: the number of parameters vary by upload. This test could be parameterized
    if not destination[0]:
      raise ValueError(
          'Missing destination information. Received {}'.format(str(destination)))

  @utils.safe_process(logger=logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader"))
  def process(self, elements, **kwargs) -> None:
    """
    Args:
       elements: List of dict with two elements: 'execution' and 'row'. All executions must be equal.
    """
    if not self.active:
      logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader").warning('Skipping upload to ads, parameters not configured.')
      return

    any_execution=elements[0]['execution']
    if any_execution.destination.destination_type is self.get_action_type():
      self._assert_execution_is_valid(elements, any_execution)
      user_list_service=self._get_user_list_service(any_execution.account_config.google_ads_account_id)
      list_id=self._create_list_if_it_does_not_exist(user_list_service,
                                                            any_execution.destination.destination_metadata[0],
                                                            self.get_list_definition(any_execution.account_config, any_execution.destination.destination_metadata[0]))

      rows=self.get_filtered_rows(
          utils.extract_rows(elements), self.get_row_keys())
      logging.getLogger("megalista.GoogleAdsCustomerMatchAbstractUploader").warning('Uploading {} rows to Google Ads'.format(len(rows)))
      mutate_members_operation={
        'operand': {
          'userListId': list_id,
          'membersList': rows
        },
        'operator': any_execution.destination.destination_metadata[1]
      }
      user_list_service.mutateMembers([mutate_members_operation])
    yield elements

  def get_filtered_rows(self, rows:List[Any], keys: List[str]) -> List[Dict[str, Any]]:
      return [{key: row.get(key) for key in keys} for row in rows]

  def get_list_definition(self, account_config:AccountConfig, list_name:str) -> Dict[str, Any]:
    pass

  def get_row_keys(self) -> List[str]:
    pass

  def get_action_type(self) -> DestinationType:
    pass<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import csv

import attr
from attr.validators import optional
from attr.validators import instance_of as is_a

BLAT_GENOMES = {
    'anopheles_gambiae',
    'arabidopsis_thaliana',
    'bombyx_mori',
    'caenorhabditis_elegans',
    'dictyostelium_discoideum',
    'drosophila_melanogaster',
    'homo_sapiens',
    'mus_musculus',
    'plasmodium_falciparum',
    'rattus_norvegicus'
    'saccharomyces_cerevisiae',
    'schizosaccharomyces_pombe',
}

EXAMPLE_LOCATIONS = {
    'homo_sapiens': {
        'chromosome': 'X',
        'start': 73819307,
        'end': 73856333,
    },
    'mus_musculus': {
        'chromosome': 1,
        'start': 86351908,
        'end': 86352200,
    },
    'danio_rerio': {
        'chromosome': 9,
        'start': 7633910,
        'end': 7634210,
    },
    'bos_taurus': {
        'chromosome': 15,
        'start': 82197673,
        'end': 82197837,
    },
    'rattus_norvegicus': {
        'chromosome': 'X',
        'start': 118277628,
        'end': 118277850,
    },
    'felis_catus': {
        'chromosome': 'X',
        'start': 18058223,
        'end': 18058546,
    },
    'macaca_mulatta': {
        'chromosome': 1,
        'start': 146238837,
        'end': 146238946,
    },
    'pan_troglodytes': {
        'chromosome': 11,
        'start': 78369004,
        'end': 78369219,
    },
    'canis_familiaris': {
        'chromosome': 19,
        'start': 22006909,
        'end': 22007119,
    },
    'gallus_gallus': {
        'chromosome': 9,
        'start': 15676031,
        'end': 15676160,
    },
    'xenopus_tropicalis': {
        'chromosome': 'NC_006839',
        'start': 11649,
        'end': 11717,
    },
    'saccharomyces_cerevisiae': {
        'chromosome': 'XII',
        'start': 856709,
        'end': 856919,
    },
    'schizosaccharomyces_pombe': {
        'chromosome': 'I',
        'start': 540951,
        'end': 544327,
    },
    'caenorhabditis_elegans': {
        'chromosome': 'III',
        'start': 11467363,
        'end': 11467705,
    },
    'drosophila_melanogaster': {
        'chromosome': '3R',
        'start': 7474331,
        'end': 7475217,
    },
    'bombyx_mori': {
        'chromosome': 'scaf16',
        'start': 6180018,
        'end': 6180422,
    },
    'anopheles_gambiae': {
        'chromosome': '2R',
        'start': 34644956,
        'end': 34645131,
    },
    'dictyostelium_discoideum': {
        'chromosome': 2,
        'start': 7874546,
        'end': 7876498,
    },
    'plasmodium_falciparum': {
        'chromosome': 13,
        'start': 2796339,
        'end': 2798488,
    },
    'arabidopsis_thaliana': {
        'chromosome': 2,
        'start': 18819643,
        'end': 18822629,
    }
}

def reconcile_taxids(taxid):
    """
    Sometimes Ensembl taxid and ENA/Expert Database taxid do not match,
    so to reconcile the differences, taxids in the ensembl_assembly table
    are overriden to match other data.
    """

    taxid = str(taxid)
    if taxid == '284812':  # Ensembl assembly for Schizosaccharomyces pombe
        return 4896  # Pombase and ENA xrefs for Schizosaccharomyces pombe
    return int(taxid)

class InvalidDomain(Exception):
    """
    Raised when we cannot compute a URL for a domain.
    """
    pass

@attr.s()
class AssemblyExample(object):
    chromosome = attr.ib(validator=is_a(basestring))
    start = attr.ib(validator=is_a(int))
    end = attr.ib(validator=is_a(int))

    @classmethod
    def build(cls, raw, example_locations):
        key = raw['species.url'].lower()
        example = example_locations.get(key, None)
        if not example:
            return None

        return cls(
            chromosome=example['chromosome'],
            start=example['start'],
            end=example['end'],
        )

@attr.s()
class AssemblyInfo(object):
    assembly_id = attr.ib(validator=is_a(basestring))
    assembly_full_name = attr.ib(validator=is_a(basestring))
    gca_accession = attr.ib(validator=is_a(basestring))
    assembly_ucsc = attr.ib(validator=optional(is_a(basestring)))
    common_name = attr.ib(validator=optional(is_a(basestring)))
    taxid = attr.ib(validator=is_a(int))
    ensembl_url = attr.ib(validator=is_a(basestring))
    division = attr.ib(validator=is_a(basestring))
    blat_mapping = attr.ib(validator=is_a(bool))
    example = attr.ib(validator=optional(is_a(AssemblyExample)))

    @classmethod
    def build(cls, raw, example_locations):
        url = raw['species.url'].lower()
        is_mapped = url in BLAT_GENOMES

        return cls(
            assembly_id=raw['assembly.default'],
            assembly_full_name=raw['assembly.name'],
            gca_accession=raw.get('assembly.accession', None),
            assembly_ucsc=raw.get('assembly.ucsc_alias', None),
            common_name=raw.get('species.common_name', None),
            taxid=reconcile_taxids(raw['species.taxonomy_id']),
            ensembl_url=url,
            division=raw['species.division'],
            blat_mapping=is_mapped,
            example=AssemblyExample.build(raw, example_locations),
        )

    @property
    def domain_url(self):
        """Given E! division, returns E!/E! Genomes url."""

        if self.division == 'Ensembl':
            return 'ensembl.org'
        if self.division == 'EnsemblPlants':
            return 'plants.ensembl.org'
        if self.division == 'EnsemblMetazoa':
            return 'metazoa.ensembl.org'
        if self.division == 'EnsemblBacteria':
            return 'bacteria.ensembl.org'
        if self.division == 'EnsemblFungi':
            return 'fungi.ensembl.org'
        if self.division == 'EnsemblProtists':
            return 'protists.ensembl.org'
        raise InvalidDomain()

    def writeable(self):
        chromosome = None
        start = None
        end = None
        if self.example:
            chromosome = self.example.chromosome
            start = self.example.start
            end = self.example.end

        return [
            self.assembly_id,
            self.assembly_full_name,
            self.gca_accession,
            self.assembly_ucsc,
            self.common_name,
            self.taxid,
            self.domain_url,
            self.division,
            self.domain_url,
            chromosome,
            start,
            end,
            int(self.blat_mapping),
        ]

def parse(handle, example_locations):
    """
    This parses the TSV file for assembly information and produces an array
    of the data to write to the database.
    """

    reader = csv.reader(handle, delimiter='\t')
    return AssemblyInfo.build(dict(reader), example_locations)

def write(handle, output):
    """
    Parse the given input handle and write the readable data to the CSV.
    """

    data = parse(handle, EXAMPLE_LOCATIONS)
    csv.writer(output).writerow(data.writeable())
<commit_msg>Ensure chromosome is always a string<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import csv

import attr
from attr.validators import optional
from attr.validators import instance_of as is_a

BLAT_GENOMES = {
    'anopheles_gambiae',
    'arabidopsis_thaliana',
    'bombyx_mori',
    'caenorhabditis_elegans',
    'dictyostelium_discoideum',
    'drosophila_melanogaster',
    'homo_sapiens',
    'mus_musculus',
    'plasmodium_falciparum',
    'rattus_norvegicus'
    'saccharomyces_cerevisiae',
    'schizosaccharomyces_pombe',
}

EXAMPLE_LOCATIONS = {
    'homo_sapiens': {
        'chromosome': 'X',
        'start': 73819307,
        'end': 73856333,
    },
    'mus_musculus': {
        'chromosome': 1,
        'start': 86351908,
        'end': 86352200,
    },
    'danio_rerio': {
        'chromosome': 9,
        'start': 7633910,
        'end': 7634210,
    },
    'bos_taurus': {
        'chromosome': 15,
        'start': 82197673,
        'end': 82197837,
    },
    'rattus_norvegicus': {
        'chromosome': 'X',
        'start': 118277628,
        'end': 118277850,
    },
    'felis_catus': {
        'chromosome': 'X',
        'start': 18058223,
        'end': 18058546,
    },
    'macaca_mulatta': {
        'chromosome': 1,
        'start': 146238837,
        'end': 146238946,
    },
    'pan_troglodytes': {
        'chromosome': 11,
        'start': 78369004,
        'end': 78369219,
    },
    'canis_familiaris': {
        'chromosome': 19,
        'start': 22006909,
        'end': 22007119,
    },
    'gallus_gallus': {
        'chromosome': 9,
        'start': 15676031,
        'end': 15676160,
    },
    'xenopus_tropicalis': {
        'chromosome': 'NC_006839',
        'start': 11649,
        'end': 11717,
    },
    'saccharomyces_cerevisiae': {
        'chromosome': 'XII',
        'start': 856709,
        'end': 856919,
    },
    'schizosaccharomyces_pombe': {
        'chromosome': 'I',
        'start': 540951,
        'end': 544327,
    },
    'caenorhabditis_elegans': {
        'chromosome': 'III',
        'start': 11467363,
        'end': 11467705,
    },
    'drosophila_melanogaster': {
        'chromosome': '3R',
        'start': 7474331,
        'end': 7475217,
    },
    'bombyx_mori': {
        'chromosome': 'scaf16',
        'start': 6180018,
        'end': 6180422,
    },
    'anopheles_gambiae': {
        'chromosome': '2R',
        'start': 34644956,
        'end': 34645131,
    },
    'dictyostelium_discoideum': {
        'chromosome': 2,
        'start': 7874546,
        'end': 7876498,
    },
    'plasmodium_falciparum': {
        'chromosome': 13,
        'start': 2796339,
        'end': 2798488,
    },
    'arabidopsis_thaliana': {
        'chromosome': 2,
        'start': 18819643,
        'end': 18822629,
    }
}

def reconcile_taxids(taxid):
    """
    Sometimes Ensembl taxid and ENA/Expert Database taxid do not match,
    so to reconcile the differences, taxids in the ensembl_assembly table
    are overriden to match other data.
    """

    taxid = str(taxid)
    if taxid == '284812':  # Ensembl assembly for Schizosaccharomyces pombe
        return 4896  # Pombase and ENA xrefs for Schizosaccharomyces pombe
    return int(taxid)

class InvalidDomain(Exception):
    """
    Raised when we cannot compute a URL for a domain.
    """
    pass

@attr.s()
class AssemblyExample(object):
    chromosome = attr.ib(validator=is_a(basestring))
    start = attr.ib(validator=is_a(int))
    end = attr.ib(validator=is_a(int))

    @classmethod
    def build(cls, raw, example_locations):
        key = raw['species.url'].lower()
        example = example_locations.get(key, None)
        if not example:
            return None

        return cls(
            chromosome=str(example['chromosome']),
            start=example['start'],
            end=example['end'],
        )

@attr.s()
class AssemblyInfo(object):
    assembly_id = attr.ib(validator=is_a(basestring))
    assembly_full_name = attr.ib(validator=is_a(basestring))
    gca_accession = attr.ib(validator=is_a(basestring))
    assembly_ucsc = attr.ib(validator=optional(is_a(basestring)))
    common_name = attr.ib(validator=optional(is_a(basestring)))
    taxid = attr.ib(validator=is_a(int))
    ensembl_url = attr.ib(validator=is_a(basestring))
    division = attr.ib(validator=is_a(basestring))
    blat_mapping = attr.ib(validator=is_a(bool))
    example = attr.ib(validator=optional(is_a(AssemblyExample)))

    @classmethod
    def build(cls, raw, example_locations):
        url = raw['species.url'].lower()
        is_mapped = url in BLAT_GENOMES

        return cls(
            assembly_id=raw['assembly.default'],
            assembly_full_name=raw['assembly.name'],
            gca_accession=raw.get('assembly.accession', None),
            assembly_ucsc=raw.get('assembly.ucsc_alias', None),
            common_name=raw.get('species.common_name', None),
            taxid=reconcile_taxids(raw['species.taxonomy_id']),
            ensembl_url=url,
            division=raw['species.division'],
            blat_mapping=is_mapped,
            example=AssemblyExample.build(raw, example_locations),
        )

    @property
    def domain_url(self):
        """Given E! division, returns E!/E! Genomes url."""

        if self.division == 'Ensembl':
            return 'ensembl.org'
        if self.division == 'EnsemblPlants':
            return 'plants.ensembl.org'
        if self.division == 'EnsemblMetazoa':
            return 'metazoa.ensembl.org'
        if self.division == 'EnsemblBacteria':
            return 'bacteria.ensembl.org'
        if self.division == 'EnsemblFungi':
            return 'fungi.ensembl.org'
        if self.division == 'EnsemblProtists':
            return 'protists.ensembl.org'
        raise InvalidDomain()

    def writeable(self):
        chromosome = None
        start = None
        end = None
        if self.example:
            chromosome = self.example.chromosome
            start = self.example.start
            end = self.example.end

        return [
            self.assembly_id,
            self.assembly_full_name,
            self.gca_accession,
            self.assembly_ucsc,
            self.common_name,
            self.taxid,
            self.domain_url,
            self.division,
            self.domain_url,
            chromosome,
            start,
            end,
            int(self.blat_mapping),
        ]

def parse(handle, example_locations):
    """
    This parses the TSV file for assembly information and produces an array
    of the data to write to the database.
    """

    reader = csv.reader(handle, delimiter='\t')
    return AssemblyInfo.build(dict(reader), example_locations)

def write(handle, output):
    """
    Parse the given input handle and write the readable data to the CSV.
    """

    data = parse(handle, EXAMPLE_LOCATIONS)
    csv.writer(output).writerow(data.writeable())
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import apache_beam as beam
import logging

from typing import Dict, Any, List

from uploaders import google_ads_utils as ads_utils
from uploaders import utils as utils
from utils.execution import Action
from utils.oauth_credentials import OAuthCredentials

class GoogleAdsCustomerMatchAbstractUploaderDoFn(beam.DoFn):

  def __init__(self, oauth_credentials: OAuthCredentials, developer_token: str, customer_id: str):
    super().__init__()
    self.oauth_credentials = oauth_credentials
    self.developer_token = developer_token
    self.customer_id = customer_id
    self.active = True
    if self.developer_token is None or self.customer_id is None:
      self.active = False
    self._user_list_id_cache = {}

  def start_bundle(self):
    pass

  def _create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    if self._user_list_id_cache.get(list_name) is None:
      self._user_list_id_cache[list_name] = \
        self._do_create_list_if_it_does_not_exist(
            user_list_service, list_name, list_definition)

    return self._user_list_id_cache[list_name]

  def _do_create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    response = user_list_service.get([{
      'fields': ['Id', 'Name'],
      'predicates': [{
        'field': 'Name',
        'operator': 'EQUALS',
        'values': [list_name]
      }]
    }])
    if (len(response.entries) == 0):
      logging.getLogger().info('%s list does not exist, creating...' % list_name)
      result = user_list_service.mutate([{
        'operator': 'ADD',
        **list_definition
      }])
      id = result['value'][0]['id']
      logging.getLogger().info('List %s created with id: %d' % (list_name, id))
    else:
      id = response.entries[0]['id']
      logging.getLogger().info('List found %s with id: %d' % (list_name, id))
    return id

  # just to facilitate mocking
  def _get_user_list_service(self):
    return ads_utils.get_ads_service(
      'AdwordsUserListService', 'v201809', self.oauth_credentials, self.developer_token.get(), self.customer_id.get())

  def _assert_execution_is_valid(self, elements, any_execution) -> None:
    ads_utils.assert_elements_have_same_execution(elements)
    ads_utils.assert_right_type_action(any_execution, self.get_action_type())
    destination= any_execution.destination_metadata
    if not destination[0]:
      raise ValueError(
          'Missing destination information. Received {}'.format(str(destination)))

  def process(self, elements, **kwargs) -> None:
    """
    Args:
       elements: List of dict with two elements: 'execution' and 'row'. All executions must be equal.
    """
    if not self.active:
      logging.getLogger().warning('Skipping upload to ads, parameters not configured.')
      return

    if len(elements) == 0:
      logging.getLogger().warning('Skipping upload to ads, received no elements.')
      return
    any_execution=elements[0]['execution']
    self._assert_execution_is_valid(elements, any_execution)
    user_list_service=self._get_user_list_service()
    list_id=self._create_list_if_it_does_not_exist(user_list_service,
                                                          any_execution.destination_metadata[0],
                                                          self.get_list_definition(any_execution.destination_metadata[0]))

    rows=self.get_filtered_rows(
        utils.extract_rows(elements), self.get_row_keys())
    logging.getLogger().warning('Uploading {} rows to Google Ads'.format(len(rows)))
    mutate_members_operation={
      'operand': {
        'userListId': list_id,
        'membersList': rows
      },
      'operator': any_execution.destination_metadata[1]
    }
    user_list_service.mutateMembers([mutate_members_operation])

  def get_filtered_rows(self, rows:List[Any], keys: List[str]) -> List[Dict[str, Any]]:
      return [{key: row.get(key) for key in keys} for row in rows]

  def get_list_definition(self, list_name:str) -> Dict[str, Any]:
    pass

  def get_row_keys(self) -> List[str]:
    pass

  def get_action_type(self) -> Action:
    pass<commit_msg>Enabling piping across uploaders<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import apache_beam as beam
import logging

from typing import Dict, Any, List

from uploaders import google_ads_utils as ads_utils
from uploaders import utils as utils
from utils.execution import Action
from utils.oauth_credentials import OAuthCredentials

class GoogleAdsCustomerMatchAbstractUploaderDoFn(beam.DoFn):

  def __init__(self, oauth_credentials: OAuthCredentials, developer_token: str, customer_id: str):
    super().__init__()
    self.oauth_credentials = oauth_credentials
    self.developer_token = developer_token
    self.customer_id = customer_id
    self.active = True
    if self.developer_token is None or self.customer_id is None:
      self.active = False
    self._user_list_id_cache = {}

  def start_bundle(self):
    pass

  def _create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    if self._user_list_id_cache.get(list_name) is None:
      self._user_list_id_cache[list_name] = \
        self._do_create_list_if_it_does_not_exist(
            user_list_service, list_name, list_definition)

    return self._user_list_id_cache[list_name]

  def _do_create_list_if_it_does_not_exist(self, user_list_service, list_name: str, list_definition: Dict[str, Any]) -> str:
    response = user_list_service.get([{
      'fields': ['Id', 'Name'],
      'predicates': [{
        'field': 'Name',
        'operator': 'EQUALS',
        'values': [list_name]
      }]
    }])
    if (len(response.entries) == 0):
      logging.getLogger().info('%s list does not exist, creating...' % list_name)
      result = user_list_service.mutate([{
        'operator': 'ADD',
        **list_definition
      }])
      id = result['value'][0]['id']
      logging.getLogger().info('List %s created with id: %d' % (list_name, id))
    else:
      id = response.entries[0]['id']
      logging.getLogger().info('List found %s with id: %d' % (list_name, id))
    return id

  # just to facilitate mocking
  def _get_user_list_service(self):
    return ads_utils.get_ads_service(
      'AdwordsUserListService', 'v201809', self.oauth_credentials, self.developer_token.get(), self.customer_id.get())

  def _assert_execution_is_valid(self, elements, any_execution) -> None:
    ads_utils.assert_elements_have_same_execution(elements)
    destination= any_execution.destination_metadata
    if not destination[0]:
      raise ValueError(
          'Missing destination information. Received {}'.format(str(destination)))

  def process(self, elements, **kwargs) -> None:
    """
    Args:
       elements: List of dict with two elements: 'execution' and 'row'. All executions must be equal.
    """
    if not self.active:
      logging.getLogger().warning('Skipping upload to ads, parameters not configured.')
      return

    if len(elements) == 0:
      logging.getLogger().warning('Skipping upload to ads, received no elements.')
      return

    any_execution=elements[0]['execution']
    if any_execution.action is self.get_action_type():
      self._assert_execution_is_valid(elements, any_execution)
      user_list_service=self._get_user_list_service()
      list_id=self._create_list_if_it_does_not_exist(user_list_service,
                                                            any_execution.destination_metadata[0],
                                                            self.get_list_definition(any_execution.destination_metadata[0]))

      rows=self.get_filtered_rows(
          utils.extract_rows(elements), self.get_row_keys())
      logging.getLogger().warning('Uploading {} rows to Google Ads'.format(len(rows)))
      mutate_members_operation={
        'operand': {
          'userListId': list_id,
          'membersList': rows
        },
        'operator': any_execution.destination_metadata[1]
      }
      user_list_service.mutateMembers([mutate_members_operation])
    yield elements

  def get_filtered_rows(self, rows:List[Any], keys: List[str]) -> List[Dict[str, Any]]:
      return [{key: row.get(key) for key in keys} for row in rows]

  def get_list_definition(self, list_name:str) -> Dict[str, Any]:
    pass

  def get_row_keys(self) -> List[str]:
    pass

  def get_action_type(self) -> Action:
    pass<|endoftext|>


<commit_before># Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX IPython notebook formatter integration.

Note: these APIs are **experimental** and major changes to interface and
functionality are expected.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Standard Imports

from six.moves import builtins
from typing import Callable, List, Optional, Text, Tuple, Type, Union

from tfx.components.base.base_component import BaseComponent
from tfx.orchestration.interactive.execution_result import ExecutionResult
from tfx.types.artifact import Artifact
from tfx.types.channel import Channel
from tfx.types.component_spec import _PropertyDictWrapper

STATIC_HTML_CONTENTS = u"""

"""

class NotebookFormatter(object):
  """Formats a TFX component in the context of an interactive notebook."""

  _DEFAULT_TITLE_FORMAT = ('%s',
                           ['__class__.__name__'])

  def __init__(self,
               cls: Type[object],
               attributes: List[Text] = None,
               title_format: Tuple[Text, List[Union[Text, Callable]]] = None):  # pylint: disable=g-bare-generic
    """Constructs a NotebookFormatter.

    Args:
      cls: The TFX class to be formated by this NotebookFormatter instance.
      attributes: A list of string attributes that are to be displayed by this
        formatter. Can be a nested field specifier with nested attribute names
        separated by "." (e.g. to get `obj.a.b`, specify the attribute string
        "a.b").
      title_format: A 2-tuple consisting of (1) a format string and (2) a list
        of either string attribute names (possible of nested field specifiers as
        in "attributes" above) or callback callable objects taking as input the
        object to be formatted and returning the value for that position of the
        format string. If not specified, the default title format will be used.
    """
    self.cls = cls
    self.attributes = attributes or []
    self.title_format = title_format or NotebookFormatter._DEFAULT_TITLE_FORMAT

  def _extended_getattr(self, obj: object, property_name: Text) -> object:
    """Get a possibly nested attribute of a given object."""
    if callable(property_name):
      return property_name(obj)
    parts = property_name.split('.')
    current = obj
    for part in parts:
      current = getattr(current, part)
    return current

  def render(self,
             obj: object,
             expanded: bool = True,
             seen_elements: Optional[set] = None) -> Text:  # pylint: disable=g-bare-generic
    """Render a given object as an HTML string.

    Args:
      obj: The object to be rendered.
      expanded: Whether the object is to be expanded by default.
      seen_elements: Optionally, a set of seen elements to not re-render to
        prevent a rendering cycle.

    Returns:
      Formatted HTML string representing the object, for notebook display.
    """
    seen_elements = seen_elements or set()
    if id(obj) in seen_elements:
      return '(recursion in rendering object)'
    seen_elements.add(id(obj))
    if not isinstance(obj, self.cls):
      raise ValueError('Expected object of type %s but got %s.' %
                       (self.cls, obj))
    seen_elements.remove(id(obj))
    return STATIC_HTML_CONTENTS + (
        '
'
        '
'
        ''
        '%s at 0x%x
%s'
        '
') % (' expanded' if expanded else ' collapsed',
                     self.render_title(obj), id(obj),
                     self.render_attributes(obj, seen_elements))

  def render_title(self, obj: object) -> Text:
    """Render the title section of an object."""
    title_format = self.title_format
    values = []
    for property_name in title_format[1]:
      values.append(self._extended_getattr(obj, property_name))
    return title_format[0] % tuple(values)

  def render_value(self,
                   value: object,
                   seen_elements: set) -> object:  # pylint: disable=g-bare-generic
    """Render the value section of an object."""
    if isinstance(value, _PropertyDictWrapper):
      value = value.get_all()
    if isinstance(value, dict):
      value = self.render_dict(value, seen_elements)
    if isinstance(value, list):
      value = self.render_list(value, seen_elements)
    for cls in value.__class__.mro():
      if cls in FORMATTER_REGISTRY:
        value = FORMATTER_REGISTRY[cls].render(
            value, expanded=False, seen_elements=seen_elements)
        break
    return value

  def render_attributes(self,
                        obj: object,
                        seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render the attributes section of an object."""
    attr_trs = []
    for property_name in self.attributes:
      value = self._extended_getattr(obj, property_name)
      value = self.render_value(value, seen_elements)
      attr_trs.append(
          ('
.%s	'
           '%s
') % (property_name, value))
    return '
%s
' % ''.join(attr_trs)

  def render_dict(self,
                  obj: dict,  # pylint: disable=g-bare-generic
                  seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render a dictionary table."""
    if not obj:
      return '{}'
    attr_trs = []
    for key, value in obj.items():
      value = self.render_value(value, seen_elements)
      attr_trs.append(('
[%r]	'
                       '%s
') % (key, value))
    return '
%s
' % ''.join(attr_trs)

  def render_list(self,
                  obj: list,  # pylint: disable=g-bare-generic
                  seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render a list table."""
    if not obj:
      return '[]'
    attr_trs = []
    for i, value in enumerate(obj):
      value = self.render_value(value, seen_elements)
      attr_trs.append(('
[%d]	'
                       '%s
') % (i, value))
    return '
%s
' % ''.join(attr_trs)

def _create_formatters(formatters_spec):
  result = {}
  for cls, kwargs in formatters_spec.items():
    formatter = NotebookFormatter(cls, **kwargs)
    result[cls] = formatter
  return result

FORMATTER_REGISTRY = _create_formatters({
    Artifact: {
        'attributes': ['type_name', 'uri', 'span', 'split'],
        'title_format': (
            ('Artifact of type '
             '%r (uri: %s)'),
            ['type_name', 'uri']),
    },
    BaseComponent: {
        'attributes': ['inputs', 'outputs', 'exec_properties']
    },
    Channel: {
        'attributes': ['type_name', '_artifacts'],
        'title_format': (
            ('Channel of type '
             '%r (%d artifact%s)'),
            [
                'type_name', lambda o: len(o._artifacts),  # pylint: disable=protected-access
                lambda o: '' if len(o._artifacts) == 1 else 's'  # pylint: disable=protected-access
            ]),
    },
    ExecutionResult: {
        'attributes': [
            'execution_id', 'component', 'component.inputs', 'component.outputs'
        ]
    },
})

def register_formatters():
  """Register HTML notebook formatters for TFX classes.

  This method registers HTML formatters for TFX classes for display in
  IPython / Jupyter / Colab notebooks. No action will be performed if called
  outside a notebook environment.
  """
  if getattr(builtins, '__IPYTHON__', None):
    try:
      ipython = __import__('IPython.core.getipython').get_ipython()
    except ImportError:
      return
    html_formatter = ipython.display_formatter.formatters['text/html']
    for cls, formatter in FORMATTER_REGISTRY.items():
      html_formatter.for_type(cls, formatter.render)
<commit_msg>Update interactive notebook formatter registration to be skipped outside notebook context.<commit_after># Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TFX IPython notebook formatter integration.

Note: these APIs are **experimental** and major changes to interface and
functionality are expected.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Standard Imports

from six.moves import builtins
from typing import Callable, List, Optional, Text, Tuple, Type, Union

from tfx.components.base.base_component import BaseComponent
from tfx.orchestration.interactive.execution_result import ExecutionResult
from tfx.types.artifact import Artifact
from tfx.types.channel import Channel
from tfx.types.component_spec import _PropertyDictWrapper

STATIC_HTML_CONTENTS = u"""

"""

class NotebookFormatter(object):
  """Formats a TFX component in the context of an interactive notebook."""

  _DEFAULT_TITLE_FORMAT = ('%s',
                           ['__class__.__name__'])

  def __init__(self,
               cls: Type[object],
               attributes: List[Text] = None,
               title_format: Tuple[Text, List[Union[Text, Callable]]] = None):  # pylint: disable=g-bare-generic
    """Constructs a NotebookFormatter.

    Args:
      cls: The TFX class to be formated by this NotebookFormatter instance.
      attributes: A list of string attributes that are to be displayed by this
        formatter. Can be a nested field specifier with nested attribute names
        separated by "." (e.g. to get `obj.a.b`, specify the attribute string
        "a.b").
      title_format: A 2-tuple consisting of (1) a format string and (2) a list
        of either string attribute names (possible of nested field specifiers as
        in "attributes" above) or callback callable objects taking as input the
        object to be formatted and returning the value for that position of the
        format string. If not specified, the default title format will be used.
    """
    self.cls = cls
    self.attributes = attributes or []
    self.title_format = title_format or NotebookFormatter._DEFAULT_TITLE_FORMAT

  def _extended_getattr(self, obj: object, property_name: Text) -> object:
    """Get a possibly nested attribute of a given object."""
    if callable(property_name):
      return property_name(obj)
    parts = property_name.split('.')
    current = obj
    for part in parts:
      current = getattr(current, part)
    return current

  def render(self,
             obj: object,
             expanded: bool = True,
             seen_elements: Optional[set] = None) -> Text:  # pylint: disable=g-bare-generic
    """Render a given object as an HTML string.

    Args:
      obj: The object to be rendered.
      expanded: Whether the object is to be expanded by default.
      seen_elements: Optionally, a set of seen elements to not re-render to
        prevent a rendering cycle.

    Returns:
      Formatted HTML string representing the object, for notebook display.
    """
    seen_elements = seen_elements or set()
    if id(obj) in seen_elements:
      return '(recursion in rendering object)'
    seen_elements.add(id(obj))
    if not isinstance(obj, self.cls):
      raise ValueError('Expected object of type %s but got %s.' %
                       (self.cls, obj))
    seen_elements.remove(id(obj))
    return STATIC_HTML_CONTENTS + (
        '
'
        '
'
        ''
        '%s at 0x%x
%s'
        '
') % (' expanded' if expanded else ' collapsed',
                     self.render_title(obj), id(obj),
                     self.render_attributes(obj, seen_elements))

  def render_title(self, obj: object) -> Text:
    """Render the title section of an object."""
    title_format = self.title_format
    values = []
    for property_name in title_format[1]:
      values.append(self._extended_getattr(obj, property_name))
    return title_format[0] % tuple(values)

  def render_value(self,
                   value: object,
                   seen_elements: set) -> object:  # pylint: disable=g-bare-generic
    """Render the value section of an object."""
    if isinstance(value, _PropertyDictWrapper):
      value = value.get_all()
    if isinstance(value, dict):
      value = self.render_dict(value, seen_elements)
    if isinstance(value, list):
      value = self.render_list(value, seen_elements)
    for cls in value.__class__.mro():
      if cls in FORMATTER_REGISTRY:
        value = FORMATTER_REGISTRY[cls].render(
            value, expanded=False, seen_elements=seen_elements)
        break
    return value

  def render_attributes(self,
                        obj: object,
                        seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render the attributes section of an object."""
    attr_trs = []
    for property_name in self.attributes:
      value = self._extended_getattr(obj, property_name)
      value = self.render_value(value, seen_elements)
      attr_trs.append(
          ('
.%s	'
           '%s
') % (property_name, value))
    return '
%s
' % ''.join(attr_trs)

  def render_dict(self,
                  obj: dict,  # pylint: disable=g-bare-generic
                  seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render a dictionary table."""
    if not obj:
      return '{}'
    attr_trs = []
    for key, value in obj.items():
      value = self.render_value(value, seen_elements)
      attr_trs.append(('
[%r]	'
                       '%s
') % (key, value))
    return '
%s
' % ''.join(attr_trs)

  def render_list(self,
                  obj: list,  # pylint: disable=g-bare-generic
                  seen_elements: set) -> Text:  # pylint: disable=g-bare-generic
    """Render a list table."""
    if not obj:
      return '[]'
    attr_trs = []
    for i, value in enumerate(obj):
      value = self.render_value(value, seen_elements)
      attr_trs.append(('
[%d]	'
                       '%s
') % (i, value))
    return '
%s
' % ''.join(attr_trs)

def _create_formatters(formatters_spec):
  result = {}
  for cls, kwargs in formatters_spec.items():
    formatter = NotebookFormatter(cls, **kwargs)
    result[cls] = formatter
  return result

FORMATTER_REGISTRY = _create_formatters({
    Artifact: {
        'attributes': ['type_name', 'uri', 'span', 'split'],
        'title_format': (
            ('Artifact of type '
             '%r (uri: %s)'),
            ['type_name', 'uri']),
    },
    BaseComponent: {
        'attributes': ['inputs', 'outputs', 'exec_properties']
    },
    Channel: {
        'attributes': ['type_name', '_artifacts'],
        'title_format': (
            ('Channel of type '
             '%r (%d artifact%s)'),
            [
                'type_name', lambda o: len(o._artifacts),  # pylint: disable=protected-access
                lambda o: '' if len(o._artifacts) == 1 else 's'  # pylint: disable=protected-access
            ]),
    },
    ExecutionResult: {
        'attributes': [
            'execution_id', 'component', 'component.inputs', 'component.outputs'
        ]
    },
})

def register_formatters():
  """Register HTML notebook formatters for TFX classes.

  This method registers HTML formatters for TFX classes for display in
  IPython / Jupyter / Colab notebooks. No action will be performed if called
  outside a notebook environment.
  """
  if getattr(builtins, '__IPYTHON__', None):
    # Skip registration if (1) IPython is not installed or (2) if IPython is
    # installed but we are not running in the notebook context (in this case,
    # get_ipython() returns None).
    try:
      ipython = __import__('IPython.core.getipython').get_ipython()
      if not ipython:
        return
    except ImportError:
      return
    html_formatter = ipython.display_formatter.formatters['text/html']
    for cls, formatter in FORMATTER_REGISTRY.items():
      html_formatter.for_type(cls, formatter.render)
<|endoftext|>


<commit_before>import os

import dj_database_url

BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
PROJECT_ROOT = os.path.dirname(os.path.dirname(BASE_DIR))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*=k@kj(!ba$+m1h@j8v3ctelp@w@1u1aqj2x^+)80&u!gt!ub5'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',

    'pipeline',

    'multisites',
    'tidbits',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

ROOT_URLCONF = 'superpunch.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',

                # Custom Context Processors
                'multisites.context_processors.sites',
            ],
        },
    },
]

WSGI_APPLICATION = 'superpunch.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

# dj-database-url
# https://github.com/kennethreitz/dj-database-url/

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get(
        'DATABASE_URL',
        'postgres://:@127.0.0.1:5432/superpunch'
    )),
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Site Frameworks
SITE_ID = 1
<commit_msg>fix error : should not specify SITE_ID to support multisites application<commit_after>import os

import dj_database_url

BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
PROJECT_ROOT = os.path.dirname(os.path.dirname(BASE_DIR))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '*=k@kj(!ba$+m1h@j8v3ctelp@w@1u1aqj2x^+)80&u!gt!ub5'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',

    'pipeline',

    'multisites',
    'tidbits',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

ROOT_URLCONF = 'superpunch.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',

                # Custom Context Processors
                'multisites.context_processors.sites',
            ],
        },
    },
]

WSGI_APPLICATION = 'superpunch.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

# dj-database-url
# https://github.com/kennethreitz/dj-database-url/

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get(
        'DATABASE_URL',
        'postgres://:@127.0.0.1:5432/superpunch'
    )),
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True
<|endoftext|>


<commit_before>#
# Copyright (C) 2013 RoboIME
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
from numpy import array, linspace
from numpy.linalg import norm

from .. import Skill
from ...utils.mathutils import exp
from ...utils.geom import Point, Line
from ...utils.pidcontroller import PidController

def force_equation(direction, distance, floor, factor, power):
    return direction * (floor + factor / distance) ** power

class Goto(Skill):
    """
    Sends the robot to a specified point with a specified orientation with no
    regard to the position of any other objects on the field.
    """

    # attraction coefificients
    attraction_factor = 6.0
    attraction_power = 2.3
    attraction_floor = 80.0

    # repulsion coefificients
    repulsion_factor = 8.0
    repulsion_power = 3.3
    repulsion_floor = 0.0

    # magnetic coefificients
    magnetic_factor = 9.0
    magnetic_power = 3.1
    magnetic_floor = 0.0

    # delta_speed coefificients
    delta_speed_factor = 2.0
    delta_speed_power = 1.4
    delta_speed_floor = 0.0

    # minimum distance to use on the equation
    # anything smaller is capped to this
    min_distance = 1e-5

    # distance to consider target arrival
    arrive_distance = 1e-3
    max_angle_error = 1.0

    # ignore other forces if attraction
    # force is as least this high:
    min_force_to_ignore_others = 100000

    # control params
    g = 9.80665
    mi = 0.1
    exp_k = 6

    #def __init__(self, robot, target=None, angle=None, final_target=None, referential=None, deterministic=True, avoid_collisions=True, **kwargs):
    def __init__(self, robot, target=None, angle=None, final_target=None, referential=None, ignore_defense_area=False, deterministic=True, avoid_collisions=False, use_norm_pid=False, separate_axis_control=False, **kwargs):
        """
        final_target: the ultimate position the robot should attain
        target: the intermediate position you're ACTUALLY sending the robot to
        angle: the orientation the robot should have at target

        final_target is there so that, on a tactic, you can make the robot follow a curve to final_target
        while not stopping near each target due to its extreme proximity to it by continuously adjusting
        target to be ever closer to final_target. These intermediate steps are not handled by this class,
        and the exact curve you want the robot to follow (and hence all the intermediate targets that you'll
        be switching in) should be specified in your own class.
        """
        super(Goto, self).__init__(robot, deterministic=deterministic, **kwargs)
        #TODO: find the right parameters
        self.angle_controller = PidController(kp=1., ki=.0, kd=.0, integ_max=.5, output_max=360)
        self.norm_controller = PidController(kp=.1, ki=0.01, kd=.5, integ_max=50., output_max=1.2)
        self.x_controller = PidController(kp=.3, ki=0.01, kd=.05, integ_max=5., output_max=.8)
        self.y_controller = PidController(kp=.3, ki=0.01, kd=.05, integ_max=5., output_max=.8)
        self.use_norm_pid = use_norm_pid
        self.separate_axis_control = separate_axis_control
        #self.angle_controller = PidController(kp=1.324, ki=0, kd=0, integ_max=6.55, output_max=1000)
        self._angle = angle
        self.target = target
        # self.final_target = final_target if final_target is not None else self.target
        self.final_target = final_target or target
        self.referential = referential
        self.avoid_collisions = avoid_collisions
        self.ignore_defense_area = ignore_defense_area
        self.avoid_collisions = False

    def arrived(self):
        return norm(array(self.robot) - array(self.target)) <= self.arrive_distance

    def oriented(self):
        return (180 + self.angle - self.robot.angle) % 360 - 180 <= self.max_angle_error

    @property
    def angle(self):
        if callable(self._angle):
            return self._angle()
        else:
            return self._angle or self.robot.angle

    @angle.setter
    def angle(self, angle):
        self._angle = angle

    @property
    def target(self):
        if callable(self._target):
            return self._target()
        else:
            return self._target or self.robot

    @target.setter
    def target(self, target):
        self._target = target

    @property
    def final_target(self):
        if callable(self._final_target):
            return self._final_target()
        else:
            return self._final_target or self.target

    @final_target.setter
    def final_target(self, target):
        self._final_target = target

    def attraction_force(self):
        # the error vector from the robot to the target point
        delta = array(self.target) - array(self.robot)

        # attractive force
        dist = max(self.robot.distance(self.final_target), self.min_distance)

        # normalize delta
        delta /= dist

        #attraction_force = delta * (1 + 1 / (dist / self.attraction_factor) ** self.attraction_power)
        #attraction_force = delta * (self.attraction_floor + (self.attraction_factor / dist) ** self.attraction_power)
        attraction_force = force_equation(delta, dist, self.attraction_floor, self.attraction_factor, self.attraction_power)

        return attraction_force

    def other_forces(self):
        robot = self.robot
        for other in filter(lambda other: other is not robot, self.world.iterrobots()):
            # difference of position
            delta = array(robot) - array(other)

            # considered distance
            dist = norm(delta) + robot.radius + other.radius

            # cap the distance to a minimum of 1mm
            dist = max(dist, self.min_distance)

            # normalize the delta
            delta /= norm(delta)

            # perpendicular delta
            pdelta = array((delta[1], -delta[0]))

            # normalized difference of speeds of the robots
            sdelta = other.speed - robot.speed
            sdelta /= max(norm(sdelta), self.min_distance)

            # calculating each force
            #repulsion_force = delta * (self.repulsion_factor / dist) ** self.repulsion_power
            repulsion_force = force_equation(delta, dist, 0, self.repulsion_factor, self.repulsion_power)
            #magnetic_force = pdelta * (self.magnetic_factor / dist) ** self.magnetic_power
            magnetic_force = force_equation(pdelta, dist, 0, self.magnetic_factor, self.magnetic_power)
            #delta_speed_force = sdelta * (self.delta_speed_factor / dist) ** self.delta_speed_power
            delta_speed_force = force_equation(sdelta, dist, 0, self.delta_speed_factor, self.delta_speed_power)
            yield (repulsion_force, magnetic_force, delta_speed_force)

    def _step(self):
        l = Line(self.robot, self.final_target)
        self.target = self.final_target if l.length < 0.30 else Point(self.robot.x + 0.30 * (l.coords[1][0] - l.coords[0][0]) / l.length, self.robot.y + 0.30 * (l.coords[1][1] - l.coords[0][1]) / l.length)

        r = self.robot
        t = self.target
        f_t = self.final_target
        #ref = self.referential if self.referential is not None else f_t
        # TODO: Check if target is within the boundaries of the field (augmented of the robot's radius).

        # check whether the point we want to go to will make the robot be in the defense area
        # if so then we'll go to the nearest point that meets that constraint
        #print "ball is in area?", r.world.is_in_defense_area(body=self.world.ball.buffer(r.radius), color=r.color)
        if not self.ignore_defense_area and r.world.is_in_defense_area(body=t.buffer(r.radius), color=r.color):
            self.target = self.point_away_from_defense_area
            t = self.target

        # angle control using PID controller
        if self.angle is not None and self.robot.angle is not None:
            self.angle_controller.input = (180 + self.angle - self.robot.angle) % 360 - 180
            #self.angle_controller.input = (180 + 0 - self.robot.angle) % 360 - 180
            self.angle_controller.feedback = 0.0
            self.angle_controller.step()
            va = self.angle_controller.output
        else:
            va = 0.0

        # the error vector from the robot to the target point
        error = array(t) - array(r)

        # the gradient of the field
        gradient = self.attraction_force()
        if norm(gradient) < self.min_force_to_ignore_others:
            gradient += sum(map(sum, self.other_forces()))
        # WTF
        if self.separate_axis_control:
            delta = array(f_t) - array(r)
            ghost_point = norm(delta) * gradient / norm(gradient)
            print gradient
            print ghost_point
            self.x_controller.input = ghost_point[0]
            self.y_controller.input = ghost_point[1]
            self.x_controller.feedback = 0.
            self.y_controller.feedback = 0.
            self.x_controller.step()
            self.y_controller.step()
            r.action.absolute_speeds = self.x_controller.output, self.y_controller.output, va
            print r.action.absolute_speeds

        else:
            if self.use_norm_pid:
                distance = norm(array(f_t) - array(r))
                self.norm_controller.input = distance
                self.norm_controller.feedback = 0.
                self.norm_controller.step()
                out = self.norm_controller.output
            else:
                # some crazy equation that makes the robot converge to the target point
                a_max = self.mi * self.g
                v_max = r.max_speed
                cte = (self.exp_k * a_max / (v_max * v_max))
                out = v_max * (1 - exp(-cte * r.distance(f_t)))
            # v is the speed vector resulting from that equation
            #print out
            if self.avoid_collisions:
                v = out * gradient / norm(gradient)
            else:
                # v = out * normalized error
                # must check if error is zero lengthed
                v = error
                ne = norm(error)
                if ne != 0:
                    v *= out / ne

            # increase by referential speed
            # not working, must think about it
            #if ref is not None:
            #    v += ref.speed

            # at last set the action accordingly
            r.action.absolute_speeds = v[0], v[1], va

    @property
    def point_away_from_defense_area(self):
        # FIXME: Only works if robot is inside defense area (which, honestly, is the only place you should ever be using this).
        # FIXME: I think I broke it, needs fixing, reviewing and commenting
        #defense_area = self.world.augmented_defense_area(self.robot, self.robot.color).boundary
        defense_area = self.world.augmented_defense_area(self.robot, self.robot.color)
        distance = self.target.distance(defense_area)
        #buffered_circumference = self.target.buffer(distance)
        buffered_circumference = self.target.buffer(self.robot.radius)
        intersection = buffered_circumference.intersection(defense_area).centroid
        l = intersection
        if not intersection.is_empty:
        #    if abs(self.target.x) < abs(self.world.goal(self.robot.color).x):
            #print 'case 1'
            error = array(intersection) - array(self.target)
            diff = distance * error / norm(error)
            #print "returning point from defense area"
            #return Point(array(self.target) + diff)
            return Point(0, 0)
        #    else:
        #        print 'case 2'
        #        return Point(self.target.x - 2 * sign(self.goal.x) * self.world.defense_radius, self.robot.y)
        #        #diff = -1 * distance * error / norm(error)
        #        #print "returning point from defense area"
        #        #return Point(array(self.target) + 2 * diff)
        else:
            return self.target
<commit_msg>Fixed goalkeeper staying in the goal middle<commit_after>#
# Copyright (C) 2013 RoboIME
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
from numpy import array, linspace
from numpy.linalg import norm

from .. import Skill
from ...utils.mathutils import exp
from ...utils.geom import Point, Line
from ...utils.pidcontroller import PidController

def force_equation(direction, distance, floor, factor, power):
    return direction * (floor + factor / distance) ** power

class Goto(Skill):
    """
    Sends the robot to a specified point with a specified orientation with no
    regard to the position of any other objects on the field.
    """

    # attraction coefificients
    attraction_factor = 6.0
    attraction_power = 2.3
    attraction_floor = 80.0

    # repulsion coefificients
    repulsion_factor = 8.0
    repulsion_power = 3.3
    repulsion_floor = 0.0

    # magnetic coefificients
    magnetic_factor = 9.0
    magnetic_power = 3.1
    magnetic_floor = 0.0

    # delta_speed coefificients
    delta_speed_factor = 2.0
    delta_speed_power = 1.4
    delta_speed_floor = 0.0

    # minimum distance to use on the equation
    # anything smaller is capped to this
    min_distance = 1e-5

    # distance to consider target arrival
    arrive_distance = 1e-3
    max_angle_error = 1.0

    # ignore other forces if attraction
    # force is as least this high:
    min_force_to_ignore_others = 100000

    # control params
    g = 9.80665
    mi = 0.1
    exp_k = 6

    #def __init__(self, robot, target=None, angle=None, final_target=None, referential=None, deterministic=True, avoid_collisions=True, **kwargs):
    def __init__(self, robot, target=None, angle=None, final_target=None, referential=None, ignore_defense_area=False, deterministic=True, avoid_collisions=False, use_norm_pid=False, separate_axis_control=False, **kwargs):
        """
        final_target: the ultimate position the robot should attain
        target: the intermediate position you're ACTUALLY sending the robot to
        angle: the orientation the robot should have at target

        final_target is there so that, on a tactic, you can make the robot follow a curve to final_target
        while not stopping near each target due to its extreme proximity to it by continuously adjusting
        target to be ever closer to final_target. These intermediate steps are not handled by this class,
        and the exact curve you want the robot to follow (and hence all the intermediate targets that you'll
        be switching in) should be specified in your own class.
        """
        super(Goto, self).__init__(robot, deterministic=deterministic, **kwargs)
        #TODO: find the right parameters
        self.angle_controller = PidController(kp=1., ki=.0, kd=.0, integ_max=.5, output_max=360)
        self.norm_controller = PidController(kp=.1, ki=0.01, kd=.5, integ_max=50., output_max=1.2)
        self.x_controller = PidController(kp=.3, ki=0.01, kd=.05, integ_max=5., output_max=.8)
        self.y_controller = PidController(kp=.3, ki=0.01, kd=.05, integ_max=5., output_max=.8)
        self.use_norm_pid = use_norm_pid
        self.separate_axis_control = separate_axis_control
        #self.angle_controller = PidController(kp=1.324, ki=0, kd=0, integ_max=6.55, output_max=1000)
        self._angle = angle
        self.target = target
        # self.final_target = final_target if final_target is not None else self.target
        self.final_target = final_target
        self.referential = referential
        self.avoid_collisions = avoid_collisions
        self.ignore_defense_area = ignore_defense_area
        self.avoid_collisions = False
        self.next_target = target

    def arrived(self):
        return norm(array(self.robot) - array(self.target)) <= self.arrive_distance

    def oriented(self):
        return (180 + self.angle - self.robot.angle) % 360 - 180 <= self.max_angle_error

    @property
    def angle(self):
        if callable(self._angle):
            return self._angle()
        else:
            return self._angle or self.robot.angle

    @angle.setter
    def angle(self, angle):
        self._angle = angle

    @property
    def target(self):
        if callable(self._target):
            return self._target()
        else:
            return self._target or self.robot

    @target.setter
    def target(self, target):
        self._target = target

    @property
    def final_target(self):
        if callable(self._final_target):
            return self._final_target()
        else:
            return self._final_target or self.target

    @final_target.setter
    def final_target(self, target):
        self._final_target = target

    def attraction_force(self):
        # the error vector from the robot to the target point
        delta = array(self.target) - array(self.robot)

        # attractive force
        dist = max(self.robot.distance(self.final_target), self.min_distance)

        # normalize delta
        delta /= dist

        #attraction_force = delta * (1 + 1 / (dist / self.attraction_factor) ** self.attraction_power)
        #attraction_force = delta * (self.attraction_floor + (self.attraction_factor / dist) ** self.attraction_power)
        attraction_force = force_equation(delta, dist, self.attraction_floor, self.attraction_factor, self.attraction_power)

        return attraction_force

    def other_forces(self):
        robot = self.robot
        for other in filter(lambda other: other is not robot, self.world.iterrobots()):
            # difference of position
            delta = array(robot) - array(other)

            # considered distance
            dist = norm(delta) + robot.radius + other.radius

            # cap the distance to a minimum of 1mm
            dist = max(dist, self.min_distance)

            # normalize the delta
            delta /= norm(delta)

            # perpendicular delta
            pdelta = array((delta[1], -delta[0]))

            # normalized difference of speeds of the robots
            sdelta = other.speed - robot.speed
            sdelta /= max(norm(sdelta), self.min_distance)

            # calculating each force
            #repulsion_force = delta * (self.repulsion_factor / dist) ** self.repulsion_power
            repulsion_force = force_equation(delta, dist, 0, self.repulsion_factor, self.repulsion_power)
            #magnetic_force = pdelta * (self.magnetic_factor / dist) ** self.magnetic_power
            magnetic_force = force_equation(pdelta, dist, 0, self.magnetic_factor, self.magnetic_power)
            #delta_speed_force = sdelta * (self.delta_speed_factor / dist) ** self.delta_speed_power
            delta_speed_force = force_equation(sdelta, dist, 0, self.delta_speed_factor, self.delta_speed_power)
            yield (repulsion_force, magnetic_force, delta_speed_force)

    def _step(self):
        l = Line(self.robot, self.target)
        self.next_target = self.target if l.length < 0.30 else Point(self.robot.x + 0.30 * (l.coords[1][0] - l.coords[0][0]) / l.length, self.robot.y + 0.30 * (l.coords[1][1] - l.coords[0][1]) / l.length)

        r = self.robot
        t = self.next_target
        f_t = self.final_target
        #ref = self.referential if self.referential is not None else f_t
        # TODO: Check if target is within the boundaries of the field (augmented of the robot's radius).

        # check whether the point we want to go to will make the robot be in the defense area
        # if so then we'll go to the nearest point that meets that constraint
        #print "ball is in area?", r.world.is_in_defense_area(body=self.world.ball.buffer(r.radius), color=r.color)
        if not self.ignore_defense_area and r.world.is_in_defense_area(body=t.buffer(r.radius), color=r.color):
            self.target = self.point_away_from_defense_area
            t = self.target

        # angle control using PID controller
        if self.angle is not None and self.robot.angle is not None:
            self.angle_controller.input = (180 + self.angle - self.robot.angle) % 360 - 180
            #self.angle_controller.input = (180 + 0 - self.robot.angle) % 360 - 180
            self.angle_controller.feedback = 0.0
            self.angle_controller.step()
            va = self.angle_controller.output
        else:
            va = 0.0

        # the error vector from the robot to the target point
        error = array(t) - array(r)

        # the gradient of the field
        gradient = self.attraction_force()
        if norm(gradient) < self.min_force_to_ignore_others:
            gradient += sum(map(sum, self.other_forces()))
        # WTF
        if self.separate_axis_control:
            delta = array(f_t) - array(r)
            ghost_point = norm(delta) * gradient / norm(gradient)
            print gradient
            print ghost_point
            self.x_controller.input = ghost_point[0]
            self.y_controller.input = ghost_point[1]
            self.x_controller.feedback = 0.
            self.y_controller.feedback = 0.
            self.x_controller.step()
            self.y_controller.step()
            r.action.absolute_speeds = self.x_controller.output, self.y_controller.output, va
            print r.action.absolute_speeds

        else:
            if self.use_norm_pid:
                distance = norm(array(f_t) - array(r))
                self.norm_controller.input = distance
                self.norm_controller.feedback = 0.
                self.norm_controller.step()
                out = self.norm_controller.output
            else:
                # some crazy equation that makes the robot converge to the target point
                a_max = self.mi * self.g
                v_max = r.max_speed
                cte = (self.exp_k * a_max / (v_max * v_max))
                out = v_max * (1 - exp(-cte * r.distance(f_t)))
            # v is the speed vector resulting from that equation
            #print out
            if self.avoid_collisions:
                v = out * gradient / norm(gradient)
            else:
                # v = out * normalized error
                # must check if error is zero lengthed
                v = error
                ne = norm(error)
                if ne != 0:
                    v *= out / ne

            # increase by referential speed
            # not working, must think about it
            #if ref is not None:
            #    v += ref.speed

            # at last set the action accordingly
            r.action.absolute_speeds = v[0], v[1], va

    @property
    def point_away_from_defense_area(self):
        # FIXME: Only works if robot is inside defense area (which, honestly, is the only place you should ever be using this).
        # FIXME: I think I broke it, needs fixing, reviewing and commenting
        #defense_area = self.world.augmented_defense_area(self.robot, self.robot.color).boundary
        defense_area = self.world.augmented_defense_area(self.robot, self.robot.color)
        distance = self.target.distance(defense_area)
        #buffered_circumference = self.target.buffer(distance)
        buffered_circumference = self.target.buffer(self.robot.radius)
        intersection = buffered_circumference.intersection(defense_area).centroid
        l = intersection
        if not intersection.is_empty:
        #    if abs(self.target.x) < abs(self.world.goal(self.robot.color).x):
            #print 'case 1'
            error = array(intersection) - array(self.target)
            diff = distance * error / norm(error)
            #print "returning point from defense area"
            #return Point(array(self.target) + diff)
            return Point(0, 0)
        #    else:
        #        print 'case 2'
        #        return Point(self.target.x - 2 * sign(self.goal.x) * self.world.defense_radius, self.robot.y)
        #        #diff = -1 * distance * error / norm(error)
        #        #print "returning point from defense area"
        #        #return Point(array(self.target) + 2 * diff)
        else:
            return self.target
<|endoftext|>


<commit_before>from IzVerifier.izspecs.izproperties import IzProperties

__author__ = 'fcanas'

import unittest

from IzVerifier.izspecs.containers.izconditions import IzConditions
from IzVerifier.izspecs.containers.izstrings import IzStrings
from IzVerifier.izspecs.containers.izvariables import IzVariables
from IzVerifier.izspecs.verifiers.seeker import Seeker
from IzVerifier.izverifier import IzVerifier
from IzVerifier.izspecs.containers.constants import *

path1 = 'data/sample_installer_iz5/izpack/'
path2 = 'data/sample_installer_iz5/resources/'
source_path2 = 'data/sample_code_base'
pom = 'data/sample_installer_iz5/pom.xml'

class TestSeeker(unittest.TestCase):
    """
    Basic testing of seeker class.
    """

    def setUp(self):
        args = {
            'specs_path': path1,
            'sources': [source_path2],
            'resources_path': path2,
            'pom': pom,
            'specs': ['conditions', 'strings', 'variables']
        }
        self.verifier = IzVerifier(args)
        self.seeker = Seeker(self.verifier.paths)
        langpack = self.verifier.paths.get_langpack_path()
        self.conditions = IzConditions(self.verifier.paths.get_path('conditions'))
        self.variables = IzVariables(self.verifier.paths.get_path('variables'))
        self.strings = IzStrings(langpack)

    def test_findStringReference(self):
        """
        Find specified string reference across some spec files.
        """
        props = {
            'path': self.verifier.paths.root,
            'id': 'some.user.panel.title',
            'specs': ['data/sample_installer_iz5/izpack/install.xml',
                      'data/sample_installer_iz5/resources/userInputSpec.xml'],
            'filter_fn': self.strings.has_reference,
            'attributes': self.strings.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.strings.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.strings.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findConditionReference(self):
        """
        Find specified condition reference across some spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'id': 'some.condition.1',
            'specs': map(self.verifier.paths.get_path, self.conditions.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.conditions.has_reference,
            'attributes': self.conditions.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.conditions.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.conditions.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findVariableReference(self):
        """
        Find specified variable reference across some spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'id': 'some.user.password',
            'specs': map(self.verifier.paths.get_path, self.variables.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.variables.has_reference,
            'attributes': self.variables.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.variables.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.strings.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findAllStringsInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.strings.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.strings.has_reference,
            'attributes': self.strings.properties[ATTRIBUTES],
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 7, msg=str(len(hits)) + '!=7')

    def test_findAllConditionsInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.conditions.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.conditions.has_reference,
            'attributes': self.conditions.properties[ATTRIBUTES],
            'white_list_patterns': []
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 4)

    def test_findAllVariablesInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.variables.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.variables.has_reference,
            'attributes': self.variables.properties[ATTRIBUTES],
            'white_list_patterns': []
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 2)

    def test_findAllStringsInSource(self):
        """
        Search source code for izpack string references.
        """
        hits = self.seeker.find_references_in_source(
            patterns=self.strings.properties[PATTERNS],
            path_list=[source_path2],
            white_list_patterns=self.strings.properties[WHITE_LIST_PATTERNS]
        )
        num = len(hits)
        self.assertTrue(num == 8)

    def test_processKeys(self):
        """
        Test the process_keys method.
        """
        # hits list: (key from source, location, expected output)
        hits = [('idata.getString("string.key.1"', 'path/to/file.java', 'string.key.1'),
                ('System.out.println(idata.getString("string.key.2"', 'path/to/file.java', 'string.key.2'),
                ('"string.key.3"', 'path/to/file.java', 'string.key.3'),
                ('somevar + "string.key.4"', 'path/to/file.java', None),
            ('key1', 'data/sample_code_base/src/com/sample/installer/Foo.java', 'some.string.1')
        ]

        seeker = Seeker(None)
        for hit in hits:
            key = seeker.process_key(hit[0], hit[1], ['string.in.whitelist'], 'idata.getString\(.*|System.*.println\(.*')
            print str(hit[2]) + " => " + str(key)
            self.assertEquals(hit[2], key)

<commit_msg>Updated test case to work with new output from process_key<commit_after>from IzVerifier.izspecs.izproperties import IzProperties

__author__ = 'fcanas'

import unittest

from IzVerifier.izspecs.containers.izconditions import IzConditions
from IzVerifier.izspecs.containers.izstrings import IzStrings
from IzVerifier.izspecs.containers.izvariables import IzVariables
from IzVerifier.izspecs.verifiers.seeker import Seeker
from IzVerifier.izverifier import IzVerifier
from IzVerifier.izspecs.containers.constants import *

path1 = 'data/sample_installer_iz5/izpack/'
path2 = 'data/sample_installer_iz5/resources/'
source_path2 = 'data/sample_code_base'
pom = 'data/sample_installer_iz5/pom.xml'

class TestSeeker(unittest.TestCase):
    """
    Basic testing of seeker class.
    """

    def setUp(self):
        args = {
            'specs_path': path1,
            'sources': [source_path2],
            'resources_path': path2,
            'pom': pom,
            'specs': ['conditions', 'strings', 'variables']
        }
        self.verifier = IzVerifier(args)
        self.seeker = Seeker(self.verifier.paths)
        langpack = self.verifier.paths.get_langpack_path()
        self.conditions = IzConditions(self.verifier.paths.get_path('conditions'))
        self.variables = IzVariables(self.verifier.paths.get_path('variables'))
        self.strings = IzStrings(langpack)

    def test_findStringReference(self):
        """
        Find specified string reference across some spec files.
        """
        props = {
            'path': self.verifier.paths.root,
            'id': 'some.user.panel.title',
            'specs': ['data/sample_installer_iz5/izpack/install.xml',
                      'data/sample_installer_iz5/resources/userInputSpec.xml'],
            'filter_fn': self.strings.has_reference,
            'attributes': self.strings.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.strings.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.strings.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findConditionReference(self):
        """
        Find specified condition reference across some spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'id': 'some.condition.1',
            'specs': map(self.verifier.paths.get_path, self.conditions.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.conditions.has_reference,
            'attributes': self.conditions.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.conditions.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.conditions.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findVariableReference(self):
        """
        Find specified variable reference across some spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'id': 'some.user.password',
            'specs': map(self.verifier.paths.get_path, self.variables.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.variables.has_reference,
            'attributes': self.variables.properties[ATTRIBUTES],
            'transformer': lambda x: x,
            'patterns': self.variables.properties[PATTERNS],
            'source_paths': [],
            'white_list_patterns': self.strings.properties[WHITE_LIST_PATTERNS]
        }
        results = self.seeker.find_id_references(props)
        hits = len(results)
        self.assertEquals(hits, 1, msg=str(hits) + '!=1')

    def test_findAllStringsInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.strings.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.strings.has_reference,
            'attributes': self.strings.properties[ATTRIBUTES],
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 7, msg=str(len(hits)) + '!=7')

    def test_findAllConditionsInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.conditions.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.conditions.has_reference,
            'attributes': self.conditions.properties[ATTRIBUTES],
            'white_list_patterns': []
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 4)

    def test_findAllVariablesInSpecs(self):
        """
        Finds all strings referenced in installer spec files.
        """

        props = {
            'path': self.verifier.paths.root,
            'specs': map(self.verifier.paths.get_path, self.variables.properties[REFERENCE_SPEC_FILES]),
            'filter_fn': self.variables.has_reference,
            'attributes': self.variables.properties[ATTRIBUTES],
            'white_list_patterns': []
        }

        hits = self.seeker.search_specs_for_attributes(props)
        self.assertTrue(len(hits) == 2)

    def test_findAllStringsInSource(self):
        """
        Search source code for izpack string references.
        """
        hits = self.seeker.find_references_in_source(
            patterns=self.strings.properties[PATTERNS],
            path_list=[source_path2],
            white_list_patterns=self.strings.properties[WHITE_LIST_PATTERNS]
        )
        num = len(hits)
        self.assertTrue(num == 8)

    def test_processKeys(self):
        """
        Test the process_keys method.
        """
        # hits list: (key from source, location, expected output)
        hits = [('idata.getString("string.key.1"', 'path/to/file.java', 'string.key.1'),
                ('System.out.println(idata.getString("string.key.2"', 'path/to/file.java', 'string.key.2'),
                ('"string.key.3"', 'path/to/file.java', 'string.key.3'),
                ('somevar + "string.key.4"', 'path/to/file.java', None),
                ('key1', 'data/sample_code_base/src/com/sample/installer/Foo.java', 'some.string.1')
        ]

        seeker = Seeker(None)
        for hit in hits:
            key = seeker.process_key(hit, ['string.in.whitelist'], 'idata.getString\(.*|System.*.println\(.*')
            if key is None:
                print str(hit[2]) + " => " + str(key)
                self.assertEquals(hit[2], key)
            else:
                print str(hit[2]) + " => " + str(key[0])
                self.assertEquals(hit[2], key[0])

<|endoftext|>