<commit_before>import pytest
from .common import *  # NOQA
from .test_rke_cluster_provisioning import HOST_NAME
from .test_rke_cluster_provisioning import create_and_validate_custom_host
from .test_rke_cluster_provisioning import rke_config

namespace = {"p_client": None, "ns": None, "cluster": None, "project": None,
             "nodes": []}
backup_info = {"backupname": None, "backup_id": None, "workload": None,
               "backupfilename": None, "etcdbackupdata": None}

@if_test_all_snapshot
def test_bkp_restore_s3_recover_validate():
    """
    - This test create 1 cluster with s3 backups enabled
        - 1 ControlPlane/Worker Node
        - 3 etcd nodes
    - Creates an Ingress pointing to a workload
    - Snapshots the cluster and checks the backup is in S3
    - Stops the etcd nodes in ec2
    - Waits for the cluster to go into unavailable state
    - Removes all 3 etcd nodes
    - Creates 3 new etcd nodes and waits until the cluster asks to restore from backup.
    - Restores from S3 backup
    - Cluster is validated after it gets in Active state.
    - Checks the Ingress created before the backup is functional after restore.
    - Removes cluster if RANCHER_CLEANUP_CLUSTER=True
    """

    cluster = namespace["cluster"]
    client = namespace["client"]
    ns, b_info = validate_backup_create(namespace, backup_info, "s3")
    ips_to_remove = []
    etcd_nodes = get_etcd_nodes(client, cluster)
    assert len(etcd_nodes) > 0, "Make sure we have etcd nodes in the cluster"

    # stop the etcd ec2 instances
    [stop_node_from_ec2(etcd_node.externalIpAddress)
     for etcd_node in etcd_nodes]
    # wait for cluster to get into unavailable state
    cluster = wait_for_cluster_unavailable(client, cluster)
    for etcd_node in etcd_nodes:
        ips_to_remove.append(etcd_node.customConfig['internalAddress'])
        client.delete(etcd_node)
    # Also remove the ec2 instances
    for ip_to_remove in ips_to_remove:
        delete_node_from_ec2(ip_to_remove)
        namespace["nodes"] = [node for node
                              in namespace["nodes"]
                              if node.private_ip_address != ip_to_remove]
    ips_to_remove.clear()
    cluster = client.reload(cluster)
    wait_for_cluster_node_count(client, cluster, 1)
    # Add completely new etcd nodes to the cluster
    cluster = add_new_etcd_nodes(client, cluster)
    cluster = client.reload(cluster)
    wait_for_cluster_node_count(client, cluster, 4)
    # This message is expected to appear after we add new etcd nodes
    # The cluster will require the user to perform a backup to recover
    # this is appears in the cluster object in cluster.transitioningMessage
    message = "Please restore your cluster from backup"
    cluster = wait_for_cluster_transitioning_message(client, cluster, message)
    etcd_nodes = get_etcd_nodes(client, cluster)
    assert len(etcd_nodes) == 3, "Make sure the cluster now has 3 etcd nodes"
    cluster = restore_backup_to_recover(client, cluster, b_info)
    # validate the ingress that was created in the first backup
    # after restoring and recovering the cluster
    validate_ingress(namespace["p_client"], cluster,
                     [b_info["workload"]], ns["host"],
                     "/name.html")

@pytest.fixture(scope='module', autouse=True)
def create_project_client_and_cluster_s3_three_etcd(request):
    node_roles = [
        ["controlplane", "worker"],
        ["etcd"], ["etcd"], ["etcd"],
    ]
    rke_config["services"]["etcd"]["backupConfig"] = {
        "enabled": "true",
        "intervalHours": 6,
        "retention": 3,
        "type": "backupConfig",
        "s3BackupConfig": {
            "type": "s3BackupConfig",
            "accessKey": AWS_ACCESS_KEY_ID,
            "secretKey": AWS_SECRET_ACCESS_KEY,
            "bucketName": AWS_S3_BUCKET_NAME,
            "folder": AWS_S3_BUCKET_FOLDER_NAME,
            "region": AWS_REGION,
            "endpoint": "s3.amazonaws.com"
        }
    }
    cluster, aws_nodes = create_and_validate_custom_host(node_roles)
    client = get_user_client()
    namespace["nodes"].extend(aws_nodes)

    p, ns = create_project_and_ns(USER_TOKEN, cluster, "testrecover")
    p_client = get_project_client_for_token(p, USER_TOKEN)
    c_client = get_cluster_client_for_token(cluster, USER_TOKEN)

    namespace["p_client"] = p_client
    namespace["ns"] = ns
    namespace["cluster"] = cluster
    namespace["project"] = p
    namespace["c_client"] = c_client
    namespace["client"] = client

    def fin():
        cluster_cleanup(client, cluster, namespace["nodes"])
    request.addfinalizer(fin)

def wait_for_cluster_unavailable(client, cluster):
    return wait_for_condition(client, cluster,
                              lambda x: x.state == "unavailable",
                              lambda x: 'State is: ' + x.state,
                              timeout=DEFAULT_CLUSTER_STATE_TIMEOUT)

def wait_for_cluster_transitioning_message(client, cluster, message):
    start = time.time()
    while message not in cluster.transitioningMessage:
        print(cluster.transitioningMessage)
        time.sleep(5)
        cluster = client.reload(cluster)
        # We are waiting 4 minutes for the transitioning message to appear
        # this could be impacted by environmental factors
        if time.time() - start > DEFAULT_CLUSTER_STATE_TIMEOUT:
            raise Exception('Timeout waiting for condition')
    return cluster

def add_new_etcd_nodes(client, cluster, no_of_nodes=3):
    aws_nodes = AmazonWebServices().create_multiple_nodes(
        no_of_nodes, random_test_name(HOST_NAME))
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, ["etcd"],
                                             aws_node)
        print("Docker run command: " + docker_run_cmd)
        aws_node.roles.append("etcd")

        result = aws_node.execute_command(docker_run_cmd)
        namespace["nodes"].append(aws_node)
        print(result)
    return cluster

def delete_node_from_ec2(internal_ip):

    filters = [
        {'Name': 'private-ip-address',
         'Values': [internal_ip]}
    ]
    aws_node = AmazonWebServices().get_nodes(filters)
    if len(aws_node) > 0:
        AmazonWebServices().delete_node(aws_node[0])

def stop_node_from_ec2(address):
    filters = [
        {'Name': 'ip-address',
         'Values': [address]}
    ]
    aws_node = AmazonWebServices().get_nodes(filters)
    if len(aws_node) > 0:
        AmazonWebServices().stop_node(aws_node[0])

def restore_backup_to_recover(client, cluster, b_info):
    cluster.restoreFromEtcdBackup(etcdBackupId=b_info["backup_id"])
    return validate_cluster(client, cluster, intermediate_state="updating",
                            check_intermediate_state=True,
                            skipIngresscheck=False)

def get_etcd_nodes(client, cluster):
    nodes = client.list_node(clusterId=cluster.id).data
    return [node for node in nodes if node.etcd is True]

def get_worker_nodes(client, cluster):
    nodes = client.list_node(clusterId=cluster.id).data
    return [node for node in nodes if node.worker is True]
<commit_msg>s3 backup recover test improvements<commit_after>import pytest
from .common import *  # NOQA
from .test_rke_cluster_provisioning import HOST_NAME
from .test_rke_cluster_provisioning import create_and_validate_custom_host
from .test_rke_cluster_provisioning import rke_config

namespace = {"p_client": None, "ns": None, "cluster": None, "project": None,
             "nodes": []}
backup_info = {"backupname": None, "backup_id": None, "workload": None,
               "backupfilename": None, "etcdbackupdata": None}

@if_test_all_snapshot
def test_bkp_restore_s3_recover_validate():
    """
    - This test create 1 cluster with s3 backups enabled
        - 1 ControlPlane/worker node
        - 2 worker nodes
        - 3 etcd nodes
    - Creates an Ingress pointing to a workload
    - Snapshots the cluster and checks the backup is in S3
    - Stops the etcd nodes in ec2
    - Waits for the cluster to go into unavailable state
    - Removes all 3 etcd nodes
    - Creates 3 new etcd nodes and waits until the cluster
      asks to restore from backup.
    - Restores from S3 backup
    - Cluster is validated after it gets in Active state.
    - Checks the Ingress created before the backup is functional after restore.
    - Removes cluster if RANCHER_CLEANUP_CLUSTER=True
    """

    cluster = namespace["cluster"]
    client = namespace["client"]
    ns, b_info = validate_backup_create(namespace, backup_info, "s3")
    ips_to_remove = []
    etcd_nodes = get_etcd_nodes(client, cluster)
    assert len(etcd_nodes) > 0, "Make sure we have etcd nodes in the cluster"

    # stop the etcd ec2 instances
    [stop_node_from_ec2(etcd_node.externalIpAddress)
     for etcd_node in etcd_nodes]
    # wait for cluster to get into unavailable state
    cluster = wait_for_cluster_unavailable(client, cluster)
    for etcd_node in etcd_nodes:
        ips_to_remove.append(etcd_node.customConfig['internalAddress'])
        client.delete(etcd_node)
    # Also remove the ec2 instances
    for ip_to_remove in ips_to_remove:
        delete_node_from_ec2(ip_to_remove)
        namespace["nodes"] = [node for node
                              in namespace["nodes"]
                              if node.private_ip_address != ip_to_remove]
    ips_to_remove.clear()
    cluster = client.reload(cluster)
    wait_for_cluster_node_count(client, cluster, 3)
    # Add completely new etcd nodes to the cluster
    cluster = add_new_etcd_nodes(client, cluster)
    cluster = client.reload(cluster)
    wait_for_cluster_node_count(client, cluster, 6)
    # This message is expected to appear after we add new etcd nodes
    # The cluster will require the user to perform a backup to recover
    # this is appears in the cluster object in cluster.transitioningMessage
    message = "Please restore your cluster from backup"
    cluster = wait_for_cluster_transitioning_message(client, cluster, message)
    etcd_nodes = get_etcd_nodes(client, cluster)
    assert len(etcd_nodes) == 3, "Make sure the cluster now has 3 etcd nodes"
    cluster = restore_backup_to_recover(client, cluster, b_info)
    # validate the ingress that was created in the first backup
    # after restoring and recovering the cluster
    validate_ingress(namespace["p_client"], cluster,
                     [b_info["workload"]], ns["host"],
                     "/name.html")

@pytest.fixture(scope='module', autouse=True)
def create_project_client_and_cluster_s3_three_etcd(request):
    node_roles = [
        ["controlplane", "worker"],
        ["etcd"], ["etcd"], ["etcd"],
        ["worker"], ["worker"]
    ]
    rke_config["services"]["etcd"]["backupConfig"] = {
        "enabled": "true",
        "intervalHours": 6,
        "retention": 3,
        "type": "backupConfig",
        "s3BackupConfig": {
            "type": "s3BackupConfig",
            "accessKey": AWS_ACCESS_KEY_ID,
            "secretKey": AWS_SECRET_ACCESS_KEY,
            "bucketName": AWS_S3_BUCKET_NAME,
            "folder": AWS_S3_BUCKET_FOLDER_NAME,
            "region": AWS_REGION,
            "endpoint": "s3.amazonaws.com"
        }
    }
    cluster, aws_nodes = create_and_validate_custom_host(
        node_roles,
        random_cluster_name=True
    )
    client = get_user_client()
    namespace["nodes"].extend(aws_nodes)

    p, ns = create_project_and_ns(USER_TOKEN, cluster, "testrecover")
    p_client = get_project_client_for_token(p, USER_TOKEN)
    c_client = get_cluster_client_for_token(cluster, USER_TOKEN)

    namespace["p_client"] = p_client
    namespace["ns"] = ns
    namespace["cluster"] = cluster
    namespace["project"] = p
    namespace["c_client"] = c_client
    namespace["client"] = client

    def fin():
        cluster_cleanup(client, cluster, namespace["nodes"])
    request.addfinalizer(fin)

def wait_for_cluster_unavailable(client, cluster):
    return wait_for_condition(client, cluster,
                              lambda x: x.state == "unavailable",
                              lambda x: 'State is: ' + x.state,
                              timeout=DEFAULT_CLUSTER_STATE_TIMEOUT)

def wait_for_cluster_transitioning_message(client, cluster, message):
    start = time.time()
    while message not in cluster.transitioningMessage:
        print(cluster.transitioningMessage)
        time.sleep(5)
        cluster = client.reload(cluster)
        # We are waiting 4 minutes for the transitioning message to appear
        # this could be impacted by environmental factors
        if time.time() - start > DEFAULT_CLUSTER_STATE_TIMEOUT:
            raise Exception('Timeout waiting for condition')
    return cluster

def add_new_etcd_nodes(client, cluster, no_of_nodes=3):
    aws_nodes = AmazonWebServices().create_multiple_nodes(
        no_of_nodes, random_test_name(HOST_NAME))
    for aws_node in aws_nodes:
        docker_run_cmd = \
            get_custom_host_registration_cmd(client, cluster, ["etcd"],
                                             aws_node)
        print("Docker run command: " + docker_run_cmd)
        aws_node.roles.append("etcd")

        result = aws_node.execute_command(docker_run_cmd)
        namespace["nodes"].append(aws_node)
        print(result)
    return cluster

def delete_node_from_ec2(internal_ip):

    filters = [
        {'Name': 'private-ip-address',
         'Values': [internal_ip]}
    ]
    aws_node = AmazonWebServices().get_nodes(filters)
    if len(aws_node) > 0:
        AmazonWebServices().delete_node(aws_node[0])

def stop_node_from_ec2(address):
    filters = [
        {'Name': 'ip-address',
         'Values': [address]}
    ]
    aws_node = AmazonWebServices().get_nodes(filters)
    if len(aws_node) > 0:
        AmazonWebServices().stop_node(aws_node[0])

def restore_backup_to_recover(client, cluster, b_info):
    cluster.restoreFromEtcdBackup(etcdBackupId=b_info["backup_id"])
    return validate_cluster(client, cluster, intermediate_state="updating",
                            check_intermediate_state=True,
                            skipIngresscheck=False)

def get_etcd_nodes(client, cluster):
    nodes = client.list_node(clusterId=cluster.id).data
    return [node for node in nodes if node.etcd is True]

def get_worker_nodes(client, cluster):
    nodes = client.list_node(clusterId=cluster.id).data
    return [node for node in nodes if node.worker is True]
<|endoftext|>


<commit_before>import re
from itertools import izip_longest

import arcpy

from ..exceptions import MapLayerError, DataSourceUpdateError, UnsupportedLayerError, ChangeDataSourcesError

def change_data_sources(map, data_sources):
    """ """
    errors = []
    data_tables = arcpy.mapping.ListTableViews(map)
    layers_by_df = [arcpy.mapping.ListLayers(df) for df in arcpy.mapping.ListDataFrames(map)]

    if not 'layers' in data_sources or not 'tableViews' in data_sources:
        raise ChangeDataSourcesError("Data sources dictionary does not contain both 'layers' and 'tableViews' keys")

    for layers, layer_sources in izip_longest(layers_by_df, data_sources["layers"]):

        if layer_sources == None or len(layers) != len(layer_sources):
            raise ChangeDataSourcesError("Number of layers does not match number of data sources.")

        for layer, layer_source in izip_longest(layers, layer_sources):
            try:
                if layer_source == None:
                    continue

                if not layer.supports("dataSource") or not layer.supports("workspacePath"):
                    #error on layers that we can't change
                    raise UnsupportedLayerError(layer = layer)

                if layer.supports("dataSource"):
                    print(u"Layer '{0}': Current datasource: '{1}'".format(layer.longName, layer.dataSource).encode("ascii", "ignore"))

                print(u"Layer '{0}': Attempting to change workspace path".format(layer.longName).encode("ascii", "ignore"))
                _change_data_source(layer, layer_source["workspacePath"], layer_source.get("datasetName"), layer_source.get("workspaceType"), layer_source.get("schema"))
                print(u"Layer '{0}': Workspace path updated to: '{1}'".format(layer.name, layer_source["workspacePath"]).encode("ascii", "ignore"))

                if layer.supports("dataSource"):
                    print(u"Layer '{0}': New datasource: '{1}'".format(layer.longName, layer.dataSource).encode("ascii", "ignore"))

            except MapLayerError as mle:
                errors.append(mle)

    if not len(data_tables) == len(data_sources['tableViews']):
        raise ChangeDataSourcesError("Number of data tables does not match number of data table data sources.")

    for data_table, layer_source in izip_longest(data_tables, data_sources['tableViews']):
        try:
            if layer_source == None:
                continue

            print(u"Data Table '{0}': Attempting to change workspace path".format(data_table.name).encode("ascii", "ignore"))
            _change_data_source(data_table, layer_source["workspacePath"], layer_source.get("datasetName"), layer_source.get("workspaceType"), layer_source.get("schema"))
            print(u"Data Table '{0}': Workspace path updated to: '{1}'".format(data_table.name, layer_source["workspacePath"]).encode("ascii", "ignore"))

        except MapLayerError as mle:
            errors.append(mle)

    if not len(errors) == 0:
        raise ChangeDataSourcesError("A number of errors were encountered whilst change layer data sources.", errors)

def create_replacement_data_sources_list(document_data_sources_list, data_source_templates, raise_exception_no_change = False):
    template_sets = [dict(template.items() + [("matchCriteria", set(template["matchCriteria"].items()))]) for template in data_source_templates]

    def match_new_data_source(item):
        if item == None:
            return None

        new_conn = None
        for template in template_sets:
            if template["matchCriteria"].issubset(set(item.items())):
                new_conn = template["dataSource"]
                break
        if new_conn == None and raise_exception_no_change:
            raise RuntimeError("No matching data source was found for layer")
        return new_conn

    return {
        "layers": [[match_new_data_source(layer) for layer in df] for df in document_data_sources_list["layers"]],
        "tableViews": [match_new_data_source(table) for table in document_data_sources_list["tableViews"]]
    }

def list_document_data_sources(map):
    """List the data sources for each layer or table view of the specified map.

    Outputs a dictionary containing two keys, "layers" and "tableViews".

    "layers" contains an array, with each element representing a data frame (as another array) that contains a
    dictionary of layer details relevant to that layer's connection to its data source.

    "tableViews" is also an array, where each element is a dictionary of table view details relevant to that table
    view's connection to its data source.

    The order of array elements is as displayed in the ArcMap table of contents.

    An example of the output format is the following::

        {
            "layers": [
                [
                    # Data frame number one
                    {
                        # Layer number one
                        "name":          "Layer Name",
                        "longName":      "Layer Group/Layer Name",
                        "datasetName":   "(Optional) dataset name",
                        "dataSource":    "(Optional) data source name",
                        "serviceType":   "(Optional) service type, e.g. SDE, MapServer, IMS",
                        "userName":      "(Optional) user name",
                        "server":        "(Optional) server address/hostname",
                        "service":       "(Optional) name or number of the ArcSDE Service",
                        "database":      "(Optional) name of the database",
                        "workspacePath": "(Optional) workspace path"
                    },
                    # ...more layers
                ],
                # ...more data frames
            ],
            "tableViews": [
                {
                    "datasetName": "dataset name",
                    "dataSource": "data source",
                    "definitionQuery": "definition query on the table",
                    "workspacePath": "workspace path"
                }
            ]
        }

    :param map: The map to gather data source connection details about
    :type map: arcpy.mapping.MapDocument
    :returns: dict

    """

    return {
        "layers": [[_get_layer_details(layer) for layer in arcpy.mapping.ListLayers(df)] for df in arcpy.mapping.ListDataFrames(map)],
        "tableViews": [_get_table_details(table) for table in arcpy.mapping.ListTableViews(map)]
    }

def validate_map(map):
    """Analyse the map for broken layers and return a boolean indicating if it is in a valid state or not.

    Lists broken layers on the shell output.

    :param map: The map to be validated
    :type map: arcpy.mapping.MapDocument
    :returns: Boolean, True if valid, False if there are one or more broken layers

    """

    broken_layers = arcpy.mapping.ListBrokenDataSources(map)

    if len(broken_layers) > 0:
        print(u"Map '{0}': Broken data sources:".format(map.title))
        for layer in broken_layers:
            print(u" {0}".format(layer.name))
            if not hasattr(layer, "supports"):
                #probably a TableView
                print(u"  workspace: {0}".format(layer.workspacePath))
                print(u"  datasource: {0}".format(layer.dataSource))
                continue

            #Some sort of layer
            if layer.supports("workspacePath"):
                print(u"  workspace: {0}".format(layer.workspacePath))
            if layer.supports("dataSource"):
                print(u"  datasource: {0}".format(layer.dataSource))

        return False

    return True

def _change_data_source(layer, workspace_path, dataset_name = None, workspace_type = None, schema = None):
    try:
        if ((not hasattr(layer, "supports") or layer.supports("workspacePath")) and
            (dataset_name == None and workspace_type == None and schema == None)):

            # Tests if layer is actually a layer object (i.e. has a "support" function) or table view (which doesn't,
            # but always supports "workspacePath").  Can't test on type (arcpy.mapping.TableView) as that doesn't work
            # on ArcPy 10.0

            layer.findAndReplaceWorkspacePath("", workspace_path, validate = False)
            return

        kwargs = { "validate": False }

        if dataset_name == None and hasattr(layer, "supports") and layer.supports("datasetName"):
            if layer.supports("workspacePath"):
                dataset_name = layer.dataSource.replace(layer.workspacePath, "")
            else:
                dataset_name = layer.datasetName

        if dataset_name != None:
            if (schema != None):
                ds_user, ds_name, fc_user, fc_name = _parse_data_source(dataset_name)

                dataset_name = "{0}.{1}".format(schema, fc_name)

            kwargs["dataset_name"] = dataset_name

        if workspace_type != None:
            kwargs["workspace_type"] = workspace_type

        layer.replaceDataSource(workspace_path, **kwargs)

    except Exception, e:
        raise DataSourceUpdateError("Exception raised internally by ArcPy", layer, e)

    if hasattr(layer, "isBroken") and layer.isBroken:
        raise DataSourceUpdateError("Layer is now broken.", layer)

def _get_layer_details(layer):
    if layer.isGroupLayer and not layer.isNetworkAnalystLayer:
        return None

    details = {
        "name": layer.name,
        "longName": layer.longName
    }

    if layer.supports("datasetName"):
        details["datasetName"] = layer.datasetName

    if layer.supports("dataSource"):
        details["dataSource"] = layer.dataSource

    if layer.supports("serviceProperties"):
        details["serviceType"] = layer.serviceProperties["ServiceType"]
        details["userName"] = layer.serviceProperties["UserName"]

        if layer.serviceProperties["ServiceType"].upper() == "SDE":
            details["server"] = layer.serviceProperties["Server"]
            details["service"] = layer.serviceProperties["Service"]
            details["database"] = layer.serviceProperties["Database"]

    if layer.supports("workspacePath"):
        details["workspacePath"] = layer.workspacePath

    return details

def _get_table_details(table):
    return {
        "datasetName": table.datasetName,
        "dataSource": table.dataSource,
        "definitionQuery": table.definitionQuery,
        "workspacePath": table.workspacePath
    }

def _parse_data_source(data_source):
    """Takes a string describing a data source and returns a four-part tuple describing the dataset username, dataset
    name, feature class username and feature class name"""

    dataset_regex = re.compile(
                        r"^(?:\\)?(?P<ds_user>[\w]*?)(?:\.)?(?P<ds_name>[\w]*?(?=\\))(?:\\)?(?P<fc_user>[\w]*?(?=\.))(?:\.)(?P<fc_name>[\w]*?)$",
                        re.IGNORECASE)

    r = dataset_regex.search(data_source)

    if r == None:
        feature_class_regex = re.compile(
                                r"^(?:\\)?(?P<fc_user>[\w]*?(?=\.))(?:\.)(?P<fc_name>[\w]*?)$",
                                re.IGNORECASE)
        r = feature_class_regex.search(data_source)

    if r == None:
        return (None, None, None, data_source)

    r = r.groupdict()

    return (r.get("ds_user"), r.get("ds_name"), r.get("fc_user"), r.get("fc_name"))
<commit_msg>Switched print statements to logger<commit_after>import logging
import re
from itertools import izip_longest

import arcpy

from ..exceptions import MapLayerError, DataSourceUpdateError, UnsupportedLayerError, ChangeDataSourcesError

# Configure module logging
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

def change_data_sources(map, data_sources):
    """ """
    errors = []
    data_tables = arcpy.mapping.ListTableViews(map)
    layers_by_df = [arcpy.mapping.ListLayers(df) for df in arcpy.mapping.ListDataFrames(map)]

    if not 'layers' in data_sources or not 'tableViews' in data_sources:
        raise ChangeDataSourcesError("Data sources dictionary does not contain both 'layers' and 'tableViews' keys")

    for layers, layer_sources in izip_longest(layers_by_df, data_sources["layers"]):

        if layer_sources == None or len(layers) != len(layer_sources):
            raise ChangeDataSourcesError("Number of layers does not match number of data sources.")

        for layer, layer_source in izip_longest(layers, layer_sources):
            try:
                if layer_source == None:
                    continue

                if not layer.supports("dataSource") or not layer.supports("workspacePath"):
                    #error on layers that we can't change
                    raise UnsupportedLayerError(layer = layer)

                if layer.supports("dataSource"):
                    logger.debug(u"Layer '{0}': Current datasource: '{1}'".format(layer.longName, layer.dataSource).encode("ascii", "ignore"))

                logger.debug(u"Layer '{0}': Attempting to change workspace path".format(layer.longName).encode("ascii", "ignore"))
                _change_data_source(layer, layer_source["workspacePath"], layer_source.get("datasetName"), layer_source.get("workspaceType"), layer_source.get("schema"))
                logger.debug(u"Layer '{0}': Workspace path updated to: '{1}'".format(layer.name, layer_source["workspacePath"]).encode("ascii", "ignore"))

                if layer.supports("dataSource"):
                    logger.debug(u"Layer '{0}': New datasource: '{1}'".format(layer.longName, layer.dataSource).encode("ascii", "ignore"))

            except MapLayerError as mle:
                errors.append(mle)

    if not len(data_tables) == len(data_sources['tableViews']):
        raise ChangeDataSourcesError("Number of data tables does not match number of data table data sources.")

    for data_table, layer_source in izip_longest(data_tables, data_sources['tableViews']):
        try:
            if layer_source == None:
                continue

            logger.debug(u"Data Table '{0}': Attempting to change workspace path".format(data_table.name).encode("ascii", "ignore"))
            _change_data_source(data_table, layer_source["workspacePath"], layer_source.get("datasetName"), layer_source.get("workspaceType"), layer_source.get("schema"))
            logger.debug(u"Data Table '{0}': Workspace path updated to: '{1}'".format(data_table.name, layer_source["workspacePath"]).encode("ascii", "ignore"))

        except MapLayerError as mle:
            errors.append(mle)

    if not len(errors) == 0:
        raise ChangeDataSourcesError("A number of errors were encountered whilst change layer data sources.", errors)

def create_replacement_data_sources_list(document_data_sources_list, data_source_templates, raise_exception_no_change = False):
    template_sets = [dict(template.items() + [("matchCriteria", set(template["matchCriteria"].items()))]) for template in data_source_templates]

    def match_new_data_source(item):
        if item == None:
            return None

        new_conn = None
        for template in template_sets:
            if template["matchCriteria"].issubset(set(item.items())):
                new_conn = template["dataSource"]
                break
        if new_conn == None and raise_exception_no_change:
            raise RuntimeError("No matching data source was found for layer")
        return new_conn

    return {
        "layers": [[match_new_data_source(layer) for layer in df] for df in document_data_sources_list["layers"]],
        "tableViews": [match_new_data_source(table) for table in document_data_sources_list["tableViews"]]
    }

def list_document_data_sources(map):
    """List the data sources for each layer or table view of the specified map.

    Outputs a dictionary containing two keys, "layers" and "tableViews".

    "layers" contains an array, with each element representing a data frame (as another array) that contains a
    dictionary of layer details relevant to that layer's connection to its data source.

    "tableViews" is also an array, where each element is a dictionary of table view details relevant to that table
    view's connection to its data source.

    The order of array elements is as displayed in the ArcMap table of contents.

    An example of the output format is the following::

        {
            "layers": [
                [
                    # Data frame number one
                    {
                        # Layer number one
                        "name":          "Layer Name",
                        "longName":      "Layer Group/Layer Name",
                        "datasetName":   "(Optional) dataset name",
                        "dataSource":    "(Optional) data source name",
                        "serviceType":   "(Optional) service type, e.g. SDE, MapServer, IMS",
                        "userName":      "(Optional) user name",
                        "server":        "(Optional) server address/hostname",
                        "service":       "(Optional) name or number of the ArcSDE Service",
                        "database":      "(Optional) name of the database",
                        "workspacePath": "(Optional) workspace path"
                    },
                    # ...more layers
                ],
                # ...more data frames
            ],
            "tableViews": [
                {
                    "datasetName": "dataset name",
                    "dataSource": "data source",
                    "definitionQuery": "definition query on the table",
                    "workspacePath": "workspace path"
                }
            ]
        }

    :param map: The map to gather data source connection details about
    :type map: arcpy.mapping.MapDocument
    :returns: dict

    """

    return {
        "layers": [[_get_layer_details(layer) for layer in arcpy.mapping.ListLayers(df)] for df in arcpy.mapping.ListDataFrames(map)],
        "tableViews": [_get_table_details(table) for table in arcpy.mapping.ListTableViews(map)]
    }

def validate_map(map):
    """Analyse the map for broken layers and return a boolean indicating if it is in a valid state or not.

    Lists broken layers on the shell output.

    :param map: The map to be validated
    :type map: arcpy.mapping.MapDocument
    :returns: Boolean, True if valid, False if there are one or more broken layers

    """

    broken_layers = arcpy.mapping.ListBrokenDataSources(map)

    if len(broken_layers) > 0:
        logger.debug(u"Map '{0}': Broken data sources:".format(map.title))
        for layer in broken_layers:
            logger.debug(u" {0}".format(layer.name))
            if not hasattr(layer, "supports"):
                #probably a TableView
                logger.debug(u"  workspace: {0}".format(layer.workspacePath))
                logger.debug(u"  datasource: {0}".format(layer.dataSource))
                continue

            #Some sort of layer
            if layer.supports("workspacePath"):
                logger.debug(u"  workspace: {0}".format(layer.workspacePath))
            if layer.supports("dataSource"):
                logger.debug(u"  datasource: {0}".format(layer.dataSource))

        return False

    return True

def _change_data_source(layer, workspace_path, dataset_name = None, workspace_type = None, schema = None):
    try:
        if ((not hasattr(layer, "supports") or layer.supports("workspacePath")) and
            (dataset_name == None and workspace_type == None and schema == None)):

            # Tests if layer is actually a layer object (i.e. has a "support" function) or table view (which doesn't,
            # but always supports "workspacePath").  Can't test on type (arcpy.mapping.TableView) as that doesn't work
            # on ArcPy 10.0

            layer.findAndReplaceWorkspacePath("", workspace_path, validate = False)
            return

        kwargs = { "validate": False }

        if dataset_name == None and hasattr(layer, "supports") and layer.supports("datasetName"):
            if layer.supports("workspacePath"):
                dataset_name = layer.dataSource.replace(layer.workspacePath, "")
            else:
                dataset_name = layer.datasetName

        if dataset_name != None:
            if (schema != None):
                ds_user, ds_name, fc_user, fc_name = _parse_data_source(dataset_name)

                dataset_name = "{0}.{1}".format(schema, fc_name)

            kwargs["dataset_name"] = dataset_name

        if workspace_type != None:
            kwargs["workspace_type"] = workspace_type

        layer.replaceDataSource(workspace_path, **kwargs)

    except Exception, e:
        raise DataSourceUpdateError("Exception raised internally by ArcPy", layer, e)

    if hasattr(layer, "isBroken") and layer.isBroken:
        raise DataSourceUpdateError("Layer is now broken.", layer)

def _get_layer_details(layer):
    if layer.isGroupLayer and not layer.isNetworkAnalystLayer:
        return None

    details = {
        "name": layer.name,
        "longName": layer.longName
    }

    if layer.supports("datasetName"):
        details["datasetName"] = layer.datasetName

    if layer.supports("dataSource"):
        details["dataSource"] = layer.dataSource

    if layer.supports("serviceProperties"):
        details["serviceType"] = layer.serviceProperties["ServiceType"]
        details["userName"] = layer.serviceProperties["UserName"]

        if layer.serviceProperties["ServiceType"].upper() == "SDE":
            details["server"] = layer.serviceProperties["Server"]
            details["service"] = layer.serviceProperties["Service"]
            details["database"] = layer.serviceProperties["Database"]

    if layer.supports("workspacePath"):
        details["workspacePath"] = layer.workspacePath

    return details

def _get_table_details(table):
    return {
        "datasetName": table.datasetName,
        "dataSource": table.dataSource,
        "definitionQuery": table.definitionQuery,
        "workspacePath": table.workspacePath
    }

def _parse_data_source(data_source):
    """Takes a string describing a data source and returns a four-part tuple describing the dataset username, dataset
    name, feature class username and feature class name"""

    dataset_regex = re.compile(
                        r"^(?:\\)?(?P<ds_user>[\w]*?)(?:\.)?(?P<ds_name>[\w]*?(?=\\))(?:\\)?(?P<fc_user>[\w]*?(?=\.))(?:\.)(?P<fc_name>[\w]*?)$",
                        re.IGNORECASE)

    r = dataset_regex.search(data_source)

    if r == None:
        feature_class_regex = re.compile(
                                r"^(?:\\)?(?P<fc_user>[\w]*?(?=\.))(?:\.)(?P<fc_name>[\w]*?)$",
                                re.IGNORECASE)
        r = feature_class_regex.search(data_source)

    if r == None:
        return (None, None, None, data_source)

    r = r.groupdict()

    return (r.get("ds_user"), r.get("ds_name"), r.get("fc_user"), r.get("fc_name"))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
import time
import numpy as np

import pycopia.OS.Linux.Input as input

from geometry_msgs.msg import PointStamped
from art_msgs.msg import Touch

class Slot:

    def __init__(self, slot_id=None, track_id=None):
        if slot_id is None:
            self.slot_id = -1
        else:
            self.slot_id = slot_id

        if track_id is None:
            self.track_id = -1
        else:
            self.track_id = track_id

        self.x = 0
        self.y = 0

    def __eq__(self, other):
        return self.slot_id == other.slot_id

class ArtTouchDriver:

    def __init__(self):
        self.x = 0
        self.y = 0
        self.touch = False
        self.touch_id = -1
        self.device = input.EventDevice("/dev/input/event17")
        self.touch_pub = rospy.Publisher("/art/interface/touchtable/touch", Touch, queue_size=1)
        self.slots = []
        self.slot = None
        self.to_delete_id = -1

    def get_slot_by_id(self, slot_id):
        for slot in self.slots:
            if slot.slot_id == slot_id:
                return slot
        return None

    def process(self):
        # print 1 if self.device._eventq else 0
        event = self.device.read()
        print event
        while True:
            if event.evtype == 3 and event.code == 47 and event.value >= 0:
                # MT_SLOT
                self.slot = self.get_slot_by_id(event.value)
                if self.slot is None:
                    self.slot = Slot(slot_id=event.value)
                    self.slots.append(self.slot)

            elif event.evtype == 3 and event.code == 57 and event.value >= 0:
                # MT_TRACK_ID start
                if self.slot is None:
                    self.slot = Slot(track_id=event.value, slot_id=0)
                    self.slots.append(self.slot)

            elif event.evtype == 3 and event.code == 57 and event.value < 0:
                # MT_TRACK_ID end
                if self.slot is not None:
                    self.to_delete_id = self.slot.track_id
                    self.slots.remove(self.slot)
                    self.slot = None

            elif event.evtype == 3 and event.code == 53:
                # x position
                self.slot.x = event.value
                print("x")

            elif event.evtype == 3 and event.code == 54:
                # y position
                self.slot.y = event.value
                print("y")

            elif event.evtype == 0:

                touch = Touch()
                if self.slot is None:
                    touch.id = self.to_delete_id
                    touch.touch = False
                else:
                    touch.touch = True
                    touch.id = self.slot.track_id
                    touch.point = PointStamped()
                    touch.point.point.x = self.slot.x
                    touch.point.point.y = self.slot.y

                self.touch_pub.publish(touch)

            else:
                pass
            # print event

            if not self.device._eventq:
                break
            event = self.device.read()

if __name__ == '__main__':
    rospy.init_node('art_touch_driver')

    rospy.loginfo('Waiting for other nodes to come up...')

    rospy.loginfo('Ready!')

    try:
        node = ArtTouchDriver()
        rate = rospy.Rate(1000)

        while not rospy.is_shutdown():
            node.process()
            rate.sleep()
    except rospy.ROSInterruptException:
        pass
<commit_msg>art_touch_driver: multitouch working<commit_after>#!/usr/bin/env python

import rospy
import time
import numpy as np

import pycopia.OS.Linux.Input as input

from geometry_msgs.msg import PointStamped
from art_msgs.msg import Touch

class Slot:

    def __init__(self, slot_id=None, track_id=None):
        if slot_id is None:
            self.slot_id = -1
        else:
            self.slot_id = slot_id

        if track_id is None:
            self.track_id = -1
        else:
            self.track_id = track_id

        self.x = 0
        self.y = 0

    def __eq__(self, other):
        return self.slot_id == other.slot_id

class ArtTouchDriver:

    def __init__(self):
        self.x = 0
        self.y = 0
        self.touch = False
        self.touch_id = -1
        self.device = input.EventDevice("/dev/input/event17")
        self.touch_pub = rospy.Publisher("/art/interface/touchtable/touch", Touch, queue_size=1)
        self.slots = []
        self.slot = None
        self.to_delete_id = -1

    def get_slot_by_id(self, slot_id):
        for slot in self.slots:
            if slot.slot_id == slot_id:
                return slot
        return None

    def process(self):
        # print 1 if self.device._eventq else 0
        event = self.device.read()
        print event
        while True:
            if event.evtype == 3 and event.code == 47 and event.value >= 0:
                # MT_SLOT
                self.slot = self.get_slot_by_id(event.value)
                if self.slot is None:
                    self.slot = Slot(slot_id=event.value)
                    self.slots.append(self.slot)

            elif event.evtype == 3 and event.code == 57 and event.value >= 0:
                # MT_TRACK_ID start
                if self.slot is None:
                    self.slot = Slot(track_id=event.value, slot_id=0)
                    self.slots.append(self.slot)
                else:
                    self.slot.track_id = event.value

            elif event.evtype == 3 and event.code == 57 and event.value < 0:
                # MT_TRACK_ID end
                if self.slot is not None:
                    self.to_delete_id = self.slot.track_id
                    self.slots.remove(self.slot)
                    self.slot = None

            elif event.evtype == 3 and event.code == 53:
                # x position
                self.slot.x = event.value
                print("x")

            elif event.evtype == 3 and event.code == 54:
                # y position
                self.slot.y = event.value
                print("y")

            elif event.evtype == 0:

                touch = Touch()
                if self.slot is None:
                    touch.id = self.to_delete_id
                    touch.touch = False
                else:
                    touch.touch = True
                    touch.id = self.slot.track_id
                    touch.point = PointStamped()
                    touch.point.point.x = self.slot.x
                    touch.point.point.y = self.slot.y

                self.touch_pub.publish(touch)

            else:
                pass
            # print event

            if not self.device._eventq:
                break
            event = self.device.read()

if __name__ == '__main__':
    rospy.init_node('art_touch_driver')

    rospy.loginfo('Waiting for other nodes to come up...')

    rospy.loginfo('Ready!')

    try:
        node = ArtTouchDriver()
        rate = rospy.Rate(1000)

        while not rospy.is_shutdown():
            node.process()
            rate.sleep()
    except rospy.ROSInterruptException:
        pass
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A `traverse` visitor for processing documentation."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import six

from tensorflow_docs.api_generator import tf_inspect

class DocGeneratorVisitor(object):
  """A visitor that generates docs for a python object when __call__ed."""

  def __init__(self):
    """Make a visitor.

    This visitor expects to be called on each node in the api. It is passed the
    path to an object, the object, and the filtered list of the object's
    children. (see the `__call__` method for details.

    This object accumulates the various data-structures necessary to build the
    docs, including (see the property definitions for details.):

    In the decsription below "master name" is the object's preferred fully
    qualified name.

    Params:
      index: A mapping from master names to python python objects.
      tree: A mapping from master names to a list if attribute names.
      reverse_index: Mapping from python object ids to master names.
        Note that this doesn't work for python numbers, strings or tuples.
      duplicate_of: A mapping from a fully qualified names to the object's
        master name. The master names are not included as keys.
      duplicates: A mapping from master names to lists of other fully qualified
        names for the object.
    """
    self._index = {}
    self._tree = {}
    self._reverse_index = None
    self._duplicates = None
    self._duplicate_of = None

  @property
  def index(self):
    """A map from fully qualified names to objects to be documented.

    The index is filled when the visitor is passed to `traverse`.

    Returns:
      The index filled by traversal.
    """
    return self._index

  @property
  def tree(self):
    """A map from fully qualified names to all its child names for traversal.

    The full name to member names map is filled when the visitor is passed to
    `traverse`.

    Returns:
      The full name to member name map filled by traversal.
    """
    return self._tree

  @property
  def reverse_index(self):
    """A map from `id(object)` to the preferred fully qualified name.

    This map only contains non-primitive objects (no numbers or strings) present
    in `index` (for primitive objects, `id()` doesn't quite do the right thing).

    It is computed when it, `duplicate_of`, or `duplicates` are first accessed.

    Returns:
      The `id(object)` to full name map.
    """
    self._maybe_find_duplicates()
    return self._reverse_index

  @property
  def duplicate_of(self):
    """A map from duplicate full names to a preferred fully qualified name.

    This map only contains names that are not themself a preferred name.

    It is computed when it, `reverse_index`, or `duplicates` are first accessed.

    Returns:
      The map from duplicate name to preferred name.
    """
    self._maybe_find_duplicates()
    return self._duplicate_of

  @property
  def duplicates(self):
    """A map from preferred full names to a list of all names for this symbol.

    This function returns a map from preferred (master) name for a symbol to a
    lexicographically sorted list of all aliases for that name (incl. the master
    name). Symbols without duplicate names do not appear in this map.

    It is computed when it, `reverse_index`, or `duplicate_of` are first
    accessed.

    Returns:
      The map from master name to list of all duplicate names.
    """
    self._maybe_find_duplicates()
    return self._duplicates

  def __call__(self, parent_path, parent, children):
    """Visitor interface, see `tensorflow/tools/common:traverse` for details.

    This method is called for each symbol found in a traversal using
    `tensorflow/tools/common:traverse`. It should not be called directly in
    user code.

    Args:
      parent_path: A tuple of strings. The fully qualified path to a symbol
        found during traversal.
      parent: The Python object referenced by `parent_name`.
      children: A list of `(name, py_object)` pairs enumerating, in alphabetical
        order, the children (as determined by `tf_inspect.getmembers`) of
          `parent`. `name` is the local name of `py_object` in `parent`.

    Returns:
      The list of children, with any __metaclass__ removed.

    Raises:
      RuntimeError: If this visitor is called with a `parent` that is not a
        class or module.
    """
    parent_name = '.'.join(parent_path)
    self._index[parent_name] = parent
    self._tree[parent_name] = []

    if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)):
      raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name,
                                                                   parent))

    for i, (name, child) in enumerate(list(children)):
      # Don't document these names
      if name in ['__metaclass__', '__setattr__', '__delattr__']:
        del children[i]
        continue

      full_name = '.'.join([parent_name, name]) if parent_name else name
      self._index[full_name] = child
      self._tree[parent_name].append(name)

    return children

  def _score_name(self, name):
    """Return a tuple of scores indicating how to sort for the best name.

    This function is meant to be used as the `key` to the `sorted` function.

    This sorting in order:
      Prefers names refering to the defining class, over a subclass.
      Prefers names that are not in "contrib".
      prefers submodules to the root namespace.
      Prefers short names `tf.thing` over `tf.a.b.c.thing`
      Sorts lexicographically on name parts.

    Args:
      name: the full name to score, for example `tf.estimator.Estimator`

    Returns:
      A tuple of scores. When sorted the preferred name will have the lowest
      value.
    """
    parts = name.split('.')
    short_name = parts[-1]
    if len(parts) == 1:
      return (-99, -99, -99, short_name)

    container = self._index['.'.join(parts[:-1])]

    defining_class_score = 1
    if tf_inspect.isclass(container):
      if short_name in container.__dict__:
        # prefer the defining class
        defining_class_score = -1

    contrib_score = -1
    if 'contrib' in parts:
      contrib_score = 1

    while parts:
      container = self._index['.'.join(parts)]
      if tf_inspect.ismodule(container):
        break
      parts.pop()

    module_length = len(parts)

    if len(parts) == 2:
      # `tf.submodule.thing` is better than `tf.thing`
      module_length_score = -1
    else:
      # shorter is better
      module_length_score = module_length

    return (defining_class_score, contrib_score, module_length_score, name)

  def _maybe_find_duplicates(self):
    """Compute data structures containing information about duplicates.

    Find duplicates in `index` and decide on one to be the "master" name.

    Computes a reverse_index mapping each object id to its master name.

    Also computes a map `duplicate_of` from aliases to their master name (the
    master name itself has no entry in this map), and a map `duplicates` from
    master names to a lexicographically sorted list of all aliases for that name
    (incl. the master name).

    All these are computed and set as fields if they haven't already.
    """
    if self._reverse_index is not None:
      return

    # Maps the id of a symbol to its fully qualified name. For symbols that have
    # several aliases, this map contains the first one found.
    # We use id(py_object) to get a hashable value for py_object. Note all
    # objects in _index are in memory at the same time so this is safe.
    reverse_index = {}

    # Make a preliminary duplicates map. For all sets of duplicate names, it
    # maps the first name found to a list of all duplicate names.
    raw_duplicates = {}
    for full_name, py_object in six.iteritems(self._index):
      # We cannot use the duplicate mechanism for some constants, since e.g.,
      # id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants
      # have no usable docstring and won't be documented automatically.
      if (py_object is not None and
          not isinstance(py_object, six.integer_types + six.string_types +
                         (six.binary_type, six.text_type, float, complex, bool))
          and py_object is not ()):  # pylint: disable=literal-comparison
        object_id = id(py_object)
        if object_id in reverse_index:
          master_name = reverse_index[object_id]
          if master_name in raw_duplicates:
            raw_duplicates[master_name].append(full_name)
          else:
            raw_duplicates[master_name] = [master_name, full_name]
        else:
          reverse_index[object_id] = full_name
    # Decide on master names, rewire duplicates and make a duplicate_of map
    # mapping all non-master duplicates to the master name. The master symbol
    # does not have an entry in this map.
    duplicate_of = {}
    # Duplicates maps the main symbols to the set of all duplicates of that
    # symbol (incl. itself).
    duplicates = {}
    for names in raw_duplicates.values():
      names = sorted(names)
      # Choose the master name with a lexical sort on the tuples returned by
      # by _score_name.
      master_name = min(names, key=self._score_name)

      duplicates[master_name] = names
      for name in names:
        if name != master_name:
          duplicate_of[name] = master_name

      # Set the reverse index to the canonical name.
      reverse_index[id(self._index[master_name])] = master_name

    self._duplicate_of = duplicate_of
    self._duplicates = duplicates
    self._reverse_index = reverse_index
<commit_msg>Add a score preferring symbols inside keras.<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A `traverse` visitor for processing documentation."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import six

from tensorflow_docs.api_generator import tf_inspect

class DocGeneratorVisitor(object):
  """A visitor that generates docs for a python object when __call__ed."""

  def __init__(self):
    """Make a visitor.

    This visitor expects to be called on each node in the api. It is passed the
    path to an object, the object, and the filtered list of the object's
    children. (see the `__call__` method for details.

    This object accumulates the various data-structures necessary to build the
    docs, including (see the property definitions for details.):

    In the decsription below "master name" is the object's preferred fully
    qualified name.

    Params:
      index: A mapping from master names to python python objects.
      tree: A mapping from master names to a list if attribute names.
      reverse_index: Mapping from python object ids to master names.
        Note that this doesn't work for python numbers, strings or tuples.
      duplicate_of: A mapping from a fully qualified names to the object's
        master name. The master names are not included as keys.
      duplicates: A mapping from master names to lists of other fully qualified
        names for the object.
    """
    self._index = {}
    self._tree = {}
    self._reverse_index = None
    self._duplicates = None
    self._duplicate_of = None

  @property
  def index(self):
    """A map from fully qualified names to objects to be documented.

    The index is filled when the visitor is passed to `traverse`.

    Returns:
      The index filled by traversal.
    """
    return self._index

  @property
  def tree(self):
    """A map from fully qualified names to all its child names for traversal.

    The full name to member names map is filled when the visitor is passed to
    `traverse`.

    Returns:
      The full name to member name map filled by traversal.
    """
    return self._tree

  @property
  def reverse_index(self):
    """A map from `id(object)` to the preferred fully qualified name.

    This map only contains non-primitive objects (no numbers or strings) present
    in `index` (for primitive objects, `id()` doesn't quite do the right thing).

    It is computed when it, `duplicate_of`, or `duplicates` are first accessed.

    Returns:
      The `id(object)` to full name map.
    """
    self._maybe_find_duplicates()
    return self._reverse_index

  @property
  def duplicate_of(self):
    """A map from duplicate full names to a preferred fully qualified name.

    This map only contains names that are not themself a preferred name.

    It is computed when it, `reverse_index`, or `duplicates` are first accessed.

    Returns:
      The map from duplicate name to preferred name.
    """
    self._maybe_find_duplicates()
    return self._duplicate_of

  @property
  def duplicates(self):
    """A map from preferred full names to a list of all names for this symbol.

    This function returns a map from preferred (master) name for a symbol to a
    lexicographically sorted list of all aliases for that name (incl. the master
    name). Symbols without duplicate names do not appear in this map.

    It is computed when it, `reverse_index`, or `duplicate_of` are first
    accessed.

    Returns:
      The map from master name to list of all duplicate names.
    """
    self._maybe_find_duplicates()
    return self._duplicates

  def __call__(self, parent_path, parent, children):
    """Visitor interface, see `tensorflow/tools/common:traverse` for details.

    This method is called for each symbol found in a traversal using
    `tensorflow/tools/common:traverse`. It should not be called directly in
    user code.

    Args:
      parent_path: A tuple of strings. The fully qualified path to a symbol
        found during traversal.
      parent: The Python object referenced by `parent_name`.
      children: A list of `(name, py_object)` pairs enumerating, in alphabetical
        order, the children (as determined by `tf_inspect.getmembers`) of
          `parent`. `name` is the local name of `py_object` in `parent`.

    Returns:
      The list of children, with any __metaclass__ removed.

    Raises:
      RuntimeError: If this visitor is called with a `parent` that is not a
        class or module.
    """
    parent_name = '.'.join(parent_path)
    self._index[parent_name] = parent
    self._tree[parent_name] = []

    if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)):
      raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name,
                                                                   parent))

    for i, (name, child) in enumerate(list(children)):
      # Don't document these names
      if name in ['__metaclass__', '__setattr__', '__delattr__']:
        del children[i]
        continue

      full_name = '.'.join([parent_name, name]) if parent_name else name
      self._index[full_name] = child
      self._tree[parent_name].append(name)

    return children

  def _score_name(self, name):
    """Return a tuple of scores indicating how to sort for the best name.

    This function is meant to be used as the `key` to the `sorted` function.

    This returns a score tuple composed of the following scores:
      defining_class: Prefers method names pointing into the defining class,
        over a subclass (`ParentClass.method` over `Subclass.method`, if it
        referrs to the same method implementation).
      experimental: Prefers names that are not in "contrib" or "experimental".
      keras: Prefers keras names to non-keras names.
      module_length: Prefers submodules (tf.sub.thing) over the root namespace
        (tf.thing) over deeply nested paths (tf.a.b.c.thing)
      name: Fallback, sorts lexicographically on the full_name.

    Args:
      name: the full name to score, for example `tf.estimator.Estimator`

    Returns:
      A tuple of scores. When sorted the preferred name will have the lowest
      value.
    """
    parts = name.split('.')
    short_name = parts[-1]
    if len(parts) == 1:
      return (-99, -99, -99, -99, short_name)

    container = self._index['.'.join(parts[:-1])]

    defining_class_score = 1
    if tf_inspect.isclass(container):
      if short_name in container.__dict__:
        # prefer the defining class
        defining_class_score = -1

    experimental_score = -1
    if 'contrib' in parts or any('experimental' in part for part in parts):
      experimental_score = 1

    keras_score = 1
    if 'keras' in parts:
      keras_score = -1

    while parts:
      container = self._index['.'.join(parts)]
      if tf_inspect.ismodule(container):
        break
      parts.pop()

    module_length = len(parts)

    if len(parts) == 2:
      # `tf.submodule.thing` is better than `tf.thing`
      module_length_score = -1
    else:
      # shorter is better
      module_length_score = module_length

    return (defining_class_score, experimental_score, keras_score,
            module_length_score, name)

  def _maybe_find_duplicates(self):
    """Compute data structures containing information about duplicates.

    Find duplicates in `index` and decide on one to be the "master" name.

    Computes a reverse_index mapping each object id to its master name.

    Also computes a map `duplicate_of` from aliases to their master name (the
    master name itself has no entry in this map), and a map `duplicates` from
    master names to a lexicographically sorted list of all aliases for that name
    (incl. the master name).

    All these are computed and set as fields if they haven't already.
    """
    if self._reverse_index is not None:
      return

    # Maps the id of a symbol to its fully qualified name. For symbols that have
    # several aliases, this map contains the first one found.
    # We use id(py_object) to get a hashable value for py_object. Note all
    # objects in _index are in memory at the same time so this is safe.
    reverse_index = {}

    # Make a preliminary duplicates map. For all sets of duplicate names, it
    # maps the first name found to a list of all duplicate names.
    raw_duplicates = {}
    for full_name, py_object in six.iteritems(self._index):
      # We cannot use the duplicate mechanism for some constants, since e.g.,
      # id(c1) == id(c2) with c1=1, c2=1. This is unproblematic since constants
      # have no usable docstring and won't be documented automatically.
      if (py_object is not None and
          not isinstance(py_object, six.integer_types + six.string_types +
                         (six.binary_type, six.text_type, float, complex, bool))
          and py_object is not ()):  # pylint: disable=literal-comparison
        object_id = id(py_object)
        if object_id in reverse_index:
          master_name = reverse_index[object_id]
          if master_name in raw_duplicates:
            raw_duplicates[master_name].append(full_name)
          else:
            raw_duplicates[master_name] = [master_name, full_name]
        else:
          reverse_index[object_id] = full_name
    # Decide on master names, rewire duplicates and make a duplicate_of map
    # mapping all non-master duplicates to the master name. The master symbol
    # does not have an entry in this map.
    duplicate_of = {}
    # Duplicates maps the main symbols to the set of all duplicates of that
    # symbol (incl. itself).
    duplicates = {}
    for names in raw_duplicates.values():
      names = sorted(names)
      # Choose the master name with a lexical sort on the tuples returned by
      # by _score_name.
      master_name = min(names, key=self._score_name)

      duplicates[master_name] = names
      for name in names:
        if name != master_name:
          duplicate_of[name] = master_name

      # Set the reverse index to the canonical name.
      reverse_index[id(self._index[master_name])] = master_name

    self._duplicate_of = duplicate_of
    self._duplicates = duplicates
    self._reverse_index = reverse_index
<|endoftext|>


<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import logging

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.utils import translation

from videos.models import VIDEO_TYPE, VIDEO_TYPE_YOUTUBE
from .videos.types import (
    video_type_registrar, UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION
)
from teams.models import Team
from teams.moderation_const import APPROVED, UNMODERATED
from auth.models import CustomUser as User

from utils.metrics import Meter

logger = logging.getLogger(__name__)

# for now, they kind of match
ACCOUNT_TYPES = VIDEO_TYPE
AMARA_CREDIT = translation.ugettext("Subtitles by the Amara.org community")
AMARA_DESCRIPTION_CREDIT = translation.ugettext(
    "Help us caption and translate this video on Amara.org!")

def youtube_sync(video, language):
    """
    Used on debug page for video.

    Simplified version of what's found in
    ``ThirdPartyAccount.mirror_on_third_party``.  It doesn't bother checking if
    we should be syncing this or not.  Only does the new Youtube/Amara
    integration syncing.
    """
    version = language.latest_version()

    always_push_account = ThirdPartyAccount.objects.always_push_account()

    for vurl in video.videourl_set.all():
        vt = video_type_registrar.video_type_for_url(vurl.url)

        try:
            vt.update_subtitles(version, always_push_account)
            Meter('youtube.push.success').inc()
        except:
            Meter('youtube.push.fail').inc()
            logger.error('Always pushing to youtoube has failed.', extra={
                'video': video.video_id,
                'vurl': vurl.pk
            })
        finally:
            Meter('youtube.push.request').inc()

def check_authorization(video):
    """
    Make sure that a video can have its subtitles synced to Youtube.  This
    doesn't take into account any language/version information.

    Return a tuple of (is_authorized, ignore_new_syncing_logic).
    """
    team_video = video.get_team_video()
    ignore_new_syncing_logic = False

    if team_video:
        team = team_video.team
        has_linked_youtube_account = team.third_party_accounts.filter(
                type=VIDEO_TYPE_YOUTUBE).exists()
        if has_linked_youtube_account:
            # Ignore the new syncing logic.  Use the linked Youtube account
            # to publish subtitles to Youtube.
            ignore_new_syncing_logic = True
        else:
            # The assumption is that it's the partner's official Youtube
            # account and they don't want it messed up with strange subs.
            return False, None
    else:
        # If a video isn't part of a team but the video's Youtube URL is linked
        # to a team third party account, we don't sync to Youtube.
        yt_url = video.videourl_set.filter(type=VIDEO_TYPE_YOUTUBE)
        if yt_url.exists():
            usernames = [url.owner_username for url in yt_url]
            linked_accounts = ThirdPartyAccount.objects.filter(
                    username__in=usernames)

            if linked_accounts.exists():
                if any(a.is_team_account for a in linked_accounts):
                    return False, None

    return True, ignore_new_syncing_logic

def can_be_synced(version):
    """
    Determine if a subtitle version can be synced to Youtube.

    A version must be public, synced and complete; it must also be either
    "approved" or "unmoderated".
    """
    if version:
        if not version.is_public or not version.is_synced():
            # We can't mirror unsynced or non-public versions.
            return False

        if not version.language.is_complete:
            # Don't sync incomplete languages
            return False

        status = version.moderation_status

        if (status != APPROVED) and (status != UNMODERATED):
            return False

    return True

def translate_string(string, language='en'):
    """
    If a translation for the specified language doesn't exist, return the
    English version.
    """
    cur_language = translation.get_language()
    try:
        translation.activate(language)
        text = translation.ugettext(string)
    finally:
        translation.activate(cur_language)
    return text

def get_amara_credit_text(language='en'):
    return translate_string(AMARA_CREDIT, language)

def add_amara_description_credit(old_description, video_url, language='en'):
    """
    Prepend the credit to the existing description.
    """
    credit = translate_string(AMARA_DESCRIPTION_CREDIT, language)

    if old_description.startswith(credit):
        return old_description

    return "%s\n\n%s\n\n%s" % (
        credit,
        video_url,
        old_description or ""
    )

class ThirdPartyAccountManager(models.Manager):

    def always_push_account(self):
        """
        Get the ThirdPartyAccount that is able to push to any video on Youtube.
        Raise ``ImproperlyConfigured`` if it can't be found.
        """
        username = getattr(settings, 'YOUTUBE_ALWAYS_PUSH_USERNAME')

        try:
            return self.get(username=username)
        except ThirdPartyAccount.DoesNotExist:
            raise ImproperlyConfigured("Can't find youtube account")

    def mirror_on_third_party(self, video, language, action, version=None):
        """
        Does the specified action (video.types.UPDATE_VERSION_ACTION or
                                   video.types.DELETE_LANGUAGE_ACTION)
        on the original account (e.g. Youtube video).
        For example, to update a given version to Youtube:
             ThirdPartyAccountManager.objects.mirror_on_third_party(
                       video, language, "update_subtitles", version)
        For deleting, we only delete languages, so it should be
              ThirdPartyAccountManager.objects.mirror_on_third_party(
                        video, language, "delete_subtitles")
        This method is 'safe' to call, meaning that we only do syncing if there
        are matching third party credentials for this video.
        The update will only be done if the version is synced
        """
        if action not in [UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION]:
            raise NotImplementedError(
                "Mirror to third party does not support the %s action" % action)

        if not version and action == UPDATE_VERSION_ACTION:
            raise ValueError("You need to pass a version when updating subs")

        if not can_be_synced(version):
            return

        is_authorized, ignore_new_syncing_logic = check_authorization(video)

        if not is_authorized:
            return

        try:
            rule = YoutubeSyncRule.objects.all()[0]
            should_sync = rule.should_sync(video)
            always_push_account = self.always_push_account()
        except IndexError:
            should_sync = False

        for vurl in video.videourl_set.all():
            already_updated = False
            vt = video_type_registrar.video_type_for_url(vurl.url)

            if should_sync and not ignore_new_syncing_logic:
                try:
                    vt.update_subtitles(version, always_push_account)
                    already_updated = True
                    Meter('youtube.push.success').inc()
                except:
                    Meter('youtube.push.fail').inc()
                    logger.error('Pushing to youtoube has failed.', extra={
                        'video': video.video_id,
                        'vurl': vurl.pk
                    })
                finally:
                    Meter('youtube.push.request').inc()

            username = vurl.owner_username

            if not username:
                continue
            try:
                account = ThirdPartyAccount.objects.get(type=vurl.type, username=username)
            except ThirdPartyAccount.DoesNotExist:
                continue

            if hasattr(vt, action):
                if action == UPDATE_VERSION_ACTION and not already_updated:
                    vt.update_subtitles(version, account)
                elif action == DELETE_LANGUAGE_ACTION:
                    vt.delete_subtitles(language, account)

class ThirdPartyAccount(models.Model):
    """
    Links a third party account (e.g. YouTube's') to a certain video URL
    This allows us to push changes in Unisubs back to that video provider.
    The user links a video on unisubs to his youtube account. Once edits to
    any languages are done, we push those back to Youtube.
    For know, this only supports Youtube, but nothing is stopping it from
    working with others.
    """
    type = models.CharField(max_length=10, choices=ACCOUNT_TYPES)
    # this is the third party account user name, eg the youtube user
    username  = models.CharField(max_length=255, db_index=True,
                                 null=False, blank=False)
    oauth_access_token = models.CharField(max_length=255, db_index=True,
                                          null=False, blank=False)
    oauth_refresh_token = models.CharField(max_length=255, db_index=True,
                                           null=False, blank=False)

    objects = ThirdPartyAccountManager()

    class Meta:
        unique_together = ("type", "username")

    def __unicode__(self):
        return '%s - %s' % (self.get_type_display(), self.username)

    @property
    def is_team_account(self):
        return self.teams.exists()

    @property
    def is_individual_account(self):
        return self.users.exists()

class YoutubeSyncRule(models.Model):
    """
    An instance of this class determines which Youtube videos should be synced
    back to Youtube via the new integration.

    There should only ever be one instance of this class in the database.

    You should run a query and then call it like this:

        rule = YoutubeSyncRule.objects.all()[0]
        rule.should_sync(video)

    Where ``video`` is a ``videos.models.Video`` instance.

    ``team`` should be a comma-separated list of team slugs that you want to
    sync.  ``user`` should be a comma-separated list of usernames of users
    whose videos should be synced.  ``video`` is a list of video ids of
    videos that should be synced.

    You can also specify a wildcard "*" to any of the above to match any teams,
    any users, or any videos.
    """
    team = models.TextField(default='', blank=True,
            help_text='Comma separated list of slugs')
    user = models.TextField(default='', blank=True,
            help_text='Comma separated list of usernames')
    video = models.TextField(default='', blank=True,
            help_text='Comma separated list of video ids')

    def __unicode__(self):
        return 'Youtube sync rule'

    def team_in_list(self, team):
        if not team:
            return False
        teams = self.team.split(',')
        if '*' in teams:
            return True
        return team in teams

    def user_in_list(self, user):
        if not user:
            return False

        users = self.user.split(',')

        if '*' in users:
            return True
        return user.username in users

    def video_in_list(self, pk):
        pks = self.video.split(',')
        if '*' in pks:
            return True
        if len(pks) == 1 and pks[0] == '':
            return False
        return pk in pks

    def should_sync(self, video):
        tv = video.get_team_video()
        team = None
        if tv:
            team = tv.team.slug

        return self.team_in_list(team) or \
                self.user_in_list(video.user) or \
                self.video_in_list(video.video_id)

    def _clean(self, name):
        if name not in ['team', 'user']:
            return
        field  = getattr(self, name)
        values = set(field.split(','))
        values = [v for v in values if v != '*']
        if len(values) == 1 and values[0] == '':
            return []
        return values

    def clean(self):
        teams = self._clean('team')
        users = self._clean('user')

        if len(teams) != Team.objects.filter(slug__in=teams).count():
            raise ValidationError("One or more teams not found")

        if len(users) != User.objects.filter(username__in=users).count():
            raise ValidationError("One or more users not found")
<commit_msg>Only one newline between link and text<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import logging

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.utils import translation

from videos.models import VIDEO_TYPE, VIDEO_TYPE_YOUTUBE
from .videos.types import (
    video_type_registrar, UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION
)
from teams.models import Team
from teams.moderation_const import APPROVED, UNMODERATED
from auth.models import CustomUser as User

from utils.metrics import Meter

logger = logging.getLogger(__name__)

# for now, they kind of match
ACCOUNT_TYPES = VIDEO_TYPE
AMARA_CREDIT = translation.ugettext("Subtitles by the Amara.org community")
AMARA_DESCRIPTION_CREDIT = translation.ugettext(
    "Help us caption and translate this video on Amara.org!")

def youtube_sync(video, language):
    """
    Used on debug page for video.

    Simplified version of what's found in
    ``ThirdPartyAccount.mirror_on_third_party``.  It doesn't bother checking if
    we should be syncing this or not.  Only does the new Youtube/Amara
    integration syncing.
    """
    version = language.latest_version()

    always_push_account = ThirdPartyAccount.objects.always_push_account()

    for vurl in video.videourl_set.all():
        vt = video_type_registrar.video_type_for_url(vurl.url)

        try:
            vt.update_subtitles(version, always_push_account)
            Meter('youtube.push.success').inc()
        except:
            Meter('youtube.push.fail').inc()
            logger.error('Always pushing to youtoube has failed.', extra={
                'video': video.video_id,
                'vurl': vurl.pk
            })
        finally:
            Meter('youtube.push.request').inc()

def check_authorization(video):
    """
    Make sure that a video can have its subtitles synced to Youtube.  This
    doesn't take into account any language/version information.

    Return a tuple of (is_authorized, ignore_new_syncing_logic).
    """
    team_video = video.get_team_video()
    ignore_new_syncing_logic = False

    if team_video:
        team = team_video.team
        has_linked_youtube_account = team.third_party_accounts.filter(
                type=VIDEO_TYPE_YOUTUBE).exists()
        if has_linked_youtube_account:
            # Ignore the new syncing logic.  Use the linked Youtube account
            # to publish subtitles to Youtube.
            ignore_new_syncing_logic = True
        else:
            # The assumption is that it's the partner's official Youtube
            # account and they don't want it messed up with strange subs.
            return False, None
    else:
        # If a video isn't part of a team but the video's Youtube URL is linked
        # to a team third party account, we don't sync to Youtube.
        yt_url = video.videourl_set.filter(type=VIDEO_TYPE_YOUTUBE)
        if yt_url.exists():
            usernames = [url.owner_username for url in yt_url]
            linked_accounts = ThirdPartyAccount.objects.filter(
                    username__in=usernames)

            if linked_accounts.exists():
                if any(a.is_team_account for a in linked_accounts):
                    return False, None

    return True, ignore_new_syncing_logic

def can_be_synced(version):
    """
    Determine if a subtitle version can be synced to Youtube.

    A version must be public, synced and complete; it must also be either
    "approved" or "unmoderated".
    """
    if version:
        if not version.is_public or not version.is_synced():
            # We can't mirror unsynced or non-public versions.
            return False

        if not version.language.is_complete:
            # Don't sync incomplete languages
            return False

        status = version.moderation_status

        if (status != APPROVED) and (status != UNMODERATED):
            return False

    return True

def translate_string(string, language='en'):
    """
    If a translation for the specified language doesn't exist, return the
    English version.
    """
    cur_language = translation.get_language()
    try:
        translation.activate(language)
        text = translation.ugettext(string)
    finally:
        translation.activate(cur_language)
    return text

def get_amara_credit_text(language='en'):
    return translate_string(AMARA_CREDIT, language)

def add_amara_description_credit(old_description, video_url, language='en'):
    """
    Prepend the credit to the existing description.
    """
    credit = translate_string(AMARA_DESCRIPTION_CREDIT, language)

    if old_description.startswith(credit):
        return old_description

    return "%s\n%s\n\n%s" % (
        credit,
        video_url,
        old_description or ""
    )

class ThirdPartyAccountManager(models.Manager):

    def always_push_account(self):
        """
        Get the ThirdPartyAccount that is able to push to any video on Youtube.
        Raise ``ImproperlyConfigured`` if it can't be found.
        """
        username = getattr(settings, 'YOUTUBE_ALWAYS_PUSH_USERNAME')

        try:
            return self.get(username=username)
        except ThirdPartyAccount.DoesNotExist:
            raise ImproperlyConfigured("Can't find youtube account")

    def mirror_on_third_party(self, video, language, action, version=None):
        """
        Does the specified action (video.types.UPDATE_VERSION_ACTION or
                                   video.types.DELETE_LANGUAGE_ACTION)
        on the original account (e.g. Youtube video).
        For example, to update a given version to Youtube:
             ThirdPartyAccountManager.objects.mirror_on_third_party(
                       video, language, "update_subtitles", version)
        For deleting, we only delete languages, so it should be
              ThirdPartyAccountManager.objects.mirror_on_third_party(
                        video, language, "delete_subtitles")
        This method is 'safe' to call, meaning that we only do syncing if there
        are matching third party credentials for this video.
        The update will only be done if the version is synced
        """
        if action not in [UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION]:
            raise NotImplementedError(
                "Mirror to third party does not support the %s action" % action)

        if not version and action == UPDATE_VERSION_ACTION:
            raise ValueError("You need to pass a version when updating subs")

        if not can_be_synced(version):
            return

        is_authorized, ignore_new_syncing_logic = check_authorization(video)

        if not is_authorized:
            return

        try:
            rule = YoutubeSyncRule.objects.all()[0]
            should_sync = rule.should_sync(video)
            always_push_account = self.always_push_account()
        except IndexError:
            should_sync = False

        for vurl in video.videourl_set.all():
            already_updated = False
            vt = video_type_registrar.video_type_for_url(vurl.url)

            if should_sync and not ignore_new_syncing_logic:
                try:
                    vt.update_subtitles(version, always_push_account)
                    already_updated = True
                    Meter('youtube.push.success').inc()
                except:
                    Meter('youtube.push.fail').inc()
                    logger.error('Pushing to youtoube has failed.', extra={
                        'video': video.video_id,
                        'vurl': vurl.pk
                    })
                finally:
                    Meter('youtube.push.request').inc()

            username = vurl.owner_username

            if not username:
                continue
            try:
                account = ThirdPartyAccount.objects.get(type=vurl.type, username=username)
            except ThirdPartyAccount.DoesNotExist:
                continue

            if hasattr(vt, action):
                if action == UPDATE_VERSION_ACTION and not already_updated:
                    vt.update_subtitles(version, account)
                elif action == DELETE_LANGUAGE_ACTION:
                    vt.delete_subtitles(language, account)

class ThirdPartyAccount(models.Model):
    """
    Links a third party account (e.g. YouTube's') to a certain video URL
    This allows us to push changes in Unisubs back to that video provider.
    The user links a video on unisubs to his youtube account. Once edits to
    any languages are done, we push those back to Youtube.
    For know, this only supports Youtube, but nothing is stopping it from
    working with others.
    """
    type = models.CharField(max_length=10, choices=ACCOUNT_TYPES)
    # this is the third party account user name, eg the youtube user
    username  = models.CharField(max_length=255, db_index=True,
                                 null=False, blank=False)
    oauth_access_token = models.CharField(max_length=255, db_index=True,
                                          null=False, blank=False)
    oauth_refresh_token = models.CharField(max_length=255, db_index=True,
                                           null=False, blank=False)

    objects = ThirdPartyAccountManager()

    class Meta:
        unique_together = ("type", "username")

    def __unicode__(self):
        return '%s - %s' % (self.get_type_display(), self.username)

    @property
    def is_team_account(self):
        return self.teams.exists()

    @property
    def is_individual_account(self):
        return self.users.exists()

class YoutubeSyncRule(models.Model):
    """
    An instance of this class determines which Youtube videos should be synced
    back to Youtube via the new integration.

    There should only ever be one instance of this class in the database.

    You should run a query and then call it like this:

        rule = YoutubeSyncRule.objects.all()[0]
        rule.should_sync(video)

    Where ``video`` is a ``videos.models.Video`` instance.

    ``team`` should be a comma-separated list of team slugs that you want to
    sync.  ``user`` should be a comma-separated list of usernames of users
    whose videos should be synced.  ``video`` is a list of video ids of
    videos that should be synced.

    You can also specify a wildcard "*" to any of the above to match any teams,
    any users, or any videos.
    """
    team = models.TextField(default='', blank=True,
            help_text='Comma separated list of slugs')
    user = models.TextField(default='', blank=True,
            help_text='Comma separated list of usernames')
    video = models.TextField(default='', blank=True,
            help_text='Comma separated list of video ids')

    def __unicode__(self):
        return 'Youtube sync rule'

    def team_in_list(self, team):
        if not team:
            return False
        teams = self.team.split(',')
        if '*' in teams:
            return True
        return team in teams

    def user_in_list(self, user):
        if not user:
            return False

        users = self.user.split(',')

        if '*' in users:
            return True
        return user.username in users

    def video_in_list(self, pk):
        pks = self.video.split(',')
        if '*' in pks:
            return True
        if len(pks) == 1 and pks[0] == '':
            return False
        return pk in pks

    def should_sync(self, video):
        tv = video.get_team_video()
        team = None
        if tv:
            team = tv.team.slug

        return self.team_in_list(team) or \
                self.user_in_list(video.user) or \
                self.video_in_list(video.video_id)

    def _clean(self, name):
        if name not in ['team', 'user']:
            return
        field  = getattr(self, name)
        values = set(field.split(','))
        values = [v for v in values if v != '*']
        if len(values) == 1 and values[0] == '':
            return []
        return values

    def clean(self):
        teams = self._clean('team')
        users = self._clean('user')

        if len(teams) != Team.objects.filter(slug__in=teams).count():
            raise ValidationError("One or more teams not found")

        if len(users) != User.objects.filter(username__in=users).count():
            raise ValidationError("One or more users not found")
<|endoftext|>