<commit_before># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utils.

This has been tested with python 2.7.
"""

from __future__ import print_function

import subprocess
import re
import os

import constants

def id_from_name(name):
    """Extract the id from the resource name.

    Args:
        name (str): The name of the resource, formatted as
            "${RESOURCE_TYPE}/${RESOURCE_ID}".

    Returns:
        str: The resource id.
    """
    if not name or not '/' in name:
        return name
    return name[name.index('/')+1:]

def print_banner(text):
    """Print a banner.

    Args:
        text (str): Text to put in the banner.
    """
    print('')
    print('+-------------------------------------------------------')
    print('|  %s' % text)
    print('+-------------------------------------------------------')
    print('')

def get_forseti_version():
    """Get Forseti version from version file.

    Returns:
        str: The version.
    """
    version = None
    version_re = re.compile(constants.VERSIONFILE_REGEX)
    version_file = os.path.join(
        constants.FORSETI_SRC_PATH, '__init__.py')
    with open(version_file, 'rt') as vfile:
        for line in vfile:
            version_match = version_re.match(line)
            if version_match:
                version = version_match.group(1)
                break
    return version

def get_remote_branches():
    """Get remote git branches.

    Returns:
        list: A list of branches.
    """
    branches = []
    return_code, out, err = run_command(
        ['git', 'branch', '-r'])
    if return_code:
        print(err)
    else:
        out = out.strip()
        if out:
            branches = [b.strip() for b in out.split('\n')]
    return branches

def checkout_git_branch():
    """Let user choose git branch and check it out.

    Returns:
        str: The checked out branch, if exists.
    """
    branches = get_remote_branches()
    choice_index = -1
    while choice_index < 0 or choice_index > len(branches):
        branches = get_remote_branches()
        print('Remote branches:')
        for (i, branch) in enumerate(branches):
            print('[%s] %s' % (i+1, branch[len('origin/'):]))
        try:
            choice_index = int(raw_input(
                'Enter your numerical choice: ').strip())
        except ValueError:
            print('Invalid input choice, try again.')
    branch = branches[choice_index-1][len('origin/'):]
    return_code, _, err = run_command(
        ['git', 'checkout', branch])
    if return_code:
        print(err)
    return branch

def full_service_acct_email(account_id, project_id):
    """Generate the full service account email.

    Args:
        account_id (str): The service account id, i.e. the part before
            the "@".
        project_id (str): The project id the service account belongs to

    Returns:
        str: The full service account email.
    """
    return constants.SERVICE_ACCT_EMAIL_FMT.format(
        account_id, project_id)

def format_resource_id(resource_type, resource_id):
    """Format the resource id as $RESOURCE_TYPE/$RESOURCE_ID.

    Args:
        resource_type (str): The resource type.
        resource_id (str): The resource id.

    Returns:
        str: The formatted resource id.
    """
    return '%s/%s' % (resource_type, resource_id)

def format_service_acct_id(prefix, modifier, timestamp, project_id):
    """Format the service account ids.

    Args:
        prefix (str): The prefix of the account id
        modifier (str): Access level of the account
        timestamp (str): Timestamp of the class
        project_id (str): Id of the project on GCP

    Returns:
        str: Service account id
    """

    return full_service_acct_email(
        constants.SERVICE_ACCT_FMT.format(
            prefix, modifier, timestamp), project_id)

def infer_version(advanced_mode):
    """Infer the Forseti version, or ask user to input one not listed.

    Args:
        advanced_mode (bool): Whether or not the installer is in advanced mode

    Returns:
        str: Selected Forseti branch
    """
    return_code, out, err = run_command(
        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    if return_code:
        print(err)
        print('Will try to infer the Forseti version instead.')
    else:
        branch = out.strip()

    user_choice = None
    if not branch or branch == 'master':
        version = get_forseti_version()
        if version:
            branch = 'v%s' % version

    if not advanced_mode:
        user_choice = 'y'

    while user_choice != 'y' and user_choice != 'n':
        user_choice = raw_input(
            'Install Forseti branch/tag %s? (y/n) '
            % branch).lower().strip()

    if user_choice == 'n':
        branch = checkout_git_branch()
        if branch:
            branch = branch
        else:
            print('No branch was chosen; using %s' % branch)

    print('Forseti branch/tag: %s' % branch)
    return branch

def get_zone_from_bucket_location(bucket_location):
    """Get zone from bucket location.

    Args:
        bucket_location (str): Bucket location

    Returns:
        str: Zone for that given bucket location
    """

    return '{}-c'.format(bucket_location)

def get_choice_id(choices, print_function):
    """Get choice id from user.

    Args:
        choices (list): A list of choices
        print_function (method): Print function
    Returns:
        int: choice id
    """
    while True:
        for (i, choice) in enumerate(choices):
            print_function(i, choice)

        choice_input = raw_input(
            'Enter the number of your choice: ').strip()

        try:
            choice_index = int(choice_input)
            if choice_index < 1 or choice_index > len(choices):
                raise ValueError
            else:
                break
        except ValueError:
            print('Invalid choice "%s", try again' % choice_input)
    return choice_index

def extract_timestamp_from_name(instance_name, include_date=False):
    """Extract timestamp from instance name.

    Example instance_name: forseti-security-server-20180207125916-vm
    Example output: '125916' if include_date is false,
                    '20180207125916' otherwise
    Args:
        instance_name (str): Name of the instance
        include_date (bool): Include date in the timestamp

    Returns:
        str: Timestamp
    """
    if instance_name is None:
        return ''

    if include_date:
        return instance_name.split('-')[-2]
    return instance_name.split('-')[-2][8:]

def run_command(cmd_args):
    """Wrapper to run a command in subprocess.

    Args:
        cmd_args (list): The list of command arguments.

    Returns:
        int: The return code. 0 is "ok", anything else is "error".
        str: Output, if command was successful.
        err: Error output, if there was an error.
    """
    proc = subprocess.Popen(cmd_args,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    return proc.returncode, out, err

def sanitize_conf_values(conf_values):
    """Sanitize the forseti_conf values not to be zero-length strings.

    Args:
        conf_values (dict): The conf values to replace in the
            forseti_conf_server.yaml.

    Returns:
        dict: The sanitized values.
    """
    for key in conf_values.keys():
        if not conf_values[key]:
            conf_values[key] = '""'
    return conf_values

def merge_dict(base_dict, target_dict,
               fields_to_ignore=None, field_identifiers=None):
    """Merge target_dict into base_dict.

    Note: base_dict will be modified during the merge process

    Args:
        base_dict (dict): Base dictionary.
        target_dict (dict): Target dictionary.
        fields_to_ignore (list): Fields to ignore (keep in base_dict)
        field_identifiers (dict): Identifiers for fields
    """
    # Init default values for fields_to_ignore and field_identifiers
    # if they don't already exists
    if fields_to_ignore is None:
        fields_to_ignore = []
    if field_identifiers is None:
        field_identifiers = {}

    for key, val in base_dict.iteritems():
        # print (key + "      ///////        " + str(val))
        if key in target_dict:
            # If target_dict has the same key, we check if the value is
            # an instance of dictionary. If it is we merge recursively and
            # if it's not, we do a simple replace if the key is not in
            # fields_to_ignore
            if key in fields_to_ignore:
                continue
            if isinstance(val, dict):
                merge_dict(val, target_dict.get(key),
                           fields_to_ignore, field_identifiers)
            elif isinstance(val, list) and val:
                # If the list has at least one item, we check for the type
                # of the first item, if it's a dictionary, we invoke
                # merge_dict_list, otherwise we will do a simple merge of
                # 2 lists
                if isinstance(val[0], dict):
                    identifier = field_identifiers.get(key)
                    merge_dict_list(val, target_dict.get(key), identifier,
                                    fields_to_ignore, field_identifiers)
                else:
                    # Cast the merged list to a set to remove duplicate
                    # and cast it back to a list afterward
                    base_dict[key] = list(set(val + target_dict.get(key)))
            else:
                base_dict[key] = target_dict.get(key)

    for key, val in target_dict.iteritems():
        if key not in base_dict and key not in fields_to_ignore:
            # If this is a key we have only in target but not in base, we add
            # it to the base_dict
            base_dict[key] = val

def merge_dict_list(base_dict_list, target_dict_list, identifier,
                    fields_to_ignore, field_identifiers):
    """Merge target_dict_list into base_dict_list.

    Note: base_dict_list will be modified during the merge process.

    Args:
        base_dict_list (list): Base dictionary.
        target_dict_list (list): Target dictionary.
        identifier (str): Current field identifier.
        fields_to_ignore (list): Fields to ignore (keep in base_dict).
        field_identifiers (dict): Identifiers for fields.
    """

    if not identifier:
        # Doesn't have an identifier, append all the items from
        # target_dict_list to base_dict_list
        base_dict_list.extend(target_dict_list)
        return

    # Sort both base_dict_list and target_dict_list by the identifier
    base_dict_list.sort(key=lambda k: k[identifier])
    target_dict_list.sort(key=lambda k: k[identifier])

    # Merge them
    base_counter = 0
    target_counter = 0
    max_items = len(base_dict_list) + len(target_dict_list)
    for _ in range(0, max_items):
        cur_base_dict = (None if len(base_dict_list) <= base_counter
                         else base_dict_list[base_counter])
        cur_taget_dict = (None if len(target_dict_list) <= target_counter
                          else target_dict_list[target_counter])

        if target_counter >= len(target_dict_list):
            break
        elif (base_counter >= len(base_dict_list) or
              cur_base_dict[identifier] > cur_taget_dict[identifier]):
            # cur_target_object only exists in target_dict_list,
            # add it to base_dict_list, increment target_counter
            base_dict_list.append(cur_taget_dict)
            target_counter += 1
        elif cur_base_dict[identifier] < cur_taget_dict[identifier]:
            # cur_base_dict object only exists in base_dict_list,
            # increment base_counter
            base_counter += 1
        else:
            # They have the same identifier, merge them,
            # increment both counters
            merge_dict(cur_base_dict, cur_taget_dict,
                       fields_to_ignore, field_identifiers)
            base_counter += 1
            target_counter += 1
<commit_msg>Added handles when merging dictionary lists without an identifier<commit_after># Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utils.

This has been tested with python 2.7.
"""

from __future__ import print_function

import subprocess
import re
import os

import constants

def id_from_name(name):
    """Extract the id from the resource name.

    Args:
        name (str): The name of the resource, formatted as
            "${RESOURCE_TYPE}/${RESOURCE_ID}".

    Returns:
        str: The resource id.
    """
    if not name or not '/' in name:
        return name
    return name[name.index('/')+1:]

def print_banner(text):
    """Print a banner.

    Args:
        text (str): Text to put in the banner.
    """
    print('')
    print('+-------------------------------------------------------')
    print('|  %s' % text)
    print('+-------------------------------------------------------')
    print('')

def get_forseti_version():
    """Get Forseti version from version file.

    Returns:
        str: The version.
    """
    version = None
    version_re = re.compile(constants.VERSIONFILE_REGEX)
    version_file = os.path.join(
        constants.FORSETI_SRC_PATH, '__init__.py')
    with open(version_file, 'rt') as vfile:
        for line in vfile:
            version_match = version_re.match(line)
            if version_match:
                version = version_match.group(1)
                break
    return version

def get_remote_branches():
    """Get remote git branches.

    Returns:
        list: A list of branches.
    """
    branches = []
    return_code, out, err = run_command(
        ['git', 'branch', '-r'])
    if return_code:
        print(err)
    else:
        out = out.strip()
        if out:
            branches = [b.strip() for b in out.split('\n')]
    return branches

def checkout_git_branch():
    """Let user choose git branch and check it out.

    Returns:
        str: The checked out branch, if exists.
    """
    branches = get_remote_branches()
    choice_index = -1
    while choice_index < 0 or choice_index > len(branches):
        branches = get_remote_branches()
        print('Remote branches:')
        for (i, branch) in enumerate(branches):
            print('[%s] %s' % (i+1, branch[len('origin/'):]))
        try:
            choice_index = int(raw_input(
                'Enter your numerical choice: ').strip())
        except ValueError:
            print('Invalid input choice, try again.')
    branch = branches[choice_index-1][len('origin/'):]
    return_code, _, err = run_command(
        ['git', 'checkout', branch])
    if return_code:
        print(err)
    return branch

def full_service_acct_email(account_id, project_id):
    """Generate the full service account email.

    Args:
        account_id (str): The service account id, i.e. the part before
            the "@".
        project_id (str): The project id the service account belongs to

    Returns:
        str: The full service account email.
    """
    return constants.SERVICE_ACCT_EMAIL_FMT.format(
        account_id, project_id)

def format_resource_id(resource_type, resource_id):
    """Format the resource id as $RESOURCE_TYPE/$RESOURCE_ID.

    Args:
        resource_type (str): The resource type.
        resource_id (str): The resource id.

    Returns:
        str: The formatted resource id.
    """
    return '%s/%s' % (resource_type, resource_id)

def format_service_acct_id(prefix, modifier, timestamp, project_id):
    """Format the service account ids.

    Args:
        prefix (str): The prefix of the account id
        modifier (str): Access level of the account
        timestamp (str): Timestamp of the class
        project_id (str): Id of the project on GCP

    Returns:
        str: Service account id
    """

    return full_service_acct_email(
        constants.SERVICE_ACCT_FMT.format(
            prefix, modifier, timestamp), project_id)

def infer_version(advanced_mode):
    """Infer the Forseti version, or ask user to input one not listed.

    Args:
        advanced_mode (bool): Whether or not the installer is in advanced mode

    Returns:
        str: Selected Forseti branch
    """
    return_code, out, err = run_command(
        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    if return_code:
        print(err)
        print('Will try to infer the Forseti version instead.')
    else:
        branch = out.strip()

    user_choice = None
    if not branch or branch == 'master':
        version = get_forseti_version()
        if version:
            branch = 'v%s' % version

    if not advanced_mode:
        user_choice = 'y'

    while user_choice != 'y' and user_choice != 'n':
        user_choice = raw_input(
            'Install Forseti branch/tag %s? (y/n) '
            % branch).lower().strip()

    if user_choice == 'n':
        branch = checkout_git_branch()
        if branch:
            branch = branch
        else:
            print('No branch was chosen; using %s' % branch)

    print('Forseti branch/tag: %s' % branch)
    return branch

def get_zone_from_bucket_location(bucket_location):
    """Get zone from bucket location.

    Args:
        bucket_location (str): Bucket location

    Returns:
        str: Zone for that given bucket location
    """

    return '{}-c'.format(bucket_location)

def get_choice_id(choices, print_function):
    """Get choice id from user.

    Args:
        choices (list): A list of choices
        print_function (method): Print function
    Returns:
        int: choice id
    """
    while True:
        for (i, choice) in enumerate(choices):
            print_function(i, choice)

        choice_input = raw_input(
            'Enter the number of your choice: ').strip()

        try:
            choice_index = int(choice_input)
            if choice_index < 1 or choice_index > len(choices):
                raise ValueError
            else:
                break
        except ValueError:
            print('Invalid choice "%s", try again' % choice_input)
    return choice_index

def extract_timestamp_from_name(instance_name, include_date=False):
    """Extract timestamp from instance name.

    Example instance_name: forseti-security-server-20180207125916-vm
    Example output: '125916' if include_date is false,
                    '20180207125916' otherwise
    Args:
        instance_name (str): Name of the instance
        include_date (bool): Include date in the timestamp

    Returns:
        str: Timestamp
    """
    if instance_name is None:
        return ''

    if include_date:
        return instance_name.split('-')[-2]
    return instance_name.split('-')[-2][8:]

def run_command(cmd_args):
    """Wrapper to run a command in subprocess.

    Args:
        cmd_args (list): The list of command arguments.

    Returns:
        int: The return code. 0 is "ok", anything else is "error".
        str: Output, if command was successful.
        err: Error output, if there was an error.
    """
    proc = subprocess.Popen(cmd_args,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate()
    return proc.returncode, out, err

def sanitize_conf_values(conf_values):
    """Sanitize the forseti_conf values not to be zero-length strings.

    Args:
        conf_values (dict): The conf values to replace in the
            forseti_conf_server.yaml.

    Returns:
        dict: The sanitized values.
    """
    for key in conf_values.keys():
        if not conf_values[key]:
            conf_values[key] = '""'
    return conf_values

def merge_dict(base_dict, target_dict,
               fields_to_ignore=None, field_identifiers=None):
    """Merge target_dict into base_dict.

    Note: base_dict will be modified during the merge process

    Args:
        base_dict (dict): Base dictionary.
        target_dict (dict): Target dictionary.
        fields_to_ignore (list): Fields to ignore (keep in base_dict)
        field_identifiers (dict): Identifiers for fields
    """
    # Init default values for fields_to_ignore and field_identifiers
    # if they don't already exists
    if fields_to_ignore is None:
        fields_to_ignore = []
    if field_identifiers is None:
        field_identifiers = {}

    for key, val in base_dict.iteritems():
        # print (key + "      ///////        " + str(val))
        if key in target_dict:
            # If target_dict has the same key, we check if the value is
            # an instance of dictionary. If it is we merge recursively and
            # if it's not, we do a simple replace if the key is not in
            # fields_to_ignore
            if key in fields_to_ignore:
                continue
            if isinstance(val, dict):
                merge_dict(val, target_dict.get(key),
                           fields_to_ignore, field_identifiers)
            elif isinstance(val, list) and val:
                # If the list has at least one item, we check for the type
                # of the first item, if it's a dictionary, we invoke
                # merge_dict_list, otherwise we will do a simple merge of
                # 2 lists
                if isinstance(val[0], dict):
                    identifier = field_identifiers.get(key)
                    merge_dict_list(val, target_dict.get(key), identifier,
                                    fields_to_ignore, field_identifiers)
                else:
                    # Cast the merged list to a set to remove duplicate
                    # and cast it back to a list afterward
                    base_dict[key] = list(set(val + target_dict.get(key)))
            else:
                base_dict[key] = target_dict.get(key)

    for key, val in target_dict.iteritems():
        if key not in base_dict and key not in fields_to_ignore:
            # If this is a key we have only in target but not in base, we add
            # it to the base_dict
            base_dict[key] = val

def merge_dict_list(base_dict_list, target_dict_list, identifier,
                    fields_to_ignore, field_identifiers):
    """Merge target_dict_list into base_dict_list.

    Note: base_dict_list will be modified during the merge process.

    Args:
        base_dict_list (list): Base dictionary.
        target_dict_list (list): Target dictionary.
        identifier (str): Current field identifier.
        fields_to_ignore (list): Fields to ignore (keep in base_dict).
        field_identifiers (dict): Identifiers for fields.
    """

    if not identifier or identifier not in base_dict_list:
        # Doesn't have an identifier, append all the dictionary objects from
        # target_dict_list to base_dict_list if base_dict_list doesn't have
        # them already
        for target_dict in target_dict_list:
            if target_dict not in base_dict_list:
                base_dict_list.append(target_dict)
        return

    # Sort both base_dict_list and target_dict_list by the identifier
    base_dict_list.sort(key=lambda k: k[identifier])
    target_dict_list.sort(key=lambda k: k[identifier])

    # Merge them
    base_counter = 0
    target_counter = 0
    max_items = len(base_dict_list) + len(target_dict_list)
    for _ in range(0, max_items):
        cur_base_dict = (None if len(base_dict_list) <= base_counter
                         else base_dict_list[base_counter])
        cur_taget_dict = (None if len(target_dict_list) <= target_counter
                          else target_dict_list[target_counter])

        if target_counter >= len(target_dict_list):
            break
        elif (base_counter >= len(base_dict_list) or
              cur_base_dict[identifier] > cur_taget_dict[identifier]):
            # cur_target_object only exists in target_dict_list,
            # add it to base_dict_list, increment target_counter
            base_dict_list.append(cur_taget_dict)
            target_counter += 1
        elif cur_base_dict[identifier] < cur_taget_dict[identifier]:
            # cur_base_dict object only exists in base_dict_list,
            # increment base_counter
            base_counter += 1
        else:
            # They have the same identifier, merge them,
            # increment both counters
            merge_dict(cur_base_dict, cur_taget_dict,
                       fields_to_ignore, field_identifiers)
            base_counter += 1
            target_counter += 1
<|endoftext|>


<commit_before>import pytest
from spacy.pipeline._parser_internals.nonproj import ancestors, contains_cycle
from spacy.pipeline._parser_internals.nonproj import is_nonproj_tree, is_nonproj_arc
from spacy.pipeline._parser_internals import nonproj
from spacy.tokens import Doc

@pytest.fixture
def tree():
    return [1, 2, 2, 4, 5, 2, 2]

@pytest.fixture
def cyclic_tree():
    return [1, 2, 2, 4, 5, 3, 2]

@pytest.fixture
def partial_tree():
    return [1, 2, 2, 4, 5, None, 7, 4, 2]

@pytest.fixture
def nonproj_tree():
    return [1, 2, 2, 4, 5, 2, 7, 4, 2]

@pytest.fixture
def proj_tree():
    return [1, 2, 2, 4, 5, 2, 7, 5, 2]

@pytest.fixture
def multirooted_tree():
    return [3, 2, 0, 3, 3, 7, 7, 3, 7, 10, 7, 10, 11, 12, 18, 16, 18, 17, 12, 3]

def test_parser_ancestors(tree, cyclic_tree, partial_tree, multirooted_tree):
    assert [a for a in ancestors(3, tree)] == [4, 5, 2]
    assert [a for a in ancestors(3, cyclic_tree)] == [4, 5, 3, 4, 5, 3, 4]
    assert [a for a in ancestors(3, partial_tree)] == [4, 5, None]
    assert [a for a in ancestors(17, multirooted_tree)] == []

def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree):
    assert contains_cycle(tree) is None
    assert contains_cycle(cyclic_tree) == {3, 4, 5}
    assert contains_cycle(partial_tree) is None
    assert contains_cycle(multirooted_tree) is None

def test_parser_is_nonproj_arc(cyclic_tree, nonproj_tree, partial_tree, multirooted_tree):
    assert is_nonproj_arc(0, nonproj_tree) is False
    assert is_nonproj_arc(1, nonproj_tree) is False
    assert is_nonproj_arc(2, nonproj_tree) is False
    assert is_nonproj_arc(3, nonproj_tree) is False
    assert is_nonproj_arc(4, nonproj_tree) is False
    assert is_nonproj_arc(5, nonproj_tree) is False
    assert is_nonproj_arc(6, nonproj_tree) is False
    assert is_nonproj_arc(7, nonproj_tree) is True
    assert is_nonproj_arc(8, nonproj_tree) is False
    assert is_nonproj_arc(7, partial_tree) is False
    assert is_nonproj_arc(17, multirooted_tree) is False
    assert is_nonproj_arc(16, multirooted_tree) is True
    with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
        is_nonproj_arc(6, cyclic_tree)

def test_parser_is_nonproj_tree(
    proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
):
    assert is_nonproj_tree(proj_tree) is False
    assert is_nonproj_tree(nonproj_tree) is True
    assert is_nonproj_tree(partial_tree) is False
    assert is_nonproj_tree(multirooted_tree) is True
    with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
        is_nonproj_tree(cyclic_tree)

def test_parser_pseudoprojectivity(en_vocab):
    def deprojectivize(proj_heads, deco_labels):
        words = ["whatever "] * len(proj_heads)
        doc = Doc(en_vocab, words=words, deps=deco_labels, heads=proj_heads)
        nonproj.deprojectivize(doc)
        return [t.head.i for t in doc], [token.dep_ for token in doc]

    # fmt: off
    tree = [1, 2, 2]
    nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2]
    nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
    cyclic_tree = [1, 2, 2, 4, 5, 3, 2]
    labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"]
    labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"]
    cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"]
    # fmt: on
    assert nonproj.decompose("X||Y") == ("X", "Y")
    assert nonproj.decompose("X") == ("X", "")
    assert nonproj.is_decorated("X||Y") is True
    assert nonproj.is_decorated("X") is False
    nonproj._lift(0, tree)
    assert tree == [2, 2, 2]
    assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree) == 7
    assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10
    # fmt: off
    proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels)
    with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
        nonproj.projectivize(cyclic_tree, cyclic_labels)
    assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2]
    assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
                           "nsubj", "acl||dobj", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == nonproj_tree
    assert undeco_labels == labels
    proj_heads, deco_labels = nonproj.projectivize(nonproj_tree2, labels2)
    assert proj_heads == [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1]
    assert deco_labels == ["advmod||aux", "root", "det", "nsubj", "advmod",
                           "det", "dobj", "det", "nmod", "aux", "nmod||dobj",
                           "advmod", "det", "amod", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == nonproj_tree2
    assert undeco_labels == labels2
    # if decoration is wrong such that there is no head with the desired label
    # the structure is kept and the label is undecorated
    proj_heads = [1, 2, 2, 4, 5, 2, 7, 5, 2]
    deco_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj",
                   "acl||iobj", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == proj_heads
    assert undeco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
                             "nsubj", "acl", "punct"]
    # if there are two potential new heads, the first one is chosen even if
    # it's wrong
    proj_heads = [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1]
    deco_labels = ["advmod||aux", "root", "det", "aux", "advmod", "det",
                   "dobj", "det", "nmod", "aux", "nmod||dobj", "advmod",
                   "det", "amod", "punct"]

    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == [3, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
    assert undeco_labels == ["advmod", "root", "det", "aux", "advmod", "det",
                             "dobj", "det", "nmod", "aux", "nmod", "advmod",
                             "det", "amod", "punct"]
    # fmt: on
<commit_msg>Auto-format code with black (#10945)<commit_after>import pytest
from spacy.pipeline._parser_internals.nonproj import ancestors, contains_cycle
from spacy.pipeline._parser_internals.nonproj import is_nonproj_tree, is_nonproj_arc
from spacy.pipeline._parser_internals import nonproj
from spacy.tokens import Doc

@pytest.fixture
def tree():
    return [1, 2, 2, 4, 5, 2, 2]

@pytest.fixture
def cyclic_tree():
    return [1, 2, 2, 4, 5, 3, 2]

@pytest.fixture
def partial_tree():
    return [1, 2, 2, 4, 5, None, 7, 4, 2]

@pytest.fixture
def nonproj_tree():
    return [1, 2, 2, 4, 5, 2, 7, 4, 2]

@pytest.fixture
def proj_tree():
    return [1, 2, 2, 4, 5, 2, 7, 5, 2]

@pytest.fixture
def multirooted_tree():
    return [3, 2, 0, 3, 3, 7, 7, 3, 7, 10, 7, 10, 11, 12, 18, 16, 18, 17, 12, 3]

def test_parser_ancestors(tree, cyclic_tree, partial_tree, multirooted_tree):
    assert [a for a in ancestors(3, tree)] == [4, 5, 2]
    assert [a for a in ancestors(3, cyclic_tree)] == [4, 5, 3, 4, 5, 3, 4]
    assert [a for a in ancestors(3, partial_tree)] == [4, 5, None]
    assert [a for a in ancestors(17, multirooted_tree)] == []

def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree):
    assert contains_cycle(tree) is None
    assert contains_cycle(cyclic_tree) == {3, 4, 5}
    assert contains_cycle(partial_tree) is None
    assert contains_cycle(multirooted_tree) is None

def test_parser_is_nonproj_arc(
    cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
):
    assert is_nonproj_arc(0, nonproj_tree) is False
    assert is_nonproj_arc(1, nonproj_tree) is False
    assert is_nonproj_arc(2, nonproj_tree) is False
    assert is_nonproj_arc(3, nonproj_tree) is False
    assert is_nonproj_arc(4, nonproj_tree) is False
    assert is_nonproj_arc(5, nonproj_tree) is False
    assert is_nonproj_arc(6, nonproj_tree) is False
    assert is_nonproj_arc(7, nonproj_tree) is True
    assert is_nonproj_arc(8, nonproj_tree) is False
    assert is_nonproj_arc(7, partial_tree) is False
    assert is_nonproj_arc(17, multirooted_tree) is False
    assert is_nonproj_arc(16, multirooted_tree) is True
    with pytest.raises(
        ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]"
    ):
        is_nonproj_arc(6, cyclic_tree)

def test_parser_is_nonproj_tree(
    proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree
):
    assert is_nonproj_tree(proj_tree) is False
    assert is_nonproj_tree(nonproj_tree) is True
    assert is_nonproj_tree(partial_tree) is False
    assert is_nonproj_tree(multirooted_tree) is True
    with pytest.raises(
        ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]"
    ):
        is_nonproj_tree(cyclic_tree)

def test_parser_pseudoprojectivity(en_vocab):
    def deprojectivize(proj_heads, deco_labels):
        words = ["whatever "] * len(proj_heads)
        doc = Doc(en_vocab, words=words, deps=deco_labels, heads=proj_heads)
        nonproj.deprojectivize(doc)
        return [t.head.i for t in doc], [token.dep_ for token in doc]

    # fmt: off
    tree = [1, 2, 2]
    nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2]
    nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
    cyclic_tree = [1, 2, 2, 4, 5, 3, 2]
    labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"]
    labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"]
    cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"]
    # fmt: on
    assert nonproj.decompose("X||Y") == ("X", "Y")
    assert nonproj.decompose("X") == ("X", "")
    assert nonproj.is_decorated("X||Y") is True
    assert nonproj.is_decorated("X") is False
    nonproj._lift(0, tree)
    assert tree == [2, 2, 2]
    assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree) == 7
    assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10
    # fmt: off
    proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels)
    with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'):
        nonproj.projectivize(cyclic_tree, cyclic_labels)
    assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2]
    assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
                           "nsubj", "acl||dobj", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == nonproj_tree
    assert undeco_labels == labels
    proj_heads, deco_labels = nonproj.projectivize(nonproj_tree2, labels2)
    assert proj_heads == [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1]
    assert deco_labels == ["advmod||aux", "root", "det", "nsubj", "advmod",
                           "det", "dobj", "det", "nmod", "aux", "nmod||dobj",
                           "advmod", "det", "amod", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == nonproj_tree2
    assert undeco_labels == labels2
    # if decoration is wrong such that there is no head with the desired label
    # the structure is kept and the label is undecorated
    proj_heads = [1, 2, 2, 4, 5, 2, 7, 5, 2]
    deco_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj",
                   "acl||iobj", "punct"]
    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == proj_heads
    assert undeco_labels == ["det", "nsubj", "root", "det", "dobj", "aux",
                             "nsubj", "acl", "punct"]
    # if there are two potential new heads, the first one is chosen even if
    # it's wrong
    proj_heads = [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1]
    deco_labels = ["advmod||aux", "root", "det", "aux", "advmod", "det",
                   "dobj", "det", "nmod", "aux", "nmod||dobj", "advmod",
                   "det", "amod", "punct"]

    deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels)
    assert deproj_heads == [3, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1]
    assert undeco_labels == ["advmod", "root", "det", "aux", "advmod", "det",
                             "dobj", "det", "nmod", "aux", "nmod", "advmod",
                             "det", "amod", "punct"]
    # fmt: on
<|endoftext|>


<commit_before>
# Establish the common DatabaseOperations instance, which we call 'db'.
# Much thanks to cmkmrr for a lot of the code base here

from django.conf import settings
import sys

# A few aliases, because there's FQMNs now
engine_modules = {
    'django.db.backends.postgresql_psycopg2': 'postgresql_psycopg2',
    'django.db.backends.sqlite3': 'sqlite3',
    'django.db.backends.mysql': 'mysql',
    'django.db.backends.oracle': 'oracle',
    'sql_server.pyodbc': 'sql_server.pyodbc', #django-pyodbc
    'sqlserver_ado': 'sql_server.pyodbc', #django-mssql
    'django.contrib.gis.db.backends.postgis': 'postgresql_psycopg2',
    'django.contrib.gis.db.backends.spatialite': 'sqlite3',
    'django.contrib.gis.db.backends.mysql': 'mysql',
    'django.contrib.gis.db.backends.oracle': 'oracle',
}

# First, work out if we're multi-db or not, and which databases we have
try:
    from django.db import DEFAULT_DB_ALIAS
except ImportError:
    #### 1.1 or below ####
    # We'll 'fake' multi-db; set the default alias
    DEFAULT_DB_ALIAS = 'default'
    # SOUTH_DATABASE_ADAPTER is an optional override if you have a different module
    engine = getattr(settings, "SOUTH_DATABASE_ADAPTER", "south.db.%s" % settings.DATABASE_ENGINE)
    # And then, we have one database with one engine
    db_engines = {DEFAULT_DB_ALIAS: engine}
else:
    #### 1.2 or above ####
    # Loop over the defined databases, gathering up their engines
    db_engines = dict([
        # Note we check to see if contrib.gis has overridden us.
        (alias, "south.db.%s" % engine_modules.get(db_settings['ENGINE'], None))
        for alias, db_settings in settings.DATABASES.items()
    ])
    # Update with any overrides
    db_engines.update(getattr(settings, "SOUTH_DATABASE_ADAPTERS", {}))
    # Check there's no None engines, or...
    for alias, engine in db_engines.items():
        if engine is None:
            # They've used a backend we don't support
            sys.stderr.write(
                (
                    "There is no South database module for your database backend '%s'. " + \
                    "Please either choose a supported database, check for " + \
                    "SOUTH_DATABASE_ADAPTER[S] settings, " + \
                    "or remove South from INSTALLED_APPS.\n"
                ) % (settings.DATABASES[alias]['ENGINE'],)
            )
            sys.exit(1)

# Now, turn that into a dict of <alias: south db module>
dbs = {}
try:
    for alias, module_name in db_engines.items():
        module = __import__(module_name, {}, {}, [''])
        dbs[alias] = module.DatabaseOperations(alias)
except ImportError:
    # This error should only be triggered on 1.1 and below.
    sys.stderr.write(
        (
            "There is no South database module '%s' for your database. " + \
            "Please either choose a supported database, check for " + \
            "SOUTH_DATABASE_ADAPTER[S] settings, " + \
            "or remove South from INSTALLED_APPS.\n"
        ) % (module_name,)
    )
    sys.exit(1)

# Finally, to make old migrations work, keep 'db' around as the default database
db = dbs[DEFAULT_DB_ALIAS]
<commit_msg>skip non-found adapters<commit_after>
# Establish the common DatabaseOperations instance, which we call 'db'.
# Much thanks to cmkmrr for a lot of the code base here

from django.conf import settings
import sys

# A few aliases, because there's FQMNs now
engine_modules = {
    'django.db.backends.postgresql_psycopg2': 'postgresql_psycopg2',
    'django.db.backends.sqlite3': 'sqlite3',
    'django.db.backends.mysql': 'mysql',
    'django.db.backends.oracle': 'oracle',
    'sql_server.pyodbc': 'sql_server.pyodbc', #django-pyodbc
    'sqlserver_ado': 'sql_server.pyodbc', #django-mssql
    'django.contrib.gis.db.backends.postgis': 'postgresql_psycopg2',
    'django.contrib.gis.db.backends.spatialite': 'sqlite3',
    'django.contrib.gis.db.backends.mysql': 'mysql',
    'django.contrib.gis.db.backends.oracle': 'oracle',
}

# First, work out if we're multi-db or not, and which databases we have
try:
    from django.db import DEFAULT_DB_ALIAS
except ImportError:
    #### 1.1 or below ####
    # We'll 'fake' multi-db; set the default alias
    DEFAULT_DB_ALIAS = 'default'
    # SOUTH_DATABASE_ADAPTER is an optional override if you have a different module
    engine = getattr(settings, "SOUTH_DATABASE_ADAPTER", "south.db.%s" % settings.DATABASE_ENGINE)
    # And then, we have one database with one engine
    db_engines = {DEFAULT_DB_ALIAS: engine}
else:
    #### 1.2 or above ####
    # Loop over the defined databases, gathering up their engines
    db_engines = dict([
        # Note we check to see if contrib.gis has overridden us.
        (alias, "south.db.%s" % engine_modules[db_settings['ENGINE']])
        for alias, db_settings in settings.DATABASES.items()
        if db_settings['ENGINE'] in engine_modules
    ])
    # Update with any overrides
    db_engines.update(getattr(settings, "SOUTH_DATABASE_ADAPTERS", {}))
    # Check there's no None engines, or...
    for alias, engine in db_engines.items():
        if engine is None:
            # They've used a backend we don't support
            sys.stderr.write(
                (
                    "There is no South database module for your database backend '%s'. " + \
                    "Please either choose a supported database, check for " + \
                    "SOUTH_DATABASE_ADAPTER[S] settings, " + \
                    "or remove South from INSTALLED_APPS.\n"
                ) % (settings.DATABASES[alias]['ENGINE'],)
            )
            sys.exit(1)

# Now, turn that into a dict of <alias: south db module>
dbs = {}
try:
    for alias, module_name in db_engines.items():
        module = __import__(module_name, {}, {}, [''])
        dbs[alias] = module.DatabaseOperations(alias)
except ImportError:
    # This error should only be triggered on 1.1 and below.
    sys.stderr.write(
        (
            "There is no South database module '%s' for your database. " + \
            "Please either choose a supported database, check for " + \
            "SOUTH_DATABASE_ADAPTER[S] settings, " + \
            "or remove South from INSTALLED_APPS.\n"
        ) % (module_name,)
    )
    sys.exit(1)

# Finally, to make old migrations work, keep 'db' around as the default database
db = dbs[DEFAULT_DB_ALIAS]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""upload_docs

Implements a Distutils 'upload_docs' subcommand (upload documentation to
PyPI's packages.python.org).
"""

import os
import socket
import zipfile
import httplib
import base64
import urlparse
import tempfile
import cStringIO as StringIO

from distutils import log
from distutils.errors import DistutilsOptionError
from distutils.command.upload import upload

class upload_docs(upload):

    description = 'Upload documentation to PyPI'

    user_options = [
        ('repository=', 'r',
         "url of repository [default: %s]" % upload.DEFAULT_REPOSITORY),
        ('show-response', None,
         'display full response text from server'),
        ('upload-dir=', None, 'directory to upload'),
        ]
    boolean_options = upload.boolean_options

    def initialize_options(self):
        upload.initialize_options(self)
        self.upload_dir = None

    def finalize_options(self):
        upload.finalize_options(self)
        if self.upload_dir is None:
            build = self.get_finalized_command('build')
            self.upload_dir = os.path.join(build.build_base, 'docs')
            self.mkpath(self.upload_dir)
        self.ensure_dirname('upload_dir')
        self.announce('Using upload directory %s' % self.upload_dir)

    def create_zipfile(self):
        name = self.distribution.metadata.get_name()
        tmp_dir = tempfile.mkdtemp()
        tmp_file = os.path.join(tmp_dir, "%s.zip" % name)
        zip_file = zipfile.ZipFile(tmp_file, "w")
        for root, dirs, files in os.walk(self.upload_dir):
            if root == self.upload_dir and not files:
                raise DistutilsOptionError(
                    "no files found in upload directory '%s'"
                    % self.upload_dir)
            for name in files:
                full = os.path.join(root, name)
                relative = root[len(self.upload_dir):].lstrip(os.path.sep)
                dest = os.path.join(relative, name)
                zip_file.write(full, dest)
        zip_file.close()
        return tmp_file

    def run(self):
        zip_file = self.create_zipfile()
        self.upload_file(zip_file)

    def upload_file(self, filename):
        content = open(filename, 'rb').read()
        meta = self.distribution.metadata
        data = {
            ':action': 'doc_upload',
            'name': meta.get_name(),
            'content': (os.path.basename(filename), content),
        }
        # set up the authentication
        auth = "Basic " + base64.encodestring(
            self.username + ":" + self.password).strip()

        # Build up the MIME payload for the POST data
        boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
        sep_boundary = '\n--' + boundary
        end_boundary = sep_boundary + '--'
        body = StringIO.StringIO()
        for key, value in data.items():
            # handle multiple entries for the same name
            if type(value) != type([]):
                value = [value]
            for value in value:
                if type(value) is tuple:
                    fn = ';filename="%s"' % value[0]
                    value = value[1]
                else:
                    fn = ""
                value = str(value)
                body.write(sep_boundary)
                body.write('\nContent-Disposition: form-data; name="%s"'%key)
                body.write(fn)
                body.write("\n\n")
                body.write(value)
                if value and value[-1] == '\r':
                    body.write('\n')  # write an extra newline (lurve Macs)
        body.write(end_boundary)
        body.write("\n")
        body = body.getvalue()

        self.announce("Submitting documentation to %s" % (self.repository),
                      log.INFO)

        # build the Request
        # We can't use urllib2 since we need to send the Basic
        # auth right with the first request
        schema, netloc, url, params, query, fragments = \
            urlparse.urlparse(self.repository)
        assert not params and not query and not fragments
        if schema == 'http':
            http = httplib.HTTPConnection(netloc)
        elif schema == 'https':
            http = httplib.HTTPSConnection(netloc)
        else:
            raise AssertionError("unsupported schema "+schema)

        data = ''
        loglevel = log.INFO
        try:
            http.connect()
            http.putrequest("POST", url)
            http.putheader('Content-type',
                           'multipart/form-data; boundary=%s'%boundary)
            http.putheader('Content-length', str(len(body)))
            http.putheader('Authorization', auth)
            http.endheaders()
            http.send(body)
        except socket.error, e:
            self.announce(str(e), log.ERROR)
            return

        r = http.getresponse()
        if r.status == 200:
            self.announce('Server response (%s): %s' % (r.status, r.reason),
                          log.INFO)
        elif r.status == 301:
            location = r.getheader('Location')
            if location is None:
                location = 'http://packages.python.org/%s/' % meta.get_name()
            self.announce('Upload successful. Visit %s' % location,
                          log.INFO)
        else:
            self.announce('Upload failed (%s): %s' % (r.status, r.reason),
                          log.ERROR)
        if self.show_response:
            print '-'*75, r.read(), '-'*75
<commit_msg>Change upload_docs so that its use of base64 does not fail under Python 3.<commit_after># -*- coding: utf-8 -*-
"""upload_docs

Implements a Distutils 'upload_docs' subcommand (upload documentation to
PyPI's packages.python.org).
"""

import os
import socket
import zipfile
import httplib
import base64
import urlparse
import tempfile
import cStringIO as StringIO

from distutils import log
from distutils.errors import DistutilsOptionError
from distutils.command.upload import upload

class upload_docs(upload):

    description = 'Upload documentation to PyPI'

    user_options = [
        ('repository=', 'r',
         "url of repository [default: %s]" % upload.DEFAULT_REPOSITORY),
        ('show-response', None,
         'display full response text from server'),
        ('upload-dir=', None, 'directory to upload'),
        ]
    boolean_options = upload.boolean_options

    def initialize_options(self):
        upload.initialize_options(self)
        self.upload_dir = None

    def finalize_options(self):
        upload.finalize_options(self)
        if self.upload_dir is None:
            build = self.get_finalized_command('build')
            self.upload_dir = os.path.join(build.build_base, 'docs')
            self.mkpath(self.upload_dir)
        self.ensure_dirname('upload_dir')
        self.announce('Using upload directory %s' % self.upload_dir)

    def create_zipfile(self):
        name = self.distribution.metadata.get_name()
        tmp_dir = tempfile.mkdtemp()
        tmp_file = os.path.join(tmp_dir, "%s.zip" % name)
        zip_file = zipfile.ZipFile(tmp_file, "w")
        for root, dirs, files in os.walk(self.upload_dir):
            if root == self.upload_dir and not files:
                raise DistutilsOptionError(
                    "no files found in upload directory '%s'"
                    % self.upload_dir)
            for name in files:
                full = os.path.join(root, name)
                relative = root[len(self.upload_dir):].lstrip(os.path.sep)
                dest = os.path.join(relative, name)
                zip_file.write(full, dest)
        zip_file.close()
        return tmp_file

    def run(self):
        zip_file = self.create_zipfile()
        self.upload_file(zip_file)

    def upload_file(self, filename):
        content = open(filename, 'rb').read()
        meta = self.distribution.metadata
        data = {
            ':action': 'doc_upload',
            'name': meta.get_name(),
            'content': (os.path.basename(filename), content),
        }
        # set up the authentication
        credentials = self.username + ':' + self.password
        try:  # base64 only works with bytes in Python 3.
            encoded_creds = base64.encodebytes(credentials.encode('utf8'))
            auth = b"Basic"
        except AttributeError:
            encoded_creds = base64.encodestring(credentials)
            auth = "Basic"
        auth += encoded_creds.strip()

        # Build up the MIME payload for the POST data
        boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
        sep_boundary = '\n--' + boundary
        end_boundary = sep_boundary + '--'
        body = StringIO.StringIO()
        for key, value in data.items():
            # handle multiple entries for the same name
            if type(value) != type([]):
                value = [value]
            for value in value:
                if type(value) is tuple:
                    fn = ';filename="%s"' % value[0]
                    value = value[1]
                else:
                    fn = ""
                value = str(value)
                body.write(sep_boundary)
                body.write('\nContent-Disposition: form-data; name="%s"'%key)
                body.write(fn)
                body.write("\n\n")
                body.write(value)
                if value and value[-1] == '\r':
                    body.write('\n')  # write an extra newline (lurve Macs)
        body.write(end_boundary)
        body.write("\n")
        body = body.getvalue()

        self.announce("Submitting documentation to %s" % (self.repository),
                      log.INFO)

        # build the Request
        # We can't use urllib2 since we need to send the Basic
        # auth right with the first request
        schema, netloc, url, params, query, fragments = \
            urlparse.urlparse(self.repository)
        assert not params and not query and not fragments
        if schema == 'http':
            http = httplib.HTTPConnection(netloc)
        elif schema == 'https':
            http = httplib.HTTPSConnection(netloc)
        else:
            raise AssertionError("unsupported schema "+schema)

        data = ''
        loglevel = log.INFO
        try:
            http.connect()
            http.putrequest("POST", url)
            http.putheader('Content-type',
                           'multipart/form-data; boundary=%s'%boundary)
            http.putheader('Content-length', str(len(body)))
            http.putheader('Authorization', auth)
            http.endheaders()
            http.send(body)
        except socket.error, e:
            self.announce(str(e), log.ERROR)
            return

        r = http.getresponse()
        if r.status == 200:
            self.announce('Server response (%s): %s' % (r.status, r.reason),
                          log.INFO)
        elif r.status == 301:
            location = r.getheader('Location')
            if location is None:
                location = 'http://packages.python.org/%s/' % meta.get_name()
            self.announce('Upload successful. Visit %s' % location,
                          log.INFO)
        else:
            self.announce('Upload failed (%s): %s' % (r.status, r.reason),
                          log.ERROR)
        if self.show_response:
            print '-'*75, r.read(), '-'*75
<|endoftext|>


<commit_before># firebird

import datetime

from django.db import connection, models
from django.core.management.color import no_style
from django.db.utils import DatabaseError

from south.db import generic

class DatabaseOperations(generic.DatabaseOperations):
    backend_name = 'firebird'
    alter_string_set_type = 'ALTER %(column)s TYPE %(type)s'
    alter_string_set_default =  'ALTER %(column)s SET DEFAULT %(default)s;'
    alter_string_drop_null = ''
    add_column_string = 'ALTER TABLE %s ADD %s;'
    delete_column_string = 'ALTER TABLE %s DROP %s;'
    allows_combined_alters = False

    def _fill_constraint_cache(self, db_name, table_name):
        self._constraint_cache.setdefault(db_name, {})
        self._constraint_cache[db_name][table_name] = {}

        rows = self.execute("""
            SELECT
                rc.RDB$CONSTRAINT_NAME,
                rc.RDB$CONSTRAINT_TYPE,
                cc.RDB$TRIGGER_NAME
            FROM rdb$relation_constraints rc
            JOIN rdb$check_constraints cc
            ON rc.rdb$constraint_name = cc.rdb$constraint_name
            WHERE rc.rdb$constraint_type = 'NOT NULL'
            AND rc.rdb$relation_name = '%s'
            """ % table_name)

        for constraint, kind, column in rows:
           self._constraint_cache[db_name][table_name].setdefault(column, set())
           self._constraint_cache[db_name][table_name][column].add((kind, constraint))
        return

    def _alter_column_set_null(self, table_name, column_name, is_null):
        sql = """
            UPDATE RDB$RELATION_FIELDS SET RDB$NULL_FLAG = %(null_flag)s
            WHERE RDB$FIELD_NAME = '%(column)s'
            AND RDB$RELATION_NAME = '%(table_name)s'
        """
        null_flag = 'NULL' if is_null else '1'
        return sql % {
            'null_flag': null_flag,
            'column': column_name.upper(),
            'table_name': table_name.upper()
        }

    def _column_has_default(self, params):
        sql = """
            SELECT a.RDB$DEFAULT_VALUE
            FROM RDB$RELATION_FIELDS a
            WHERE a.RDB$FIELD_NAME = '%(column)s'
            AND a.RDB$RELATION_NAME = '%(table_name)s'
        """
        value = self.execute(sql % params)
        return True if value else False

    def _alter_set_defaults(self, field, name, params, sqls):
        "Subcommand of alter_column that sets default values (overrideable)"
        # Next, set any default
        if not field.null and field.has_default():
            default = field.get_default()
            sqls.append(('ALTER COLUMN %s SET DEFAULT %%s ' % (self.quote_name(name),), [default]))
        elif self._column_has_default(params):
            sqls.append(('ALTER COLUMN %s DROP DEFAULT' % (self.quote_name(name),), []))

    @generic.invalidate_table_constraints
    def create_table(self, table_name, fields):
        qn = self.quote_name(table_name)
        columns = []
        autoinc_sql = ''

        for field_name, field in fields:
            col = self.column_sql(table_name, field_name, field)
            if not col:
                continue
            #col = self.adj_column_sql(col)

            columns.append(col)
            if isinstance(field, models.AutoField):
                autoinc_sql = connection.ops.autoinc_sql(table_name, field_name)

        sql = 'CREATE TABLE %s (%s);' % (qn, ', '.join([col for col in columns]))
        self.execute(sql)
        if autoinc_sql:
            self.execute(autoinc_sql[0])
            self.execute(autoinc_sql[1])

    def column_sql(self, table_name, field_name, field, tablespace='', with_name=True, field_prepared=False):
        """
        Creates the SQL snippet for a column. Used by add_column and add_table.
        """

        # If the field hasn't already been told its attribute name, do so.
        if not field_prepared:
            field.set_attributes_from_name(field_name)

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()

        # Possible hook to fiddle with the fields (e.g. defaults & TEXT on MySQL)
        field = self._field_sanity(field)

        try:
            sql = field.db_type(connection=self._get_connection())
        except TypeError:
            sql = field.db_type()

        if sql:
            # Some callers, like the sqlite stuff, just want the extended type.
            if with_name:
                field_output = [self.quote_name(field.column), sql]
            else:
                field_output = [sql]

            if field.primary_key:
                field_output.append('NOT NULL PRIMARY KEY')
            elif field.unique:
                # Just use UNIQUE (no indexes any more, we have delete_unique)
                field_output.append('UNIQUE')

            sql = ' '.join(field_output)
            sqlparams = ()

            # if the field is "NOT NULL" and a default value is provided, create the column with it
            # this allows the addition of a NOT NULL field to a table with existing rows
            if not getattr(field, '_suppress_default', False):
                if field.has_default():
                    default = field.get_default()
                    # If the default is actually None, don't add a default term
                    if default is not None:
                        # If the default is a callable, then call it!
                        if callable(default):
                            default = default()
                        # Now do some very cheap quoting. TODO: Redesign return values to avoid this.
                        if isinstance(default, basestring):
                            default = "'%s'" % default.replace("'", "''")
                        elif isinstance(default, (datetime.date, datetime.time, datetime.datetime)):
                            default = "'%s'" % default
                        # Escape any % signs in the output (bug #317)
                        if isinstance(default, basestring):
                            default = default.replace("%", "%%")
                        # Add it in
                        sql += " DEFAULT %s"
                        sqlparams = (default)
                elif (not field.null and field.blank) or (field.get_default() == ''):
                    if field.empty_strings_allowed and self._get_connection().features.interprets_empty_strings_as_nulls:
                        sql += " DEFAULT ''"
                    # Error here would be nice, but doesn't seem to play fair.
                    #else:
                    #    raise ValueError("Attempting to add a non null column that isn't character based without an explicit default value.")

            # Firebird need set not null after of default value keyword
            if not field.primary_key and not field.null:
                sql += ' NOT NULL'

            if field.rel and self.supports_foreign_keys:
                self.add_deferred_sql(
                    self.foreign_key_sql(
                        table_name,
                        field.column,
                        field.rel.to._meta.db_table,
                        field.rel.to._meta.get_field(field.rel.field_name).column
                    )
                )

        # Things like the contrib.gis module fields have this in 1.1 and below
        if hasattr(field, 'post_create_sql'):
            for stmt in field.post_create_sql(no_style(), table_name):
                self.add_deferred_sql(stmt)

        # In 1.2 and above, you have to ask the DatabaseCreation stuff for it.
        # This also creates normal indexes in 1.1.
        if hasattr(self._get_connection().creation, "sql_indexes_for_field"):
            # Make a fake model to pass in, with only db_table
            model = self.mock_model("FakeModelForGISCreation", table_name)
            for stmt in self._get_connection().creation.sql_indexes_for_field(model, field, no_style()):
                self.add_deferred_sql(stmt)

        if sql:
            return sql % sqlparams
        else:
            return None

    def _drop_constraints(self, table_name, name, field):
        if self.has_check_constraints:
            check_constraints = self._constraints_affecting_columns(table_name, [name], "CHECK")
            for constraint in check_constraints:
                self.execute(self.delete_check_sql % {
                    'table': self.quote_name(table_name),
                    'constraint': self.quote_name(constraint),
                })

        # Drop or add UNIQUE constraint
        unique_constraint = list(self._constraints_affecting_columns(table_name, [name], "UNIQUE"))
        if field.unique and not unique_constraint:
            self.create_unique(table_name, [name])
        elif not field.unique and unique_constraint:
            self.delete_unique(table_name, [name])

        # Drop all foreign key constraints
        try:
            self.delete_foreign_key(table_name, name)
        except ValueError:
            # There weren't any
            pass

    @generic.invalidate_table_constraints
    def alter_column(self, table_name, name, field, explicit_name=True, ignore_constraints=False):
        """
        Alters the given column name so it will match the given field.
        Note that conversion between the two by the database must be possible.
        Will not automatically add _id by default; to have this behavour, pass
        explicit_name=False.

        @param table_name: The name of the table to add the column to
        @param name: The name of the column to alter
        @param field: The new field definition to use
        """

        if self.dry_run:
            return

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()

        # Add _id or whatever if we need to
        field.set_attributes_from_name(name)
        if not explicit_name:
            name = field.column
        else:
            field.column = name

        if not ignore_constraints:
            # Drop all check constraints. Note that constraints will be added back
            # with self.alter_string_set_type and self.alter_string_drop_null.
            self._drop_constraints(table_name, name, field)

        # First, change the type
        params = {
            "column": self.quote_name(name),
            "type": self._db_type_for_alter_column(field),
            "table_name": table_name
        }

        # SQLs is a list of (SQL, values) pairs.
        sqls = []
        sqls_extra = []

        # Only alter the column if it has a type (Geometry ones sometimes don't)
        if params["type"] is not None:
            sqls.append((self.alter_string_set_type % params, []))

        # Add any field- and backend- specific modifications
        self._alter_add_column_mods(field, name, params, sqls)

        # Next, nullity: modified, firebird doesn't support DROP NOT NULL
        sqls_extra.append(self._alter_column_set_null(table_name, name, field.null))

        # Next, set any default
        self._alter_set_defaults(field, name, params, sqls)

        # Finally, actually change the column
        if self.allows_combined_alters:
            sqls, values = zip(*sqls)
            self.execute(
                "ALTER TABLE %s %s;" % (self.quote_name(table_name), ", ".join(sqls)),
                generic.flatten(values),
            )
        else:
            # Databases like e.g. MySQL don't like more than one alter at once.
            for sql, values in sqls:
                try:
                    self.execute("ALTER TABLE %s %s;" % (self.quote_name(table_name), sql), values)
                except DatabaseError as e:
                    print e

        # Execute extra sql, which don't need ALTER TABLE statement
        for sql in sqls_extra:
            self.execute(sql)

        if not ignore_constraints:
            # Add back FK constraints if needed
            if field.rel and self.supports_foreign_keys:
                self.execute(
                    self.foreign_key_sql(
                        table_name,
                        field.column,
                        field.rel.to._meta.db_table,
                        field.rel.to._meta.get_field(field.rel.field_name).column
                    )
                )

    @generic.copy_column_constraints
    @generic.delete_column_constraints
    def rename_column(self, table_name, old, new):
        if old == new:
            # Short-circuit out
            return []

        self.execute('ALTER TABLE %s ALTER %s TO %s;' % (
            self.quote_name(table_name),
            self.quote_name(old),
            self.quote_name(new),
        ))
<commit_msg>Fix exception handling for python2.5 compatibility<commit_after># firebird

import datetime

from django.db import connection, models
from django.core.management.color import no_style
from django.db.utils import DatabaseError

from south.db import generic

class DatabaseOperations(generic.DatabaseOperations):
    backend_name = 'firebird'
    alter_string_set_type = 'ALTER %(column)s TYPE %(type)s'
    alter_string_set_default =  'ALTER %(column)s SET DEFAULT %(default)s;'
    alter_string_drop_null = ''
    add_column_string = 'ALTER TABLE %s ADD %s;'
    delete_column_string = 'ALTER TABLE %s DROP %s;'
    allows_combined_alters = False

    def _fill_constraint_cache(self, db_name, table_name):
        self._constraint_cache.setdefault(db_name, {})
        self._constraint_cache[db_name][table_name] = {}

        rows = self.execute("""
            SELECT
                rc.RDB$CONSTRAINT_NAME,
                rc.RDB$CONSTRAINT_TYPE,
                cc.RDB$TRIGGER_NAME
            FROM rdb$relation_constraints rc
            JOIN rdb$check_constraints cc
            ON rc.rdb$constraint_name = cc.rdb$constraint_name
            WHERE rc.rdb$constraint_type = 'NOT NULL'
            AND rc.rdb$relation_name = '%s'
            """ % table_name)

        for constraint, kind, column in rows:
           self._constraint_cache[db_name][table_name].setdefault(column, set())
           self._constraint_cache[db_name][table_name][column].add((kind, constraint))
        return

    def _alter_column_set_null(self, table_name, column_name, is_null):
        sql = """
            UPDATE RDB$RELATION_FIELDS SET RDB$NULL_FLAG = %(null_flag)s
            WHERE RDB$FIELD_NAME = '%(column)s'
            AND RDB$RELATION_NAME = '%(table_name)s'
        """
        null_flag = 'NULL' if is_null else '1'
        return sql % {
            'null_flag': null_flag,
            'column': column_name.upper(),
            'table_name': table_name.upper()
        }

    def _column_has_default(self, params):
        sql = """
            SELECT a.RDB$DEFAULT_VALUE
            FROM RDB$RELATION_FIELDS a
            WHERE a.RDB$FIELD_NAME = '%(column)s'
            AND a.RDB$RELATION_NAME = '%(table_name)s'
        """
        value = self.execute(sql % params)
        return True if value else False

    def _alter_set_defaults(self, field, name, params, sqls):
        "Subcommand of alter_column that sets default values (overrideable)"
        # Next, set any default
        if not field.null and field.has_default():
            default = field.get_default()
            sqls.append(('ALTER COLUMN %s SET DEFAULT %%s ' % (self.quote_name(name),), [default]))
        elif self._column_has_default(params):
            sqls.append(('ALTER COLUMN %s DROP DEFAULT' % (self.quote_name(name),), []))

    @generic.invalidate_table_constraints
    def create_table(self, table_name, fields):
        qn = self.quote_name(table_name)
        columns = []
        autoinc_sql = ''

        for field_name, field in fields:
            col = self.column_sql(table_name, field_name, field)
            if not col:
                continue
            #col = self.adj_column_sql(col)

            columns.append(col)
            if isinstance(field, models.AutoField):
                autoinc_sql = connection.ops.autoinc_sql(table_name, field_name)

        sql = 'CREATE TABLE %s (%s);' % (qn, ', '.join([col for col in columns]))
        self.execute(sql)
        if autoinc_sql:
            self.execute(autoinc_sql[0])
            self.execute(autoinc_sql[1])

    def column_sql(self, table_name, field_name, field, tablespace='', with_name=True, field_prepared=False):
        """
        Creates the SQL snippet for a column. Used by add_column and add_table.
        """

        # If the field hasn't already been told its attribute name, do so.
        if not field_prepared:
            field.set_attributes_from_name(field_name)

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()

        # Possible hook to fiddle with the fields (e.g. defaults & TEXT on MySQL)
        field = self._field_sanity(field)

        try:
            sql = field.db_type(connection=self._get_connection())
        except TypeError:
            sql = field.db_type()

        if sql:
            # Some callers, like the sqlite stuff, just want the extended type.
            if with_name:
                field_output = [self.quote_name(field.column), sql]
            else:
                field_output = [sql]

            if field.primary_key:
                field_output.append('NOT NULL PRIMARY KEY')
            elif field.unique:
                # Just use UNIQUE (no indexes any more, we have delete_unique)
                field_output.append('UNIQUE')

            sql = ' '.join(field_output)
            sqlparams = ()

            # if the field is "NOT NULL" and a default value is provided, create the column with it
            # this allows the addition of a NOT NULL field to a table with existing rows
            if not getattr(field, '_suppress_default', False):
                if field.has_default():
                    default = field.get_default()
                    # If the default is actually None, don't add a default term
                    if default is not None:
                        # If the default is a callable, then call it!
                        if callable(default):
                            default = default()
                        # Now do some very cheap quoting. TODO: Redesign return values to avoid this.
                        if isinstance(default, basestring):
                            default = "'%s'" % default.replace("'", "''")
                        elif isinstance(default, (datetime.date, datetime.time, datetime.datetime)):
                            default = "'%s'" % default
                        # Escape any % signs in the output (bug #317)
                        if isinstance(default, basestring):
                            default = default.replace("%", "%%")
                        # Add it in
                        sql += " DEFAULT %s"
                        sqlparams = (default)
                elif (not field.null and field.blank) or (field.get_default() == ''):
                    if field.empty_strings_allowed and self._get_connection().features.interprets_empty_strings_as_nulls:
                        sql += " DEFAULT ''"
                    # Error here would be nice, but doesn't seem to play fair.
                    #else:
                    #    raise ValueError("Attempting to add a non null column that isn't character based without an explicit default value.")

            # Firebird need set not null after of default value keyword
            if not field.primary_key and not field.null:
                sql += ' NOT NULL'

            if field.rel and self.supports_foreign_keys:
                self.add_deferred_sql(
                    self.foreign_key_sql(
                        table_name,
                        field.column,
                        field.rel.to._meta.db_table,
                        field.rel.to._meta.get_field(field.rel.field_name).column
                    )
                )

        # Things like the contrib.gis module fields have this in 1.1 and below
        if hasattr(field, 'post_create_sql'):
            for stmt in field.post_create_sql(no_style(), table_name):
                self.add_deferred_sql(stmt)

        # In 1.2 and above, you have to ask the DatabaseCreation stuff for it.
        # This also creates normal indexes in 1.1.
        if hasattr(self._get_connection().creation, "sql_indexes_for_field"):
            # Make a fake model to pass in, with only db_table
            model = self.mock_model("FakeModelForGISCreation", table_name)
            for stmt in self._get_connection().creation.sql_indexes_for_field(model, field, no_style()):
                self.add_deferred_sql(stmt)

        if sql:
            return sql % sqlparams
        else:
            return None

    def _drop_constraints(self, table_name, name, field):
        if self.has_check_constraints:
            check_constraints = self._constraints_affecting_columns(table_name, [name], "CHECK")
            for constraint in check_constraints:
                self.execute(self.delete_check_sql % {
                    'table': self.quote_name(table_name),
                    'constraint': self.quote_name(constraint),
                })

        # Drop or add UNIQUE constraint
        unique_constraint = list(self._constraints_affecting_columns(table_name, [name], "UNIQUE"))
        if field.unique and not unique_constraint:
            self.create_unique(table_name, [name])
        elif not field.unique and unique_constraint:
            self.delete_unique(table_name, [name])

        # Drop all foreign key constraints
        try:
            self.delete_foreign_key(table_name, name)
        except ValueError:
            # There weren't any
            pass

    @generic.invalidate_table_constraints
    def alter_column(self, table_name, name, field, explicit_name=True, ignore_constraints=False):
        """
        Alters the given column name so it will match the given field.
        Note that conversion between the two by the database must be possible.
        Will not automatically add _id by default; to have this behavour, pass
        explicit_name=False.

        @param table_name: The name of the table to add the column to
        @param name: The name of the column to alter
        @param field: The new field definition to use
        """

        if self.dry_run:
            return

        # hook for the field to do any resolution prior to it's attributes being queried
        if hasattr(field, 'south_init'):
            field.south_init()

        # Add _id or whatever if we need to
        field.set_attributes_from_name(name)
        if not explicit_name:
            name = field.column
        else:
            field.column = name

        if not ignore_constraints:
            # Drop all check constraints. Note that constraints will be added back
            # with self.alter_string_set_type and self.alter_string_drop_null.
            self._drop_constraints(table_name, name, field)

        # First, change the type
        params = {
            "column": self.quote_name(name),
            "type": self._db_type_for_alter_column(field),
            "table_name": table_name
        }

        # SQLs is a list of (SQL, values) pairs.
        sqls = []
        sqls_extra = []

        # Only alter the column if it has a type (Geometry ones sometimes don't)
        if params["type"] is not None:
            sqls.append((self.alter_string_set_type % params, []))

        # Add any field- and backend- specific modifications
        self._alter_add_column_mods(field, name, params, sqls)

        # Next, nullity: modified, firebird doesn't support DROP NOT NULL
        sqls_extra.append(self._alter_column_set_null(table_name, name, field.null))

        # Next, set any default
        self._alter_set_defaults(field, name, params, sqls)

        # Finally, actually change the column
        if self.allows_combined_alters:
            sqls, values = zip(*sqls)
            self.execute(
                "ALTER TABLE %s %s;" % (self.quote_name(table_name), ", ".join(sqls)),
                generic.flatten(values),
            )
        else:
            # Databases like e.g. MySQL don't like more than one alter at once.
            for sql, values in sqls:
                try:
                    self.execute("ALTER TABLE %s %s;" % (self.quote_name(table_name), sql), values)
                except DatabaseError, e:
                    print e

        # Execute extra sql, which don't need ALTER TABLE statement
        for sql in sqls_extra:
            self.execute(sql)

        if not ignore_constraints:
            # Add back FK constraints if needed
            if field.rel and self.supports_foreign_keys:
                self.execute(
                    self.foreign_key_sql(
                        table_name,
                        field.column,
                        field.rel.to._meta.db_table,
                        field.rel.to._meta.get_field(field.rel.field_name).column
                    )
                )

    @generic.copy_column_constraints
    @generic.delete_column_constraints
    def rename_column(self, table_name, old, new):
        if old == new:
            # Short-circuit out
            return []

        self.execute('ALTER TABLE %s ALTER %s TO %s;' % (
            self.quote_name(table_name),
            self.quote_name(old),
            self.quote_name(new),
        ))
<|endoftext|>


<commit_before>'''
Created on Jun 18, 2012

@author: Sjoerd
'''

from django.contrib import admin
from django import forms
from django.db import models

from models import ComicSite,Page

class PageAdminForm(forms.ModelForm):
    move = forms.CharField(widget=forms.Select)
    move.required = False
    move.widget.choices=(
                         (models.BLANK_CHOICE_DASH[0]),
                         ('FIRST', 'First'),
                         ('UP', 'Up'),
                         ('DOWN', 'Down'),
                         ('LAST', 'Last'),
                        )
    class Meta:
        model = Page

class PageAdmin(admin.ModelAdmin):
    """Define the admin interface for pages"""
    form = PageAdminForm

    def save_model(self, request, obj, form, change):
        obj.save()
        move = form.cleaned_data['move']
        obj.move(move)

    #Show these page params in admin overview list
    list_display = ('title','ComicSite','order')

admin.site.register(ComicSite)
admin.site.register(Page,PageAdmin)

<commit_msg>ComicSiteAdmin is now based on guardian. This allows permissions to be set for each page individually in admin panel<commit_after>'''
Created on Jun 18, 2012

@author: Sjoerd
'''

from django.contrib import admin
from django import forms
from django.db import models

from guardian.admin import GuardedModelAdmin
from models import ComicSite,Page

class PageAdminForm(forms.ModelForm):
    move = forms.CharField(widget=forms.Select)
    move.required = False
    move.widget.choices=(
                         (models.BLANK_CHOICE_DASH[0]),
                         ('FIRST', 'First'),
                         ('UP', 'Up'),
                         ('DOWN', 'Down'),
                         ('LAST', 'Last'),
                        )
    class Meta:
        model = Page

class PageAdmin(GuardedModelAdmin):
    """Define the admin interface for pages"""
    form = PageAdminForm

    def save_model(self, request, obj, form, change):
        obj.save()
        move = form.cleaned_data['move']
        obj.move(move)

    #Show these page params in admin overview list
    list_display = ('title','ComicSite','order')

admin.site.register(ComicSite)
admin.site.register(Page,PageAdmin)

<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
queued python client
Author: Tim Henderson
Email: tadh@case.edu
Copyright 2013 All Right Reserved

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.

 * Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

 * Neither the name of the queued nor the names of its contributors may be
   used to endorse or promote products derived from this software without
   specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
'''

import sys, os, socket, threading, time
from collections import deque

class Queue(object):

    def __init__(self, host, port, debug=False):
        self.host = host
        self.port = port
        self.debug = debug
        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.conn.connect((self.host, self.port))
        self.queue_lock = threading.Lock()
        self.lines = deque()
        self.lines_cv = threading.Condition()
        self.read_thread = threading.Thread(target=self.listen)
        self.read_thread.daemon = True
        self.read_thread.start()

    def close(self):
        with self.queue_lock:
            self.conn.shutdown(socket.SHUT_RDWR)
            self.read_thread.join()
            if self.debug:
                print >>sys.stderr, "closed"

    def enque(self, data):
        with self.queue_lock:
            if self.debug:
                print >>sys.stderr, "queuing"
            msg = "ENQUE " + data.encode('base64').replace('\n', '') + '\n'
            self.conn.send(msg)
            if self.debug:
                print >>sys.stderr, "wait for response"
            self.get_enque_response()
            if self.debug:
                print >>sys.stderr, "recieved OK"

    def get_enque_response(self):
        cmd, data = self.get_line()
        if cmd == "ERROR":
            raise Exception(data)
        elif cmd != "OK":
            raise Exception, "bad command recieved %s" % cmd

    def deque(self):
        with self.queue_lock:
            self.conn.send("DEQUE\n")
            return self.get_deque_response()

    def get_deque_response(self):
        cmd, data = self.get_line()
        if cmd == "ERROR":
            if data == "queue is empty":
                raise IndexError(data)
            raise Exception(data)
        elif cmd != "ITEM":
            raise Exception, "bad command recieved %s" % cmd
        assert cmd == "ITEM"
        return data

    def listen(self):
        chunk = ''
        while True:
            while "\n" not in chunk:
                try:
                    data = self.conn.recv(4096*4)
                    if not data: return
                    chunk += data
                except Exception, e:
                    print >>sys.stderr, e
                    return
            line, chunk = chunk.split('\n', 1)
            with self.lines_cv:
                self.lines.append(line)
                self.lines_cv.notify()

    def get_line(self):
        with self.lines_cv:
            while len(self.lines) <= 0:
                self.lines_cv.wait()
            line = self.lines.popleft()
        return self.process_line(line)

    def process_line(self, line):
        split = line.split(' ', 1)
        command = split[0]
        rest = None
        if len(split) > 1:
            rest = split[1].decode('base64')

        return command, rest

def _loop(queue):
    while True:
        try:
            line = raw_input('> ')
        except:
            break
        split = line.split(' ', 1)
        command = split[0]
        data = None
        if len(split) > 1:
            data = split[1]
        if command == 'enque' and data is not None:
            queue.enque(data)
        elif command == 'enque' and data is None:
            print >>sys.stderr, "bad input, need data"
        elif command == "deque":
            try:
                print queue.deque()
            except IndexError:
                print "queue is empty"

def main():
    queue = Queue('localhost', 9001)
    try:
        _loop(queue)
    finally:
        queue.close()

if __name__ == '__main__':
    main()

<commit_msg>removed some debug statements from py-client<commit_after>#!/usr/bin/env python

'''
queued python client
Author: Tim Henderson
Email: tadh@case.edu
Copyright 2013 All Right Reserved

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

 * Redistributions of source code must retain the above copyright notice,
   this list of conditions and the following disclaimer.

 * Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

 * Neither the name of the queued nor the names of its contributors may be
   used to endorse or promote products derived from this software without
   specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
'''

import sys, os, socket, threading, time
from collections import deque

class Queue(object):

    def __init__(self, host, port, debug=False):
        self.host = host
        self.port = port
        self.debug = debug
        self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.conn.connect((self.host, self.port))
        self.queue_lock = threading.Lock()
        self.lines = deque()
        self.lines_cv = threading.Condition()
        self.read_thread = threading.Thread(target=self.listen)
        self.read_thread.daemon = True
        self.read_thread.start()

    def close(self):
        with self.queue_lock:
            self.conn.shutdown(socket.SHUT_RDWR)
            self.read_thread.join()
            if self.debug:
                print >>sys.stderr, "closed"

    def enque(self, data):
        with self.queue_lock:
            msg = "ENQUE " + data.encode('base64').replace('\n', '') + '\n'
            self.conn.send(msg)
            self.get_enque_response()

    def get_enque_response(self):
        cmd, data = self.get_line()
        if cmd == "ERROR":
            raise Exception(data)
        elif cmd != "OK":
            raise Exception, "bad command recieved %s" % cmd

    def deque(self):
        with self.queue_lock:
            self.conn.send("DEQUE\n")
            return self.get_deque_response()

    def get_deque_response(self):
        cmd, data = self.get_line()
        if cmd == "ERROR":
            if data == "queue is empty":
                raise IndexError(data)
            raise Exception(data)
        elif cmd != "ITEM":
            raise Exception, "bad command recieved %s" % cmd
        assert cmd == "ITEM"
        return data

    def listen(self):
        chunk = ''
        while True:
            while "\n" not in chunk:
                try:
                    data = self.conn.recv(4096*4)
                    if not data: return
                    chunk += data
                except Exception, e:
                    print >>sys.stderr, e
                    return
            line, chunk = chunk.split('\n', 1)
            with self.lines_cv:
                self.lines.append(line)
                self.lines_cv.notify()

    def get_line(self):
        with self.lines_cv:
            while len(self.lines) <= 0:
                self.lines_cv.wait()
            line = self.lines.popleft()
        return self.process_line(line)

    def process_line(self, line):
        split = line.split(' ', 1)
        command = split[0]
        rest = None
        if len(split) > 1:
            rest = split[1].decode('base64')

        return command, rest

def _loop(queue):
    while True:
        try:
            line = raw_input('> ')
        except:
            break
        split = line.split(' ', 1)
        command = split[0]
        data = None
        if len(split) > 1:
            data = split[1]
        if command == 'enque' and data is not None:
            queue.enque(data)
        elif command == 'enque' and data is None:
            print >>sys.stderr, "bad input, need data"
        elif command == "deque":
            try:
                print queue.deque()
            except IndexError:
                print "queue is empty"

def main():
    queue = Queue('localhost', 9001)
    try:
        _loop(queue)
    finally:
        queue.close()

if __name__ == '__main__':
    main()

<|endoftext|>