<commit_before>import logging
import re
import sys

# Tested on Azure 2.0 rc5 API
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import (
    ComputeManagementClient,
    )
from azure.mgmt.resource.subscriptions import SubscriptionClient
from msrestazure.azure_exceptions import CloudError
from simplestreams.json2streams import (
    dict_to_item,
    Item,
    )
from simplestreams import mirrors
from simplestreams import util

CANONICAL = 'Canonical'
MS_VSTUDIO = 'MicrosoftVisualStudio'
MS_SERVER = 'MicrosoftWindowsServer'
UBUNTU_SERVER = 'UbuntuServer'
WINDOWS = 'Windows'
WINDOWS_SERVER = 'WindowsServer'
IMAGE_SPEC = [
    ('win81', MS_VSTUDIO, WINDOWS, '8.1-Enterprise-N'),
    ('win10', MS_VSTUDIO, WINDOWS, '10-Enterprise-N'),
    ('win2012', MS_SERVER, WINDOWS_SERVER, '2012-Datacenter'),
    ('win2012r2', MS_SERVER, WINDOWS_SERVER, '2012-R2-Datacenter'),
    ('centos7', 'OpenLogic', 'CentOS', '7.1'),
]

ITEM_NAMES = {
    "australiaeast": "auee1i3",
    "australiasoutheast": "ause1i3",
    "brazilsouth": "brss1i3",
    "canadacentral": "cacc1i3",
    "canadaeast": "caee1i3",
    "centralindia": "incc1i3",
    "centralus": "uscc1i3",
    "chinaeast": "cnee1i3",
    "chinanorth": "cnnn1i3",
    "eastasia": "asee1i3",
    "eastus2": "usee2i3",
    "eastus": "usee1i3",
    "japaneast": "jpee1i3",
    "japanwest": "jpww1i3",
    "northcentralus": "usnc1i3",
    "northeurope": "eunn1i3",
    "southcentralus": "ussc1i3",
    "southeastasia": "asse1i3",
    "southindia": "inss1i3",
    "uknorth": "gbnn1i3",
    "uksouth": "gbss1i3",
    "uksouth2": "gbss2i3",
    "ukwest": "gbww1i3",
    "westcentralus": "uswc1i3",
    "westeurope": "euww1i3",
    "westindia": "inww1i3",
    "westus2": "usww2i3",
    "westus": "usww1i3",
}

# Thorough investigation has not found an equivalent for these in the
# Azure-ARM image repository.
EXPECTED_MISSING = frozenset({
    ('12.04.2-LTS', '12.04.201212180'),
    ('16.04.0-LTS','16.04.201611220'),
    ('16.04.0-LTS', '16.04.201611300'),
    })

class MissingImage(Exception):
    """Raised when an expected image is not present."""

class UnexpectedImage(Exception):
    """Raised when an image not expected is present."""

def get_azure_credentials(all_credentials):
    """Return the subscription_id and credentials for Azure.

    Takes a dict where key is the cloud name, expected to be formatted like
    cloud-city's credentials.
    """
    azure_dict = all_credentials['azure']['credentials']
    subscription_id = azure_dict['subscription-id']
    return subscription_id, ServicePrincipalCredentials(
        client_id=azure_dict['application-id'],
        secret=azure_dict['application-password'],
        tenant=azure_dict['tenant-id'],
        subscription_id=azure_dict['subscription-id'],
        )

def parse_id(item_id):
    """Parse an old-style item-id to determine sku and version.

    From old-style ID, we ignore the first 32 chars.
    From "Ubuntu-14_04-LTS-amd64-server-20140416.1-en-us-30GB", we extract
    "14_04-LTS", "20140416" and "1"
    We convert to the sku 14.04.0-LTS and the version to 201404161.  (If there
    was no ".1", we'd use "0".)
    Note that for LTS only, the sku's version number always has a third digit
    (patchlevel).

    For 16.04.0-LTS, anything with beta in the id has the sku overriden to
    '16.04-beta'
    """
    match = re.match(
        '^.{32}__Ubuntu-(.*)-.*-server-(\d+)(\.(\d+))?(\-([^\d]*)\d+)?'
        '-.{2}-.{2}-\d+GB$', item_id)
    sku = match.group(1)
    sku = sku.replace('_', '.')
    # Prepare to manipulate the sku's version number.
    sku_sections = sku.split('-')
    if match.group(6) == 'beta' and len(sku_sections) > 1:
        sku_sections[1:] = ['beta']
    sku_num = sku_sections[0]
    sku_num_parts = sku_num.split('.')
    # foo-LTS always has a three-part version number, but for non-LTS, no third
    # digit should be added.
    if len(sku_num_parts) < 3 and sku_sections[1:] == ['LTS']:
        sku_num_parts.append('0')
    sku_num = '.'.join(sku_num_parts)
    sku = '-'.join([sku_num] + sku_sections[1:])
    version = '.'.join(sku_num_parts[0:2] + [match.group(2)])
    number = match.group(4)
    if number is not None:
        version += number
    else:
        version += '0'
    return sku, version

def arm_image_exists(client, location, full_spec):
    """Return True if the full_spec exists on Azure-ARM, else False."""
    try:
        client.virtual_machine_images.get(location, *full_spec)
    except CloudError as e:
        if e.message != 'Artifact: VMImage was not found.':
            raise
        return False
    else:
        return True

def convert_item_to_arm(item, urn, endpoint, region):
    """Return the ARM equivalent of an item, given a urn + endpoint."""
    data = dict(item.data)
    data.pop('crsn', None)
    data.update({'id': urn, 'endpoint': endpoint, 'region': region})
    return Item(item.content_id, item.product_name, item.version_name,
                item.item_name, data=data)

def sku_version_items(items):
    """Return a sorted list of tuples of (sku, version, item).

    The items' ids must support parse_id.
    """
    sort_items = []
    for item in items:
        sku, version = parse_id(item.data['id'])
        sort_items.append((sku, version, item))
    sort_items.sort()
    return sort_items

def convert_cloud_images_items(client, locations, items):
    """Convert cloud-images Azure data to Azure-ARM data."""
    arm_items = []
    endpoint = client.config.base_url
    location_map = dict((l.display_name, l.name) for l in locations)
    unknown_locations = set()
    for sku, version, item in sku_version_items(items):
        location_display_name = item.data['region']
        location = location_map.get(location_display_name)
        if location is None:
            unknown_locations.add(location_display_name)
            continue
        full_spec = (CANONICAL, UBUNTU_SERVER, sku, version)
        urn = ':'.join(full_spec)
        if not arm_image_exists(client, location, full_spec):
            if (sku, version) not in EXPECTED_MISSING:
                raise MissingImage('{} not in {}\n'.format(urn, location))
            continue
        if (sku, version) in EXPECTED_MISSING:
            raise UnexpectedImage(
                'Unexpectedly found {} in {}\n'.format(urn, location))
        arm_items.append(convert_item_to_arm(item, urn, endpoint, location))
    return arm_items, unknown_locations

def make_spec_items(client, full_spec, locations):
    """Return Items for all versions this spec in all Azure locations.

    full_spec is the spec to use for looking up versions.
    locations is a list of Azure Locations.
    """
    endpoint = client.config.base_url
    spec = full_spec[1:]
    location_versions = {}
    for location in locations:
        logging.info('Retrieving image data in {}'.format(
            location.display_name))
        try:
            versions = client.virtual_machine_images.list(location.name,
                                                          *spec)
        except CloudError:
            template = 'Could not find {} {} {} in {location}'
            logging.warning(template.format(
                *spec, location=location.display_name))
            continue
        for version in versions:
            location_versions.setdefault(
                version.name, set()).add(location.name)
    lv2 = sorted(location_versions.items(), key=lambda x: [
        int(ns) for ns in x[0].split('.')])
    for num, (version, v_locations) in enumerate(lv2):
        # Sort in theoretical version number order, not lexicographically
        width = len('{}'.format(len(versions)))
        for location in v_locations:
            version_name = '{:0{}d}'.format(num, width)
            yield make_item(version_name, version, full_spec, location,
                            endpoint)

def make_item(version_name, urn_version, full_spec, location_name, endpoint):
    """Make a simplestreams Item for a version.

    Version name is the simplestreams version_name.
    urn_version is the Azure version.name, used to generate the URN ID.
    full_spec is the spec that was used to list the versions.
    location_name is the Azure display name.
    endpoint is the URL used as an Azure endpoint.

    The item_name is looked up from ITEM_NAMES.
    """
    URN = ':'.join(full_spec[1:] + (urn_version,))
    product_name = 'com.ubuntu.cloud:server:{}:amd64'.format(full_spec[0])
    return Item(
        'com.ubuntu.cloud:released:azure',
        product_name,
        version_name, ITEM_NAMES[location_name], {
            'arch': 'amd64',
            'virt': 'Hyper-V',
            'region': location_name,
            'id': URN,
            'label': 'release',
            'endpoint': endpoint,
            'release': full_spec[0],
            'version': full_spec[0],
            }
        )

class ItemList(mirrors.BasicMirrorWriter):
    """A class that can retrieve the items from a given url.

    Based on sstream-query.
    """

    @classmethod
    def items_from_url(cls, url):
        source = mirrors.UrlMirrorReader(url)
        target = cls()
        target.sync(source, 'com.ubuntu.cloud:released:azure.sjson')
        return target.items

    def __init__(self):
        self.config = {}
        self.items = []

    def load_products(self, path, content_id):
        pass

    def insert_item(self, data, src, target, pedigree, contentsource):
        data = util.products_exdata(src, pedigree)
        self.items.append(dict_to_item(data))

def make_azure_items(all_credentials):
    """Make simplestreams Items for existing Azure images.

    All versions of all images matching IMAGE_SPEC will be returned.

    all_credentials is a dict of credentials in the credentials.yaml
    structure, used to create Azure credentials.
    """
    subscription_id, credentials = get_azure_credentials(all_credentials)
    sub_client = SubscriptionClient(credentials)
    client = ComputeManagementClient(credentials, subscription_id)
    locations = sub_client.subscriptions.list_locations(subscription_id)
    ci_items = ItemList.items_from_url(
        'http://cloud-images.ubuntu.com/releases/streams/v1')
    items, unknown_locations = convert_cloud_images_items(
        client, locations, ci_items)
    sys.stderr.write('Unknown locations: {}\n'.format(', '.join(
        sorted(unknown_locations))))
    items.extend(find_spec_items(client, locations))
    return items

def find_spec_items(client, locations):
    """Make simplestreams Items for existing Azure images.

    All versions of all images matching IMAGE_SPEC will be returned.

    all_credentials is a dict of credentials in the credentials.yaml
    structure, used to create Azure credentials.
    """
    items = []
    for full_spec in IMAGE_SPEC:
        items.extend(make_spec_items(client, full_spec, locations))
    return items
<commit_msg>Start using :latest for ubuntu.<commit_after>import logging
import re
import sys

# Tested on Azure 2.0 rc5 API
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import (
    ComputeManagementClient,
    )
from azure.mgmt.resource.subscriptions import SubscriptionClient
from msrestazure.azure_exceptions import CloudError
from simplestreams.json2streams import (
    dict_to_item,
    Item,
    )
from simplestreams import mirrors
from simplestreams import util

from build_package import juju_series

CANONICAL = 'Canonical'
MS_VSTUDIO = 'MicrosoftVisualStudio'
MS_SERVER = 'MicrosoftWindowsServer'
UBUNTU_SERVER = 'UbuntuServer'
WINDOWS = 'Windows'
WINDOWS_SERVER = 'WindowsServer'
IMAGE_SPEC = [
    ('win81', MS_VSTUDIO, WINDOWS, '8.1-Enterprise-N'),
    ('win10', MS_VSTUDIO, WINDOWS, '10-Enterprise-N'),
    ('win2012', MS_SERVER, WINDOWS_SERVER, '2012-Datacenter'),
    ('win2012r2', MS_SERVER, WINDOWS_SERVER, '2012-R2-Datacenter'),
    ('centos7', 'OpenLogic', 'CentOS', '7.1'),
]

ITEM_NAMES = {
    "australiaeast": "auee1i3",
    "australiasoutheast": "ause1i3",
    "brazilsouth": "brss1i3",
    "canadacentral": "cacc1i3",
    "canadaeast": "caee1i3",
    "centralindia": "incc1i3",
    "centralus": "uscc1i3",
    "chinaeast": "cnee1i3",
    "chinanorth": "cnnn1i3",
    "eastasia": "asee1i3",
    "eastus2": "usee2i3",
    "eastus": "usee1i3",
    "japaneast": "jpee1i3",
    "japanwest": "jpww1i3",
    "northcentralus": "usnc1i3",
    "northeurope": "eunn1i3",
    "southcentralus": "ussc1i3",
    "southeastasia": "asse1i3",
    "southindia": "inss1i3",
    "uknorth": "gbnn1i3",
    "uksouth": "gbss1i3",
    "uksouth2": "gbss2i3",
    "ukwest": "gbww1i3",
    "westcentralus": "uswc1i3",
    "westeurope": "euww1i3",
    "westindia": "inww1i3",
    "westus2": "usww2i3",
    "westus": "usww1i3",
}

# Thorough investigation has not found an equivalent for these in the
# Azure-ARM image repository.
EXPECTED_MISSING = frozenset({
    ('12.04.2-LTS', '12.04.201212180'),
    ('16.04.0-LTS','16.04.201611220'),
    ('16.04.0-LTS', '16.04.201611300'),
    })

class MissingImage(Exception):
    """Raised when an expected image is not present."""

class UnexpectedImage(Exception):
    """Raised when an image not expected is present."""

def get_azure_credentials(all_credentials):
    """Return the subscription_id and credentials for Azure.

    Takes a dict where key is the cloud name, expected to be formatted like
    cloud-city's credentials.
    """
    azure_dict = all_credentials['azure']['credentials']
    subscription_id = azure_dict['subscription-id']
    return subscription_id, ServicePrincipalCredentials(
        client_id=azure_dict['application-id'],
        secret=azure_dict['application-password'],
        tenant=azure_dict['tenant-id'],
        subscription_id=azure_dict['subscription-id'],
        )

def parse_id(item_id):
    """Parse an old-style item-id to determine sku and version.

    From old-style ID, we ignore the first 32 chars.
    From "Ubuntu-14_04-LTS-amd64-server-20140416.1-en-us-30GB", we extract
    "14_04-LTS", "20140416" and "1"
    We convert to the sku 14.04.0-LTS and the version to 201404161.  (If there
    was no ".1", we'd use "0".)
    Note that for LTS only, the sku's version number always has a third digit
    (patchlevel).

    For 16.04.0-LTS, anything with beta in the id has the sku overriden to
    '16.04-beta'
    """
    match = re.match(
        '^.{32}__Ubuntu-(.*)-.*-server-(\d+)(\.(\d+))?(\-([^\d]*)\d+)?'
        '-.{2}-.{2}-\d+GB$', item_id)
    sku = match.group(1)
    sku = sku.replace('_', '.')
    # Prepare to manipulate the sku's version number.
    sku_sections = sku.split('-')
    if match.group(6) == 'beta' and len(sku_sections) > 1:
        sku_sections[1:] = ['beta']
    sku_num = sku_sections[0]
    sku_num_parts = sku_num.split('.')
    # foo-LTS always has a three-part version number, but for non-LTS, no third
    # digit should be added.
    if len(sku_num_parts) < 3 and sku_sections[1:] == ['LTS']:
        sku_num_parts.append('0')
    sku_num = '.'.join(sku_num_parts)
    sku = '-'.join([sku_num] + sku_sections[1:])
    version = '.'.join(sku_num_parts[0:2] + [match.group(2)])
    number = match.group(4)
    if number is not None:
        version += number
    else:
        version += '0'
    return sku, version

def arm_image_exists(client, location, full_spec):
    """Return True if the full_spec exists on Azure-ARM, else False."""
    try:
        client.virtual_machine_images.get(location, *full_spec)
    except CloudError as e:
        if e.message != 'Artifact: VMImage was not found.':
            raise
        return False
    else:
        return True

def convert_item_to_arm(item, urn, endpoint, region):
    """Return the ARM equivalent of an item, given a urn + endpoint."""
    data = dict(item.data)
    data.pop('crsn', None)
    data.update({'id': urn, 'endpoint': endpoint, 'region': region})
    return Item(item.content_id, item.product_name, item.version_name,
                item.item_name, data=data)

def sku_version_items(items):
    """Return a sorted list of tuples of (sku, version, item).

    The items' ids must support parse_id.
    """
    sort_items = []
    for item in items:
        sku, version = parse_id(item.data['id'])
        sort_items.append((sku, version, item))
    sort_items.sort()
    return sort_items

def convert_cloud_images_items(client, locations, items):
    """Convert cloud-images Azure data to Azure-ARM data."""
    arm_items = []
    endpoint = client.config.base_url
    location_map = dict((l.display_name, l.name) for l in locations)
    unknown_locations = set()
    for sku, version, item in sku_version_items(items):
        location_display_name = item.data['region']
        location = location_map.get(location_display_name)
        if location is None:
            unknown_locations.add(location_display_name)
            continue
        full_spec = (CANONICAL, UBUNTU_SERVER, sku, version)
        urn = ':'.join(full_spec)
        if not arm_image_exists(client, location, full_spec):
            if (sku, version) not in EXPECTED_MISSING:
                raise MissingImage('{} not in {}\n'.format(urn, location))
            continue
        if (sku, version) in EXPECTED_MISSING:
            raise UnexpectedImage(
                'Unexpectedly found {} in {}\n'.format(urn, location))
        arm_items.append(convert_item_to_arm(item, urn, endpoint, location))
    return arm_items, unknown_locations

def make_spec_items(client, full_spec, locations):
    """Return Items for all versions this spec in all Azure locations.

    full_spec is the spec to use for looking up versions.
    locations is a list of Azure Locations.
    """
    endpoint = client.config.base_url
    spec = full_spec[1:]
    location_versions = {}
    for location in locations:
        logging.info('Retrieving image data in {}'.format(
            location.display_name))
        try:
            versions = client.virtual_machine_images.list(location.name,
                                                          *spec)
        except CloudError:
            template = 'Could not find {} {} {} in {location}'
            logging.warning(template.format(
                *spec, location=location.display_name))
            continue
        for version in versions:
            location_versions.setdefault(
                version.name, set()).add(location.name)
    lv2 = sorted(location_versions.items(), key=lambda x: [
        int(ns) for ns in x[0].split('.')])
    for num, (version, v_locations) in enumerate(lv2):
        # Sort in theoretical version number order, not lexicographically
        width = len('{}'.format(len(versions)))
        for location in v_locations:
            version_name = '{:0{}d}'.format(num, width)
            yield make_item(version_name, version, full_spec, location,
                            endpoint)

def make_item(version_name, urn_version, full_spec, location_name, endpoint,
              stream='released', item_version=None):
    """Make a simplestreams Item for a version.

    Version name is the simplestreams version_name.
    urn_version is the Azure version.name, used to generate the URN ID.
    full_spec is the spec that was used to list the versions.
    location_name is the Azure display name.
    endpoint is the URL used as an Azure endpoint.

    The item_name is looked up from ITEM_NAMES.
    """
    URN = ':'.join(full_spec[1:] + (urn_version,))
    product_name = 'com.ubuntu.cloud:server:{}:amd64'.format(full_spec[0])
    if item_version is None:
        item_version = full_spec[0]
    return Item(
        'com.ubuntu.cloud:{}:azure'.format(stream),
        product_name,
        version_name, ITEM_NAMES[location_name], {
            'arch': 'amd64',
            'virt': 'Hyper-V',
            'region': location_name,
            'id': URN,
            'label': 'release',
            'endpoint': endpoint,
            'release': full_spec[0],
            'version': item_version,
            }
        )

class ItemList(mirrors.BasicMirrorWriter):
    """A class that can retrieve the items from a given url.

    Based on sstream-query.
    """

    @classmethod
    def items_from_url(cls, url):
        source = mirrors.UrlMirrorReader(url)
        target = cls()
        target.sync(source, 'com.ubuntu.cloud:released:azure.sjson')
        return target.items

    def __init__(self):
        self.config = {}
        self.items = []

    def load_products(self, path, content_id):
        pass

    def insert_item(self, data, src, target, pedigree, contentsource):
        data = util.products_exdata(src, pedigree)
        self.items.append(dict_to_item(data))

def make_azure_items(all_credentials):
    """Make simplestreams Items for existing Azure images.

    All versions of all images matching IMAGE_SPEC will be returned.

    all_credentials is a dict of credentials in the credentials.yaml
    structure, used to create Azure credentials.
    """
    subscription_id, credentials = get_azure_credentials(all_credentials)
    sub_client = SubscriptionClient(credentials)
    client = ComputeManagementClient(credentials, subscription_id)
    locations = sub_client.subscriptions.list_locations(subscription_id)
    items = find_ubuntu_items(client, locations)
    items.extend(find_spec_items(client, locations))
    return items

def find_ubuntu_items(client, locations):
    """Make simplestreams Items for existing Azure images.

    All versions of all images matching IMAGE_SPEC will be returned.

    all_credentials is a dict of credentials in the credentials.yaml
    structure, used to create Azure credentials.
    """
    items = []
    for location in locations:
        skus = client.virtual_machine_images.list_skus(
            location.name, CANONICAL, UBUNTU_SERVER)
        for sku in skus:
            match = re.match(r'(\d\d\.\d\d)(\.\d+)?-?(.*)', sku.name)
            if match is None:
                logging.warning('Skipping {}'.format(sku.name))
                continue
            tag = match.group(3)
            if tag in ('DAILY', 'DAILY-LTS'):
                stream = 'daily'
            elif tag in ('', 'LTS'):
                stream = 'released'
            else:
                print sku.name
                continue
            minor_version = match.group(1)
            try:
                full_spec = (
                    juju_series.get_name(minor_version), CANONICAL,
                    UBUNTU_SERVER, sku.name)
            except KeyError:
                continue
            items.append(make_item(sku.name, 'latest', full_spec,
                                   location.name, client.config.base_url,
                                   item_version=minor_version, stream=stream))
    return items

def find_spec_items(client, locations):
    """Make simplestreams Items for existing Azure images.

    All versions of all images matching IMAGE_SPEC will be returned.

    all_credentials is a dict of credentials in the credentials.yaml
    structure, used to create Azure credentials.
    """
    items = []
    for full_spec in IMAGE_SPEC:
        items.extend(make_spec_items(client, full_spec, locations))
    return items
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from cms.utils.i18n import get_default_language
from django.conf import settings
from django.core.urlresolvers import reverse
from django.middleware.locale import LocaleMiddleware
from django.utils import translation
import re
import urllib
import urlparse

SUPPORTED = dict(settings.CMS_LANGUAGES)

HAS_LANG_PREFIX_RE = re.compile(r"^/(%s)/.*" % "|".join([re.escape(l[0]) for l in settings.CMS_LANGUAGES]))

def has_lang_prefix(path):
    check = HAS_LANG_PREFIX_RE.match(path)
    if check is not None:
        return check.group(1)
    else:
        return False

def patch_response(content, pages_root, language):
    # Customarily user pages are served from http://the.server.com/~username/
    # When a user uses django-cms for his pages, the '~' of the url appears quoted in href links.
    # We have to quote pages_root for the regular expression to match.
    #
    # The used regex is quite complex. The exact pattern depends on the used settings.
    # The regex extracts the path of the url without the leading page root, but only matches urls
    # that don't already contain a language string or aren't considered multilingual.
    #
    # Here is an annotated example pattern (_r_ is a shorthand for the value of pages_root):
    #   pattern:        ]+)href=("|\')(?=_r_)(?!(/fr/|/de/|/en/|/pt-br/|/media/|/media/admin/))(_r_(.*?))("|\')(.*?)>
    #                     |-\1--|     |-\2-|       |---------------------\3---------------------|    | |-\5--|||-\6-||-\7-|
    #                                                                                |---\4---|
    #   input (_r_=/):
    #   matched groups: (u' ', None, u'/admin/password_change/', u'admin/password_change/', u' class="foo"')
    #
    # Notice that (?=...) and (?!=...) do not consume input or produce a group in the match object.
    # If the regex matches, the extracted path we want is stored in the fourth group (\4).
    quoted_root = urllib.quote(pages_root)
    ignore_paths = ['%s%s/' % (quoted_root, l[0]) for l in settings.CMS_LANGUAGES]
    ignore_paths += [settings.MEDIA_URL, settings.STATIC_URL]
    if getattr(settings,'STATIC_URL', False):
        ignore_paths += [settings.STATIC_URL]

    HREF_URL_FIX_RE = re.compile(ur']+)href=("|\')(?=%s)(?!(%s))(%s(.*?))("|\')(.*?)>' % (
        quoted_root,
        "|".join([re.escape(p) for p in ignore_paths]),
        quoted_root
    ))

    # Unlike in href links, the '~' (see above) the '~' in form actions appears unquoted.
    #
    # For understanding this regex, please read the documentation for HREF_URL_FIX_RE above.

    ignore_paths = ['%s%s/' % (pages_root, l[0]) for l in settings.CMS_LANGUAGES]
    ignore_paths += [settings.MEDIA_URL, settings.STATIC_URL]
    if getattr(settings,'STATIC_URL', False):
        ignore_paths += [settings.STATIC_URL]
    FORM_URL_FIX_RE = re.compile(ur'<form([^>]+)action=("|\')(?=%s)(?!(%s))(%s(.*?))("|\')(.*?)>' % (
        pages_root,
        "|".join([re.escape(p) for p in ignore_paths]),
        pages_root
    ))

    content = HREF_URL_FIX_RE.sub(ur'' % (language, pages_root), content)
    content = FORM_URL_FIX_RE.sub(ur'<form\1action=\2%s%s/\5\6\7>' % (pages_root, language), content).encode("utf8")
    return content

class MultilingualURLMiddleware(object):
    def get_language_from_request (self,request):
        changed = False
        prefix = has_lang_prefix(request.path_info)
        if prefix:
            request.path = "/" + "/".join(request.path.split("/")[2:])
            request.path_info = "/" + "/".join(request.path_info.split("/")[2:])
            t = prefix
            if t in SUPPORTED:
                lang = t
                if hasattr(request, "session"):
                    request.session["django_language"] = lang
                changed = True
        else:
            lang = translation.get_language_from_request(request)
        if not changed:
            if hasattr(request, "session"):
                lang = request.session.get("django_language", None)
                if lang in SUPPORTED and lang is not None:
                    return lang
            elif "django_language" in request.COOKIES.keys():
                lang = request.COOKIES.get("django_language", None)
                if lang in SUPPORTED and lang is not None:
                    return lang
            if not lang:
                lang = translation.get_language_from_request(request)
        lang = get_default_language(lang)
        return lang

    def process_request(self, request):
        language = self.get_language_from_request(request)
        translation.activate(language)
        request.LANGUAGE_CODE = language

    def process_response(self, request, response):
        language = getattr(request, 'LANGUAGE_CODE', self.get_language_from_request(request))
        local_middleware = LocaleMiddleware()
        response =local_middleware.process_response(request, response)
        path = unicode(request.path)

        # note: pages_root is assumed to end in '/'.
        #       testing this and throwing an exception otherwise, would probably be a good idea

        if not path.startswith(settings.MEDIA_URL) and \
                not path.startswith(settings.STATIC_URL) and \
                not (getattr(settings,'STATIC_URL', False) and path.startswith(settings.STATIC_URL)) and \
                response.status_code == 200 and \
                response._headers['content-type'][1].split(';')[0] == "text/html":
            pages_root = urllib.unquote(reverse("pages-root"))
            try:
                decoded_response = response.content.decode('utf-8')
            except UnicodeDecodeError:
                decoded_response = response.content

            response.content = patch_response(
                decoded_response,
                pages_root,
                request.LANGUAGE_CODE
            )

        if (response.status_code == 301 or response.status_code == 302 ):
            location = response['Location']
            if location.startswith('.'):
                location = urlparse.urljoin(request.path, location)
                response['Location'] = location
            if not has_lang_prefix(location) and location.startswith("/") and \
                    not location.startswith(settings.MEDIA_URL) and \
                    not (getattr(settings,'STATIC_URL', False) and location.startswith(settings.STATIC_URL)) and \
                    not location.startswith(settings.STATIC_URL):
                response['Location'] = "/%s%s" % (language, location)
        response.set_cookie("django_language", language)
        return response
<commit_msg>removed silly line<commit_after># -*- coding: utf-8 -*-
from cms.utils.i18n import get_default_language
from django.conf import settings
from django.core.urlresolvers import reverse
from django.middleware.locale import LocaleMiddleware
from django.utils import translation
import re
import urllib
import urlparse

SUPPORTED = dict(settings.CMS_LANGUAGES)

HAS_LANG_PREFIX_RE = re.compile(r"^/(%s)/.*" % "|".join([re.escape(l[0]) for l in settings.CMS_LANGUAGES]))

def has_lang_prefix(path):
    check = HAS_LANG_PREFIX_RE.match(path)
    if check is not None:
        return check.group(1)
    else:
        return False

def patch_response(content, pages_root, language):
    # Customarily user pages are served from http://the.server.com/~username/
    # When a user uses django-cms for his pages, the '~' of the url appears quoted in href links.
    # We have to quote pages_root for the regular expression to match.
    #
    # The used regex is quite complex. The exact pattern depends on the used settings.
    # The regex extracts the path of the url without the leading page root, but only matches urls
    # that don't already contain a language string or aren't considered multilingual.
    #
    # Here is an annotated example pattern (_r_ is a shorthand for the value of pages_root):
    #   pattern:        ]+)href=("|\')(?=_r_)(?!(/fr/|/de/|/en/|/pt-br/|/media/|/media/admin/))(_r_(.*?))("|\')(.*?)>
    #                     |-\1--|     |-\2-|       |---------------------\3---------------------|    | |-\5--|||-\6-||-\7-|
    #                                                                                |---\4---|
    #   input (_r_=/):
    #   matched groups: (u' ', None, u'/admin/password_change/', u'admin/password_change/', u' class="foo"')
    #
    # Notice that (?=...) and (?!=...) do not consume input or produce a group in the match object.
    # If the regex matches, the extracted path we want is stored in the fourth group (\4).
    quoted_root = urllib.quote(pages_root)
    ignore_paths = ['%s%s/' % (quoted_root, l[0]) for l in settings.CMS_LANGUAGES]
    ignore_paths += [settings.MEDIA_URL, settings.STATIC_URL]
    if getattr(settings,'STATIC_URL', False):
        ignore_paths += [settings.STATIC_URL]

    HREF_URL_FIX_RE = re.compile(ur']+)href=("|\')(?=%s)(?!(%s))(%s(.*?))("|\')(.*?)>' % (
        quoted_root,
        "|".join([re.escape(p) for p in ignore_paths]),
        quoted_root
    ))

    # Unlike in href links, the '~' (see above) the '~' in form actions appears unquoted.
    #
    # For understanding this regex, please read the documentation for HREF_URL_FIX_RE above.

    ignore_paths = ['%s%s/' % (pages_root, l[0]) for l in settings.CMS_LANGUAGES]
    ignore_paths += [settings.MEDIA_URL, settings.STATIC_URL]
    if getattr(settings,'STATIC_URL', False):
        ignore_paths += [settings.STATIC_URL]
    FORM_URL_FIX_RE = re.compile(ur'<form([^>]+)action=("|\')(?=%s)(?!(%s))(%s(.*?))("|\')(.*?)>' % (
        pages_root,
        "|".join([re.escape(p) for p in ignore_paths]),
        pages_root
    ))

    content = HREF_URL_FIX_RE.sub(ur'' % (language, pages_root), content)
    content = FORM_URL_FIX_RE.sub(ur'<form\1action=\2%s%s/\5\6\7>' % (pages_root, language), content).encode("utf8")
    return content

class MultilingualURLMiddleware(object):
    def get_language_from_request (self,request):
        changed = False
        prefix = has_lang_prefix(request.path_info)
        if prefix:
            request.path = "/" + "/".join(request.path.split("/")[2:])
            request.path_info = "/" + "/".join(request.path_info.split("/")[2:])
            t = prefix
            if t in SUPPORTED:
                lang = t
                if hasattr(request, "session"):
                    request.session["django_language"] = lang
                changed = True
        else:
            lang = translation.get_language_from_request(request)
        if not changed:
            if hasattr(request, "session"):
                lang = request.session.get("django_language", None)
                if lang in SUPPORTED and lang is not None:
                    return lang
            elif "django_language" in request.COOKIES.keys():
                lang = request.COOKIES.get("django_language", None)
                if lang in SUPPORTED and lang is not None:
                    return lang
            if not lang:
                lang = translation.get_language_from_request(request)
        lang = get_default_language(lang)
        return lang

    def process_request(self, request):
        language = self.get_language_from_request(request)
        translation.activate(language)
        request.LANGUAGE_CODE = language

    def process_response(self, request, response):
        language = getattr(request, 'LANGUAGE_CODE', self.get_language_from_request(request))
        local_middleware = LocaleMiddleware()
        response =local_middleware.process_response(request, response)
        path = unicode(request.path)

        # note: pages_root is assumed to end in '/'.
        #       testing this and throwing an exception otherwise, would probably be a good idea

        if not path.startswith(settings.MEDIA_URL) and \
                not path.startswith(settings.STATIC_URL) and \
                not (getattr(settings,'STATIC_URL', False) and path.startswith(settings.STATIC_URL)) and \
                response.status_code == 200 and \
                response._headers['content-type'][1].split(';')[0] == "text/html":
            pages_root = urllib.unquote(reverse("pages-root"))
            try:
                decoded_response = response.content.decode('utf-8')
            except UnicodeDecodeError:
                decoded_response = response.content

            response.content = patch_response(
                decoded_response,
                pages_root,
                request.LANGUAGE_CODE
            )

        if (response.status_code == 301 or response.status_code == 302 ):
            location = response['Location']
            if location.startswith('.'):
                location = urlparse.urljoin(request.path, location)
                response['Location'] = location
            if not has_lang_prefix(location) and location.startswith("/") and \
                    not location.startswith(settings.MEDIA_URL) and \
                    not (getattr(settings,'STATIC_URL', False) and location.startswith(settings.STATIC_URL)):
                response['Location'] = "/%s%s" % (language, location)
        response.set_cookie("django_language", language)
        return response
<|endoftext|>


<commit_before>#!/usr/bin/env python26

#
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
#

#
import logging, sys, os, ConfigParser
from datetime import date
from Stylesheet import *

# make the current working directory where ever the scripts are.
abspath = os.path.abspath(sys.argv[0])
dname = os.path.dirname(abspath)
os.chdir(dname)

# create base logger
# FIXME: move into a common file.
log = logging.getLogger('cssbot')
log.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
today = date.today()
#log_date = "%d%02d%02d" % (today.year, today.month, today.day)
log_date = "%d%02d" % (today.year, today.month)
fh = logging.FileHandler('log/cssbot-%s.log' % log_date)
fh.setLevel(logging.INFO)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s : [%(levelname)s] %(name)s : %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
log.addHandler(fh)
log.addHandler(ch)

# read the config file.
config = ConfigParser.ConfigParser()
config.read("cssbot.cfg")

# exec the css update.
s = Stylesheet(config)
s.generate_and_post()

<commit_msg>add in log to denote the forced css update<commit_after>#!/usr/bin/env python26

#
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
#

#
import logging, sys, os, ConfigParser
from datetime import date
from Stylesheet import *

# make the current working directory where ever the scripts are.
abspath = os.path.abspath(sys.argv[0])
dname = os.path.dirname(abspath)
os.chdir(dname)

# create base logger
# FIXME: move into a common file.
log = logging.getLogger('cssbot')
log.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
today = date.today()
#log_date = "%d%02d%02d" % (today.year, today.month, today.day)
log_date = "%d%02d" % (today.year, today.month)
fh = logging.FileHandler('log/cssbot-%s.log' % log_date)
fh.setLevel(logging.INFO)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s : [%(levelname)s] %(name)s : %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
log.addHandler(fh)
log.addHandler(ch)

# read the config file.
config = ConfigParser.ConfigParser()
config.read("cssbot.cfg")

# exec the css update.
log.warn("forcing update of css")
s = Stylesheet(config)
s.generate_and_post()

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# The azure lib checks the environment for two vars that can be sourced
# or defined before the command.
# export AZURE_STORAGE_ACCOUNT=tcontepub
# export AZURE_STORAGE_ACCESS_KEY='secret key'

from __future__ import print_function

import base64
from collections import namedtuple
import hashlib
import mimetypes
from operator import attrgetter
from optparse import OptionParser
import os
import sys

from azure.storage import BlobService

mimetypes.init()

OK = 0
BAD_ARGS = 1
UNKNOWN_COMMAND = 2
NO_PUBLISHED_FILES = 3
NO_LOCAL_FILES = 4
UNKNOWN_PURPOSE = 5

LIST = 'list'
PUBLISH = 'publish'
DELETE = 'delete'
RELEASE = 'release'
PROPOSED = 'proposed'
DEVEL = 'devel'
TESTING = 'testing'
PURPOSES = (RELEASE, PROPOSED, DEVEL, TESTING)
JUJU_DIST = 'juju-tools'
CHUNK_SIZE = 4 * 1024 * 1024

SIGNED_EXTS = ('.sjson', '.gpg')

SyncFile = namedtuple(
    'SyncFile', ['path', 'size', 'md5content', 'mimetype', 'local_path'])

def get_prefix(purpose):
    """Return the top-level dir name for the purpose."""
    if purpose == RELEASE:
        return 'tools'
    else:
        return purpose

def get_published_files(purpose, blob_service):
    """Return the SyncFile info about the published files."""
    prefix = get_prefix(purpose)
    files = []
    for blob in blob_service.list_blobs(
            JUJU_DIST, prefix=prefix, include='metadata'):
        sync_file = SyncFile(
            path=blob.name, md5content=blob.properties.content_md5,
            size=blob.properties.content_length,
            mimetype=blob.properties.content_type, local_path='')
        files.append(sync_file)
    return sorted(files, key=attrgetter('path'))

def get_local_files(purpose, local_dir):
    """Return SyncFile info about the files in the local tools tree."""
    if not os.path.isdir(local_dir):
        print('%s not found.' % local_dir)
        return None
    if purpose == RELEASE:
        replacements = (local_dir + '/', '')
    else:
        replacements = (local_dir, purpose)
    found = []
    for path, subdirs, files in os.walk(local_dir):
        for name in files:
            local_path = os.path.join(path, name)
            publish_path = local_path.replace(*replacements)
            if 'mirror' in name or os.path.islink(local_path):
                # The mirror files only belong on streams.canonical.com.
                continue
            ignore, extension = os.path.splitext(name)
            if purpose == TESTING and extension in SIGNED_EXTS:
                continue
            size = os.path.getsize(local_path)
            md5content = get_md5content(local_path)
            mimetype, encoding = mimetypes.guess_type(name)
            sync_file = SyncFile(
                path=publish_path, size=size, md5content=md5content,
                mimetype=mimetype, local_path=local_path)
            found.append(sync_file)
    return sorted(found, key=attrgetter('path'))

def get_md5content(local_path):
    """Return the base64 encoded md5 digest for the local file."""
    md5 = hashlib.md5()
    with open(local_path, mode='rb') as local_file:
        md5.update(local_file.read())
    base64_md5 = base64.encodestring(md5.digest()).strip()
    return base64_md5

def publish_local_file(purpose, blob_service, sync_file):
    """Published the local file to the release proposed, or testing location.

    The file is broken down into blocks that can be uploaded within
    the azure restrictions. The blocks are then assembled into a blob
    with the md5 content (base64 encoded digest).
    """
    blob_service.put_blob(JUJU_DIST, sync_file.path, '', 'BlockBlob')
    block_ids = []
    index = 0
    with open(sync_file.local_path, 'rb') as local_file:
        while True:
            data = local_file.read(CHUNK_SIZE)
            if data:
                block_id = base64.b64encode(str(index))
                blob_service.put_block(
                    JUJU_DIST, sync_file.path, data, block_id)
                block_ids.append(block_id)
                index += 1
            else:
                break
    blob_service.put_block_list(
        JUJU_DIST, sync_file.path, block_ids,
        x_ms_blob_content_type=sync_file.mimetype,
        x_ms_blob_content_md5=sync_file.md5content)

def list_published_files(purpose):
    """List the testing, proposed, or release files."""
    blob_service = BlobService()
    published_files = get_published_files(purpose, blob_service)
    if published_files is None:
        return NO_PUBLISHED_FILES
    for sync_file in published_files:
        print(
            '%s %s %s' % (
                sync_file.path, sync_file.size, sync_file.md5content))
    return OK

def publish_files(purpose, local_dir, options):
    """Publish the streams to the location for the intended purpose."""
    blob_service = BlobService()
    if local_dir.endswith('/'):
        local_dir = local_dir[:-1]
    print("Looking for published files in %s" % purpose)
    published_files = get_published_files(purpose, blob_service)
    if published_files is None:
        return NO_PUBLISHED_FILES
    print("Looking for local files in %s" % local_dir)
    local_files = get_local_files(purpose, local_dir)
    if local_files is None:
        return NO_LOCAL_FILES
    if options.verbose:
        for lf in local_files:
            print(lf.path)
    published_dict = dict(
        (sync_file.path, sync_file) for sync_file in published_files)
    for sync_file in local_files:
        if sync_file.path not in published_dict:
            print('%s is new.' % sync_file.path)
        elif published_dict[sync_file.path].md5content != sync_file.md5content:
            print('%s is different.' % sync_file.path)
            if options.verbose:
                print(
                    '  published:%s != local:%s.' % (
                        published_dict[sync_file.path].md5content,
                        sync_file.md5content))
        else:
            if options.verbose:
                print("Nothing to do: %s == %s" % (
                    sync_file.path, published_dict[sync_file.path].md5content))
            continue
        if not options.dry_run:
            publish_local_file(purpose, blob_service, sync_file)
    return OK

def delete_files(purpose, files, options):
    prefix = get_prefix(purpose)
    blob_service = BlobService()
    for path in files:
        if prefix is not None:
            path = '%s/%s' % (prefix, path)
        print("Deleting %s" % path)
        blob_service.delete_blob(JUJU_DIST, path)
    return OK

def main():
    """Execute the commands from the command line."""
    parser = get_option_parser()
    (options, args) = parser.parse_args(args=sys.argv[1:])
    if len(args) <= 1:
        parser.print_usage()
        return BAD_ARGS
    command = args[0]
    purpose = args[1]
    if purpose not in PURPOSES:
        return UNKNOWN_PURPOSE
    if command == LIST:
        return list_published_files(purpose)
    elif command == PUBLISH:
        if len(args) != 3:
            parser.print_usage()
            return 1
        return publish_files(purpose, args[2], options)
    elif command == DELETE:
        return delete_files(purpose, args[2:], options)
    else:
        # The command is not known.
        return UNKNOWN_COMMAND

def get_option_parser():
    """Return the option parser for this program."""
    usage = (
        "usage: %prog <list | publish | delete> "
        "<{}> [local-tools]".format(' | '.join(PURPOSES)))
    parser = OptionParser(usage=usage)
    parser.add_option(
        "-d", "--dry-run", action="store_true", dest="dry_run")
    parser.add_option(
        "-v", "--verbose", action="store_true", dest="verbose")
    parser.set_defaults(
        dry_run=False,
        verbose=False
    )
    return parser

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Switch to argument parser.<commit_after>#!/usr/bin/python
#
# The azure lib checks the environment for two vars that can be sourced
# or defined before the command.
# export AZURE_STORAGE_ACCOUNT=tcontepub
# export AZURE_STORAGE_ACCESS_KEY='secret key'

from __future__ import print_function

from argparse import ArgumentParser
import base64
from collections import namedtuple
import hashlib
import mimetypes
from operator import attrgetter
import os
import sys

from azure.storage import BlobService

mimetypes.init()

OK = 0
BAD_ARGS = 1
UNKNOWN_COMMAND = 2
NO_PUBLISHED_FILES = 3
NO_LOCAL_FILES = 4
UNKNOWN_PURPOSE = 5

LIST = 'list'
PUBLISH = 'publish'
DELETE = 'delete'
COMMANDS = (LIST, PUBLISH, DELETE)
RELEASE = 'release'
PROPOSED = 'proposed'
DEVEL = 'devel'
TESTING = 'testing'
PURPOSES = (RELEASE, PROPOSED, DEVEL, TESTING)
JUJU_DIST = 'juju-tools'
CHUNK_SIZE = 4 * 1024 * 1024

SIGNED_EXTS = ('.sjson', '.gpg')

SyncFile = namedtuple(
    'SyncFile', ['path', 'size', 'md5content', 'mimetype', 'local_path'])

def get_prefix(purpose):
    """Return the top-level dir name for the purpose."""
    if purpose == RELEASE:
        return 'tools'
    else:
        return purpose

def get_published_files(purpose, blob_service):
    """Return the SyncFile info about the published files."""
    prefix = get_prefix(purpose)
    files = []
    for blob in blob_service.list_blobs(
            JUJU_DIST, prefix=prefix, include='metadata'):
        sync_file = SyncFile(
            path=blob.name, md5content=blob.properties.content_md5,
            size=blob.properties.content_length,
            mimetype=blob.properties.content_type, local_path='')
        files.append(sync_file)
    return sorted(files, key=attrgetter('path'))

def get_local_files(purpose, local_dir):
    """Return SyncFile info about the files in the local tools tree."""
    if not os.path.isdir(local_dir):
        print('%s not found.' % local_dir)
        return None
    if purpose == RELEASE:
        replacements = (local_dir + '/', '')
    else:
        replacements = (local_dir, purpose)
    found = []
    for path, subdirs, files in os.walk(local_dir):
        for name in files:
            local_path = os.path.join(path, name)
            publish_path = local_path.replace(*replacements)
            if 'mirror' in name or os.path.islink(local_path):
                # The mirror files only belong on streams.canonical.com.
                continue
            ignore, extension = os.path.splitext(name)
            if purpose == TESTING and extension in SIGNED_EXTS:
                continue
            size = os.path.getsize(local_path)
            md5content = get_md5content(local_path)
            mimetype, encoding = mimetypes.guess_type(name)
            sync_file = SyncFile(
                path=publish_path, size=size, md5content=md5content,
                mimetype=mimetype, local_path=local_path)
            found.append(sync_file)
    return sorted(found, key=attrgetter('path'))

def get_md5content(local_path):
    """Return the base64 encoded md5 digest for the local file."""
    md5 = hashlib.md5()
    with open(local_path, mode='rb') as local_file:
        md5.update(local_file.read())
    base64_md5 = base64.encodestring(md5.digest()).strip()
    return base64_md5

def publish_local_file(purpose, blob_service, sync_file):
    """Published the local file to the release proposed, or testing location.

    The file is broken down into blocks that can be uploaded within
    the azure restrictions. The blocks are then assembled into a blob
    with the md5 content (base64 encoded digest).
    """
    blob_service.put_blob(JUJU_DIST, sync_file.path, '', 'BlockBlob')
    block_ids = []
    index = 0
    with open(sync_file.local_path, 'rb') as local_file:
        while True:
            data = local_file.read(CHUNK_SIZE)
            if data:
                block_id = base64.b64encode(str(index))
                blob_service.put_block(
                    JUJU_DIST, sync_file.path, data, block_id)
                block_ids.append(block_id)
                index += 1
            else:
                break
    blob_service.put_block_list(
        JUJU_DIST, sync_file.path, block_ids,
        x_ms_blob_content_type=sync_file.mimetype,
        x_ms_blob_content_md5=sync_file.md5content)

def list_published_files(purpose):
    """List the testing, proposed, or release files."""
    blob_service = BlobService()
    published_files = get_published_files(purpose, blob_service)
    if published_files is None:
        return NO_PUBLISHED_FILES
    for sync_file in published_files:
        print(
            '%s %s %s' % (
                sync_file.path, sync_file.size, sync_file.md5content))
    return OK

def publish_files(purpose, local_dir, args):
    """Publish the streams to the location for the intended purpose."""
    blob_service = BlobService()
    if local_dir.endswith('/'):
        local_dir = local_dir[:-1]
    print("Looking for published files in %s" % purpose)
    published_files = get_published_files(purpose, blob_service)
    if published_files is None:
        return NO_PUBLISHED_FILES
    print("Looking for local files in %s" % local_dir)
    local_files = get_local_files(purpose, local_dir)
    if local_files is None:
        return NO_LOCAL_FILES
    if args.verbose:
        for lf in local_files:
            print(lf.path)
    published_dict = dict(
        (sync_file.path, sync_file) for sync_file in published_files)
    for sync_file in local_files:
        if sync_file.path not in published_dict:
            print('%s is new.' % sync_file.path)
        elif published_dict[sync_file.path].md5content != sync_file.md5content:
            print('%s is different.' % sync_file.path)
            if args.verbose:
                print(
                    '  published:%s != local:%s.' % (
                        published_dict[sync_file.path].md5content,
                        sync_file.md5content))
        else:
            if args.verbose:
                print("Nothing to do: %s == %s" % (
                    sync_file.path, published_dict[sync_file.path].md5content))
            continue
        if not args.dry_run:
            publish_local_file(purpose, blob_service, sync_file)
    return OK

def delete_files(purpose, files, args):
    prefix = get_prefix(purpose)
    blob_service = BlobService()
    for path in files:
        if prefix is not None:
            path = '%s/%s' % (prefix, path)
        print("Deleting %s" % path)
        if not args.dry_run:
            blob_service.delete_blob(JUJU_DIST, path)
    return OK

def main():
    """Execute the commands from the command line."""
    parser = get_option_parser()
    args = parser.parse_args()
    if args.purpose not in PURPOSES:
        return UNKNOWN_PURPOSE
    if args.command not in COMMANDS:
        return UNKNOWN_COMMAND
    elif args.command == LIST:
        return list_published_files(args.purpose)
    elif args.command == PUBLISH:
        if args.path == None or len(args.path) != 1:
            parser.print_usage()
            return BAD_ARGS
        return publish_files(args.purpose, args.path[0], args)
    elif args.command == DELETE:
        if args.path == None:
            parser.print_usage()
            return BAD_ARGS
        return delete_files(args.purpose, args.path, args)

def get_option_parser():
    """Return the option parser for this program."""
    parser = ArgumentParser("Manage objects in Azure blob storage")
    parser.add_argument(
        "-d", "--dry-run", action="store_true", default=False,
        help="Do not publish or delete")
    parser.add_argument(
        '-v', '--verbose', action="store_true", default=False,
        help='Increse verbosity.')
    parser.add_argument('command', help="<list | publish | delete>")
    parser.add_argument('purpose', help="<{}>".format(' | '.join(PURPOSES)))
    parser.add_argument('path', nargs="*", default=None,
        help='The path to publish or files to delete')
    return parser

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>from cms.models import Localisation
from cms.tests.base import BaseCmsTestCase

from unicore.content import models as eg_models

class LocalisationTestCase(BaseCmsTestCase):

    def setUp(self):
        self.workspace = self.mk_workspace()
        self.workspace.setup_custom_mapping(eg_models.Localisation, {
            'properties': {
                'locale': {
                    'type': 'string',
                    'index': 'not_analyzed',
                }
            }
        })

    def test_create_localisation(self):
        l = Localisation._for('spa_ES')
        l.save()
        self.assertEquals(l.language_code, 'spa')
        self.assertEquals(l.country_code, 'ES')
        self.assertEquals(Localisation.objects.all().count(), 1)
        self.assertEquals(self.workspace.S(eg_models.Localisation).count(), 1)

        [eg_local] = self.workspace.S(eg_models.Localisation).everything()
        self.assertEquals(eg_local.locale, 'spa_ES')
<commit_msg>add deletion to tests<commit_after>from cms.models import Localisation
from cms.tests.base import BaseCmsTestCase

from unicore.content import models as eg_models

class LocalisationTestCase(BaseCmsTestCase):

    def setUp(self):
        self.workspace = self.mk_workspace()
        self.workspace.setup_custom_mapping(eg_models.Localisation, {
            'properties': {
                'locale': {
                    'type': 'string',
                    'index': 'not_analyzed',
                }
            }
        })

    def test_create_localisation(self):
        l = Localisation._for('spa_ES')
        l.save()
        self.assertEquals(l.language_code, 'spa')
        self.assertEquals(l.country_code, 'ES')
        self.assertEquals(Localisation.objects.all().count(), 1)
        self.assertEquals(self.workspace.S(eg_models.Localisation).count(), 1)

        [eg_local] = self.workspace.S(eg_models.Localisation).everything()
        self.assertEquals(eg_local.locale, 'spa_ES')

        l.delete()
        self.assertEquals(self.workspace.S(eg_models.Localisation).count(), 0)
<|endoftext|>


<commit_before>"""
    Copyright (c) 2015 Ad Schellevis
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    Crawler class to find module (require/include) dependencies
"""
import os
import os.path

class DependancyCrawler(object):
    """ Legacy dependency crawler and grapher
    """
    def __init__(self, root):
        """ init
        :param root: start crawling at
        :return:
        """
        self._all_dependencies = {}
        self._all_dependencies_src = {}
        self._all_functions = {}
        self.root = root

    def get_dependency_by_src(self, src_filename):
        """ dependencies are stored by a single name, this method maps a filename back to it's name
                usually the basename of the file.
        :param src_filename:
        :return:
        """
        if src_filename in self._all_dependencies_src:
            return self._all_dependencies_src[src_filename]
        else:
            return None

    def fetch_php_modules(self, src_filename):
        # create a new list for this base filename
        base_filename = os.path.basename(src_filename)
        if base_filename in self._all_dependencies:
            base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)
        self._all_dependencies[base_filename] = []
        self._all_dependencies_src[src_filename] = base_filename

        source_data = open(src_filename).read()
        # fetch all include, include_once, require, require_once statements and
        # add dependencies to object dependency list.
        for tag in ('include', 'require'):
            data = source_data
            while True:
                startpos = data.find(tag)
                if startpos == -1:
                    break
                else:
                    strlen = data[startpos:].find(';')
                    if strlen > -1:
                        # parse (single) statement, check if this could be an include type command
                        dep_stmt = data[startpos-1:strlen+startpos]
                        if dep_stmt[0] in (' ', '\n'):
                            dep_stmt = dep_stmt[1:].replace("'", '"')
                            if dep_stmt.find('\n') == -1 and dep_stmt.count('"') == 2:
                                dep_filename = dep_stmt.split('"')[1]
                                if dep_filename not in self._all_dependencies[base_filename]:
                                    self._all_dependencies[base_filename].append(dep_filename)
                        data = data[strlen+startpos:]

    def fetch_php_functions(self, src_filename):
        """ find php functions
        :param src_filename:
        :return:
        """
        base_filename = os.path.basename(src_filename)
        if base_filename in self._all_functions:
            base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)

        function_list = []
        for line in open(src_filename,'r').read().split('\n'):
            if line.find('function ') > -1 and line.find('(') > -1:
                if line.find('*') > -1 and line.find('function') > line.find('*'):
                    continue
                function_nm = line.split('(')[0].split(' ')[-1]
                function_list.append(function_nm)

        self._all_functions[base_filename] = function_list

    def find_files(self, analyse_dirs=('etc','www', 'captiveportal', 'sbin')):
        """
        :param analyse_dirs: directories to analyse
        :return:
        """
        for analyse_dir in analyse_dirs:
            analyse_dir = ('%s/%s' % (self.root, analyse_dir)).replace('//', '/')
            for wroot, wdirs, wfiles in os.walk(analyse_dir):
                for src_filename in wfiles:
                    src_filename = '%s/%s' % (wroot, src_filename)
                    if src_filename.split('.')[-1] in ('php', 'inc','class') \
                            or open(src_filename).read(1024).find('/bin/php') > -1:
                        yield src_filename

    def crawl(self):
        """ Crawl through legacy code
        :param analyse_dirs: only analyse these directories
        :return: None
        """
        for src_filename in self.find_files():
            self.fetch_php_modules(src_filename)
            self.fetch_php_functions(src_filename)

    def where_used(self, src):
        """
        :param src: source object name (base name)
        :return: dictionary containing files and functions
        """
        where_used_lst={}
        for src_filename in self.find_files():
            data = open(src_filename,'r').read().replace('\n',' ')
            use_list = []
            for function in self._all_functions[src]:
                if data.find(' %s(' % (function)) > -1 or \
                                data.find('!%s ') > -1 or \
                                data.find('!%s(') > -1 or \
                                data.find('(%s(') > -1 or \
                                data.find('(%s ') > -1 or \
                                data.find(' %s ' % (function)) > -1:
                    use_list.append(function)

            if len(use_list) > 0:
                where_used_lst[src_filename] = sorted(use_list)

        return where_used_lst

    def get_total_files(self):
        """ get total number of analysed files
        :return: int
        """
        return len(self._all_dependencies)

    def get_total_dependencies(self):
        """ get total number of dependencies
        :return: int
        """
        count = 0
        for src_filename in self._all_dependencies:
            count += len(self._all_dependencies[src_filename])
        return count

    def get_files(self):
        """ retrieve all analysed files as iterator (ordered by name)
        :return: iterator
        """
        for src_filename in sorted(self._all_dependencies):
            yield src_filename

    def trace(self, src_filename, parent_filename=None, result=None, level=0):
        """ trace dependencies (recursive)
        :param src_filename:
        :param parent_filename:
        :param result:
        :param level:
        :return:
        """
        if result is None:
            result = {}
        if src_filename not in result:
            result[src_filename] = {'level': level, 'dup': list(), 'parent': parent_filename}
        else:
            result[src_filename]['dup'].append(parent_filename)
            return

        if src_filename in self._all_dependencies:
            for dependency in self._all_dependencies[src_filename]:
                self.trace(dependency, src_filename, result, level=level+1)

        return result

    def file_info(self, src_filename):
        """ retrieve file info, like maximum recursive depth and number of duplicate dependencies
        :param src_filename:
        :return:
        """
        result = {'levels': 0,'dup_count':0}
        if src_filename in self._all_dependencies:
            data = self.trace(src_filename)
            for dep_filename in data:
                if data[dep_filename]['level'] > result['levels']:
                    result['levels'] = data[dep_filename]['level']
                result['dup_count'] += len(data[dep_filename]['dup'])

        return result

    def generate_dot(self, filename_to_inspect):
        """ convert trace data to do graph
        :param filename_to_inspect: source filename to generate graph for
        :return: string (dot) data
        """
        trace_data = self.trace(filename_to_inspect)
        result = list()
        result.append('digraph dependencies {')
        result.append('\toverlap=scale;')
        nodes = {}
        for level in range(100):
            for src_filename in trace_data:
                if trace_data[src_filename]['level'] == level:
                    if trace_data[src_filename]['parent'] is not None:
                        result.append('\tedge [color=black style=filled];')
                        result.append('\t"%s" -> "%s" [weight=%d];' % (trace_data[src_filename]['parent'],
                                                                       src_filename, trace_data[src_filename]['level']))
                        if len(trace_data[src_filename]['dup']) > 0:
                            for target in trace_data[src_filename]['dup']:
                                result.append('\tedge [color=red style=dotted];')
                                result.append('\t"%s" -> "%s";' % (target, src_filename))

                    if trace_data[src_filename]['parent'] is None:
                        nodes[src_filename] = '[shape=Mdiamond]'
                    elif len(trace_data[src_filename]['dup']) > 0:
                        nodes[src_filename] = '[shape=box,style=filled,color=".7 .3 1.0"]'
                    else:
                        nodes[src_filename] = '[shape=box]'

        for node in nodes:
            result.append('\t"%s" %s;' % (node, nodes[node]))

        result.append('}')
        return '\n'.join(result)

    @staticmethod
    def generate_index_html(filelist):
        html_body = "<title></title>

Name
\n%s"
        html_row = '
%s
\n'
        html = html_body % ('\n'.join(map(lambda x: html_row % (x, x), sorted(filelist))))

        return html

<commit_msg>(dev) fix function search<commit_after>"""
    Copyright (c) 2015 Ad Schellevis
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    Crawler class to find module (require/include) dependencies
"""
import os
import os.path

class DependancyCrawler(object):
    """ Legacy dependency crawler and grapher
    """
    def __init__(self, root):
        """ init
        :param root: start crawling at
        :return:
        """
        self._all_dependencies = {}
        self._all_dependencies_src = {}
        self._all_functions = {}
        self.root = root

    def get_dependency_by_src(self, src_filename):
        """ dependencies are stored by a single name, this method maps a filename back to it's name
                usually the basename of the file.
        :param src_filename:
        :return:
        """
        if src_filename in self._all_dependencies_src:
            return self._all_dependencies_src[src_filename]
        else:
            return None

    def fetch_php_modules(self, src_filename):
        # create a new list for this base filename
        base_filename = os.path.basename(src_filename)
        if base_filename in self._all_dependencies:
            base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)
        self._all_dependencies[base_filename] = []
        self._all_dependencies_src[src_filename] = base_filename

        source_data = open(src_filename).read()
        # fetch all include, include_once, require, require_once statements and
        # add dependencies to object dependency list.
        for tag in ('include', 'require'):
            data = source_data
            while True:
                startpos = data.find(tag)
                if startpos == -1:
                    break
                else:
                    strlen = data[startpos:].find(';')
                    if strlen > -1:
                        # parse (single) statement, check if this could be an include type command
                        dep_stmt = data[startpos-1:strlen+startpos]
                        if dep_stmt[0] in (' ', '\n'):
                            dep_stmt = dep_stmt[1:].replace("'", '"')
                            if dep_stmt.find('\n') == -1 and dep_stmt.count('"') == 2:
                                dep_filename = dep_stmt.split('"')[1]
                                if dep_filename not in self._all_dependencies[base_filename]:
                                    self._all_dependencies[base_filename].append(dep_filename)
                        data = data[strlen+startpos:]

    def fetch_php_functions(self, src_filename):
        """ find php functions
        :param src_filename:
        :return:
        """
        base_filename = os.path.basename(src_filename)
        if base_filename in self._all_functions:
            base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)

        function_list = []
        for line in open(src_filename,'r').read().split('\n'):
            if line.find('function ') > -1 and line.find('(') > -1:
                if line.find('*') > -1 and line.find('function') > line.find('*'):
                    continue
                function_nm = line.split('(')[0].split(' ')[-1].strip()
                function_list.append(function_nm)

        self._all_functions[base_filename] = function_list

    def find_files(self, analyse_dirs=('etc','www', 'captiveportal', 'sbin')):
        """
        :param analyse_dirs: directories to analyse
        :return:
        """
        for analyse_dir in analyse_dirs:
            analyse_dir = ('%s/%s' % (self.root, analyse_dir)).replace('//', '/')
            for wroot, wdirs, wfiles in os.walk(analyse_dir):
                for src_filename in wfiles:
                    src_filename = '%s/%s' % (wroot, src_filename)
                    if src_filename.split('.')[-1] in ('php', 'inc','class') \
                            or open(src_filename).read(1024).find('/bin/php') > -1:
                        yield src_filename

    def crawl(self):
        """ Crawl through legacy code
        :param analyse_dirs: only analyse these directories
        :return: None
        """
        for src_filename in self.find_files():
            self.fetch_php_modules(src_filename)
            self.fetch_php_functions(src_filename)

    def where_used(self, src):
        """
        :param src: source object name (base name)
        :return: dictionary containing files and functions
        """
        where_used_lst={}
        for src_filename in self.find_files():
            data = open(src_filename,'r').read().replace('\n',' ')
            use_list = []
            for function in self._all_functions[src]:
                if data.find(' %s(' % (function)) > -1 or \
                                data.find('!%s ' % (function)) > -1 or \
                                data.find('!%s(' % (function)) > -1 or \
                                data.find('(%s(' % (function)) > -1 or \
                                data.find('(%s ' % (function)) > -1 or \
                                data.find(' %s ' % (function)) > -1:
                    use_list.append(function)

            if len(use_list) > 0:
                where_used_lst[src_filename] = sorted(use_list)

        return where_used_lst

    def get_total_files(self):
        """ get total number of analysed files
        :return: int
        """
        return len(self._all_dependencies)

    def get_total_dependencies(self):
        """ get total number of dependencies
        :return: int
        """
        count = 0
        for src_filename in self._all_dependencies:
            count += len(self._all_dependencies[src_filename])
        return count

    def get_files(self):
        """ retrieve all analysed files as iterator (ordered by name)
        :return: iterator
        """
        for src_filename in sorted(self._all_dependencies):
            yield src_filename

    def trace(self, src_filename, parent_filename=None, result=None, level=0):
        """ trace dependencies (recursive)
        :param src_filename:
        :param parent_filename:
        :param result:
        :param level:
        :return:
        """
        if result is None:
            result = {}
        if src_filename not in result:
            result[src_filename] = {'level': level, 'dup': list(), 'parent': parent_filename}
        else:
            result[src_filename]['dup'].append(parent_filename)
            return

        if src_filename in self._all_dependencies:
            for dependency in self._all_dependencies[src_filename]:
                self.trace(dependency, src_filename, result, level=level+1)

        return result

    def file_info(self, src_filename):
        """ retrieve file info, like maximum recursive depth and number of duplicate dependencies
        :param src_filename:
        :return:
        """
        result = {'levels': 0,'dup_count':0}
        if src_filename in self._all_dependencies:
            data = self.trace(src_filename)
            for dep_filename in data:
                if data[dep_filename]['level'] > result['levels']:
                    result['levels'] = data[dep_filename]['level']
                result['dup_count'] += len(data[dep_filename]['dup'])

        return result

    def generate_dot(self, filename_to_inspect):
        """ convert trace data to do graph
        :param filename_to_inspect: source filename to generate graph for
        :return: string (dot) data
        """
        trace_data = self.trace(filename_to_inspect)
        result = list()
        result.append('digraph dependencies {')
        result.append('\toverlap=scale;')
        nodes = {}
        for level in range(100):
            for src_filename in trace_data:
                if trace_data[src_filename]['level'] == level:
                    if trace_data[src_filename]['parent'] is not None:
                        result.append('\tedge [color=black style=filled];')
                        result.append('\t"%s" -> "%s" [weight=%d];' % (trace_data[src_filename]['parent'],
                                                                       src_filename, trace_data[src_filename]['level']))
                        if len(trace_data[src_filename]['dup']) > 0:
                            for target in trace_data[src_filename]['dup']:
                                result.append('\tedge [color=red style=dotted];')
                                result.append('\t"%s" -> "%s";' % (target, src_filename))

                    if trace_data[src_filename]['parent'] is None:
                        nodes[src_filename] = '[shape=Mdiamond]'
                    elif len(trace_data[src_filename]['dup']) > 0:
                        nodes[src_filename] = '[shape=box,style=filled,color=".7 .3 1.0"]'
                    else:
                        nodes[src_filename] = '[shape=box]'

        for node in nodes:
            result.append('\t"%s" %s;' % (node, nodes[node]))

        result.append('}')
        return '\n'.join(result)

    @staticmethod
    def generate_index_html(filelist):
        html_body = "<title></title>

Name
\n%s"
        html_row = '
%s
\n'
        html = html_body % ('\n'.join(map(lambda x: html_row % (x, x), sorted(filelist))))

        return html

<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

import sys
import re

"""Baby Names exercise

Define the extract_names() function below and change main()
to call it.

For writing regex, it's nice to include a copy of the target
text for inspiration.

Here's what the html looks like in the baby.html files:
...

Popularity in 1990

....

1	Michael	Jessica

2	Christopher	Ashley

3	Matthew	Brittany
...

Suggested milestones for incremental development:
 -Extract the year and print it
 -Extract the names and rank numbers and just print them
 -Get the names data into a dict and print it
 -Build the [year, 'name rank', ... ] list and print it
 -Fix main() to use the extract_names list
"""

def extract_names(filename):
    """
    Given a file name for baby.html, returns a list starting with the year string
    followed by the name-rank strings in alphabetical order.
    ['2006', 'Aaliyah 91', Aaron 57', 'Abagail 895', ' ...]
    """
    # +++your code here+++
    return

def main():
    # This command-line parsing code is provided.
    # Make a list of command line arguments, omitting the [0] element
    # which is the script itself.
    args = sys.argv[1:]

    if not args:
        print('usage: [--summaryfile] file [file ...]')
        sys.exit(1)

    # Notice the summary flag and remove it from args if it is present.
    summary = False
    if args[0] == '--summaryfile':
        summary = True
        del args[0]

        # +++your code here+++
        # For each filename, get the names, then either print the text output
        # or write it to a summary file

if __name__ == '__main__':
    main()
<commit_msg>babynames resolved It's able to provide the required output without using dict, however dict will be implement in the future to query names and get its rank.<commit_after>#!/usr/bin/python
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

import sys
import re

"""Baby Names exercise

Define the extract_names() function below and change main()
to call it.

For writing regex, it's nice to include a copy of the target
text for inspiration.

Here's what the html looks like in the baby.html files:
...

Popularity in 1990

....

1	Michael	Jessica

2	Christopher	Ashley

3	Matthew	Brittany
...

Suggested milestones for incremental development:
 -Extract the year and print it
 -Extract the names and rank numbers and just print them
 -Get the names data into a dict and print it
 -Build the [year, 'name rank', ... ] list and print it
 -Fix main() to use the extract_names list
"""

def extract_names(filename):
    """
    Given a file name for baby.html, returns a list starting with the year string
    followed by the name-rank strings in alphabetical order.
    ['2006', 'Aaliyah 91', Aaron 57', 'Abagail 895', ' ...]
    """
    name_list = []
    # regex to capture the year in the text file
    regex1 = 'name="year".*value="([0-9]+)">'
    # regex to capture rank, male and female names
    regex2 = '([0-9]+)	([A-Za-z]+)	([A-Za-z]+)	'
    # iterate over the given files
    for file in filename:
        baby_names = []
        with open(file, 'r') as fp:
            text = fp.read()

            year = re.search(regex1, text)
            year = year.group(1)
            # In case year is found proceed otherwise the pattern may incorrect
            # it's better to abort
            if year:
                baby_names.append(year)
            else:
                assert "Unable to find year on file - different pattern"
            # Hunt for the names and append to the list
            for line in text.splitlines():
                name = re.search(regex2, line)
                if name:
                    rank = name.group(1)
                    male_name = name.group(2)
                    female_name = name.group(3)

                    baby_names.append('{} {}'.format(male_name, rank))
                    baby_names.append('{} {}'.format(female_name, rank))

        fp.close()

        baby_names.sort()

        name_list.append(baby_names)

    return name_list

def main():
    # This command-line parsing code is provided.
    # Make a list of command line arguments, omitting the [0] element
    # which is the script itself.
    args = sys.argv[1:]

    if not args:
        print('usage: [--summaryfile] file [file ...]')
        sys.exit(1)

    # Notice the summary flag and remove it from args if it is present.
    summary = False
    if args[0] == '--summaryfile':
        summary = True
        del args[0]
        # Print each one of the list returned by extract_names function
        for name_list in extract_names(args):
            print(name_list)

if __name__ == '__main__':
    main()
<|endoftext|>