<commit_before>import os
import base64
from collections import defaultdict
from netaddr import IPAddress, IPNetwork
from django.db.models import F, Q
from planetstack.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.site import Deployment, SiteDeployments
from core.models.slice import Slice, SliceDeployments
from core.models.userdeployments import UserDeployments
from util.logger import Logger, logging

logger = Logger(level=logging.INFO)

class SyncSliceDeployments(OpenStackSyncStep):
    provides=[SliceDeployments]
    requested_interval=0

    def fetch_pending(self, deleted):
        if (deleted):
            return SliceDeployments.deleted_objects.all()
        else:
            # slice deployments are not visible to users. We must ensure
            # slices are deployed at all deploymets available to their site.
            site_deployments = SiteDeployments.objects.all()
            site_deploy_lookup = defaultdict(list)
            for site_deployment in site_deployments:
                site_deploy_lookup[site_deployment.site].append(site_deployment.deployment)

            slice_deployments = SliceDeployments.objects.all()
            slice_deploy_lookup = defaultdict(list)
            for slice_deployment in slice_deployments:
                slice_deploy_lookup[slice_deployment.slice].append(slice_deployment.deployment)

            all_deployments = Deployment.objects.all()
            for slice in Slice.objects.all():
                # slices are added to all deployments for now
                expected_deployments = all_deployments
                #expected_deployments = site_deploy_lookup[slice.site]
                for expected_deployment in expected_deployments:
                    if slice not in slice_deploy_lookup or \
                       expected_deployment not in slice_deploy_lookup[slice]:
                        sd = SliceDeployments(slice=slice, deployment=expected_deployment)
                        sd.save()

            # now we can return all slice deployments that need to be enacted
            return SliceDeployments.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))

    def get_next_subnet(self, deployment=None):
        # limit ourself to 10.0.x.x for now
        valid_subnet = lambda net: net.startswith('10.0')
        driver = self.driver.admin_driver(deployment=deployment)
        subnets = driver.shell.quantum.list_subnets()['subnets']
        ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
                if valid_subnet(subnet['cidr'])]
        ints.sort()
        if ints:
            last_ip = IPAddress(ints[-1])
        else:
            last_ip = IPAddress('10.0.0.1')
        last_ip = IPAddress(ints[-1])
        last_network = IPNetwork(str(last_ip) + "/24")
        next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
        return next_network

    def sync_record(self, slice_deployment):
        logger.info("sync'ing slice deployment %s" % slice_deployment)
        if not slice_deployment.tenant_id:
            nova_fields = {'tenant_name': slice_deployment.slice.name,
                   'description': slice_deployment.slice.description,
                   'enabled': slice_deployment.slice.enabled}
            driver = self.driver.admin_driver(deployment=slice_deployment.deployment.name)
            tenant = driver.create_tenant(**nova_fields)
            slice_deployment.tenant_id = tenant.id

            # XXX give caller an admin role at the tenant they've created
            deployment_users = UserDeployments.objects.filter(user=slice_deployment.slice.creator,
                                                             deployment=slice_deployment.deployment)
            if not deployment_users:
                logger.info("slice createor %s has not accout at deployment %s" % (slice_deployment.slice.creator, slice_deployment.deployment.name))
            else:
                deployment_user = deployment_users[0]
                # lookup user id at this deployment
                kuser= driver.shell.keystone.users.find(email=slice_deployment.slice.creator.email)

                # add required roles at the slice's tenant
                driver.add_user_role(kuser.id, tenant.id, 'admin')

                # refresh credentials using this tenant
                client_driver = self.driver.client_driver(caller=deployment_user.user,
                                                          tenant=tenant.name,
                                                          deployment=slice_deployment.deployment.name)

        if slice_deployment.id and slice_deployment.tenant_id:
            # update existing tenant
            driver = self.driver.admin_driver(deployment=slice_deployment.deployment.name)
            driver.update_tenant(slice_deployment.tenant_id,
                                 description=slice_deployment.slice.description,
                                 enabled=slice_deployment.slice.enabled)

        if slice_deployment.tenant_id:
            # update slice/tenant quota
            driver = self.driver.client_driver(deployment=slice_deployment.deployment.name, tenant=slice_deployment.slice.name)
            driver.shell.nova.quotas.update(tenant_id=slice_deployment.tenant_id, instances=int(slice_deployment.slice.max_slivers))

        slice_deployment.save()

    def delete_record(self, slice_deployment):
        user = User.objects.get(id=slice_deployment.slice.creator.id)
        driver = OpenStackDriver().admin_driver(deployment=slice_deployment.deployment.name)
        client_driver = driver.client_driver(caller=user,
                                             tenant=slice_deployment.slice.name,
                                             deployment=slice_deployment.deployment.name)

        if slice_deployment.router_id and slice_deployment.subnet_id:
            client_driver.delete_router_interface(slice_deployment.router_id, slice_deployment.subnet_id)
        if slice_deployment.subnet_id:
            client_driver.delete_subnet(slice_deployment.subnet_id)
        if slice_deployment.router_id:
            client_driver.delete_router(slice_deployment.router_id)
        if slice_deployment.network_id:
            client_driver.delete_network(slice_deployment.network_id)
        if slice_deployment.tenant_id:
            driver.delete_tenant(slice_deployment.tenant_id)
        # delete external route
        #subnet = None
        #subnets = client_driver.shell.quantum.list_subnets()['subnets']
        #for snet in subnets:
        #    if snet['id'] == slice_deployment.subnet_id:
        #        subnet = snet
        #if subnet:
        #    driver.delete_external_route(subnet)
<commit_msg>Drop Slice policy code from Observer<commit_after>import os
import base64
from collections import defaultdict
from netaddr import IPAddress, IPNetwork
from django.db.models import F, Q
from planetstack.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.site import Deployment, SiteDeployments
from core.models.slice import Slice, SliceDeployments
from core.models.userdeployments import UserDeployments
from util.logger import Logger, logging

logger = Logger(level=logging.INFO)

class SyncSliceDeployments(OpenStackSyncStep):
    provides=[SliceDeployments]
    requested_interval=0

    def fetch_pending(self, deleted):
        if (deleted):
            return SliceDeployments.deleted_objects.all()
        else:
            return SliceDeployments.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None))

    def get_next_subnet(self, deployment=None):
        # limit ourself to 10.0.x.x for now
        valid_subnet = lambda net: net.startswith('10.0')
        driver = self.driver.admin_driver(deployment=deployment)
        subnets = driver.shell.quantum.list_subnets()['subnets']
        ints = [int(IPNetwork(subnet['cidr']).ip) for subnet in subnets \
                if valid_subnet(subnet['cidr'])]
        ints.sort()
        if ints:
            last_ip = IPAddress(ints[-1])
        else:
            last_ip = IPAddress('10.0.0.1')
        last_ip = IPAddress(ints[-1])
        last_network = IPNetwork(str(last_ip) + "/24")
        next_network = IPNetwork(str(IPAddress(last_network) + last_network.size) + "/24")
        return next_network

    def sync_record(self, slice_deployment):
        logger.info("sync'ing slice deployment %s" % slice_deployment)
        if not slice_deployment.tenant_id:
            nova_fields = {'tenant_name': slice_deployment.slice.name,
                   'description': slice_deployment.slice.description,
                   'enabled': slice_deployment.slice.enabled}
            driver = self.driver.admin_driver(deployment=slice_deployment.deployment.name)
            tenant = driver.create_tenant(**nova_fields)
            slice_deployment.tenant_id = tenant.id

            # XXX give caller an admin role at the tenant they've created
            deployment_users = UserDeployments.objects.filter(user=slice_deployment.slice.creator,
                                                             deployment=slice_deployment.deployment)
            if not deployment_users:
                logger.info("slice createor %s has not accout at deployment %s" % (slice_deployment.slice.creator, slice_deployment.deployment.name))
            else:
                deployment_user = deployment_users[0]
                # lookup user id at this deployment
                kuser= driver.shell.keystone.users.find(email=slice_deployment.slice.creator.email)

                # add required roles at the slice's tenant
                driver.add_user_role(kuser.id, tenant.id, 'admin')

                # refresh credentials using this tenant
                client_driver = self.driver.client_driver(caller=deployment_user.user,
                                                          tenant=tenant.name,
                                                          deployment=slice_deployment.deployment.name)

        if slice_deployment.id and slice_deployment.tenant_id:
            # update existing tenant
            driver = self.driver.admin_driver(deployment=slice_deployment.deployment.name)
            driver.update_tenant(slice_deployment.tenant_id,
                                 description=slice_deployment.slice.description,
                                 enabled=slice_deployment.slice.enabled)

        if slice_deployment.tenant_id:
            # update slice/tenant quota
            driver = self.driver.client_driver(deployment=slice_deployment.deployment.name, tenant=slice_deployment.slice.name)
            driver.shell.nova.quotas.update(tenant_id=slice_deployment.tenant_id, instances=int(slice_deployment.slice.max_slivers))

        slice_deployment.save()

    def delete_record(self, slice_deployment):
        user = User.objects.get(id=slice_deployment.slice.creator.id)
        driver = OpenStackDriver().admin_driver(deployment=slice_deployment.deployment.name)
        client_driver = driver.client_driver(caller=user,
                                             tenant=slice_deployment.slice.name,
                                             deployment=slice_deployment.deployment.name)

        if slice_deployment.router_id and slice_deployment.subnet_id:
            client_driver.delete_router_interface(slice_deployment.router_id, slice_deployment.subnet_id)
        if slice_deployment.subnet_id:
            client_driver.delete_subnet(slice_deployment.subnet_id)
        if slice_deployment.router_id:
            client_driver.delete_router(slice_deployment.router_id)
        if slice_deployment.network_id:
            client_driver.delete_network(slice_deployment.network_id)
        if slice_deployment.tenant_id:
            driver.delete_tenant(slice_deployment.tenant_id)
        # delete external route
        #subnet = None
        #subnets = client_driver.shell.quantum.list_subnets()['subnets']
        #for snet in subnets:
        #    if snet['id'] == slice_deployment.subnet_id:
        #        subnet = snet
        #if subnet:
        #    driver.delete_external_route(subnet)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Executable and reusable sample for retrieving a list of feeds."""

import argparse
from typing import Mapping, Any

from google.auth.transport import requests

from common import chronicle_auth
from common import regions

CHRONICLE_API_BASE_URL = "https://backstory.googleapis.com"

def generate_files(http_session: requests.AuthorizedSession,
                   name: str) -> Mapping[str, Any]:
  """Retrieves all forwarders for the tenant.

  Args:
    http_session: Authorized session for HTTP requests.
    name: Unique name for the forwarder.

  Returns:
    Array containing each line of the forwarder's content.

  Raises:
    requests.exceptions.HTTPError: HTTP request resulted in an error
      (response.status_code >= 400).
  """
  url = f"{CHRONICLE_API_BASE_URL}/v2/{name}:generateForwarderFiles"

  response = http_session.request("GET", url)
  # Expected server response:
  # {
  #   "config": "<CONFIG FILE CONTENTS>",
  #   "auth": "<AUTH FILE CONTENTS>",
  # }

  if response.status_code >= 400:
    print(response.text)
  response.raise_for_status()
  return response.json()

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  chronicle_auth.add_argument_credentials_file(parser)
  regions.add_argument_region(parser)
  parser.add_argument(
      "-n",
      "--name",
      type=str,
      required=True,
      help="resource name for the Forwarder (forwarders/{UUID})")
  parser.add_argument(
      "-o",
      "--output",
      type=str,
      help="Writes configuration files to the specified output directory.")

  args = parser.parse_args()
  CHRONICLE_API_BASE_URL = regions.url(CHRONICLE_API_BASE_URL, args.region)
  session = chronicle_auth.initialize_http_session(args.credentials_file)
  res = generate_files(session, args.name)

  verbose = args.verbose
  output_dir = args.output
  if "config" in res:
    config = res["config"]
    if output_dir:
      with open(output_dir + "/forwarder.conf", "w") as f:
        f.write(config)
    else:
      print("forwarder.conf:\n", config)
  if "auth" in res:
    auth = res["auth"]
    if output_dir:
      with open(output_dir + "/forwarder_auth.conf", "w") as f:
        f.write(auth)
    else:
      print("forwarder_auth.conf:\n", auth)
<commit_msg>Internal change<commit_after>#!/usr/bin/env python3

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Executable and reusable sample for retrieving a list of feeds."""

import argparse
from typing import Mapping, Any

from google.auth.transport import requests

from common import chronicle_auth
from common import regions

CHRONICLE_API_BASE_URL = "https://backstory.googleapis.com"

def generate_files(http_session: requests.AuthorizedSession,
                   name: str) -> Mapping[str, Any]:
  """Retrieves all forwarders for the tenant.

  Args:
    http_session: Authorized session for HTTP requests.
    name: Unique name for the forwarder.

  Returns:
    Array containing each line of the forwarder's content.

  Raises:
    requests.exceptions.HTTPError: HTTP request resulted in an error
      (response.status_code >= 400).
  """
  url = f"{CHRONICLE_API_BASE_URL}/v2/{name}:generateForwarderFiles"

  response = http_session.request("GET", url)
  # Expected server response:
  # {
  #   "config": "<CONFIG FILE CONTENTS>",
  #   "auth": "<AUTH FILE CONTENTS>",
  # }

  if response.status_code >= 400:
    print(response.text)
  response.raise_for_status()
  return response.json()

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  chronicle_auth.add_argument_credentials_file(parser)
  regions.add_argument_region(parser)
  parser.add_argument(
      "-n",
      "--name",
      type=str,
      required=True,
      help="resource name for the Forwarder (forwarders/{UUID})")
  parser.add_argument(
      "-o",
      "--output",
      type=str,
      help="Writes configuration files to the specified output directory.")

  args = parser.parse_args()
  CHRONICLE_API_BASE_URL = regions.url(CHRONICLE_API_BASE_URL, args.region)
  session = chronicle_auth.initialize_http_session(args.credentials_file)
  res = generate_files(session, args.name)

  output_dir = args.output
  if "config" in res:
    config = res["config"]
    if output_dir:
      with open(output_dir + "/forwarder.conf", "w") as f:
        f.write(config)
    else:
      print("forwarder.conf:\n", config)
  if "auth" in res:
    auth = res["auth"]
    if output_dir:
      with open(output_dir + "/forwarder_auth.conf", "w") as f:
        f.write(auth)
    else:
      print("forwarder_auth.conf:\n", auth)
<|endoftext|>


<commit_before>
from fparser import api

def test_reproduce_issue():
    source_str = '''\
      subroutine bndfp()
      use m_struc_def
C-
C
C
C
C
C
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    s, u, c, e = tree[:3]+tree[-1:]
    assert s.span==(1,1),`s.span`
    assert u.span==(2,2),`u.span`
    assert c.span==(3,3),`c.span`
    assert e.span==(9,9),`e.span`

def test_reproduce_issue_fix77():
    source_str = '''\
      subroutine foo()
      real a
c
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=True)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert comment.span==(3,3)
    assert end.span==(5,5),`end.span`

def test_reproduce_issue_fix90():
    source_str = '''\
      subroutine foo()
      real a
c 1
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment,end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert end.span==(5,5),`end.span`

    source_str = '''\
      subroutine foo()
      real a
c-
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment,end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert end.span==(5,5),`end.span`

    source_str = '''\
      subroutine foo()
      real a
c
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert comment.span == (3,3)
    assert end.span==(5,5),`end.span`

def test_comment_cont_fix90():
    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c
     & a
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c 2
     &,b
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, ab, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert ab.span==(2,6),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(7,7)
<commit_msg>fixed test after import<commit_after>
from fparser import api

def test_reproduce_issue():
    source_str = '''\
      subroutine bndfp()
      use m_struc_def
C-
C
C
C
C
C
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    s, u, c, e = tree[:3]+tree[-1:]
    assert s.span==(1,1),`s.span`
    assert u.span==(2,2),`u.span`
    assert c.span==(3,3),`c.span`
    assert e.span==(9,9),`e.span`

def test_reproduce_issue_fix77():
    source_str = '''\
      subroutine foo()
      real a
c
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=True)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(5,5),`comment.span`
    assert end.span==(5,5),`end.span`

def test_reproduce_issue_fix90():
    source_str = '''\
      subroutine foo()
      real a
c 1
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment,end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert end.span==(5,5),`end.span`

    source_str = '''\
      subroutine foo()
      real a
c-
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment,end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert end.span==(5,5),`end.span`

    source_str = '''\
      subroutine foo()
      real a
c
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,2),`a.span`
    assert comment.span == (3,3)
    assert end.span==(5,5),`end.span`

def test_comment_cont_fix90():
    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c
     & a
c 2
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, a, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert a.span==(2,4),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(6,6)

    source_str = '''\
          subroutine foo()
      real
c 1
     & a
c 2
     &,b
      end
'''
    tree = api.get_reader(source_str, isfree=False, isstrict=False)
    tree = list(tree)
    foo, ab, comment, end = tree[:3]+tree[-1:]
    assert foo.span==(1,1)
    assert ab.span==(2,6),`a.span`
    assert comment.span==(3,3),`comment.span`
    assert end.span==(7,7)
<|endoftext|>


<commit_before># Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import monocyte.handler

REMOVE_WARNING = "WARNING: region '%s' not allowed!"
IGNORED_REGIONS = ["cn-north-1", "us-gov-west-1", "us-east-1", "us-west-2"]
ALLOWED_REGIONS_STARTS_WITH = "eu"

class Monocyte(object):

    def __init__(self):
        self.problematic_resources = []

    def is_region_allowed(self, region):
        return region.lower().startswith(ALLOWED_REGIONS_STARTS_WITH)

    def is_region_ignored(self, region):
        return region.lower() in IGNORED_REGIONS

    def is_region_handled(self, region):
        return not self.is_region_allowed(region) and not self.is_region_ignored(region)

    def search_and_destroy_unwanted_resources(self, handler_names, dry_run=True):
        if dry_run:
            print(" DRY RUN " * 8)
            print()

        handler_classes = fetch_all_handler_classes()
        specific_handlers = self.instanciate_handlers(handler_classes, handler_names, dry_run)

        print("              aws handlers: {0}".format(" -> ".join(handler_names)))
        print("allowed regions start with: {0}".format(ALLOWED_REGIONS_STARTS_WITH))
        print("           ignored regions: {0}".format(" ".join(IGNORED_REGIONS)))

        for specific_handler in specific_handlers:
            print("\n---- checking %s resources" % specific_handler.name)
            self.handle_service(specific_handler)

        if self.problematic_resources:
            print("\nproblems encountered while deleting the following resources:")
            for resource, service_handler, exception in self.problematic_resources:
                print("{0:10s} {1}".format(resource.region, service_handler.name))
            return 1
        return 0

    def handle_service(self, specific_handler):
        for resource in specific_handler.fetch_unwanted_resources():
            if not self.is_region_allowed(resource.region):
                print("\n%s\n\t%s" % (
                    specific_handler.to_string(resource),
                    REMOVE_WARNING % resource.region))
                try:
                    specific_handler.delete(resource)
                except BaseException as e:
                    print("\t{0}".format(e))
                    self.problematic_resources.append((resource, specific_handler, e))

    def instanciate_handlers(self, handler_classes, handler_names, dry_run):
        return [handler_classes["monocyte.handler." + handler_name](self.is_region_handled, dry_run)
                for handler_name in handler_names]

def fetch_all_handler_classes():
    subclasses = {}
    work = [monocyte.handler.Handler]
    while work:
        parent = work.pop()
        for child in parent.__subclasses__():
            if child not in subclasses:
                class_name = "%s.%s" % (child.__module__, child.__name__)
                subclasses[class_name] = child
                work.append(child)
    return subclasses
<commit_msg>fixes typo.<commit_after># Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import monocyte.handler

REMOVE_WARNING = "WARNING: region '%s' not allowed!"
IGNORED_REGIONS = ["cn-north-1", "us-gov-west-1", "us-east-1", "us-west-2"]
ALLOWED_REGIONS_STARTS_WITH = "eu"

class Monocyte(object):

    def __init__(self):
        self.problematic_resources = []

    def is_region_allowed(self, region):
        return region.lower().startswith(ALLOWED_REGIONS_STARTS_WITH)

    def is_region_ignored(self, region):
        return region.lower() in IGNORED_REGIONS

    def is_region_handled(self, region):
        return not self.is_region_allowed(region) and not self.is_region_ignored(region)

    def search_and_destroy_unwanted_resources(self, handler_names, dry_run=True):
        if dry_run:
            print(" DRY RUN " * 8)
            print()

        handler_classes = fetch_all_handler_classes()
        specific_handlers = self.instantiate_handlers(handler_classes, handler_names, dry_run)

        print("              aws handlers: {0}".format(" -> ".join(handler_names)))
        print("allowed regions start with: {0}".format(ALLOWED_REGIONS_STARTS_WITH))
        print("           ignored regions: {0}".format(" ".join(IGNORED_REGIONS)))

        for specific_handler in specific_handlers:
            print("\n---- checking %s resources" % specific_handler.name)
            self.handle_service(specific_handler)

        if self.problematic_resources:
            print("\nproblems encountered while deleting the following resources:")
            for resource, service_handler, exception in self.problematic_resources:
                print("{0:10s} {1}".format(resource.region, service_handler.name))
            return 1
        return 0

    def handle_service(self, specific_handler):
        for resource in specific_handler.fetch_unwanted_resources():
            if not self.is_region_allowed(resource.region):
                print("\n%s\n\t%s" % (
                    specific_handler.to_string(resource),
                    REMOVE_WARNING % resource.region))
                try:
                    specific_handler.delete(resource)
                except BaseException as e:
                    print("\t{0}".format(e))
                    self.problematic_resources.append((resource, specific_handler, e))

    def instantiate_handlers(self, handler_classes, handler_names, dry_run):
        return [handler_classes["monocyte.handler." + handler_name](self.is_region_handled, dry_run)
                for handler_name in handler_names]

def fetch_all_handler_classes():
    subclasses = {}
    work = [monocyte.handler.Handler]
    while work:
        parent = work.pop()
        for child in parent.__subclasses__():
            if child not in subclasses:
                class_name = "%s.%s" % (child.__module__, child.__name__)
                subclasses[class_name] = child
                work.append(child)
    return subclasses
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess
import os
import sys
try:
    import ConfigParser as configparser  # Python2
except ImportError:
    import configparser

def load_config():
    # https://docs.python.org/3.4/library/configparser.html
    path = os.path.dirname(__file__)
    config_file = os.path.join(path, 'config.ini')
    assert os.path.exists(config_file), config_file

    config = configparser.ConfigParser()
    config.read(config_file)  # donator.cfg merged here

    #print(config['DEFAULT']['DISC'])
    #print(config['DEFAULT'].getboolean('DISC'))
    #print(config['donator']['email'])

    # TODO set fallback values if config is empty
    # https://docs.python.org/3.4/library/configparser.html#fallback-values

    return config

def get_user_input(sdx_path):
    # XXX configurable user input fields
    config_erase = raw_input("Are you sure to erase \"{0}\"? [y/N] ".format(sdx_path))
    return config_erase

def erasetor(dev):
    subprocess.call(["ls",dev])

def do_erasure(sdx):
    config = load_config()
    erase = config.get('DEFAULT', 'ERASE')

    if erase == "yes":
        print erasetor(sdx)
    elif erase == "ask":
        erase = get_user_input(sdx)
        if erase.lower().strip() == "y" or erase.lower().strip() == "yes":
            print erasetor(sdx)

def main(argv=None):
    device = sys.argv[1]
    do_erasure(device)

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<commit_msg>Data erasure (need to uncomment subprocess...)<commit_after>#!/usr/bin/env python
import subprocess
import os
import sys
try:
    import ConfigParser as configparser  # Python2
except ImportError:
    import configparser

def load_config():
    # https://docs.python.org/3.4/library/configparser.html
    path = os.path.dirname(__file__)
    config_file = os.path.join(path, 'config.ini')
    assert os.path.exists(config_file), config_file

    config = configparser.ConfigParser()
    config.read(config_file)  # donator.cfg merged here

    #print(config['DEFAULT']['DISC'])
    #print(config['DEFAULT'].getboolean('DISC'))
    #print(config['donator']['email'])

    # TODO set fallback values if config is empty
    # https://docs.python.org/3.4/library/configparser.html#fallback-values

    return config

def get_user_input(sdx_path):
    # XXX configurable user input fields
    config_erase = raw_input("Are you sure to erase \"{0}\"? [y/N] ".format(sdx_path))
    return config_erase

def erasetor(dev):
    #erasuring with 0 bits all the harddrive
    #subprocess.call(["shred","-zvn","o",dev"])
    subprocess.call(["ls",dev])

def do_erasure(sdx):
    config = load_config()
    erase = config.get('DEFAULT', 'ERASE')

    if erase == "yes":
        print erasetor(sdx)
    elif erase == "ask":
        erase = get_user_input(sdx)
        if erase.lower().strip() == "y" or erase.lower().strip() == "yes":
            print erasetor(sdx)

def main(argv=None):
    device = sys.argv[1]
    do_erasure(device)

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>from sympy import Eq

from devito.dimension import SubDimension
from devito.equation import DOMAIN, INTERIOR
from devito.ir.equations.algorithms import dimension_sort
from devito.ir.support import (IterationSpace, DataSpace, Interval, IntervalGroup,
                               Any, Stencil, detect_accesses, detect_oobs, detect_io,
                               force_directions, detect_flow_directions, build_intervals,
                               build_iterators)
from devito.symbolics import FrozenExpr
from devito.tools import Pickable, as_tuple, split

__all__ = ['LoweredEq', 'ClusterizedEq', 'DummyEq']

class IREq(object):

    """
    A mixin providing operations common to all :mod:`ir` equation types.
    """

    _state = ('is_Increment', 'ispace', 'dspace', 'conditionals')

    @property
    def is_Scalar(self):
        return self.lhs.is_Symbol

    @property
    def is_Tensor(self):
        return self.lhs.is_Indexed

    @property
    def is_Increment(self):
        return self._is_Increment

    @property
    def ispace(self):
        return self._ispace

    @property
    def dspace(self):
        return self._dspace

    @property
    def dimensions(self):
        # Note: some dimensions may be in the iteration space but not in the
        # data space (e.g., a DerivedDimension); likewise, some dimensions may
        # be in the data space but not in the iteration space (e.g., when a
        # function is indexed with integers only)
        return set(self.dspace.dimensions) | set(self.ispace.dimensions)

    @property
    def conditionals(self):
        return as_tuple(self._conditionals)

    @property
    def directions(self):
        return self.ispace.directions

    @property
    def dtype(self):
        return self.lhs.dtype

    @property
    def grid(self):
        return self.lhs.function.grid if self.is_Tensor else None

    @property
    def state(self):
        return {i: getattr(self, i) for i in self._state}

class LoweredEq(Eq, IREq):

    """
    LoweredEq(sympy.Eq)
    LoweredEq(devito.LoweredEq, **kwargs)
    LoweredEq(lhs, rhs, **kwargs)

    A SymPy equation with associated :class:`IterationSpace` and
    :class:`DataSpace`.

    When created as ``LoweredEq(sympy.Eq)``, the iteration and data spaces are
    automatically derived from analysis of ``expr``.

    When created as ``LoweredEq(devito.LoweredEq, **kwargs)``, the keyword
    arguments can be anything that appears in ``LoweredEq._state`` (i.e.,
    ispace, dspace, ...).

    When created as ``LoweredEq(lhs, rhs, **kwargs)``, *all* keywords in
    ``LoweredEq._state`` must appear in ``kwargs``.
    """

    _state = IREq._state + ('reads', 'writes')

    def __new__(cls, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], LoweredEq):
            # origin: LoweredEq(devito.LoweredEq, **kwargs)
            input_expr = args[0]
            expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.get(i) or getattr(input_expr, i))
            return expr
        elif len(args) == 1 and isinstance(args[0], Eq):
            # origin: LoweredEq(sympy.Eq)
            input_expr = expr = args[0]
        elif len(args) == 2:
            expr = Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
            return expr
        else:
            raise ValueError("Cannot construct LoweredEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))

        # Well-defined dimension ordering
        dimensions = dimension_sort(expr, key=lambda d: not d.is_Time)
        conditionals, ordering = split(dimensions, lambda d: d.is_Conditional)

        # Introduce space sub-dimensions if need to
        region = getattr(input_expr, '_region', DOMAIN)
        if region == INTERIOR:
            mapper = {i: SubDimension.middle("%si" % i, i, 1, 1)
                      for i in ordering if i.is_Space}
            expr = expr.xreplace(mapper)
            for k, v in mapper.items():
                ordering.insert(ordering.index(k) + 1, v)

        # Analyze the expression
        mapper = detect_accesses(expr)
        oobs = detect_oobs(mapper)

        # The iteration space is constructed so that information always flows
        # from an iteration to another (i.e., no anti-dependences are created)
        directions, _ = force_directions(detect_flow_directions(expr), lambda i: Any)
        iterators = build_iterators(mapper)
        intervals = build_intervals(Stencil.union(*mapper.values()))
        intervals = sorted(intervals, key=lambda i: ordering.index(i.dim))
        ispace = IterationSpace([i.zero() for i in intervals], iterators, directions)

        # The data space is relative to the computational domain
        intervals = [i if i.dim in oobs else i.zero() for i in intervals]
        intervals += [Interval(i, 0, 0) for i in ordering if i not in ispace.dimensions]
        parts = {k: IntervalGroup(build_intervals(v)) for k, v in mapper.items() if k}
        dspace = DataSpace(intervals, parts)

        # Finally create the LoweredEq with all metadata attached
        expr = super(LoweredEq, cls).__new__(cls, expr.lhs, expr.rhs, evaluate=False)
        expr._is_Increment = getattr(input_expr, 'is_Increment', False)
        expr._dspace = dspace
        expr._ispace = ispace
        expr._conditionals = conditionals
        expr._reads, expr._writes = detect_io(expr)

        return expr

    @property
    def reads(self):
        return self._reads

    @property
    def writes(self):
        return self._writes

    def xreplace(self, rules):
        return LoweredEq(self.lhs.xreplace(rules), self.rhs.xreplace(rules), **self.state)

    def func(self, *args):
        return super(LoweredEq, self).func(*args, **self.state, evaluate=False)

class ClusterizedEq(Eq, IREq, FrozenExpr, Pickable):

    """
    ClusterizedEq(devito.IREq, **kwargs)
    ClusterizedEq(lhs, rhs, **kwargs)

    A SymPy equation with associated :class:`IterationSpace` and
    :class:`DataSpace`.

    There are two main differences between a :class:`LoweredEq` and a
    ClusterizedEq: ::

        * In a ClusterizedEq, the iteration and data spaces must *always*
          be provided by the caller.
        * A ClusterizedEq is "frozen", meaning that any call to ``xreplace``
          will not trigger re-evaluation (e.g., mathematical simplification)
          of the expression.

    These two properties make a ClusterizedEq suitable for use in a
    :class:`Cluster`.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            # origin: ClusterizedEq(expr, **kwargs)
            input_expr = args[0]
            expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                v = kwargs[i] if i in kwargs else getattr(input_expr, i, None)
                setattr(expr, '_%s' % i, v)
        elif len(args) == 2:
            # origin: ClusterizedEq(lhs, rhs, **kwargs)
            expr = Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
        else:
            raise ValueError("Cannot construct ClusterizedEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))
        return expr

    def func(self, *args, **kwargs):
        return super(ClusterizedEq, self).func(*args, **self.state)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = IREq._state
    __reduce_ex__ = Pickable.__reduce_ex__

class DummyEq(ClusterizedEq):

    """
    DummyEq(expr)
    DummyEq(lhs, rhs)

    A special :class:`ClusterizedEq` that tracks no iteration or data spaces.
    """

    def __new__(cls, *args):
        if len(args) == 1:
            input_expr = args[0]
            assert isinstance(input_expr, Eq)
            obj = LoweredEq(input_expr)
        elif len(args) == 2:
            obj = LoweredEq(Eq(*args, evaluate=False))
        else:
            raise ValueError("Cannot construct DummyEq from args=%s" % str(args))
        return ClusterizedEq.__new__(cls, obj, ispace=obj.ispace, dspace=obj.dspace)
<commit_msg>ir: Refactor Equation to exploit PartialOrderTuple<commit_after>from sympy import Eq

from devito.dimension import SubDimension
from devito.equation import DOMAIN, INTERIOR
from devito.ir.equations.algorithms import dimension_sort
from devito.ir.support import (IterationSpace, DataSpace, Interval, IntervalGroup,
                               Any, Stencil, detect_accesses, detect_oobs, detect_io,
                               force_directions, detect_flow_directions, build_intervals,
                               build_iterators)
from devito.symbolics import FrozenExpr
from devito.tools import Pickable, as_tuple

__all__ = ['LoweredEq', 'ClusterizedEq', 'DummyEq']

class IREq(object):

    """
    A mixin providing operations common to all :mod:`ir` equation types.
    """

    _state = ('is_Increment', 'ispace', 'dspace', 'conditionals')

    @property
    def is_Scalar(self):
        return self.lhs.is_Symbol

    @property
    def is_Tensor(self):
        return self.lhs.is_Indexed

    @property
    def is_Increment(self):
        return self._is_Increment

    @property
    def ispace(self):
        return self._ispace

    @property
    def dspace(self):
        return self._dspace

    @property
    def dimensions(self):
        # Note: some dimensions may be in the iteration space but not in the
        # data space (e.g., a DerivedDimension); likewise, some dimensions may
        # be in the data space but not in the iteration space (e.g., when a
        # function is indexed with integers only)
        return set(self.dspace.dimensions) | set(self.ispace.dimensions)

    @property
    def conditionals(self):
        return as_tuple(self._conditionals)

    @property
    def directions(self):
        return self.ispace.directions

    @property
    def dtype(self):
        return self.lhs.dtype

    @property
    def grid(self):
        return self.lhs.function.grid if self.is_Tensor else None

    @property
    def state(self):
        return {i: getattr(self, i) for i in self._state}

class LoweredEq(Eq, IREq):

    """
    LoweredEq(sympy.Eq)
    LoweredEq(devito.LoweredEq, **kwargs)
    LoweredEq(lhs, rhs, **kwargs)

    A SymPy equation with associated :class:`IterationSpace` and
    :class:`DataSpace`.

    When created as ``LoweredEq(sympy.Eq)``, the iteration and data spaces are
    automatically derived from analysis of ``expr``.

    When created as ``LoweredEq(devito.LoweredEq, **kwargs)``, the keyword
    arguments can be anything that appears in ``LoweredEq._state`` (i.e.,
    ispace, dspace, ...).

    When created as ``LoweredEq(lhs, rhs, **kwargs)``, *all* keywords in
    ``LoweredEq._state`` must appear in ``kwargs``.
    """

    _state = IREq._state + ('reads', 'writes')

    def __new__(cls, *args, **kwargs):
        if len(args) == 1 and isinstance(args[0], LoweredEq):
            # origin: LoweredEq(devito.LoweredEq, **kwargs)
            input_expr = args[0]
            expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.get(i) or getattr(input_expr, i))
            return expr
        elif len(args) == 1 and isinstance(args[0], Eq):
            # origin: LoweredEq(sympy.Eq)
            input_expr = expr = args[0]
        elif len(args) == 2:
            expr = Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
            return expr
        else:
            raise ValueError("Cannot construct LoweredEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))

        # Well-defined dimension ordering
        ordering = dimension_sort(expr)

        # Introduce space sub-dimensions if need to
        region = getattr(input_expr, '_region', DOMAIN)
        if region == INTERIOR:
            mapper = {i: SubDimension.middle("%si" % i, i, 1, 1)
                      for i in ordering if i.is_Space}
            expr = expr.xreplace(mapper)
            for k, v in mapper.items():
                ordering.insert(ordering.index(k) + 1, v)

        # Analyze the expression
        mapper = detect_accesses(expr)
        oobs = detect_oobs(mapper)
        conditionals = [i for i in ordering if i.is_Conditional]

        # The iteration space is constructed so that information always flows
        # from an iteration to another (i.e., no anti-dependences are created)
        directions, _ = force_directions(detect_flow_directions(expr), lambda i: Any)
        iterators = build_iterators(mapper)
        intervals = build_intervals(Stencil.union(*mapper.values()))
        intervals = IntervalGroup(intervals, relations=ordering.relations)
        ispace = IterationSpace(intervals.zero(), iterators, directions)

        # The data space is relative to the computational domain. Note that we
        # are deliberately dropping the intervals ordering (by turning `intervals`
        # into a list), as this is irrelevant (even more: dangerous) for data spaces
        intervals = [i if i.dim in oobs else i.zero() for i in intervals]
        intervals += [Interval(i, 0, 0) for i in ordering
                      if i not in ispace.dimensions + conditionals]
        parts = {k: IntervalGroup(build_intervals(v)) for k, v in mapper.items() if k}
        dspace = DataSpace(intervals, parts)

        # Finally create the LoweredEq with all metadata attached
        expr = super(LoweredEq, cls).__new__(cls, expr.lhs, expr.rhs, evaluate=False)
        expr._is_Increment = getattr(input_expr, 'is_Increment', False)
        expr._dspace = dspace
        expr._ispace = ispace
        expr._conditionals = tuple(conditionals)
        expr._reads, expr._writes = detect_io(expr)

        return expr

    @property
    def reads(self):
        return self._reads

    @property
    def writes(self):
        return self._writes

    def xreplace(self, rules):
        return LoweredEq(self.lhs.xreplace(rules), self.rhs.xreplace(rules), **self.state)

    def func(self, *args):
        return super(LoweredEq, self).func(*args, **self.state, evaluate=False)

class ClusterizedEq(Eq, IREq, FrozenExpr, Pickable):

    """
    ClusterizedEq(devito.IREq, **kwargs)
    ClusterizedEq(lhs, rhs, **kwargs)

    A SymPy equation with associated :class:`IterationSpace` and
    :class:`DataSpace`.

    There are two main differences between a :class:`LoweredEq` and a
    ClusterizedEq: ::

        * In a ClusterizedEq, the iteration and data spaces must *always*
          be provided by the caller.
        * A ClusterizedEq is "frozen", meaning that any call to ``xreplace``
          will not trigger re-evaluation (e.g., mathematical simplification)
          of the expression.

    These two properties make a ClusterizedEq suitable for use in a
    :class:`Cluster`.
    """

    def __new__(cls, *args, **kwargs):
        if len(args) == 1:
            # origin: ClusterizedEq(expr, **kwargs)
            input_expr = args[0]
            expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
            for i in cls._state:
                v = kwargs[i] if i in kwargs else getattr(input_expr, i, None)
                setattr(expr, '_%s' % i, v)
        elif len(args) == 2:
            # origin: ClusterizedEq(lhs, rhs, **kwargs)
            expr = Eq.__new__(cls, *args, evaluate=False)
            for i in cls._state:
                setattr(expr, '_%s' % i, kwargs.pop(i))
        else:
            raise ValueError("Cannot construct ClusterizedEq from args=%s "
                             "and kwargs=%s" % (str(args), str(kwargs)))
        return expr

    def func(self, *args, **kwargs):
        return super(ClusterizedEq, self).func(*args, **self.state)

    # Pickling support
    _pickle_args = ['lhs', 'rhs']
    _pickle_kwargs = IREq._state
    __reduce_ex__ = Pickable.__reduce_ex__

class DummyEq(ClusterizedEq):

    """
    DummyEq(expr)
    DummyEq(lhs, rhs)

    A special :class:`ClusterizedEq` that tracks no iteration or data spaces.
    """

    def __new__(cls, *args):
        if len(args) == 1:
            input_expr = args[0]
            assert isinstance(input_expr, Eq)
            obj = LoweredEq(input_expr)
        elif len(args) == 2:
            obj = LoweredEq(Eq(*args, evaluate=False))
        else:
            raise ValueError("Cannot construct DummyEq from args=%s" % str(args))
        return ClusterizedEq.__new__(cls, obj, ispace=obj.ispace, dspace=obj.dspace)
<|endoftext|>


<commit_before>import new
import re

import _base
from html5lib import ihatexml
from html5lib import constants
from html5lib.constants import namespaces

tag_regexp = re.compile("{([^}]*)}(.*)")

moduleCache = {}

def getETreeModule(ElementTreeImplementation, fullTree=False):
    name = "_" + ElementTreeImplementation.__name__+"builder"
    if name in moduleCache:
        return moduleCache[name]
    else:
        mod = new.module("_" + ElementTreeImplementation.__name__+"builder")
        objs = getETreeBuilder(ElementTreeImplementation, fullTree)
        mod.__dict__.update(objs)
        moduleCache[name] = mod
        return mod

def getETreeBuilder(ElementTreeImplementation, fullTree=False):
    ElementTree = ElementTreeImplementation
    class Element(_base.Node):
        def __init__(self, name, namespace=None):
            self._name = name
            self._namespace = namespace
            self._element = ElementTree.Element(self._getETreeTag(name,
                                                                  namespace))
            if namespace is None:
                self.nameTuple = namespaces["html"], self._name
            else:
                self.nameTuple = self._namespace, self._name
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getETreeTag(self, name, namespace):
            if namespace is None:
                etree_tag = name
            else:
                etree_tag = "{%s}%s"%(namespace, name)
            return etree_tag

        def _setName(self, name):
            self._name = name
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getName(self):
            return self._name

        name = property(_getName, _setName)

        def _setNamespace(self, namespace):
            self._namespace = namespace
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getNamespace(self):
            return self._namespace

        namespace = property(_getNamespace, _setNamespace)

        def _getAttributes(self):
            return self._element.attrib

        def _setAttributes(self, attributes):
            #Delete existing attributes first
            #XXX - there may be a better way to do this...
            for key in self._element.attrib.keys():
                del self._element.attrib[key]
            for key, value in attributes.iteritems():
                if isinstance(key, tuple):
                    name = "{%s}%s"%(key[2], key[1])
                else:
                    name = key
                self._element.set(name, value)

        attributes = property(_getAttributes, _setAttributes)

        def _getChildNodes(self):
            return self._childNodes
        def _setChildNodes(self, value):
            del self._element[:]
            self._childNodes = []
            for element in value:
                self.insertChild(element)

        childNodes = property(_getChildNodes, _setChildNodes)

        def hasContent(self):
            """Return true if the node has children or text"""
            return bool(self._element.text or self._element.getchildren())

        def appendChild(self, node):
            self._childNodes.append(node)
            self._element.append(node._element)
            node.parent = self

        def insertBefore(self, node, refNode):
            index = self._element.getchildren().index(refNode._element)
            self._element.insert(index, node._element)
            node.parent = self

        def removeChild(self, node):
            self._element.remove(node._element)
            node.parent=None

        def insertText(self, data, insertBefore=None):
            if not(len(self._element)):
                if not self._element.text:
                    self._element.text = ""
                self._element.text += data
            elif insertBefore is None:
                #Insert the text as the tail of the last child element
                if not self._element[-1].tail:
                    self._element[-1].tail = ""
                self._element[-1].tail += data
            else:
                #Insert the text before the specified node
                children = self._element.getchildren()
                index = children.index(insertBefore._element)
                if index > 0:
                    if not self._element[index-1].tail:
                        self._element[index-1].tail = ""
                    self._element[index-1].tail += data
                else:
                    if not self._element.text:
                        self._element.text = ""
                    self._element.text += data

        def cloneNode(self):
            element = Element(self.name)
            for name, value in self.attributes.iteritems():
                element.attributes[name] = value
            return element

        def reparentChildren(self, newParent):
            if newParent.childNodes:
                newParent.childNodes[-1]._element.tail += self._element.text
            else:
                if not newParent._element.text:
                    newParent._element.text = ""
                if self._element.text is not None:
                    newParent._element.text += self._element.text
            self._element.text = ""
            _base.Node.reparentChildren(self, newParent)

    class Comment(Element):
        def __init__(self, data):
            #Use the superclass constructor to set all properties on the
            #wrapper element
            self._element = ElementTree.Comment(data)
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getData(self):
            return self._element.text

        def _setData(self, value):
            self._element.text = value

        data = property(_getData, _setData)

    class DocumentType(Element):
        def __init__(self, name, publicId, systemId):
            Element.__init__(self, "<!DOCTYPE>")
            self._element.text = name
            self.publicId = publicId
            self.systemId = systemId

        def _getPublicId(self):
            return self._element.get(u"publicId", "")

        def _setPublicId(self, value):
            if value is not None:
                self._element.set(u"publicId", value)

        publicId = property(_getPublicId, _setPublicId)

        def _getSystemId(self):
            return self._element.get(u"systemId", "")

        def _setSystemId(self, value):
            if value is not None:
                self._element.set(u"systemId", value)

        systemId = property(_getSystemId, _setSystemId)

    class Document(Element):
        def __init__(self):
            Element.__init__(self, "<DOCUMENT_ROOT>")

    class DocumentFragment(Element):
        def __init__(self):
            Element.__init__(self, "<DOCUMENT_FRAGMENT>")

    def testSerializer(element):
        rv = []
        finalText = None
        def serializeElement(element, indent=0):
            if not(hasattr(element, "tag")):
                element = element.getroot()
            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "<DOCUMENT_ROOT>":
                rv.append("#document")
                if element.text:
                    rv.append("|%s\"%s\""%(' '*(indent+2), element.text))
                if element.tail:
                    finalText = element.tail
            elif type(element.tag) == type(ElementTree.Comment):
                rv.append("|%s<!-- %s -->"%(' '*indent, element.text))
            else:
                nsmatch = tag_regexp.match(element.tag)

                if nsmatch is None:
                    name = element.tag
                else:
                    ns, name = nsmatch.groups()
                    prefix = constants.prefixes[ns]
                    name = "%s %s"%(prefix, name)
                rv.append("|%s<%s>"%(' '*indent, name))

                if hasattr(element, "attrib"):
                    for name, value in element.attrib.iteritems():
                        nsmatch = tag_regexp.match(name)
                        if nsmatch is not None:
                            ns, name = nsmatch.groups()
                            prefix = constants.prefixes[ns]
                            name = "%s %s"%(prefix, name)
                        rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
                if element.text:
                    rv.append("|%s\"%s\"" %(' '*(indent+2), element.text))
            indent += 2
            for child in element.getchildren():
                serializeElement(child, indent)
            if element.tail:
                rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail))
        serializeElement(element, 0)

        if finalText is not None:
            rv.append("|%s\"%s\""%(' '*2, finalText))

        return "\n".join(rv)

    def tostring(element):
        """Serialize an element and its child nodes to a string"""
        rv = []
        finalText = None
        filter = ihatexml.InfosetFilter()
        def serializeElement(element):
            if type(element) == type(ElementTree.ElementTree):
                element = element.getroot()

            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "<DOCUMENT_ROOT>":
                if element.text:
                    rv.append(element.text)
                if element.tail:
                    finalText = element.tail

                for child in element.getchildren():
                    serializeElement(child)

            elif type(element.tag) == type(ElementTree.Comment):
                rv.append("<!--%s-->"%(element.text,))
            else:
                #This is assumed to be an ordinary element
                if not element.attrib:
                    rv.append("<%s>"%(filter.fromXmlName(element.tag),))
                else:
                    attr = " ".join(["%s=\"%s\""%(
                                filter.fromXmlName(name), value)
                                     for name, value in element.attrib.iteritems()])
                    rv.append("<%s %s>"%(element.tag, attr))
                if element.text:
                    rv.append(element.text)

                for child in element.getchildren():
                    serializeElement(child)

                rv.append("</%s>"%(element.tag,))

            if element.tail:
                rv.append(element.tail)

        serializeElement(element)

        if finalText is not None:
            rv.append("%s\""%(' '*2, finalText))

        return "".join(rv)

    class TreeBuilder(_base.TreeBuilder):
        documentClass = Document
        doctypeClass = DocumentType
        elementClass = Element
        commentClass = Comment
        fragmentClass = DocumentFragment

        def testSerializer(self, element):
            return testSerializer(element)

        def getDocument(self):
            if fullTree:
                return self.document._element
            else:
                return self.document._element.find("html")

        def getFragment(self):
            return _base.TreeBuilder.getFragment(self)._element

    return locals()
<commit_msg>Fix etree treebuilder cloneNode to work with namespaces.<commit_after>import new
import re

import _base
from html5lib import ihatexml
from html5lib import constants
from html5lib.constants import namespaces

tag_regexp = re.compile("{([^}]*)}(.*)")

moduleCache = {}

def getETreeModule(ElementTreeImplementation, fullTree=False):
    name = "_" + ElementTreeImplementation.__name__+"builder"
    if name in moduleCache:
        return moduleCache[name]
    else:
        mod = new.module("_" + ElementTreeImplementation.__name__+"builder")
        objs = getETreeBuilder(ElementTreeImplementation, fullTree)
        mod.__dict__.update(objs)
        moduleCache[name] = mod
        return mod

def getETreeBuilder(ElementTreeImplementation, fullTree=False):
    ElementTree = ElementTreeImplementation
    class Element(_base.Node):
        def __init__(self, name, namespace=None):
            self._name = name
            self._namespace = namespace
            self._element = ElementTree.Element(self._getETreeTag(name,
                                                                  namespace))
            if namespace is None:
                self.nameTuple = namespaces["html"], self._name
            else:
                self.nameTuple = self._namespace, self._name
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getETreeTag(self, name, namespace):
            if namespace is None:
                etree_tag = name
            else:
                etree_tag = "{%s}%s"%(namespace, name)
            return etree_tag

        def _setName(self, name):
            self._name = name
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getName(self):
            return self._name

        name = property(_getName, _setName)

        def _setNamespace(self, namespace):
            self._namespace = namespace
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getNamespace(self):
            return self._namespace

        namespace = property(_getNamespace, _setNamespace)

        def _getAttributes(self):
            return self._element.attrib

        def _setAttributes(self, attributes):
            #Delete existing attributes first
            #XXX - there may be a better way to do this...
            for key in self._element.attrib.keys():
                del self._element.attrib[key]
            for key, value in attributes.iteritems():
                if isinstance(key, tuple):
                    name = "{%s}%s"%(key[2], key[1])
                else:
                    name = key
                self._element.set(name, value)

        attributes = property(_getAttributes, _setAttributes)

        def _getChildNodes(self):
            return self._childNodes
        def _setChildNodes(self, value):
            del self._element[:]
            self._childNodes = []
            for element in value:
                self.insertChild(element)

        childNodes = property(_getChildNodes, _setChildNodes)

        def hasContent(self):
            """Return true if the node has children or text"""
            return bool(self._element.text or self._element.getchildren())

        def appendChild(self, node):
            self._childNodes.append(node)
            self._element.append(node._element)
            node.parent = self

        def insertBefore(self, node, refNode):
            index = self._element.getchildren().index(refNode._element)
            self._element.insert(index, node._element)
            node.parent = self

        def removeChild(self, node):
            self._element.remove(node._element)
            node.parent=None

        def insertText(self, data, insertBefore=None):
            if not(len(self._element)):
                if not self._element.text:
                    self._element.text = ""
                self._element.text += data
            elif insertBefore is None:
                #Insert the text as the tail of the last child element
                if not self._element[-1].tail:
                    self._element[-1].tail = ""
                self._element[-1].tail += data
            else:
                #Insert the text before the specified node
                children = self._element.getchildren()
                index = children.index(insertBefore._element)
                if index > 0:
                    if not self._element[index-1].tail:
                        self._element[index-1].tail = ""
                    self._element[index-1].tail += data
                else:
                    if not self._element.text:
                        self._element.text = ""
                    self._element.text += data

        def cloneNode(self):
            element = Element(self.name, self.namespace)
            for name, value in self.attributes.iteritems():
                element.attributes[name] = value
            return element

        def reparentChildren(self, newParent):
            if newParent.childNodes:
                newParent.childNodes[-1]._element.tail += self._element.text
            else:
                if not newParent._element.text:
                    newParent._element.text = ""
                if self._element.text is not None:
                    newParent._element.text += self._element.text
            self._element.text = ""
            _base.Node.reparentChildren(self, newParent)

    class Comment(Element):
        def __init__(self, data):
            #Use the superclass constructor to set all properties on the
            #wrapper element
            self._element = ElementTree.Comment(data)
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getData(self):
            return self._element.text

        def _setData(self, value):
            self._element.text = value

        data = property(_getData, _setData)

    class DocumentType(Element):
        def __init__(self, name, publicId, systemId):
            Element.__init__(self, "<!DOCTYPE>")
            self._element.text = name
            self.publicId = publicId
            self.systemId = systemId

        def _getPublicId(self):
            return self._element.get(u"publicId", "")

        def _setPublicId(self, value):
            if value is not None:
                self._element.set(u"publicId", value)

        publicId = property(_getPublicId, _setPublicId)

        def _getSystemId(self):
            return self._element.get(u"systemId", "")

        def _setSystemId(self, value):
            if value is not None:
                self._element.set(u"systemId", value)

        systemId = property(_getSystemId, _setSystemId)

    class Document(Element):
        def __init__(self):
            Element.__init__(self, "<DOCUMENT_ROOT>")

    class DocumentFragment(Element):
        def __init__(self):
            Element.__init__(self, "<DOCUMENT_FRAGMENT>")

    def testSerializer(element):
        rv = []
        finalText = None
        def serializeElement(element, indent=0):
            if not(hasattr(element, "tag")):
                element = element.getroot()
            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "<DOCUMENT_ROOT>":
                rv.append("#document")
                if element.text:
                    rv.append("|%s\"%s\""%(' '*(indent+2), element.text))
                if element.tail:
                    finalText = element.tail
            elif type(element.tag) == type(ElementTree.Comment):
                rv.append("|%s<!-- %s -->"%(' '*indent, element.text))
            else:
                nsmatch = tag_regexp.match(element.tag)

                if nsmatch is None:
                    name = element.tag
                else:
                    ns, name = nsmatch.groups()
                    prefix = constants.prefixes[ns]
                    name = "%s %s"%(prefix, name)
                rv.append("|%s<%s>"%(' '*indent, name))

                if hasattr(element, "attrib"):
                    for name, value in element.attrib.iteritems():
                        nsmatch = tag_regexp.match(name)
                        if nsmatch is not None:
                            ns, name = nsmatch.groups()
                            prefix = constants.prefixes[ns]
                            name = "%s %s"%(prefix, name)
                        rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
                if element.text:
                    rv.append("|%s\"%s\"" %(' '*(indent+2), element.text))
            indent += 2
            for child in element.getchildren():
                serializeElement(child, indent)
            if element.tail:
                rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail))
        serializeElement(element, 0)

        if finalText is not None:
            rv.append("|%s\"%s\""%(' '*2, finalText))

        return "\n".join(rv)

    def tostring(element):
        """Serialize an element and its child nodes to a string"""
        rv = []
        finalText = None
        filter = ihatexml.InfosetFilter()
        def serializeElement(element):
            if type(element) == type(ElementTree.ElementTree):
                element = element.getroot()

            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "<DOCUMENT_ROOT>":
                if element.text:
                    rv.append(element.text)
                if element.tail:
                    finalText = element.tail

                for child in element.getchildren():
                    serializeElement(child)

            elif type(element.tag) == type(ElementTree.Comment):
                rv.append("<!--%s-->"%(element.text,))
            else:
                #This is assumed to be an ordinary element
                if not element.attrib:
                    rv.append("<%s>"%(filter.fromXmlName(element.tag),))
                else:
                    attr = " ".join(["%s=\"%s\""%(
                                filter.fromXmlName(name), value)
                                     for name, value in element.attrib.iteritems()])
                    rv.append("<%s %s>"%(element.tag, attr))
                if element.text:
                    rv.append(element.text)

                for child in element.getchildren():
                    serializeElement(child)

                rv.append("</%s>"%(element.tag,))

            if element.tail:
                rv.append(element.tail)

        serializeElement(element)

        if finalText is not None:
            rv.append("%s\""%(' '*2, finalText))

        return "".join(rv)

    class TreeBuilder(_base.TreeBuilder):
        documentClass = Document
        doctypeClass = DocumentType
        elementClass = Element
        commentClass = Comment
        fragmentClass = DocumentFragment

        def testSerializer(self, element):
            return testSerializer(element)

        def getDocument(self):
            if fullTree:
                return self.document._element
            else:
                return self.document._element.find("html")

        def getFragment(self):
            return _base.TreeBuilder.getFragment(self)._element

    return locals()
<|endoftext|>