<commit_before>"""Export advocate europe 2016 proposal.

This is registered as console script 'export_ae_proposals_2016'
in setup.py.
"""

import argparse
import csv
import inspect
import textwrap
from functools import partial

from pyramid.registry import Registry
from pyramid.paster import bootstrap
from substanced.util import find_service
from adhocracy_core.utils import create_filename

from adhocracy_core.utils import get_sheet_field
from adhocracy_core.interfaces import search_query

from pyramid.traversal import resource_path

from adhocracy_core.sheets.title import ITitle
from adhocracy_core.sheets.metadata import IMetadata
from adhocracy_mercator.resources.mercator2 import IMercatorProposal
from adhocracy_core.sheets.principal import IUserBasic
from adhocracy_core.sheets.principal import IUserExtended
from adhocracy_mercator.sheets.mercator2 import IUserInfo
from adhocracy_mercator.sheets.mercator2 import IOrganizationInfo
from adhocracy_mercator.sheets.mercator2 import IPitch
from adhocracy_mercator.sheets.mercator2 import IMercatorSubResources
from adhocracy_mercator.sheets.mercator2 import IPartners
from adhocracy_mercator.sheets.mercator2 import ITopic
from adhocracy_mercator.sheets.mercator2 import IDuration
from adhocracy_mercator.sheets.mercator2 import ILocation
from adhocracy_mercator.sheets.mercator2 import IStatus
from adhocracy_mercator.sheets.mercator2 import IChallenge
from adhocracy_mercator.sheets.mercator2 import IGoal
from adhocracy_mercator.sheets.mercator2 import IPlan
from adhocracy_mercator.sheets.mercator2 import ITarget
from adhocracy_mercator.sheets.mercator2 import ITeam
from adhocracy_mercator.sheets.mercator2 import IExtraInfo
from adhocracy_mercator.sheets.mercator2 import IConnectionCohesion
from adhocracy_mercator.sheets.mercator2 import IDifference
from adhocracy_mercator.sheets.mercator2 import IPracticalRelevance
from adhocracy_mercator.sheets.mercator2 import IFinancialPlanning
from adhocracy_mercator.sheets.mercator2 import IExtraFunding
from adhocracy_mercator.sheets.mercator2 import ICommunity

def export_proposals():
    """
    Export all proposals from database and write them to csv file.

    --limited restricts the export to a few fields.
    """
    doc = textwrap.dedent(inspect.getdoc(export_proposals))
    parser = argparse.ArgumentParser(description=doc)
    parser.add_argument('config')
    parser.add_argument('-l',
                        '--limited',
                        help='only export a limited subset of all fields',
                        action='store_true')
    args = parser.parse_args()
    env = bootstrap(args.config)
    root = env['root']
    registry = env['registry']
    _export_proposals(root, registry, args.limited)
    env['closer']()

def _export_proposals(root, registry, limited):
    catalogs = find_service(root, 'catalogs')
    query = search_query._replace(interfaces=IMercatorProposal,
                                  resolve=True,
                                  )
    proposals = catalogs.search(query).elements

    filename = create_filename(directory='./var/export',
                               prefix='ae-2016-proposals',
                               suffix='.csv')
    result_file = open(filename, 'w', newline='')
    wr = csv.writer(result_file, delimiter=';', quotechar='"',
                    quoting=csv.QUOTE_MINIMAL)

    include_field = True
    exclude_field = False
    fields = \
        [('URL', include_field,
          partial(_get_proposal_url, registry)),
         ('Creation date', include_field,
          partial(_get_creation_date)),
         ('Title', include_field,
          partial(_get_sheet_field, ITitle, 'title')),
         ('Creator name', include_field,
          partial(_get_creator_name)),
         ('Creator email', include_field,
          partial(_get_creator_email)),
         ('First name', include_field,
          partial(_get_sheet_field, IUserInfo, 'first_name')),
         ('Last name', include_field,
          partial(_get_sheet_field, IUserInfo, 'last_name')),
         ('Organisation name', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'name')),
         ('Organisation city', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'city')),
         ('Organisation country', include_field,
          partial(_get_sheet_field, IOrganizationInfo, 'country')),
         ('Organisation help request', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'help_request')),
         ('Organisation registration date', exclude_field,
          partial(_get_date, IOrganizationInfo, 'registration_date')),
         ('Organisation website', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'website')),
         ('Organisation status', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'status')),
         ('Organisation status other', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'status_other')),
         ('Pitch', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPitch, 'pitch', 'pitch')),
         ('Partner1 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_name')),
         ('Partner1 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_website')),
         ('Partner1 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_country')),
         ('Partner2 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_name')),
         ('Partner2 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_website')),
         ('Partner2 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_country')),
         ('Partner3 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_name')),
         ('Partner3 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_website')),
         ('Partner3 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_country')),
         ('Others partners', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'other_partners')),
         ('Topics', exclude_field,
          lambda proposal: ' '.join(
              get_sheet_field(proposal, ITopic, 'topic'))),
         ('Topic other', exclude_field,
          partial(_get_sheet_field, ITopic, 'topic_other')),
         ('Duration', exclude_field,
          lambda proposal: str(_get_sheet_field_from_subresource(
              IDuration, 'duration', 'duration', proposal))),
         ('Location', exclude_field,
          partial(_get_sheet_field, ILocation, 'location')),
         ('Is online', exclude_field,
          lambda proposal: str(
              get_sheet_field(proposal, ILocation, 'is_online'))),
         ('Link to Ruhr', exclude_field,
          partial(_get_sheet_field, ILocation, 'link_to_ruhr')),
         ('Status', exclude_field,
          partial(_get_sheet_field, IStatus, 'status')),
         ('Challenge', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IChallenge, 'challenge', 'challenge')),
         ('Goal', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IGoal, 'goal', 'goal')),
         ('Plan', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPlan, 'plan', 'plan')),
         ('Target', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  ITarget, 'target', 'target')),
         ('Team', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  ITeam, 'team', 'team')),
         ('Extra info', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IExtraInfo, 'extrainfo', 'extrainfo')),
         ('Connection cohesion', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IConnectionCohesion,
                  'connectioncohesion', 'connection_cohesion')),
         ('Difference', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IDifference,
                  'difference', 'difference')),
         ('Practical relevance', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPracticalRelevance,
                  'practicalrelevance', 'practicalrelevance')),
         ('Budget', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IFinancialPlanning, 'budget'))),
         ('Requested funding', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IFinancialPlanning, 'requested_funding'))),
         ('Major expenses', exclude_field,
          partial(_get_sheet_field,
                  IFinancialPlanning, 'major_expenses')),
         ('Other sources of income', exclude_field,
          partial(_get_sheet_field, IExtraFunding, 'other_sources')),
         ('Secured', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IExtraFunding, 'secured'))),
         ('Reach out', exclude_field,
          partial(_get_sheet_field, ICommunity, 'expected_feedback')),
         ('Heard from', exclude_field,
          lambda proposal: ' '.join(get_sheet_field(
              proposal, ICommunity, 'heard_froms'))),
         ('Heard from other', exclude_field,
          lambda proposal: ' '.join(get_sheet_field(
              proposal, ICommunity, 'heard_from_other')))]

    wr.writerow([name for (name, is_included, _) in fields if is_included])

    for proposal in proposals:

        result = []
        append_field = partial(_append_field, result)

        for name, is_in, get_field in fields:
            if is_in or not limited:
                append_field(get_field(proposal))

        wr.writerow(result)

    print('Exported mercator proposals to %s' % filename)

def _get_user_fields(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    name = get_sheet_field(creator, IUserBasic, 'name')
    email = get_sheet_field(creator, IUserExtended, 'email')
    return name, email

def _get_sheet_field_from_subresource(sheet,
                                      sheet_field,
                                      sub_sheet_field,
                                      proposal):
    sub_resource = get_sheet_field(proposal,
                                   IMercatorSubResources,
                                   sheet_field)
    return get_sheet_field(sub_resource, sheet, sub_sheet_field)

def _normalize_text(s: str) -> str:
    """Normalize text to put it in CVS."""
    return s.replace(';', '')

def _append_field(result, content):
    result.append(_normalize_text(content))

def _get_date(sheet, field, resource):
    value = get_sheet_field(resource, sheet, field)
    if not value:
        return ''
    date = value.date().strftime('%d.%m.%Y')
    return date

def _get_creation_date(proposal):
    creation_date = get_sheet_field(
        proposal,
        IMetadata,
        'item_creation_date')
    date = creation_date.date().strftime('%d.%m.%Y')
    return date

def _get_proposal_url(registry: Registry,
                      proposal: IMercatorProposal) -> str:
    path = resource_path(proposal)
    frontend_url = registry.settings.get('adhocracy.frontend_url')
    return frontend_url + '/r' + path

def _get_sheet_field(sheet, field, resource):
    return get_sheet_field(resource, sheet, field)

def _get_creator_name(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    name = get_sheet_field(creator, IUserBasic, 'name')
    return name

def _get_creator_email(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    email = get_sheet_field(creator, IUserExtended, 'email')
    return email
<commit_msg>Fix bug in AE2016 export script<commit_after>"""Export advocate europe 2016 proposal.

This is registered as console script 'export_ae_proposals_2016'
in setup.py.
"""

import argparse
import csv
import inspect
import textwrap
from functools import partial

from pyramid.registry import Registry
from pyramid.paster import bootstrap
from substanced.util import find_service
from adhocracy_core.utils import create_filename

from adhocracy_core.utils import get_sheet_field
from adhocracy_core.interfaces import search_query

from pyramid.traversal import resource_path

from adhocracy_core.sheets.title import ITitle
from adhocracy_core.sheets.metadata import IMetadata
from adhocracy_mercator.resources.mercator2 import IMercatorProposal
from adhocracy_core.sheets.principal import IUserBasic
from adhocracy_core.sheets.principal import IUserExtended
from adhocracy_mercator.sheets.mercator2 import IUserInfo
from adhocracy_mercator.sheets.mercator2 import IOrganizationInfo
from adhocracy_mercator.sheets.mercator2 import IPitch
from adhocracy_mercator.sheets.mercator2 import IMercatorSubResources
from adhocracy_mercator.sheets.mercator2 import IPartners
from adhocracy_mercator.sheets.mercator2 import ITopic
from adhocracy_mercator.sheets.mercator2 import IDuration
from adhocracy_mercator.sheets.mercator2 import ILocation
from adhocracy_mercator.sheets.mercator2 import IStatus
from adhocracy_mercator.sheets.mercator2 import IChallenge
from adhocracy_mercator.sheets.mercator2 import IGoal
from adhocracy_mercator.sheets.mercator2 import IPlan
from adhocracy_mercator.sheets.mercator2 import ITarget
from adhocracy_mercator.sheets.mercator2 import ITeam
from adhocracy_mercator.sheets.mercator2 import IExtraInfo
from adhocracy_mercator.sheets.mercator2 import IConnectionCohesion
from adhocracy_mercator.sheets.mercator2 import IDifference
from adhocracy_mercator.sheets.mercator2 import IPracticalRelevance
from adhocracy_mercator.sheets.mercator2 import IFinancialPlanning
from adhocracy_mercator.sheets.mercator2 import IExtraFunding
from adhocracy_mercator.sheets.mercator2 import ICommunity

def export_proposals():
    """
    Export all proposals from database and write them to csv file.

    --limited restricts the export to a few fields.
    """
    doc = textwrap.dedent(inspect.getdoc(export_proposals))
    parser = argparse.ArgumentParser(description=doc)
    parser.add_argument('config')
    parser.add_argument('-l',
                        '--limited',
                        help='only export a limited subset of all fields',
                        action='store_true')
    args = parser.parse_args()
    env = bootstrap(args.config)
    root = env['root']
    registry = env['registry']
    _export_proposals(root, registry, args.limited)
    env['closer']()

def _export_proposals(root, registry, limited):
    catalogs = find_service(root, 'catalogs')
    query = search_query._replace(interfaces=IMercatorProposal,
                                  resolve=True,
                                  )
    proposals = catalogs.search(query).elements

    filename = create_filename(directory='./var/export',
                               prefix='ae-2016-proposals',
                               suffix='.csv')
    result_file = open(filename, 'w', newline='')
    wr = csv.writer(result_file, delimiter=';', quotechar='"',
                    quoting=csv.QUOTE_MINIMAL)

    include_field = True
    exclude_field = False
    fields = \
        [('URL', include_field,
          partial(_get_proposal_url, registry)),
         ('Creation date', include_field,
          partial(_get_creation_date)),
         ('Title', include_field,
          partial(_get_sheet_field, ITitle, 'title')),
         ('Creator name', include_field,
          partial(_get_creator_name)),
         ('Creator email', include_field,
          partial(_get_creator_email)),
         ('First name', include_field,
          partial(_get_sheet_field, IUserInfo, 'first_name')),
         ('Last name', include_field,
          partial(_get_sheet_field, IUserInfo, 'last_name')),
         ('Organisation name', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'name')),
         ('Organisation city', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'city')),
         ('Organisation country', include_field,
          partial(_get_sheet_field, IOrganizationInfo, 'country')),
         ('Organisation help request', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'help_request')),
         ('Organisation registration date', exclude_field,
          partial(_get_date, IOrganizationInfo, 'registration_date')),
         ('Organisation website', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'website')),
         ('Organisation status', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'status')),
         ('Organisation status other', exclude_field,
          partial(_get_sheet_field, IOrganizationInfo, 'status_other')),
         ('Pitch', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPitch, 'pitch', 'pitch')),
         ('Partner1 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_name')),
         ('Partner1 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_website')),
         ('Partner1 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner1_country')),
         ('Partner2 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_name')),
         ('Partner2 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_website')),
         ('Partner2 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner2_country')),
         ('Partner3 name', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_name')),
         ('Partner3 website', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_website')),
         ('Partner3 country', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'partner3_country')),
         ('Others partners', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPartners, 'partners', 'other_partners')),
         ('Topics', exclude_field,
          lambda proposal: ' '.join(
              get_sheet_field(proposal, ITopic, 'topic'))),
         ('Topic other', exclude_field,
          partial(_get_sheet_field, ITopic, 'topic_other')),
         ('Duration', exclude_field,
          lambda proposal: str(_get_sheet_field_from_subresource(
              IDuration, 'duration', 'duration', proposal))),
         ('Location', exclude_field,
          partial(_get_sheet_field, ILocation, 'location')),
         ('Is online', exclude_field,
          lambda proposal: str(
              get_sheet_field(proposal, ILocation, 'is_online'))),
         ('Link to Ruhr', exclude_field,
          partial(_get_sheet_field, ILocation, 'link_to_ruhr')),
         ('Status', exclude_field,
          partial(_get_sheet_field, IStatus, 'status')),
         ('Challenge', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IChallenge, 'challenge', 'challenge')),
         ('Goal', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IGoal, 'goal', 'goal')),
         ('Plan', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPlan, 'plan', 'plan')),
         ('Target', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  ITarget, 'target', 'target')),
         ('Team', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  ITeam, 'team', 'team')),
         ('Extra info', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IExtraInfo, 'extrainfo', 'extrainfo')),
         ('Connection cohesion', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IConnectionCohesion,
                  'connectioncohesion', 'connection_cohesion')),
         ('Difference', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IDifference,
                  'difference', 'difference')),
         ('Practical relevance', exclude_field,
          partial(_get_sheet_field_from_subresource,
                  IPracticalRelevance,
                  'practicalrelevance', 'practicalrelevance')),
         ('Budget', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IFinancialPlanning, 'budget'))),
         ('Requested funding', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IFinancialPlanning, 'requested_funding'))),
         ('Major expenses', exclude_field,
          partial(_get_sheet_field,
                  IFinancialPlanning, 'major_expenses')),
         ('Other sources of income', exclude_field,
          partial(_get_sheet_field, IExtraFunding, 'other_sources')),
         ('Secured', exclude_field,
          lambda proposal: str(get_sheet_field(
              proposal, IExtraFunding, 'secured'))),
         ('Reach out', exclude_field,
          partial(_get_sheet_field, ICommunity, 'expected_feedback')),
         ('Heard from', exclude_field,
          lambda proposal: ' '.join(get_sheet_field(
              proposal, ICommunity, 'heard_froms'))),
         ('Heard from other', exclude_field,
          lambda proposal: ' '.join(get_sheet_field(
              proposal, ICommunity, 'heard_from_other')))]

    wr.writerow([name for (name, is_included, _) in fields
                 if is_included or not limited])

    for proposal in proposals:

        result = []
        append_field = partial(_append_field, result)

        for name, is_in, get_field in fields:
            if is_in or not limited:
                append_field(get_field(proposal))

        wr.writerow(result)

    print('Exported mercator proposals to %s' % filename)

def _get_user_fields(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    name = get_sheet_field(creator, IUserBasic, 'name')
    email = get_sheet_field(creator, IUserExtended, 'email')
    return name, email

def _get_sheet_field_from_subresource(sheet,
                                      sheet_field,
                                      sub_sheet_field,
                                      proposal):
    sub_resource = get_sheet_field(proposal,
                                   IMercatorSubResources,
                                   sheet_field)
    return get_sheet_field(sub_resource, sheet, sub_sheet_field)

def _normalize_text(s: str) -> str:
    """Normalize text to put it in CVS."""
    return s.replace(';', '')

def _append_field(result, content):
    result.append(_normalize_text(content))

def _get_date(sheet, field, resource):
    value = get_sheet_field(resource, sheet, field)
    if not value:
        return ''
    date = value.date().strftime('%d.%m.%Y')
    return date

def _get_creation_date(proposal):
    creation_date = get_sheet_field(
        proposal,
        IMetadata,
        'item_creation_date')
    date = creation_date.date().strftime('%d.%m.%Y')
    return date

def _get_proposal_url(registry: Registry,
                      proposal: IMercatorProposal) -> str:
    path = resource_path(proposal)
    frontend_url = registry.settings.get('adhocracy.frontend_url')
    return frontend_url + '/r' + path

def _get_sheet_field(sheet, field, resource):
    return get_sheet_field(resource, sheet, field)

def _get_creator_name(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    name = get_sheet_field(creator, IUserBasic, 'name')
    return name

def _get_creator_email(proposal):
    creator = get_sheet_field(proposal, IMetadata, 'creator')
    email = get_sheet_field(creator, IUserExtended, 'email')
    return email
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import socket
import math

sys.path.append("../")
#from TorCtl import *
from TorCtl import TorUtil, PathSupport, TorCtl
from TorCtl.TorUtil import control_port, control_host, control_pass
from TorCtl.TorUtil import *
from TorCtl.PathSupport import *
import atexit

TorUtil.loglevel = "INFO"

def cleanup(c, f):
  print "Resetting FetchUselessDescriptors to "+f
  c.set_option("FetchUselessDescriptors", f)

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((control_host,control_port))
c = Connection(s)
c.debug(file("control.log", "w"))
c.authenticate(control_pass)
FUDValue = c.get_option("FetchUselessDescriptors")[0][1]
c.set_option("FetchUselessDescriptors", "1")
atexit.register(cleanup, *(c, FUDValue))
nslist = c.get_network_status()
sorted_rlist = c.read_routers(c.get_network_status())

sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i

fast_rst = FlagsRestriction(["Fast"], [])
exit_rst = FlagsRestriction(["Exit"], [])
dir_rst = FlagsRestriction(["V2Dir"], [])
heavy_exits = OrNodeRestriction(
      [ExitPolicyRestriction("255.255.255.255", 6881),
      ExitPolicyRestriction("255.255.255.255", 6889),
      ExitPolicyRestriction("255.255.255.255", 6346),
      ExitPolicyRestriction("255.255.255.255", 25)])

def check(start, stop):
  pct_rst = PercentileRestriction(start, stop, sorted_rlist)
  bw = 0
  nodes = 0
  exits = 0
  exit_bw = 0
  heavy = 0
  dirs = 0
  nodes_up = 0.0000000005 # shhh. dirty hack for div 0
  up = 0

  for r in sorted_rlist:
    if pct_rst.r_is_ok(r) and fast_rst.r_is_ok(r):
      nodes += 1
      bw += r.bw
      if r.uptime > 0:
        nodes_up += 1.0
        up += r.uptime
      if exit_rst.r_is_ok(r):
        exits += 1
        exit_bw += r.bw
      if heavy_exits.r_is_ok(r):
        heavy += 1
      if dir_rst.r_is_ok(r):
        dirs += 1

  print str(start)+"-"+str(stop)+": N: "+str(nodes)+", Bw: "+str(round(bw/(1024*1024.0), 2))+", X: "+str(exits)+", XBw: "+str(round(exit_bw/(1024*1024.0),2))+", BT: "+str(heavy)+", Dirs:"+str(dirs)+", Up: "+str(round(up/nodes_up/60/60/24, 2))

for i in xrange(0,100,3):
  check(i,i+3)

def check_entropy(rlist, clipping_point):
  clipped = 0
  clipped_bw = 0.0
  exits = 0
  nodes = 0
  bw = 0.0
  exit_bw = 0.0
  pure_entropy = 0.0
  clipped_entropy = 0.0
  uniform_entropy = 0.0
  for r in rlist:
    if not fast_rst.r_is_ok(r):
      continue
    if r.bw > clipping_point:
      clipped += 1
      clipped_bw += clipping_point
    else:
      clipped_bw += r.bw
    nodes += 1
    bw += r.bw
    if exit_rst.r_is_ok(r):
      exits += 1
      exit_bw += r.bw

  for r in rlist:
    if not fast_rst.r_is_ok(r):
      continue
    if r.bw < 2:
      continue
    pure_entropy += (r.bw/bw)*math.log(r.bw/bw, 2)
    uniform_entropy += (1.0/nodes)*math.log(1.0/nodes, 2)

    rbw = 0
    if r.bw > clipping_point:
      rbw = clipping_point
    else:
      rbw = r.bw
    clipped_entropy += (rbw/clipped_bw)*math.log(rbw/clipped_bw, 2)

  print "Uniform entropy: " + str(-uniform_entropy)
  print "Raw entropy: " + str(-pure_entropy)
  print "Clipped entropy: " + str(-clipped_entropy)
  print "Nodes: "+str(nodes)+", Exits: "+str(exits)+" Total bw: "+str(round(bw/(1024.0*1024),2))+", Exit Bw: "+str(round(exit_bw/(1024.0*1024),2))
  print "Clipped: "+str(clipped)+", bw: "+str(round(clipped_bw/(1024.0*1024),2))

check_entropy(sorted_rlist, 1500000)
<commit_msg><commit_after>#!/usr/bin/python
import sys
import socket
import math

sys.path.append("../")
#from TorCtl import *
from TorCtl import TorUtil, PathSupport, TorCtl
from TorCtl.TorUtil import control_port, control_host, control_pass
from TorCtl.TorUtil import *
from TorCtl.PathSupport import *
import atexit

TorUtil.loglevel = "INFO"

def cleanup(c, f):
  print "Resetting FetchUselessDescriptors to "+f
  c.set_option("FetchUselessDescriptors", f)

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((control_host,control_port))
c = Connection(s)
c.debug(file("control.log", "w"))
c.authenticate(control_pass)
FUDValue = c.get_option("FetchUselessDescriptors")[0][1]
c.set_option("FetchUselessDescriptors", "1")
atexit.register(cleanup, *(c, FUDValue))
nslist = c.get_network_status()
sorted_rlist = c.read_routers(c.get_network_status())

sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i

fast_rst = FlagsRestriction(["Fast"], [])
exit_rst = FlagsRestriction(["Exit"], [])
dir_rst = FlagsRestriction(["V2Dir"], [])
heavy_exits = OrNodeRestriction(
      [ExitPolicyRestriction("255.255.255.255", 6881),
      ExitPolicyRestriction("255.255.255.255", 6889),
      ExitPolicyRestriction("255.255.255.255", 6346),
      ExitPolicyRestriction("255.255.255.255", 25)])

def check(start, stop):
  pct_rst = PercentileRestriction(start, stop, sorted_rlist)
  bw = 0
  nodes = 0
  exits = 0
  exit_bw = 0
  heavy = 0
  dirs = 0
  nodes_up = 0.0000000005 # shhh. dirty hack for div 0
  up = 0

  for r in sorted_rlist:
    if pct_rst.r_is_ok(r) and fast_rst.r_is_ok(r):
      nodes += 1
      bw += r.bw
      if r.uptime > 0:
        nodes_up += 1.0
        up += r.uptime
      if exit_rst.r_is_ok(r):
        exits += 1
        exit_bw += r.bw
      if heavy_exits.r_is_ok(r):
        heavy += 1
      if dir_rst.r_is_ok(r):
        dirs += 1

  print str(start)+"-"+str(stop)+": N: "+str(nodes)+", Bw: "+str(round(bw/(1024*1024.0), 2))+", X: "+str(exits)+", XBw: "+str(round(exit_bw/(1024*1024.0),2))+", BT: "+str(heavy)+", Dirs:"+str(dirs)+", Up: "+str(round(up/nodes_up/60/60/24, 2))

for i in xrange(0,100,3):
  check(i,i+3)

def check_entropy(rlist, clipping_point):
  clipped = 0
  clipped_bw = 0.0
  exits = 0
  nodes = 0
  bw = 0.0
  exit_bw = 0.0
  pure_entropy = 0.0
  clipped_entropy = 0.0
  uniform_entropy = 0.0

  guard_entropy = 0.0
  mid_entropy = 0.0
  exit_entropy = 0.0

  ggen = BwWeightedGenerator(rlist, FlagsRestriction(["Guard", "Fast", "Valid", "Running"]), 3, guard=True)
  mgen = BwWeightedGenerator(rlist, FlagsRestriction(["Valid", "Fast", "Running"]), 3)
  egen = BwWeightedGenerator(rlist, FlagsRestriction(["Exit", "Fast", "Valid", "Running"]), 3, exit=True)

  for r in rlist:
    if not fast_rst.r_is_ok(r):
      continue
    if r.bw > clipping_point:
      clipped += 1
      clipped_bw += clipping_point
    else:
      clipped_bw += r.bw
    nodes += 1
    bw += r.bw
    if exit_rst.r_is_ok(r):
      exits += 1
      exit_bw += r.bw

  for r in rlist:
    if not fast_rst.r_is_ok(r):
      continue
    if r.bw < 2:
      continue
    pure_entropy += (r.bw/bw)*math.log(r.bw/bw, 2)
    uniform_entropy += (1.0/nodes)*math.log(1.0/nodes, 2)
    gbw = r.bw
    mbw = r.bw
    ebw = r.bw
    if "Exit" in r.flags:
      gbw *= ggen.exit_weight
      mbw *= mgen.exit_weight
      ebw *= egen.exit_weight
    if "Guard" in r.flags:
      gbw *= ggen.guard_weight
      mbw *= mgen.guard_weight
      ebw *= egen.guard_weight

    if gbw > 2: guard_entropy += (gbw/bw)*math.log(gbw/bw, 2)
    if mbw > 2: mid_entropy += (mbw/bw)*math.log(mbw/bw, 2)
    if ebw > 2: exit_entropy += (ebw/bw)*math.log(ebw/bw, 2)

    rbw = 0
    if r.bw > clipping_point:
      rbw = clipping_point
    else:
      rbw = r.bw
    clipped_entropy += (rbw/clipped_bw)*math.log(rbw/clipped_bw, 2)

  print "Uniform entropy: " + str(-uniform_entropy)
  print "Raw entropy: " + str(-pure_entropy)
  print "Clipped entropy: " + str(-clipped_entropy)

  print "Guard entropy: " + str(-guard_entropy)
  print "Middle entropy: " + str(-mid_entropy)
  print "Exit entropy: " + str(-exit_entropy)

  print "Nodes: "+str(nodes)+", Exits: "+str(exits)+" Total bw: "+str(round(bw/(1024.0*1024),2))+", Exit Bw: "+str(round(exit_bw/(1024.0*1024),2))
  print "Clipped: "+str(clipped)+", bw: "+str(round(clipped_bw/(1024.0*1024),2))

check_entropy(sorted_rlist, 1500000)
<|endoftext|>


<commit_before>import json
from itertools import product, chain

from troposphere import Ref, Parameter, FindInMap, Output
import troposphere.ec2 as ec2
import boto.vpc
import boto
from environmentbase import EnvironmentBase
from ipcalc import IP, Network
import resources as res
from patterns import ha_nat

import netaddr
from toolz import groupby, assoc

class NetworkBase(EnvironmentBase):
    """
    EnvironmentBase controller containing a root template with all of the base networking infrastructure
    for a common deployment within AWS. This is intended to be the 'base' stack for deploying child stacks
    """

    def construct_network(self):
        """
        Main function to construct VPC, subnets, security groups, NAT instances, etc
        """
        network_config = self.config.get('network', {})

        self.template._azs = []

        az_count = int(network_config.get('az_count', '2'))

        self.stack_outputs = {}

        # Simple mapping of AZs to NATs, to prevent creating duplicates
        self.az_nat_mapping = {}

        self.add_vpc_az_mapping(boto_config=self.config.get('boto', {}), az_count=az_count)
        self.add_network_cidr_mapping(network_config=network_config)
        self.create_network_components(network_config=network_config)

        self.template._common_security_group = self.template.add_resource(ec2.SecurityGroup('commonSecurityGroup',
            GroupDescription='Security Group allows ingress and egress for common usage patterns throughout this deployed infrastructure.',
            VpcId=self.template.vpc_id,
            SecurityGroupEgress=[ec2.SecurityGroupRule(
                        FromPort='80',
                        ToPort='80',
                        IpProtocol='tcp',
                        CidrIp='0.0.0.0/0'),
                    ec2.SecurityGroupRule(
                        FromPort='443',
                        ToPort='443',
                        IpProtocol='tcp',
                        CidrIp='0.0.0.0/0'),
                    ec2.SecurityGroupRule(
                        FromPort='123',
                        ToPort='123',
                        IpProtocol='udp',
                        CidrIp='0.0.0.0/0')],
            SecurityGroupIngress= [
                    ec2.SecurityGroupRule(
                        FromPort='22',
                        ToPort='22',
                        IpProtocol='tcp',
                        CidrIp=FindInMap('networkAddresses', 'vpcBase', 'cidr'))]))
        self.template.add_output(Output('commonSecurityGroupId', Value=self.template.common_security_group))

        for x in range(0, az_count):
            self.template._azs.append(FindInMap('RegionMap', Ref('AWS::Region'), 'az' + str(x) + 'Name'))

    def create_action(self):
        """
        Override EnvironmentBase.create_action() to construct the network components before the create_hook()
        """
        self.load_config()
        self.initialize_template()
        self.construct_network()
        self.create_hook()
        self.serialize_templates()

    def add_vpc_az_mapping(self,
                           boto_config,
                           az_count=2):
        """
        Method gets the AZs within the given account where subnets can be created/deployed
        This is necessary due to some accounts having 4 subnets available within ec2 classic and only 3 within vpc
        which causes the Select by index method of picking azs unpredictable for all accounts
        @param boto_config [dict] collection of boto configuration values as set by the configuration file
        @param az_count [int] number of AWS availability zones to include in the VPC mapping
        """
        az_dict = {}
        region_list = []
        aws_auth_info = {}
        if 'aws_access_key_id' in boto_config and 'aws_secret_access_key' in boto_config:
            aws_auth_info['aws_access_key_id'] = boto_config.get('aws_access_key_id')
            aws_auth_info['aws_secret_access_key'] = boto_config.get('aws_secret_access_key')
        conn = boto.vpc.connect_to_region(region_name=boto_config.get('region_name', 'us-east-1'), **aws_auth_info)
        for region in conn.get_all_regions():
            if region.name == 'ap-northeast-2':
                # AWS added a new region in Seul, and while waiting for boto to
                # release a new version this hack solves the region error
                continue
            region_list.append(region.name)
            az_list = boto.vpc.connect_to_region(region.name, **aws_auth_info).get_all_zones()
            if len(az_list) > 1:
                temp_dict = {}
                x = 0
                for availability_zone in az_list:
                    temp_dict['az' + str(x) + 'Name'] = availability_zone.name
                    x += 1
                if len(temp_dict) >= az_count:
                    az_dict[region.name] = {}
                    for item in temp_dict:
                        self.template.add_region_map_value(region.name, item, temp_dict[item])

    def create_network_components(self, network_config=None):
        """
        Method creates a network with the specified number of public and private subnets within the
        VPC cidr specified by the networkAddresses CloudFormation mapping.
        @param network_config [dict] collection of network parameters for creating the VPC network
        """
        ## make VPC
        if 'network_name' in network_config:
            network_name = network_config.get('network_name')
        else:
            network_name = self.__class__.__name__

        self.template._vpc_id = self.template.add_resource(ec2.VPC('vpc',
                CidrBlock=FindInMap('networkAddresses', 'vpcBase', 'cidr'),
                EnableDnsSupport=True,
                EnableDnsHostnames=True,
                Tags=[ec2.Tag(key='Name', value=network_name)]))

        self.template.add_output(Output('vpcId', Value=self.template.vpc_id))

        self.template._vpc_cidr = FindInMap('networkAddresses', 'vpcBase', 'cidr')

        self.template._igw = self.template.add_resource(ec2.InternetGateway('vpcIgw'))

        ## add IGW
        igw_title = 'igwVpcAttachment'
        self.template._vpc_gateway_attachment = self.template.add_resource(ec2.VPCGatewayAttachment(
            igw_title,
            InternetGatewayId=self.template.igw,
            VpcId=self.template.vpc_id))

        self.gateway_hook()

        ## make Subnets
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))

        # Iterate through each subnet type for each AZ and add subnets, routing tables, routes, and NATs as necessary
        for index in range(int(network_config.get('az_count', 2))):
            az = index
            az_key = 'AZ{}'.format(az)

            for ind, subnet_config in enumerate(network_config.get('subnet_config', {})):
                subnet_type = subnet_config.get('type', 'private')
                subnet_layer = subnet_config.get('name')

                AvailabilityZone = FindInMap('RegionMap', Ref('AWS::Region'), 'az' + str(index) + 'Name')
                CidrBlock = self.template.mappings['networkAddresses'][az_key][subnet_layer]

                # Save the subnet references to the template object
                if subnet_type not in self.template._subnets:
                    self.template._subnets[subnet_type] = {}

                if subnet_layer not in self.template._subnets[subnet_type]:
                    self.template._subnets[subnet_type][subnet_layer] = []

                # Create the subnet
                subnet_name = subnet_layer + 'AZ' + str(index)
                subnet = self.template.add_resource(ec2.Subnet(
                    subnet_name,
                    AvailabilityZone=AvailabilityZone,
                    VpcId=self.template.vpc_id,
                    CidrBlock=CidrBlock,
                    Tags=[ec2.Tag(key='network', value=subnet_type),
                          ec2.Tag(key='Name', value=' '.join([subnet_layer, 'AZ:', str(index)])),
                        ]))

                self.template._subnets[subnet_type][subnet_layer].append(subnet)

                self.template.add_output(Output(subnet_name, Value=CidrBlock))

                # Create the routing table
                route_table = self.template.add_resource(ec2.RouteTable(
                    subnet_layer + 'AZ' + str(index) + 'RouteTable',
                    VpcId=self.template.vpc_id))

                # Create the NATs and egress rules
                self.create_subnet_egress(index, route_table, igw_title, subnet_type, subnet_layer)

                # Associate the routing table with the subnet
                self.template.add_resource(ec2.SubnetRouteTableAssociation(
                    subnet_layer + 'AZ' + str(index) + 'EgressRouteTableAssociation',
                    RouteTableId=Ref(route_table),
                    SubnetId=Ref(subnet)))

        self.template.manual_parameter_bindings['vpcId'] = self.template.vpc_id
        self.template.manual_parameter_bindings['vpcCidr'] = self.template.vpc_cidr
        self.template.manual_parameter_bindings['internetGateway'] = self.template.igw
        self.template.manual_parameter_bindings['igwVpcAttachment'] = self.template.vpc_gateway_attachment

    def create_subnet_egress(self, index, route_table, igw_title, subnet_type, subnet_layer):
        """
        Create an egress route for the subnet with the given index and type
        Override to create egress routes for other subnet types
        Creates the NAT instances in the public subnets
        """

        # For public subnets, create the route to the IGW
        if subnet_type == 'public':
            self.template.add_resource(ec2.Route(subnet_layer + 'AZ' + str(index) + 'EgressRoute',
                DependsOn=[igw_title],
                DestinationCidrBlock='0.0.0.0/0',
                GatewayId=self.template.igw,
                RouteTableId=Ref(route_table)))

        # For private subnets, create a NAT instance in a public subnet in the same AZ
        elif subnet_type == 'private':

            # If we have already created a NAT in this AZ, skip it
            if self.az_nat_mapping.get(index):
                return

            nat_instance_type = self.config['nat']['instance_type']
            nat_enable_ntp = self.config['nat']['enable_ntp']
            extra_user_data = self.config['nat'].get('extra_user_data')
            ha_nat = self.create_nat(
                index,
                nat_instance_type,
                nat_enable_ntp,
                name='HaNat' + str(index),
                extra_user_data=extra_user_data)

            # We merge the NAT template into the root template
            self.template.merge(ha_nat)

            # Save the reference to the HA NAT, so we don't recreate it if we hit another private subnet in this AZ
            self.az_nat_mapping[index] = ha_nat

    def gateway_hook(self):
        """
        Override to allow subclasses to create VPGs and similar components during network creation
        """
        pass

    def create_nat(self, index, nat_instance_type, enable_ntp, name, extra_user_data=None):
        """
        Override to customize your NAT instance. The returned object must be a
        subclass of ha_nat.HaNat.
        """
        return ha_nat.HaNat(
            index,
            nat_instance_type,
            enable_ntp,
            name=name,
            extra_user_data=extra_user_data)

    def _get_subnet_config_w_az(self, network_config):
        az_count = int(network_config.get('az_count', 2))
        subnet_config = network_config.get('subnet_config', {})

        for subnet in subnet_config:
            for az in range(az_count):
                newsubnet = assoc(subnet, 'AZ', az)
                yield newsubnet

    def _get_subnet_config_w_cidr(self, network_config):
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))
        network_cidr_size = str(network_config.get('network_cidr_size', '20'))
        first_network_address_block = str(network_config.get('first_network_address_block', network_cidr_base))

        ret_val = {}
        base_cidr = network_cidr_base + '/' + network_cidr_size
        net = netaddr.IPNetwork(base_cidr)

        grouped_subnet = groupby('size', self._get_subnet_config_w_az(network_config))
        subnet_groups = sorted(grouped_subnet.items())
        available_cidrs = []

        for subnet_size, subnet_configs in subnet_groups:
            newcidrs = net.subnet(int(subnet_size))

            for subnet_config in subnet_configs:
                try:
                    cidr = newcidrs.next()
                except StopIteration as e:
                    net = chain(*reversed(available_cidrs)).next()
                    newcidrs = net.subnet(int(subnet_size))
                    cidr = newcidrs.next()

                new_config = assoc(subnet_config, 'cidr', str(cidr))
                yield new_config
            else:
                net = newcidrs.next()
                available_cidrs.append(newcidrs)

    def add_network_cidr_mapping(self,
                                 network_config):
        """
        Method calculates and adds a CloudFormation mapping that is used to set VPC and Subnet CIDR blocks.
        Calculated based on CIDR block sizes and additionally checks to ensure all network segments
        fit inside of the specified overall VPC CIDR.
        @param network_config [dict] dictionary of values containing data for creating
        """
        az_count = int(network_config.get('az_count', '2'))
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))
        network_cidr_size = str(network_config.get('network_cidr_size', '20'))
        first_network_address_block = str(network_config.get('first_network_address_block', network_cidr_base))

        ret_val = {}
        base_cidr = network_cidr_base + '/' + network_cidr_size
        cidr_info = Network(base_cidr)
        ret_val['vpcBase'] = {'cidr': base_cidr}
        current_base_address = first_network_address_block

        subnet_config = self._get_subnet_config_w_cidr(network_config)
        subnet_config = list(subnet_config)

        for index, subnet_config in enumerate(subnet_config):
            subnet_type = subnet_config.get('type', 'private')
            subnet_size = subnet_config.get('size', '22')
            subnet_name = subnet_config.get('name', 'subnet')
            subnet_az = subnet_config.get('AZ', '-1')
            subnet_cidr = subnet_config.get('cidr', 'ERROR')
            az_key = 'AZ{}'.format(subnet_az)

            # TODO: check for subnet collisions

            if az_key not in ret_val:
                ret_val[az_key] = dict()
            if subnet_name not in ret_val[az_key]:
                ret_val[az_key][subnet_name] = dict()
            ret_val[az_key][subnet_name] = subnet_cidr

        return self.template.add_mapping('networkAddresses', ret_val)

    def add_vpn_gateway(self,
                        vpn_conf):
        """
        Not surprisingly, adds a VPN gateway to the network created by this template.
        @param vpn_conf [dict] - collection of vpn-level configuration values.
        """
        if 'vpn_name' in vpn_conf:
            vpn_name = vpn_conf.get('vpn_name')
        else:
            vpn_name = self.__class__.__name__ + 'Gateway'

        gateway = self.template.add_resource(ec2.VPNGateway('vpnGateway',
            Type=vpn_conf.get('vpn_type', 'ipsec.1'),
            Tags=[ec2.Tag(key='Name', value=vpn_name)]))

        gateway_connection = self.template.add_resource(ec2.VPCGatewayAttachment('vpnGatewayAttachment',
            VpcId=self.template.vpc_id,
            InternetGatewayId=self.template.igw,
            VpnGatewayId=gateway))
<commit_msg>initial refactor of networkbase.add_vpc_az_mapping()<commit_after>import json
from itertools import product, chain

from troposphere import Ref, Parameter, FindInMap, Output
import troposphere.ec2 as ec2
import boto.vpc
import boto
from environmentbase import EnvironmentBase
from ipcalc import IP, Network
import resources as res
from patterns import ha_nat

import netaddr
from toolz import groupby, assoc

AWS_MAPPING = dict(
                    [(u'eu-west-1', ['eu-west-1a', 'eu-west-1b', 'eu-west-1c']),
                    (u'ap-southeast-1', ['ap-southeast-1a', 'ap-southeast-1b']),
                    (u'ap-southeast-2', ['ap-southeast-2a', 'ap-southeast-2b']),
                    (u'eu-central-1', ['eu-central-1a', 'eu-central-1b']),
                    (u'ap-northeast-1', ['ap-northeast-1a', 'ap-northeast-1c']),
                    (u'us-east-1', ['us-east-1b', 'us-east-1c', 'us-east-1d', 'us-east-1e']),
                    (u'sa-east-1', ['sa-east-1a', 'sa-east-1b', 'sa-east-1c']),
                    (u'us-west-1', ['us-west-1b', 'us-west-1c']),
                    (u'us-west-2', ['us-west-2a', 'us-west-2b', 'us-west-2c']),
                    ]
                )
AWS_REGIONS = AWS_MAPPING.keys()

class NetworkBase(EnvironmentBase):
    """
    EnvironmentBase controller containing a root template with all of the base networking infrastructure
    for a common deployment within AWS. This is intended to be the 'base' stack for deploying child stacks
    """

    def construct_network(self):
        """
        Main function to construct VPC, subnets, security groups, NAT instances, etc
        """
        network_config = self.config.get('network', {})

        self.template._azs = []

        az_count = int(network_config.get('az_count', '2'))

        self.stack_outputs = {}

        # Simple mapping of AZs to NATs, to prevent creating duplicates
        self.az_nat_mapping = {}

        self.add_vpc_az_mapping(boto_config=self.config.get('boto', {}), az_count=az_count)
        self.add_network_cidr_mapping(network_config=network_config)
        self.create_network_components(network_config=network_config)

        self.template._common_security_group = self.template.add_resource(ec2.SecurityGroup('commonSecurityGroup',
            GroupDescription='Security Group allows ingress and egress for common usage patterns throughout this deployed infrastructure.',
            VpcId=self.template.vpc_id,
            SecurityGroupEgress=[ec2.SecurityGroupRule(
                        FromPort='80',
                        ToPort='80',
                        IpProtocol='tcp',
                        CidrIp='0.0.0.0/0'),
                    ec2.SecurityGroupRule(
                        FromPort='443',
                        ToPort='443',
                        IpProtocol='tcp',
                        CidrIp='0.0.0.0/0'),
                    ec2.SecurityGroupRule(
                        FromPort='123',
                        ToPort='123',
                        IpProtocol='udp',
                        CidrIp='0.0.0.0/0')],
            SecurityGroupIngress= [
                    ec2.SecurityGroupRule(
                        FromPort='22',
                        ToPort='22',
                        IpProtocol='tcp',
                        CidrIp=FindInMap('networkAddresses', 'vpcBase', 'cidr'))]))
        self.template.add_output(Output('commonSecurityGroupId', Value=self.template.common_security_group))

        for x in range(0, az_count):
            self.template._azs.append(FindInMap('RegionMap', Ref('AWS::Region'), 'az' + str(x) + 'Name'))

    def create_action(self):
        """
        Override EnvironmentBase.create_action() to construct the network components before the create_hook()
        """
        self.load_config()
        self.initialize_template()
        self.construct_network()
        self.create_hook()
        self.serialize_templates()

    def add_vpc_az_mapping(self,
                           boto_config,
                           az_count=2):
        """
        Method gets the AZs within the given account where subnets can be created/deployed
        This is necessary due to some accounts having 4 subnets available within ec2 classic and only 3 within vpc
        which causes the Select by index method of picking azs unpredictable for all accounts
        @param boto_config [dict] collection of boto configuration values as set by the configuration file
        @param az_count [int] number of AWS availability zones to include in the VPC mapping
        """
        az_dict = {}
        regions_names = self._get_aws_regions(boto_config)

        for region_name in regions_names:
            if region_name == 'ap-northeast-2':
                # AWS added a new region in Seul, and while waiting for boto to
                # release a new version this hack solves the region error
                continue
            az_list = self._get_aws_zones(region_name)
            if len(az_list) > 1:
                temp_dict = {}
                x = 0
                for availability_zone_name in az_list:
                    temp_dict['az' + str(x) + 'Name'] = availability_zone_name
                    x += 1
                if len(temp_dict) >= az_count:
                    az_dict[region_name] = {}
                    for item in temp_dict:
                        self.template.add_region_map_value(region_name, item, temp_dict[item])

    def _get_aws_regions(self, boto_config):
        if self.globals['print_debug']:
            regions_names = AWS_REGIONS
        else:
            conn = boto.vpc.connect_to_region(region_name=boto_config.get('region_name', 'us-east-1'))
            regions_names = [region.name for region in conn.get_all_regions()]
        return regions_names

    def _get_aws_zones(self, region_name):
        if self.globals['print_debug']:
            return AWS_MAPPING[region_name]
        else:
            return [az.name for az in boto.vpc.connect_to_region(region_name).get_all_zones()]

    def create_network_components(self, network_config=None):
        """
        Method creates a network with the specified number of public and private subnets within the
        VPC cidr specified by the networkAddresses CloudFormation mapping.
        @param network_config [dict] collection of network parameters for creating the VPC network
        """
        ## make VPC
        if 'network_name' in network_config:
            network_name = network_config.get('network_name')
        else:
            network_name = self.__class__.__name__

        self.template._vpc_id = self.template.add_resource(ec2.VPC('vpc',
                CidrBlock=FindInMap('networkAddresses', 'vpcBase', 'cidr'),
                EnableDnsSupport=True,
                EnableDnsHostnames=True,
                Tags=[ec2.Tag(key='Name', value=network_name)]))

        self.template.add_output(Output('vpcId', Value=self.template.vpc_id))

        self.template._vpc_cidr = FindInMap('networkAddresses', 'vpcBase', 'cidr')

        self.template._igw = self.template.add_resource(ec2.InternetGateway('vpcIgw'))

        ## add IGW
        igw_title = 'igwVpcAttachment'
        self.template._vpc_gateway_attachment = self.template.add_resource(ec2.VPCGatewayAttachment(
            igw_title,
            InternetGatewayId=self.template.igw,
            VpcId=self.template.vpc_id))

        self.gateway_hook()

        ## make Subnets
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))

        # Iterate through each subnet type for each AZ and add subnets, routing tables, routes, and NATs as necessary
        for index in range(int(network_config.get('az_count', 2))):
            az = index
            az_key = 'AZ{}'.format(az)

            for ind, subnet_config in enumerate(network_config.get('subnet_config', {})):
                subnet_type = subnet_config.get('type', 'private')
                subnet_layer = subnet_config.get('name')

                AvailabilityZone = FindInMap('RegionMap', Ref('AWS::Region'), 'az' + str(index) + 'Name')
                CidrBlock = self.template.mappings['networkAddresses'][az_key][subnet_layer]

                # Save the subnet references to the template object
                if subnet_type not in self.template._subnets:
                    self.template._subnets[subnet_type] = {}

                if subnet_layer not in self.template._subnets[subnet_type]:
                    self.template._subnets[subnet_type][subnet_layer] = []

                # Create the subnet
                subnet_name = subnet_layer + 'AZ' + str(index)
                subnet = self.template.add_resource(ec2.Subnet(
                    subnet_name,
                    AvailabilityZone=AvailabilityZone,
                    VpcId=self.template.vpc_id,
                    CidrBlock=CidrBlock,
                    Tags=[ec2.Tag(key='network', value=subnet_type),
                          ec2.Tag(key='Name', value=' '.join([subnet_layer, 'AZ:', str(index)])),
                        ]))

                self.template._subnets[subnet_type][subnet_layer].append(subnet)

                self.template.add_output(Output(subnet_name, Value=CidrBlock))

                # Create the routing table
                route_table = self.template.add_resource(ec2.RouteTable(
                    subnet_layer + 'AZ' + str(index) + 'RouteTable',
                    VpcId=self.template.vpc_id))

                # Create the NATs and egress rules
                self.create_subnet_egress(index, route_table, igw_title, subnet_type, subnet_layer)

                # Associate the routing table with the subnet
                self.template.add_resource(ec2.SubnetRouteTableAssociation(
                    subnet_layer + 'AZ' + str(index) + 'EgressRouteTableAssociation',
                    RouteTableId=Ref(route_table),
                    SubnetId=Ref(subnet)))

        self.template.manual_parameter_bindings['vpcId'] = self.template.vpc_id
        self.template.manual_parameter_bindings['vpcCidr'] = self.template.vpc_cidr
        self.template.manual_parameter_bindings['internetGateway'] = self.template.igw
        self.template.manual_parameter_bindings['igwVpcAttachment'] = self.template.vpc_gateway_attachment

    def create_subnet_egress(self, index, route_table, igw_title, subnet_type, subnet_layer):
        """
        Create an egress route for the subnet with the given index and type
        Override to create egress routes for other subnet types
        Creates the NAT instances in the public subnets
        """

        # For public subnets, create the route to the IGW
        if subnet_type == 'public':
            self.template.add_resource(ec2.Route(subnet_layer + 'AZ' + str(index) + 'EgressRoute',
                DependsOn=[igw_title],
                DestinationCidrBlock='0.0.0.0/0',
                GatewayId=self.template.igw,
                RouteTableId=Ref(route_table)))

        # For private subnets, create a NAT instance in a public subnet in the same AZ
        elif subnet_type == 'private':

            # If we have already created a NAT in this AZ, skip it
            if self.az_nat_mapping.get(index):
                return

            nat_instance_type = self.config['nat']['instance_type']
            nat_enable_ntp = self.config['nat']['enable_ntp']
            extra_user_data = self.config['nat'].get('extra_user_data')
            ha_nat = self.create_nat(
                index,
                nat_instance_type,
                nat_enable_ntp,
                name='HaNat' + str(index),
                extra_user_data=extra_user_data)

            # We merge the NAT template into the root template
            self.template.merge(ha_nat)

            # Save the reference to the HA NAT, so we don't recreate it if we hit another private subnet in this AZ
            self.az_nat_mapping[index] = ha_nat

    def gateway_hook(self):
        """
        Override to allow subclasses to create VPGs and similar components during network creation
        """
        pass

    def create_nat(self, index, nat_instance_type, enable_ntp, name, extra_user_data=None):
        """
        Override to customize your NAT instance. The returned object must be a
        subclass of ha_nat.HaNat.
        """
        return ha_nat.HaNat(
            index,
            nat_instance_type,
            enable_ntp,
            name=name,
            extra_user_data=extra_user_data)

    def _get_subnet_config_w_az(self, network_config):
        az_count = int(network_config.get('az_count', 2))
        subnet_config = network_config.get('subnet_config', {})

        for subnet in subnet_config:
            for az in range(az_count):
                newsubnet = assoc(subnet, 'AZ', az)
                yield newsubnet

    def _get_subnet_config_w_cidr(self, network_config):
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))
        network_cidr_size = str(network_config.get('network_cidr_size', '20'))
        first_network_address_block = str(network_config.get('first_network_address_block', network_cidr_base))

        ret_val = {}
        base_cidr = network_cidr_base + '/' + network_cidr_size
        net = netaddr.IPNetwork(base_cidr)

        grouped_subnet = groupby('size', self._get_subnet_config_w_az(network_config))
        subnet_groups = sorted(grouped_subnet.items())
        available_cidrs = []

        for subnet_size, subnet_configs in subnet_groups:
            newcidrs = net.subnet(int(subnet_size))

            for subnet_config in subnet_configs:
                try:
                    cidr = newcidrs.next()
                except StopIteration as e:
                    net = chain(*reversed(available_cidrs)).next()
                    newcidrs = net.subnet(int(subnet_size))
                    cidr = newcidrs.next()

                new_config = assoc(subnet_config, 'cidr', str(cidr))
                yield new_config
            else:
                net = newcidrs.next()
                available_cidrs.append(newcidrs)

    def add_network_cidr_mapping(self,
                                 network_config):
        """
        Method calculates and adds a CloudFormation mapping that is used to set VPC and Subnet CIDR blocks.
        Calculated based on CIDR block sizes and additionally checks to ensure all network segments
        fit inside of the specified overall VPC CIDR.
        @param network_config [dict] dictionary of values containing data for creating
        """
        az_count = int(network_config.get('az_count', '2'))
        network_cidr_base = str(network_config.get('network_cidr_base', '172.16.0.0'))
        network_cidr_size = str(network_config.get('network_cidr_size', '20'))
        first_network_address_block = str(network_config.get('first_network_address_block', network_cidr_base))

        ret_val = {}
        base_cidr = network_cidr_base + '/' + network_cidr_size
        cidr_info = Network(base_cidr)
        ret_val['vpcBase'] = {'cidr': base_cidr}
        current_base_address = first_network_address_block

        subnet_config = self._get_subnet_config_w_cidr(network_config)
        subnet_config = list(subnet_config)

        for index, subnet_config in enumerate(subnet_config):
            subnet_type = subnet_config.get('type', 'private')
            subnet_size = subnet_config.get('size', '22')
            subnet_name = subnet_config.get('name', 'subnet')
            subnet_az = subnet_config.get('AZ', '-1')
            subnet_cidr = subnet_config.get('cidr', 'ERROR')
            az_key = 'AZ{}'.format(subnet_az)

            # TODO: check for subnet collisions

            if az_key not in ret_val:
                ret_val[az_key] = dict()
            if subnet_name not in ret_val[az_key]:
                ret_val[az_key][subnet_name] = dict()
            ret_val[az_key][subnet_name] = subnet_cidr

        return self.template.add_mapping('networkAddresses', ret_val)

    def add_vpn_gateway(self,
                        vpn_conf):
        """
        Not surprisingly, adds a VPN gateway to the network created by this template.
        @param vpn_conf [dict] - collection of vpn-level configuration values.
        """
        if 'vpn_name' in vpn_conf:
            vpn_name = vpn_conf.get('vpn_name')
        else:
            vpn_name = self.__class__.__name__ + 'Gateway'

        gateway = self.template.add_resource(ec2.VPNGateway('vpnGateway',
            Type=vpn_conf.get('vpn_type', 'ipsec.1'),
            Tags=[ec2.Tag(key='Name', value=vpn_name)]))

        gateway_connection = self.template.add_resource(ec2.VPCGatewayAttachment('vpnGatewayAttachment',
            VpcId=self.template.vpc_id,
            InternetGatewayId=self.template.igw,
            VpnGatewayId=gateway))
<|endoftext|>


<commit_before>from pprint import pprint

from django.contrib.auth.mixins import PermissionRequiredMixin
from django.db import connections
from django.shortcuts import render
from django.views import View

from fo2.connections import db_cursor_so

from utils.functions.views import cleanned_fields_to_context

from estoque import classes, forms, models

class Transferencia(PermissionRequiredMixin, View):

    Form_class = forms.TransferenciaForm
    template_name = 'estoque/transferencia.html'
    title_name = 'Movimentações'

    cleanned_fields_to_context = cleanned_fields_to_context

    def __init__(self):
        self.permission_required = 'estoque.can_transferencia'
        self.context = {'titulo': self.title_name}

    def valid_tipo(self):
        try:
            tip_mov = models.TipoMovStq.objects.get(codigo=self.kwargs['tipo'])
        except models.TipoMovStq.DoesNotExist as e:
            self.context.update({
                'erro_input': True,
                'erro_msg':
                    f'Tipo de movimento de estoque "{self.kwargs["tipo"]}" '
                    'não cadastrado.',
            })
            return
        return tip_mov

    def get_tipo(self):
        self.tip_mov = self.valid_tipo()
        if self.tip_mov:
            self.context.update({
                'tipo': self.tip_mov.codigo,
                'titulo': self.tip_mov.descricao,
            })

    def mount_context(self):
        self.cursor = db_cursor_so(self.request)

        try:
            transf = classes.Transfere(
                self.cursor, self.request, self.tip_mov,
                *(self.context[f] for f in [
                    'nivel', 'ref', 'tam',
                    'cores' if self.context['cores'] else 'cor',
                    'qtd', 'deposito_origem', 'deposito_destino',
                    'nova_ref', 'novo_tam',
                    'novas_cores' if self.context['novas_cores'] else 'nova_cor',
                    'num_doc', 'descricao']),
            )
        except Exception as e:
            self.context.update({
                'erro_input': True,
                'erro_msg': e,
            })
            return

        self.context.update({
            'mov_origem': self.tip_mov.trans_saida != 0,
            'mov_destino': self.tip_mov.trans_entrada != 0,
            'itens_saida': transf.itens_saida,
            'itens_entrada': transf.itens_entrada,
            'num_doc': transf.num_doc,
        })

        if 'executa' in self.request.POST:
            try:
                transf.exec()
            except Exception as e:
                self.context.update({
                    'erro_exec': True,
                    'erro_msg': e,
                })
                return
            self.context.update({
                'sucesso_msg': f"{self.context['titulo']} executada."
            })

    def get(self, request, *args, **kwargs):
        self.request = request
        self.get_tipo()
        if not self.tip_mov:
            return render(request, self.template_name, self.context)
        self.context['form'] = self.Form_class(
            user=request.user, tipo_mov=self.tip_mov)
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        self.request = request
        self.get_tipo()
        self.context['form'] = self.Form_class(
            request.POST, user=self.request.user, tipo_mov=self.tip_mov)
        if self.context['form'].is_valid():
            self.cleanned_fields_to_context()
            if self.tip_mov:
                self.mount_context()
            self.context['form'] = self.Form_class(
                self.context, user=self.request.user, tipo_mov=self.tip_mov)
        else:
            self.context.update({
                'erro_input': True,
                'erro_msg':
                    'Erro no preenchimento dos campos.',
            })
        return render(request, self.template_name, self.context)
<commit_msg>class Transferencia passando cursor para TransferenciaForm<commit_after>from pprint import pprint

from django.contrib.auth.mixins import PermissionRequiredMixin
from django.shortcuts import render
from django.views import View

from fo2.connections import db_cursor_so

from utils.functions.views import cleanned_fields_to_context

from estoque import classes, forms, models

class Transferencia(PermissionRequiredMixin, View):

    Form_class = forms.TransferenciaForm
    template_name = 'estoque/transferencia.html'
    title_name = 'Movimentações'

    cleanned_fields_to_context = cleanned_fields_to_context

    def __init__(self):
        self.permission_required = 'estoque.can_transferencia'
        self.context = {'titulo': self.title_name}

    def valid_tipo(self):
        try:
            tip_mov = models.TipoMovStq.objects.get(codigo=self.kwargs['tipo'])
        except models.TipoMovStq.DoesNotExist as e:
            self.context.update({
                'erro_input': True,
                'erro_msg':
                    f'Tipo de movimento de estoque "{self.kwargs["tipo"]}" '
                    'não cadastrado.',
            })
            return
        return tip_mov

    def get_tipo(self):
        self.tip_mov = self.valid_tipo()
        if self.tip_mov:
            self.context.update({
                'tipo': self.tip_mov.codigo,
                'titulo': self.tip_mov.descricao,
            })

    def mount_context(self):
        try:
            transf = classes.Transfere(
                self.cursor, self.request, self.tip_mov,
                *(self.context[f] for f in [
                    'nivel', 'ref', 'tam',
                    'cores' if self.context['cores'] else 'cor',
                    'qtd', 'deposito_origem', 'deposito_destino',
                    'nova_ref', 'novo_tam',
                    'novas_cores' if self.context['novas_cores'] else 'nova_cor',
                    'num_doc', 'descricao']),
            )
        except Exception as e:
            self.context.update({
                'erro_input': True,
                'erro_msg': e,
            })
            return

        self.context.update({
            'mov_origem': self.tip_mov.trans_saida != 0,
            'mov_destino': self.tip_mov.trans_entrada != 0,
            'itens_saida': transf.itens_saida,
            'itens_entrada': transf.itens_entrada,
            'num_doc': transf.num_doc,
        })

        if 'executa' in self.request.POST:
            try:
                transf.exec()
            except Exception as e:
                self.context.update({
                    'erro_exec': True,
                    'erro_msg': e,
                })
                return
            self.context.update({
                'sucesso_msg': f"{self.context['titulo']} executada."
            })

    def get(self, request, *args, **kwargs):
        self.request = request
        self.cursor = db_cursor_so(request)
        self.get_tipo()
        if not self.tip_mov:
            return render(request, self.template_name, self.context)
        self.context['form'] = self.Form_class(
            cursor=self.cursor, user=request.user, tipo_mov=self.tip_mov)
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        self.request = request
        self.cursor = db_cursor_so(request)
        self.get_tipo()
        self.context['form'] = self.Form_class(
            request.POST, cursor=self.cursor, user=self.request.user, tipo_mov=self.tip_mov)
        if self.context['form'].is_valid():
            self.cleanned_fields_to_context()
            if self.tip_mov:
                self.mount_context()
            self.context['form'] = self.Form_class(
                self.context, cursor=self.cursor, user=self.request.user, tipo_mov=self.tip_mov)
        else:
            self.context.update({
                'erro_input': True,
                'erro_msg':
                    'Erro no preenchimento dos campos.',
            })
        return render(request, self.template_name, self.context)
<|endoftext|>


<commit_before># file django/forms/fields.py
#
#   Copyright 2010 Emory University General Library
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import re

from django.core.validators import RegexValidator
from django.forms import CharField, ChoiceField
from django.forms.widgets import Select, TextInput, Widget
from django.utils.safestring import mark_safe

# regular expression to validate and parse W3C dates
W3C_DATE_RE = re.compile(r'^(?P<year>\d{4})(?:-(?P<month>[0-1]\d)(?:-(?P<day>[0-3]\d))?)?$')
validate_w3c_date = RegexValidator(W3C_DATE_RE,
    u'Enter a valid W3C date in one of these formats: YYYY, YYYY-MM, or YYYY-MM-DD',
    'invalid')

class W3CDateWidget(Widget):
    '''Multi-part date widget that generates three text input boxes for year,
    month, and day.  Expects and generates dates in any of these W3C formats,
    depending on which fields are filled in: YYYY-MM-DD, YYYY-MM, or YYYY.
    '''

    # based in part on SelectDateWidget from django.forms.extras.widgets
    month_field = '%s_month'
    day_field = '%s_day'
    year_field = '%s_year'

    def value_from_datadict(self, data, files, name):
        '''Generate a single value from multi-part form data.  Constructs a W3C
        date based on values that are set, leaving out day and month if they are
        not present.

        :param data: dictionary of data submitted by the form
        :param files: - unused
        :param name: base name of the form field
        :returns: string value
        '''

        y = data.get(self.year_field % name)
        m = data.get(self.month_field % name)
        d = data.get(self.day_field % name)

        if y == 'YYYY':
            y = ''

        if m == 'MM':
            m = ''

        if d == 'DD':
            d = ''

        date = y
        if m:
            date += '-%s' % m
            if d:
                date += '-%s' % d
        return date

    def render(self, name, value, attrs=None):
        '''Render the widget as HTML inputs for display on a form.

        :param name: form field base name
        :param value: date value
        :param attrs: - unused
        :returns: HTML text with three inputs for year/month/day
        '''

        # expects a value in format YYYY-MM-DD or YYYY-MM or YYYY (or empty/None)
        year, month, day = 'YYYY', 'MM', 'DD'
        if value:
            # use the regular expression to pull out year, month, and day values
            # if regular expression does not match, inputs will be empty
            match = W3C_DATE_RE.match(value)
            if match:
                date_parts = match.groupdict()
                year = date_parts['year']
                month = date_parts['month']
                day = date_parts['day']

        year_html = self.create_textinput(name, self.year_field, year, size=4, title='4-digit year', onClick='javascript:if(this.value == "YYYY") { this.value = "" };')
        month_html = self.create_textinput(name, self.month_field, month, size=2, title='2-digit month', onClick='javascript:if(this.value == "MM") { this.value = "" };')
        day_html = self.create_textinput(name, self.day_field, day, size=2, title='2-digit day', onClick='javascript:if(this.value == "DD") { this.value = "" };')

        output = [year_html, month_html, day_html]
        #output = [month_html, day_html, year_html]

        return mark_safe(u' / \n'.join(output))

    def create_textinput(self, name, field, value, **extra_attrs):
        '''Generate and render a :class:`django.forms.widgets.TextInput` for
        a single year, month, or day input.

        If size is specified in the extra attributes, it will also be used to
        set the maximum length of the field.

        :param name: base name of the input field
        :param field: pattern for this field (used with name to generate input name)
        :param value: initial value for the field
        :param extra_attrs: any extra widget attributes
        :returns: rendered HTML output for the text input
        '''
        if 'id' in self.attrs:
            id_ = self.attrs['id']
        else:
            id_ = 'id_%s' % name

        # use size to set maximum length
        if 'size' in extra_attrs:
            extra_attrs['maxlength'] = extra_attrs['size']
        local_attrs = self.build_attrs(id=field % id_, **extra_attrs)
        input = TextInput()
        return input.render(field % name, value, local_attrs)

class W3CDateField(CharField):
    '''W3C date field that uses a :class:`W3CDateWidget` for presentation
    and uses a simple regular expression to do basic validation on the input
    (but does not actually test that it is a valid date).
    '''
    widget = W3CDateWidget
    default_error_messages = {
        'invalid':  u'Enter a date in one of these formats: YYYY, YYYY-MM, or YYYY-MM-DD',
    }
    default_validators = [validate_w3c_date]

class DynamicSelect(Select):
    '''A :class:`~django.forms.widgets.Select` widget whose choices are not
    static, but instead generated dynamically when referenced.

    :param choices: callable; this will be called to generate choices each
                    time they are referenced.
    '''

    def __init__(self, attrs=None, choices=None):
        # Skip right over Select and go to its parents. Select just sets
        # self.choices, which will break since it's a property here.
        super(Select, self).__init__(attrs)

        if choices is None:
            choices = lambda: ()
        self._choices = choices

    def _get_choices(self):
        return self._choices()
    choices = property(_get_choices)

class DynamicChoiceField(ChoiceField):
    '''A :class:`django.forms.ChoiceField` whose choices are not static, but
    instead generated dynamically when referenced.

    :param choices: callable; this will be called to generate choices each
                    time they are referenced
    '''

    widget = DynamicSelect

    def __init__(self, choices=None, widget=None, *args, **kwargs):
        # ChoiceField.__init__ tries to set static choices, which won't
        # work since our choices are dynamic, so we're going to have to skip
        # over it.

        # First normalize our choices
        if choices is None:
            choices = lambda: ()
        self._choices = choices

        # Then normalize our widget, constructing it with our choices
        # function if we need to construct it.
        if widget is None:
            widget = self.widget
        if isinstance(widget, type):
            widget = widget(choices=choices)

        # Now call call super.__init__(), but bypass ChoiceField.
        # ChoiceField just sets static choices manually and then calls its
        # own super. We don't have static choices, so ChoiceField.__init__()
        # would break if we called it. Skip over ChoiceField and go straight
        # to *its* super.__init__().
        super(ChoiceField, self).__init__(widget=widget, *args, **kwargs)

    def _get_choices(self):
        return self._choices()
    choices = property(_get_choices)
<commit_msg>document rationale for order of w3cdate field widget display<commit_after># file django/forms/fields.py
#
#   Copyright 2010 Emory University General Library
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import re

from django.core.validators import RegexValidator
from django.forms import CharField, ChoiceField
from django.forms.widgets import Select, TextInput, Widget
from django.utils.safestring import mark_safe

# regular expression to validate and parse W3C dates
W3C_DATE_RE = re.compile(r'^(?P<year>\d{4})(?:-(?P<month>[0-1]\d)(?:-(?P<day>[0-3]\d))?)?$')
validate_w3c_date = RegexValidator(W3C_DATE_RE,
    u'Enter a valid W3C date in one of these formats: YYYY, YYYY-MM, or YYYY-MM-DD',
    'invalid')

class W3CDateWidget(Widget):
    '''Multi-part date widget that generates three text input boxes for year,
    month, and day.  Expects and generates dates in any of these W3C formats,
    depending on which fields are filled in: YYYY-MM-DD, YYYY-MM, or YYYY.
    '''

    # based in part on SelectDateWidget from django.forms.extras.widgets
    month_field = '%s_month'
    day_field = '%s_day'
    year_field = '%s_year'

    def value_from_datadict(self, data, files, name):
        '''Generate a single value from multi-part form data.  Constructs a W3C
        date based on values that are set, leaving out day and month if they are
        not present.

        :param data: dictionary of data submitted by the form
        :param files: - unused
        :param name: base name of the form field
        :returns: string value
        '''

        y = data.get(self.year_field % name)
        m = data.get(self.month_field % name)
        d = data.get(self.day_field % name)

        if y == 'YYYY':
            y = ''

        if m == 'MM':
            m = ''

        if d == 'DD':
            d = ''

        date = y
        if m:
            date += '-%s' % m
            if d:
                date += '-%s' % d
        return date

    def render(self, name, value, attrs=None):
        '''Render the widget as HTML inputs for display on a form.

        :param name: form field base name
        :param value: date value
        :param attrs: - unused
        :returns: HTML text with three inputs for year/month/day
        '''

        # expects a value in format YYYY-MM-DD or YYYY-MM or YYYY (or empty/None)
        year, month, day = 'YYYY', 'MM', 'DD'
        if value:
            # use the regular expression to pull out year, month, and day values
            # if regular expression does not match, inputs will be empty
            match = W3C_DATE_RE.match(value)
            if match:
                date_parts = match.groupdict()
                year = date_parts['year']
                month = date_parts['month']
                day = date_parts['day']

        year_html = self.create_textinput(name, self.year_field, year, size=4, title='4-digit year', onClick='javascript:if(this.value == "YYYY") { this.value = "" };')
        month_html = self.create_textinput(name, self.month_field, month, size=2, title='2-digit month', onClick='javascript:if(this.value == "MM") { this.value = "" };')
        day_html = self.create_textinput(name, self.day_field, day, size=2, title='2-digit day', onClick='javascript:if(this.value == "DD") { this.value = "" };')

        # display widget fields in YYYY-MM-DD order to match W3C date format,
        # and putting required field(s) on the left
        output = [year_html, month_html, day_html]

        return mark_safe(u' / \n'.join(output))

    def create_textinput(self, name, field, value, **extra_attrs):
        '''Generate and render a :class:`django.forms.widgets.TextInput` for
        a single year, month, or day input.

        If size is specified in the extra attributes, it will also be used to
        set the maximum length of the field.

        :param name: base name of the input field
        :param field: pattern for this field (used with name to generate input name)
        :param value: initial value for the field
        :param extra_attrs: any extra widget attributes
        :returns: rendered HTML output for the text input
        '''
        if 'id' in self.attrs:
            id_ = self.attrs['id']
        else:
            id_ = 'id_%s' % name

        # use size to set maximum length
        if 'size' in extra_attrs:
            extra_attrs['maxlength'] = extra_attrs['size']
        local_attrs = self.build_attrs(id=field % id_, **extra_attrs)
        input = TextInput()
        return input.render(field % name, value, local_attrs)

class W3CDateField(CharField):
    '''W3C date field that uses a :class:`W3CDateWidget` for presentation
    and uses a simple regular expression to do basic validation on the input
    (but does not actually test that it is a valid date).
    '''
    widget = W3CDateWidget
    default_error_messages = {
        'invalid':  u'Enter a date in one of these formats: YYYY, YYYY-MM, or YYYY-MM-DD',
    }
    default_validators = [validate_w3c_date]

class DynamicSelect(Select):
    '''A :class:`~django.forms.widgets.Select` widget whose choices are not
    static, but instead generated dynamically when referenced.

    :param choices: callable; this will be called to generate choices each
                    time they are referenced.
    '''

    def __init__(self, attrs=None, choices=None):
        # Skip right over Select and go to its parents. Select just sets
        # self.choices, which will break since it's a property here.
        super(Select, self).__init__(attrs)

        if choices is None:
            choices = lambda: ()
        self._choices = choices

    def _get_choices(self):
        return self._choices()
    choices = property(_get_choices)

class DynamicChoiceField(ChoiceField):
    '''A :class:`django.forms.ChoiceField` whose choices are not static, but
    instead generated dynamically when referenced.

    :param choices: callable; this will be called to generate choices each
                    time they are referenced
    '''

    widget = DynamicSelect

    def __init__(self, choices=None, widget=None, *args, **kwargs):
        # ChoiceField.__init__ tries to set static choices, which won't
        # work since our choices are dynamic, so we're going to have to skip
        # over it.

        # First normalize our choices
        if choices is None:
            choices = lambda: ()
        self._choices = choices

        # Then normalize our widget, constructing it with our choices
        # function if we need to construct it.
        if widget is None:
            widget = self.widget
        if isinstance(widget, type):
            widget = widget(choices=choices)

        # Now call call super.__init__(), but bypass ChoiceField.
        # ChoiceField just sets static choices manually and then calls its
        # own super. We don't have static choices, so ChoiceField.__init__()
        # would break if we called it. Skip over ChoiceField and go straight
        # to *its* super.__init__().
        super(ChoiceField, self).__init__(widget=widget, *args, **kwargs)

    def _get_choices(self):
        return self._choices()
    choices = property(_get_choices)
<|endoftext|>