<commit_before># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
OpenFlow doesn't know anything about Topology, and Topology doesn't
know anything about OpenFlow.  This module knows something about both,
and hooks the two of them together.

Specifically, this module is somewhat like an adapter that listens to
events from other parts of the openflow substem (such as discovery), and
uses them to populate and manipulate Topology.
"""

from pox.lib.revent.revent import *
import libopenflow_01 as of
from pox.openflow import *
from pox.core import core
from pox.topology.topology import *
from pox.openflow.discovery import *
from pox.openflow.flow_table import FlowTable
from pox.lib.util import dpidToStr
from pox.lib.addresses import *

# After a switch disconnects, it has this many seconds to reconnect in
# order to reactivate the same OpenFlowSwitch object.  After this, if
# it reconnects, it will be a new switch object.
RECONNECT_TIMEOUT = 30

log = core.getLogger()

class OpenFlowTopology (EventMixin):
  """
  Listens to various OpenFlow-specific events and uses those to manipulate
  Topology accordingly.
  """

  # Won't boot up OpenFlowTopology until all of these components are loaded
  # into pox.core. Note though that these components won't be loaded
  # proactively; they must be specified on the command line (with the
  # exception of openflow which usally loads automatically)
  _wantComponents = set(['openflow','topology','openflow_discovery'])

  def __init__ (self):
    """ Note that self.topology is initialized in _resolveComponents """
    super(EventMixin, self).__init__()
    if not core.listenToDependencies(self, self._wantComponents):
      self.listenTo(core)

  def _handle_openflow_discovery_LinkEvent (self, event):
    """
    The discovery module simply sends out LLDP packets, and triggers LinkEvents
    for discovered switches. It's our job to take these LinkEvents and update
    pox.topology.
    """
    if self.topology is None: return
    link = event.link
    sw1 = self.topology.getEntityByID(link.dpid1)
    sw2 = self.topology.getEntityByID(link.dpid2)
    if sw1 is None or sw2 is None: return
    if link.port1 not in sw1.ports or link.port2 not in sw2.ports: return
    if event.added:
      sw1.ports[link.port1].addEntity(sw2, single=True)
      sw2.ports[link.port2].addEntity(sw1, single=True)
    elif event.removed:
      sw1.ports[link.port1].entities.remove(sw2)
      sw2.ports[link.port2].entities.remove(sw1)

  def _handle_ComponentRegistered (self, event):
    """
    A component was registered with pox.core. If we were dependent on it,
    check again if all of our dependencies are now satisfied so we can boot.
    """
    if core.listenToDependencies(self, self._wantComponents):
      return EventRemove

  def _handle_openflow_ConnectionUp (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    add = False
    if sw is None:
      sw = OpenFlowSwitch(event.dpid)
      add = True
    else:
      if sw.connection is not None:
        log.warn("Switch %s connected, but... it's already connected!" %
                 (dpidToStr(event.dpid),))
    #sw._setConnection(event.connection, event.ofp)
    log.info("Switch " + dpidToStr(event.dpid) + " connected")
    if add:
      self.topology.addEntity(sw)
      sw.raiseEvent(SwitchJoin, sw)

  def _handle_openflow_ConnectionDown (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    if sw is None:
      log.warn("Switch %s disconnected, but... it doesn't exist!" %
               (dpidToStr(event.dpid),))
    else:
      if sw.connection is None:
        log.warn("Switch %s disconnected, but... it's wasn't connected!" %
                 (dpidToStr(event.dpid),))
      sw.connection = None
      log.info("Switch " + str(event.dpid) + " disconnected")

class OpenFlowPort (Port):
  """
  A subclass of topology.Port for OpenFlow switch ports.

  Adds the notion of "connected entities", which the default
  ofp_phy_port class does not have.

  Note: Not presently used.
  """
  def __init__ (self, ofp):
    # Passed an ofp_phy_port
    Port.__init__(self, ofp.port_no, ofp.hw_addr, ofp.name)
    self.isController = self.number == of.OFPP_CONTROLLER
    self._update(ofp)
    self.exists = True
    self.entities = set()

  def _update (self, ofp):
    assert self.name == ofp.name
    assert self.number == ofp.port_no
    self.hwAddr = EthAddr(ofp.hw_addr)
    self._config = ofp.config
    self._state = ofp.state

  def __contains__ (self, item):
    """ True if this port connects to the specified entity """
    return item in self.entities

  def addEntity (self, entity, single = False):
    # Invariant (not currently enforced?):
    #   len(self.entities) <= 2  ?
    if single:
      self.entities = set([entity])
    else:
      self.entities.add(entity)

  def to_ofp_phy_port(self):
    return of.ofp_phy_port(port_no = self.number, hw_addr = self.hwAddr,
                           name = self.name, config = self._config,
                           state = self._state)

  def __repr__ (self):
    return "<Port #" + str(self.number) + ">"

class OpenFlowSwitch (EventMixin, Switch):
  """
  OpenFlowSwitches are Topology entities (inheriting from topology.Switch)

  OpenFlowSwitches are persistent; that is, if a switch reconnects, the
  Connection field of the original OpenFlowSwitch object will simply be
  reset to refer to the new connection.

  For now, OpenFlowSwitch is primarily a proxy to its underlying connection
  object. Later, we'll possibly add more explicit operations the client can
  perform.

  Note that for the purposes of the debugger, we can interpose on
  a switch entity by enumerating all listeners for the events listed
  below, and triggering mock events for those listeners.
  """
  _eventMixin_events = set([
    SwitchJoin, # Defined in pox.topology
    SwitchLeave,

    PortStatus, # Defined in libopenflow_01
    FlowRemoved,
    PacketIn,
    BarrierIn,
  ])

  def __init__ (self, dpid):
    Switch.__init__(self)
    EventMixin.__init__(self)
    self.dpid = dpid
    self.ports = {}
    self.flow_table = FlowTable()
    self.capabilities = 0
    self._connection = None
    self._listeners = []
    self._reconnectTimeout = None # Timer for reconnection
    #self.xid_generator = count.count( (dpid & 0xFFFF) << 16 + 1)

  def _setConnection (self, connection, ofp=None):
    ''' ofp - a FeaturesReply message '''
    if self._connection: self._connection.removeListeners(self._listeners)
    self._listeners = []
    self._connection = connection
    if self._reconnectTimeout is not None:
      self._reconnectTimeout.cancel()
      self._reconnectTimeout = None
    if connection is None:
      self._reconnectTimeout = Timer(RECONNECT_TIMEOUT,
                                     self._timer_ReconnectTimeout)
    if ofp is not None:
      # update capabilities
      self.capabilities = ofp.capabilities
      # update all ports
      untouched = set(self.ports.keys())
      for p in ofp.ports:
        if p.port_no in self.ports:
          self.ports[p.port_no]._update(p)
          untouched.remove(p.port_no)
        else:
          self.ports[p.port_no] = OpenFlowPort(p)
      for p in untouched:
        self.ports[p].exists = False
        del self.ports[p]
    if connection is not None:
      self._listeners = self.listenTo(connection, prefix="con")
      self.raiseEvent(SwitchConnectionUp(switch=self, connection = connection))
    else:
      self.raiseEvent(SwitchConnectionDown(switch=self))

  def _timer_ReconnectTimeout (self):
    """ Called if we've been disconnected for RECONNECT_TIMEOUT seconds """
    self._reconnectTimeout = None
    core.topology.removeEntity(self)
    self.raiseEvent(SwitchLeave, self)

  def _handle_con_PortStatus (self, event):
    p = event.ofp.desc
    if event.ofp.reason == of.OFPPR_DELETE:
      if p.port_no in self.ports:
        self.ports[p.port_no].exists = False
        del self.ports[p.port_no]
    elif event.ofp.reason == of.OFPPR_MODIFY:
      self.ports[p.port_no]._update(p)
    else:
      assert event.ofp.reason == of.OFPPR_ADD
      assert p.port_no not in self.ports
      self.ports[p.port_no] = OpenFlowPort(p)
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_ConnectionDown (self, event):
    self._setConnection(None)

  def _handle_con_PacketIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_BarrierIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_FlowRemoved (self, event):
    self.raiseEvent(event)
    self.flowTable.removeFlow(event)
    event.halt = False

  def findPortForEntity (self, entity):
    for p in self.ports.itervalues():
      if entity in p:
        return p
    return None

  def send(self, *args, **kw):
    return self._connection.send(*args, **kw)

  def read(self):
   return self._connection.read()

  def __repr__ (self):
    return "<%s %s>" % (self.__class__.__name__, dpidToStr(self.dpid))

  @property
  def connected(self):
    return self._connection != None

def launch ():
  if not core.hasComponent("openflow_topology"):
    core.register("openflow_topology", OpenFlowTopology())
<commit_msg>openflow.topology: fixing bug where OpenFlowSwitch.connection was misnamed<commit_after># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
OpenFlow doesn't know anything about Topology, and Topology doesn't
know anything about OpenFlow.  This module knows something about both,
and hooks the two of them together.

Specifically, this module is somewhat like an adapter that listens to
events from other parts of the openflow substem (such as discovery), and
uses them to populate and manipulate Topology.
"""

from pox.lib.revent.revent import *
import libopenflow_01 as of
from pox.openflow import *
from pox.core import core
from pox.topology.topology import *
from pox.openflow.discovery import *
from pox.openflow.flow_table import FlowTable
from pox.lib.util import dpidToStr
from pox.lib.addresses import *

# After a switch disconnects, it has this many seconds to reconnect in
# order to reactivate the same OpenFlowSwitch object.  After this, if
# it reconnects, it will be a new switch object.
RECONNECT_TIMEOUT = 30

log = core.getLogger()

class OpenFlowTopology (EventMixin):
  """
  Listens to various OpenFlow-specific events and uses those to manipulate
  Topology accordingly.
  """

  # Won't boot up OpenFlowTopology until all of these components are loaded
  # into pox.core. Note though that these components won't be loaded
  # proactively; they must be specified on the command line (with the
  # exception of openflow which usally loads automatically)
  _wantComponents = set(['openflow','topology','openflow_discovery'])

  def __init__ (self):
    """ Note that self.topology is initialized in _resolveComponents """
    super(EventMixin, self).__init__()
    if not core.listenToDependencies(self, self._wantComponents):
      self.listenTo(core)

  def _handle_openflow_discovery_LinkEvent (self, event):
    """
    The discovery module simply sends out LLDP packets, and triggers LinkEvents
    for discovered switches. It's our job to take these LinkEvents and update
    pox.topology.
    """
    if self.topology is None: return
    link = event.link
    sw1 = self.topology.getEntityByID(link.dpid1)
    sw2 = self.topology.getEntityByID(link.dpid2)
    if sw1 is None or sw2 is None: return
    if link.port1 not in sw1.ports or link.port2 not in sw2.ports: return
    if event.added:
      sw1.ports[link.port1].addEntity(sw2, single=True)
      sw2.ports[link.port2].addEntity(sw1, single=True)
    elif event.removed:
      sw1.ports[link.port1].entities.remove(sw2)
      sw2.ports[link.port2].entities.remove(sw1)

  def _handle_ComponentRegistered (self, event):
    """
    A component was registered with pox.core. If we were dependent on it,
    check again if all of our dependencies are now satisfied so we can boot.
    """
    if core.listenToDependencies(self, self._wantComponents):
      return EventRemove

  def _handle_openflow_ConnectionUp (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    add = False
    if sw is None:
      sw = OpenFlowSwitch(event.dpid)
      add = True
    else:
      if sw._connection is not None:
        log.warn("Switch %s connected, but... it's already connected!" %
                 (dpidToStr(event.dpid),))
    #sw._setConnection(event.connection, event.ofp)
    log.info("Switch " + dpidToStr(event.dpid) + " connected")
    if add:
      self.topology.addEntity(sw)
      sw.raiseEvent(SwitchJoin, sw)

  def _handle_openflow_ConnectionDown (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    if sw is None:
      log.warn("Switch %s disconnected, but... it doesn't exist!" %
               (dpidToStr(event.dpid),))
    else:
      if sw._connection is None:
        log.warn("Switch %s disconnected, but... it's wasn't connected!" %
                 (dpidToStr(event.dpid),))
      sw._connection = None
      log.info("Switch " + str(event.dpid) + " disconnected")

class OpenFlowPort (Port):
  """
  A subclass of topology.Port for OpenFlow switch ports.

  Adds the notion of "connected entities", which the default
  ofp_phy_port class does not have.

  Note: Not presently used.
  """
  def __init__ (self, ofp):
    # Passed an ofp_phy_port
    Port.__init__(self, ofp.port_no, ofp.hw_addr, ofp.name)
    self.isController = self.number == of.OFPP_CONTROLLER
    self._update(ofp)
    self.exists = True
    self.entities = set()

  def _update (self, ofp):
    assert self.name == ofp.name
    assert self.number == ofp.port_no
    self.hwAddr = EthAddr(ofp.hw_addr)
    self._config = ofp.config
    self._state = ofp.state

  def __contains__ (self, item):
    """ True if this port connects to the specified entity """
    return item in self.entities

  def addEntity (self, entity, single = False):
    # Invariant (not currently enforced?):
    #   len(self.entities) <= 2  ?
    if single:
      self.entities = set([entity])
    else:
      self.entities.add(entity)

  def to_ofp_phy_port(self):
    return of.ofp_phy_port(port_no = self.number, hw_addr = self.hwAddr,
                           name = self.name, config = self._config,
                           state = self._state)

  def __repr__ (self):
    return "<Port #" + str(self.number) + ">"

class OpenFlowSwitch (EventMixin, Switch):
  """
  OpenFlowSwitches are Topology entities (inheriting from topology.Switch)

  OpenFlowSwitches are persistent; that is, if a switch reconnects, the
  Connection field of the original OpenFlowSwitch object will simply be
  reset to refer to the new connection.

  For now, OpenFlowSwitch is primarily a proxy to its underlying connection
  object. Later, we'll possibly add more explicit operations the client can
  perform.

  Note that for the purposes of the debugger, we can interpose on
  a switch entity by enumerating all listeners for the events listed
  below, and triggering mock events for those listeners.
  """
  _eventMixin_events = set([
    SwitchJoin, # Defined in pox.topology
    SwitchLeave,

    PortStatus, # Defined in libopenflow_01
    FlowRemoved,
    PacketIn,
    BarrierIn,
  ])

  def __init__ (self, dpid):
    Switch.__init__(self)
    EventMixin.__init__(self)
    self.dpid = dpid
    self.ports = {}
    self.flow_table = FlowTable()
    self.capabilities = 0
    self._connection = None
    self._listeners = []
    self._reconnectTimeout = None # Timer for reconnection
    #self.xid_generator = count.count( (dpid & 0xFFFF) << 16 + 1)

  def _setConnection (self, connection, ofp=None):
    ''' ofp - a FeaturesReply message '''
    if self._connection: self._connection.removeListeners(self._listeners)
    self._listeners = []
    self._connection = connection
    if self._reconnectTimeout is not None:
      self._reconnectTimeout.cancel()
      self._reconnectTimeout = None
    if connection is None:
      self._reconnectTimeout = Timer(RECONNECT_TIMEOUT,
                                     self._timer_ReconnectTimeout)
    if ofp is not None:
      # update capabilities
      self.capabilities = ofp.capabilities
      # update all ports
      untouched = set(self.ports.keys())
      for p in ofp.ports:
        if p.port_no in self.ports:
          self.ports[p.port_no]._update(p)
          untouched.remove(p.port_no)
        else:
          self.ports[p.port_no] = OpenFlowPort(p)
      for p in untouched:
        self.ports[p].exists = False
        del self.ports[p]
    if connection is not None:
      self._listeners = self.listenTo(connection, prefix="con")
      self.raiseEvent(SwitchConnectionUp(switch=self, connection = connection))
    else:
      self.raiseEvent(SwitchConnectionDown(switch=self))

  def _timer_ReconnectTimeout (self):
    """ Called if we've been disconnected for RECONNECT_TIMEOUT seconds """
    self._reconnectTimeout = None
    core.topology.removeEntity(self)
    self.raiseEvent(SwitchLeave, self)

  def _handle_con_PortStatus (self, event):
    p = event.ofp.desc
    if event.ofp.reason == of.OFPPR_DELETE:
      if p.port_no in self.ports:
        self.ports[p.port_no].exists = False
        del self.ports[p.port_no]
    elif event.ofp.reason == of.OFPPR_MODIFY:
      self.ports[p.port_no]._update(p)
    else:
      assert event.ofp.reason == of.OFPPR_ADD
      assert p.port_no not in self.ports
      self.ports[p.port_no] = OpenFlowPort(p)
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_ConnectionDown (self, event):
    self._setConnection(None)

  def _handle_con_PacketIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_BarrierIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_FlowRemoved (self, event):
    self.raiseEvent(event)
    self.flowTable.removeFlow(event)
    event.halt = False

  def findPortForEntity (self, entity):
    for p in self.ports.itervalues():
      if entity in p:
        return p
    return None

  def send(self, *args, **kw):
    return self._connection.send(*args, **kw)

  def read(self, *args, *kw):
   return self._connection.read(*args, **kw)

  def __repr__ (self):
    return "<%s %s>" % (self.__class__.__name__, dpidToStr(self.dpid))

  @property
  def connected(self):
    return self._connection != None

def launch ():
  if not core.hasComponent("openflow_topology"):
    core.register("openflow_topology", OpenFlowTopology())
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ppcheckout

Usage:
  ppcheckout [options] <projectid> [<projectname>]
  ppcheckout -h | --help
  ppcheckout --version

Automates the initial setup of a pgdp.net post processing project.

Examples:
  ppcheckout projectID509579cf64be8
  ppcheckout projectID509579cf64be8 proj-name

Options:
  -q, --quiet           Print less text.
  -v, --verbose         Print more text.
  -h, --help            Show help.
  --version             Show version.
"""

from docopt import docopt
from robobrowser import RoboBrowser
import urllib
import logging
import re
import shutil
import os
import zipfile
import glob
import shlex
import subprocess

VERSION="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

def main():
    args = docopt(__doc__, version="ppcheckout v{}".format(VERSION))

    # Configure logging
    logLevel = logging.INFO #default
    if args['--verbose']:
        logLevel = logging.DEBUG
    elif args['--quiet']:
        logLevel = logging.ERROR

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
    logging.debug(args)

    # Process command line arguments
    projectId = args['<projectid>']
    projectName = args['<projectname>']
    projectId = re.search("(projectID[A-Fa-f0-9]+)",projectId).group(1)
    logging.info("Checking out project {} to {}".format(projectId,projectName))

    # Parse project page for title and author
    logging.info("Parsing project page for {}".format(projectId))
    projectURL = "http://www.pgdp.net/c/project.php?id={}".format(projectId)
    browser = RoboBrowser()
    browser.open(projectURL)
    projectInfo = browser.find('table',id='project_info_table')
    title = projectInfo.find('b',text="Title").parent.next_sibling.string
    author = projectInfo.find('b',text="Author").parent.next_sibling.string

    if not projectName:
        projectName = generateProjectName(title)

    # Download project files
    logging.info("Downloading project files for '{}' by {}".format(title,author))
    imagesURL = "http://www.pgdp.net/c/tools/download_images.php?projectid={}&dummy={}images.zip".format(projectId, projectId)
    textURL = "http://www.pgdp.net/projects/{}/{}.zip".format(projectId, projectId)
    downloadFile(imagesURL,"images.zip")
    downloadFile(textURL,"text.zip")

    # Init project skeleton
    logging.info("Building project base structure")
    srcDir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),"_NEW_PROJECT_TEMPLATE"))
    dstDir = os.path.abspath(projectName)
    shutil.copytree(srcDir, dstDir)

    # Copy/unzip project files
    logging.info("Adding project files")
    zipText = zipfile.ZipFile("text.zip","r")
    zipImages = zipfile.ZipFile("images.zip","r")
    zipText.extractall(path=os.path.abspath("{}/originals/".format(projectName)))
    zipImages.extractall(path=os.path.abspath("{}/pngs/".format(projectName)))
    moveFiles(glob.glob(os.path.abspath("{}/pngs/*.jpg".format(projectName))), os.path.abspath("{}/originals/illustrations/".format(projectName)))
    shutil.copy(os.path.abspath("{}/originals/{}".format(projectName,"{}.txt".format(projectId))),os.path.abspath("{}/{}-src.txt".format(projectName,projectName)))
    moveFiles(glob.glob(os.path.abspath("*.zip")), os.path.abspath("{}/originals/".format(projectName)))

    # Convert illustrations
    files = glob.glob(os.path.abspath("{}/originals/illustrations/*.jpg".format(projectName)))
    logging.info("Converting illustrations to lossless format")
    for f in files:
        shellCommand("mogrify -format png {}".format(f))

    # Update Makefile with project name
    projectPath = os.path.abspath("{}/".format(projectName))
    projectFilePath = os.path.abspath("{}/{}-src.txt".format(projectName,projectName))
    prependText('PROJECTNAME={}'.format(projectName), os.path.abspath("{}/Makefile".format(projectName))

    # Initialize git repo
    shellCommand("git init",cwd=projectPath)
    shellCommand("git add {}-src.txt".format(projectName),cwd=projectPath)
    shellCommand('git commit -m "ppcheckout: Initial version"',cwd=projectPath)

    # Convert to UTF-8
    shellCommand("recode ISO-8859-1..UTF-8 {}".format(os.path.abspath("{}/{}-src.txt".format(projectName,projectName))))
    prependText('', projectFilePath) # convert to native line endings
    shellCommand('git commit -am "ppcheckout: Convert to UTF-8, native line endings"',cwd=projectPath)

    # Add title
    titleTxt = ".dt {}, by {}—A Project Gutenberg eBook\n".format(title,author)
    prependText(titleTxt, projectFilePath)
    shellCommand('git commit -am "ppcheckout: Add title"',cwd=projectPath)

    return

def prependText( s, fn ):
    with open(fn,'r') as f:
        t = f.read()
    with open(fn,'w') as f:
        f.write(s)
        f.write(t)

def shellCommand( s, cwd=None ):
    logging.info("Executing shell command: {}".format(s))
    cl = shlex.split(s)
    proc=subprocess.Popen(cl,cwd=cwd)
    proc.wait()
    if( proc.returncode != 0 ):
        logging.error("Command failed: {}".format(s))

def moveFiles( files, dest ):
    for f in files:
        logging.info("Moving file {} to {}".format(f,dest))
        shutil.move(f,dest)

def downloadFile( src, dest ):
    logging.info("Saving {} to {}".format(src,dest))
    try:
        urllib.request.urlretrieve(src,dest)
    except urllib.error.HTTPError as e:
        if e.code == 404:
            logging.error("Error downloading {}\nHTTP 404: Not Found {}".format(dest,src))
            pass
        else:
            raise

def generateProjectName( s ):
    name = re.sub("P[123]","",s)
    name = name.lower()
    name = re.sub(" +[&] +","-",name)
    name = re.sub("[^a-z0-9- ]","",name)
    name = re.sub("^(the |\w )","",name)
    name = re.sub("^ +","",name)
    name = re.sub(" {2,}","",name)
    name = re.sub(" ","-",name)
    name = re.sub("-{2,}","-",name)
    name = re.sub("(^-|-$)","",name)
    words = name.split("-")
    wc = len(words)
    maxNameLength = 30
    while( len(name) > maxNameLength and wc > 0 ):
        name = "-".join(words[0:min(wc,len(words))])
        wc = wc-1

    return name

if __name__ == "__main__":
    main()

#inputs
#projectID or project URL
#local project name (used for project folder name, naming of src file..)

#download
#good_words.txt (http://www.pgdp.net/projects/$PROJECTID/good_words.txt)
#bad_words.txt (http://www.pgdp.net/projects/$PROJECTID/bad_words.txt)
#zipped images (http://www.pgdp.net/c/tools/download_images.php?projectid=$PROJECTID&dummy=$PROJECTIDimages.zip
#zipped text (http://www.pgdp.net/projects/$PROJECTID/$PROJECTID.zip)

#hit checkout button
#Not sure about this.. cant accidentally check back in. User will need to visit project page for projectID anyways.

#create project skeleton
#$PROJECTID/
#    pngs/
#        001.png
#        ...
#    originals/
#        illustrations/
#            001.png
#            ...
#        good_words.txt
#        bad_words.txt
#        $PROJECTID.txt
#        $PROJECTID_comments.html
#    images/
#        001.jpg
#    good_words.txt
#    bad_words.txt
#    $PROJECTID-src.txt
#    Makefile
#    notes.txt

#create project folders
#copy word lists to originals/
#unzip text to originals/
#unzip images to illustrations/
#move .png page scans to ../pngs/ (how to differ these from illustrations reliably?)
#convert remaining images in illustrations/ to lossless format (like .png or .tif)
#- Unzip files
#   - Unzip all project files into project/originals
#   - Move files into appropriate directories
#- Copy proofed text from originals to project_root/projectname-src.txt
#- Update PROJECTNAME in Makefile
#- Set up version control for project
#   - git init
#   - git add projectname-src.txt
#   - git commit -m "Initial version"

# prepend title line to project text
# print .dt line for user to verify name is set correctly
#   - git commit -am "Add title"

<commit_msg>Use __version__<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ppcheckout

Usage:
  ppcheckout [options] <projectid> [<projectname>]
  ppcheckout -h | --help
  ppcheckout --version

Automates the initial setup of a pgdp.net post processing project.

Examples:
  ppcheckout projectID509579cf64be8
  ppcheckout projectID509579cf64be8 proj-name

Options:
  -q, --quiet           Print less text.
  -v, --verbose         Print more text.
  -h, --help            Show help.
  --version             Show version.
"""

from docopt import docopt
from robobrowser import RoboBrowser
import urllib
import logging
import re
import shutil
import os
import zipfile
import glob
import shlex
import subprocess

__version__="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

def main():
    args = docopt(__doc__, version="ppcheckout v{}".format(__version__))

    # Configure logging
    logLevel = logging.INFO #default
    if args['--verbose']:
        logLevel = logging.DEBUG
    elif args['--quiet']:
        logLevel = logging.ERROR

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
    logging.debug(args)

    # Process command line arguments
    projectId = args['<projectid>']
    projectName = args['<projectname>']
    projectId = re.search("(projectID[A-Fa-f0-9]+)",projectId).group(1)
    logging.info("Checking out project {} to {}".format(projectId,projectName))

    # Parse project page for title and author
    logging.info("Parsing project page for {}".format(projectId))
    projectURL = "http://www.pgdp.net/c/project.php?id={}".format(projectId)
    browser = RoboBrowser()
    browser.open(projectURL)
    projectInfo = browser.find('table',id='project_info_table')
    title = projectInfo.find('b',text="Title").parent.next_sibling.string
    author = projectInfo.find('b',text="Author").parent.next_sibling.string

    if not projectName:
        projectName = generateProjectName(title)

    # Download project files
    logging.info("Downloading project files for '{}' by {}".format(title,author))
    imagesURL = "http://www.pgdp.net/c/tools/download_images.php?projectid={}&dummy={}images.zip".format(projectId, projectId)
    textURL = "http://www.pgdp.net/projects/{}/{}.zip".format(projectId, projectId)
    downloadFile(imagesURL,"images.zip")
    downloadFile(textURL,"text.zip")

    # Init project skeleton
    logging.info("Building project base structure")
    srcDir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)),"_NEW_PROJECT_TEMPLATE"))
    dstDir = os.path.abspath(projectName)
    shutil.copytree(srcDir, dstDir)

    # Copy/unzip project files
    logging.info("Adding project files")
    zipText = zipfile.ZipFile("text.zip","r")
    zipImages = zipfile.ZipFile("images.zip","r")
    zipText.extractall(path=os.path.abspath("{}/originals/".format(projectName)))
    zipImages.extractall(path=os.path.abspath("{}/pngs/".format(projectName)))
    moveFiles(glob.glob(os.path.abspath("{}/pngs/*.jpg".format(projectName))), os.path.abspath("{}/originals/illustrations/".format(projectName)))
    shutil.copy(os.path.abspath("{}/originals/{}".format(projectName,"{}.txt".format(projectId))),os.path.abspath("{}/{}-src.txt".format(projectName,projectName)))
    moveFiles(glob.glob(os.path.abspath("*.zip")), os.path.abspath("{}/originals/".format(projectName)))

    # Convert illustrations
    files = glob.glob(os.path.abspath("{}/originals/illustrations/*.jpg".format(projectName)))
    logging.info("Converting illustrations to lossless format")
    for f in files:
        shellCommand("mogrify -format png {}".format(f))

    # Update Makefile with project name
    projectPath = os.path.abspath("{}/".format(projectName))
    projectFilePath = os.path.abspath("{}/{}-src.txt".format(projectName,projectName))
    prependText('PROJECTNAME={}'.format(projectName), os.path.abspath("{}/Makefile".format(projectName))

    # Initialize git repo
    shellCommand("git init",cwd=projectPath)
    shellCommand("git add {}-src.txt".format(projectName),cwd=projectPath)
    shellCommand('git commit -m "ppcheckout: Initial version"',cwd=projectPath)

    # Convert to UTF-8
    shellCommand("recode ISO-8859-1..UTF-8 {}".format(os.path.abspath("{}/{}-src.txt".format(projectName,projectName))))
    prependText('', projectFilePath) # convert to native line endings
    shellCommand('git commit -am "ppcheckout: Convert to UTF-8, native line endings"',cwd=projectPath)

    # Add title
    titleTxt = ".dt {}, by {}—A Project Gutenberg eBook\n".format(title,author)
    prependText(titleTxt, projectFilePath)
    shellCommand('git commit -am "ppcheckout: Add title"',cwd=projectPath)

    return

def prependText( s, fn ):
    with open(fn,'r') as f:
        t = f.read()
    with open(fn,'w') as f:
        f.write(s)
        f.write(t)

def shellCommand( s, cwd=None ):
    logging.info("Executing shell command: {}".format(s))
    cl = shlex.split(s)
    proc=subprocess.Popen(cl,cwd=cwd)
    proc.wait()
    if( proc.returncode != 0 ):
        logging.error("Command failed: {}".format(s))

def moveFiles( files, dest ):
    for f in files:
        logging.info("Moving file {} to {}".format(f,dest))
        shutil.move(f,dest)

def downloadFile( src, dest ):
    logging.info("Saving {} to {}".format(src,dest))
    try:
        urllib.request.urlretrieve(src,dest)
    except urllib.error.HTTPError as e:
        if e.code == 404:
            logging.error("Error downloading {}\nHTTP 404: Not Found {}".format(dest,src))
            pass
        else:
            raise

def generateProjectName( s ):
    name = re.sub("P[123]","",s)
    name = name.lower()
    name = re.sub(" +[&] +","-",name)
    name = re.sub("[^a-z0-9- ]","",name)
    name = re.sub("^(the |\w )","",name)
    name = re.sub("^ +","",name)
    name = re.sub(" {2,}","",name)
    name = re.sub(" ","-",name)
    name = re.sub("-{2,}","-",name)
    name = re.sub("(^-|-$)","",name)
    words = name.split("-")
    wc = len(words)
    maxNameLength = 30
    while( len(name) > maxNameLength and wc > 0 ):
        name = "-".join(words[0:min(wc,len(words))])
        wc = wc-1

    return name

if __name__ == "__main__":
    main()

#inputs
#projectID or project URL
#local project name (used for project folder name, naming of src file..)

#download
#good_words.txt (http://www.pgdp.net/projects/$PROJECTID/good_words.txt)
#bad_words.txt (http://www.pgdp.net/projects/$PROJECTID/bad_words.txt)
#zipped images (http://www.pgdp.net/c/tools/download_images.php?projectid=$PROJECTID&dummy=$PROJECTIDimages.zip
#zipped text (http://www.pgdp.net/projects/$PROJECTID/$PROJECTID.zip)

#hit checkout button
#Not sure about this.. cant accidentally check back in. User will need to visit project page for projectID anyways.

#create project skeleton
#$PROJECTID/
#    pngs/
#        001.png
#        ...
#    originals/
#        illustrations/
#            001.png
#            ...
#        good_words.txt
#        bad_words.txt
#        $PROJECTID.txt
#        $PROJECTID_comments.html
#    images/
#        001.jpg
#    good_words.txt
#    bad_words.txt
#    $PROJECTID-src.txt
#    Makefile
#    notes.txt

#create project folders
#copy word lists to originals/
#unzip text to originals/
#unzip images to illustrations/
#move .png page scans to ../pngs/ (how to differ these from illustrations reliably?)
#convert remaining images in illustrations/ to lossless format (like .png or .tif)
#- Unzip files
#   - Unzip all project files into project/originals
#   - Move files into appropriate directories
#- Copy proofed text from originals to project_root/projectname-src.txt
#- Update PROJECTNAME in Makefile
#- Set up version control for project
#   - git init
#   - git add projectname-src.txt
#   - git commit -m "Initial version"

# prepend title line to project text
# print .dt line for user to verify name is set correctly
#   - git commit -am "Add title"

<|endoftext|>


<commit_before>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
from types import MethodType, FunctionType

from robot.errors import DataError, FrameworkError
from robot import utils

if utils.is_jython:
    from javaargcoercer import ArgumentCoercer

class _KeywordArguments(object):
    _type = 'Keyword'

    def __init__(self, argument_source, kw_or_lib_name):
        self.names, self.defaults, self.varargs, self.minargs, self.maxargs \
            = self._determine_args(argument_source)
        self._name = kw_or_lib_name

    def resolve(self, args, variables=None):
        posargs, namedargs = self._get_argument_resolver().resolve(args, variables)
        self.check_arg_limits(posargs, namedargs)
        return posargs, namedargs

    def check_arg_limits(self, args, namedargs={}):
        arg_count = len(args) + len(namedargs)
        if not self.minargs <= arg_count <= self.maxargs:
            self._raise_inv_args(arg_count)
        return args

    def _raise_inv_args(self, arg_count):
        minend = utils.plural_or_not(self.minargs)
        if self.minargs == self.maxargs:
            exptxt = "%d argument%s" % (self.minargs, minend)
        elif self.maxargs != sys.maxint:
            exptxt = "%d to %d arguments" % (self.minargs, self.maxargs)
        else:
            exptxt = "at least %d argument%s" % (self.minargs, minend)
        raise DataError("%s '%s' expected %s, got %d."
                        % (self._type, self._name, exptxt, arg_count))

class PythonKeywordArguments(_KeywordArguments):

    def _get_argument_resolver(self):
        return PythonKeywordArgumentResolver(self)

    def _determine_args(self, handler_method):
        args, defaults, varargs = self._get_arg_spec(handler_method)
        minargs = len(args) - len(defaults)
        maxargs = varargs is not None and sys.maxint or len(args)
        return args, defaults, varargs, minargs, maxargs

    def _get_arg_spec(self, handler):
        """Returns info about args in a tuple (args, defaults, varargs)

        args     - tuple of all accepted arguments
        defaults - tuple of default values
        varargs  - name of the argument accepting varargs or None
        """
        # Code below is based on inspect module's getargs and getargspec
        # methods. See their documentation and/or source for more details.
        if type(handler) is MethodType:
            func = handler.im_func
            first_arg = 1        # this drops 'self' from methods' args
        elif type(handler) is FunctionType:
            func = handler
            first_arg = 0
        else:
            raise FrameworkError("Only MethodType and FunctionType accepted. "
                                 "Got '%s' instead." % type(handler))
        co = func.func_code
        nargs = co.co_argcount
        args = co.co_varnames[first_arg:nargs]
        defaults = func.func_defaults
        if defaults is None:
            defaults = ()
        if co.co_flags & 4:                      # 4 == CO_VARARGS
            varargs =  co.co_varnames[nargs]
        else:
            varargs = None
        return args, defaults, varargs

class JavaKeywordArguments(_KeywordArguments):

    def __init__(self, handler_method, name):
        _KeywordArguments.__init__(self, handler_method, name)
        self.arg_coercer = ArgumentCoercer(self._get_signatures(handler_method))

    def _get_argument_resolver(self):
        return JavaKeywordArgumentResolver(self)

    def _determine_args(self, handler_method):
        signatures = self._get_signatures(handler_method)
        minargs, maxargs = self._get_arg_limits(signatures)
        return [], [], None, minargs, maxargs

    def _get_signatures(self, handler):
        co = self._get_code_object(handler)
        return co.argslist[:co.nargs]

    def _get_code_object(self, handler):
        try:
            return handler.im_func
        except AttributeError:
            return handler

    def _get_arg_limits(self, signatures):
        if len(signatures) == 1:
            return self._get_single_sig_arg_limits(signatures[0])
        else:
            return self._get_multi_sig_arg_limits(signatures)

    def _get_single_sig_arg_limits(self, signature):
        args = signature.args
        if len(args) > 0 and args[-1].isArray():
            mina = len(args) - 1
            maxa = sys.maxint
        else:
            mina = maxa = len(args)
        return mina, maxa

    def _get_multi_sig_arg_limits(self, signatures):
        mina = maxa = None
        for sig in signatures:
            argc = len(sig.args)
            if mina is None or argc < mina:
                mina = argc
            if maxa is None or argc > maxa:
                maxa = argc
        return mina, maxa

class DynamicKeywordArguments(_KeywordArguments):

    def _get_argument_resolver(self):
        return PythonKeywordArgumentResolver(self)

    def _determine_args(self, argspec):
        args, defaults, varargs = self._get_arg_spec(argspec)
        minargs = len(args) - len(defaults)
        maxargs = varargs is not None and sys.maxint or len(args)
        return args, defaults, varargs, minargs, maxargs

    def _get_arg_spec(self, argspec):
        if argspec is None:
            return [], [], '<unknown>'
        try:
            if utils.is_str(argspec):
                raise TypeError
            return self._parse_arg_spec(list(argspec))
        except TypeError:
            raise TypeError('Argument specification should be list/array of Strings.')

    def _parse_arg_spec(self, argspec):
        if argspec == []:
            return [], [], None
        args = []
        defaults = []
        vararg = None
        for token in argspec:
            if vararg is not None:
                raise TypeError
            if token.startswith('*'):
                vararg = token[1:]
                continue
            if token.count('=') > 0:
                arg, default = token.split('=', 1)
                args.append(arg)
                defaults.append(default)
                continue
            if defaults:
                raise TypeError
            args.append(token)
        return args, defaults, vararg

class UserKeywordArguments(_KeywordArguments):

    def __init__(self, argnames, defaults, vararg, minargs, maxargs, name):
        self.names = list(argnames) # Python 2.5 does not support indexing tuples
        self.defaults = defaults
        self._vararg = vararg
        self.minargs = minargs
        self.maxargs = maxargs
        self._name = name

    def set_to(self, variables, arguments):
        template_with_defaults = self._template_for(variables)
        template, positional, named = self._fill(template_with_defaults, arguments)
        self._check_missing_args(template, len(arguments))
        self.check_arg_limits(template)
        self._set_variables(variables, template)
        return self._resolve_variables(positional, named, variables)

    def _resolve_variables(self, positional, named, variables):
        posargs = variables.replace_list(positional)
        for name, value in named.items():
            named[name] = variables.replace_scalar(value)
        return posargs, named

    def _check_missing_args(self, template, arg_count):
        for a in template:
            if isinstance(a, _MissingArg):
                self._raise_inv_args(arg_count)

    def _template_for(self, variables):
        return [ _MissingArg() for _ in range(len(self.names)-len(self.defaults)) ] \
                + list(self.defaults)

    def _set_variables(self, variables, arg_values):
        if self._vararg:
            variables[self._vararg] = variables.replace_list(self._get_varargs(arg_values))
            arg_values = arg_values[:len(self.names)]
        for name, value in zip(self.names, arg_values):
            variables[name] = variables.replace_scalar(value)

    def _fill(self, template, arguments):
        varargs = arguments[len(template):]
        arg_resolver = UserKeywordArgumentResolver(self)
        positional, named = arg_resolver.resolve(arguments[:len(template)])
        for name, value in named.items():
            template[self.names.index(name)] = value
        for index, value in enumerate(positional):
            template[index] = value
        return template + varargs, positional, named

    def _get_varargs(self, args):
        return args[len(self.names):]

class PythonInitArguments(PythonKeywordArguments):
    _type = 'Test Library'

class JavaInitArguments(JavaKeywordArguments):
    _type = 'Test Library'

    def resolve(self, args):
        self.check_arg_limits(args)
        return args, {}

class _MissingArg(object):
    def __getattr__(self, name):
        raise RuntimeError()

class _ArgumentResolver(object):

    def __init__(self, arguments):
        self._arguments = arguments
        self._mand_arg_count = len(arguments.names) - len(arguments.defaults)

    def resolve(self, values, variables=None):
        positional, named = self._resolve_argument_usage(values)
        return self._resolve_variables(positional, named, variables)

    def _resolve_argument_usage(self, values):
        positional = []
        named = {}
        named_args_allowed = True
        for arg in reversed(self._optional(values)):
            if named_args_allowed and self._is_named(arg):
                name, value = self._parse_named(arg)
                if name in named:
                    raise RuntimeError('Keyword argument %s repeated.' % name)
                named[name] = value
            else:
                positional.append(self._parse_positional(arg))
                named_args_allowed = False
        positional = self._mandatory(values) + list(reversed(positional))
        return positional, named

    def _optional(self, values):
        return values[self._mand_arg_count:]

    def _mandatory(self, values):
        return values[:self._mand_arg_count]

    def _is_named(self, arg):
        if self._is_str_with_kwarg_sep(arg):
            name, _ = self._split_from_kwarg_sep(arg)
            return self._is_arg_name(name)
        return False

    def _parse_named(self, arg):
        name, value = self._split_from_kwarg_sep(arg)
        return self._coerce(name), value

    def _is_str_with_kwarg_sep(self, arg):
        if not isinstance(arg, basestring):
            return False
        if not '=' in arg:
            return False
        return True

    def _split_from_kwarg_sep(self, arg):
        return arg.split('=', 1)

    def _parse_positional(self, argstr):
        if self._is_str_with_kwarg_sep(argstr):
            name, _ = self._split_from_kwarg_sep(argstr)
            if self._is_arg_name(name[:-1]):
                return argstr.replace('\\=', '=')
        return argstr

    def _is_arg_name(self, name):
        return self._arg_name(name) in self._arguments.names

    def _resolve_variables(self, posargs, kwargs, variables):
        posargs = self._replace_list(posargs, variables)
        for name, value in kwargs.items():
            kwargs[name] = self._replace_scalar(value, variables)
        return posargs, kwargs

    def _replace_list(self, values, variables):
        return variables.replace_list(values) if variables else values

    def _replace_scalar(self, value, variables):
        return variables.replace_scalar(value) if variables else value

class UserKeywordArgumentResolver(_ArgumentResolver):

    def _arg_name(self, name):
        return '${%s}' % name

    def _coerce(self, name):
        return '${%s}' % name

class PythonKeywordArgumentResolver(_ArgumentResolver):

    def _arg_name(self, name):
        return name

    def _coerce(self, name):
        return str(name)

class JavaKeywordArgumentResolver(object):

    def __init__(self, arguments):
        self._arguments = arguments
        self._minargs, self._maxargs = arguments.minargs, arguments.maxargs

    def resolve(self, values, variables):
        values = variables.replace_list(values)
        self._arguments.check_arg_limits(values)
        return self._handle_varargs_and_coerce(values)

    def _handle_varargs_and_coerce(self, args):
        if self._maxargs == sys.maxint:
            args = self._handle_varargs(args)
        return self._arguments.arg_coercer(args), {}

    def _handle_varargs(self, args):
        if len(args) == self._minargs:
            args.append([])
        elif len(args) == self._minargs + 1 and utils.is_list(args[-1]):
            pass
        else:
            varargs = args[self._minargs:]
            args = args[:self._minargs]
            args.append(varargs)
        return args
<commit_msg>almost got it working...<commit_after>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
from types import MethodType, FunctionType

from robot.errors import DataError, FrameworkError
from robot import utils

if utils.is_jython:
    from javaargcoercer import ArgumentCoercer

class _KeywordArguments(object):
    _type = 'Keyword'

    def __init__(self, argument_source, kw_or_lib_name):
        self.names, self.defaults, self.varargs, self.minargs, self.maxargs \
            = self._determine_args(argument_source)
        self._name = kw_or_lib_name

    def resolve(self, args, variables=None):
        posargs, namedargs = self._get_argument_resolver().resolve(args, variables)
        self.check_arg_limits(posargs, namedargs)
        return posargs, namedargs

    def check_arg_limits(self, args, namedargs={}):
        arg_count = len(args) + len(namedargs)
        if not self.minargs <= arg_count <= self.maxargs:
            self._raise_inv_args(arg_count)
        return args

    def _raise_inv_args(self, arg_count):
        minend = utils.plural_or_not(self.minargs)
        if self.minargs == self.maxargs:
            exptxt = "%d argument%s" % (self.minargs, minend)
        elif self.maxargs != sys.maxint:
            exptxt = "%d to %d arguments" % (self.minargs, self.maxargs)
        else:
            exptxt = "at least %d argument%s" % (self.minargs, minend)
        raise DataError("%s '%s' expected %s, got %d."
                        % (self._type, self._name, exptxt, arg_count))

class PythonKeywordArguments(_KeywordArguments):

    def _get_argument_resolver(self):
        return PythonKeywordArgumentResolver(self)

    def _determine_args(self, handler_method):
        args, defaults, varargs = self._get_arg_spec(handler_method)
        minargs = len(args) - len(defaults)
        maxargs = varargs is not None and sys.maxint or len(args)
        return args, defaults, varargs, minargs, maxargs

    def _get_arg_spec(self, handler):
        """Returns info about args in a tuple (args, defaults, varargs)

        args     - tuple of all accepted arguments
        defaults - tuple of default values
        varargs  - name of the argument accepting varargs or None
        """
        # Code below is based on inspect module's getargs and getargspec
        # methods. See their documentation and/or source for more details.
        if type(handler) is MethodType:
            func = handler.im_func
            first_arg = 1        # this drops 'self' from methods' args
        elif type(handler) is FunctionType:
            func = handler
            first_arg = 0
        else:
            raise FrameworkError("Only MethodType and FunctionType accepted. "
                                 "Got '%s' instead." % type(handler))
        co = func.func_code
        nargs = co.co_argcount
        args = co.co_varnames[first_arg:nargs]
        defaults = func.func_defaults
        if defaults is None:
            defaults = ()
        if co.co_flags & 4:                      # 4 == CO_VARARGS
            varargs =  co.co_varnames[nargs]
        else:
            varargs = None
        return args, defaults, varargs

class JavaKeywordArguments(_KeywordArguments):

    def __init__(self, handler_method, name):
        _KeywordArguments.__init__(self, handler_method, name)
        self.arg_coercer = ArgumentCoercer(self._get_signatures(handler_method))

    def _get_argument_resolver(self):
        return JavaKeywordArgumentResolver(self)

    def _determine_args(self, handler_method):
        signatures = self._get_signatures(handler_method)
        minargs, maxargs = self._get_arg_limits(signatures)
        return [], [], None, minargs, maxargs

    def _get_signatures(self, handler):
        co = self._get_code_object(handler)
        return co.argslist[:co.nargs]

    def _get_code_object(self, handler):
        try:
            return handler.im_func
        except AttributeError:
            return handler

    def _get_arg_limits(self, signatures):
        if len(signatures) == 1:
            return self._get_single_sig_arg_limits(signatures[0])
        else:
            return self._get_multi_sig_arg_limits(signatures)

    def _get_single_sig_arg_limits(self, signature):
        args = signature.args
        if len(args) > 0 and args[-1].isArray():
            mina = len(args) - 1
            maxa = sys.maxint
        else:
            mina = maxa = len(args)
        return mina, maxa

    def _get_multi_sig_arg_limits(self, signatures):
        mina = maxa = None
        for sig in signatures:
            argc = len(sig.args)
            if mina is None or argc < mina:
                mina = argc
            if maxa is None or argc > maxa:
                maxa = argc
        return mina, maxa

class DynamicKeywordArguments(_KeywordArguments):

    def _get_argument_resolver(self):
        return PythonKeywordArgumentResolver(self)

    def _determine_args(self, argspec):
        args, defaults, varargs = self._get_arg_spec(argspec)
        minargs = len(args) - len(defaults)
        maxargs = varargs is not None and sys.maxint or len(args)
        return args, defaults, varargs, minargs, maxargs

    def _get_arg_spec(self, argspec):
        if argspec is None:
            return [], [], '<unknown>'
        try:
            if utils.is_str(argspec):
                raise TypeError
            return self._parse_arg_spec(list(argspec))
        except TypeError:
            raise TypeError('Argument specification should be list/array of Strings.')

    def _parse_arg_spec(self, argspec):
        if argspec == []:
            return [], [], None
        args = []
        defaults = []
        vararg = None
        for token in argspec:
            if vararg is not None:
                raise TypeError
            if token.startswith('*'):
                vararg = token[1:]
                continue
            if token.count('=') > 0:
                arg, default = token.split('=', 1)
                args.append(arg)
                defaults.append(default)
                continue
            if defaults:
                raise TypeError
            args.append(token)
        return args, defaults, vararg

class UserKeywordArguments(_KeywordArguments):

    def __init__(self, argnames, defaults, vararg, minargs, maxargs, name):
        self.names = list(argnames) # Python 2.5 does not support indexing tuples
        self.defaults = defaults
        self._vararg = vararg
        self.minargs = minargs
        self.maxargs = maxargs
        self._name = name

    def set_to(self, variables, arguments):
        template_with_defaults = self._template_for(variables)
        template, positional, named = self._fill(template_with_defaults, arguments, variables)
        self._check_missing_args(template, len(arguments))
        self.check_arg_limits(template)
        self._set_variables(variables, template)
        return template, named

    def _check_missing_args(self, template, arg_count):
        for a in template:
            if isinstance(a, _MissingArg):
                self._raise_inv_args(arg_count)

    def _template_for(self, variables):
        return [ _MissingArg() for _ in range(len(self.names)-len(self.defaults)) ] \
                + variables.replace_list(list(self.defaults))

    def _set_variables(self, variables, arg_values):
        if self._vararg:
            variables[self._vararg] = self._get_varargs(arg_values)
            arg_values = arg_values[:len(self.names)]
        for name, value in zip(self.names, arg_values):
            variables[name] = value

    def _fill(self, template, arguments, variables):
        arg_resolver = UserKeywordArgumentResolver(self)
        positional, named = arg_resolver.resolve(arguments)
        positional = variables.replace_list(positional)
        varargs = positional[len(self.names):]
        positional = positional[:len(self.names)]
        for name, value in named.items():
            template[self.names.index(name)] = variables.replace_scalar(value)
        for index, value in enumerate(positional):
            template[index] = value
        return template + varargs, positional, named

    def _get_varargs(self, args):
        return args[len(self.names):]

class PythonInitArguments(PythonKeywordArguments):
    _type = 'Test Library'

class JavaInitArguments(JavaKeywordArguments):
    _type = 'Test Library'

    def resolve(self, args):
        self.check_arg_limits(args)
        return args, {}

class _MissingArg(object):
    def __getattr__(self, name):
        raise RuntimeError()

class _ArgumentResolver(object):

    def __init__(self, arguments):
        self._arguments = arguments
        self._mand_arg_count = len(arguments.names) - len(arguments.defaults)

    def resolve(self, values, variables=None):
        positional, named = self._resolve_argument_usage(values)
        return self._resolve_variables(positional, named, variables)

    def _resolve_argument_usage(self, values):
        positional = []
        named = {}
        named_args_allowed = True
        for arg in reversed(self._optional(values)):
            if named_args_allowed and self._is_named(arg):
                name, value = self._parse_named(arg)
                if name in named:
                    raise RuntimeError('Keyword argument %s repeated.' % name)
                named[name] = value
            else:
                positional.append(self._parse_positional(arg))
                named_args_allowed = False
        positional = self._mandatory(values) + list(reversed(positional))
        return positional, named

    def _optional(self, values):
        return values[self._mand_arg_count:]

    def _mandatory(self, values):
        return values[:self._mand_arg_count]

    def _is_named(self, arg):
        if self._is_str_with_kwarg_sep(arg):
            name, _ = self._split_from_kwarg_sep(arg)
            return self._is_arg_name(name)
        return False

    def _parse_named(self, arg):
        name, value = self._split_from_kwarg_sep(arg)
        return self._coerce(name), value

    def _is_str_with_kwarg_sep(self, arg):
        if not isinstance(arg, basestring):
            return False
        if not '=' in arg:
            return False
        return True

    def _split_from_kwarg_sep(self, arg):
        return arg.split('=', 1)

    def _parse_positional(self, argstr):
        if self._is_str_with_kwarg_sep(argstr):
            name, _ = self._split_from_kwarg_sep(argstr)
            if self._is_arg_name(name[:-1]):
                return argstr.replace('\\=', '=')
        return argstr

    def _is_arg_name(self, name):
        return self._arg_name(name) in self._arguments.names

    def _resolve_variables(self, posargs, kwargs, variables):
        posargs = self._replace_list(posargs, variables)
        for name, value in kwargs.items():
            kwargs[name] = self._replace_scalar(value, variables)
        return posargs, kwargs

    def _replace_list(self, values, variables):
        return variables.replace_list(values) if variables else values

    def _replace_scalar(self, value, variables):
        return variables.replace_scalar(value) if variables else value

class UserKeywordArgumentResolver(_ArgumentResolver):

    def _arg_name(self, name):
        return '${%s}' % name

    def _coerce(self, name):
        return '${%s}' % name

class PythonKeywordArgumentResolver(_ArgumentResolver):

    def _arg_name(self, name):
        return name

    def _coerce(self, name):
        return str(name)

class JavaKeywordArgumentResolver(object):

    def __init__(self, arguments):
        self._arguments = arguments
        self._minargs, self._maxargs = arguments.minargs, arguments.maxargs

    def resolve(self, values, variables):
        values = variables.replace_list(values)
        self._arguments.check_arg_limits(values)
        return self._handle_varargs_and_coerce(values)

    def _handle_varargs_and_coerce(self, args):
        if self._maxargs == sys.maxint:
            args = self._handle_varargs(args)
        return self._arguments.arg_coercer(args), {}

    def _handle_varargs(self, args):
        if len(args) == self._minargs:
            args.append([])
        elif len(args) == self._minargs + 1 and utils.is_list(args[-1]):
            pass
        else:
            varargs = args[self._minargs:]
            args = args[:self._minargs]
            args.append(varargs)
        return args
<|endoftext|>


<commit_before>import flask
import redis
import collections
import json
import numpy as np

app = flsk ask.Flask(__name__)
conn = redis.Redis()

def buildHistogram():
    keys = conn.keys()
    values = conn.mget(keys)
    c = collections.Counter(values)
    z = sum(c.values())
    return {k:v/float(z) for k,v in c.items()}

@app.route("/")
def histogram():
    h = buildHistogram()
    return json.dumps(h)

@app.route("/entropy")
def entropy():
    h = buildHistogram()
    return -sum([p*np.log(p) for p in h.values()])

@app.route("/probability")
def probability():
    city = request.args.get('city', '')
    ref = request.args.get('referrer', '')
    d = conn.hget(city,ref)
    return d

if __name__ == "__main__":
    app.debug = True
    app.run()
<commit_msg>Update city-bot-api.py for a spelling error<commit_after>import flask
import redis
import collections
import json
import numpy as np

app = flask.Flask(__name__)
conn = redis.Redis()

def buildHistogram():
    keys = conn.keys()
    values = conn.mget(keys)
    c = collections.Counter(values)
    z = sum(c.values())
    return {k:v/float(z) for k,v in c.items()}

@app.route("/")
def histogram():
    h = buildHistogram()
    return json.dumps(h)

@app.route("/entropy")
def entropy():
    h = buildHistogram()
    return -sum([p*np.log(p) for p in h.values()])

@app.route("/probability")
def probability():
    city = request.args.get('city', '')
    ref = request.args.get('referrer', '')
    d = conn.hget(city,ref)
    return d

if __name__ == "__main__":
    app.debug = True
    app.run()
<|endoftext|>


<commit_before># Copyright (C) 2009 Umea University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A plugin that allows you to use SAML2 SSO as authentication
and SAML2 attribute aggregations as metadata collector in your
WSGI application.

"""
import cgi
import os
import shelve

from paste.httpexceptions import HTTPTemporaryRedirect
from paste.httpexceptions import HTTPNotImplemented
from paste.httpexceptions import HTTPInternalServerError
from paste.request import parse_dict_querystring
from paste.request import construct_url

from zope.interface import implements

from repoze.who.interfaces import IChallenger, IIdentifier, IAuthenticator
from repoze.who.interfaces import IMetadataProvider
from repoze.who.plugins.form import FormPluginBase

from saml2 import BINDING_HTTP_REDIRECT
from saml2.client import Saml2Client
#from saml2.attribute_resolver import AttributeResolver
from saml2.config import SPConfig
from saml2.s_utils import sid
#from saml2.population import Population

def construct_came_from(environ):
    """ The URL that the user used when the process where interupted
    for single-sign-on processing. """

    came_from = environ.get("PATH_INFO")
    qstr = environ.get("QUERY_STRING","")
    if qstr:
        came_from += '?' + qstr
    return came_from

# FormPluginBase defines the methods remember and forget
def cgi_field_storage_to_dict( field_storage ):
    """Get a plain dictionary, rather than the '.value' system used by the
    cgi module."""

    params = {}
    for key in field_storage.keys():
        try:
            params[ key ] = field_storage[ key ].value
        except AttributeError:
            if isinstance(field_storage[ key ], basestring):
                params[key] = field_storage[key]

    return params

class SAML2Plugin(FormPluginBase):

    implements(IChallenger, IIdentifier, IAuthenticator, IMetadataProvider)

    def __init__(self, rememberer_name, config, saml_client,
                virtual_organization, wayf, _cache, debug, sid_store=None):
        FormPluginBase.__init__(self)

        self.rememberer_name = rememberer_name
        self.debug = debug
        self.wayf = wayf
        self.saml_client = saml_client

        self.conf = config
        self.srv = self.conf["service"]["sp"]
        self.log = None

        if virtual_organization:
            self.vorg = virtual_organization
            self.vorg_conf = None
#            try:
#                self.vorg_conf = self.conf[
#                                "virtual_organization"][virtual_organization]
#            except KeyError:
#                self.vorg = None
        else:
            self.vorg = None

        try:
            self.metadata = self.conf["metadata"]
        except KeyError:
            self.metadata = None
        if sid_store:
            self.outstanding_queries = shelve.open(sid_store, writeback=True)
        else:
            self.outstanding_queries = {}
        self.iam = os.uname()[1]

    def _pick_idp(self, environ, came_from):
        """
        If more than one idp and if none is selected, I have to do wayf or
        disco
        """

        idps = self.conf.idps()

        #self.log and self.log.info("IdP URL: %s" % idps)

        if len( idps ) == 1:
            idp_url = idps[0]["single_sign_on_service"][BINDING_HTTP_REDIRECT]
        elif len( idps ) == 0:
            return (1, HTTPInternalServerError(detail='Misconfiguration'))
        else:
            if self.wayf:
                wayf_selected = environ.get('s2repose.wayf_selected','')
                if not wayf_selected:
                    sid_ = sid()
                    self.outstanding_queries[sid_] = came_from
                    self.log.info("Redirect to WAYF function: %s" % self.wayf)
                    #self.log.info("env: %s" % (environ,))
                    return (1, HTTPTemporaryRedirect(headers = [('Location',
                                                "%s?%s" % (self.wayf, sid_))]))
                else:
                    self.log.info("Choosen IdP: '%s'" % wayf_selected)
                    try:
                        idp_url = self.srv["idp"][wayf_selected][
                                                    "single_sign_on_service"][
                                                        BINDING_HTTP_REDIRECT]
                    except KeyError:
                        return (1,
                            HTTPNotImplemented(
                                detail="Do not know how to talk to '%s'!" & (
                                                            wayf_selected,)))
            else:
                return (1, HTTPNotImplemented(detail='No WAYF present!'))

        return (0, idp_url)

    #### IChallenger ####
    def challenge(self, environ, _status, _app_headers, _forget_headers):

        # this challenge consist in loggin out
        if environ.has_key('rwpc.logout'):
            # ignore right now?
            pass

        self.log = environ.get('repoze.who.logger','')
        self.saml_client.log = self.log

        # Which page was accessed to get here
        came_from = construct_came_from(environ)
        environ["myapp.came_from"] = came_from
        if self.debug:
            self.log and self.log.info("[sp.challenge] RelayState >> %s" % came_from)

        # Am I part of a virtual organization ?
        try:
            vorg = environ["myapp.vo"]
        except KeyError:
            vorg = self.vorg
        self.log and self.log.info("[sp.challenge] VO: %s" % vorg)

        # If more than one idp and if none is selected, I have to do wayf
        (done, response) = self._pick_idp(environ, came_from)
        if done:
            return response
        else:
            idp_url = response
            # Do the AuthnRequest
            (sid_, result) = self.saml_client.authenticate(location=idp_url,
                                                    relay_state=came_from,
                                                    vorg=vorg)

            # remember the request
            self.outstanding_queries[sid_] = came_from

            if self.debug:
                self.log and self.log.info('sc returned: %s' % (result,))
            if isinstance(result, tuple):
                return HTTPTemporaryRedirect(headers=[result])
            else :
                return HTTPInternalServerError(detail='Incorrect returned data')

    def _get_post(self, environ):
        """
        Get the posted information

        :param environ: A dictionary
        """

        post = {}

        post_env = environ.copy()
        post_env['QUERY_STRING'] = ''

        length = int(environ["CONTENT_LENGTH"])
        try:
            body = environ["wsgi.input"].read(length)
        except Exception, excp:
            if self.debug and self.log:
                self.log.info("Exception while reading post: %s" % (excp,))
                raise

        from StringIO import StringIO
        environ['wsgi.input'] = StringIO(body)
        environ['s2repoze.body'] = body

        try:
            post = cgi.FieldStorage(
                fp=environ['wsgi.input'],
                environ=post_env,
                keep_blank_values=True
            )
        except Exception, excp:
            if self.debug and self.log:
                self.log.info("Exception (II): %s" % (excp,))
                raise

        if self.debug:
            self.log and self.log.info('identify post: %s' % (post,))

        return post

    def _construct_identity(self, session_info):
        identity = {}
        identity["login"] = session_info["name_id"]
        identity["password"] = ""
        identity['repoze.who.userid'] = session_info["name_id"]
        identity["user"] = session_info["ava"]
        if self.debug and self.log:
            self.log.info("Identity: %s" % identity)

        return identity

    def _eval_authn_response(self, environ, post):
        self.log and self.log.info("Got AuthN response, checking..")

        print "Outstanding: %s" % (self.outstanding_queries,)
        try:
            # Evaluate the response, returns a AuthnResponse instance
            try:
                authresp = self.saml_client.response(post,
                                                    self.conf["entityid"],
                                                    self.outstanding_queries,
                                                    self.log)
            except Exception, excp:
                self.log and self.log.error("Exception: %s" % (excp,))
                raise

            session_info = authresp.session_info()
        except TypeError, excp:
            self.log and self.log.error("Exception: %s" % (excp,))
            return None

        if session_info["came_from"]:
            if self.debug:
                self.log and self.log.info(
                            "came_from << %s" % session_info["came_from"])
            try:
                path, query = session_info["came_from"].split('?')
                environ["PATH_INFO"] = path
                environ["QUERY_STRING"] = query
            except ValueError:
                environ["PATH_INFO"] = session_info["came_from"]

        self.log and self.log.info("Session_info: %s" % session_info)
        return session_info

    #### IIdentifier ####
    def identify(self, environ):
        self.log = environ.get('repoze.who.logger', '')
        self.saml_client.log = self.log

        if "CONTENT_LENGTH" not in environ or not environ["CONTENT_LENGTH"]:
            if self.debug:
                self.log and self.log.info('[identify] get or empty post')
            return {}

        # if self.log:
        #     self.log.info("ENVIRON: %s" % environ)
        #     self.log.info("self: %s" % (self.__dict__,))

        uri = environ.get('REQUEST_URI', construct_url(environ))

        if self.debug:
            #self.log and self.log.info("environ.keys(): %s" % environ.keys())
            #self.log and self.log.info("Environment: %s" % environ)
            self.log and self.log.info('[sp.identify] uri: %s' % (uri,))

        query = parse_dict_querystring(environ)
        if self.debug:
            self.log and self.log.info('[sp.identify] query: %s' % (query,))

        post = self._get_post(environ)

        if self.debug:
            self.log and self.log.info('[sp.identify] post keys: %s' % (post.keys(),))

        # Not for me, put the post back where next in line can find it
        try:
            if not post.has_key("SAMLResponse"):
                self.log.info("[sp.identify] --- NOT SAMLResponse ---")
                environ["post.fieldstorage"] = post
                return {}
            else:
                self.log.info("[sp.identify] --- SAMLResponse ---")
        except TypeError, exc:
            self.log.error("[sp.identify] Exception: %s" % (exc,))
            environ["post.fieldstorage"] = post
            return {}

        # check for SAML2 authN response
        #if self.debug:
        try:
            session_info = self._eval_authn_response(environ,
                                            cgi_field_storage_to_dict(post))
        except Exception, exc:
            return None

        if session_info:
            environ["s2repoze.sessioninfo"] = session_info
            name_id = session_info["name_id"]
            # contruct and return the identity
            identity = {}
            identity["login"] = name_id
            identity["password"] = ""
            identity['repoze.who.userid'] = name_id
            identity["user"] = self.saml_client.users.get_identity(name_id)[0]

            self.log.info("[sp.identify] IDENTITY: %s" % (identity,))
            return identity
        else:
            return None

    # IMetadataProvider
    def add_metadata(self, environ, identity):
        """ Add information to the knowledge I have about the user """
        subject_id = identity['repoze.who.userid']

        self.log = environ.get('repoze.who.logger','')
        self.saml_client.log = self.log

        if self.debug and self.log:
            self.log.info(
                "[add_metadata] for %s" % subject_id)
            try:
                self.log.info(
                    "Issuers: %s" % self.saml_client.users.sources(subject_id))
            except KeyError:
                pass

        if "user" not in identity:
            identity["user"] = {}
        try:
            (ava, _) = self.saml_client.users.get_identity(subject_id)
            #now = time.gmtime()
            if self.debug:
                self.log and self.log.info("Adding %s" % ava)
            identity["user"].update(ava)
        except KeyError:
            pass

        if "pysaml2_vo_expanded" not in identity:
            # is this a Virtual Organization situation
            if self.vorg:
                if self.vorg.do_vo_aggregation(subject_id):
                    # Get the extended identity
                    identity["user"] = self.saml_client.users.get_identity(
                                                                subject_id)[0]
                    # Only do this once, mark that the identity has been
                    # expanded
                    identity["pysaml2_vo_expanded"] = 1

        if self.debug:
            self.log and self.log.info("[Adding] identity %s" % (identity,))

# @return
# used 2 times : one to get the ticket, the other to validate it
    def _service_url(self, environ, qstr=None):
        if qstr != None:
            url = construct_url(environ, querystring = qstr)
        else:
            url = construct_url(environ)
        return url

    #### IAuthenticatorPlugin ####
    def authenticate(self, _environ, identity=None):
        if identity:
            return identity.get('login', None)
        else:
            return None

def make_plugin(rememberer_name=None, # plugin for remember
                 cache= "", # cache
                 # Which virtual organization to support
                 virtual_organization="",
                 saml_conf="",
                 wayf="",
                 debug=0,
                 sid_store="",
                 identity_cache=""
                 ):

    if saml_conf is "":
        raise ValueError(
            'must include saml_conf in configuration')

    if rememberer_name is None:
        raise ValueError(
             'must include rememberer_name in configuration')

    config = SPConfig()
    config.load_file(saml_conf)

    scl = Saml2Client(config, identity_cache=identity_cache)

    plugin = SAML2Plugin(rememberer_name, config, scl,
                virtual_organization, wayf, cache, debug, sid_store)
    return plugin

# came_from = re.sub(r'ticket=[^&]*&?', '', came_from)

<commit_msg>More informative logging message<commit_after># Copyright (C) 2009 Umea University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A plugin that allows you to use SAML2 SSO as authentication
and SAML2 attribute aggregations as metadata collector in your
WSGI application.

"""
import cgi
import os
import shelve

from paste.httpexceptions import HTTPTemporaryRedirect
from paste.httpexceptions import HTTPNotImplemented
from paste.httpexceptions import HTTPInternalServerError
from paste.request import parse_dict_querystring
from paste.request import construct_url

from zope.interface import implements

from repoze.who.interfaces import IChallenger, IIdentifier, IAuthenticator
from repoze.who.interfaces import IMetadataProvider
from repoze.who.plugins.form import FormPluginBase

from saml2 import BINDING_HTTP_REDIRECT
from saml2.client import Saml2Client
#from saml2.attribute_resolver import AttributeResolver
from saml2.config import SPConfig
from saml2.s_utils import sid
#from saml2.population import Population

def construct_came_from(environ):
    """ The URL that the user used when the process where interupted
    for single-sign-on processing. """

    came_from = environ.get("PATH_INFO")
    qstr = environ.get("QUERY_STRING","")
    if qstr:
        came_from += '?' + qstr
    return came_from

# FormPluginBase defines the methods remember and forget
def cgi_field_storage_to_dict( field_storage ):
    """Get a plain dictionary, rather than the '.value' system used by the
    cgi module."""

    params = {}
    for key in field_storage.keys():
        try:
            params[ key ] = field_storage[ key ].value
        except AttributeError:
            if isinstance(field_storage[ key ], basestring):
                params[key] = field_storage[key]

    return params

class SAML2Plugin(FormPluginBase):

    implements(IChallenger, IIdentifier, IAuthenticator, IMetadataProvider)

    def __init__(self, rememberer_name, config, saml_client,
                virtual_organization, wayf, _cache, debug, sid_store=None):
        FormPluginBase.__init__(self)

        self.rememberer_name = rememberer_name
        self.debug = debug
        self.wayf = wayf
        self.saml_client = saml_client

        self.conf = config
        self.srv = self.conf["service"]["sp"]
        self.log = None

        if virtual_organization:
            self.vorg = virtual_organization
            self.vorg_conf = None
#            try:
#                self.vorg_conf = self.conf[
#                                "virtual_organization"][virtual_organization]
#            except KeyError:
#                self.vorg = None
        else:
            self.vorg = None

        try:
            self.metadata = self.conf["metadata"]
        except KeyError:
            self.metadata = None
        if sid_store:
            self.outstanding_queries = shelve.open(sid_store, writeback=True)
        else:
            self.outstanding_queries = {}
        self.iam = os.uname()[1]

    def _pick_idp(self, environ, came_from):
        """
        If more than one idp and if none is selected, I have to do wayf or
        disco
        """

        idps = self.conf.idps()

        #self.log and self.log.info("IdP URL: %s" % idps)

        if len( idps ) == 1:
            idp_url = idps[0]["single_sign_on_service"][BINDING_HTTP_REDIRECT]
        elif len( idps ) == 0:
            return (1, HTTPInternalServerError(detail='Misconfiguration'))
        else:
            if self.wayf:
                wayf_selected = environ.get('s2repose.wayf_selected','')
                if not wayf_selected:
                    sid_ = sid()
                    self.outstanding_queries[sid_] = came_from
                    self.log.info("Redirect to WAYF function: %s" % self.wayf)
                    #self.log.info("env: %s" % (environ,))
                    return (1, HTTPTemporaryRedirect(headers = [('Location',
                                                "%s?%s" % (self.wayf, sid_))]))
                else:
                    self.log.info("Choosen IdP: '%s'" % wayf_selected)
                    try:
                        idp_url = self.srv["idp"][wayf_selected][
                                                    "single_sign_on_service"][
                                                        BINDING_HTTP_REDIRECT]
                    except KeyError:
                        return (1,
                            HTTPNotImplemented(
                                detail="Do not know how to talk to '%s'!" & (
                                                            wayf_selected,)))
            else:
                return (1, HTTPNotImplemented(detail='No WAYF present!'))

        return (0, idp_url)

    #### IChallenger ####
    def challenge(self, environ, _status, _app_headers, _forget_headers):

        # this challenge consist in loggin out
        if environ.has_key('rwpc.logout'):
            # ignore right now?
            pass

        self.log = environ.get('repoze.who.logger','')
        self.saml_client.log = self.log

        # Which page was accessed to get here
        came_from = construct_came_from(environ)
        environ["myapp.came_from"] = came_from
        if self.debug:
            self.log and self.log.info("[sp.challenge] RelayState >> %s" % came_from)

        # Am I part of a virtual organization ?
        try:
            vorg = environ["myapp.vo"]
        except KeyError:
            vorg = self.vorg
        self.log and self.log.info("[sp.challenge] VO: %s" % vorg)

        # If more than one idp and if none is selected, I have to do wayf
        (done, response) = self._pick_idp(environ, came_from)
        if done:
            return response
        else:
            idp_url = response
            # Do the AuthnRequest
            (sid_, result) = self.saml_client.authenticate(location=idp_url,
                                                    relay_state=came_from,
                                                    vorg=vorg)

            # remember the request
            self.outstanding_queries[sid_] = came_from

            if self.debug:
                self.log and self.log.info('sc returned: %s' % (result,))
            if isinstance(result, tuple):
                return HTTPTemporaryRedirect(headers=[result])
            else :
                return HTTPInternalServerError(detail='Incorrect returned data')

    def _get_post(self, environ):
        """
        Get the posted information

        :param environ: A dictionary
        """

        post = {}

        post_env = environ.copy()
        post_env['QUERY_STRING'] = ''

        length = int(environ["CONTENT_LENGTH"])
        try:
            body = environ["wsgi.input"].read(length)
        except Exception, excp:
            if self.debug and self.log:
                self.log.info("Exception while reading post: %s" % (excp,))
                raise

        from StringIO import StringIO
        environ['wsgi.input'] = StringIO(body)
        environ['s2repoze.body'] = body

        try:
            post = cgi.FieldStorage(
                fp=environ['wsgi.input'],
                environ=post_env,
                keep_blank_values=True
            )
        except Exception, excp:
            if self.debug and self.log:
                self.log.info("Exception (II): %s" % (excp,))
                raise

        if self.debug:
            self.log and self.log.info('identify post: %s' % (post,))

        return post

    def _construct_identity(self, session_info):
        identity = {}
        identity["login"] = session_info["name_id"]
        identity["password"] = ""
        identity['repoze.who.userid'] = session_info["name_id"]
        identity["user"] = session_info["ava"]
        if self.debug and self.log:
            self.log.info("Identity: %s" % identity)

        return identity

    def _eval_authn_response(self, environ, post):
        self.log and self.log.info("Got AuthN response, checking..")

        print "Outstanding: %s" % (self.outstanding_queries,)
        try:
            # Evaluate the response, returns a AuthnResponse instance
            try:
                authresp = self.saml_client.response(post,
                                                    self.conf["entityid"],
                                                    self.outstanding_queries,
                                                    self.log)
            except Exception, excp:
                self.log and self.log.error("Exception: %s" % (excp,))
                raise

            session_info = authresp.session_info()
        except TypeError, excp:
            self.log and self.log.error("Exception: %s" % (excp,))
            return None

        if session_info["came_from"]:
            if self.debug:
                self.log and self.log.info(
                            "came_from << %s" % session_info["came_from"])
            try:
                path, query = session_info["came_from"].split('?')
                environ["PATH_INFO"] = path
                environ["QUERY_STRING"] = query
            except ValueError:
                environ["PATH_INFO"] = session_info["came_from"]

        self.log and self.log.info("Session_info: %s" % session_info)
        return session_info

    #### IIdentifier ####
    def identify(self, environ):
        self.log = environ.get('repoze.who.logger', '')
        self.saml_client.log = self.log

        if "CONTENT_LENGTH" not in environ or not environ["CONTENT_LENGTH"]:
            if self.debug:
                self.log and self.log.info('[identify] get or empty post')
            return {}

        # if self.log:
        #     self.log.info("ENVIRON: %s" % environ)
        #     self.log.info("self: %s" % (self.__dict__,))

        uri = environ.get('REQUEST_URI', construct_url(environ))

        if self.debug:
            #self.log and self.log.info("environ.keys(): %s" % environ.keys())
            #self.log and self.log.info("Environment: %s" % environ)
            self.log and self.log.info('[sp.identify] uri: %s' % (uri,))

        query = parse_dict_querystring(environ)
        if self.debug:
            self.log and self.log.info('[sp.identify] query: %s' % (query,))

        post = self._get_post(environ)

        if self.debug:
            self.log and self.log.info('[sp.identify] post keys: %s' % (post.keys(),))

        # Not for me, put the post back where next in line can find it
        try:
            if not post.has_key("SAMLResponse"):
                self.log.info("[sp.identify] --- NOT SAMLResponse ---")
                environ["post.fieldstorage"] = post
                return {}
            else:
                self.log.info("[sp.identify] --- SAMLResponse ---")
        except TypeError, exc:
            self.log.error("[sp.identify] Exception: %s" % (exc,))
            environ["post.fieldstorage"] = post
            return {}

        # check for SAML2 authN response
        #if self.debug:
        try:
            session_info = self._eval_authn_response(environ,
                                            cgi_field_storage_to_dict(post))
        except Exception, exc:
            return None

        if session_info:
            environ["s2repoze.sessioninfo"] = session_info
            name_id = session_info["name_id"]
            # contruct and return the identity
            identity = {}
            identity["login"] = name_id
            identity["password"] = ""
            identity['repoze.who.userid'] = name_id
            identity["user"] = self.saml_client.users.get_identity(name_id)[0]

            self.log.info("[sp.identify] IDENTITY: %s" % (identity,))
            return identity
        else:
            return None

    # IMetadataProvider
    def add_metadata(self, environ, identity):
        """ Add information to the knowledge I have about the user """
        subject_id = identity['repoze.who.userid']

        self.log = environ.get('repoze.who.logger','')
        self.saml_client.log = self.log

        if self.debug and self.log:
            self.log.info(
                "[add_metadata] for %s" % subject_id)
            try:
                self.log.info(
                    "Issuers: %s" % self.saml_client.users.sources(subject_id))
            except KeyError:
                pass

        if "user" not in identity:
            identity["user"] = {}
        try:
            (ava, _) = self.saml_client.users.get_identity(subject_id)
            #now = time.gmtime()
            if self.debug:
                self.log and self.log.info("[add_metadata] adds: %s" % ava)
            identity["user"].update(ava)
        except KeyError:
            pass

        if "pysaml2_vo_expanded" not in identity:
            # is this a Virtual Organization situation
            if self.vorg:
                if self.vorg.do_vo_aggregation(subject_id):
                    # Get the extended identity
                    identity["user"] = self.saml_client.users.get_identity(
                                                                subject_id)[0]
                    # Only do this once, mark that the identity has been
                    # expanded
                    identity["pysaml2_vo_expanded"] = 1

        if self.debug:
            self.log and self.log.info("[add_metadata] returns: %s" % (dict(identity),))

# @return
# used 2 times : one to get the ticket, the other to validate it
    def _service_url(self, environ, qstr=None):
        if qstr != None:
            url = construct_url(environ, querystring = qstr)
        else:
            url = construct_url(environ)
        return url

    #### IAuthenticatorPlugin ####
    def authenticate(self, _environ, identity=None):
        if identity:
            return identity.get('login', None)
        else:
            return None

def make_plugin(rememberer_name=None, # plugin for remember
                 cache= "", # cache
                 # Which virtual organization to support
                 virtual_organization="",
                 saml_conf="",
                 wayf="",
                 debug=0,
                 sid_store="",
                 identity_cache=""
                 ):

    if saml_conf is "":
        raise ValueError(
            'must include saml_conf in configuration')

    if rememberer_name is None:
        raise ValueError(
             'must include rememberer_name in configuration')

    config = SPConfig()
    config.load_file(saml_conf)

    scl = Saml2Client(config, identity_cache=identity_cache)

    plugin = SAML2Plugin(rememberer_name, config, scl,
                virtual_organization, wayf, cache, debug, sid_store)
    return plugin

# came_from = re.sub(r'ticket=[^&]*&?', '', came_from)

<|endoftext|>