<commit_before>from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame, PolygonAsset
from math import pi, cos, sin, atan2, sqrt
import time
SCREEN_WIDTH1 = 640
hit = 0
t=1
SCREEN_HEIGHT = 480
black = Color(0, 1)
speed = 10
speed1=4
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
swordlist = []
thinline = LineStyle(1, black)
white = Color(0xffffff, 1)
gray = Color(0x8c8c8c, 1)
noline = LineStyle(0, black)
thinline1 = LineStyle(1, white)
dead_asset = RectangleAsset(SCREEN_WIDTH1, SCREEN_HEIGHT, noline, red)
cf = PolygonAsset(((-10,-15),(10,-15),(10,15),(-10,15)), thinline, gray)
ms = PolygonAsset(((-7.5,-11.5),(7.5,-11.5),(7.5,11.5),(-7.5,11.5)), thinline, red)
smlSword = PolygonAsset(((-2.5,-5),(2.5,-5),(2.5,5),(-2.5,5)), thinline, red)
class MC(Sprite):
    def __init__(self, position, lives):
        super().__init__(cf, position)
        self.moves = speed1
        SpaceGame.listenKeyEvent("keydown", "d", self.dKey)
        SpaceGame.listenKeyEvent("keydown", "a", self.aKey)
        SpaceGame.listenKeyEvent("keydown", "s", self.sKey)
        SpaceGame.listenKeyEvent("keydown", "w", self.wKey)
        SpaceGame.listenKeyEvent("keydown", "q", self.qKey)
        SpaceGame.listenKeyEvent("keydown", "e", self.eKey)
        self.fxcenter = self.fycenter = 0.5
        self.lives = lives
        self.start=0
        self.end=0
        self.dead = 0
    def dKey(self, event):
        if self.moves > 0 and self.x +speed*cos(self.rotation)<SCREEN_WIDTH1 and self.y -speed*sin(self.rotation) >0:
            self.x += speed*cos(self.rotation)
            self.y -= speed*sin(self.rotation)
            self.moves -=1
    def aKey(self, event):
        if self.moves > 0 and self.x -speed*cos(self.rotation)>0 and self.y +speed*sin(self.rotation) <SCREEN_HEIGHT:
            self.x -= speed*cos(self.rotation)
            self.y += speed*sin(self.rotation)

            self.moves -=1
    def sKey(self, event):
        if self.moves > 0 and self.x +speed*cos((pi/2)-self.rotation)<SCREEN_WIDTH1 and self.y +speed*sin((pi/2)-self.rotation) <SCREEN_HEIGHT:
            self.x += speed*cos((pi/2)-self.rotation)
            self.y += speed*sin((pi/2)-self.rotation)
            self.moves -=1
    def wKey(self, event):
        if self.moves > 0 and self.x -speed*cos((pi/2)-self.rotation)>0 and self.y -speed*sin((pi/2)-self.rotation) >0:
           self.x -= speed*cos((pi/2)-self.rotation)
           self.y -= speed*sin((pi/2)-self.rotation)
           self.moves-=1
    def qKey(self, event):
        if self.rotation == 3*(pi/2):
            self.rotation = 0
        else:
            self.rotation+= pi/2
    def eKey(self, event):
        if self.rotation == -3*(pi/2):
            self.rotation = 0
        else:
            self.rotation-= pi/2
    def step(self):
        self.moves = speed1
    def hit(self):
        self.start = time.time()
        self.lives -=1
        self.dead = Sprite(dead_asset, (1,1))
        global t
        t=0
        self.end = self.start + .5
    def hit2(self):
        elapsed = time.time()
        if elapsed > self.end:
            self.dead.destroy()
            global t
            t =1

class meleeSprite(Sprite):
    def __init__(self, position):
        super().__init__(ms, position)
        self.fxcenter = self.fycenter = 0.5
    def step(self):
        self.rotation = pi/2-atan2((self.y-MC1.y), (self.x-MC1.x))
        self.x -= 10*cos(atan2(self.y-MC1.y, self.x-MC1.x))
        self.y -= 10*sin(atan2(self.y-MC1.y, self.x-MC1.x))
        if MC1.rotation == 0 or MC1.rotation == pi:
            if sqrt((self.x-MC1.x)**2+(self.y-MC1.y)**2) <= 20:
                swordlist.append(sword(((self.x-15*cos(atan2(self.y-MC1.y, self.x-MC1.x))), (self.y-15*sin(atan2(self.y-MC1.y, self.x-MC1.x)))), self.rotation))
                MC1.hit()
class sword(Sprite):
    def __init__(self, position, rotation):
        super().__init__(smlSword, position)
        self.fxcenter = self.fycenter = 0.5
        self.rotation = rotation
    def step(self):
        self.x = self.y = 10000

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, white)

        bg = Sprite(bg_asset, (0,0))

    def step(self):
        global swordlist
        if turn == 1:
            if len(swordlist)>0:
                for x in swordlist:
                    x.destroy()

            swordlist=[]
            meleeSprite1.step()

            MC1.step()

            global turn
            turn = 0
        if t == 0:
            MC1.hit2()

myapp = SpaceGame(SCREEN_WIDTH1, SCREEN_HEIGHT)

turn = 0
def turnProgress ():
    global turn
    turn=1
def spaceKey (event):

    turnProgress()

myapp.listenKeyEvent('keydown', 'space', spaceKey)
MC1=MC((320,240), 4)
meleeSprite1=meleeSprite((300,240))

myapp.run()<commit_msg>Updated from Brython Server: 4/25/2017 2:05:13 PM<commit_after>from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame, PolygonAsset
from math import pi, cos, sin, atan2, sqrt
import time
SCREEN_WIDTH1 = 640
hit = 0
t=1
SCREEN_HEIGHT = 480
black = Color(0, 1)
speed = 10
speed1=4
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
swordlist = []
thinline = LineStyle(1, black)
white = Color(0xffffff, 1)
gray = Color(0x8c8c8c, 1)
noline = LineStyle(0, black)
thinline1 = LineStyle(1, white)
dead_asset = RectangleAsset(SCREEN_WIDTH1, SCREEN_HEIGHT, noline, red)
cf = PolygonAsset(((-10,-15),(10,-15),(10,15),(-10,15)), thinline, gray)
ms = PolygonAsset(((-7.5,-11.5),(7.5,-11.5),(7.5,11.5),(-7.5,11.5)), thinline, red)
smlSword = PolygonAsset(((-2.5,-5),(2.5,-5),(2.5,5),(-2.5,5)), thinline, red)
class MC(Sprite):
    def __init__(self, position, lives):
        super().__init__(cf, position)
        self.moves = speed1
        SpaceGame.listenKeyEvent("keydown", "d", self.dKey)
        SpaceGame.listenKeyEvent("keydown", "a", self.aKey)
        SpaceGame.listenKeyEvent("keydown", "s", self.sKey)
        SpaceGame.listenKeyEvent("keydown", "w", self.wKey)
        SpaceGame.listenKeyEvent("keydown", "q", self.qKey)
        SpaceGame.listenKeyEvent("keydown", "e", self.eKey)
        self.fxcenter = self.fycenter = 0.5
        self.lives = lives
        self.start=0
        self.end=0
        self.dead = 0
    def dKey(self, event):
        if self.moves > 0 and self.x +speed*cos(self.rotation)<SCREEN_WIDTH1 and self.y -speed*sin(self.rotation) >0:
            self.x += speed*cos(self.rotation)
            self.y -= speed*sin(self.rotation)
            self.moves -=1
    def aKey(self, event):
        if self.moves > 0 and self.x -speed*cos(self.rotation)>0 and self.y +speed*sin(self.rotation) <SCREEN_HEIGHT:
            self.x -= speed*cos(self.rotation)
            self.y += speed*sin(self.rotation)

            self.moves -=1
    def sKey(self, event):
        if self.moves > 0 and self.x +speed*cos((pi/2)-self.rotation)<SCREEN_WIDTH1 and self.y +speed*sin((pi/2)-self.rotation) <SCREEN_HEIGHT:
            self.x += speed*cos((pi/2)-self.rotation)
            self.y += speed*sin((pi/2)-self.rotation)
            self.moves -=1
    def wKey(self, event):
        if self.moves > 0 and self.x -speed*cos((pi/2)-self.rotation)>0 and self.y -speed*sin((pi/2)-self.rotation) >0:
           self.x -= speed*cos((pi/2)-self.rotation)
           self.y -= speed*sin((pi/2)-self.rotation)
           self.moves-=1
    def qKey(self, event):
        if self.rotation == 3*(pi/2):
            self.rotation = 0
        else:
            self.rotation+= pi/2
    def eKey(self, event):
        if self.rotation == -3*(pi/2):
            self.rotation = 0
        else:
            self.rotation-= pi/2
    def step(self):
        self.moves = speed1
    def hit(self):
        self.start = time.time()
        self.lives -=1
        self.dead = Sprite(dead_asset, (1,1))
        global t
        t=0
        self.end = self.start + .5
    def hit2(self):
        elapsed = time.time()
        if elapsed > self.end:
            self.dead.destroy()
            global t
            t =1

class meleeSprite(Sprite):
    def __init__(self, position):
        super().__init__(ms, position)
        self.fxcenter = self.fycenter = 0.5
    def step(self):
        self.rotation = pi/2-atan2((self.y-MC1.y), (self.x-MC1.x))
        self.x -= 10*cos(atan2(self.y-MC1.y, self.x-MC1.x))
        self.y -= 10*sin(atan2(self.y-MC1.y, self.x-MC1.x))
        if MC1.rotation == 0 or MC1.rotation == pi:
            if sqrt((self.x-MC1.x)**2+(self.y-MC1.y)**2) <= 20:
                swordlist.append(sword(((self.x-15*cos(atan2(self.y-MC1.y, self.x-MC1.x))), (self.y-15*sin(atan2(self.y-MC1.y, self.x-MC1.x)))), self.rotation))
                MC1.hit()
class sword(Sprite):
    def __init__(self, position, rotation):
        super().__init__(smlSword, position)
        self.fxcenter = self.fycenter = 0.5
        self.rotation = rotation
    def step(self):
        self.x = self.y = 10000

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, .2)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)

        bg = Sprite(bg_asset, (0,0))

    def step(self):
        global swordlist
        if turn == 1:
            if len(swordlist)>0:
                for x in swordlist:
                    x.destroy()

            swordlist=[]
            meleeSprite1.step()

            MC1.step()

            global turn
            turn = 0
        if t == 0:
            MC1.hit2()

myapp = SpaceGame(SCREEN_WIDTH1, SCREEN_HEIGHT)

turn = 0
def turnProgress ():
    global turn
    turn=1
def spaceKey (event):

    turnProgress()

myapp.listenKeyEvent('keydown', 'space', spaceKey)
MC1=MC((320,240), 4)
meleeSprite1=meleeSprite((300,240))

myapp.run()<|endoftext|>


<commit_before>import httplib, socket, sys, threading, urlparse
import Pyro.core, Pyro.protocol

CONNECTION_TIMEOUT = 30
EMPTY_QUEUE_WAIT = 5
MAX_DEPTH = 10
STOP_CRAWLE = False

def runCrawle(argv, handler, auth=None):
    """The typical to start CRAWL-E"""
    try:
        rmi_url = argv[1].strip()
        threads = int(argv[2])
    except:
        sys.stderr.write("Usage: %s rmi_url threads\n" % argv[0])
        sys.exit(1)

    controller = Controller(handler=handler, auth=auth, RMI_URL=rmi_url,
                            numThreads=threads)
    controller.start()
    try:
        controller.join()
    except KeyboardInterrupt:
        controller.stop()

class Auth(object):
    """An _abstract_ class for handling authentication.
    Python doesn't have real abstract classes, but its cool

    The variable terminate is checked periodically to see if the threads
    which hold this class should shutdown.
    """

    terminate = False

    def getNextHeader(self, url):
        """Return the next header to use in the HTTP request.

	Additionally this function can be used to verify that the user is
	still logged in through the URL.
	"""
	raise NotImplementedError(' '.join(('Auth.getNextHeader must be',
					    'defined in a subclass')))

    def isValidBody(self, body):
        """Function called by to verify the body of the request is as expected.

	By default this function always returns True unless it is extended in
        a subclass.
	"""
	return True

    def stop(self):
        """Called when getNextURL returns none.

	This is useful to avoid sleeping and to allow threads to close
	cleanly. This function need not be extended by a subclass, however
	subclasses should check the value of terminate in getNextHeader.
	"""
	self.terminate = True

    def save(self):
        """This function should be extended in a subclass to handle
        termination.
        """
	sys.stderr.write("Auth.save() is not implemented\n")
        sys.stderr.flush()

class Handler(object):
    """An _abstract_ class for handling what urls to retrieve and how to
    parse and save them. The functions of this class need to be designed in
    such a way so that they are threadsafe as multiple threads will have
    access to the same instance.
    """

    def process(self, info, rmi):
        """Process is called after the request has been made. It needs to be
        implemented by a subclass.

        Keyword Arguments:
        info -- a dictionary containing:
                    status -- the returned HTTP status
                    body -- the page content
                    url -- the initial requested url
                    final_url -- the final url after redirects
                    headers -- the HTTP response headers
        rmi -- the PythonRMI object.
        """
        raise NotImplementedError(' '.join(('Handler.process must be defined',
                                            'in a subclass')))

    def isValidURL(self, url):
        """This function is called before processing a redirect. Implementing
        this function in a subclass can greatly improve performace.
        """
        return True

class HTTPConnectionQueue(object):
    """This class handles the queue of sockets for a particular address.

    Expand on this.
    """

    def __init__(self, address):
        """Constructs a HTTPConnectionQueue object.

        Keyword Arguments:
        address -- The address for which this object maps to.
        """
        self.address = address
        self.lock = threading.Lock()
        self.queue = []
        self.size = 0

    def closeConnections(self):
        self.lock.acquire()
        sys.stderr.write("Closing %d connections\n" % len(self.queue))
        sys.stderr.flush()
        for connection in self.queue:
            connection.close()
        self.lock.release()

    def getConnection(self):
        """Return a HTTPConnection object for the appropriate address.

        First try to return the object from the queue, however if the queue
        is empty create a new socket object to return.

        Dynamically add new field to HTTPConnection called requestCount to
        keep track of the number of requests made with the specific connection.
        """
        self.lock.acquire()
        if self.size is 0:
            self.lock.release()
            new = httplib.HTTPConnection(self.address)
            new.requestCount = 1
            return new
        connection = self.queue.pop(0)
        self.size -= 1
        self.lock.release()
        return connection

    def putSocket(self, connection):
        if STOP_CRAWLE:
            connection.close()
            return
        """Put the HTTPConenction object back on the queue."""
        self.lock.acquire()
        self.size += 1
        connection.requestCount += 1
        self.queue.append(connection)
        self.lock.release()

class HTTPConnectionControl(object):
    """This class handles HTTPConnectionQueues by storing a queue in a
    dictionary with the address as the index to the dictionary. Additionally
    this class handles resetting the connection when it reaches a specified
    request limit."""

    socket.setdefaulttimeout(CONNECTION_TIMEOUT)

    def __init__(self, handler, auth=None, requestLimit=-1, doRedirect=True):
        """Constructs the HTTPConnection Control object. These objects are to
        be shared between each thread.

        Keyword Arguments:
        handler -- The Handler class for checking if a url is valid.
        auth -- The Auth class for getting the next header (Default None).
        requestLimit -- The maximum number of requests a HTTP session can make.
                        When reached, the connection is closed and a new
                        connection is established in its place (Default -1).
        doRedirect -- If true, the redirected URL will automatically be placed
                      back on the queue. Otherwise the Handler process
                      function will have to handle a return with a HTTP
                      redirect status.
        """

        self.connectionQueues = {}
        self.lock = threading.Lock()
        self.handler = handler
        self.auth = auth
        self.requestLimit = requestLimit
        self.doRedirect = doRedirect

    def __resetConnection(self, connection, address):
        """Restart the connection."""
        connection.close()
        connection = httplib.HTTPConnection(address)
        connection.requestCount = 1

    def stop(self):
        """Used to indicate to stop processing requests"""
        [q.closeConnections() for q in self.connectionQueues.itervalues()]

    def request(self, url, depth):
        """Handles the request to the server.

        On success, return a dictionary containing:
        status, headers, body, url, final_url
        as described in handler	process.

        On failure the status values are negative and mean the following:
            -1.0 -- Response not ready
            -1.1 -- Bad status line
            -1.2 -- Socket error
            -1.3 -- Unhandled exception
            -2   -- Stop has been called
            -3   -- Redirect depth has been exceeded
            -4   -- Unsupported protocol (only HTTP currently supported)
            -5   -- Auth class returned None
            -6   -- gethostbyname failed
        """
        if STOP_CRAWLE:
            return {'status':-2, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        if depth > MAX_DEPTH:
            return {'status':-3, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        protocol, domain, request, _, parameters, _ = urlparse.urlparse(url)
        if parameters != '':
            request = '?'.join((request, parameters))

        try:
            address = socket.gethostbyname(domain)
        except socket.error:
            return {'status':-6, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        self.lock.acquire()
        try:
            connectionQueue = self.connectionQueues[address]
        except:
            connectionQueue = HTTPConnectionQueue(address)
            self.connectionQueues[address] = connectionQueue
        self.lock.release()
        connection = connectionQueue.getConnection()

        if connection.requestCount is self.requestLimit:
            self.__resetConnection(connection,address)

        if self.auth:
            headers = self.auth.getNextHeader(url)
            if headers is None:
                return {'status':-5, 'headers':'', 'body':'', 'url':url,
                        'final_url':url}
        else:
            headers = {}
        headers['Host'] = domain

        # This should be deleted at somepoint when https is handled
        # It needs to be here because it needs to be checked after
        # auth.getNextHeader in cases where reauthentication is
        # required.
        if protocol != 'http':
            return {'status':-4, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        # This needs to be checked here after we checked
        # our auth header, otherwise we might not know
        # when to log back in, or put the url back on the
        # queue
        if not self.handler.isValidURL(url):
            return None

        try:
            connection.request('GET', request, '', headers)
            response = connection.getresponse()
            body = response.read()
            connectionQueue.putSocket(connection)
        except httplib.ResponseNotReady:
            sys.stderr.write(' '.join(('A previous request did not call'
                                       'read(). This shouldn\'t happen\n')))
            sys.stderr.flush()
            connection.close()
            return {'status':-1.0, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        except httplib.BadStatusLine:
            connection.close()
            return {'status':-1.1, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        except socket.error:
            connection.close()
            return {'status':-1.2, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        except:
            sys.stderr.write('Unhandled exception -- FIXY TIME\n')
            sys.stderr.flush()
            connection.close()
            return {'status':-1.3, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        # Handle redirecting by first verifying the URL
        # and then making the request. Return the response.
        if self.doRedirect and response.status in (301,302):
            url1 = urlparse.urljoin(url, response.getheader('Location'))
            retryReturn = self.request(url1, depth + 1)
            if retryReturn == None:
                return None
            retryReturn['url'] = url
            return retryReturn

        if self.auth and not self.auth.isValidBody(body):
            return {'status':-5, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        toReturn = {}
        toReturn['status'] = response.status
        try: # This only works in Python 2.5
            toReturn['headers'] = response.getheaders()
        except:
            toReturn['headers'] = None
        toReturn['body'] = body
        toReturn['url'] = url
        toReturn['final_url'] = url
        return toReturn

class ControlThread(threading.Thread):
    """A single thread of control"""

    stop_wait_event = threading.Event()

    def __init__(self, connectionControl, handler, RMI_URL, auth=None):
        """Sets up the ControlThread.

        Keyword Arguments:
        connectionControl -- A HTTPConnectionControl object. This object is
                             shared amongst the threads
        handler -- The handler class for parsing the returned information
        RMI_URL -- The RMI initilization URL. Each thread needs to connect
                   separately to the RMI server.
        auth -- The optional auth class which can provide cookies, deal with
                rate limiting, etc.
        """
        threading.Thread.__init__(self)
        self.connectionControl = connectionControl
        self.handler = handler;
        self.auth = auth
        self.rmi = Pyro.core.getProxyForURI(RMI_URL)

    def run(self):
        """This is the execution order of a single thread.

        The threads will stop when STOP_CRAWLE becomes true, when the RMI
        server is shutdown, and when a returned url is None.
        """

        global STOP_CRAWLE
        while not STOP_CRAWLE:
            # Get the URL to retrieve
            try:
                url = self.rmi.get()
            except Pyro.protocol.ProtocolError:
                if not STOP_CRAWLE:
                    sys.stderr.write("Queue unreachable - stopping CRAWL-E\n")
                    sys.stderr.flush()
                    STOP_CRAWLE = True
                break

            if url is None:
                ControlThread.stop_wait_event.clear()
                ControlThread.stop_wait_event.wait(EMPTY_QUEUE_WAIT)
                if ControlThread.stop_wait_event.isSet():
                    continue

                if self.auth:
                    self.auth.stop()
                if not STOP_CRAWLE:
                    sys.stderr.write("Queue empty - stopping CRAWL-E\n")
                    sys.stderr.flush()
                    STOP_CRAWLE = True
                break

            response = self.connectionControl.request(url, 0)
            if response:
                self.handler.process(response, self.rmi)

            # Now release waiting threads
            ControlThread.stop_wait_event.set()

class Controller(object):
    """The primary controller manages all the threads."""

    def __init__(self, handler, RMI_URL, auth=None, numThreads=1,
                 requestLimit=-1, doRedirect=True):
        """Create the controller object

        Keyword Arguments:
        handler -- The Handler class each thread will use for processing
        RMI_URL -- The url to the RMI server
        auth -- The optional Auth class (Default None)
        numThreads -- The number of threads to spwan (Default 1)
        requestLimit -- See HTTPConnectionControl
        doRedirect -- If true redirects will transparently be handled
        """
        self.threads = []
        self.connection_ctrl = HTTPConnectionControl(auth=auth,
                                                     handler=handler,
                                                     requestLimit=requestLimit,
                                                     doRedirect=doRedirect)
        self.auth = auth
        self.handler = handler
        self.already_stopped = False

        for x in range(numThreads):
            thread = ControlThread(handler=handler,
                                   RMI_URL=RMI_URL, auth=auth,
                                   connectionControl = self.connection_ctrl)
            self.threads.append(thread)

    def start(self):
        """Starts all threads"""
        for thread in self.threads:
            thread.start()

    def join(self):
        """Join on all threads"""
        count = 0
        for thread in self.threads:
            while 1:
                thread.join(1)
                if not thread.isAlive():
                    break
            count += 1
            sys.stderr.write("%d threads closed\r" % count)
            sys.stderr.flush()
        sys.stderr.write("                        \n")
        sys.stderr.flush()

    def stop(self):
        """Stops all threads gracefully"""
        global STOP_CRAWLE
        STOP_CRAWLE = True
        sys.stderr.write("Stop received\n")
        sys.stderr.flush()
        self.connection_ctrl.stop()
        self.join()
        if self.auth:
            self.auth.save()

class BrowserAuth(Auth):
    """A simple Auth class which sets the HTTP USER-AGENT"""

    _USER_AGENT = ' '.join(('Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-0;',
                            'en-US; rv:1.8.1.7) Gecko/20070914',
                            'Firefox/2.0.0.7'))

    def __init__(self, user_agent=None):
        """Constructs the BrowserAuth class

        Keyword Arguments:
        user_agent -- The desired user_agent string to use.
        """

        if user_agent is None:
            self.headers = {'USER-AGENT':self._USER_AGENT}
        else:
            self.headers = {'USER-AGENT':user_agent}

    def getNextHeader(self,url):
        """Return the header containing the USER-AGENT field"""
        return self.headers

    def save(self):
        """Required, does nothing"""
        pass

class VisitURLHandler(Handler):
    """Very simple example handler which simply visits the page.

    This handler just demonstrates how to interact with the queue.
    """

    def process(self, info, rmi):
        if info['status'] != 200:
            print "putting %s back on queue" % info['url']
            rmi.put(info['url'])

if __name__ == '__main__':
    """Basic example of how to start CRAWL-E. The assumption is that
    a queue is running and the url for it is known."""
    runCrawle(sys.argv, handler=VisitURLHandler())
<commit_msg>CRAWL-E now supports ports other than port 80 which I apparently previously neglected.<commit_after>import httplib, socket, sys, threading, urlparse
import Pyro.core, Pyro.protocol

CONNECTION_TIMEOUT = 30
EMPTY_QUEUE_WAIT = 5
MAX_DEPTH = 10
STOP_CRAWLE = False

def runCrawle(argv, handler, auth=None):
    """The typical to start CRAWL-E"""
    try:
        rmi_url = argv[1].strip()
        threads = int(argv[2])
    except:
        sys.stderr.write("Usage: %s rmi_url threads\n" % argv[0])
        sys.exit(1)

    controller = Controller(handler=handler, auth=auth, RMI_URL=rmi_url,
                            numThreads=threads)
    controller.start()
    try:
        controller.join()
    except KeyboardInterrupt:
        controller.stop()

class Auth(object):
    """An _abstract_ class for handling authentication.
    Python doesn't have real abstract classes, but its cool

    The variable terminate is checked periodically to see if the threads
    which hold this class should shutdown.
    """

    terminate = False

    def getNextHeader(self, url):
        """Return the next header to use in the HTTP request.

	Additionally this function can be used to verify that the user is
	still logged in through the URL.
	"""
	raise NotImplementedError(' '.join(('Auth.getNextHeader must be',
					    'defined in a subclass')))

    def isValidBody(self, body):
        """Function called by to verify the body of the request is as expected.

	By default this function always returns True unless it is extended in
        a subclass.
	"""
	return True

    def stop(self):
        """Called when getNextURL returns none.

	This is useful to avoid sleeping and to allow threads to close
	cleanly. This function need not be extended by a subclass, however
	subclasses should check the value of terminate in getNextHeader.
	"""
	self.terminate = True

    def save(self):
        """This function should be extended in a subclass to handle
        termination.
        """
	sys.stderr.write("Auth.save() is not implemented\n")
        sys.stderr.flush()

class Handler(object):
    """An _abstract_ class for handling what urls to retrieve and how to
    parse and save them. The functions of this class need to be designed in
    such a way so that they are threadsafe as multiple threads will have
    access to the same instance.
    """

    def process(self, info, rmi):
        """Process is called after the request has been made. It needs to be
        implemented by a subclass.

        Keyword Arguments:
        info -- a dictionary containing:
                    status -- the returned HTTP status
                    body -- the page content
                    url -- the initial requested url
                    final_url -- the final url after redirects
                    headers -- the HTTP response headers
        rmi -- the PythonRMI object.
        """
        raise NotImplementedError(' '.join(('Handler.process must be defined',
                                            'in a subclass')))

    def isValidURL(self, url):
        """This function is called before processing a redirect. Implementing
        this function in a subclass can greatly improve performace.
        """
        return True

class HTTPConnectionQueue(object):
    """This class handles the queue of sockets for a particular address.

    Expand on this.
    """

    def __init__(self, address):
        """Constructs a HTTPConnectionQueue object.

        Keyword Arguments:
        address -- The address for which this object maps to.
        """
        self.address = address
        self.lock = threading.Lock()
        self.queue = []
        self.size = 0

    def closeConnections(self):
        self.lock.acquire()
        sys.stderr.write("Closing %d connections\n" % len(self.queue))
        sys.stderr.flush()
        for connection in self.queue:
            connection.close()
        self.lock.release()

    def getConnection(self):
        """Return a HTTPConnection object for the appropriate address.

        First try to return the object from the queue, however if the queue
        is empty create a new socket object to return.

        Dynamically add new field to HTTPConnection called requestCount to
        keep track of the number of requests made with the specific connection.
        """
        self.lock.acquire()
        if self.size is 0:
            self.lock.release()
            new = httplib.HTTPConnection(*self.address)
            new.requestCount = 1
            return new
        connection = self.queue.pop(0)
        self.size -= 1
        self.lock.release()
        return connection

    def putSocket(self, connection):
        if STOP_CRAWLE:
            connection.close()
            return
        """Put the HTTPConenction object back on the queue."""
        self.lock.acquire()
        self.size += 1
        connection.requestCount += 1
        self.queue.append(connection)
        self.lock.release()

class HTTPConnectionControl(object):
    """This class handles HTTPConnectionQueues by storing a queue in a
    dictionary with the address as the index to the dictionary. Additionally
    this class handles resetting the connection when it reaches a specified
    request limit."""

    socket.setdefaulttimeout(CONNECTION_TIMEOUT)

    def __init__(self, handler, auth=None, requestLimit=-1, doRedirect=True):
        """Constructs the HTTPConnection Control object. These objects are to
        be shared between each thread.

        Keyword Arguments:
        handler -- The Handler class for checking if a url is valid.
        auth -- The Auth class for getting the next header (Default None).
        requestLimit -- The maximum number of requests a HTTP session can make.
                        When reached, the connection is closed and a new
                        connection is established in its place (Default -1).
        doRedirect -- If true, the redirected URL will automatically be placed
                      back on the queue. Otherwise the Handler process
                      function will have to handle a return with a HTTP
                      redirect status.
        """

        self.connectionQueues = {}
        self.lock = threading.Lock()
        self.handler = handler
        self.auth = auth
        self.requestLimit = requestLimit
        self.doRedirect = doRedirect

    def __resetConnection(self, connection, address):
        """Restart the connection."""
        connection.close()
        connection = httplib.HTTPConnection(address)
        connection.requestCount = 1

    def stop(self):
        """Used to indicate to stop processing requests"""
        [q.closeConnections() for q in self.connectionQueues.itervalues()]

    def request(self, url, depth):
        """Handles the request to the server.

        On success, return a dictionary containing:
        status, headers, body, url, final_url
        as described in handler	process.

        On failure the status values are negative and mean the following:
            -1.0 -- Response not ready
            -1.1 -- Bad status line
            -1.2 -- Socket error
            -1.3 -- Unhandled exception
            -2   -- Stop has been called
            -3   -- Redirect depth has been exceeded
            -4   -- Unsupported scheme (only HTTP currently supported)
            -5   -- Auth class returned None
            -6   -- gethostbyname failed
        """
        if STOP_CRAWLE:
            return {'status':-2, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        if depth > MAX_DEPTH:
            return {'status':-3, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        u = urlparse.urlparse(url)
        request = urlparse.urlunparse(('', '', u.path, u.params, u.query, ''))

        try:
            address = (socket.gethostbyname(u.hostname), u.port)
        except socket.error:
            return {'status':-6, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        self.lock.acquire()
        try:
            connectionQueue = self.connectionQueues[address]
        except:
            connectionQueue = HTTPConnectionQueue(address)
            self.connectionQueues[address] = connectionQueue
        self.lock.release()
        connection = connectionQueue.getConnection()

        if connection.requestCount is self.requestLimit:
            self.__resetConnection(connection, address)

        if self.auth:
            headers = self.auth.getNextHeader(url)
            if headers is None:
                return {'status':-5, 'headers':'', 'body':'', 'url':url,
                        'final_url':url}
        else:
            headers = {}
        headers['Host'] = u.hostname

        # This should be deleted at somepoint when https is handled
        # It needs to be here because it needs to be checked after
        # auth.getNextHeader in cases where reauthentication is
        # required.
        if u.scheme != 'http':
            return {'status':-4, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        # This needs to be checked here after we checked
        # our auth header, otherwise we might not know
        # when to log back in, or put the url back on the
        # queue
        if not self.handler.isValidURL(url):
            return None

        try:
            connection.request('GET', request, '', headers)
            response = connection.getresponse()
            body = response.read()
            connectionQueue.putSocket(connection)
        except httplib.ResponseNotReady:
            sys.stderr.write(' '.join(('A previous request did not call'
                                       'read(). This shouldn\'t happen\n')))
            sys.stderr.flush()
            connection.close()
            return {'status':-1.0, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        except httplib.BadStatusLine:
            connection.close()
            return {'status':-1.1, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}
        except socket.error, e:
            connection.close()
            return {'status':-1.2, 'headers':'', 'body':'', 'url':url,
                    'final_url':url, 'extra':e}
        except:
            sys.stderr.write('Unhandled exception -- FIXY TIME\n')
            sys.stderr.flush()
            connection.close()
            return {'status':-1.3, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        # Handle redirecting by first verifying the URL
        # and then making the request. Return the response.
        if self.doRedirect and response.status in (301,302):
            url1 = urlparse.urljoin(url, response.getheader('Location'))
            retryReturn = self.request(url1, depth + 1)
            if retryReturn == None:
                return None
            retryReturn['url'] = url
            return retryReturn

        if self.auth and not self.auth.isValidBody(body):
            return {'status':-5, 'headers':'', 'body':'', 'url':url,
                    'final_url':url}

        toReturn = {}
        toReturn['status'] = response.status
        try: # This only works in Python 2.5
            toReturn['headers'] = response.getheaders()
        except:
            toReturn['headers'] = None
        toReturn['body'] = body
        toReturn['url'] = url
        toReturn['final_url'] = url
        return toReturn

class ControlThread(threading.Thread):
    """A single thread of control"""

    stop_wait_event = threading.Event()

    def __init__(self, connectionControl, handler, RMI_URL, auth=None):
        """Sets up the ControlThread.

        Keyword Arguments:
        connectionControl -- A HTTPConnectionControl object. This object is
                             shared amongst the threads
        handler -- The handler class for parsing the returned information
        RMI_URL -- The RMI initilization URL. Each thread needs to connect
                   separately to the RMI server.
        auth -- The optional auth class which can provide cookies, deal with
                rate limiting, etc.
        """
        threading.Thread.__init__(self)
        self.connectionControl = connectionControl
        self.handler = handler;
        self.auth = auth
        self.rmi = Pyro.core.getProxyForURI(RMI_URL)

    def run(self):
        """This is the execution order of a single thread.

        The threads will stop when STOP_CRAWLE becomes true, when the RMI
        server is shutdown, and when a returned url is None.
        """

        global STOP_CRAWLE
        while not STOP_CRAWLE:
            # Get the URL to retrieve
            try:
                url = self.rmi.get()
            except Pyro.protocol.ProtocolError:
                if not STOP_CRAWLE:
                    sys.stderr.write("Queue unreachable - stopping CRAWL-E\n")
                    sys.stderr.flush()
                    STOP_CRAWLE = True
                break

            if url is None:
                ControlThread.stop_wait_event.clear()
                ControlThread.stop_wait_event.wait(EMPTY_QUEUE_WAIT)
                if ControlThread.stop_wait_event.isSet():
                    continue

                if self.auth:
                    self.auth.stop()
                if not STOP_CRAWLE:
                    sys.stderr.write("Queue empty - stopping CRAWL-E\n")
                    sys.stderr.flush()
                    STOP_CRAWLE = True
                break

            response = self.connectionControl.request(url, 0)
            if response:
                self.handler.process(response, self.rmi)

            # Now release waiting threads
            ControlThread.stop_wait_event.set()

class Controller(object):
    """The primary controller manages all the threads."""

    def __init__(self, handler, RMI_URL, auth=None, numThreads=1,
                 requestLimit=-1, doRedirect=True):
        """Create the controller object

        Keyword Arguments:
        handler -- The Handler class each thread will use for processing
        RMI_URL -- The url to the RMI server
        auth -- The optional Auth class (Default None)
        numThreads -- The number of threads to spwan (Default 1)
        requestLimit -- See HTTPConnectionControl
        doRedirect -- If true redirects will transparently be handled
        """
        self.threads = []
        self.connection_ctrl = HTTPConnectionControl(auth=auth,
                                                     handler=handler,
                                                     requestLimit=requestLimit,
                                                     doRedirect=doRedirect)
        self.auth = auth
        self.handler = handler
        self.already_stopped = False

        for x in range(numThreads):
            thread = ControlThread(handler=handler,
                                   RMI_URL=RMI_URL, auth=auth,
                                   connectionControl = self.connection_ctrl)
            self.threads.append(thread)

    def start(self):
        """Starts all threads"""
        for thread in self.threads:
            thread.start()

    def join(self):
        """Join on all threads"""
        count = 0
        for thread in self.threads:
            while 1:
                thread.join(1)
                if not thread.isAlive():
                    break
            count += 1
            sys.stderr.write("%d threads closed\r" % count)
            sys.stderr.flush()
        sys.stderr.write("                        \n")
        sys.stderr.flush()

    def stop(self):
        """Stops all threads gracefully"""
        global STOP_CRAWLE
        STOP_CRAWLE = True
        sys.stderr.write("Stop received\n")
        sys.stderr.flush()
        self.connection_ctrl.stop()
        self.join()
        if self.auth:
            self.auth.save()

class BrowserAuth(Auth):
    """A simple Auth class which sets the HTTP USER-AGENT"""

    _USER_AGENT = ' '.join(('Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-0;',
                            'en-US; rv:1.8.1.7) Gecko/20070914',
                            'Firefox/2.0.0.7'))

    def __init__(self, user_agent=None):
        """Constructs the BrowserAuth class

        Keyword Arguments:
        user_agent -- The desired user_agent string to use.
        """

        if user_agent is None:
            self.headers = {'USER-AGENT':self._USER_AGENT}
        else:
            self.headers = {'USER-AGENT':user_agent}

    def getNextHeader(self,url):
        """Return the header containing the USER-AGENT field"""
        return self.headers

    def save(self):
        """Required, does nothing"""
        pass

class VisitURLHandler(Handler):
    """Very simple example handler which simply visits the page.

    This handler just demonstrates how to interact with the queue.
    """

    def process(self, info, rmi):
        if info['status'] != 200:
            print "putting %s back on queue" % info['url']
            rmi.put(info['url'])

if __name__ == '__main__':
    """Basic example of how to start CRAWL-E. The assumption is that
    a queue is running and the url for it is known."""
    runCrawle(sys.argv, handler=VisitURLHandler())
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import os.path
import sys
import subprocess

OUTPUT_DIR = '/Users/matt/Desktop/mp3/'

def main():
    path = os.getcwd()
    filenames = [
        filename
        for filename
        in os.listdir(path)
        if filename.endswith('.m4a')
        ]

    for filename in filenames:
        subprocess.call([
            "ffmpeg", "-i",
            os.path.join(path, filename),
            "-acodec", "libmp3lame", "-ab", "256k",
            os.path.join(OUTPUT_DIR, '%s.mp3' % filename[:-4])
            ])

    return 0

if __name__ == '__main__':
    status = main()
    sys.exit(status)<commit_msg>add flat file input<commit_after>#!/usr/bin/env python

import os
import os.path
import sys
import subprocess

#OUTPUT_DIR = '/Users/matt/Desktop/mp3/'
OUTPUT_DIR = 'c:/test/'

def covert_and_save(path):
     filenames = [
        filename
        for filename
        in os.listdir(path)
        if filename.endswith('.m4a')
        ]

    for filename in filenames:
        source = os.path.join(path, filename)
        destination = os.path.join(OUTPUT_DIR, '%s.mp3' % filename[:-4])

        subprocess.call([
            "ffmpeg", "-i",
            source,
            "-acodec", "libmp3lame", "-ab", "256k",
            destination)
            ])

def flat_file(text_file):
    for dir in text_file:
        convert_and_save(dir)

    return 0

def main():
    path = os.getcwd()
    convert_and_save(path)

    return 0

if __name__ == '__main__':
    status = main()
    sys.exit(status)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

y_pos_dist = norm(500, 10)

cluster_x_dists = {
    'A': uniform(0, 50),
    'B': uniform(30, 50),
    'C': uniform(60, 50)
}

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    "message uno",
    "message dos",
]

class Population(object):
    """A set of connected Entities. Handles message passing and displaying."""

    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []
        self.connectivity_matrix = None

    def pass_message(self, recipient, task, sender):
        """
        Displays current state and routes the message one connection
        down. Handles the case where the recipient is the target. Calls
        Entity.receive_task(). <recipient> is the index of the recipient
        Entity.
        """

        self.display(recipient, task.target)

        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)

            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        """Initializes each Entity's adjacency list."""

        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self, current=None, target=None):
        """
        Plots the state of the task. If <show> = False, doesn't plot
        anything and the simulation can run faster.
        """

        if not self.show:
            return

        # Scatter plot of points, color coded by class
        pts = self.points
        size = 35
        for cluster, color in cluster_colors.iteritems():
            class_points = [x for x in pts if x.cluster == cluster]
            plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=size)

        # Draw the connections
        if self.connectivity_matrix is not None:
            for start_ix, connections in enumerate(self.connectivity_matrix):
                for connect_ix, connected in enumerate(connections):
                    if connected and connect_ix != start_ix:
                        plt.plot(*zip(
                            (pts[start_ix].x, pts[start_ix].y),
                            (pts[connect_ix].x, pts[connect_ix].y)),
                            c='k', linewidth=0.5)

        # Show where the message is going and where it currently is
        if current and target:
            plt.scatter(pts[current].x, pts[current].y, c='m', s=150)
            plt.scatter(pts[target].x,  pts[target].y,  c='y', s=190)

        plt.show()

    def initiate_task(self, fixed_pair=None):
        """Initializes a task, and calls pass_message()."""

        start, end = [-1, -1] if not fixed_pair else fixed_pair

        if fixed_pair and start == end:
            print("changing fixed pair: they're the same!")

        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()

        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))

        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        """
        Clear every Entity's history. Called when the package is stuck
        in a cycle.
        """

        for point in self.points:
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    """Represents a message to be routed."""

    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""

    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index

        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)

        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        """
        Builds the adjacency list for this Entity. <connectivity_vector>
        is a row from an adjacency matrix.
        """

        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //

        self.value = 0
        self.sent = []

def make_points(cluster, size):
    """Creates a set of points using y_pos_dist and cluster_x_dists."""

    ys = y_pos_dist.rvs(size)
    xs = cluster_x_dists[cluster].rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population(cluster_sizes):
    """Creates a Population and sets its connections. Uses make_points."""

    points = []
    for cluster, count in cluster_sizes.iteritems():
        points += make_points(cluster, count)

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)

    population.set_connections()
    return population

def get_connectivity_matrix(points):
    """
    Computes the connectivity matrix of <points>. Each point is
    connected to each other within a radius.
    """

    points_arr = np.array([[p.x, p.y] for p in points])
    distance_mat = euclidean_distances(points_arr, points_arr)

    min_nonzero = lambda r: min(r[r > 0])
    # axis=1 is the horizontal axis (row-wise)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)

    # WHY 2.2 ?
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff

    return connectivity_matrix

def run(cluster_sizes):
    """Creates and sets up a Population and runs initiate_task()."""

    pop = make_population(cluster_sizes)
    pop.initiate_task()

if __name__ == '__main__':
    cluster_sizes = {
        'A': 10,
        'B': 15,
        'C': 12
    }

    run(cluster_sizes)
<commit_msg>Refactor: making y_pos_dist and cluster_x_dists arguments to run() makes it easier to test when running simulations interactively.<commit_after># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    "message uno",
    "message dos",
]

class Population(object):
    """A set of connected Entities. Handles message passing and displaying."""

    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []
        self.connectivity_matrix = None

    def pass_message(self, recipient, task, sender):
        """
        Displays current state and routes the message one connection
        down. Handles the case where the recipient is the target. Calls
        Entity.receive_task(). <recipient> is the index of the recipient
        Entity.
        """

        self.display(recipient, task.target)

        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)

            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        """Initializes each Entity's adjacency list."""

        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self, current=None, target=None):
        """
        Plots the state of the task. If <show> = False, doesn't plot
        anything and the simulation can run faster.
        """

        if not self.show:
            return

        # Scatter plot of points, color coded by class
        pts = self.points
        size = 35
        for cluster, color in cluster_colors.iteritems():
            class_points = [x for x in pts if x.cluster == cluster]
            plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=size)

        # Draw the connections
        if self.connectivity_matrix is not None:
            for start_ix, connections in enumerate(self.connectivity_matrix):
                for connect_ix, connected in enumerate(connections):
                    if connected and connect_ix != start_ix:
                        plt.plot(*zip(
                            (pts[start_ix].x, pts[start_ix].y),
                            (pts[connect_ix].x, pts[connect_ix].y)),
                            c='k', linewidth=0.5)

        # Show where the message is going and where it currently is
        if current and target:
            plt.scatter(pts[current].x, pts[current].y, c='m', s=150)
            plt.scatter(pts[target].x,  pts[target].y,  c='y', s=190)

        plt.show()

    def initiate_task(self, fixed_pair=None):
        """Initializes a task, and calls pass_message()."""

        start, end = [-1, -1] if not fixed_pair else fixed_pair

        if fixed_pair and start == end:
            print("changing fixed pair: they're the same!")

        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()

        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))

        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        """
        Clear every Entity's history. Called when the package is stuck
        in a cycle.
        """

        for point in self.points:
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    """Represents a message to be routed."""

    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""

    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index

        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)

        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        """
        Builds the adjacency list for this Entity. <connectivity_vector>
        is a row from an adjacency matrix.
        """

        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //

        self.value = 0
        self.sent = []

def make_points(cluster, size, y_dist, x_dist):
    """Creates a set of points using y_dist and x_dist to draw the location."""

    ys = y_dist.rvs(size)
    xs = x_dist.rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population(y_pos_dist, cluster_x_dists, cluster_sizes):
    """Creates a Population and sets its connections. Uses make_points."""

    points = []
    for cluster, size in cluster_sizes.iteritems():
        points += make_points(cluster, size, y_pos_dist, cluster_x_dists[cluster])

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)

    population.set_connections()
    return population

def get_connectivity_matrix(points):
    """
    Computes the connectivity matrix of <points>. Each point is
    connected to each other within a radius.
    """

    points_arr = np.array([[p.x, p.y] for p in points])
    distance_mat = euclidean_distances(points_arr, points_arr)

    min_nonzero = lambda r: min(r[r > 0])
    # axis=1 is the horizontal axis (row-wise)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)

    # WHY 2.2 ?
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff

    return connectivity_matrix

def run(y_pos_dist, cluster_x_dists, cluster_sizes):
    """Creates and sets up a Population and runs initiate_task()."""

    pop = make_population(y_pos_dist, cluster_x_dists, cluster_sizes)
    pop.initiate_task()

if __name__ == '__main__':

    y_pos_dist = norm(500, 10)

    cluster_x_dists = {
        'A': uniform(0, 50),
        'B': uniform(30, 50),
        'C': uniform(60, 50)
    }

    cluster_sizes = {
        'A': 10,
        'B': 15,
        'C': 12
    }

    run(y_pos_dist, cluster_x_dists, cluster_sizes)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

y_pos_dist = norm(500, 10)
cluster_x_dists = {
    'A': uniform(0, 50),
    'B': uniform(30, 50),
    'C': uniform(60, 50)
}

cluster_sizes = {
    'A': 10,
    'B': 15,
    'C': 12
}

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    'go fish',
    "don't go fish"
]

class Population(object):
    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []

    def pass_message(self, recipient, task, sender):
        display(self.points, self.connectivity_matrix, recipient, task.target, show=self.show)
        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)
            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self):
        display(self.points, self.connectivity_matrix, show=self.show)

    def initiate_task(self, fixed_pair=None):
        start, end = ([-1, -1] if not fixed_pair else fixed_pair)
        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()
        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))
        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        for point in self.points:
            # point.adjacencies = list(set(point.adjacencies))
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""
    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def compare(self, other):
        pass

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index
        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)
        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //
        self.value = 0
        self.sent = []

def make_points(cluster, size):
    ys = y_pos_dist.rvs(size)
    xs = cluster_x_dists[cluster].rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population():
    points = []
    for cluster, count in cluster_sizes.iteritems():
        points += make_points(cluster, count)

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)
    population.set_connections()
    return population

def display(points, connectivity_matrix=None, current_ix=None, target_ix=None, show=True):
    if not show:
        return
    for cluster, color in cluster_colors.iteritems():
        class_points = [x for x in points if x.cluster == cluster]
        plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=35)

    if connectivity_matrix is not None:
        for start_ix, connections in enumerate(connectivity_matrix):
            for connect_ix, connected in enumerate(connections):
                if connected and connect_ix != start_ix:
                    plt.plot(*zip(
                        (points[start_ix].x, points[start_ix].y),
                        (points[connect_ix].x, points[connect_ix].y)),
                        c='k', linewidth=0.5)
    if current_ix and target_ix:
        plt.scatter(points[current_ix].x, points[current_ix].y, c='m', s=150)
        plt.scatter(points[target_ix].x, points[target_ix].y, c='y', s=190)

    plt.show()

def get_connectivity_matrix(points):
    points_arr = np.array([[p.x, p.y] for p in points])
    min_nonzero = lambda r: min(r[r > 0])
    distance_mat = euclidean_distances(points_arr, points_arr)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff
    return connectivity_matrix

<commit_msg>basic usage<commit_after># -*- coding: utf-8 -*-
"""
    abm_sketch
    ~~~~

    Initial implementation of message passing in an agent graph
"""

from scipy.stats.distributions import norm
from scipy.stats.distributions import uniform
from sklearn.metrics.pairwise import euclidean_distances
from matplotlib.pylab import plt
import numpy as np
from random import choice
from uuid import uuid4
from collections import defaultdict

y_pos_dist = norm(500, 10)
cluster_x_dists = {
    'A': uniform(0, 50),
    'B': uniform(30, 50),
    'C': uniform(60, 50)
}

cluster_sizes = {
    'A': 10,
    'B': 15,
    'C': 12
}

cluster_colors = {
    'A': 'r',
    'B': 'b',
    'C': 'g'
}

MESSAGES = [
    "message one",
    "message dos",
]

class Population(object):
    def __init__(self):
        self.points = []
        self.path = []
        self.show = True
        self.success_lens = []

    def pass_message(self, recipient, task, sender):
        display(self.points, self.connectivity_matrix, recipient, task.target, show=self.show)
        self.path.append(recipient)
        if recipient == task.target:
            # import ipdb ; ipdb.set_trace#z()  # breakpoint 223cdcc4 //

            k = float(len(self.path))
            for ix, point in enumerate(self.path):
                i_value = (task.value / k) + ((k - ix) * 5)
                self.points[point].award(i_value)
            self.success_lens.append(len(self.path))
            self.path = []

        self.points[recipient].receive_task(task, sender)

    def set_connections(self):
        self.connectivity_matrix = get_connectivity_matrix(self.points)
        for index, point in enumerate(self.points):
            point.set_adjacencies(self.connectivity_matrix[index])

    def display(self):
        display(self.points, self.connectivity_matrix, show=self.show)

    def initiate_task(self, fixed_pair=None):
        start, end = ([-1, -1] if not fixed_pair else fixed_pair)
        while start == end:
            [start, end] = np.random.randint(len(self.points), size=2).tolist()
        message = choice(MESSAGES)
        print('new task: %s' % message)
        print('starting at %s and aiming for %s' % (start, end))
        task = Task(end, message)
        self.pass_message(start, task, len(self.points))

    def clear(self):
        for point in self.points:
            # point.adjacencies = list(set(point.adjacencies))
            adjs = point.adjacencies, set(point.adjacencies)
            if len(adjs[0]) != len(adjs[1]):
                print("clearing %s" % point.index + '---' * 20)
                print(point.adjacencies, set(point.adjacencies))
            point.sent = []

class Task(object):
    def __init__(self, target, message):
        self.target = target
        self.message = message
        self.id = uuid4()
        self.value = 100

class Entity(object):
    """An entity in our world"""
    def __init__(self, population, index, x, y, cluster):
        self.x = x
        self.y = y
        self.index = index
        self.cluster = cluster
        self.adjacencies = []
        self.population = population
        self.task_attempt_map = defaultdict(lambda: [index])
        self.sent = []
        self.value = 0

    def receive_task(self, task, sender):
        if sender not in self.task_attempt_map[task.id]:
            self.task_attempt_map[task.id].append(sender)
        print(sender, self.sent)
        if sender in self.sent:
            print('popping %s' % self.sent.pop(self.sent.index(sender)))
        if set(self.adjacencies).issubset(set(self.task_attempt_map[task.id])):
            print 'caught in a cycle! bailing!'
            self.population.clear()
            return
        if self.index == task.target:
            print 'message delivered! %s' % task.message
        else:
            self.pass_message(task, sender)

    def compare(self, other):
        pass

    def pass_message(self, task, sender):
        next_recipient = task.target if task.target in self.adjacencies else self.index
        while next_recipient in self.task_attempt_map[task.id]:
            next_recipient = choice(self.adjacencies)
        self.task_attempt_map[task.id].append(next_recipient)
        print "passing message from %s to %s" % (self.index, next_recipient)
        self.sent.append(next_recipient)
        self.population.pass_message(next_recipient, task, self.index)

    def set_adjacencies(self, connectivity_vector):
        for connect_ix, connected in enumerate(connectivity_vector):
            if connected and connect_ix != self.index:
                self.adjacencies.append(connect_ix)

    def award(self, value):
        self.value += value
        print(self.value, self.index, self.adjacencies)

        for adj in self.sent:
            assert adj in self.adjacencies
            u_val = uniform.rvs(0, 100, 1)
            print(u_val, u_val < value)
            if u_val < value:
                self.adjacencies.append(adj)
                # import ipdb ; ipdb.set_trace() #z()  # breakpoint f35d9e23 //
        self.value = 0
        self.sent = []

def make_points(cluster, size):
    ys = y_pos_dist.rvs(size)
    xs = cluster_x_dists[cluster].rvs(size)
    return list(zip(xs, ys, [cluster] * size))

def make_population():
    points = []
    for cluster, count in cluster_sizes.iteritems():
        points += make_points(cluster, count)

    population = Population()
    for ix, point in enumerate(points):
        pt = Entity(population, ix, *point)
        population.points.append(pt)
    population.set_connections()
    return population

def display(points, connectivity_matrix=None, current_ix=None, target_ix=None, show=True):
    if not show:
        return
    for cluster, color in cluster_colors.iteritems():
        class_points = [x for x in points if x.cluster == cluster]
        plt.scatter([p.x for p in class_points], [p.y for p in class_points], c=color, s=35)

    if connectivity_matrix is not None:
        for start_ix, connections in enumerate(connectivity_matrix):
            for connect_ix, connected in enumerate(connections):
                if connected and connect_ix != start_ix:
                    plt.plot(*zip(
                        (points[start_ix].x, points[start_ix].y),
                        (points[connect_ix].x, points[connect_ix].y)),
                        c='k', linewidth=0.5)
    if current_ix and target_ix:
        plt.scatter(points[current_ix].x, points[current_ix].y, c='m', s=150)
        plt.scatter(points[target_ix].x, points[target_ix].y, c='y', s=190)

    plt.show()

def get_connectivity_matrix(points):
    points_arr = np.array([[p.x, p.y] for p in points])
    min_nonzero = lambda r: min(r[r > 0])
    distance_mat = euclidean_distances(points_arr, points_arr)
    min_neighbor_distances = np.apply_along_axis(min_nonzero, axis=1, arr=distance_mat)
    neighbor_cutoff = np.mean(min_neighbor_distances) * 2.2
    connectivity_matrix = distance_mat < neighbor_cutoff
    return connectivity_matrix

def run():
    pop = make_population()
    pop.initiate_task()

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from places.models import Place
import os
import json

class Command(BaseCommand):
    args = '<filename>'
    help = 'Imports places from json file to "place" table'

    def handle(self, *args, **options):
        if len(args) <= 0:
            raise CommandError('please specify file name')
        if not os.path.exists(args[0]):
            raise CommandError("file %s doesn't exist" % args[0])
        with open(args[0], 'r') as f:
            result = json.load(f)
        for i in result:
            try:
                place = Place.objects.get(vendor_id=i['id'])
                print "place id #%s already exist... skipping" %i["id"]
            except Place.DoesNotExist:
                place = Place()
                place.vendor_id = i["id"]
                place.data = json.dumps(i)
                place.save()
                print "place id #%s added" %i["id"]<commit_msg>minor string fixes<commit_after>from django.core.management.base import BaseCommand, CommandError
from places.models import Place
import os
import json

class Command(BaseCommand):
    args = '<filename>'
    help = 'Imports places from json file to "place" table'

    def handle(self, *args, **options):
        if len(args) <= 0:
            raise CommandError('please specify file name')
        if not os.path.exists(args[0]):
            raise CommandError("file %s doesn't exist" % args[0])
        with open(args[0], 'r') as f:
            result = json.load(f)
        for i in result:
            try:
                place = Place.objects.get(vendor_id=i['id'])
                print "place id #%s already exist... skipping" % i["id"]
            except Place.DoesNotExist:
                place = Place()
                place.vendor_id = i["id"]
                place.data = json.dumps(i)
                place.save()
                print "place id #%s added" % i["id"]
<|endoftext|>


<commit_before>from optparse import make_option

from django.core.management.base import BaseCommand
from django.core.exceptions import ValidationError
from django.utils.six.moves import input
from django.db.models.signals import post_save, post_delete
from django.conf import settings

from cms.models import (
    Post, Category, Localisation, auto_save_post_to_git,
    auto_save_category_to_git, auto_delete_post_to_git,
    auto_delete_category_to_git, auto_save_localisation_to_git,
    auto_delete_localisation_to_git)

from elasticgit import EG

from unicore.content import models as eg_models

class Command(BaseCommand):
    help = 'import all the current content from a github repository'

    option_list = BaseCommand.option_list + (
        make_option(
            '--quiet',
            action='store_true',
            dest='quiet',
            default=False,
            help='imports the data using default arguments'),
    )

    input_func = input

    def disconnect_signals(self):
        post_save.disconnect(auto_save_post_to_git, sender=Post)
        post_delete.disconnect(auto_delete_post_to_git, sender=Post)

        post_save.disconnect(auto_save_category_to_git, sender=Category)
        post_delete.disconnect(auto_delete_category_to_git, sender=Category)

        post_save.disconnect(
            auto_save_localisation_to_git, sender=Localisation)
        post_delete.disconnect(
            auto_delete_localisation_to_git, sender=Localisation)

    def reconnect_signals(self):
        post_save.connect(auto_save_post_to_git, sender=Post)
        post_delete.connect(auto_delete_post_to_git, sender=Post)

        post_save.connect(auto_save_category_to_git, sender=Category)
        post_delete.connect(auto_delete_category_to_git, sender=Category)

        post_save.connect(
            auto_save_localisation_to_git, sender=Localisation)
        post_delete.connect(
            auto_delete_localisation_to_git, sender=Localisation)

    def emit(self, message):
        if not self.quiet:
            self.stdout.write(message)

    def handle(self, *args, **options):
        self.disconnect_signals()
        self.quiet = options.get('quiet')
        workspace = EG.workspace(
            settings.GIT_REPO_PATH,
            index_prefix=settings.ELASTIC_GIT_INDEX_PREFIX)

        if not self.quiet:
            must_delete = self.get_input_data(
                'Do you want to delete existing data? Y/n: ', 'y')
        else:
            must_delete = 'y'

        if must_delete.lower() == 'y':
            self.emit('deleting existing content..')
            Localisation.objects.all().delete()
            Post.objects.all().delete()
            Category.objects.all().delete()

        self.emit('creating localisations..')
        localisations = workspace.S(eg_models.Localisation).everything()
        for l in localisations:
            Localisation._for(l.locale)

        self.emit('creating categories..')
        categories = workspace.S(eg_models.Category).everything()

        for instance in categories:
            Category.objects.create(
                slug=instance.slug,
                title=instance.title,
                subtitle=instance.subtitle,
                localisation=Localisation._for(instance.language),
                featured_in_navbar=instance.featured_in_navbar or False,
                uuid=instance.uuid,
                position=instance.position or 0,
            )

        # second pass to add related fields
        for instance in categories:
            if instance.source:
                c = Category.objects.get(uuid=instance.uuid)
                c.source = Category.objects.get(uuid=instance.source)
                c.save()

        # Manually refresh stuff because the command disables signals
        workspace.refresh_index()

        pages = workspace.S(eg_models.Page).everything()
        for instance in pages:
            primary_category = None
            if instance.primary_category:
                primary_category = Category.objects.get(
                    uuid=instance.primary_category)
            try:
                p = Post.objects.create(
                    title=instance.title,
                    subtitle=instance.subtitle,
                    slug=instance.slug,
                    description=instance.description,
                    content=instance.content,
                    created_at=instance.created_at,
                    modified_at=instance.modified_at,
                    featured_in_category=(
                        instance.featured_in_category or False),
                    featured=(
                        instance.featured or False),
                    localisation=Localisation._for(instance.language),
                    primary_category=primary_category,
                    uuid=instance.uuid,
                    position=instance.position or 0
                )
                # add the tags
                p.author_tags.add(*instance.author_tags)
            except ValidationError, e:  # pragma: no cover
                self.stderr.write('An error occured with: %s(%s)' % (
                    instance.title, instance.uuid))
                self.stderr.write(e)

        # Manually refresh stuff because the command disables signals
        workspace.refresh_index()

        # second pass to add related fields
        for instance in pages:
            if instance.source:
                p = Post.objects.get(uuid=instance.uuid)
                p.source = Post.objects.get(uuid=instance.source)
                p.save()

            if instance.linked_pages:
                p = Post.objects.get(uuid=instance.uuid)
                p.related_posts.add(*list(
                    Post.objects.filter(uuid__in=instance.linked_pages)))
                p.save()
        self.emit('done.')
        self.reconnect_signals()

    def get_input_data(self, message, default=None):
        raw_value = self.input_func(message)
        if default and raw_value == '':
            raw_value = default

        return raw_value.lower()
<commit_msg>null checking for localisation during import<commit_after>from optparse import make_option

from django.core.management.base import BaseCommand
from django.core.exceptions import ValidationError
from django.utils.six.moves import input
from django.db.models.signals import post_save, post_delete
from django.conf import settings

from cms.models import (
    Post, Category, Localisation, auto_save_post_to_git,
    auto_save_category_to_git, auto_delete_post_to_git,
    auto_delete_category_to_git, auto_save_localisation_to_git,
    auto_delete_localisation_to_git)

from elasticgit import EG

from unicore.content import models as eg_models

class Command(BaseCommand):
    help = 'import all the current content from a github repository'

    option_list = BaseCommand.option_list + (
        make_option(
            '--quiet',
            action='store_true',
            dest='quiet',
            default=False,
            help='imports the data using default arguments'),
    )

    input_func = input

    def disconnect_signals(self):
        post_save.disconnect(auto_save_post_to_git, sender=Post)
        post_delete.disconnect(auto_delete_post_to_git, sender=Post)

        post_save.disconnect(auto_save_category_to_git, sender=Category)
        post_delete.disconnect(auto_delete_category_to_git, sender=Category)

        post_save.disconnect(
            auto_save_localisation_to_git, sender=Localisation)
        post_delete.disconnect(
            auto_delete_localisation_to_git, sender=Localisation)

    def reconnect_signals(self):
        post_save.connect(auto_save_post_to_git, sender=Post)
        post_delete.connect(auto_delete_post_to_git, sender=Post)

        post_save.connect(auto_save_category_to_git, sender=Category)
        post_delete.connect(auto_delete_category_to_git, sender=Category)

        post_save.connect(
            auto_save_localisation_to_git, sender=Localisation)
        post_delete.connect(
            auto_delete_localisation_to_git, sender=Localisation)

    def emit(self, message):
        if not self.quiet:
            self.stdout.write(message)

    def handle(self, *args, **options):
        self.disconnect_signals()
        self.quiet = options.get('quiet')
        workspace = EG.workspace(
            settings.GIT_REPO_PATH,
            index_prefix=settings.ELASTIC_GIT_INDEX_PREFIX)

        if not self.quiet:
            must_delete = self.get_input_data(
                'Do you want to delete existing data? Y/n: ', 'y')
        else:
            must_delete = 'y'

        if must_delete.lower() == 'y':
            self.emit('deleting existing content..')
            Localisation.objects.all().delete()
            Post.objects.all().delete()
            Category.objects.all().delete()

        self.emit('creating localisations..')
        localisations = workspace.S(eg_models.Localisation).everything()
        for l in localisations:
            Localisation._for(l.locale)

        self.emit('creating categories..')
        categories = workspace.S(eg_models.Category).everything()

        for instance in categories:
            localisation = Localisation._for(
                instance.language) if instance.language else None
            Category.objects.create(
                slug=instance.slug,
                title=instance.title,
                subtitle=instance.subtitle,
                localisation=localisation,
                featured_in_navbar=instance.featured_in_navbar or False,
                uuid=instance.uuid,
                position=instance.position or 0,
            )

        # second pass to add related fields
        for instance in categories:
            if instance.source:
                c = Category.objects.get(uuid=instance.uuid)
                c.source = Category.objects.get(uuid=instance.source)
                c.save()

        # Manually refresh stuff because the command disables signals
        workspace.refresh_index()

        pages = workspace.S(eg_models.Page).everything()
        for instance in pages:
            primary_category = None
            if instance.primary_category:
                primary_category = Category.objects.get(
                    uuid=instance.primary_category)
            try:
                localisation = Localisation._for(
                    instance.language) if instance.language else None
                p = Post.objects.create(
                    title=instance.title,
                    subtitle=instance.subtitle,
                    slug=instance.slug,
                    description=instance.description,
                    content=instance.content,
                    created_at=instance.created_at,
                    modified_at=instance.modified_at,
                    featured_in_category=(
                        instance.featured_in_category or False),
                    featured=(
                        instance.featured or False),
                    localisation=localisation,
                    primary_category=primary_category,
                    uuid=instance.uuid,
                    position=instance.position or 0
                )
                # add the tags
                p.author_tags.add(*instance.author_tags)
            except ValidationError, e:  # pragma: no cover
                self.stderr.write('An error occured with: %s(%s)' % (
                    instance.title, instance.uuid))
                self.stderr.write(e)

        # Manually refresh stuff because the command disables signals
        workspace.refresh_index()

        # second pass to add related fields
        for instance in pages:
            if instance.source:
                p = Post.objects.get(uuid=instance.uuid)
                p.source = Post.objects.get(uuid=instance.source)
                p.save()

            if instance.linked_pages:
                p = Post.objects.get(uuid=instance.uuid)
                p.related_posts.add(*list(
                    Post.objects.filter(uuid__in=instance.linked_pages)))
                p.save()
        self.emit('done.')
        self.reconnect_signals()

    def get_input_data(self, message, default=None):
        raw_value = self.input_func(message)
        if default and raw_value == '':
            raw_value = default

        return raw_value.lower()
<|endoftext|>