<commit_before>import sys
import subprocess
import asyncio
from itertools import chain
from collections import Hashable, Iterable, Mapping, MutableMapping, Set, deque
from signal import SIGHUP, SIGINT
try:
    from asyncio import ensure_future
except ImportError:
    from asyncio import async as ensure_future

import yaml

class FrozenDict(Mapping):  # pragma: nocover
    def __init__(self, *args, **kwargs):
        self._data = dict(*args, **kwargs)
        self._hash = None

    def __getitem__(self, key):
        return self._data[key]

    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return iter(self._data)

    def __hash__(self):
        if not self._hash:
            self._hash = hash(frozenset(self._data.items()))

        return self._hash

    def __repr__(self):
        return '{}({})'.format(type(self).__name__, repr(self._data))

    def __eq__(self, other):
        if isinstance(other, type(self)):
            return self._data == other._data
        elif isinstance(other, Mapping):
            return self._data == other

class AsyncBase():
    def __init__(self, loop=None, executor=None, **kwargs):
        self._loop = loop or  asyncio.get_event_loop()
        self._executor = executor

    def delay(self, fn, *args):
        return self._loop.run_in_executor(self._executor, fn, *args)

class AsyncLoopSupervisor(object):
    def __init__(self, loop, timeout=65):
        self.loop = loop  # type: asyncio.AbstractEventLoop
        self.timeout = timeout  # type: int
        self._timeout_handle = None

    @staticmethod
    def _interrupt():
        raise KeyboardInterrupt

    def supervise(self, run_until):
        run_fut = ensure_future(run_until, loop=self.loop)
        self.loop.add_signal_handler(SIGHUP, self._interrupt)
        self.loop.add_signal_handler(SIGINT, self._interrupt)

        try:
            while True:
                try:
                    return self.loop.run_until_complete(run_fut)
                except KeyboardInterrupt:
                    if not self._timeout_handle:
                        self._timeout_handle = self.loop.call_later(
                            self.timeout, self.loop.stop)

                    run_fut.cancel()
        finally:
            self.loop.remove_signal_handler(SIGHUP)
            self.loop.remove_signal_handler(SIGINT)

            if hasattr(self.loop, 'shutdown_asyncgens'):
                self.loop.run_until_complete(self.loop.shutdown_asyncgens())

            if self._timeout_handle:
                self._timeout_handle.cancel()
                self._timeout_handle = None

            self.loop.stop()

def is_collection(v):
    return isinstance(v, Iterable) and not isinstance(v, (bytes, str))

def wrap_as_coll(v, coll=list):
    if is_collection(v):
        return coll(v)
    else:
        return coll([v])

def _merge_mutable_mapping(left, right):
    res = left.copy()
    for k, right_v in right.items():
        if k in res:
            res[k] = deep_merge(res[k], right_v)
        else:
            res[k] = right_v

    return res

def _merge_mapping(left, right):
    memo = set()
    for k, left_v in left.items():
        if k in right:
            v = deep_merge(left_v, right[k])
        else:
            v = left_v

        yield k, v
        memo.add(k)

    for k, right_v in right.items():
        if k not in memo:
            yield k, right_v

def deep_merge(left, right):
    if isinstance(left, Mapping):
        if not isinstance(right, Mapping):
            raise ValueError('Cannot merge Mapping and non-Mapping')

        if isinstance(left, MutableMapping) \
           and callable(getattr(left, 'copy', None)):
            return _merge_mutable_mapping(left, right)
        else:
            tpe = type(left)
            return tpe(_merge_mapping(left, right))
    elif isinstance(left, Set):
        if not is_collection(right):
            raise ValueError('Cannot merge Set and non-Collection')

        return left | set(right)
    elif is_collection(left):
        if not is_collection(right):
            raise ValueError('Cannot merge Collection and non-Collection')

        tpe = type(left)
        return tpe(chain(left, right))
    else:
        return right

def freeze(obj):
    if isinstance(obj, Hashable):
        return obj
    elif isinstance(obj, bytearray):
        return bytes(obj)
    elif isinstance(obj, Mapping):
        return FrozenDict((k, freeze(v)) for (k, v) in obj.items())
    elif isinstance(obj, Set):
        return frozenset(freeze(v) for v in obj)
    elif is_collection(obj):
        return tuple(freeze(v) for v in obj)
    else:  # pragma: nocover
        raise ValueError('Cannot freeze object of type {}'.format(type(obj)))

class TopologicalSortError(ValueError):
    def __init__(self, cycles):
        self.cycles = cycles

def topological_sort(nodes, edges):
    result = []
    edges = {dest: set(sources) for (dest, sources) in edges.items()}
    leaves = deque(filter(lambda n: n not in edges, nodes))

    while leaves:
        result.append(set())
        while leaves:
            leaf = leaves.popleft()
            result[-1].add(leaf)

            for dest, sources in edges.items():
                try:
                    sources.remove(leaf)
                except KeyError:
                    pass

        for dest in list(edges):
            if not edges[dest]:
                del edges[dest]
                leaves.append(dest)

    if edges:
        raise TopologicalSortError(edges)

    return result

@asyncio.coroutine
def async_subprocess_run(program, *args, input=None, stdout=subprocess.PIPE,
                         stderr=None, loop=None, **kwargs):
    loop = loop or asyncio.get_event_loop()
    cmd = [program, *args]
    proc = yield from asyncio.create_subprocess_exec(
        *cmd, stdout=stdout, stderr=stderr, loop=loop, **kwargs)
    out, err = yield from proc.communicate(input)
    ret = yield from proc.wait()

    if ret != 0:
        exc = subprocess.CalledProcessError(ret, cmd, output=out)
        # These attributes are not added automatically pre-Py3.5
        if not hasattr(exc, 'stdout'):
            exc.stdout = stdout and out
        if not hasattr(exc, 'stderr'):
            exc.stderr = stderr and err

        raise exc

    return out, err
<commit_msg>Fix python3.4 error -  args list<commit_after>import sys
import subprocess
import asyncio
from itertools import chain
from collections import Hashable, Iterable, Mapping, MutableMapping, Set, deque
from signal import SIGHUP, SIGINT
try:
    from asyncio import ensure_future
except ImportError:
    from asyncio import async as ensure_future

import yaml

class FrozenDict(Mapping):  # pragma: nocover
    def __init__(self, *args, **kwargs):
        self._data = dict(*args, **kwargs)
        self._hash = None

    def __getitem__(self, key):
        return self._data[key]

    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return iter(self._data)

    def __hash__(self):
        if not self._hash:
            self._hash = hash(frozenset(self._data.items()))

        return self._hash

    def __repr__(self):
        return '{}({})'.format(type(self).__name__, repr(self._data))

    def __eq__(self, other):
        if isinstance(other, type(self)):
            return self._data == other._data
        elif isinstance(other, Mapping):
            return self._data == other

class AsyncBase():
    def __init__(self, loop=None, executor=None, **kwargs):
        self._loop = loop or  asyncio.get_event_loop()
        self._executor = executor

    def delay(self, fn, *args):
        return self._loop.run_in_executor(self._executor, fn, *args)

class AsyncLoopSupervisor(object):
    def __init__(self, loop, timeout=65):
        self.loop = loop  # type: asyncio.AbstractEventLoop
        self.timeout = timeout  # type: int
        self._timeout_handle = None

    @staticmethod
    def _interrupt():
        raise KeyboardInterrupt

    def supervise(self, run_until):
        run_fut = ensure_future(run_until, loop=self.loop)
        self.loop.add_signal_handler(SIGHUP, self._interrupt)
        self.loop.add_signal_handler(SIGINT, self._interrupt)

        try:
            while True:
                try:
                    return self.loop.run_until_complete(run_fut)
                except KeyboardInterrupt:
                    if not self._timeout_handle:
                        self._timeout_handle = self.loop.call_later(
                            self.timeout, self.loop.stop)

                    run_fut.cancel()
        finally:
            self.loop.remove_signal_handler(SIGHUP)
            self.loop.remove_signal_handler(SIGINT)

            if hasattr(self.loop, 'shutdown_asyncgens'):
                self.loop.run_until_complete(self.loop.shutdown_asyncgens())

            if self._timeout_handle:
                self._timeout_handle.cancel()
                self._timeout_handle = None

            self.loop.stop()

def is_collection(v):
    return isinstance(v, Iterable) and not isinstance(v, (bytes, str))

def wrap_as_coll(v, coll=list):
    if is_collection(v):
        return coll(v)
    else:
        return coll([v])

def _merge_mutable_mapping(left, right):
    res = left.copy()
    for k, right_v in right.items():
        if k in res:
            res[k] = deep_merge(res[k], right_v)
        else:
            res[k] = right_v

    return res

def _merge_mapping(left, right):
    memo = set()
    for k, left_v in left.items():
        if k in right:
            v = deep_merge(left_v, right[k])
        else:
            v = left_v

        yield k, v
        memo.add(k)

    for k, right_v in right.items():
        if k not in memo:
            yield k, right_v

def deep_merge(left, right):
    if isinstance(left, Mapping):
        if not isinstance(right, Mapping):
            raise ValueError('Cannot merge Mapping and non-Mapping')

        if isinstance(left, MutableMapping) \
           and callable(getattr(left, 'copy', None)):
            return _merge_mutable_mapping(left, right)
        else:
            tpe = type(left)
            return tpe(_merge_mapping(left, right))
    elif isinstance(left, Set):
        if not is_collection(right):
            raise ValueError('Cannot merge Set and non-Collection')

        return left | set(right)
    elif is_collection(left):
        if not is_collection(right):
            raise ValueError('Cannot merge Collection and non-Collection')

        tpe = type(left)
        return tpe(chain(left, right))
    else:
        return right

def freeze(obj):
    if isinstance(obj, Hashable):
        return obj
    elif isinstance(obj, bytearray):
        return bytes(obj)
    elif isinstance(obj, Mapping):
        return FrozenDict((k, freeze(v)) for (k, v) in obj.items())
    elif isinstance(obj, Set):
        return frozenset(freeze(v) for v in obj)
    elif is_collection(obj):
        return tuple(freeze(v) for v in obj)
    else:  # pragma: nocover
        raise ValueError('Cannot freeze object of type {}'.format(type(obj)))

class TopologicalSortError(ValueError):
    def __init__(self, cycles):
        self.cycles = cycles

def topological_sort(nodes, edges):
    result = []
    edges = {dest: set(sources) for (dest, sources) in edges.items()}
    leaves = deque(filter(lambda n: n not in edges, nodes))

    while leaves:
        result.append(set())
        while leaves:
            leaf = leaves.popleft()
            result[-1].add(leaf)

            for dest, sources in edges.items():
                try:
                    sources.remove(leaf)
                except KeyError:
                    pass

        for dest in list(edges):
            if not edges[dest]:
                del edges[dest]
                leaves.append(dest)

    if edges:
        raise TopologicalSortError(edges)

    return result

@asyncio.coroutine
def async_subprocess_run(program, *args, input=None, stdout=subprocess.PIPE,
                         stderr=None, loop=None, **kwargs):
    loop = loop or asyncio.get_event_loop()
    cmd = [program] + list(args)
    proc = yield from asyncio.create_subprocess_exec(
        *cmd, stdout=stdout, stderr=stderr, loop=loop, **kwargs)
    out, err = yield from proc.communicate(input)
    ret = yield from proc.wait()

    if ret != 0:
        exc = subprocess.CalledProcessError(ret, cmd, output=out)
        # These attributes are not added automatically pre-Py3.5
        if not hasattr(exc, 'stdout'):
            exc.stdout = stdout and out
        if not hasattr(exc, 'stderr'):
            exc.stderr = stderr and err

        raise exc

    return out, err
<|endoftext|>


<commit_before>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)&$$app_token=%s'
    try:
    count_url = '%s?$select=count(*)&$$app_token=%s' % (d['api_url'], app_token)
    count_data = requests.get(count_url).json()
    d['number_of_rows'] = count_data[0]['count']

    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<commit_msg>auto commit<commit_after>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)&$$app_token=%s'
    try:
    count_url = '%s?$select=count(*)&$$app_token=%s' % (d['api_url'], app_token)
    count_data = requests.get(count_url).json()
    d['number_of_rows'] = count_data[0]['count']
    except:
    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<|endoftext|>


<commit_before>""" This module contains the AI search algorithm """

# pylint: disable=import-error
from res import types
from src import coordinate
from src import historynode
from src import rules

class AI(object):
    """ Class that stores the AI algorithm """
    def __init__(self):
        self.arbiter = rules.Rules()
        self.weightA = 1.0
        self.weightB = 1.0
        self.moveCount = 0

    def findBestMove(self,
                     theGame,
                     gooseP,
                     searchPly,
                     minimum=-10000,
                     maximum=10000):
        """ Main alpha-beta minimax algorithm to find best move """
        allMoves = self.getAllMovesForPlayer(theGame, gooseP)
        self.moveCount += 1
        searchPly -= 1
        if searchPly > 0 and not theGame.winningState:
            if gooseP:
                allMoves.sort(key=lambda x: x.score, reverse=gooseP)
                for move in allMoves:
                    result = self.findBestMove(move,
                                               not gooseP,
                                               searchPly,
                                               minimum,
                                               maximum).score
                    move.score = result
                    if result > minimum:
                        minimum = result
                    if result > maximum:
                        move.score = maximum
                        return move
            else:
                allMoves.sort(key=lambda x: x.score, reverse=gooseP)
                for move in allMoves:
                    result = self.findBestMove(move,
                                               not gooseP,
                                               searchPly,
                                               minimum,
                                               maximum).score
                    move.score = result
                    if result < maximum:
                        maximum = result
                    if move.score < minimum:
                        move.score = minimum
                        return move
        return getHighestOrLowestScoreMove(allMoves, gooseP)

    def getAllMovesForPlayer(self, theGame, gooseP):
        """GooseP == True means it's the Goose player's turn. Otherwise fox"""
        moves = []
        for location in getTupleOfAllCoordinates():
            if gooseP:
                moves.extend(self.getMovesForGoosePiece(theGame, location))
            else:
                moves.extend(self.getMovesForFoxPiece(theGame, location))
        return moves

    def getMovesForGoosePiece(self, theGame, gooseLocation):
        """ This returns a GameNode for every legal move of a given goose """
        moveList = []
        if (theGame.getState(gooseLocation) not in (types.GOOSE,
                                                    types.SUPERGOOSE)):
            # Don't examine a piece that's not a goose or supergoose
            return moveList

        for direction in range(1, 9):
            gooseDestination = getCoordinateFromDirection(gooseLocation,
                                                          direction)
            if (gooseDestination and
                    self.arbiter.legalMoveP(theGame,
                                            gooseLocation,
                                            gooseDestination)):
                gooseType = theGame.getState(gooseLocation)
                moveResult = transferNode(theGame)
                finalGooseType = rules.resultingGoose(gooseType,
                                                      gooseDestination)
                moveResult.setState(gooseDestination, finalGooseType)
                moveResult.setState(gooseLocation, types.EMPTY)
                moveResult.score = self.evaluationFunction(moveResult)
                moveResult.determineWinningState()
                moveResult.leafP = True
                moveResult.rootP = False
                moveList.append(moveResult)
        return moveList

    def getMovesForFoxPiece(self, theGame, foxLocation):
        """ Returns a GameNode for every legal move of a given fox. theGame
        is the current board position. """
        moveList = []
        if theGame.getState(foxLocation) != types.FOX:
            # Don't examine a piece that's not a fox
            return moveList
        moveList = self.getAllFoxCaptures(theGame, foxLocation)

        if len(moveList) == 0:
            # These are regular non-capture moves for the fox, if no captures
            # were found
            for direction in range(1, 9):
                foxDestination = getCoordinateFromDirection(foxLocation,
                                                            direction)
                if (foxDestination and
                        self.arbiter.legalMoveP(theGame,
                                                foxLocation,
                                                foxDestination)):
                    resultMove = transferNode(theGame)
                    resultMove.setState(foxDestination, types.FOX)
                    resultMove.setState(foxLocation, types.EMPTY)
                    resultMove.score = self.evaluationFunction(resultMove)
                    resultMove.determineWinningState()
                    resultMove.leafP = True
                    resultMove.rootP = False
                    moveList.append(resultMove)
        return moveList

    def getAllFoxCaptures(self, theGame, location):
        """ This recursively finds all available captures for a single fox and
        returns the list of captures"""
        captureList = []
        x_board = location.get_x_board()
        y_board = location.get_y_board()
        for direction in range(1, 9):
            if self.arbiter.isACaptureP(theGame, location, direction):
                deltaX = rules.findXCoordinateFromDirection(direction)
                deltaY = rules.findYCoordinateFromDirection(direction)
                newMoveNode = transferNode(theGame)
                destination = coordinate.Coordinate(x_board + deltaX,
                                                    y_board + deltaY)
                rules.makeCapture(newMoveNode, location, destination)
                newMoveNode.score = self.evaluationFunction(newMoveNode)
                newMoveNode.determineWinningState()
                newMoveNode.leafP = True
                newMoveNode.rootP = False
                newMoveNode.isCapture = True
                captureList.append(newMoveNode)
                nextCapture = self.getAllFoxCaptures(newMoveNode, destination)
                if nextCapture:
                    captureList.extend(nextCapture)
        return captureList

    def evaluationFunction(self, theGame):
        """ This function takes a game state and returns a score for the
        position. A positive score favors the geese, and a negative score
        favors the foxes. """
        valueA = 0.0
        valueB = 0.0
        victoryPoints = 0
        totalScore = 0.0

        for location in getTupleOfAllCoordinates():
            if theGame.getState(location) == types.GOOSE:
                # Reward goose player for having material on the board
                valueA += 1
                # Reward Goose player for moving to the first row
                valueA += (7 - location.get_y_board()) * 0.1
            elif theGame.getState(location) == types.SUPERGOOSE:
                valueA += 2
                if (3 <= location.get_x_board() <= 5 and
                    1 <= location.get_y_board() <= 3):
                    # Reward Goose player for occupying victory zone
                    valueB += 4 - location.get_y_board()
                    victoryPoints += 1
            elif theGame.getState(location) == types.FOX:
                # Reward fox player for being near the 1st row
                valueA -= (7 - location.get_y_board()) * 0.2

        valueA -= 20
        valueB *= victoryPoints
        totalScore += self.weightA * valueA + self.weightB * valueB
        if theGame.geeseWinP():
            totalScore += 1000
        elif theGame.foxesWinP():
            totalScore -= 1000

        return totalScore

def getTupleOfAllCoordinates():
    return (coordinate.Coordinate(3, 7), coordinate.Coordinate(4, 7),
            coordinate.Coordinate(5, 7), coordinate.Coordinate(3, 6),
            coordinate.Coordinate(4, 6), coordinate.Coordinate(5, 6),
            coordinate.Coordinate(1, 5), coordinate.Coordinate(2, 5),
            coordinate.Coordinate(3, 5), coordinate.Coordinate(4, 5),
            coordinate.Coordinate(5, 5), coordinate.Coordinate(6, 5),
            coordinate.Coordinate(7, 5), coordinate.Coordinate(1, 4),
            coordinate.Coordinate(2, 4), coordinate.Coordinate(3, 4),
            coordinate.Coordinate(4, 4), coordinate.Coordinate(5, 4),
            coordinate.Coordinate(6, 4), coordinate.Coordinate(7, 4),
            coordinate.Coordinate(1, 3), coordinate.Coordinate(2, 3),
            coordinate.Coordinate(3, 3), coordinate.Coordinate(4, 3),
            coordinate.Coordinate(5, 3), coordinate.Coordinate(6, 3),
            coordinate.Coordinate(7, 3), coordinate.Coordinate(3, 2),
            coordinate.Coordinate(4, 2), coordinate.Coordinate(5, 2),
            coordinate.Coordinate(3, 1), coordinate.Coordinate(4, 1),
            coordinate.Coordinate(5, 1))

def transferNode(startNode):
    """ Copies input historynode to a new one and returns that.
    Basically performs a deep copy."""
    endNode = historynode.HistoryNode()
    for location in getTupleOfAllCoordinates():
        endNode.setState(location, startNode.getState(location))
    return endNode

# pylint: disable=too-many-return-statements
def getCoordinateFromDirection(currentLocation, direction):
    """ Gets a coordinate delta from a current one & direction """

    xBoard = currentLocation.get_x_board()
    yBoard = currentLocation.get_y_board()

    if direction == 1:
        return getCoordinateHelper(xBoard, yBoard + 1)
    elif direction == 2:
        return getCoordinateHelper(xBoard + 1, yBoard + 1)
    elif direction == 3:
        return getCoordinateHelper(xBoard + 1, yBoard)
    elif direction == 4:
        return getCoordinateHelper(xBoard + 1, yBoard - 1)
    elif direction == 5:
        return getCoordinateHelper(xBoard, yBoard - 1)
    elif direction == 6:
        return getCoordinateHelper(xBoard - 1, yBoard - 1)
    elif direction == 7:
        return getCoordinateHelper(xBoard - 1, yBoard)
    elif direction == 8:
        return getCoordinateHelper(xBoard - 1, yBoard + 1)
    else:
        raise ValueError

def getCoordinateHelper(xBoard, yBoard):
    """ Wrap the error handling """
    try:
        return coordinate.Coordinate(xBoard, yBoard)
    except ValueError:
        return None

def getHighestOrLowestScoreMove(moves, gooseP):
    """ Returns the highest/lowest scored move depending on the player """
    if gooseP:
        return max(moves, key=lambda x: x.score)
    else:
        return min(moves, key=lambda x: x.score)
<commit_msg>Add configurable condition to make search algorithm pick moves randomly in the event of a tie<commit_after>""" This module contains the AI search algorithm """

from random import shuffle

# pylint: disable=import-error
from res import types
from src import coordinate
from src import historynode
from src import rules

class AI(object):
    """ Class that stores the AI algorithm """
    def __init__(self):
        self.arbiter = rules.Rules()
        self.weightA = 1.0
        self.weightB = 1.0
        self.moveCount = 0
        self.random = True

    def findBestMove(self,
                     theGame,
                     gooseP,
                     searchPly,
                     minimum=-10000,
                     maximum=10000):
        """ Main alpha-beta minimax algorithm to find best move """
        allMoves = self.getAllMovesForPlayer(theGame, gooseP)
        if self.random:
            shuffle(allMoves)
        self.moveCount += 1
        searchPly -= 1
        if searchPly > 0 and not theGame.winningState:
            if gooseP:
                allMoves.sort(key=lambda x: x.score, reverse=gooseP)
                for move in allMoves:
                    result = self.findBestMove(move,
                                               not gooseP,
                                               searchPly,
                                               minimum,
                                               maximum).score
                    move.score = result
                    if result > minimum:
                        minimum = result
                    if result > maximum:
                        move.score = maximum
                        return move
            else:
                allMoves.sort(key=lambda x: x.score, reverse=gooseP)
                for move in allMoves:
                    result = self.findBestMove(move,
                                               not gooseP,
                                               searchPly,
                                               minimum,
                                               maximum).score
                    move.score = result
                    if result < maximum:
                        maximum = result
                    if move.score < minimum:
                        move.score = minimum
                        return move
        return getHighestOrLowestScoreMove(allMoves, gooseP)

    def getAllMovesForPlayer(self, theGame, gooseP):
        """GooseP == True means it's the Goose player's turn. Otherwise fox"""
        moves = []
        for location in getTupleOfAllCoordinates():
            if gooseP:
                moves.extend(self.getMovesForGoosePiece(theGame, location))
            else:
                moves.extend(self.getMovesForFoxPiece(theGame, location))
        if not gooseP:
            captureMoves = list(filter(lambda x: x.isCapture, moves))
            if len(captureMoves) > 0:
                return captureMoves
        return moves

    def getMovesForGoosePiece(self, theGame, gooseLocation):
        """ This returns a GameNode for every legal move of a given goose """
        moveList = []
        if (theGame.getState(gooseLocation) not in (types.GOOSE,
                                                    types.SUPERGOOSE)):
            # Don't examine a piece that's not a goose or supergoose
            return moveList

        for direction in range(1, 9):
            gooseDestination = getCoordinateFromDirection(gooseLocation,
                                                          direction)
            if (gooseDestination and
                    self.arbiter.legalMoveP(theGame,
                                            gooseLocation,
                                            gooseDestination)):
                gooseType = theGame.getState(gooseLocation)
                moveResult = transferNode(theGame)
                finalGooseType = rules.resultingGoose(gooseType,
                                                      gooseDestination)
                moveResult.setState(gooseDestination, finalGooseType)
                moveResult.setState(gooseLocation, types.EMPTY)
                moveResult.score = self.evaluationFunction(moveResult)
                moveResult.determineWinningState()
                moveResult.leafP = True
                moveResult.rootP = False
                moveList.append(moveResult)
        return moveList

    def getMovesForFoxPiece(self, theGame, foxLocation):
        """ Returns a GameNode for every legal move of a given fox. theGame
        is the current board position. """
        moveList = []
        if theGame.getState(foxLocation) != types.FOX:
            # Don't examine a piece that's not a fox
            return moveList
        moveList = self.getAllFoxCaptures(theGame, foxLocation)

        if len(moveList) == 0:
            # These are regular non-capture moves for the fox, if no captures
            # were found
            for direction in range(1, 9):
                foxDestination = getCoordinateFromDirection(foxLocation,
                                                            direction)
                if (foxDestination and
                        self.arbiter.legalMoveP(theGame,
                                                foxLocation,
                                                foxDestination)):
                    resultMove = transferNode(theGame)
                    resultMove.setState(foxDestination, types.FOX)
                    resultMove.setState(foxLocation, types.EMPTY)
                    resultMove.score = self.evaluationFunction(resultMove)
                    resultMove.determineWinningState()
                    resultMove.leafP = True
                    resultMove.rootP = False
                    moveList.append(resultMove)
        return moveList

    def getAllFoxCaptures(self, theGame, location):
        """ This recursively finds all available captures for a single fox and
        returns the list of captures"""
        captureList = []
        x_board = location.get_x_board()
        y_board = location.get_y_board()
        for direction in range(1, 9):
            if self.arbiter.isACaptureP(theGame, location, direction):
                deltaX = rules.findXCoordinateFromDirection(direction)
                deltaY = rules.findYCoordinateFromDirection(direction)
                newMoveNode = transferNode(theGame)
                destination = coordinate.Coordinate(x_board + deltaX,
                                                    y_board + deltaY)
                rules.makeCapture(newMoveNode, location, destination)
                newMoveNode.score = self.evaluationFunction(newMoveNode)
                newMoveNode.determineWinningState()
                newMoveNode.leafP = True
                newMoveNode.rootP = False
                newMoveNode.isCapture = True
                captureList.append(newMoveNode)
                nextCapture = self.getAllFoxCaptures(newMoveNode, destination)
                if nextCapture:
                    captureList.extend(nextCapture)
        return captureList

    def evaluationFunction(self, theGame):
        """ This function takes a game state and returns a score for the
        position. A positive score favors the geese, and a negative score
        favors the foxes. """
        valueA = 0.0
        valueB = 0.0
        victoryPoints = 0
        totalScore = 0.0

        for location in getTupleOfAllCoordinates():
            if theGame.getState(location) == types.GOOSE:
                # Reward goose player for having material on the board
                valueA += 1
                # Reward Goose player for moving to the first row
                valueA += (7 - location.get_y_board()) * 0.1
            elif theGame.getState(location) == types.SUPERGOOSE:
                valueA += 2
                if (3 <= location.get_x_board() <= 5 and
                    1 <= location.get_y_board() <= 3):
                    # Reward Goose player for occupying victory zone
                    valueB += 4 - location.get_y_board()
                    victoryPoints += 1
            elif theGame.getState(location) == types.FOX:
                # Reward fox player for being near the 1st row
                valueA -= (7 - location.get_y_board()) * 0.2

        valueA -= 20
        valueB *= victoryPoints
        totalScore += self.weightA * valueA + self.weightB * valueB
        if theGame.geeseWinP():
            totalScore += 1000
        elif theGame.foxesWinP():
            totalScore -= 1000

        return totalScore

def getTupleOfAllCoordinates():
    return (coordinate.Coordinate(3, 7), coordinate.Coordinate(4, 7),
            coordinate.Coordinate(5, 7), coordinate.Coordinate(3, 6),
            coordinate.Coordinate(4, 6), coordinate.Coordinate(5, 6),
            coordinate.Coordinate(1, 5), coordinate.Coordinate(2, 5),
            coordinate.Coordinate(3, 5), coordinate.Coordinate(4, 5),
            coordinate.Coordinate(5, 5), coordinate.Coordinate(6, 5),
            coordinate.Coordinate(7, 5), coordinate.Coordinate(1, 4),
            coordinate.Coordinate(2, 4), coordinate.Coordinate(3, 4),
            coordinate.Coordinate(4, 4), coordinate.Coordinate(5, 4),
            coordinate.Coordinate(6, 4), coordinate.Coordinate(7, 4),
            coordinate.Coordinate(1, 3), coordinate.Coordinate(2, 3),
            coordinate.Coordinate(3, 3), coordinate.Coordinate(4, 3),
            coordinate.Coordinate(5, 3), coordinate.Coordinate(6, 3),
            coordinate.Coordinate(7, 3), coordinate.Coordinate(3, 2),
            coordinate.Coordinate(4, 2), coordinate.Coordinate(5, 2),
            coordinate.Coordinate(3, 1), coordinate.Coordinate(4, 1),
            coordinate.Coordinate(5, 1))

def transferNode(startNode):
    """ Copies input historynode to a new one and returns that.
    Basically performs a deep copy."""
    endNode = historynode.HistoryNode()
    for location in getTupleOfAllCoordinates():
        endNode.setState(location, startNode.getState(location))
    return endNode

# pylint: disable=too-many-return-statements
def getCoordinateFromDirection(currentLocation, direction):
    """ Gets a coordinate delta from a current one & direction """

    xBoard = currentLocation.get_x_board()
    yBoard = currentLocation.get_y_board()

    if direction == 1:
        return getCoordinateHelper(xBoard, yBoard + 1)
    elif direction == 2:
        return getCoordinateHelper(xBoard + 1, yBoard + 1)
    elif direction == 3:
        return getCoordinateHelper(xBoard + 1, yBoard)
    elif direction == 4:
        return getCoordinateHelper(xBoard + 1, yBoard - 1)
    elif direction == 5:
        return getCoordinateHelper(xBoard, yBoard - 1)
    elif direction == 6:
        return getCoordinateHelper(xBoard - 1, yBoard - 1)
    elif direction == 7:
        return getCoordinateHelper(xBoard - 1, yBoard)
    elif direction == 8:
        return getCoordinateHelper(xBoard - 1, yBoard + 1)
    else:
        raise ValueError

def getCoordinateHelper(xBoard, yBoard):
    """ Wrap the error handling """
    try:
        return coordinate.Coordinate(xBoard, yBoard)
    except ValueError:
        return None

def getHighestOrLowestScoreMove(moves, gooseP):
    """ Returns the highest/lowest scored move depending on the player """
    if gooseP:
        return max(moves, key=lambda x: x.score)
    else:
        return min(moves, key=lambda x: x.score)
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import numpy as np
import time
from keras.layers.core import Dense, Activation, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Sequential

from crimeprediction.vectorize import vectorize

def run_network(grid_size, period, crime_type=None, seasonal=False):
    vectors = vectorize(grid_size, period)
    global_start_time = time.time()
    epochs = 1
    ratio = 0.5
    sequence_length = 50
    path_to_dataset = 'household_power_consumption.txt'

    print 'Loading Data...'
    vectors = vectorize(grid_size, period)
    dim = len(vectors[0])
    result = np.array(vectors)

    print "Data  : ", result.shape

    row = round(0.5 * result.shape[0])
    train = result[:row]
    X_train = train[:-1]
    y_train = train[1:]
    test = result[row:]
    X_test = test[:-1]
    y_test = test[1:]

    X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
    X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
    print '\nData Loaded. Compiling...\n'
    print X_train
    print y_train
    model = Sequential()
    model.add(LSTM(dim, input_shape=X_train.shape[1:]))
    model.compile(loss='mse', optimizer='rmsprop',)

    print("Train...")

    try:
        model.fit(X_train, y_train, nb_epoch=1000, shuffle=False)
    except KeyboardInterrupt:
        pass

    print 'Training duration (s) : ', time.time() - global_start_time
    predicted = model.predict(X_test)

    accuracy = []
    f1scr = []

    for x, data in enumerate(y_test):
        print len(data)
        print len(predicted[x])
        correct = 0
        total = 0
        truepos = 0
        falsepos = 0
        trueneg = 0
        falseneg = 0

        for y, node in enumerate(data):
            total += 1
            if predicted[x][y] > 0:  # threshold for prediction. If prediction is greater than this value, prediction is one, zero otherwise
                if node == 1:
                    correct += 1
                    truepos += 1
                else:
                    falsepos += 1
            else:
                if node == -1:
                    correct += 1
                    trueneg += 1
                else:
                    falseneg += 1
        print "correct", correct
        print "total", total
        act = float(correct) / total
        print act
        accuracy.append(act)

        precision = truepos / float(truepos+falsepos)
        recall = truepos / float(truepos+falseneg)

        f1 = (precision * recall * 2) / float(precision + recall)

        f1scr.append(f1)

        print accuracy
        print f1

    print "Average Accuracy:", np.average(accuracy)
    print "Average F1 Score:", np.average(f1scr)

    # graph of Accuracy for each grid snapshot
    fig1 = plt.figure()
    plt.plot(accuracy)
    plt.xlabel(period.capitalize())
    plt.ylabel('Accuracy')
    fig1.savefig("Accuracy.png")
    # plt.show()

    # graph of F1 Score for each grid snapshot
    fig2 = plt.figure()
    plt.plot(f1scr)
    plt.xlabel(period.capitalize())
    plt.ylabel('F1 Score')
    fig2.savefig("F1Score.png")
    return model, y_test, predicted
<commit_msg>updated image filenames<commit_after>import matplotlib.pyplot as plt
import numpy as np
import time
from keras.layers.core import Dense, Activation, Dropout
from keras.layers.recurrent import LSTM
from keras.models import Sequential

from crimeprediction.vectorize import vectorize

def run_network(grid_size, period, crime_type=None, seasonal=False):
    vectors = vectorize(grid_size, period)
    global_start_time = time.time()
    epochs = 1
    ratio = 0.5
    sequence_length = 50
    path_to_dataset = 'household_power_consumption.txt'

    print 'Loading Data...'
    vectors = vectorize(grid_size, period)
    dim = len(vectors[0])
    result = np.array(vectors)

    print "Data  : ", result.shape

    row = round(0.5 * result.shape[0])
    train = result[:row]
    X_train = train[:-1]
    y_train = train[1:]
    test = result[row:]
    X_test = test[:-1]
    y_test = test[1:]

    X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
    X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
    print '\nData Loaded. Compiling...\n'
    print X_train
    print y_train
    model = Sequential()
    model.add(LSTM(dim, input_shape=X_train.shape[1:]))
    model.compile(loss='mse', optimizer='rmsprop',)

    print("Train...")

    try:
        model.fit(X_train, y_train, nb_epoch=1000, shuffle=False)
    except KeyboardInterrupt:
        pass

    print 'Training duration (s) : ', time.time() - global_start_time
    predicted = model.predict(X_test)

    accuracy = []
    f1scr = []

    for x, data in enumerate(y_test):
        print len(data)
        print len(predicted[x])
        correct = 0
        total = 0
        truepos = 0
        falsepos = 0
        trueneg = 0
        falseneg = 0

        for y, node in enumerate(data):
            total += 1
            if predicted[x][y] > 0:  # threshold for prediction. If prediction is greater than this value, prediction is one, zero otherwise
                if node == 1:
                    correct += 1
                    truepos += 1
                else:
                    falsepos += 1
            else:
                if node == -1:
                    correct += 1
                    trueneg += 1
                else:
                    falseneg += 1
        print "correct", correct
        print "total", total
        act = float(correct) / total
        print act
        accuracy.append(act)

        precision = truepos / float(truepos+falsepos)
        recall = truepos / float(truepos+falseneg)

        f1 = (precision * recall * 2) / float(precision + recall)

        f1scr.append(f1)

        print accuracy
        print f1

    print "Average Accuracy:", np.average(accuracy)
    print "Average F1 Score:", np.average(f1scr)
    crime_verbose = crime_type if crime_type is not None else "ALL"
    # graph of Accuracy for each grid snapshot
    fig1 = plt.figure()
    plt.plot(accuracy)
    plt.xlabel(period.capitalize())
    plt.ylabel('Accuracy')
    fig1.savefig("Accuracy_{0}_{1}_{2}_{3}.png".format(
        grid_size, crime_verbose, period, seasonal))
    # plt.show()

    # graph of F1 Score for each grid snapshot
    fig2 = plt.figure()
    plt.plot(f1scr)
    plt.xlabel(period.capitalize())
    plt.ylabel('F1 Score')
    fig2.savefig("F1Score_{0}_{1}_{2}_{3}.png".format(
        grid_size, crime_verbose, period, seasonal))
    return model, y_test, predicted
<|endoftext|>


<commit_before>"""
Unit tests for django-registration.

"""

import datetime

from django.conf import settings
from django.core import mail
from django.test import TestCase

from registration.models import RegistrationProfile

class DefaultBackendTestCase(TestCase):
    """
    Test the default registration backend.

    Running these tests successfull will require two templates to be
    created for the sending of activation emails; details on these
    templates and their contexts may be found in the documentation for
    the default backend. The setting ``ACCOUNT_ACTIVATION_DAYS`` must
    also be specified, and must be an integer.

    """
    def setUp(self):
        """
        Create an instance of the default backend for use in testing.

        """
        from registration.backends.default import DefaultBackend
        self.backend = DefaultBackend()

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register({}, 'bob', 'bob@example.com', 'secret')
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')
        self.failIf(new_user.is_active)
        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

    def test_activation(self):
        """
        Test the activation process: activating within the permitted
        window sets the account's ``is_active`` field to ``True`` and
        resets the activation key, while failing to activate within
        the permitted window forbids later activation.

        """
        # First, test with a user activating inside the activation
        # window.
        valid_user = self.backend.register({}, 'alice', 'alice@example.com', 'swordfish')
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        activated = self.backend.activate({}, valid_profile.activation_key)
        self.assertEqual(activated.username, valid_user.username)
        self.failUnless(activated.is_active)

        # Fetch the profile again to verify its activation key has
        # been reset.
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        self.assertEqual(valid_profile.activation_key, RegistrationProfile.ACTIVATED)

        # Now test again, but with a user activating outside the
        # activation window.
        expired_user = self.backend.register({}, 'bob', 'bob@example.com', 'secret')
        expired_user.date_joined = expired_user.date_joined - datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        expired_user.save()
        expired_profile = RegistrationProfile.objects.get(user=expired_user)
        self.failIf(self.backend.activate({}, expired_profile.activation_key))
        self.failUnless(expired_profile.activation_key_expired())
<commit_msg>Test the ability to open and close registration using a setting.<commit_after>"""
Unit tests for django-registration.

"""

import datetime

from django.conf import settings
from django.core import mail
from django.test import TestCase

from registration.models import RegistrationProfile

class DefaultBackendTestCase(TestCase):
    """
    Test the default registration backend.

    Running these tests successfull will require two templates to be
    created for the sending of activation emails; details on these
    templates and their contexts may be found in the documentation for
    the default backend. The setting ``ACCOUNT_ACTIVATION_DAYS`` must
    also be specified, and must be an integer.

    """
    def setUp(self):
        """
        Create an instance of the default backend for use in testing.

        """
        from registration.backends.default import DefaultBackend
        self.backend = DefaultBackend()

    def test_registration(self):
        """
        Test the registration process: registration creates a new
        inactive account and a new profile with activation key,
        populates the correct account data and sends an activation
        email.

        """
        new_user = self.backend.register({}, 'bob', 'bob@example.com', 'secret')
        self.assertEqual(new_user.username, 'bob')
        self.failUnless(new_user.check_password('secret'))
        self.assertEqual(new_user.email, 'bob@example.com')
        self.failIf(new_user.is_active)
        self.assertEqual(RegistrationProfile.objects.count(), 1)
        self.assertEqual(len(mail.outbox), 1)

    def test_activation(self):
        """
        Test the activation process: activating within the permitted
        window sets the account's ``is_active`` field to ``True`` and
        resets the activation key, while failing to activate within
        the permitted window forbids later activation.

        """
        # First, test with a user activating inside the activation
        # window.
        valid_user = self.backend.register({}, 'alice', 'alice@example.com', 'swordfish')
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        activated = self.backend.activate({}, valid_profile.activation_key)
        self.assertEqual(activated.username, valid_user.username)
        self.failUnless(activated.is_active)

        # Fetch the profile again to verify its activation key has
        # been reset.
        valid_profile = RegistrationProfile.objects.get(user=valid_user)
        self.assertEqual(valid_profile.activation_key, RegistrationProfile.ACTIVATED)

        # Now test again, but with a user activating outside the
        # activation window.
        expired_user = self.backend.register({}, 'bob', 'bob@example.com', 'secret')
        expired_user.date_joined = expired_user.date_joined - datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        expired_user.save()
        expired_profile = RegistrationProfile.objects.get(user=expired_user)
        self.failIf(self.backend.activate({}, expired_profile.activation_key))
        self.failUnless(expired_profile.activation_key_expired())

    def test_allow(self):
        """
        Test that the setting ``REGISTRATION_OPEN`` appropriately
        controls whether registration is permitted.

        """
        self.failUnless(self.backend.registration_allowed({}))
        settings.REGISTRATION_OPEN = False
        self.failIf(self.backend.registration_allowed({}))
<|endoftext|>


<commit_before>import inspect

def strip_html(s):
    from xml.etree.ElementTree import fromstring
    from html import unescape
    return unescape("".join(fromstring("<root>%s</root>" % s).itertext()))

class Hex(int):

    def __str__(self):
        return "%X" % self

    def __repr__(self):
        return "0x%x" % self

class Token(str):

    _all = {}

    def __new__(cls, name):
        try:
            return cls._all[name]
        except KeyError:
            pass
        cls._all[name] = self = super().__new__(cls, "<%s>" % name.strip("<>"))
        return self

    def __repr__(self):
        return self

def __LOCATION__():
    frame = inspect.getframeinfo(inspect.stack()[1][0])
    return "%s @ %s:%s" % (frame.function, frame.filename, frame.lineno)

def get_all_subclasses(cls):
    _all = cls.__subclasses__() + [rec_subclass
                                   for subclass in cls.__subclasses__()
                                   for rec_subclass in get_all_subclasses(subclass)]
    return [subclass for subclass in _all if not hasattr(subclass, "_%s__is_mixin" % subclass.__name__)]

def stack_level_to_get_out_of_file():
    frame = inspect.currentframe().f_back
    filename = frame.f_code.co_filename
    stack_levels = 1
    while frame.f_code.co_filename == filename:
        stack_levels += 1
        frame = frame.f_back
    return stack_levels
<commit_msg>tokens: make them comparable to strings to support cli option conversions<commit_after>import inspect

def strip_html(s):
    from xml.etree.ElementTree import fromstring
    from html import unescape
    return unescape("".join(fromstring("<root>%s</root>" % s).itertext()))

class Hex(int):

    def __str__(self):
        return "%X" % self

    def __repr__(self):
        return "0x%x" % self

class Token(str):

    _all = {}

    def __new__(cls, name):
        name = name.strip("<>")
        try:
            return cls._all[name]
        except KeyError:
            pass
        cls._all[name] = self = super().__new__(cls, "<%s>" % name)
        return self

    def __repr__(self):
        return self

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self is other
        elif isinstance(other, str):
            return self.strip("<>").lower() == other.strip("<>").lower()
        return False

    # we're already case insensitive when comparing
    def lower(self): return self

    def upper(self): return self

def __LOCATION__():
    frame = inspect.getframeinfo(inspect.stack()[1][0])
    return "%s @ %s:%s" % (frame.function, frame.filename, frame.lineno)

def get_all_subclasses(cls):
    _all = cls.__subclasses__() + [rec_subclass
                                   for subclass in cls.__subclasses__()
                                   for rec_subclass in get_all_subclasses(subclass)]
    return [subclass for subclass in _all if not hasattr(subclass, "_%s__is_mixin" % subclass.__name__)]

def stack_level_to_get_out_of_file():
    frame = inspect.currentframe().f_back
    filename = frame.f_code.co_filename
    stack_levels = 1
    while frame.f_code.co_filename == filename:
        stack_levels += 1
        frame = frame.f_back
    return stack_levels
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding: utf-8

import os
import commands
import miniparser
import util.core as core
from util.color import *

parser = miniparser.parser(version="1.0.0", description="This tools make you more usefull git")

@parser.command(description="You can show latest commit hash.")
def latest():
    commits = core.get_commits()
    print commits[0].commithash

@parser.option(
    "ls",
    description="You can show all commit hash with date, comment")
def show_all():
    commits = core.get_commits()
    for index, commit_obj in enumerate(commits):
        print "[%(index)s] %(date)s : %(hash)s - '%(comment)s'" % {
            "index": yellow("0%d"%index) if index < 10 else yellow(index),
            "date" : commit_obj.date.strftime("%y/%m/%d %H:%M:%S"),
            "hash" : green(commit_obj.commithash),
            "comment": commit_obj.comment,
        }

@parser.option(
    "get",
    description="You can get commit hash. Please set index.",
    argument_types={"index":int})
def copy_hash(index):
    commits = core.get_commits()
    if index > len(commits)-1:
        print yellow("this index is over length limit.")
        return

    print commits[index].commithash

if __name__ == "__main__":
    if core.check_exist_repo() == False:
        print yellow("There is not git repository!")
        miniparser.kill(1)

    parser.parse()
<commit_msg>added 'show' option<commit_after>#!/usr/bin/env python
#coding: utf-8

import os
import commands
import miniparser
import util.core as core
from util.color import *

parser = miniparser.parser(version="1.0.0", description="This tools make you more usefull git")

@parser.command(description="You can show latest commit hash.")
def latest():
    commits = core.get_commits()
    print commits[0].commithash

@parser.option(
    "ls",
    description="You can show all commit hash with date, comment")
def show_all():
    commits = core.get_commits()
    for index, commit_obj in enumerate(commits):
        print "[%(index)s] %(date)s : %(hash)s - '%(comment)s'" % {
            "index": yellow("0%d"%index) if index < 10 else yellow(index),
            "date" : commit_obj.date.strftime("%y/%m/%d %H:%M:%S"),
            "hash" : green(commit_obj.commithash),
            "comment": commit_obj.comment,
        }

@parser.option(
    "get",
    description="You can get commit hash. Please set index.",
    argument_types={"index":int})
def copy_hash(index):
    commits = core.get_commits()
    if index > len(commits)-1:
        print yellow("this index is over length limit.")
        return

    print commits[index].commithash

@parser.option(
    "show",
    description="You can show commit difarence",
    argument_types={"index":int})
def show_diff(index):
    commits = core.get_commits()
    if index > len(commits)-1:
        print yellow("this index is over length limit.")
        return

    lines = core.shellrun("git show %s" % commits[index]).split("\n")
    for line in lines:
        if line.startswith("+"):
            print red(line)
        elif line.startswith("-"):
            print green(line)
        elif line.startswith("diff") or line.startswith("index"):
            print yellow(line)
        else:
            print line

if __name__ == "__main__":
    if core.check_exist_repo() == False:
        print yellow("There is not git repository!")
        miniparser.kill(1)

    parser.parse()
<|endoftext|>


<commit_before>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)

    d['number_of_rows'] =
    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<commit_msg>auto commit<commit_after>import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
    # https://data.cityofnewyork.us/resource/xah7-gu5w.json
    # use permalink
    d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
    # ?$select=count(*)&$$app_token=%s'
    count_url = '%s?$select=count(*)&$$app_token=%s' % (d['api_url'],
    d['number_of_rows'] =
    modified_data.append(d)
print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<|endoftext|>


<commit_before><commit_msg>Send_email function added<commit_after>"""library functions for pipelines
"""

#--- standard library imports
#
import os
import sys
import hashlib
import subprocess
import logging
import shutil
from datetime import datetime
import smtplib
from getpass import getuser

#--- third-party imports
#
import yaml

#--- project specific imports
#/

__author__ = "Andreas Wilm"
__email__ = "wilma@gis.a-star.edu.sg"
__copyright__ = "2016 Genome Institute of Singapore"
__license__ = "The MIT License (MIT)"

SERVER = "localhost"

# global logger
LOG = logging.getLogger()

INIT = {
    'gis': "/mnt/projects/rpd/init"
}

# FIXME hack: assuming importer is just one dir down of pipeline base dir
PIPELINE_BASEDIR = os.path.join(os.path.dirname(sys.argv[0]), "..")
# assert os.path.exists(os.path.join(PIPELINE_BASEDIR, "VERSION")), (PIPELINE_BASEDIR)
# we need functions from here in Snakefile as well, where the above breaks
if not os.path.exists(os.path.join(PIPELINE_BASEDIR, "VERSION")):
    # FIXME imported from wrong directory. Make sure everyone using PIPELINE_BASEDIR fails
    del PIPELINE_BASEDIR

def get_pipeline_version():
    """determine pipeline version as defined by updir file
    """
    version_file = os.path.abspath(os.path.join(PIPELINE_BASEDIR, "VERSION"))
    with open(version_file) as fh:
        version = fh.readline().strip()
    return version

def testing_is_active():
    """checks whether this is a developers version of production
    """
    check_file = os.path.abspath(os.path.join(PIPELINE_BASEDIR, "DEVELOPERS_VERSION"))
    #LOG.debug("check_file = {}".format(check_file))
    return os.path.exists(check_file)

def get_site():
    """Determine site where we're running. Throws ValueError if unknown
    """
    # this is a bit naive... but socket.getfqdn() is also useless
    if os.path.exists("/mnt/projects/rpd/") and os.path.exists("/mnt/software"):
        return "gis"
    else:
        raise ValueError("unknown site")

def get_init_call():
    """return dotkit init call
    """
    site = get_site()
    try:
        cmd = [INIT[get_site()]]
    except KeyError:
        raise ValueError("unknown or unconfigured or site {}".format(site))

    if testing_is_active():
        cmd.append('-d')

    return cmd

def get_rpd_vars():
    """Read RPD variables set by calling and parsing output from init
    """

    cmd = get_init_call()
    try:
        res = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError:
        LOG.fatal("Couldn't call init as '{}'".format(' '.join(cmd)))
        raise

    rpd_vars = dict()
    for line in res.decode().splitlines():
        if line.startswith('export '):
            line = line.replace("export ", "")
            line = ''.join([c for c in line if c not in '";\''])
            #LOG.debug("line = {}".format(line))
            k, v = line.split('=')
            rpd_vars[k.strip()] = v.strip()
    return rpd_vars

def hash_for_fastq(fq1, fq2=None):
    """return hash for one or two fastq files based on filename only
    """
    m = hashlib.md5()
    m.update(fq1.encode())
    if fq2:
        m.update(fq2.encode())
    return m.hexdigest()

def write_dk_init(rc_file, overwrite=False):
    """creates dotkit init rc
    """
    if not overwrite:
        assert not os.path.exists(rc_file), rc_file
    with open(rc_file, 'w') as fh:
        fh.write("eval `{}`;\n".format(' '.join(get_init_call())))

def write_snakemake_init(rc_file, overwrite=False):
    """creates file which loads snakemake
    """
    if not overwrite:
        assert not os.path.exists(rc_file), rc_file
    with open(rc_file, 'w') as fh:
        fh.write("# initialize snakemake. requires pre-initialized dotkit\n")
        fh.write("reuse -q miniconda-3\n")
        #fh.write("source activate snakemake-3.5.4\n")
        #fh.write("source activate snakemake-ga622cdd-onstart\n")
        #fh.write("source activate snakemake-3.5.5-onstart\n")
        fh.write("source activate snakemake-3.5.5-g9752cd7-catch-logger-cleanup\n")

def write_snakemake_env(rc_file, config, overwrite=False):
    """creates file for use as bash prefix within snakemake
    """
    if not overwrite:
        assert not os.path.exists(rc_file), rc_file

    with open(rc_file, 'w') as fh_rc:
        fh_rc.write("# used as bash prefix within snakemake\n\n")
        fh_rc.write("# init dotkit\n")
        fh_rc.write("source dk_init.rc\n\n")

        fh_rc.write("# load modules\n")
        with open(config) as fh_cfg:
            yaml_data = yaml.safe_load(fh_cfg)
            assert "modules" in yaml_data
            for k, v in yaml_data["modules"].items():
                fh_rc.write("reuse -q {}\n".format("{}-{}".format(k, v)))

        fh_rc.write("\n")
        fh_rc.write("# unofficial bash strict has to come last\n")
        fh_rc.write("set -euo pipefail;\n")

def write_cluster_config(outdir, basedir, force_overwrite=False):
    """writes site dependend cluster config

    basedir is where to find the input template (i.e. the pipeline directory)
    """
    cluster_config_in = os.path.join(basedir, "cluster.{}.yaml".format(get_site()))
    cluster_config_out = os.path.join(outdir, "cluster.yaml")

    assert os.path.exists(cluster_config_in)
    if not force_overwrite:
        assert not os.path.exists(cluster_config_out), cluster_config_out

    shutil.copyfile(cluster_config_in, cluster_config_out)

def generate_timestamp():
    """generate ISO8601 timestamp incl microsends, but with colons
    replaced to avoid problems if used as file name
    """
    return datetime.isoformat(datetime.now()).replace(":", "-")

def get_machine_run_flowcell_id(runid_and_flowcellid):
    """return machine-id, run-id and flowcell-id from full string
    """

    runid, flowcellid = runid_and_flowcellid.split("_")
    machineid = runid.split("-")[0]
    return machineid, runid, flowcellid

def send_status_mail(pipeline_name, status, id):
    """main function"""
    print (pipeline_name + status + id)
    user_name = getuser()

    FROM = "localhost"

    if user_name == "userrig":
        # FIXME rpd@mailmain in future
        TO = ["veeravallil@gis.a-star.edu.sg"] # must be a list
    else:
        # testing at NSCC
        TO = ["{}@gis.a-star.edu.sg".format(user_name)]

    SUBJECT = "Pipeline {} completed with status {} for {}".format(
        pipeline_name, status, id)

    body = "Pipeline {} completed with status {} for {}".format(
        pipeline_name, status, id)
    body += "\nPlease check {}".format(os.getcwd())

    # FIXME add later: - send questions to rpd@mailman
    #                  - attach report on success etc
    message = """\
    From: %s
    To: %s
    Subject: %s

    %s
    """ % (FROM, ", ".join(TO), SUBJECT, body)

    # Send the mail
    try:
        server = smtplib.SMTP(SERVER)
        server.sendmail(FROM, TO, message)
        server.quit()
    except Exception:
        LOG.fatal("Sending mail failed")
        # FIXME consider exit 0 if pipeline breaks
        sys.exit(1)<|endoftext|>