<commit_before>#!/usr/bin/env dumbo

"""
Householder mrtsqr

Austin R. Benson (arbenson@stanford.edu)
David F. Gleich
Copyright (c) 2012
"""

import struct
import numpy
import math

import util
import mrmc
import sys

import dumbo
import dumbo.backends.common
from dumbo import opt

def parse_info(f):
  PICKED_SET = '(picked_set) '
  ALPHA = '(alpha) '
  TAU = '(tau) '
  SIGMA = '(sigma) '

  info = {PICKED_SET: None, ALPHA: None, TAU: None, SIGMA: None}

  # TODO(arbenson): This is ugly.  The file is small and specific, so things
  # are annoying to parse.  Make this cleaner.
  for key in info:
    info_data = open(f, 'r')
    for line in info_data:
      if len(line) > len(key) and line[:len(key)] == key:
        # Handle doubles
        if key != PICKED_SET:
          info[key] = float(line[len(key):])
        else:
          line = line[len(key):]
          line = line.lstrip('[').rstrip(']\n').split(',')
          line = [k.lstrip().lstrip('\'').rstrip('\'') for k in line]
          info[key] = line
    info_data.close()

  return (info[PICKED_SET], info[ALPHA], info[TAU], info[SIGMA])

@opt("getpath", "yes")
class HouseholderMap1(mrmc.MatrixHandler):
  def __init__(self, step, last_step, info_file=None, w_file=None, blocksize=500):
    mrmc.MatrixHandler.__init__(self)
    self.data = 0.0
    self.A_data = []

    # step of alg: i = 0, 1, ..., n-1
    self.step = step
    self.first_step = (step == 0)
    self.last_step = last_step

    if self.first_step:
      self.picked_set = []
    else:
      self.picked_set, self.alpha, self.tau, self.sigma = parse_info(info_file)
      self.last_picked = self.picked_set[-1]
      self.w = self.parse_w(w_file)

    self.nrows = 0
    self.blocksize = blocksize
    self.first_key = None
    self.first_value = None

  def parse_w(self, f):
    w = []
    for line in open(f, 'r'):
      ind = line.find(')') + 2
      w.append(float(line[ind:]))
    return w

  def store(self, key, value):
    if self.last_step:
      if key in self.picked_set:
        self.A_data.append((key, value))
    else:
      if key not in self.picked_set:
        if self.first_key is None:
          self.first_key = key
          self.first_value = value[self.step]
        else:
          self.data += value[self.step] * value[self.step]
      self.A_data.append((key, value))

  def collect(self, key, value):
    new_row = [float(v) for v in value]
    if not self.first_step:
      if key == self.last_picked:
        new_row[self.step - 1] = self.alpha
      elif key not in self.picked_set:
        new_row[self.step - 1] *= self.sigma

    if not self.first_step:
      last_picked = self.picked_set[-1]
      w = self.w[self.step:]
      if key in self.picked_set:
        if key != last_picked:
          for i, val in enumerate(w):
            new_row[self.step + i] -= val * self.tau
      else:
        for i, val in enumerate(w):
          new_row[self.step + i] -= val * self.tau * new_row[self.step - 1]

    self.nrows += 1
    # write status updates so Hadoop doesn't complain
    if self.nrows % 50000 == 0:
      self.counters['rows processed'] += 50000
    self.store(key, new_row)

  def flush(self, last_flush=False):
    for pair in self.A_data:
      key, value = pair
      if not self.last_step:
        value = struct.pack('d' * len(value), *value)
      yield ('A_matrix', key), value
    self.A_data = []

    if not self.last_step and last_flush:
      if self.first_value is not None:
        yield ('KV_output', self.first_key), [self.first_value, '']
      if self.data != 0.0:
        key = numpy.random.randint(0, 4000000000)
        yield ('KV_output', key), [self.data]

    self.nrows = 0

  def __call__(self, data):
    for key, value in data:
      self.collect_data_instance(key, value)
      if self.nrows == self.blocksize:
        for key, val in self.flush():
          yield key, val
    for key, val in self.flush(last_flush=True):
      yield key, val

class HouseholderRed2(dumbo.backends.common.MapRedBase):
  def __init__(self, step, info_file):
    if step == 0:
      self.picked_set = []
    else:
      self.picked_set, _, _, _ = parse_info(info_file)
    self.picked = None
    self.alpha = None
    self.norm = 0.0

  def collect(self, key, value, picked_possibility=False):
    if picked_possibility:
      if self.picked is None:
        self.picked = key
        self.alpha = value
      self.norm += value * value
    else:
      self.norm += value

  def close(self):
    beta = math.sqrt(self.norm)
    beta = float(-1.0) * math.copysign(beta, self.alpha)
    tau = (beta - self.alpha) / beta
    sigma = float(1.0) / (self.alpha - beta)
    self.alpha = beta
    self.picked_set.append(self.picked)
    yield 'picked_set', self.picked_set
    yield 'alpha', self.alpha
    yield 'tau', tau
    yield 'sigma', sigma

  def __call__(self, data):
    for key, values in data:
      for value in values:
        self.collect(key, float(value[0]), len(value) == 2)

    for key, val in self.close():
      yield key, val

class HouseholderMap3(mrmc.MatrixHandler):
  def __init__(self, step, info_file):
    mrmc.MatrixHandler.__init__(self)
    self.data = []
    self.keys = []
    self.output_keys = []
    self.output_vals = []
    self.step = step
    self.picked_set, self.alpha, _, self.sigma = parse_info(info_file)
    self.last_picked = self.picked_set[-1]
    self.nrows = 0

  def collect(self, key, value):
    self.keys.append(key)
    value = list(value)
    if key == self.last_picked:
      value[self.step] = self.alpha
    elif key not in self.picked_set:
      value[self.step] *= self.sigma
    self.data.append(value)
    self.nrows += 1

    # write status updates so Hadoop doesn't complain
    if self.nrows % 50000 == 0:
      self.counters['rows processed'] += 50000

  def close(self):
    for i, row in enumerate(self.data):
      key = self.keys[i]
      if key in self.picked_set:
        if key != self.last_picked:
          continue
        else:
          self.output_keys += range(self.step + 1, self.ncols)
          for val in row[self.step + 1:]:
            self.output_vals.append(val)
      else:
        self.output_keys += range(self.step + 1, self.ncols)
        mult = row[self.step]
        for val in row[self.step + 1:]:
          self.output_vals.append(mult * val)

  def __call__(self, data):
    self.collect_data(data)
    self.close()

    # output the key, value pairs for the reduce
    assert(len(self.output_keys) == len(self.output_vals))
    for i, key in enumerate(self.output_keys):
      yield key, self.output_vals[i]

class Householder4(dumbo.backends.common.MapRedBase):
  def __init__(self, isreducer):
    self.isreducer = isreducer
    if not self.isreducer:
      self.data = {}

  def __call__(self, data):
    if not self.isreducer:
      for key, value in data:
        if key not in self.data:
          self.data[key] = value
        else:
          self.data[key] += value
      for key in self.data:
        yield key, self.data[key]
    else:
      for key, values in data:
        yield key, sum(values)
<commit_msg>more optimizations<commit_after>#!/usr/bin/env dumbo

"""
Householder mrtsqr

Austin R. Benson (arbenson@stanford.edu)
David F. Gleich
Copyright (c) 2012
"""

import struct
import numpy
import math

import util
import mrmc
import sys

import dumbo
import dumbo.backends.common
from dumbo import opt

def parse_info(f):
  PICKED_SET = '(picked_set) '
  ALPHA = '(alpha) '
  TAU = '(tau) '
  SIGMA = '(sigma) '

  info = {PICKED_SET: None, ALPHA: None, TAU: None, SIGMA: None}

  # TODO(arbenson): This is ugly.  The file is small and specific, so things
  # are annoying to parse.  Make this cleaner.
  for key in info:
    info_data = open(f, 'r')
    for line in info_data:
      if len(line) > len(key) and line[:len(key)] == key:
        # Handle doubles
        if key != PICKED_SET:
          info[key] = float(line[len(key):])
        else:
          line = line[len(key):]
          line = line.lstrip('[').rstrip(']\n').split(',')
          line = [k.lstrip().lstrip('\'').rstrip('\'') for k in line]
          info[key] = line
    info_data.close()

  return (info[PICKED_SET], info[ALPHA], info[TAU], info[SIGMA])

@opt("getpath", "yes")
class HouseholderMap1(mrmc.MatrixHandler):
  def __init__(self, step, last_step, info_file=None, w_file=None, blocksize=500):
    mrmc.MatrixHandler.__init__(self)
    self.data = 0.0
    self.A_data = []

    # step of alg: i = 0, 1, ..., n-1
    self.step = step
    self.first_step = (step == 0)
    self.last_step = last_step

    if self.first_step:
      self.picked_set = []
    else:
      self.picked_set, self.alpha, self.tau, self.sigma = parse_info(info_file)
      self.last_picked = self.picked_set[-1]
      self.w = self.parse_w(w_file)

    self.nrows = 0
    self.blocksize = blocksize
    self.first_key = None
    self.first_value = None

  def parse_w(self, f):
    w = []
    for line in open(f, 'r'):
      ind = line.find(')') + 2
      w.append(float(line[ind:]))
    return w

  def store(self, key, value):
    if self.last_step:
      if key in self.picked_set:
        self.A_data.append((key, value))
    else:
      if key not in self.picked_set:
        if self.first_key is None:
          self.first_key = key
          self.first_value = value[self.step]
        else:
          self.data += value[self.step] * value[self.step]
      self.A_data.append((key, value))

  def collect(self, key, value):
    new_row = [float(v) for v in value]
    if not self.first_step:
      if key == self.last_picked:
        new_row[self.step - 1] = self.alpha
      elif key not in self.picked_set:
        new_row[self.step - 1] *= self.sigma

    if not self.first_step:
      last_picked = self.picked_set[-1]
      w = self.w[self.step:]
      if key in self.picked_set:
        if key != last_picked:
          for i, val in enumerate(w):
            new_row[self.step + i] -= val * self.tau
      else:
        for i, val in enumerate(w):
          new_row[self.step + i] -= val * self.tau * new_row[self.step - 1]

    self.nrows += 1
    # write status updates so Hadoop doesn't complain
    if self.nrows % 50000 == 0:
      self.counters['rows processed'] += 50000
    self.store(key, new_row)

  def flush(self, last_flush=False):
    for pair in self.A_data:
      key, value = pair
      if not self.last_step:
        value = struct.pack('d' * len(value), *value)
      yield ('A_matrix', key), value
    self.A_data = []

    if not self.last_step and last_flush:
      if self.first_value is not None:
        yield ('KV_output', self.first_key), [self.first_value, '']
      if self.data != 0.0:
        key = numpy.random.randint(0, 4000000000)
        yield ('KV_output', key), [self.data]

    self.nrows = 0

  def __call__(self, data):
    for key, value in data:
      self.collect_data_instance(key, value)
      if self.nrows == self.blocksize:
        for key, val in self.flush():
          yield key, val
    for key, val in self.flush(last_flush=True):
      yield key, val

class HouseholderRed2(dumbo.backends.common.MapRedBase):
  def __init__(self, step, info_file):
    if step == 0:
      self.picked_set = []
    else:
      self.picked_set, _, _, _ = parse_info(info_file)
    self.picked = None
    self.alpha = None
    self.norm = 0.0

  def collect(self, key, value, picked_possibility=False):
    if picked_possibility:
      if self.picked is None:
        self.picked = key
        self.alpha = value
      self.norm += value * value
    else:
      self.norm += value

  def close(self):
    beta = math.sqrt(self.norm)
    beta = float(-1.0) * math.copysign(beta, self.alpha)
    tau = (beta - self.alpha) / beta
    sigma = float(1.0) / (self.alpha - beta)
    self.alpha = beta
    self.picked_set.append(self.picked)
    yield 'picked_set', self.picked_set
    yield 'alpha', self.alpha
    yield 'tau', tau
    yield 'sigma', sigma

  def __call__(self, data):
    for key, values in data:
      for value in values:
        self.collect(key, float(value[0]), len(value) == 2)

    for key, val in self.close():
      yield key, val

class HouseholderMap3(mrmc.MatrixHandler):
  def __init__(self, step, info_file):
    mrmc.MatrixHandler.__init__(self)
    self.data = []
    self.keys = []
    #    self.output_keys =
    self.output_vals = {}
    self.step = step
    self.picked_set, self.alpha, _, self.sigma = parse_info(info_file)
    self.last_picked = self.picked_set[-1]
    self.nrows = 0

  def collect(self, key, value):
    self.keys.append(key)
    value = list(value)
    if key == self.last_picked:
      value[self.step] = self.alpha
    elif key not in self.picked_set:
      value[self.step] *= self.sigma
    self.data.append(value)
    self.nrows += 1

    # write status updates so Hadoop doesn't complain
    if self.nrows % 50000 == 0:
      self.counters['rows processed'] += 50000

  def close(self):
    for i in xrange(self.step + 1, self.ncols):
      self.output_vals[i] = 0.0
    for i, row in enumerate(self.data):
      key = self.keys[i]
      if key in self.picked_set:
        if key != self.last_picked:
          continue
        else:
          for i, val in enumerate(row[self.step + 1:]):
            k = self.step + 1 + i
            self.output_vals[k] += val
      else:
        self.output_keys += range(self.step + 1, self.ncols)
        mult = row[self.step]
        for i, val in enumerate(row[self.step + 1:]):
          k = self.step + 1 + i
          self.output_vals[k] += mult * val

  def __call__(self, data):
    self.collect_data(data)
    self.close()

    # output the key, value pairs for the reduce
    assert(len(self.output_keys) == len(self.output_vals))
    for i, key in enumerate(self.output_keys):
      yield key, self.output_vals[i]

class Householder4(dumbo.backends.common.MapRedBase):
  def __init__(self, isreducer):
    self.isreducer = isreducer
    if not self.isreducer:
      self.data = {}

  def __call__(self, data):
    if not self.isreducer:
      for key, value in data:
        if key not in self.data:
          self.data[key] = value
        else:
          self.data[key] += value
      for key in self.data:
        yield key, self.data[key]
    else:
      for key, values in data:
        yield key, sum(values)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

"""
Templatetags related to the contact form.
"""

from django import template

try:
    import honeypot
except ImportError:
    honeypot = None

register = template.Library()

@register.simple_tag
def antispam_fields():
    """
    Returns the HTML for any spam filters available.
    """
    content = ''
    if honeypot:
        t = template.Template('{% load honeypot %}{% render_honeypot_field %}')
        content += t.render(template.Context({}))
    return content
<commit_msg>Added render_contact_form templatetag.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

"""
Templatetags related to the contact form.
"""

from django import template

try:
    import honeypot
except ImportError:
    honeypot = None

register = template.Library()

@register.inclusion_tag('envelope/contact_form.html', takes_context=True)
def render_contact_form(context):
    """
    Renders the contact form which must be in the template context.
    """
    form = context['form']
    return {
        'form': form,
    }

@register.simple_tag
def antispam_fields():
    """
    Returns the HTML for any spam filters available.
    """
    content = ''
    if honeypot:
        t = template.Template('{% load honeypot %}{% render_honeypot_field %}')
        content += t.render(template.Context({}))
    return content
<|endoftext|>


<commit_before>from os.path import join as pjoin
from itertools import combinations,product,chain,permutations
from heapq import nlargest
from collections import defaultdict
import time

WORDS_FILE = 'words.txt'
BOARD_SIZE = 15
SPECIALS = {
    '2':[(2,1),(4,2),(6,4)],  # double letter
    '3':[(6,0),(3,3),(5,5)],  # triple letter
    '@':[(5,1),(7,3)],        # double word
    '#':[(3,0)],              # triple word
}
BINGO_BONUS = 35
ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
LETTER_VALUES = dict((x,1) for x in ALPHABET)
for v,ltrs in ((0, '.'), (2, 'DLNU'), (3, 'GHY'), (4, 'BCFMPW'),
               (5, 'KV'), (8, 'X'), (10, 'JQZ')):
  for l in ltrs:
    LETTER_VALUES[l] = v
WILDS = set()

def make_board(fh):
  board = [['_' for _ in xrange(BOARD_SIZE)] for _ in xrange(BOARD_SIZE)]
  mid = BOARD_SIZE//2
  #board[mid][mid] = '*'
  for bonus,coords in SPECIALS.iteritems():
    for i,j in coords:
      i2 = mid-i+mid
      j2 = mid-j+mid
      board[i][j] = board[j][i] = board[i2][j] = board[j][i2] = \
      board[i][j2] = board[j2][i] = board[i2][j2] = board[j2][i2] = bonus
  if fh:
    data = [x.strip('\n') for x in fh]
    assert len(data) == BOARD_SIZE and len(data[0]) == BOARD_SIZE
    for r,row in enumerate(data):
      for c,letter in enumerate(row):
        l = letter.upper()
        if 'A' <= l <= 'Z':
          board[r][c] = l
  return board

def word_to_string(word):
  return ''.join(x for x,r,c in word).upper()

def base_value(x,r,c):
 return LETTER_VALUES[x] if x in LETTER_VALUES else 0

def score_word(board,word):
  s = 0
  mult = 1
  for x,r,c in word:
    base_val = base_value(x,r,c)
    space = board[r][c]
    s += base_val
    if space == '2':
      s += base_val
    elif space == '3':
      s += 2*base_val
    elif space == '@':
      mult *= 2
    elif space == '#':
      mult *= 3
  return s*mult

def is_letter(x):
  return 'A' <= x <= 'Z' or 'a' <= x <= 'z'

def find_words(board,playdict,r,c):
  #TODO: DRY this up
  def find_horiz():
    ci = c
    while ci > 0 and is_letter(board[r][ci-1]):
      ci -= 1
    if ci > 0 and (r,ci-1) in playdict:
      return
    word = []
    while ci < BOARD_SIZE:
      if is_letter(board[r][ci]):
        word.append((board[r][ci],r,ci))
      elif (r,ci) in playdict:
        word.append((playdict[(r,ci)],r,ci))
      else:
        break
      ci += 1
    if len(word) >= 2:
      yield word

  def find_vert():
    ri = r
    while ri > 0 and is_letter(board[ri-1][c]):
      ri -= 1
    if ri > 0 and (ri-1,c) in playdict:
      return
    word = []
    while ri < BOARD_SIZE:
      if is_letter(board[ri][c]):
        word.append((board[ri][c],ri,c))
      elif (ri,c) in playdict:
        word.append((playdict[(ri,c)],ri,c))
      else:
        break
      ri += 1
    if len(word) >= 2:
      yield word
  return chain(find_horiz(),find_vert())

def all_words(board,play):
  pd = dict(((r,c),x) for x,r,c in play)
  return chain.from_iterable(find_words(board,pd,r,c) for x,r,c in play)

def score_play(board,words,play):
  score = 0
  for w in all_words(board,play):
    if word_to_string(w) not in words:
      return 0
    score += score_word(board,w)
  if len(play) == 7:
    score += BINGO_BONUS
  return score

def next_to_existing(board,play):
  for r,c in play:
    if r > 0 and is_letter(board[r-1][c]):
      return True
    if c > 0 and is_letter(board[r][c-1]):
      return True
    if r < BOARD_SIZE-1 and is_letter(board[r+1][c]):
      return True
    if c < BOARD_SIZE-1 and is_letter(board[r][c+1]):
      return True
    # also check if this is the first word
    if r == BOARD_SIZE//2 and c == BOARD_SIZE//2:
      return True
  return False

def powerset_nonempty(seq):
  s = list(seq)
  return chain.from_iterable(combinations(s, r) for r in xrange(1,len(s)+1))

def letter_combos(hand):
  lcs = defaultdict(set)
  if '.' not in hand:
    for combo_types in powerset_nonempty(hand):
      for combo in permutations(combo_types):
        lcs[len(combo)].add(combo)
  else:
    hands = (hand.replace('.',x,1) for x in ALPHABET.lower())
    for h in hands:
      for l,cs in letter_combos(h).iteritems():
        lcs[l].update(cs)
  return lcs

def valid_plays(board):
  # precompute possible play locations
  valid_plays = dict((n,[]) for n in xrange(1,8))
  for r,c in product(xrange(BOARD_SIZE),xrange(BOARD_SIZE)):
    if is_letter(board[r][c]):
      continue
    h_play,v_play = [],[]
    ri,ci = r,c
    for _ in xrange(7):
      h_play.append((r,ci))
      if next_to_existing(board,h_play):
        valid_plays[len(h_play)].append(h_play)
      ci += 1
      while ci < BOARD_SIZE and is_letter(board[r][ci]):
        ci += 1
      if ci == BOARD_SIZE:
        break
    for _ in xrange(7):
      v_play.append((ri,c))   # bug here, modifying v_play after appending it
      if next_to_existing(board,v_play):
        valid_plays[len(v_play)].append(v_play)
      ri += 1
      while ri < BOARD_SIZE and is_letter(board[ri][c]):
        ri += 1
      if ri == BOARD_SIZE:
        break
  return valid_plays

def valid_moves(valid_plays,hand):
  valid_moves = set()
  lcs = letter_combos(hand)
  print "got letter combos:", dict((n,len(lcs[n])) for n in lcs.iterkeys())
  for i in lcs.iterkeys():
    for letters,play in product(lcs[i],valid_plays[i]):
      move = tuple((l,r,c) for l,(r,c) in zip(letters,play))
      valid_moves.add(move)
  return valid_moves

def positive_scoring_moves(board,words,hand):
  t0 = time.time()
  plays = valid_plays(board)
  t1 = time.time()
  play_counts = dict((n,len(plays[n])) for n in plays.iterkeys())
  print "%f secs, valid play counts: %s" % (t1-t0, play_counts)
  moves = valid_moves(plays,hand)
  t2 = time.time()
  print "%f secs, valid move count:" % (t2-t1), len(moves)
  #TODO: reduce wordset size to only words containing at least one from the hand
  for move in moves:
    score = score_play(board,words,move)
    if score > 0:
      yield score,move
  t3 = time.time()
  print "%f secs, done" % (t3-t2)

def read_dictionary(path=''):
  return set(x.strip().upper() for x in open(pjoin(path,WORDS_FILE)))

def top_moves(board,wordlist,hand,n=20):
  assert 1 <= len(hand) <= 7
  moves = positive_scoring_moves(board,wordlist,hand.upper())
  for score,play in nlargest(n,moves):
    yield score, map(word_to_string,all_words(board,play)), play
<commit_msg>Simplifying somewhat<commit_after>from os.path import join as pjoin
from itertools import combinations,product,chain,permutations
from heapq import nlargest
from collections import defaultdict
import time

WORDS_FILE = 'words.txt'
BOARD_SIZE = 15
BINGO_BONUS = 35
ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
LETTER_VALUES = dict((x,1) for x in ALPHABET)
for v,ltrs in ((0, '.'), (2, 'DLNU'), (3, 'GHY'), (4, 'BCFMPW'),
               (5, 'KV'), (8, 'X'), (10, 'JQZ')):
  for l in ltrs:
    LETTER_VALUES[l] = v
WILDS = set()

def make_board(fh):
  # 2 = double letter, 3 = triple letter
  # @ = double word, # = triple word
  board = map(list, [
      "___#__3_3__#___",
      "__2__@___@__2__",
      "_2__2_____2__2_",
      "#__3___@___3__#",
      "__2___2_2___2__",
      "_@___3___3___@_",
      "3___2_____2___3",
      "___@_______@___",
      "3___2_____2___3",
      "_@___3___3___@_",
      "__2___2_2___2__",
      "#__3___@___3__#",
      "_2__2_____2__2_",
      "__2__@___@__2__",
      "___#__3_3__#___"])
  if fh:
    data = [x.strip('\n').upper() for x in fh]
    assert len(data) == BOARD_SIZE and len(data[0]) == BOARD_SIZE
    for r,row in enumerate(data):
      for c,letter in enumerate(row):
        if 'A' <= letter <= 'Z':
          board[r][c] = letter
  return board

def word_to_string(word):
  return ''.join(x for x,r,c in word).upper()

def score_word(board,word):
  s = 0
  mult = 1
  for x,r,c in word:
    base_val = LETTER_VALUES.get(x, 0)
    space = board[r][c]
    s += base_val
    if space == '2':
      s += base_val
    elif space == '3':
      s += 2*base_val
    elif space == '@':
      mult *= 2
    elif space == '#':
      mult *= 3
  return s*mult

def is_letter(x):
  return 'A' <= x <= 'Z' or 'a' <= x <= 'z'

def find_words(board,playdict,r,c):
  #TODO: DRY this up
  def find_horiz():
    ci = c
    while ci > 0 and is_letter(board[r][ci-1]):
      ci -= 1
    if ci > 0 and (r,ci-1) in playdict:
      return
    word = []
    while ci < BOARD_SIZE:
      if is_letter(board[r][ci]):
        word.append((board[r][ci],r,ci))
      elif (r,ci) in playdict:
        word.append((playdict[(r,ci)],r,ci))
      else:
        break
      ci += 1
    if len(word) >= 2:
      yield word

  def find_vert():
    ri = r
    while ri > 0 and is_letter(board[ri-1][c]):
      ri -= 1
    if ri > 0 and (ri-1,c) in playdict:
      return
    word = []
    while ri < BOARD_SIZE:
      if is_letter(board[ri][c]):
        word.append((board[ri][c],ri,c))
      elif (ri,c) in playdict:
        word.append((playdict[(ri,c)],ri,c))
      else:
        break
      ri += 1
    if len(word) >= 2:
      yield word
  return chain(find_horiz(),find_vert())

def all_words(board,play):
  pd = dict(((r,c),x) for x,r,c in play)
  return chain.from_iterable(find_words(board,pd,r,c) for x,r,c in play)

def score_play(board,words,play):
  score = 0
  for w in all_words(board,play):
    if word_to_string(w) not in words:
      return 0
    score += score_word(board,w)
  if len(play) == 7:
    score += BINGO_BONUS
  return score

def next_to_existing(board,play):
  for r,c in play:
    if r > 0 and is_letter(board[r-1][c]):
      return True
    if c > 0 and is_letter(board[r][c-1]):
      return True
    if r < BOARD_SIZE-1 and is_letter(board[r+1][c]):
      return True
    if c < BOARD_SIZE-1 and is_letter(board[r][c+1]):
      return True
    # also check if this is the first word
    if r == BOARD_SIZE//2 and c == BOARD_SIZE//2:
      return True
  return False

def powerset_nonempty(seq):
  s = list(seq)
  return chain.from_iterable(combinations(s, r) for r in xrange(1,len(s)+1))

def letter_combos(hand):
  lcs = defaultdict(set)
  if '.' not in hand:
    for combo_types in powerset_nonempty(hand):
      for combo in permutations(combo_types):
        lcs[len(combo)].add(combo)
  else:
    hands = (hand.replace('.',x,1) for x in ALPHABET.lower())
    for h in hands:
      for l,cs in letter_combos(h).iteritems():
        lcs[l].update(cs)
  return lcs

def valid_plays(board):
  # precompute possible play locations
  valid_plays = dict((n,[]) for n in xrange(1,8))
  for r,c in product(xrange(BOARD_SIZE),xrange(BOARD_SIZE)):
    if is_letter(board[r][c]):
      continue
    h_play,v_play = [],[]
    ri,ci = r,c
    for _ in xrange(7):
      h_play.append((r,ci))
      if next_to_existing(board,h_play):
        valid_plays[len(h_play)].append(h_play)
      ci += 1
      while ci < BOARD_SIZE and is_letter(board[r][ci]):
        ci += 1
      if ci == BOARD_SIZE:
        break
    for _ in xrange(7):
      v_play.append((ri,c))   # bug here, modifying v_play after appending it
      if next_to_existing(board,v_play):
        valid_plays[len(v_play)].append(v_play)
      ri += 1
      while ri < BOARD_SIZE and is_letter(board[ri][c]):
        ri += 1
      if ri == BOARD_SIZE:
        break
  return valid_plays

def valid_moves(valid_plays,hand):
  valid_moves = set()
  lcs = letter_combos(hand)
  print "got letter combos:", dict((n,len(lcs[n])) for n in lcs.iterkeys())
  for i in lcs.iterkeys():
    for letters,play in product(lcs[i],valid_plays[i]):
      move = tuple((l,r,c) for l,(r,c) in zip(letters,play))
      valid_moves.add(move)
  return valid_moves

def positive_scoring_moves(board,words,hand):
  t0 = time.time()
  plays = valid_plays(board)
  t1 = time.time()
  play_counts = dict((n,len(plays[n])) for n in plays.iterkeys())
  print "%f secs, valid play counts: %s" % (t1-t0, play_counts)
  moves = valid_moves(plays,hand)
  t2 = time.time()
  print "%f secs, valid move count:" % (t2-t1), len(moves)
  #TODO: reduce wordset size to only words containing at least one from the hand
  for move in moves:
    score = score_play(board,words,move)
    if score > 0:
      yield score,move
  t3 = time.time()
  print "%f secs, done" % (t3-t2)

def read_dictionary(path=''):
  return set(x.strip().upper() for x in open(pjoin(path,WORDS_FILE)))

def top_moves(board,wordlist,hand,n=20):
  assert 1 <= len(hand) <= 7
  moves = positive_scoring_moves(board,wordlist,hand.upper())
  for score,play in nlargest(n,moves):
    yield score, map(word_to_string,all_words(board,play)), play
<|endoftext|>


<commit_before>VERSION = '0.9.dev'
<commit_msg>Update version.py<commit_after>VERSION = '0.9.4'
<|endoftext|>


<commit_before>
from bs4 import BeautifulSoup
import urllib2
import nltk
import operator
class uTaboo:

    def __init__(self):
        pass

    def pickWord(file):
        return pickedWord
        pass

    def getGoogledURLS(pickedWord):
        return urls
        pass

    def fetchHTML(self,url):
    	req = urllib2.Request('http://www.google.com')
    	response = urllib2.urlopen(req)
    	rawHtml = response.read()
        return rawHtml
        pass

     	#used ntlk to remove for now. Use beautiful soup to do that as well. Details in this link
    	# http://stackoverflow.com/questions/22799990/beatifulsoup4-get-text-still-has-javascript
    def stripScript(self,rawHtml):
     	cleanedHtml=nltk.clean_html(rawHtml)
     	return cleanedHtml
     	pass

    def getAllhtmls(urls):
        #use fetchHTML(url) to obtain a concatenated list of htmldata
        pass

    def html2Text(htmlData):
    	soup = BeautifulSoup(htmlData)
        return soup.get_text()
        pass

    def getUnprocessed(pickedWord):
    	myUrls = getGoogledURLS(pickedWord)
    	textData=""
    	for myUrl in myUrls:
    		rawHtml = self.fetchHTML(myUrl)
    		neathtml=self.stripScript(rawHtml)
    		textData=textData+" " + self.html2Text(neathtml)

        return textData

    # Second Module

    def filterStopWords(listOfWords):
        pass

    def getRankedList(self,filteredListofWords):
        rankedListOfWords={}
        for i in filteredListofWords:
            rankedListOfWords[i]=0
        for i in filteredListofWords:
            rankedListOfWords[i]=rankedListOfWords[i]+1
        return rankedListOfWords

    def fetchTop6(rankedListOfWords):
        ranked = sorted(rankedListOfWords.iteritems(), key=operator.itemgetter(1),reverse=True)
        keys=[x for (x,y) in ranked]
        print keys[0:5]

    def getTabooWords(word,top6):
        #return a dictionary with word as key and a list of 6 words as value
        pass

x = uTaboo()
print x.getRankedList(['hello','shinoy','smrithi','vishnu' , 'shinoy','radhika', 'shinoy','vishnu','vishnu'])<commit_msg>updated getTabooWords, inserted function calls inside. getTabooWords now has tabooWords as 6 taboo words.<commit_after>
from bs4 import BeautifulSoup
import urllib2
import nltk
import operator
class uTaboo:

    def __init__(self):
        pass

    def pickWord(file):
        return pickedWord
        pass

    def getGoogledURLS(pickedWord):
        return urls
        pass

    def fetchHTML(self,url):
    	req = urllib2.Request('http://www.google.com')
    	response = urllib2.urlopen(req)
    	rawHtml = response.read()
        return rawHtml
        pass

     	#used ntlk to remove for now. Use beautiful soup to do that as well. Details in this link
    	# http://stackoverflow.com/questions/22799990/beatifulsoup4-get-text-still-has-javascript
    def stripScript(self,rawHtml):
     	cleanedHtml=nltk.clean_html(rawHtml)
     	return cleanedHtml
     	pass

    def getAllhtmls(urls):
        #use fetchHTML(url) to obtain a concatenated list of htmldata
        pass

    def html2Text(htmlData):
    	soup = BeautifulSoup(htmlData)
        return soup.get_text()
        pass

    def getUnprocessed(pickedWord):
    	myUrls = getGoogledURLS(pickedWord)
    	textData=""
    	for myUrl in myUrls:
    		rawHtml = self.fetchHTML(myUrl)
    		neathtml=self.stripScript(rawHtml)
    		textData=textData+" " + self.html2Text(neathtml)

        return textData

    # Second Module

    def filterStopWords(listOfWords):

        return filteredListofWords

    def getRankedList(self,filteredListofWords):
        rankedListOfWords={}
        for i in filteredListofWords:
            rankedListOfWords[i]=0
        for i in filteredListofWords:
            rankedListOfWords[i]=rankedListOfWords[i]+1
        return rankedListOfWords

    def fetchTop6(rankedListOfWords):
        ranked = sorted(rankedListOfWords.iteritems(), key=operator.itemgetter(1),reverse=True)
        tabooWords=[x for (x,y) in ranked]
        return tabooWords

    def getTabooWords(word,listOfWords):
        filteredListofWords=self.filterStopWords(listOfWords)
        rankedListOfWords=self.getRankedList(filteredListofWords)
        tabooWords=fetchTop6(rankedListOfWords)

        #return a dictionary with word as key and a list of 6 words as value
        pass

x = uTaboo()
print x.getRankedList(['hello','shinoy','smrithi','vishnu' , 'shinoy','radhika', 'shinoy','vishnu','vishnu'])<|endoftext|>


<commit_before>#
# setup.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
from setuptools import setup, find_packages
from distutils.core import Command
import sys, os

VERSION = "1.0beta"
PREFIX= ""
DESCRIPTION="""\
** Archipel Server **

Copyright (c) 2011 Antoine Mercadal

This package contains the agent you need to install on your hypervisor
in order to use them with Archipel. You need a running XMPP server in
order to use Archipel and a recent version of Libvirt.

For more information, please go to http://archipelproject.org
"""

WORKINGFOLDERS  = { "drives"    : "%s/vm/drives"  % PREFIX,
                    "iso"       : "%s/vm/iso"     % PREFIX,
                    "repo"      : "%s/vm/repo"    % PREFIX,
                    "tmp"       : "%s/vm/tmp"     % PREFIX,
                    "vmcasts"   : "%s/vm/vmcasts" % PREFIX}

def create_avatar_list(folder):
    ret = []
    for avatar in os.listdir(folder):
        ret.append("%s%s" % (folder, avatar))
    return ret

setup(name='archipel-server',
      version=VERSION,
      description="The hypervisor's agent part of Archipel",
      long_description=DESCRIPTION,
      classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Environment :: No Input/Output (Daemon)',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: End Users/Desktop',
        'Intended Audience :: Science/Research',
        'Intended Audience :: System Administrators',
        'Intended Audience :: Telecommunications Industry',
        'License :: OSI Approved :: GNU Affero General Public License v',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Topic :: Internet',
        'Topic :: System :: Emulators',
        'Topic :: System :: Operating System'],
      keywords='archipel, virtualization, libvirt, orchestration',
      author='Antoine Mercadal',
      author_email='primalmotion@archipelproject.org',
      url='http://archipelproject.org',
      license='AGPLv3',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      include_empty_folders=True,
      zip_safe=False,
      provides=["archipel"],
      install_requires=[
        "xmpppy>=0.5.0rc1",
        "sqlobject>=0.14.1",
        "apscheduler>=1.3.1"
      ],
      entry_points="""
      # -*- Entry points: -*-
      """,
      scripts = [
        'install/bin/arch-importvirtualmachine',
        'install/bin/arch-rolesnode',
        'install/bin/arch-tagnode',
        'install/bin/arch-updatedomain',
        'install/bin/runarchipel',
        ],
      data_files=[
        ('%s/var/lib/archipel/avatars'  % PREFIX, create_avatar_list("install/var/lib/archipel/avatars/")),
        ('%s/var/lib/archipel/'         % PREFIX, ['install/var/lib/archipel/names.txt']),
        ('%s/etc/init.d'                % PREFIX, ['install/etc/init.d/archipel']),
        ('%s/etc/archipel/'             % PREFIX, ['install/etc/archipel/archipel.conf', 'install/etc/archipel/vnc.pem'])
        ]
      )
<commit_msg>clean up setup.py<commit_after>#
# setup.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
from setuptools import setup, find_packages

VERSION = "1.0beta"
DESCRIPTION="""\
** Archipel Server **

Copyright (c) 2011 Antoine Mercadal

This package contains the agent you need to install on your hypervisor
in order to use them with Archipel. You need a running XMPP server in
order to use Archipel and a recent version of Libvirt.

For more information, please go to http://archipelproject.org
"""

def create_avatar_list(folder):
    ret = []
    for avatar in os.listdir(folder):
        ret.append("%s%s" % (folder, avatar))
    return ret

setup(name='archipel-server',
      version=VERSION,
      description="The hypervisor's agent part of Archipel",
      long_description=DESCRIPTION,
      classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Environment :: No Input/Output (Daemon)',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: End Users/Desktop',
        'Intended Audience :: Science/Research',
        'Intended Audience :: System Administrators',
        'Intended Audience :: Telecommunications Industry',
        'License :: OSI Approved :: GNU Affero General Public License v3',
        'Operating System :: POSIX :: Linux',
        'Programming Language :: Python',
        'Topic :: Internet',
        'Topic :: System :: Emulators',
        'Topic :: System :: Operating System'],
      keywords='archipel, virtualization, libvirt, orchestration',
      author='Antoine Mercadal',
      author_email='primalmotion@archipelproject.org',
      url='http://archipelproject.org',
      license='AGPLv3',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=False,
      provides=["archipel"],
      install_requires=[
        "xmpppy>=0.5.0rc1",
        "sqlobject>=0.14.1",
        "apscheduler>=1.3.1"
      ],
      entry_points="""
      # -*- Entry points: -*-
      """,
      scripts = [
        'install/bin/arch-importvirtualmachine',
        'install/bin/arch-rolesnode',
        'install/bin/arch-tagnode',
        'install/bin/arch-updatedomain',
        'install/bin/runarchipel',
        ],
      data_files=[
        ('/var/lib/archipel/avatars', create_avatar_list("install/var/lib/archipel/avatars/")),
        ('/var/lib/archipel/'       , ['install/var/lib/archipel/names.txt']),
        ('/etc/init.d'              , ['install/etc/init.d/archipel']),
        ('/etc/archipel/'           , ['install/etc/archipel/archipel.conf', 'install/etc/archipel/vnc.pem'])
        ]
      )
<|endoftext|>


<commit_before>import hashlib
try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit
from django.core.cache import get_cache
from django.conf import settings
from django.utils.encoding import smart_text

def invalidate_paths(paths):
    '''
    invalidate that keys for certain paths that the cache middleware would
    cache that get page at.
    '''
    keys = map(get_path_key, paths)
    cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
    cache.delete_many(keys)

def get_path_key(path):
    path = urlsplit(path)[2]
    if settings.APPEND_SLASH and not path.endswith('/'):
        path += '/'
    key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    path = hashlib.md5(smart_text(path))
    cache_key = 'dumper.invalidation.invalidate_paths.{0}.{1}'.format(
        key_prefix, path.hexdigest()
    )
    return cache_key
<commit_msg>used .encode("utf-8") for path instead of django smart_text so that python 3.3 wont complain about encoding when hashing<commit_after>import hashlib
try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit
from django.core.cache import get_cache
from django.conf import settings

def invalidate_paths(paths):
    '''
    invalidate that keys for certain paths that the cache middleware would
    cache that get page at.
    '''
    keys = map(get_path_key, paths)
    cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
    cache.delete_many(keys)

def get_path_key(path):
    path = urlsplit(path)[2]
    if settings.APPEND_SLASH and not path.endswith('/'):
        path += '/'
    key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    path = hashlib.md5(path.encode('utf-8'))
    cache_key = 'dumper.invalidation.invalidate_paths.{0}.{1}'.format(
        key_prefix, path.hexdigest()
    )
    return cache_key
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect

from django.shortcuts import render, redirect, get_object_or_404
from django.db import connection
from django.contrib.auth.decorators import login_required
from .forms import ItemForm, ItemFormMark
from datetime import timedelta

# Index page, with map and markers
def index(request):
    # temporary adding event function, to be improved later

    # if this is a POST request
    if request.method == 'POST':
        if request.user.is_authenticated():
            form = ItemForm(data=request.POST)
            #check for validation
            print("About to validate form")
            if form.is_valid():
                #form.save()
                print("form is valid")
                data = form.cleaned_data
                print(data['start_time'])
                with connection.cursor() as cursors:
                    cursors.execute("INSERT INTO fs_item (user_id, tag_id, longitude, latitude, location, description, expire_type, start_time, expiration) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);",
                                    [request.user.id, data['tag_name'], data['longitude'],data['latitude'], data['location'], data['description'], data['expiration_type'], data['start_time'] - timedelta(hours=5),data['expiration'] - timedelta(hours=5)])
                return HttpResponseRedirect('/freesources/')
            else:
                print("FAIL")
                HttpResponse("Adding new item not successful")
        else:
            return HttpResponseRedirect('/accounts/login')
    elif request.method == 'GET':
        form = ItemForm()
        form_mark = ItemFormMark()
    else:
        return 0;

	#item table inner join feedback, join tag table; pass result to index page to display on map
    with connection.cursor() as cursor:
        cursor.execute("""SELECT ev.*,u.username, t.tag_name,f.feedback_type, f_rej_f.fb as rej_false, f_rej_e.fb as rej_exp, f_confirm.fb as confirm
            FROM fs_item ev
            INNER JOIN fs_item e ON ev.item_id = e.item_id
            INNER JOIN auth_user u ON ev.user_id = u.id
            INNER JOIN fs_tag t ON ev.tag_id = t.tag_id
            LEFT JOIN fs_feedback f ON f.user_id = %s AND ev.item_id = f.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'rej_false'
            GROUP BY f.item_id
            ) f_rej_f ON f_rej_f.item_id = ev.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'rej_exp'
            GROUP BY f.item_id
            ) f_rej_e ON f_rej_e.item_id = ev.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'confirm'
            GROUP BY f.item_id
            ) f_confirm ON f_confirm.item_id = ev.item_id
            ORDER BY ev.latitude, ev.longitude
            ;""",[request.user.id])

        # cursor.execute("""SELECT COUNT(*)
        #     FROM
        #     LEFT JOIN Feedback f_rej_f ON f_rej_f.feedback_type = 'rej_false' AND ev.event_id = f_rej_f.event_id
        #     LEFT JOIN Feedback f_rej_e ON f_rej_e.feedback_type = 'rej_exp' AND ev.event_id = f_rej_e.event_id
        #     LEFT JOIN Feedback f_confirm ON f_confirm.feedback_type = 'confirm' AND ev.event_id = f_confirm.event_id
        #     GROUP BY ev.event_id;
        #     """,
        #     [request.user.id])
        items = dictfetchall(cursor)

    context = {
        'items': items,
        'form': form,
        'form_mark': form_mark,
    }
    return render(request,'freesources/index.html', context)

# transfer the raw pulled database data into dictionary format
def dictfetchall(cursor):
#    "Return all rows from a cursor as a dict"
    columns = [col[0] for col in cursor.description]
    return [
        dict(zip(columns, row))
        for row in cursor.fetchall()
    ]

# feedback adding view, require user to be logged in
@login_required
def feedback(request,fd_type,item_id):
    with connection.cursor() as cursor:
        cursor.execute("DELETE FROM fs_feedback WHERE item_id = %s AND user_id = %s",
            [item_id,request.user.id])

        cursor.execute("INSERT INTO fs_feedback (item_id,feedback_type,user_id) VALUES (%s, %s, %s)",
                      [item_id,fd_type,request.user.id])

    return HttpResponseRedirect('/freesources/')

<commit_msg>add item start and expiration time but fixed - allow for empty time value for permanent item<commit_after>from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect

from django.shortcuts import render, redirect, get_object_or_404
from django.db import connection
from django.contrib.auth.decorators import login_required
from .forms import ItemForm, ItemFormMark
from datetime import timedelta

# Index page, with map and markers
def index(request):
    # temporary adding event function, to be improved later

    # if this is a POST request
    if request.method == 'POST':
        if request.user.is_authenticated():
            form = ItemForm(data=request.POST)
            #check for validation
            print("About to validate form")
            if form.is_valid():
                #form.save()
                print("form is valid")
                data = form.cleaned_data
                print(data['start_time'])
                if(data['start_time'] == 'None'):
                    start_time = data['start_time']
                else:
                    start_time = data['start_time'] - timedelta(hours=5)
                if(data['expiration'] == 'None'):
                    expiration = data['expiration']
                else:
                    expiration = data['expiration'] - timedelta(hours=5)
                with connection.cursor() as cursors:
                    cursors.execute("INSERT INTO fs_item (user_id, tag_id, longitude, latitude, location, description, expire_type, start_time, expiration) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s);",
                                    [request.user.id, data['tag_name'], data['longitude'],data['latitude'], data['location'], data['description'], data['expiration_type'], start_time,expiration])
                return HttpResponseRedirect('/freesources/')
            else:
                print("FAIL")
                HttpResponse("Adding new item not successful")
        else:
            return HttpResponseRedirect('/accounts/login')
    elif request.method == 'GET':
        form = ItemForm()
        form_mark = ItemFormMark()
    else:
        return 0;

	#item table inner join feedback, join tag table; pass result to index page to display on map
    with connection.cursor() as cursor:
        cursor.execute("""SELECT ev.*,u.username, t.tag_name,f.feedback_type, f_rej_f.fb as rej_false, f_rej_e.fb as rej_exp, f_confirm.fb as confirm
            FROM fs_item ev
            INNER JOIN fs_item e ON ev.item_id = e.item_id
            INNER JOIN auth_user u ON ev.user_id = u.id
            INNER JOIN fs_tag t ON ev.tag_id = t.tag_id
            LEFT JOIN fs_feedback f ON f.user_id = %s AND ev.item_id = f.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'rej_false'
            GROUP BY f.item_id
            ) f_rej_f ON f_rej_f.item_id = ev.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'rej_exp'
            GROUP BY f.item_id
            ) f_rej_e ON f_rej_e.item_id = ev.item_id
            LEFT JOIN (
            SELECT f.item_id, COUNT(*) as fb FROM fs_feedback f WHERE f.feedback_type = 'confirm'
            GROUP BY f.item_id
            ) f_confirm ON f_confirm.item_id = ev.item_id
            ORDER BY ev.latitude, ev.longitude
            ;""",[request.user.id])

        # cursor.execute("""SELECT COUNT(*)
        #     FROM
        #     LEFT JOIN Feedback f_rej_f ON f_rej_f.feedback_type = 'rej_false' AND ev.event_id = f_rej_f.event_id
        #     LEFT JOIN Feedback f_rej_e ON f_rej_e.feedback_type = 'rej_exp' AND ev.event_id = f_rej_e.event_id
        #     LEFT JOIN Feedback f_confirm ON f_confirm.feedback_type = 'confirm' AND ev.event_id = f_confirm.event_id
        #     GROUP BY ev.event_id;
        #     """,
        #     [request.user.id])
        items = dictfetchall(cursor)

    context = {
        'items': items,
        'form': form,
        'form_mark': form_mark,
    }
    return render(request,'freesources/index.html', context)

# transfer the raw pulled database data into dictionary format
def dictfetchall(cursor):
#    "Return all rows from a cursor as a dict"
    columns = [col[0] for col in cursor.description]
    return [
        dict(zip(columns, row))
        for row in cursor.fetchall()
    ]

# feedback adding view, require user to be logged in
@login_required
def feedback(request,fd_type,item_id):
    with connection.cursor() as cursor:
        cursor.execute("DELETE FROM fs_feedback WHERE item_id = %s AND user_id = %s",
            [item_id,request.user.id])

        cursor.execute("INSERT INTO fs_feedback (item_id,feedback_type,user_id) VALUES (%s, %s, %s)",
                      [item_id,fd_type,request.user.id])

    return HttpResponseRedirect('/freesources/')

<|endoftext|>


<commit_before>"""
--- Part Two ---

Apparently, the file actually uses version two of the format.

In version two, the only difference is that markers within decompressed data
are decompressed. This, the documentation explains, provides much more
substantial compression capabilities, allowing many-gigabyte files to be stored
in only a few kilobytes.

For example:

(3x3)XYZ still becomes XYZXYZXYZ, as the decompressed section contains no
markers.
X(8x2)(3x3)ABCY becomes XABCABCABCABCABCABCY, because the decompressed data
from the (8x2) marker is then further decompressed, thus triggering the (3x3)
marker twice for a total of six ABC sequences.
(27x12)(20x12)(13x14)(7x10)(1x12)A decompresses into a string of A repeated
241920 times.
(25x3)(3x3)ABC(2x3)XY(5x2)PQRSTX(18x9)(3x2)TWO(5x7)SEVEN becomes 445 characters
long.
Unfortunately, the computer you brought probably doesn't have enough memory to
actually decompress the file; you'll have to come up with another way to get
its decompressed length.

What is the decompressed length of the file using this improved format?
"""
from __future__ import print_function
import argparse
import re
import string

def grab_marker(line):
    track = 0
    marker = ""
    while line[track] in "(0123456789x)":
        marker += line[track]
        if line[track] == ")":
            break
        track += 1
    return marker

def count_chars(line):
    ccount = 0
    for char in line:
        if char in "ABCDEFGHIJKLMNOPQRSTUVWXYZ":
            ccount +=  1
        else:
            return ccount
    return ccount

def yikes(line, multiplier):
    global decompress_count
    i = 0
    while i < len(line):
        #print(i)
        if line[i]=="(":
            #print("MARKER START:",line[i])
            #print("LINE:",line)
            marker = grab_marker(line[i:])
            char_count, dup = [int(y) for y in marker[1:-1].split("x")]
            ddata = line[(i + len(marker)):(i + len(marker) + char_count)]
            new_multiplier = multiplier * dup
            i += len(marker) + char_count
            #print("DDATA:",ddata)
            #print("UPDATE MULTIPLIER:",new_multiplier)
            while ddata[-1] in "(0123456789x)":
                i += 1
                ddata += line[i]
            yikes(ddata, new_multiplier)
        elif line[i] in list(string.ascii_uppercase):
            lead_char_count = count_chars(line[i:])
            #print("CHARS:",line[i:i+lead_char_count],lead_char_count,multiplier)
            decompress_count += (lead_char_count * multiplier)
            i += lead_char_count
        else:
            break

parser = argparse.ArgumentParser(description='Advent of code.')
parser.add_argument('inputfile', type=argparse.FileType('r'),
                    help='Path to input file')
args = parser.parse_args()
line = args.inputfile.read().rstrip("\n")
decompress_count = 0
#count_sequences(line)
yikes(line,1)
print("FINAL:",decompress_count)
<commit_msg>made comparisons against string.ascii_uppercase consistent<commit_after>"""
--- Part Two ---

Apparently, the file actually uses version two of the format.

In version two, the only difference is that markers within decompressed data
are decompressed. This, the documentation explains, provides much more
substantial compression capabilities, allowing many-gigabyte files to be stored
in only a few kilobytes.

For example:

(3x3)XYZ still becomes XYZXYZXYZ, as the decompressed section contains no
markers.
X(8x2)(3x3)ABCY becomes XABCABCABCABCABCABCY, because the decompressed data
from the (8x2) marker is then further decompressed, thus triggering the (3x3)
marker twice for a total of six ABC sequences.
(27x12)(20x12)(13x14)(7x10)(1x12)A decompresses into a string of A repeated
241920 times.
(25x3)(3x3)ABC(2x3)XY(5x2)PQRSTX(18x9)(3x2)TWO(5x7)SEVEN becomes 445 characters
long.
Unfortunately, the computer you brought probably doesn't have enough memory to
actually decompress the file; you'll have to come up with another way to get
its decompressed length.

What is the decompressed length of the file using this improved format?
"""
from __future__ import print_function
import argparse
import re
import string

def grab_marker(line):
    track = 0
    marker = ""
    while line[track] in "(0123456789x)":
        marker += line[track]
        if line[track] == ")":
            break
        track += 1
    return marker

def count_chars(line):
    ccount = 0
    for char in line:
        if char in string.ascii_uppercase:
            ccount +=  1
        else:
            return ccount
    return ccount

def yikes(line, multiplier):
    global decompress_count
    i = 0
    while i < len(line):
        #print(i)
        if line[i]=="(":
            #print("MARKER START:",line[i])
            #print("LINE:",line)
            marker = grab_marker(line[i:])
            char_count, dup = [int(y) for y in marker[1:-1].split("x")]
            ddata = line[(i + len(marker)):(i + len(marker) + char_count)]
            new_multiplier = multiplier * dup
            i += len(marker) + char_count
            #print("DDATA:",ddata)
            #print("UPDATE MULTIPLIER:",new_multiplier)
            while ddata[-1] in "(0123456789x)":
                i += 1
                ddata += line[i]
            yikes(ddata, new_multiplier)
        elif line[i] in string.ascii_uppercase:
            lead_char_count = count_chars(line[i:])
            #print("CHARS:",line[i:i+lead_char_count],lead_char_count,multiplier)
            decompress_count += (lead_char_count * multiplier)
            i += lead_char_count
        else:
            break

parser = argparse.ArgumentParser(description='Advent of code.')
parser.add_argument('inputfile', type=argparse.FileType('r'),
                    help='Path to input file')
args = parser.parse_args()
line = args.inputfile.read().rstrip("\n")
decompress_count = 0
#count_sequences(line)
yikes(line,1)
print("FINAL:",decompress_count)
<|endoftext|>


<commit_before>import datetime
import sys

from django.db.models.sql import aggregates as sqlaggregates
from django.db.models.sql.constants import LOOKUP_SEP, MULTI, SINGLE
from django.db.models.sql.where import AND, OR
from django.db.utils import DatabaseError, IntegrityError
from django.utils.tree import Node

from functools import wraps

from google.appengine.api.datastore import Entity, Query, Put, Get, Delete, Key
from google.appengine.api.datastore_errors import Error as GAEError
from google.appengine.api.datastore_types import Text, Category, Email, Link, \
    PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, Key, \
    Rating, BlobKey

from djangotoolbox.db.basecompiler import NonrelQuery, NonrelCompiler, \
    NonrelInsertCompiler, NonrelDeleteCompiler

# Valid query types (a dictionary is used for speedy lookups).
OPERATORS_MAP = {
    'exact': '=',
    'gt': '>',
    'gte': '>=',
    'lt': '<',
    'lte': '<=',

    # The following operators are supported with special code below:
    'isnull': None,
    'startswith': None,
    'range': None,
    'year': None,

    # TODO: support these filters
    # in
}

NEGATION_MAP = {
    'gt': '<=',
    'gte': '<',
    'lt': '>=',
    'lte': '>',
    # TODO: support these filters
    #'exact': '!=', # this might actually become individual '<' and '>' queries
}

def safe_call(func):
    @wraps(func)
    def _func(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except GAEError, e:
            raise DatabaseError, DatabaseError(str(e)), sys.exc_info()[2]
    return _func

class GAEQuery(NonrelQuery):
    # ----------------------------------------------
    # Public API
    # ----------------------------------------------
    def __init__(self, compiler, fields):
        super(GAEQuery, self).__init__(compiler, fields)
        self.inequality_field = None
        self.pk_filters = []
        pks_only = False
        if len(fields) == 1 and fields[0].primary_key:
            pks_only = True
        self.gae_query = Query(self.db_table, keys_only=pks_only)

    # This is needed for debugging
    def __repr__(self):
        return '<GAEQuery: %r ORDER %r>' % (self.gae_query, self.ordering)

    @safe_call
    def fetch(self):
        query = self.gae_query
        if self.pk_filters:
            results = self.get_matching_pk(self.pk_filters)
        else:
            low_mark, high_mark = self.limits
            if high_mark is None:
                results = query.Run(offset=low_mark, prefetch_count=25,
                                    next_count=75)
            elif high_mark > low_mark:
                results = query.Get(high_mark - low_mark, low_mark)
            else:
                results = ()

        for entity in results:
            yield self._make_entity(entity, self.fields)

    @safe_call
    def count(self, limit):
        if self.pk_filters:
            return len(self.get_matching_pk(self.pk_filters))
        return self.gae_query.Count(limit)

    @safe_call
    def delete(self):
        if self.pk_filters:
            keys = [key for key in self.pk_filters if key is not None]
        else:
            keys = self.fetch()
        Delete(keys)

    @safe_call
    def order_by(self, ordering):
        self.ordering = ordering
        gae_ordering = []
        for order in self.ordering:
            if order == '?':
                raise Error("Randomized ordering isn't supported on App Engine")
            if order.startswith('-'):
                order, direction = order[1:], Query.DESCENDING
            else:
                direction = Query.ASCENDING
            if order in (self.query.get_meta().pk.column, 'pk'):
                order = '__key__'
            gae_ordering.append((order, direction))
        self.gae_query.Order(*gae_ordering)

    # This function is used by the default add_filters() implementation
    @safe_call
    def add_filter(self, column, lookup_type, negated, db_type, value):
        query = self.gae_query

        # Emulated/converted lookups
        if column == self.query.get_meta().pk.column:
            column = '__key__'
            db_table = self.query.get_meta().db_table
            if lookup_type in ('exact', 'in'):
                if self.pk_filters:
                    raise DatabaseError("You can't apply multiple AND filters "
                                        "on the primary key. "
                                        "Did you mean __in=[...]?")
                # Optimization: batch-get by key
                if negated:
                    raise DatabaseError("You can't negate equality lookups on "
                                        "the primary key.")
                if not isinstance(value, (tuple, list)):
                    value = [value]
                self.pk_filters = [create_key(db_table, pk) for pk in value if pk]
                return
            else:
                # XXX: set db_type to 'gae_key' in order to allow
                # convert_value_for_db to recognize the value to be a Key and
                # not a str. Otherwise the key would be converted back to a
                # unicode (see convert_value_for_db)
                db_type = 'gae_key'
                key_type_error = 'Lookup values on primary keys have to be' \
                                 'a string or an integer.'
                if lookup_type == 'range':
                    if isinstance(value,(list, tuple)) and not(isinstance(
                            value[0], (basestring, int, long)) and \
                            isinstance(value[1], (basestring, int, long))):
                        raise DatabaseError(key_type_error)
                elif not isinstance(value,(basestring, int, long)):
                    raise DatabaseError(key_type_error)
                # for lookup type range we have to deal with a list
                if lookup_type == 'range':
                    value[0] = create_key(db_table, value[0])
                    value[1] = create_key(db_table, value[1])
                else:
                    value = create_key(db_table, value)

        if lookup_type not in OPERATORS_MAP:
            raise DatabaseError("Lookup type %r isn't supported" % lookup_type)

        # We check for negation after lookup_type isnull because it
        # simplifies the code. All following lookup_type checks assume
        # that they're not negated.
        if lookup_type == 'isnull':
            if (negated and value) or not value:
                # TODO/XXX: is everything greater than None?
                op = '>'
            else:
                op = '='
            value = None
        elif negated:
            try:
                op = NEGATION_MAP[lookup_type]
            except KeyError:
                raise DatabaseError("Lookup type %r can't be negated" % lookup_type)
            if self.inequality_field and column != self.inequality_field:
                raise DatabaseError("Can't have inequality filters on multiple "
                    "columns (here: %r and %r)" % (self.inequality_field, column))
            self.inequality_field = column
        elif lookup_type == 'startswith':
            op = '>='
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value)
            op = '<='
            if isinstance(value, str):
                value = value.decode('utf8')
            if isinstance(value, Key):
                value = list(value.to_path())
                if isinstance(value[-1], str):
                    value[-1] = value[-1].decode('utf8')
                value[-1] += u'\ufffd'
                value = Key.from_path(*value)
            else:
                value += u'\ufffd'
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value)
            return
        elif lookup_type in ('range', 'year'):
            op = '>='
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value[0])
            op = '<=' if lookup_type == 'range' else '<'
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value[1])
            return
        else:
            op = OPERATORS_MAP[lookup_type]

        query["%s %s" % (column, op)] = self.convert_value_for_db(db_type,
            value)

    # ----------------------------------------------
    # Internal API
    # ----------------------------------------------
    @safe_call
    def _make_entity(self, entity, fields):
        if isinstance(entity, Key):
            key = entity
            entity = {}
        else:
            key = entity.key()

        entity[self.query.get_meta().pk.column] = key
        return entity

    def get_matching_pk(self, pk_filters):
        pk_filters = [key for key in pk_filters if key is not None]
        if not pk_filters:
            return []

        results = [result for result in Get(pk_filters)
                   if result is not None
                       and self.matches_filters(result)]
        if self.ordering:
            results.sort(cmp=self.order_pk_filtered)
        low_mark, high_mark = self.limits
        if high_mark is not None and high_mark < len(results) - 1:
            results = results[:high_mark]
        if low_mark:
            results = results[low_mark:]
        return results

    def order_pk_filtered(self, lhs, rhs):
        left = dict(lhs)
        left[self.query.get_meta().pk.column] = lhs.key().to_path()
        right = dict(rhs)
        right[self.query.get_meta().pk.column] = rhs.key().to_path()
        return self._order_in_memory(left, right)

    def matches_filters(self, entity):
        item = dict(entity)
        pk = self.query.get_meta().pk
        value = self.convert_value_from_db(pk.db_type(connection=self.connection),
            entity.key())
        item[pk.column] = value
        result = self._matches_filters(item, self.query.where)
        return result

class SQLCompiler(NonrelCompiler):
    """
    A simple App Engine query: no joins, no distinct, etc.
    """
    query_class = GAEQuery

    def convert_value_from_db(self, db_type, value):
        if isinstance(value, (list, tuple)) and len(value) and \
                db_type.startswith('ListField:'):
            db_sub_type = db_type.split('ListField:')[1]
            for i, val in enumerate(value):
                value[i] = self.convert_value_from_db(db_sub_type, val)

        # the following GAE database types are all unicode subclasses, cast them
        # to unicode so they appear like pure unicode instances for django
        if isinstance(value, (Category, Email, Link, PhoneNumber, PostalAddress,
                Text, unicode)):
            value = unicode(value)
        elif isinstance(value, Blob):
            value = str(value)
        elif isinstance(value, str):
            # always retrieve strings as unicode (it is possible that old datasets
            # contain non unicode strings, nevertheless work with unicode ones)
            value = value.decode('utf-8')
        elif isinstance(value, Key):
            # for now we do not support KeyFields thus a Key has to be the own
            # primary key
            # TODO: GAE: support parents via GAEKeyField
            assert value.parent() is None, "Parents are not yet supported!"
            if db_type == 'integer':
                if value.id() is None:
                    raise DatabaseError('Wrong type for Key. Expected integer, found'
                        'None')
                else:
                    value = value.id()
            elif db_type == 'text':
                if value.name() is None:
                    raise DatabaseError('Wrong type for Key. Expected string, found'
                        'None')
                else:
                    value = value.name()
            else:
                raise DatabaseError("%s fields cannot be keys on GAE" % db_type)
#        TODO: Use long in order to simulate decimal?
#        elif isinstance(value, long):
#        elif isinstance(value, Rating):
#        elif isinstance(value, users.User):
#        elif isinstance(value, BlobKey):
#        elif isinstance(value, ByteString):
#        TODO: convert GeoPt to a field used by geo-django (or some other geo
#        app for django)
#        elif isinstance(value, GeoPt):
#        elif isinstance(value, IM):

        # here we have to check the db_type because GAE always stores datetime
        # instances
        elif db_type == 'date' and isinstance(value, datetime.datetime):
            value = value.date()
        elif db_type == 'time' and isinstance(value, datetime.datetime):
            value = value.time()
        elif db_type == 'datetime' and isinstance(value, datetime.datetime):
            value = value
        return value

    def convert_value_for_db(self, db_type, value):
        if isinstance(value, unicode):
            value = unicode(value)
        elif isinstance(value, str):
            value = str(value)
        elif isinstance(value, (list, tuple)) and len(value) and \
                db_type.startswith('ListField:'):
            db_sub_type = db_type.split('ListField:')[1]
            for i, val in enumerate(value):
                value[i] = self.convert_value_for_db(db_sub_type, val)
        if db_type == 'gae_key':
            return value
        elif db_type == 'longtext':
            # long text fields cannot be indexed on GAE so use GAE's database
            # type Text
            value = Text((isinstance(value, str) and value.decode('utf-8')) or value)
        elif db_type == 'text':
            value = (isinstance(value, str) and value.decode('utf-8')) or value
        elif db_type == 'blob':
            value = Blob(value)
        elif type(value) is str:
            # always store unicode strings
            value = value.decode('utf-8')
        elif db_type == 'date' or db_type == 'time' or db_type == 'datetime':
            # here we have to check the db_type because GAE always stores datetimes
            value = to_datetime(value)
        return value

class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler):
    @safe_call
    def insert(self, data, return_id=False):
        kwds = {}
        gae_data = {}
        for column, value in data.items():
            if column == self.query.get_meta().pk.name:
                if isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store emty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(self.query.get_meta().db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)
        return key.id_or_name()

class SQLUpdateCompiler(SQLCompiler):
    def execute_sql(self, result_type=MULTI):
        # TODO: Implement me
        raise NotImplementedError('No updates')

class SQLDeleteCompiler(NonrelDeleteCompiler, SQLCompiler):
    pass

def to_datetime(value):
    """Convert a time or date to a datetime for datastore storage.

    Args:
    value: A datetime.time, datetime.date or string object.

    Returns:
    A datetime object with date set to 1970-01-01 if value is a datetime.time
    A datetime object with date set to value.year - value.month - value.day and
    time set to 0:00 if value is a datetime.date
    """

    if value is None:
        return value
    elif isinstance(value, datetime.datetime):
        return value
    elif isinstance(value, datetime.date):
        return datetime.datetime(value.year, value.month, value.day)
    elif isinstance(value, datetime.time):
        return datetime.datetime(1970, 1, 1, value.hour, value.minute,
            value.second, value.microsecond)

def create_key(db_table, value):
    if isinstance(value, (int, long)) and value < 1:
        return None
    return Key.from_path(db_table, value)
<commit_msg>simplifications<commit_after>import datetime
import sys

from django.db.models.sql import aggregates as sqlaggregates
from django.db.models.sql.constants import LOOKUP_SEP, MULTI, SINGLE
from django.db.models.sql.where import AND, OR
from django.db.utils import DatabaseError, IntegrityError
from django.utils.tree import Node

from functools import wraps

from google.appengine.api.datastore import Entity, Query, Put, Get, Delete, Key
from google.appengine.api.datastore_errors import Error as GAEError
from google.appengine.api.datastore_types import Text, Category, Email, Link, \
    PhoneNumber, PostalAddress, Text, Blob, ByteString, GeoPt, IM, Key, \
    Rating, BlobKey

from djangotoolbox.db.basecompiler import NonrelQuery, NonrelCompiler, \
    NonrelInsertCompiler, NonrelDeleteCompiler

# Valid query types (a dictionary is used for speedy lookups).
OPERATORS_MAP = {
    'exact': '=',
    'gt': '>',
    'gte': '>=',
    'lt': '<',
    'lte': '<=',

    # The following operators are supported with special code below:
    'isnull': None,
    'startswith': None,
    'range': None,
    'year': None,

    # TODO: support these filters
    # in
}

NEGATION_MAP = {
    'gt': '<=',
    'gte': '<',
    'lt': '>=',
    'lte': '>',
    # TODO: support these filters
    #'exact': '!=', # this might actually become individual '<' and '>' queries
}

def safe_call(func):
    @wraps(func)
    def _func(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except GAEError, e:
            raise DatabaseError, DatabaseError(str(e)), sys.exc_info()[2]
    return _func

class GAEQuery(NonrelQuery):
    # ----------------------------------------------
    # Public API
    # ----------------------------------------------
    def __init__(self, compiler, fields):
        super(GAEQuery, self).__init__(compiler, fields)
        self.inequality_field = None
        self.pk_filters = []
        pks_only = False
        if len(fields) == 1 and fields[0].primary_key:
            pks_only = True
        db_table = self.query.get_meta().db_table
        self.gae_query = Query(db_table, keys_only=pks_only)

    # This is needed for debugging
    def __repr__(self):
        return '<GAEQuery: %r ORDER %r>' % (self.gae_query, self.ordering)

    @safe_call
    def fetch(self):
        query = self.gae_query
        if self.pk_filters:
            results = self.get_matching_pk()
        else:
            low_mark, high_mark = self.limits
            if high_mark is None:
                results = query.Run(offset=low_mark, prefetch_count=25,
                                    next_count=75)
            elif high_mark > low_mark:
                results = query.Get(high_mark - low_mark, low_mark)
            else:
                results = ()

        for entity in results:
            yield self._make_entity(entity, self.fields)

    @safe_call
    def count(self, limit=None):
        if self.pk_filters:
            return len(self.get_matching_pk())
        return self.gae_query.Count(limit)

    @safe_call
    def delete(self):
        if self.pk_filters:
            keys = [key for key in self.pk_filters if key is not None]
        else:
            keys = self.fetch()
        Delete(keys)

    @safe_call
    def order_by(self, ordering):
        self.ordering = ordering
        gae_ordering = []
        for order in self.ordering:
            if order == '?':
                raise Error("Randomized ordering isn't supported on App Engine")
            if order.startswith('-'):
                order, direction = order[1:], Query.DESCENDING
            else:
                direction = Query.ASCENDING
            if order in (self.query.get_meta().pk.column, 'pk'):
                order = '__key__'
            gae_ordering.append((order, direction))
        self.gae_query.Order(*gae_ordering)

    # This function is used by the default add_filters() implementation
    @safe_call
    def add_filter(self, column, lookup_type, negated, db_type, value):
        query = self.gae_query

        # Emulated/converted lookups
        if column == self.query.get_meta().pk.column:
            column = '__key__'
            db_table = self.query.get_meta().db_table
            if lookup_type in ('exact', 'in'):
                if self.pk_filters:
                    raise DatabaseError("You can't apply multiple AND filters "
                                        "on the primary key. "
                                        "Did you mean __in=[...]?")
                # Optimization: batch-get by key
                if negated:
                    raise DatabaseError("You can't negate equality lookups on "
                                        "the primary key.")
                if not isinstance(value, (tuple, list)):
                    value = [value]
                self.pk_filters = [create_key(db_table, pk) for pk in value if pk]
                return
            else:
                # XXX: set db_type to 'gae_key' in order to allow
                # convert_value_for_db to recognize the value to be a Key and
                # not a str. Otherwise the key would be converted back to a
                # unicode (see convert_value_for_db)
                db_type = 'gae_key'
                key_type_error = 'Lookup values on primary keys have to be' \
                                 'a string or an integer.'
                if lookup_type == 'range':
                    if isinstance(value,(list, tuple)) and not(isinstance(
                            value[0], (basestring, int, long)) and \
                            isinstance(value[1], (basestring, int, long))):
                        raise DatabaseError(key_type_error)
                elif not isinstance(value,(basestring, int, long)):
                    raise DatabaseError(key_type_error)
                # for lookup type range we have to deal with a list
                if lookup_type == 'range':
                    value[0] = create_key(db_table, value[0])
                    value[1] = create_key(db_table, value[1])
                else:
                    value = create_key(db_table, value)

        if lookup_type not in OPERATORS_MAP:
            raise DatabaseError("Lookup type %r isn't supported" % lookup_type)

        # We check for negation after lookup_type isnull because it
        # simplifies the code. All following lookup_type checks assume
        # that they're not negated.
        if lookup_type == 'isnull':
            if (negated and value) or not value:
                # TODO/XXX: is everything greater than None?
                op = '>'
            else:
                op = '='
            value = None
        elif negated:
            try:
                op = NEGATION_MAP[lookup_type]
            except KeyError:
                raise DatabaseError("Lookup type %r can't be negated" % lookup_type)
            if self.inequality_field and column != self.inequality_field:
                raise DatabaseError("Can't have inequality filters on multiple "
                    "columns (here: %r and %r)" % (self.inequality_field, column))
            self.inequality_field = column
        elif lookup_type == 'startswith':
            op = '>='
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value)
            op = '<='
            if isinstance(value, str):
                value = value.decode('utf8')
            if isinstance(value, Key):
                value = list(value.to_path())
                if isinstance(value[-1], str):
                    value[-1] = value[-1].decode('utf8')
                value[-1] += u'\ufffd'
                value = Key.from_path(*value)
            else:
                value += u'\ufffd'
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value)
            return
        elif lookup_type in ('range', 'year'):
            op = '>='
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value[0])
            op = '<=' if lookup_type == 'range' else '<'
            query["%s %s" % (column, op)] = self.convert_value_for_db(
                db_type, value[1])
            return
        else:
            op = OPERATORS_MAP[lookup_type]

        query["%s %s" % (column, op)] = self.convert_value_for_db(db_type,
            value)

    # ----------------------------------------------
    # Internal API
    # ----------------------------------------------
    @safe_call
    def _make_entity(self, entity, fields):
        if isinstance(entity, Key):
            key = entity
            entity = {}
        else:
            key = entity.key()

        entity[self.query.get_meta().pk.column] = key
        return entity

    def get_matching_pk(self):
        pk_filters = [key for key in self.pk_filters if key is not None]
        if not pk_filters:
            return []

        results = [result for result in Get(pk_filters)
                   if result is not None
                       and self.matches_filters(result)]
        if self.ordering:
            results.sort(cmp=self.order_pk_filtered)
        low_mark, high_mark = self.limits
        if high_mark is not None and high_mark < len(results) - 1:
            results = results[:high_mark]
        if low_mark:
            results = results[low_mark:]
        return results

    def order_pk_filtered(self, lhs, rhs):
        left = dict(lhs)
        left[self.query.get_meta().pk.column] = lhs.key().to_path()
        right = dict(rhs)
        right[self.query.get_meta().pk.column] = rhs.key().to_path()
        return self._order_in_memory(left, right)

    def matches_filters(self, entity):
        item = dict(entity)
        pk = self.query.get_meta().pk
        value = self.convert_value_from_db(pk.db_type(connection=self.connection),
            entity.key())
        item[pk.column] = value
        result = self._matches_filters(item, self.query.where)
        return result

class SQLCompiler(NonrelCompiler):
    """
    A simple App Engine query: no joins, no distinct, etc.
    """
    query_class = GAEQuery

    def convert_value_from_db(self, db_type, value):
        if isinstance(value, (list, tuple)) and len(value) and \
                db_type.startswith('ListField:'):
            db_sub_type = db_type.split('ListField:')[1]
            for i, val in enumerate(value):
                value[i] = self.convert_value_from_db(db_sub_type, val)

        # the following GAE database types are all unicode subclasses, cast them
        # to unicode so they appear like pure unicode instances for django
        if isinstance(value, (Category, Email, Link, PhoneNumber, PostalAddress,
                Text, unicode)):
            value = unicode(value)
        elif isinstance(value, Blob):
            value = str(value)
        elif isinstance(value, str):
            # always retrieve strings as unicode (it is possible that old datasets
            # contain non unicode strings, nevertheless work with unicode ones)
            value = value.decode('utf-8')
        elif isinstance(value, Key):
            # for now we do not support KeyFields thus a Key has to be the own
            # primary key
            # TODO: GAE: support parents via GAEKeyField
            assert value.parent() is None, "Parents are not yet supported!"
            if db_type == 'integer':
                if value.id() is None:
                    raise DatabaseError('Wrong type for Key. Expected integer, found'
                        'None')
                else:
                    value = value.id()
            elif db_type == 'text':
                if value.name() is None:
                    raise DatabaseError('Wrong type for Key. Expected string, found'
                        'None')
                else:
                    value = value.name()
            else:
                raise DatabaseError("%s fields cannot be keys on GAE" % db_type)
#        TODO: Use long in order to simulate decimal?
#        elif isinstance(value, long):
#        elif isinstance(value, Rating):
#        elif isinstance(value, users.User):
#        elif isinstance(value, BlobKey):
#        elif isinstance(value, ByteString):
#        TODO: convert GeoPt to a field used by geo-django (or some other geo
#        app for django)
#        elif isinstance(value, GeoPt):
#        elif isinstance(value, IM):

        # here we have to check the db_type because GAE always stores datetime
        # instances
        elif db_type == 'date' and isinstance(value, datetime.datetime):
            value = value.date()
        elif db_type == 'time' and isinstance(value, datetime.datetime):
            value = value.time()
        elif db_type == 'datetime' and isinstance(value, datetime.datetime):
            value = value
        return value

    def convert_value_for_db(self, db_type, value):
        if isinstance(value, unicode):
            value = unicode(value)
        elif isinstance(value, str):
            value = str(value)
        elif isinstance(value, (list, tuple)) and len(value) and \
                db_type.startswith('ListField:'):
            db_sub_type = db_type.split('ListField:')[1]
            for i, val in enumerate(value):
                value[i] = self.convert_value_for_db(db_sub_type, val)
        if db_type == 'gae_key':
            return value
        elif db_type == 'longtext':
            # long text fields cannot be indexed on GAE so use GAE's database
            # type Text
            value = Text((isinstance(value, str) and value.decode('utf-8')) or value)
        elif db_type == 'text':
            value = (isinstance(value, str) and value.decode('utf-8')) or value
        elif db_type == 'blob':
            value = Blob(value)
        elif type(value) is str:
            # always store unicode strings
            value = value.decode('utf-8')
        elif db_type == 'date' or db_type == 'time' or db_type == 'datetime':
            # here we have to check the db_type because GAE always stores datetimes
            value = to_datetime(value)
        return value

class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler):
    @safe_call
    def insert(self, data, return_id=False):
        kwds = {}
        gae_data = {}
        for column, value in data.items():
            if column == self.query.get_meta().pk.name:
                if isinstance(value, basestring):
                    kwds['name'] = value
                else:
                    kwds['id'] = value
            elif isinstance(value, (tuple, list)) and not len(value):
                # gae does not store emty lists (and even does not allow passing empty
                # lists to Entity.update) so skip them
                continue
            else:
                gae_data[column] = value

        entity = Entity(self.query.get_meta().db_table, **kwds)
        entity.update(gae_data)
        key = Put(entity)
        return key.id_or_name()

class SQLUpdateCompiler(SQLCompiler):
    def execute_sql(self, result_type=MULTI):
        # TODO: Implement me
        raise NotImplementedError('No updates')

class SQLDeleteCompiler(NonrelDeleteCompiler, SQLCompiler):
    pass

def to_datetime(value):
    """Convert a time or date to a datetime for datastore storage.

    Args:
    value: A datetime.time, datetime.date or string object.

    Returns:
    A datetime object with date set to 1970-01-01 if value is a datetime.time
    A datetime object with date set to value.year - value.month - value.day and
    time set to 0:00 if value is a datetime.date
    """

    if value is None:
        return value
    elif isinstance(value, datetime.datetime):
        return value
    elif isinstance(value, datetime.date):
        return datetime.datetime(value.year, value.month, value.day)
    elif isinstance(value, datetime.time):
        return datetime.datetime(1970, 1, 1, value.hour, value.minute,
            value.second, value.microsecond)

def create_key(db_table, value):
    if isinstance(value, (int, long)) and value < 1:
        return None
    return Key.from_path(db_table, value)
<|endoftext|>