<commit_before># Copyright (c) 2011 Timur Iskhodzhanov and MIPT students. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import copy
import hashlib
import logging
import os
import random
import re
import sys
import Queue # Renamed to queue in 3.0
import unittest

class FakeDHT:
  # This is a fake DHT, see http://en.wikipedia.org/wiki/Distributed_hash_table
  def __init__(self):
    self.__table = {}

  # O(log N)
  #
  # TODO(timurrrr): in fact, we don't need a precise set for a given key,
  # a non-empty subset of the most recent machines would work.
  # This can also automatically fix the abscence of Remove()
  def Append(self, key, new_values):
    # will raise exception if 'values' is not a sequence
    logging.debug("FakeDHT.Append('%s', '%s')", key, new_values)
    if key not in self.__table.keys():
      self.__table[key] = new_values
    else:
      self.__table[key].extend(new_values)

  # O(log N)
  def Get(self, key):
    ret = self.__table[key]
    logging.debug("FakeDHT.Get('%s') returns '%s'", key, ret)
    return ret

class FakeP2PNetwork:
  def __init__(self):
    # will contain (key -> (receive queue) pairs
    # where receive queue holds list of (sender_id, message) pairs.
    self.__nodes = {}
    self.__availableIDs = set(["Alice", "Brian", "Colin", "David", "Ellie"])

  # Returns new node ID
  def CreateNode(self):
    new_id = random.choice(list(self.__availableIDs))
    self.__availableIDs.remove(new_id)
    self.__nodes[new_id] = Queue.Queue()
    logging.info("New node: %s" % new_id)
    return new_id

  # Returns True on success, False on failure
  # Design flaw: we can use other node's ID as sender_id
  # TODO(timurrrr): FakeSocket
  def Send(self, sender_id, to_id, message):
    assert sender_id in self.__nodes.keys()

    if to_id not in self.__nodes.keys():
      logging.error("'%s' node is unknown" % to_id)
      return False
    self.__nodes[to_id].put((sender_id, copy.deepcopy(message)))

    return True

  # Returns (from, message) pair if present (FIFO), None if no messages are
  # available.
  # Design flaw: we can use other node's ID as receiver_id
  def Receive(self, receiver_id):
    if self.__nodes[receiver_id].empty():
      return None
    return self.__nodes[receiver_id].get()

class TorrentID:
  CHUNK_SIZE = 4

  def __init__(self, contents):
    # contents should be a string
    self.length = len(contents)
    self.filehash = hashlib.sha1(contents).hexdigest()
    self.partial_hashes = []
    for chunk_id in range(self.length / TorrentID.CHUNK_SIZE + 1):
      ph = self.__CalculatePartialHash(contents, chunk_id)
      if ph != None:
        self.partial_hashes.append(ph)
    logging.info("Created torrent:\nContents='%s'\n%s" % (contents, str(self)))

  # implicitly called by str(torrent_id)
  def __str__(self):
    return "Torrent:\n  hash=%s\n  size=%d\n  partial_hashes=[\n   %s\n  ]\n" % (
           self.filehash, self.length, ",\n   ".join(self.partial_hashes))

  @staticmethod
  def GetChunkData(contents, chunk_id):
    return contents[chunk_id * TorrentID.CHUNK_SIZE :
                (chunk_id+1) * TorrentID.CHUNK_SIZE]

  @staticmethod
  def SetChunkData(contents, chunk_id, new_data):
    idx_l = chunk_id * TorrentID.CHUNK_SIZE
    idx_r = idx_l + TorrentID.CHUNK_SIZE
    return contents[:idx_l] + new_data + contents[idx_r:]

  @staticmethod
  def GetChunkHash(chunk_data):
    return hashlib.sha1(chunk_data).hexdigest()

  @staticmethod
  def __CalculatePartialHash(contents, chunk_id):
    chunk = TorrentID.GetChunkData(contents, chunk_id)
    if len(chunk) > 0:
      return TorrentID.GetChunkHash(chunk)
    return None

  def IsKnownChunk(self, contents, chunk_id):
    return self.__CalculatePartialHash(contents, chunk_id) \
            == self.partial_hashes[chunk_id]

class TorrentClient:
  def __init__(self, network, dht):
    self.__network = network
    self.__id = network.CreateNode()

    # Will store (<chunk>_key -> set(machines which know <chunk>)
    self.__dht = dht

    # Torrent hash -> (torrent, contents) dictionary.
    # 'contents' should contain '*' for unknown bytes (TODO: this is a hack)
    self.__data = {}

    # Torrents to be downloaded
    # (torrent hash -> set of missing chunk indices)
    self.__partial_torrents = {}

    # List of finished-but-not-taken torrent hashes
    self.__downloaded = []

  def GetDownloadedTorrent(self):
    # Peek and return any downloaded torrent as a (torrent, contents) tuple
    # (if present), otherwise None.
    if len(self.__downloaded) == 0:
      return None
    ret_hash = self.__downloaded[-1]
    self.__downloaded = self.__downloaded[:-1]
    return self.__data[ret_hash]

  @staticmethod
  def __ChunkKey(torrent, chunk_id):
    assert chunk_id in range(len(torrent.partial_hashes))
    return "chunk_%s_%d" % (torrent.filehash, chunk_id)

  def AddTorrent(self, torrent, known_contents=None):
    assert torrent.filehash not in self.__data.keys()
    if known_contents:
      for chunk in range(len(torrent.partial_hashes)):
        self.__dht.Append(self.__ChunkKey(torrent, chunk), [self.__id])
      print "%s: Loaded torrent   '%s'" % (self.__id, known_contents)
    else:
      known_contents = "*" * torrent.length
      self.__partial_torrents[torrent.filehash] = \
          set(range(len(torrent.partial_hashes)))

    self.__data[torrent.filehash] = (torrent, known_contents)

  def Tick(self):
    message = self.__network.Receive(self.__id)
    if message:
      self.__ProcessMessage(message)

    if len(self.__partial_torrents.keys()) > 0:
      # Select random a torrent to download a chunk
      filehash = random.choice(self.__partial_torrents.keys())
      torrent = self.__data[filehash][0]

      # ... random chunk
      needed_chunks = self.__partial_torrents[filehash]
      chunk = random.choice(list(needed_chunks))
      chunk_key = self.__ChunkKey(torrent, chunk)

      # ... random host
      chunk_available_at = random.choice(self.__dht.Get(chunk_key))

      # Ask the host to send the chunk of that torrent
      self.__network.Send(self.__id, chunk_available_at, "give_" + chunk_key)

  def __ProcessMessage(self, msg):
    (from_id, contents) = msg
    logging.debug("Node '%s' received a message '%s' from '%s'",
                  self.__id, contents, from_id)
    m = re.match("give_chunk_([0-9a-f]+)_([0-9]+)", contents)
    if m:
      # Process "give_chunk_<hash>_<chunk>" message
      (filehash, chunk_id) = m.groups()
      if filehash not in self.__data.keys():
        logging.error("Hey, I don't know such a torrent!")
        return
      chunk_id = int(chunk_id)
      (torrent, contents) = self.__data[filehash]
      if not torrent.IsKnownChunk(contents, chunk_id):
        logging.error("Hey, I don't have this chunk!")
        logging.info("Current torrent contents are:\n  '%s'" % contents)
        return
      chunk_key = self.__ChunkKey(torrent, chunk_id)
      chunk_data = TorrentID.GetChunkData(contents, chunk_id)
      self.__network.Send(self.__id, from_id,
                          "take_%s %s" % (chunk_key, chunk_data))
      return

    m = re.match("take_chunk_([0-9a-f]+)_([0-9]+) (.*)", contents)
    if not m:
      logging.error("Couldn't parse this message '%s'", msg)
      return

    # Process "take_chunk_<hash>_<chunk> <contents>" message
    (filehash, chunk_id, chunk_data) = m.groups()
    chunk_id = int(chunk_id)
    if filehash not in self.__partial_torrents.keys():
      logging.info("Hey, I didn't want this torrent!")
      return
    needed_chunks = self.__partial_torrents[filehash]

    (torrent, known_contents) = self.__data[filehash]

    if chunk_id not in needed_chunks:
      logging.info("%s: Hey, I didn't want this chunk! %d not in %s" % \
                   (self.__id, chunk_id, str(needed_chunks)))
      logging.info("Current torrent contents are:\n  '%s'" % known_contents)
      return

    if torrent.GetChunkHash(chunk_data) != torrent.partial_hashes[chunk_id]:
      logging.error("Hash mismatch!")  # Call security?
      return

    known_contents = torrent.SetChunkData(known_contents, chunk_id, chunk_data)
    self.__data[filehash] = (torrent, known_contents)
    print "%s: New contents are '%s'" % (self.__id, known_contents)

    needed_chunks.remove(chunk_id)
    chunk_key = self.__ChunkKey(torrent, chunk_id)
    self.__dht.Append(chunk_key, [self.__id])

    if len(needed_chunks) == 0:
      logging.info("Torrent #%s download finished!" % filehash)
      self.__downloaded.append(filehash)
      self.__partial_torrents.pop(filehash)
      return

class TorrentDhtDemoTest(unittest.TestCase):
  def runTest(self):
    print  # ugly hack to force a newline

    myfile = "AAAABBBBCCCCDDDDEEEEFF"
    mytorrent = TorrentID(myfile)

    network = FakeP2PNetwork()
    dht = FakeDHT()

    clients = []
    for i in range(3):
      clients.append(TorrentClient(network, dht))
      if i == 0:  # Seeder
        clients[i].AddTorrent(mytorrent, myfile)
      else:  # others
        clients[i].AddTorrent(mytorrent)

    received_file = None
    while not received_file:
      for c in clients:
        c.Tick()  # Simulate parallel execution
      received_file = clients[1].GetDownloadedTorrent()

    self.assertEqual(received_file[0], mytorrent)
    self.assertEqual(received_file[1], myfile)

# Run the test suite.
if __name__ == '__main__':
  # replace ERROR with INFO, DEBUG, etc. and re-run. Notice any changes?
  logging.basicConfig(stream=sys.stdout,
                      level=logging.ERROR,  # Don't print anything less serious
                      format="%(asctime)s [%(levelname)s] %(message)s")
  unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
<commit_msg>Works? Reviewed at working on the wiki now.<commit_after># Copyright (c) 2012 Timur Iskhodzhanov and MIPT students. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import copy
import hashlib
import logging
import os
import random
import re
import sys
import Queue # Renamed to queue in 3.0
import unittest

class FakeDHT:
  # This is a fake DHT, see http://en.wikipedia.org/wiki/Distributed_hash_table
  def __init__(self):
    self.__table = {}

  # O(log N)
  #
  # TODO(timurrrr): in fact, we don't need a precise set for a given key,
  # a non-empty subset of the most recent machines would work.
  # This can also automatically fix the abscence of Remove()
  def Append(self, key, new_values):
    # will raise exception if 'values' is not a sequence
    logging.debug("FakeDHT.Append('%s', '%s')", key, new_values)
    if key not in self.__table.keys():
      self.__table[key] = new_values
    else:
      self.__table[key].extend(new_values)

  # O(log N)
  def Get(self, key):
    ret = self.__table[key]
    logging.debug("FakeDHT.Get('%s') returns '%s'", key, ret)
    return ret

class FakeP2PNetwork:
  def __init__(self):
    # will contain (key -> (receive queue) pairs
    # where receive queue holds list of (sender_id, message) pairs.
    self.__nodes = {}
    self.__availableIDs = set(["Alice", "Brian", "Colin", "David", "Ellie"])

  # Returns new node ID
  def CreateNode(self):
    new_id = random.choice(list(self.__availableIDs))
    self.__availableIDs.remove(new_id)
    self.__nodes[new_id] = Queue.Queue()
    logging.info("New node: %s" % new_id)
    return new_id

  # Returns True on success, False on failure
  # Design flaw: we can use other node's ID as sender_id
  # TODO(timurrrr): FakeSocket
  def Send(self, sender_id, to_id, message):
    assert sender_id in self.__nodes.keys()

    if to_id not in self.__nodes.keys():
      logging.error("'%s' node is unknown" % to_id)
      return False
    self.__nodes[to_id].put((sender_id, copy.deepcopy(message)))

    return True

  # Returns (from, message) pair if present (FIFO), None if no messages are
  # available.
  # Design flaw: we can use other node's ID as receiver_id
  def Receive(self, receiver_id):
    if self.__nodes[receiver_id].empty():
      return None
    return self.__nodes[receiver_id].get()

class TorrentID:
  CHUNK_SIZE = 4

  def __init__(self, contents):
    # contents should be a string
    self.length = len(contents)
    self.filehash = hashlib.sha1(contents).hexdigest()
    self.partial_hashes = []
    for chunk_id in range(self.length / TorrentID.CHUNK_SIZE + 1):
      ph = self.__CalculatePartialHash(contents, chunk_id)
      if ph != None:
        self.partial_hashes.append(ph)
    logging.info("Created torrent:\nContents='%s'\n%s" % (contents, str(self)))

  # implicitly called by str(torrent_id)
  def __str__(self):
    return "Torrent:\n  hash=%s\n  size=%d\n  partial_hashes=[\n   %s\n  ]\n" % (
           self.filehash, self.length, ",\n   ".join(self.partial_hashes))

  @staticmethod
  def GetChunkData(contents, chunk_id):
    return contents[chunk_id * TorrentID.CHUNK_SIZE :
                (chunk_id+1) * TorrentID.CHUNK_SIZE]

  @staticmethod
  def SetChunkData(contents, chunk_id, new_data):
    idx_l = chunk_id * TorrentID.CHUNK_SIZE
    idx_r = idx_l + TorrentID.CHUNK_SIZE
    return contents[:idx_l] + new_data + contents[idx_r:]

  @staticmethod
  def GetChunkHash(chunk_data):
    return hashlib.sha1(chunk_data).hexdigest()

  @staticmethod
  def __CalculatePartialHash(contents, chunk_id):
    chunk = TorrentID.GetChunkData(contents, chunk_id)
    if len(chunk) > 0:
      return TorrentID.GetChunkHash(chunk)
    return None

  def IsKnownChunk(self, contents, chunk_id):
    return self.__CalculatePartialHash(contents, chunk_id) \
            == self.partial_hashes[chunk_id]

class TorrentClient:
  def __init__(self, network, dht):
    self.__network = network
    self.__id = network.CreateNode()

    # Will store (<chunk>_key -> set(machines which know <chunk>)
    self.__dht = dht

    # Torrent hash -> (torrent, contents) dictionary.
    # 'contents' should contain '*' for unknown bytes (TODO: this is a hack)
    self.__data = {}

    # Torrents to be downloaded
    # (torrent hash -> set of missing chunk indices)
    self.__partial_torrents = {}

    # List of finished-but-not-taken torrent hashes
    self.__downloaded = []

  def GetDownloadedTorrent(self):
    # Peek and return any downloaded torrent as a (torrent, contents) tuple
    # (if present), otherwise None.
    if len(self.__downloaded) == 0:
      return None
    ret_hash = self.__downloaded[-1]
    self.__downloaded = self.__downloaded[:-1]
    return self.__data[ret_hash]

  @staticmethod
  def __ChunkKey(torrent, chunk_id):
    assert chunk_id in range(len(torrent.partial_hashes))
    return "chunk_%s_%d" % (torrent.filehash, chunk_id)

  def AddTorrent(self, torrent, known_contents=None):
    assert torrent.filehash not in self.__data.keys()
    if known_contents:
      for chunk in range(len(torrent.partial_hashes)):
        self.__dht.Append(self.__ChunkKey(torrent, chunk), [self.__id])
      print "%s: Loaded torrent   '%s'" % (self.__id, known_contents)
    else:
      known_contents = "*" * torrent.length
      self.__partial_torrents[torrent.filehash] = \
          set(range(len(torrent.partial_hashes)))

    self.__data[torrent.filehash] = (torrent, known_contents)

  def Tick(self):
    message = self.__network.Receive(self.__id)
    if message:
      self.__ProcessMessage(message)

    if len(self.__partial_torrents.keys()) > 0:
      # Select random a torrent to download a chunk
      filehash = random.choice(self.__partial_torrents.keys())
      torrent = self.__data[filehash][0]

      # ... random chunk
      needed_chunks = self.__partial_torrents[filehash]
      chunk = random.choice(list(needed_chunks))
      chunk_key = self.__ChunkKey(torrent, chunk)

      # ... random host
      chunk_available_at = random.choice(self.__dht.Get(chunk_key))

      # Ask the host to send the chunk of that torrent
      self.__network.Send(self.__id, chunk_available_at, "give_" + chunk_key)

  def __ProcessMessage(self, msg):
    (from_id, contents) = msg
    logging.debug("Node '%s' received a message '%s' from '%s'",
                  self.__id, contents, from_id)
    m = re.match("give_chunk_([0-9a-f]+)_([0-9]+)", contents)
    if m:
      # Process "give_chunk_<hash>_<chunk>" message
      (filehash, chunk_id) = m.groups()
      if filehash not in self.__data.keys():
        logging.error("Hey, I don't know such a torrent!")
        return
      chunk_id = int(chunk_id)
      (torrent, contents) = self.__data[filehash]
      if not torrent.IsKnownChunk(contents, chunk_id):
        logging.error("Hey, I don't have this chunk!")
        logging.info("Current torrent contents are:\n  '%s'" % contents)
        return
      chunk_key = self.__ChunkKey(torrent, chunk_id)
      chunk_data = TorrentID.GetChunkData(contents, chunk_id)
      self.__network.Send(self.__id, from_id,
                          "take_%s %s" % (chunk_key, chunk_data))
      return

    m = re.match("take_chunk_([0-9a-f]+)_([0-9]+) (.*)", contents)
    if not m:
      logging.error("Couldn't parse this message '%s'", msg)
      return

    # Process "take_chunk_<hash>_<chunk> <contents>" message
    (filehash, chunk_id, chunk_data) = m.groups()
    chunk_id = int(chunk_id)
    if filehash not in self.__partial_torrents.keys():
      logging.info("Hey, I didn't want this torrent!")
      return
    needed_chunks = self.__partial_torrents[filehash]

    (torrent, known_contents) = self.__data[filehash]

    if chunk_id not in needed_chunks:
      logging.info("%s: Hey, I didn't want this chunk! %d not in %s" % \
                   (self.__id, chunk_id, str(needed_chunks)))
      logging.info("Current torrent contents are:\n  '%s'" % known_contents)
      return

    if torrent.GetChunkHash(chunk_data) != torrent.partial_hashes[chunk_id]:
      logging.error("Hash mismatch!")  # Call security?
      return

    known_contents = torrent.SetChunkData(known_contents, chunk_id, chunk_data)
    self.__data[filehash] = (torrent, known_contents)
    print "%s: New contents are '%s'" % (self.__id, known_contents)

    needed_chunks.remove(chunk_id)
    chunk_key = self.__ChunkKey(torrent, chunk_id)
    self.__dht.Append(chunk_key, [self.__id])

    if len(needed_chunks) == 0:
      logging.info("Torrent #%s download finished!" % filehash)
      self.__downloaded.append(filehash)
      self.__partial_torrents.pop(filehash)
      return

class TorrentDhtDemoTest(unittest.TestCase):
  def runTest(self):
    print  # ugly hack to force a newline

    myfile = "AAAABBBBCCCCDDDDEEEEFF"
    mytorrent = TorrentID(myfile)

    network = FakeP2PNetwork()
    dht = FakeDHT()

    clients = []
    for i in range(3):
      clients.append(TorrentClient(network, dht))
      if i == 0:  # Seeder
        clients[i].AddTorrent(mytorrent, myfile)
      else:  # others
        clients[i].AddTorrent(mytorrent)

    received_file = None
    while not received_file:
      for c in clients:
        c.Tick()  # Simulate parallel execution
      received_file = clients[1].GetDownloadedTorrent()

    self.assertEqual(received_file[0], mytorrent)
    self.assertEqual(received_file[1], myfile)

# Run the test suite.
if __name__ == '__main__':
  # replace ERROR with INFO, DEBUG, etc. and re-run. Notice any changes?
  logging.basicConfig(stream=sys.stdout,
                      level=logging.ERROR,  # Don't print anything less serious
                      format="%(asctime)s [%(levelname)s] %(message)s")
  unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
<|endoftext|>


<commit_before>import os.path

from django.template import Template, Context, TemplateSyntaxError
from django.test import TestCase

from .utils import override_settings

__all__ = [
    "JSTemplateTagTest",
    "RawTemplateTagTest",
    "ICHTemplateTagTest",
    "DustTemplateTagTest",
]

DIR = os.path.join(os.path.dirname(__file__), "templates")

class BaseMustacheJSTagTestMixin (object):
    # The following should be true for all tag types.  Remember to define
    # tag_string for any derived test classes.

    @override_settings(MUSTACHEJS_DIRS=[DIR], DEBUG=False)
    def test_no_template(self):
        res = Template(
            "{{% load mustachejs %}}{{% {} 'notemplate' %}}".format(self.tag_string)
            ).render(Context())

        self.assertEqual(res, "")

    @override_settings(MUSTACHEJS_DIRS=[DIR], DEBUG=True)
    def test_no_template_debug(self):
        from mustachejs.loading import MustacheJSTemplateNotFound
        with self.assertRaises(MustacheJSTemplateNotFound):
            Template(
                "{{% load mustachejs %}}{{% {} 'notemplate' %}}".format(self.tag_string)
                ).render(Context())

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_no_break_out(self):
        res = Template(
                "{{% load mustachejs %}}{{% {} '../outside_dir' %}}".format(self.tag_string)
                ).render(Context())

        self.assertEqual(res, "")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_no_absolute(self):
        res = Template(
                "{{% load mustachejs %}}{{% {} '/testtemplate' %}}".format(self.tag_string)
                ).render(Context())

        self.assertEqual(res, "")

    def test_bad_args(self):
        with self.assertRaises(TemplateSyntaxError):
            Template(
                "{{% load mustachejs %}}{{% {} %}}".format(self.tag_string)
                ).render(Context())

class JSTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustachejs'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustachejs 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
             "")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustachejs templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
             "")

class ICHTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustacheich'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustacheich 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
             '')

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustacheich templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
             '')

class DustTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'dustjs'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% dustjs 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
            '')

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% dustjs templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
            '')

class RawTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustacheraw'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustacheraw 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
            "
Mustache's template full of {{ foo }} and \\.
\n")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustacheraw templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
            "
Mustache's template full of {{ foo }} and \\.
\n")
<commit_msg>Stop using implicit field names for string formatting in the tests (so that it passes on Python 2.6)<commit_after>import os.path

from django.template import Template, Context, TemplateSyntaxError
from django.test import TestCase

from .utils import override_settings

__all__ = [
    "JSTemplateTagTest",
    "RawTemplateTagTest",
    "ICHTemplateTagTest",
    "DustTemplateTagTest",
]

DIR = os.path.join(os.path.dirname(__file__), "templates")

class BaseMustacheJSTagTestMixin (object):
    # The following should be true for all tag types.  Remember to define
    # tag_string for any derived test classes.

    @override_settings(MUSTACHEJS_DIRS=[DIR], DEBUG=False)
    def test_no_template(self):
        res = Template(
            "{{% load mustachejs %}}{{% {0} 'notemplate' %}}".format(self.tag_string)
            ).render(Context())

        self.assertEqual(res, "")

    @override_settings(MUSTACHEJS_DIRS=[DIR], DEBUG=True)
    def test_no_template_debug(self):
        from mustachejs.loading import MustacheJSTemplateNotFound
        with self.assertRaises(MustacheJSTemplateNotFound):
            Template(
                "{{% load mustachejs %}}{{% {0} 'notemplate' %}}".format(self.tag_string)
                ).render(Context())

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_no_break_out(self):
        res = Template(
                "{{% load mustachejs %}}{{% {0} '../outside_dir' %}}".format(self.tag_string)
                ).render(Context())

        self.assertEqual(res, "")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_no_absolute(self):
        res = Template(
                "{{% load mustachejs %}}{{% {0} '/testtemplate' %}}".format(self.tag_string)
                ).render(Context())

        self.assertEqual(res, "")

    def test_bad_args(self):
        with self.assertRaises(TemplateSyntaxError):
            Template(
                "{{% load mustachejs %}}{{% {0} %}}".format(self.tag_string)
                ).render(Context())

class JSTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustachejs'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustachejs 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
             "")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustachejs templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
             "")

class ICHTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustacheich'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustacheich 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
             '')

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustacheich templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
             '')

class DustTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'dustjs'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% dustjs 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
            '')

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% dustjs templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
            '')

class RawTemplateTagTest(TestCase, BaseMustacheJSTagTestMixin):
    tag_string = 'mustacheraw'

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_simple(self):
        res = Template(
            "{% load mustachejs %}{% mustacheraw 'testtemplate' %}"
            ).render(Context())

        self.assertEqual(
            res,
            "
Mustache's template full of {{ foo }} and \\.
\n")

    @override_settings(MUSTACHEJS_DIRS=[DIR])
    def test_variable_template_name(self):
        res = Template(
            "{% load mustachejs %}{% mustacheraw templatename %}").render(
            Context({"templatename": "testtemplate"}))

        self.assertEqual(
            res,
            "
Mustache's template full of {{ foo }} and \\.
\n")
<|endoftext|>


<commit_before>#! /usr/bin/python2.3
# vim:sw=8:ts=8:et:nowrap

import sys
import re
import os
import string

# In Debian package python2.3-egenix-mxdatetime
import mx.DateTime

from miscfuncs import ApplyFixSubstitutions
from contextexception import ContextException

from splitheadingsspeakers import SplitHeadingsSpeakers
from splitheadingsspeakers import StampUrl

from clsinglespeech import qspeech
from parlphrases import parlPhrases

from miscfuncs import FixHTMLEntities

from filterwmsspeech import FilterWMSSpeech

# Legacy patch system, use patchfilter.py and patchtool now
fixsubs = 	[
]

# parse through the usual intro headings at the beginning of the file.
def StripWMSHeadings(headspeak, sdate):
	# check and strip the first two headings in as much as they are there
	i = 0
	if (headspeak[i][0] != 'Initial') or headspeak[i][2]:
		print headspeak[i]
		raise ContextException, 'non-conforming Initial heading '
	i += 1

	if (not re.match('written ministerial ?statements?(?i)', headspeak[i][0])) or headspeak[i][2]:
		print headspeak[i]
		raise ContextException, 'non-conforming Initial heading (looking for "written ministerial ?statements?")'
	elif (not re.search('<date>', headspeak[i][0])):
		i += 1

	if (sdate != mx.DateTime.DateTimeFrom(string.replace(headspeak[i][0], " ", " ")).date) or headspeak[i][2]:
#		if (not parlPhrases.wransmajorheadings.has_key(headspeak[i][0])) or headspeak[i][2]:
		print headspeak[i]
		print sdate
		raise ContextException, 'non-conforming second heading (looking for matching datetime)'
	else:
		i += 1

	# find the url and colnum stamps that occur before anything else
	stampurl = StampUrl(sdate)
	for j in range(0, i):
		stampurl.UpdateStampUrl(headspeak[j][1])

	if (not stampurl.stamp) or (not stampurl.pageurl) or (not stampurl.aname):
		raise ContextException('missing stamp url at beginning of file')
	return (i, stampurl)

def NormalHeadingPart(headingtxt, stampurl):
	bmajorheading = False

	if not re.search('[a-z]', headingtxt):
		bmajorheading = True
	elif re.search('_dpthd', stampurl.aname):
		bmajorheading = True

	headingtxtfx = FixHTMLEntities(headingtxt)
	qb = qspeech('nospeaker="true"', headingtxtfx, stampurl)
	if bmajorheading:
		qb.typ = 'major-heading'
	else:
		qb.typ = 'minor-heading'

	# headings become one unmarked paragraph of text
	qb.stext = [ headingtxtfx ]
	return qb

def FilterWMSSections(text, sdate):
	text = ApplyFixSubstitutions(text, sdate, fixsubs)
	# split into list of triples of (heading, pre-first speech text, [ (speaker, text) ])
	headspeak = SplitHeadingsSpeakers(text)

	(ih, stampurl) = StripWMSHeadings(headspeak, sdate)

	flatb = [ ]
	for sht in headspeak[ih:]:
		try:
			headingtxt = string.strip(sht[0])
			unspoketxt = sht[1]
			speechestxt = sht[2]

			if (not re.match('(?:<[^>]*>|\s)*$', unspoketxt)):
				raise ContextException("unspoken text under heading in WMS", stamp=stampurl, fragment=unspoketxt)
#				qb = qspeech('nospeaker="true"', unspoketxt, stampurl)
#				qb.typ = 'speech'
#				FilterDebateSpeech(qb)
#				flatb.append(qb)
#			else:
#			stampurl.UpdateStampUrl(unspoketxt)

# DEBATE
			qbh = NormalHeadingPart(headingtxt, stampurl)
			flatb.append(qbh)

                        stampurl.UpdateStampUrl(unspoketxt)
# WRANS
#			# detect if this is a major heading
#			if not re.search('[a-z]', headingtxt) and not speechestxt:
#				if not parlPhrases.wransmajorheadings.has_key(headingtxt):
#					raise ContextException("unrecognized major heading, please add to parlPhrases.wransmajorheadings", fragment = headingtxt, stamp = stampurl)
#				majheadingtxtfx = parlPhrases.wransmajorheadings[headingtxt] # no need to fix since text is from a map.
#				qbH = qspeech('nospeaker="true"', majheadingtxtfx, stampurl)
#				qbH.typ = 'major-heading'
#				qbH.stext = [ majheadingtxtfx ]
#				flatb.append(qbH)
#				continue
#			# non-major heading; to a question batch
#			if parlPhrases.wransmajorheadings.has_key(headingtxt):
#				raise Exception, ' speeches found in major heading %s' % headingtxt
#			headingtxtfx = FixHTMLEntities(headingtxt)
#			headingmark = 'nospeaker="True"'
#			bNextStartofQ = True
# DEBATES:
			for ss in speechestxt:
				qb = qspeech(ss[0], ss[1], stampurl)
				qb.typ = 'speech'
				FilterWMSSpeech(qb)
				flatb.append(qb)

		except ContextException, e:
			raise
		except Exception, e:
			# add extra stamp info to the exception
			raise ContextException(str(e), stamp=stampurl)

	return flatb

<commit_msg>Daily work autocommit<commit_after>#! /usr/bin/python2.3
# vim:sw=8:ts=8:et:nowrap

import sys
import re
import os
import string

# In Debian package python2.3-egenix-mxdatetime
import mx.DateTime

from miscfuncs import ApplyFixSubstitutions
from contextexception import ContextException

from splitheadingsspeakers import SplitHeadingsSpeakers
from splitheadingsspeakers import StampUrl

from clsinglespeech import qspeech
from parlphrases import parlPhrases

from miscfuncs import FixHTMLEntities

from filterwmsspeech import FilterWMSSpeech

# Legacy patch system, use patchfilter.py and patchtool now
fixsubs = 	[
]

# parse through the usual intro headings at the beginning of the file.
def StripWMSHeadings(headspeak, sdate):
	# check and strip the first two headings in as much as they are there
	i = 0
	if (headspeak[i][0] != 'Initial') or headspeak[i][2]:
		print headspeak[i]
		raise ContextException, 'non-conforming Initial heading '
	i += 1

	if (not re.match('written ministerial ?statements?(?i)', headspeak[i][0])) or headspeak[i][2]:
		print headspeak[i]
		raise ContextException, 'non-conforming Initial heading (looking for "written ministerial ?statements?")'
	elif (not re.search('<date>', headspeak[i][0])):
		i += 1

	if (sdate != mx.DateTime.DateTimeFrom(string.replace(headspeak[i][0], " ", " ")).date) or headspeak[i][2]:
#		if (not parlPhrases.wransmajorheadings.has_key(headspeak[i][0])) or headspeak[i][2]:
		print headspeak[i]
		print sdate
		raise ContextException, 'non-conforming second heading (looking for matching datetime)'
	else:
		i += 1

	# find the url and colnum stamps that occur before anything else
	stampurl = StampUrl(sdate)
	for j in range(0, i):
		stampurl.UpdateStampUrl(headspeak[j][1])

	if (not stampurl.stamp) or (not stampurl.pageurl) or (not stampurl.aname):
		raise ContextException('missing stamp url at beginning of file')
	return (i, stampurl)

def NormalHeadingPart(headingtxt, stampurl):
	bmajorheading = False

	if not re.search('[a-z]', headingtxt):
		bmajorheading = True
	elif re.search('_dpthd', stampurl.aname):
		bmajorheading = True
	if re.search('_sbhd', stampurl.aname):
		bmajorheading = False

	headingtxtfx = FixHTMLEntities(headingtxt)
	qb = qspeech('nospeaker="true"', headingtxtfx, stampurl)
	if bmajorheading:
		qb.typ = 'major-heading'
	else:
		qb.typ = 'minor-heading'

	# headings become one unmarked paragraph of text
	qb.stext = [ headingtxtfx ]
	return qb

def FilterWMSSections(text, sdate):
	text = ApplyFixSubstitutions(text, sdate, fixsubs)
	# split into list of triples of (heading, pre-first speech text, [ (speaker, text) ])
	headspeak = SplitHeadingsSpeakers(text)

	(ih, stampurl) = StripWMSHeadings(headspeak, sdate)

	flatb = [ ]
	for sht in headspeak[ih:]:
		try:
			headingtxt = string.strip(sht[0])
			unspoketxt = sht[1]
			speechestxt = sht[2]

			if (not re.match('(?:<[^>]*>|\s)*$', unspoketxt)):
				raise ContextException("unspoken text under heading in WMS", stamp=stampurl, fragment=unspoketxt)
#				qb = qspeech('nospeaker="true"', unspoketxt, stampurl)
#				qb.typ = 'speech'
#				FilterDebateSpeech(qb)
#				flatb.append(qb)
#			else:
#			stampurl.UpdateStampUrl(unspoketxt)

# DEBATE
			qbh = NormalHeadingPart(headingtxt, stampurl)
			flatb.append(qbh)

                        stampurl.UpdateStampUrl(unspoketxt)
# WRANS
#			# detect if this is a major heading
#			if not re.search('[a-z]', headingtxt) and not speechestxt:
#				if not parlPhrases.wransmajorheadings.has_key(headingtxt):
#					raise ContextException("unrecognized major heading, please add to parlPhrases.wransmajorheadings", fragment = headingtxt, stamp = stampurl)
#				majheadingtxtfx = parlPhrases.wransmajorheadings[headingtxt] # no need to fix since text is from a map.
#				qbH = qspeech('nospeaker="true"', majheadingtxtfx, stampurl)
#				qbH.typ = 'major-heading'
#				qbH.stext = [ majheadingtxtfx ]
#				flatb.append(qbH)
#				continue
#			# non-major heading; to a question batch
#			if parlPhrases.wransmajorheadings.has_key(headingtxt):
#				raise Exception, ' speeches found in major heading %s' % headingtxt
#			headingtxtfx = FixHTMLEntities(headingtxt)
#			headingmark = 'nospeaker="True"'
#			bNextStartofQ = True
# DEBATES:
			for ss in speechestxt:
				qb = qspeech(ss[0], ss[1], stampurl)
				qb.typ = 'speech'
				FilterWMSSpeech(qb)
				flatb.append(qb)

		except ContextException, e:
			raise
		except Exception, e:
			# add extra stamp info to the exception
			raise ContextException(str(e), stamp=stampurl)

	return flatb

<|endoftext|>


<commit_before>
from django.contrib.contenttypes.models import ContentType
from django.db import models, router
from django.db.models.fields import FieldDoesNotExist
from django.db.models.signals import m2m_changed, post_delete, post_save, pre_delete
from south.db import dbs

from mutant.models import (ModelDefinition, BaseDefinition, FieldDefinition,
    UniqueTogetherDefinition)

def model_definition_post_save(sender, instance, created, raw, **kwargs):
    if raw:
        ct_db = instance._state.db # cts should be one the same db as mds
        ct = ContentType.objects.using(ct_db).get(pk=instance.pk)
        instance.app_label, instance.model = ct.app_label, ct.model
    model_class = instance.model_class(force_create=True)
    opts = model_class._meta
    db = router.db_for_write(model_class)
    if created:
        fields = tuple((field.name, field) for field in opts.fields)
        dbs[db].create_table(opts.db_table, fields)
    else:
        old_opts = instance._model_class._meta
        if old_opts.db_table != opts.db_table:
            dbs[db].rename_table(old_opts.db_table, opts.db_table)
            # It means that the natural key has changed
            ContentType.objects.clear_cache()

post_save.connect(model_definition_post_save, ModelDefinition,
                  dispatch_uid='mutant.management.model_definition_post_save')

def model_definition_post_delete(sender, instance, **kwargs):
    model_class = instance.model_class()
    db = router.db_for_write(model_class)
    db_table = model_class._meta.db_table
    dbs[db].delete_table(db_table)

post_delete.connect(model_definition_post_delete, ModelDefinition,
                    dispatch_uid='mutant.management.model_definition_post_delete')

def base_definition_post_save(sender, instance, created, raw, **kwargs):
    base = instance.base
    if issubclass(base, models.Model):
        model_class = instance.model_def.model_class()
        opts = model_class._meta
        db = router.db_for_write(model_class)
        table_name = opts.db_table
        if created:
            for field in base._meta.fields:
                dbs[db].add_column(table_name, field.name,
                                   field, keep_default=False)
        else:
            for field in base._meta.fields:
                try:
                    old_field = opts.get_field(field.name)
                except FieldDoesNotExist:
                    dbs[db].add_column(table_name, field.name,
                                       field, keep_default=False)
                else:
                    column = old_field.get_attname_column()[1]
                    dbs[db].alter_column(table_name, column, field)

post_save.connect(base_definition_post_save, BaseDefinition,
                  dispatch_uid='mutant.management.base_definition_post_save')

def base_definition_pre_delete(sender, instance, **kwargs):
    """
    This is used to pass data required for deletion to the post_delete
    signal that is no more available thereafter.
    """
    if issubclass(instance.base, models.Model):
        model_class = instance.model_def.model_class()
        instance._state._deletion = (
            router.db_for_write(model_class),
            model_class._meta.db_table,
        )

pre_delete.connect(base_definition_pre_delete, BaseDefinition,
                   dispatch_uid='mutant.management.base_definition_pre_delete')

def base_definition_post_delete(sender, instance, **kwargs):
    if issubclass(instance.base, models.Model):
        db, table_name = instance._state._deletion
        for field in instance.base._meta.fields:
            dbs[db].delete_column(table_name, field.name)
        del instance._state._deletion

post_delete.connect(base_definition_post_delete, BaseDefinition,
                    dispatch_uid='mutant.management.base_definition_post_delete')

def unique_together_field_defs_changed(instance, action, model, **kwargs):
    columns = list(instance.field_defs.names())
    # If there's no columns and action is post_clear there's nothing to do
    if columns and action != 'post_clear':
        model_class = instance.model_def.model_class()
        db = router.db_for_write(model_class)
        table_name = model_class._meta.db_table
        if action in ('pre_add', 'pre_remove', 'pre_clear'):
            dbs[db].delete_unique(table_name, columns)
        # Safe guard againts m2m_changed.action api change
        elif action in ('post_add', 'post_remove'):
            dbs[db].create_unique(table_name, columns)

m2m_changed.connect(unique_together_field_defs_changed,
                    UniqueTogetherDefinition.field_defs.through,
                    dispatch_uid='mutant.management.unique_together_field_defs_changed')

def field_definition_post_save(sender, instance, created, raw, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    model_class = instance.model_def.model_class()
    db = router.db_for_write(model_class)
    table_name = model_class._meta.db_table
    field = instance._south_ready_field_instance()

    if created:
        if hasattr(instance._state, '_creation_default_value'):
            field.default = instance._state._creation_default_value
            delattr(instance._state, '_creation_default_value')
            keep_default = False
        else:
            keep_default = True
        dbs[db].add_column(table_name, instance.name, field, keep_default=keep_default)
    else:
        __, column = field.get_attname_column()
        old_field = instance._old_field

        # Field renaming
        old_column = old_field.get_attname_column()[1]
        if column != old_column:
            dbs[db].rename_column(table_name, old_column, column)

        for opt in ('primary_key', 'unique'):
            value = getattr(field, opt)
            if value != getattr(old_field, opt):
                method_format = ('create' if value else 'delete', opt)
                method = getattr(dbs[db], "%s_%s" % method_format)
                method(table_name, (column,))

        dbs[db].alter_column(table_name, column, field)

def field_definition_pre_delete(sender, instance, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    model_class = instance.model_def.model_class()
    opts = model_class._meta
    instance._state._deletion = (
        router.db_for_write(model_class),
        opts.db_table,
        opts.get_field(instance.name).column
    )

def field_definition_post_delete(sender, instance, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    db, table_name, name = instance._state._deletion
    dbs[db].delete_column(table_name, name)
    del instance._state._deletion

def connect_field_definition(definition):
    model = definition._meta.object_name.lower()
    post_save.connect(field_definition_post_save, sender=definition,
                      dispatch_uid="mutant.management.%s_post_save" % model)
    pre_delete.connect(field_definition_pre_delete, sender=definition,
                       dispatch_uid="mutant.management.%s_pre_delete" % model)
    post_delete.connect(field_definition_post_delete, sender=definition,
                        dispatch_uid="mutant.management.%s_post_delete" % model)

<commit_msg>Make that DDL operations are performed on all dbs that allow syncing for MutantModel and not only the db used for writting<commit_after>
from django.contrib.contenttypes.models import ContentType
from django.db import connections, models, router
from django.db.models.fields import FieldDoesNotExist
from django.db.models.signals import m2m_changed, post_delete, post_save, pre_delete
from south.db import dbs

from mutant.models import (ModelDefinition, BaseDefinition, FieldDefinition,
    UniqueTogetherDefinition)

def allow_syncdbs(model):
    for db in connections:
        if router.allow_syncdb(db, model):
            yield dbs[db]

def perform_ddl(model, action, *args, **kwargs):
    for db in allow_syncdbs(model):
        getattr(db, action)(*args, **kwargs)

def model_definition_post_save(sender, instance, created, raw, **kwargs):
    if raw:
        ct_db = instance._state.db # cts should be one the same db as mds
        ct = ContentType.objects.using(ct_db).get(pk=instance.pk)
        instance.app_label, instance.model = ct.app_label, ct.model
    model_class = instance.model_class(force_create=True)
    opts = model_class._meta
    if created:
        fields = tuple((field.name, field) for field in opts.fields)
        perform_ddl(model_class, 'create_table', opts.db_table, fields)
    else:
        old_opts = instance._model_class._meta
        if old_opts.db_table != opts.db_table:
            perform_ddl(model_class, 'rename_table', old_opts.db_table, opts.db_table)
            # It means that the natural key has changed
            ContentType.objects.clear_cache()

post_save.connect(model_definition_post_save, ModelDefinition,
                  dispatch_uid='mutant.management.model_definition_post_save')

def model_definition_post_delete(sender, instance, **kwargs):
    model_class = instance.model_class()
    table_name = model_class._meta.db_table
    perform_ddl(model_class, 'delete_table', table_name)

post_delete.connect(model_definition_post_delete, ModelDefinition,
                    dispatch_uid='mutant.management.model_definition_post_delete')

def base_definition_post_save(sender, instance, created, raw, **kwargs):
    base = instance.base
    if issubclass(base, models.Model):
        model_class = instance.model_def.model_class()
        opts = model_class._meta
        table_name = opts.db_table
        if created:
            for field in base._meta.fields:
                perform_ddl(model_class, 'add_column', table_name,
                            field.name, field, keep_default=False)
        else:
            for field in base._meta.fields:
                try:
                    old_field = opts.get_field(field.name)
                except FieldDoesNotExist:
                    perform_ddl(model_class, 'add_column', table_name,
                                field.name, field, keep_default=False)
                else:
                    column = old_field.get_attname_column()[1]
                    perform_ddl(model_class, 'alter_column', table_name,
                                column, field)

post_save.connect(base_definition_post_save, BaseDefinition,
                  dispatch_uid='mutant.management.base_definition_post_save')

def base_definition_pre_delete(sender, instance, **kwargs):
    """
    This is used to pass data required for deletion to the post_delete
    signal that is no more available thereafter.
    """
    if issubclass(instance.base, models.Model):
        model_class = instance.model_def.model_class()
        instance._state._deletion = (
            allow_syncdbs(model_class),
            model_class._meta.db_table,
        )

pre_delete.connect(base_definition_pre_delete, BaseDefinition,
                   dispatch_uid='mutant.management.base_definition_pre_delete')

def base_definition_post_delete(sender, instance, **kwargs):
    if issubclass(instance.base, models.Model):
        syncdbs, table_name = instance._state._deletion
        for field in instance.base._meta.fields:
            for db in syncdbs:
                db.delete_column(table_name, field.name)
        del instance._state._deletion

post_delete.connect(base_definition_post_delete, BaseDefinition,
                    dispatch_uid='mutant.management.base_definition_post_delete')

def unique_together_field_defs_changed(instance, action, model, **kwargs):
    columns = list(instance.field_defs.names())
    # If there's no columns and action is post_clear there's nothing to do
    if columns and action != 'post_clear':
        model_class = instance.model_def.model_class()
        table_name = model_class._meta.db_table
        if action in ('pre_add', 'pre_remove', 'pre_clear'):
            perform_ddl(model_class, 'delete_unique', table_name, columns)
        # Safe guard againts m2m_changed.action api change
        elif action in ('post_add', 'post_remove'):
            perform_ddl(model_class, 'create_unique', table_name, columns)

m2m_changed.connect(unique_together_field_defs_changed,
                    UniqueTogetherDefinition.field_defs.through,
                    dispatch_uid='mutant.management.unique_together_field_defs_changed')

def field_definition_post_save(sender, instance, created, raw, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    model_class = instance.model_def.model_class()
    table_name = model_class._meta.db_table
    field = instance._south_ready_field_instance()

    if created:
        if hasattr(instance._state, '_creation_default_value'):
            field.default = instance._state._creation_default_value
            delattr(instance._state, '_creation_default_value')
            keep_default = False
        else:
            keep_default = True
        perform_ddl(model_class, 'add_column', table_name,
                    instance.name, field, keep_default=keep_default)
    else:
        __, column = field.get_attname_column()
        old_field = instance._old_field

        # Field renaming
        old_column = old_field.get_attname_column()[1]
        if column != old_column:
            perform_ddl(model_class, 'rename_column', table_name, old_column, column)

        # Create/Drop unique and primarykey
        for opt in ('primary_key', 'unique'):
            value = getattr(field, opt)
            if value != getattr(old_field, opt):
                action_prefix = 'create' if value else 'delete'
                action = "%s_%s" % (action_prefix, opt)
                perform_ddl(model_class, action, table_name, (column,))

        perform_ddl(model_class, 'alter_column', table_name, column, field)

def field_definition_pre_delete(sender, instance, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    model_class = instance.model_def.model_class()
    opts = model_class._meta
    instance._state._deletion = (
        allow_syncdbs(model_class),
        opts.db_table,
        opts.get_field(instance.name).column
    )

def field_definition_post_delete(sender, instance, **kwargs):
    """
    This signal is connected by all FieldDefinition subclasses
    see comment in FieldDefinitionBase for more details
    """
    syncdbs, table_name, name = instance._state._deletion
    for db in syncdbs:
        db.delete_column(table_name, name)
    del instance._state._deletion

def connect_field_definition(definition):
    model = definition._meta.object_name.lower()
    post_save.connect(field_definition_post_save, sender=definition,
                      dispatch_uid="mutant.management.%s_post_save" % model)
    pre_delete.connect(field_definition_pre_delete, sender=definition,
                       dispatch_uid="mutant.management.%s_pre_delete" % model)
    post_delete.connect(field_definition_post_delete, sender=definition,
                        dispatch_uid="mutant.management.%s_post_delete" % model)

<|endoftext|>


<commit_before>from __future__ import print_function, division

import sys
from time import time

from sympy.ntheory.residue_ntheory import (discrete_log,
        _discrete_log_trial_mul, _discrete_log_shanks_steps,
        _discrete_log_pollard_rho, _discrete_log_pohlig_hellman)

# Cyclic group (Z/pZ)* with p prime, order p - 1 and generator g
data_set_1 = [
        # p, p - 1, g
        [191, 190, 19],
        [46639, 46638, 6],
        [14789363, 14789362, 2],
        [4254225211, 4254225210, 2],
        [432751500361, 432751500360, 7],
        [158505390797053, 158505390797052, 2],
        [6575202655312007, 6575202655312006, 5],
        [8430573471995353769, 8430573471995353768, 3],
        [3938471339744997827267, 3938471339744997827266, 2],
        [875260951364705563393093, 875260951364705563393092, 5],
    ]

# Cyclic sub-groups of (Z/nZ)* with prime order p and generator g
# (n, p are primes and n = 2 * p + 1)
data_set_2 = [
        # n, p, g
        [227, 113, 3],
        [2447, 1223, 2],
        [24527, 12263, 2],
        [245639, 122819, 2],
        [2456747, 1228373, 3],
        [24567899, 12283949, 3],
        [245679023, 122839511, 2],
        [2456791307, 1228395653, 3],
        [24567913439, 12283956719, 2],
        [245679135407, 122839567703, 2],
        [2456791354763, 1228395677381, 3],
        [24567913550903, 12283956775451, 2],
        [245679135509519, 122839567754759, 2],
    ]

# Cyclic sub-groups of (Z/nZ)* with smooth order o and generator g
data_set_3 = [
        # n, o, g
        [2**118, 2**116, 3],
    ]

def bench_discrete_log(data, algo=None):
    if algo is None:
        f = discrete_log
    elif algo == 'trial':
        f = _discrete_log_trial_mul
    elif algo == 'shanks':
        f = _discrete_log_shanks_steps
    elif algo == 'rho':
        f = _discrete_log_pollard_rho
    elif algo == 'ph':
        f = _discrete_log_pohlig_hellman
    else:
        raise ValueError("Argument 'algo' should be one"
                " of ('trial', 'shanks', 'rho' or 'ph')")

    for i, data in enumerate(data_set):
        for j, (n, p, g) in enumerate(data):
            t = time()
            l = f(n, pow(g, p - 1, n), g, p)
            t = time() - t
            print('[%02d-%03d] %15.10f' % (i, j, t))
            assert l == p - 1

if __name__ == '__main__':
    algo = sys.argv[1] \
            if len(sys.argv) > 1 else None
    data_set = [
            data_set_1,
            data_set_2,
            data_set_3,
        ]
    bench_discrete_log(data_set, algo)

<commit_msg>Fix argument name<commit_after>from __future__ import print_function, division

import sys
from time import time

from sympy.ntheory.residue_ntheory import (discrete_log,
        _discrete_log_trial_mul, _discrete_log_shanks_steps,
        _discrete_log_pollard_rho, _discrete_log_pohlig_hellman)

# Cyclic group (Z/pZ)* with p prime, order p - 1 and generator g
data_set_1 = [
        # p, p - 1, g
        [191, 190, 19],
        [46639, 46638, 6],
        [14789363, 14789362, 2],
        [4254225211, 4254225210, 2],
        [432751500361, 432751500360, 7],
        [158505390797053, 158505390797052, 2],
        [6575202655312007, 6575202655312006, 5],
        [8430573471995353769, 8430573471995353768, 3],
        [3938471339744997827267, 3938471339744997827266, 2],
        [875260951364705563393093, 875260951364705563393092, 5],
    ]

# Cyclic sub-groups of (Z/nZ)* with prime order p and generator g
# (n, p are primes and n = 2 * p + 1)
data_set_2 = [
        # n, p, g
        [227, 113, 3],
        [2447, 1223, 2],
        [24527, 12263, 2],
        [245639, 122819, 2],
        [2456747, 1228373, 3],
        [24567899, 12283949, 3],
        [245679023, 122839511, 2],
        [2456791307, 1228395653, 3],
        [24567913439, 12283956719, 2],
        [245679135407, 122839567703, 2],
        [2456791354763, 1228395677381, 3],
        [24567913550903, 12283956775451, 2],
        [245679135509519, 122839567754759, 2],
    ]

# Cyclic sub-groups of (Z/nZ)* with smooth order o and generator g
data_set_3 = [
        # n, o, g
        [2**118, 2**116, 3],
    ]

def bench_discrete_log(data_set, algo=None):
    if algo is None:
        f = discrete_log
    elif algo == 'trial':
        f = _discrete_log_trial_mul
    elif algo == 'shanks':
        f = _discrete_log_shanks_steps
    elif algo == 'rho':
        f = _discrete_log_pollard_rho
    elif algo == 'ph':
        f = _discrete_log_pohlig_hellman
    else:
        raise ValueError("Argument 'algo' should be one"
                " of ('trial', 'shanks', 'rho' or 'ph')")

    for i, data in enumerate(data_set):
        for j, (n, p, g) in enumerate(data):
            t = time()
            l = f(n, pow(g, p - 1, n), g, p)
            t = time() - t
            print('[%02d-%03d] %15.10f' % (i, j, t))
            assert l == p - 1

if __name__ == '__main__':
    algo = sys.argv[1] \
            if len(sys.argv) > 1 else None
    data_set = [
            data_set_1,
            data_set_2,
            data_set_3,
        ]
    bench_discrete_log(data_set, algo)

<|endoftext|>


<commit_before>import os
import warnings

from django import dispatch
from jinja2 import Environment, loaders

__all__ = ('get_env', 'need_env')

_ENV = None
_LOADERS = [] # See :fun:`_infer_loaders`.
_TEMPLATE_LIBS = []
_JINJA_I18N_EXTENSION_NAME = 'jinja2.ext.i18n'

need_env = dispatch.Signal(providing_args=['arguments', 'loaders',
                                           'filters', 'extensions'])

def _get_loaders():
    """Tries to translate each template loader given in the Django settings
    (:mod:`django.settings`) to a similarly-behaving Jinja loader.
    Warns if a similar loader cannot be found.
    Allows for Jinja2 loader instances to be placed in the template loader
    settings.
    """
    from coffin.template.loaders import jinja_loader_from_django_loader
    if _LOADERS:
        return _LOADERS
    from django.conf import settings
    for loader in settings.TEMPLATE_LOADERS:
        if isinstance(loader, basestring):
            loader_obj = jinja_loader_from_django_loader(loader)
            if loader_obj:
                _LOADERS.append(loader_obj)
            else:
                warnings.warn('Cannot translate loader: %s' % loader)
        else: # It's assumed to be a Jinja2 loader instance.
            _LOADERS.append(loader)
    return _LOADERS

def _get_templatelibs():
    """Return an iterable of template ``Library`` instances.

    Since we cannot support the {% load %} tag in Jinja, we have to
    register all libraries globally.
    """
    if _TEMPLATE_LIBS:
        return _TEMPLATE_LIBS

    from django.conf import settings
    from django.template import get_library, InvalidTemplateLibrary

    for a in settings.INSTALLED_APPS:
        try:
            path = __import__(a + '.templatetags', {}, {}, ['']).__file__
            path = os.path.dirname(path)  # we now have the templatetags/ directory
        except ImportError:
            pass
        else:
            for f in os.listdir(path):
                if f == '__init__.py':
                    continue
                if f.endswith('.py'):
                    try:
                        # TODO: will need updating when #6587 lands
                        _TEMPLATE_LIBS.append(get_library(
                            "django.templatetags.%s" % os.path.splitext(f)[0]))
                    except InvalidTemplateLibrary:
                        pass
    return _TEMPLATE_LIBS

def _get_filters():
    """Returns a list of filters to provide through Jinja. This includes
    ported versions of Django's builtin filters that Jinja is lacking,
    as well as custom filters as specified by the user in the settings.

    :return: A mapping of names to filters.
    """
    from django.conf import settings
    from coffin.template import builtins

    filters = {}
    # start with our default builtins
    for lib in builtins:
        if hasattr(lib, 'jinja2_filters'):
            filters.update(lib.jinja2_filters)

    # add the globally defined filter list
    user = getattr(settings, 'JINJA2_FILTERS', {})
    from django.core.urlresolvers import get_callable
    if isinstance(user, dict):
        for key, value in user.items():
            filters[user] = callable(value) and value or get_callable(value)
    else:
        for value in user:
            value = callable(value) and value or get_callable(value)
            filters[value.__name__] = value

    # add filters defined in application's templatetag libraries
    for lib in _get_templatelibs():
        if hasattr(lib, 'jinja2_filters'):
            filters.update(lib.jinja2_filters)

    return filters

def _get_extensions():
    from django.conf import settings
    from coffin.template import builtins

    extensions = []
    # start with our default builtins
    for lib in builtins:
        if hasattr(lib, 'jinja2_extensions'):
            extensions += lib.jinja2_extensions

    if settings.USE_I18N:
        extensions.append(_JINJA_I18N_EXTENSION_NAME)

    # add the globally defined extension list
    extensions += list(getattr(settings, 'JINJA2_EXTENSIONS', []))

    # add extensions defined in application's templatetag libraries
    for lib in _get_templatelibs():
        if hasattr(lib, 'jinja2_extensions'):
            extensions += lib.jinja2_extensions

    return extensions

def _get_all_extensions():
    from django.conf import settings
    from coffin.template import builtins
    from django.core.urlresolvers import get_callable

    extensions, filters, globals = [], {}, {}
    # start with our default builtins
    for lib in builtins:

        extensions.extend(getattr(lib, 'jinja2_extensions', []))
        filters.update(getattr(lib, 'jinja2_filters', {}))
        globals.update(getattr(lib, 'jinja2_globals', {}))

    if settings.USE_I18N:
        extensions.append(_JINJA_I18N_EXTENSION_NAME)

    # add the globally defined extension list
    extensions += list(getattr(settings, 'JINJA2_EXTENSIONS', []))

    user = getattr(settings, 'JINJA2_FILTERS', {})
    if isinstance(user, dict):
        for key, value in user.iteritems():
            filters[user] = callable(value) and value or get_callable(value)
    else:
        for value in user:
            value = callable(value) and value or get_callable(value)
            filters[value.__name__] = value

    user = getattr(settings, 'JINJA2_GLOBALS', {})
    if isinstance(user, dict):
        for key, value in user.iteritems():
            globals[user] = callable(value) and value or get_callable(value)
    else:
        for value in user:
            value = callable(value) and value or get_callable(value)
            globals[value.__name__] = value

    # add extensions defined in application's templatetag libraries
    for lib in _get_templatelibs():
        extensions.extend(getattr(lib, 'jinja2_extensions', []))
        filters.update(getattr(lib, 'jinja2_filters', {}))
        globals.update(getattr(lib, 'jinja2_globals', {}))

    return extensions, filters, globals

def get_env():
    """
    :return: A Jinja2 environment singleton.
    """
    global _ENV
    if not _ENV:
        loaders_ = _get_loaders()
        extensions, filters, globals = _get_all_extensions()
        arguments = {
            'autoescape': True,
        }

        need_env.send(sender=Environment, arguments=arguments,
                      loaders=loaders_, extensions=extensions,
                      filters=filters, globals=globals)

        if not _ENV:
            if not 'loader' in arguments:
                arguments['loader'] = loaders.ChoiceLoader(loaders_)
            if not 'extensions' in arguments:
                arguments['extensions'] = extensions

            _ENV = Environment(**arguments)
            _ENV.filters.update(filters)
            _ENV.globals.update(globals)
            from coffin.template import Template as CoffinTemplate
            _ENV.template_class = CoffinTemplate
    return _ENV<commit_msg>Force several var declarations to globals.<commit_after>import os
import warnings

from django import dispatch
from jinja2 import Environment, loaders

__all__ = ('get_env', 'need_env')

_ENV = None
_LOADERS = [] # See :fun:`_infer_loaders`.
_TEMPLATE_LIBS = []
_JINJA_I18N_EXTENSION_NAME = 'jinja2.ext.i18n'

need_env = dispatch.Signal(providing_args=['arguments', 'loaders',
                                           'filters', 'extensions',
                                           'globals'])
def _get_loaders():
    """Tries to translate each template loader given in the Django settings
    (:mod:`django.settings`) to a similarly-behaving Jinja loader.
    Warns if a similar loader cannot be found.
    Allows for Jinja2 loader instances to be placed in the template loader
    settings.
    """
    from coffin.template.loaders import jinja_loader_from_django_loader
    global _LOADERS

    if _LOADERS:
        return _LOADERS
    from django.conf import settings
    for loader in settings.TEMPLATE_LOADERS:
        if isinstance(loader, basestring):
            loader_obj = jinja_loader_from_django_loader(loader)
            if loader_obj:
                _LOADERS.append(loader_obj)
            else:
                warnings.warn('Cannot translate loader: %s' % loader)
        else: # It's assumed to be a Jinja2 loader instance.
            _LOADERS.append(loader)
    return _LOADERS

def _get_templatelibs():
    """Return an iterable of template ``Library`` instances.

    Since we cannot support the {% load %} tag in Jinja, we have to
    register all libraries globally.
    """
    global _TEMPLATE_LIBS

    if _TEMPLATE_LIBS:
        return _TEMPLATE_LIBS

    from django.conf import settings
    from django.template import get_library, InvalidTemplateLibrary

    for a in settings.INSTALLED_APPS:
        try:
            path = __import__(a + '.templatetags', {}, {}, ['']).__file__
            path = os.path.dirname(path)  # we now have the templatetags/ directory
        except ImportError:
            pass
        else:
            for f in os.listdir(path):
                if f == '__init__.py':
                    continue
                if f.endswith('.py'):
                    try:
                        # TODO: will need updating when #6587 lands
                        _TEMPLATE_LIBS.append(get_library(
                            "django.templatetags.%s" % os.path.splitext(f)[0]))
                    except InvalidTemplateLibrary:
                        pass
    return _TEMPLATE_LIBS

def _get_all_extensions():
    # TODO: should these be cached as well?
    from django.conf import settings
    from coffin.template import builtins
    from django.core.urlresolvers import get_callable

    extensions, filters, globals = [], {}, {}

    # start with our default builtins
    for lib in builtins:
        extensions.extend(getattr(lib, 'jinja2_extensions', []))
        filters.update(getattr(lib, 'jinja2_filters', {}))
        globals.update(getattr(lib, 'jinja2_globals', {}))

    if settings.USE_I18N:
        extensions.append(_JINJA_I18N_EXTENSION_NAME)

    # add the globally defined extension list
    extensions.extend(list(getattr(settings, 'JINJA2_EXTENSIONS', [])))

    user = getattr(settings, 'JINJA2_FILTERS', {})
    if isinstance(user, dict):
        for key, value in user.iteritems():
            filters[user] = callable(value) and value or get_callable(value)
    else:
        for value in user:
            value = callable(value) and value or get_callable(value)
            filters[value.__name__] = value

    user = getattr(settings, 'JINJA2_GLOBALS', {})
    if isinstance(user, dict):
        for key, value in user.iteritems():
            globals[user] = callable(value) and value or get_callable(value)
    else:
        for value in user:
            value = callable(value) and value or get_callable(value)
            globals[value.__name__] = value

    # add extensions defined in application's templatetag libraries
    for lib in _get_templatelibs():
        extensions.extend(getattr(lib, 'jinja2_extensions', []))
        filters.update(getattr(lib, 'jinja2_filters', {}))
        globals.update(getattr(lib, 'jinja2_globals', {}))

    return extensions, filters, globals

def get_env():
    """
    :return: A Jinja2 environment singleton.
    """
    global _ENV

    if not _ENV:
        loaders_ = _get_loaders()
        extensions, filters, globals = _get_all_extensions()
        arguments = {
            'autoescape': True,
        }

        need_env.send(sender=Environment, arguments=arguments,
                      loaders=loaders_, extensions=extensions,
                      filters=filters, globals=globals)

        if not _ENV:
            if not 'loader' in arguments:
                arguments['loader'] = loaders.ChoiceLoader(loaders_)
            if not 'extensions' in arguments:
                arguments['extensions'] = extensions

            _ENV = Environment(**arguments)
            _ENV.filters.update(filters)
            _ENV.globals.update(globals)
            from coffin.template import Template as CoffinTemplate
            _ENV.template_class = CoffinTemplate
    return _ENV<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2018 New Vecotr Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from twisted.internet import defer, threads
from twisted.protocols.basic import FileSender

from ._base import Responder

from synapse.util.logcontext import make_deferred_yieldable

import contextlib
import os
import logging
import shutil
import sys

logger = logging.getLogger(__name__)

class MediaStorage(object):
    """Responsible for storing/fetching files from local sources.

    Args:
        local_media_directory (str): Base path where we store media on disk
        filepaths (MediaFilePaths)
        storage_providers ([StorageProvider]): List of StorageProvider that are
            used to fetch and store files.
    """

    def __init__(self, local_media_directory, filepaths, storage_providers):
        self.local_media_directory = local_media_directory
        self.filepaths = filepaths
        self.storage_providers = storage_providers

    @defer.inlineCallbacks
    def store_file(self, source, file_info):
        """Write `source` to the on disk media store, and also any other
        configured storage providers

        Args:
            source: A file like object that should be written
            file_info (FileInfo): Info about the file to store

        Returns:
            Deferred[str]: the file path written to in the primary media store
        """
        path = self._file_info_to_path(file_info)
        fname = os.path.join(self.local_media_directory, path)

        dirname = os.path.dirname(fname)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        # Write to the main repository
        yield make_deferred_yieldable(threads.deferToThread(
            _write_file_synchronously, source, fname,
        ))

        defer.returnValue(fname)

    @contextlib.contextmanager
    def store_into_file(self, file_info):
        """Context manager used to get a file like object to write into, as
        described by file_info.

        Actually yields a 3-tuple (file, fname, finish_cb), where file is a file
        like object that can be written to, fname is the absolute path of file
        on disk, and finish_cb is a function that returns a Deferred.

        fname can be used to read the contents from after upload, e.g. to
        generate thumbnails.

        finish_cb must be called and waited on after the file has been
        successfully been written to. Should not be called if there was an
        error.

        Args:
            file_info (FileInfo): Info about the file to store

        Example:

            with media_storage.store_into_file(info) as (f, fname, finish_cb):
                # .. write into f ...
                yield finish_cb()
        """

        path = self._file_info_to_path(file_info)
        fname = os.path.join(self.local_media_directory, path)

        dirname = os.path.dirname(fname)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        finished_called = [False]

        @defer.inlineCallbacks
        def finish():
            for provider in self.storage_providers:
                yield provider.store_file(path, file_info)

            finished_called[0] = True

        try:
            with open(fname, "wb") as f:
                yield f, fname, finish
        except Exception:
            t, v, tb = sys.exc_info()
            try:
                os.remove(fname)
            except Exception:
                pass
            raise t, v, tb

        if not finished_called:
            raise Exception("Finished callback not called")

    @defer.inlineCallbacks
    def fetch_media(self, file_info):
        """Attempts to fetch media described by file_info from the local cache
        and configured storage providers.

        Args:
            file_info (FileInfo)

        Returns:
            Deferred[Responder|None]: Returns a Responder if the file was found,
                otherwise None.
        """

        path = self._file_info_to_path(file_info)
        local_path = os.path.join(self.local_media_directory, path)
        if os.path.exists(local_path):
            defer.returnValue(FileResponder(open(local_path, "rb")))

        for provider in self.storage_providers:
            res = yield provider.fetch(path, file_info)
            if res:
                defer.returnValue(res)

        defer.returnValue(None)

    def _file_info_to_path(self, file_info):
        """Converts file_info into a relative path.

        The path is suitable for storing files under a directory, e.g. used to
        store files on local FS under the base media repository directory.

        Args:
            file_info (FileInfo)

        Returns:
            str
        """
        if file_info.url_cache:
            return self.filepaths.url_cache_filepath_rel(file_info.file_id)

        if file_info.server_name:
            if file_info.thumbnail:
                return self.filepaths.remote_media_thumbnail_rel(
                    server_name=file_info.server_name,
                    file_id=file_info.file_id,
                    width=file_info.thumbnail_width,
                    height=file_info.thumbnail_height,
                    content_type=file_info.thumbnail_type,
                    method=file_info.thumbnail_method
                )
            return self.filepaths.remote_media_filepath_rel(
                file_info.server_name, file_info.file_id,
            )

        if file_info.thumbnail:
            return self.filepaths.local_media_thumbnail_rel(
                media_id=file_info.file_id,
                width=file_info.thumbnail_width,
                height=file_info.thumbnail_height,
                content_type=file_info.thumbnail_type,
                method=file_info.thumbnail_method
            )
        return self.filepaths.local_media_filepath_rel(
            file_info.file_id,
        )

def _write_file_synchronously(source, fname):
    """Write `source` to the path `fname` synchronously. Should be called
    from a thread.

    Args:
        source: A file like object to be written
        fname (str): Path to write to
    """
    dirname = os.path.dirname(fname)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    source.seek(0)  # Ensure we read from the start of the file
    with open(fname, "wb") as f:
        shutil.copyfileobj(source, f)

class FileResponder(Responder):
    """Wraps an open file that can be sent to a request.

    Args:
        open_file (file): A file like object to be streamed ot the client,
            is closed when finished streaming.
    """
    def __init__(self, open_file):
        self.open_file = open_file

    @defer.inlineCallbacks
    def write_to_consumer(self, consumer):
        yield FileSender().beginFileTransfer(self.open_file, consumer)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.open_file.close()
<commit_msg>Use the right path for url_preview thumbnails<commit_after># -*- coding: utf-8 -*-
# Copyright 2018 New Vecotr Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from twisted.internet import defer, threads
from twisted.protocols.basic import FileSender

from ._base import Responder

from synapse.util.logcontext import make_deferred_yieldable

import contextlib
import os
import logging
import shutil
import sys

logger = logging.getLogger(__name__)

class MediaStorage(object):
    """Responsible for storing/fetching files from local sources.

    Args:
        local_media_directory (str): Base path where we store media on disk
        filepaths (MediaFilePaths)
        storage_providers ([StorageProvider]): List of StorageProvider that are
            used to fetch and store files.
    """

    def __init__(self, local_media_directory, filepaths, storage_providers):
        self.local_media_directory = local_media_directory
        self.filepaths = filepaths
        self.storage_providers = storage_providers

    @defer.inlineCallbacks
    def store_file(self, source, file_info):
        """Write `source` to the on disk media store, and also any other
        configured storage providers

        Args:
            source: A file like object that should be written
            file_info (FileInfo): Info about the file to store

        Returns:
            Deferred[str]: the file path written to in the primary media store
        """
        path = self._file_info_to_path(file_info)
        fname = os.path.join(self.local_media_directory, path)

        dirname = os.path.dirname(fname)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        # Write to the main repository
        yield make_deferred_yieldable(threads.deferToThread(
            _write_file_synchronously, source, fname,
        ))

        defer.returnValue(fname)

    @contextlib.contextmanager
    def store_into_file(self, file_info):
        """Context manager used to get a file like object to write into, as
        described by file_info.

        Actually yields a 3-tuple (file, fname, finish_cb), where file is a file
        like object that can be written to, fname is the absolute path of file
        on disk, and finish_cb is a function that returns a Deferred.

        fname can be used to read the contents from after upload, e.g. to
        generate thumbnails.

        finish_cb must be called and waited on after the file has been
        successfully been written to. Should not be called if there was an
        error.

        Args:
            file_info (FileInfo): Info about the file to store

        Example:

            with media_storage.store_into_file(info) as (f, fname, finish_cb):
                # .. write into f ...
                yield finish_cb()
        """

        path = self._file_info_to_path(file_info)
        fname = os.path.join(self.local_media_directory, path)

        dirname = os.path.dirname(fname)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        finished_called = [False]

        @defer.inlineCallbacks
        def finish():
            for provider in self.storage_providers:
                yield provider.store_file(path, file_info)

            finished_called[0] = True

        try:
            with open(fname, "wb") as f:
                yield f, fname, finish
        except Exception:
            t, v, tb = sys.exc_info()
            try:
                os.remove(fname)
            except Exception:
                pass
            raise t, v, tb

        if not finished_called:
            raise Exception("Finished callback not called")

    @defer.inlineCallbacks
    def fetch_media(self, file_info):
        """Attempts to fetch media described by file_info from the local cache
        and configured storage providers.

        Args:
            file_info (FileInfo)

        Returns:
            Deferred[Responder|None]: Returns a Responder if the file was found,
                otherwise None.
        """

        path = self._file_info_to_path(file_info)
        local_path = os.path.join(self.local_media_directory, path)
        if os.path.exists(local_path):
            defer.returnValue(FileResponder(open(local_path, "rb")))

        for provider in self.storage_providers:
            res = yield provider.fetch(path, file_info)
            if res:
                defer.returnValue(res)

        defer.returnValue(None)

    def _file_info_to_path(self, file_info):
        """Converts file_info into a relative path.

        The path is suitable for storing files under a directory, e.g. used to
        store files on local FS under the base media repository directory.

        Args:
            file_info (FileInfo)

        Returns:
            str
        """
        if file_info.url_cache:
            if file_info.thumbnail:
                return self.filepaths.url_cache_thumbnail_rel(
                    media_id=file_info.file_id,
                    width=file_info.thumbnail_width,
                    height=file_info.thumbnail_height,
                    content_type=file_info.thumbnail_type,
                    method=file_info.thumbnail_method,
                )
            return self.filepaths.url_cache_filepath_rel(file_info.file_id)

        if file_info.server_name:
            if file_info.thumbnail:
                return self.filepaths.remote_media_thumbnail_rel(
                    server_name=file_info.server_name,
                    file_id=file_info.file_id,
                    width=file_info.thumbnail_width,
                    height=file_info.thumbnail_height,
                    content_type=file_info.thumbnail_type,
                    method=file_info.thumbnail_method
                )
            return self.filepaths.remote_media_filepath_rel(
                file_info.server_name, file_info.file_id,
            )

        if file_info.thumbnail:
            return self.filepaths.local_media_thumbnail_rel(
                media_id=file_info.file_id,
                width=file_info.thumbnail_width,
                height=file_info.thumbnail_height,
                content_type=file_info.thumbnail_type,
                method=file_info.thumbnail_method
            )
        return self.filepaths.local_media_filepath_rel(
            file_info.file_id,
        )

def _write_file_synchronously(source, fname):
    """Write `source` to the path `fname` synchronously. Should be called
    from a thread.

    Args:
        source: A file like object to be written
        fname (str): Path to write to
    """
    dirname = os.path.dirname(fname)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    source.seek(0)  # Ensure we read from the start of the file
    with open(fname, "wb") as f:
        shutil.copyfileobj(source, f)

class FileResponder(Responder):
    """Wraps an open file that can be sent to a request.

    Args:
        open_file (file): A file like object to be streamed ot the client,
            is closed when finished streaming.
    """
    def __init__(self, open_file):
        self.open_file = open_file

    @defer.inlineCallbacks
    def write_to_consumer(self, consumer):
        yield FileSender().beginFileTransfer(self.open_file, consumer)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.open_file.close()
<|endoftext|>


<commit_before># Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from synapse.storage.prepare_database import get_statements
from synapse.storage.engines import PostgresEngine, Sqlite3Engine

import ujson

logger = logging.getLogger(__name__)

POSTGRES_TABLE = """
CREATE TABLE IF NOT EXISTS event_search (
    event_id TEXT,
    room_id TEXT,
    sender TEXT,
    key TEXT,
    vector tsvector
);

CREATE INDEX event_search_fts_idx ON event_search USING gin(vector);
CREATE INDEX event_search_ev_idx ON event_search(event_id);
CREATE INDEX event_search_ev_ridx ON event_search(room_id);
"""

SQLITE_TABLE = (
    "CREATE VIRTUAL TABLE IF NOT EXISTS event_search"
    " USING fts4 ( event_id, room_id, sender, key, value )"
)

def run_upgrade(cur, database_engine, *args, **kwargs):
    if isinstance(database_engine, PostgresEngine):
        for statement in get_statements(POSTGRES_TABLE.splitlines()):
            cur.execute(statement)
    elif isinstance(database_engine, Sqlite3Engine):
        cur.execute(SQLITE_TABLE)
    else:
        raise Exception("Unrecognized database engine")

    cur.execute("SELECT MIN(stream_ordering) FROM events")
    rows = cur.fetchall()
    min_stream_id = rows[0][0]

    cur.execute("SELECT MAX(stream_ordering) FROM events")
    rows = cur.fetchall()
    max_stream_id = rows[0][0]

    if min_stream_id is not None and max_stream_id is not None:
        progress = {
            "target_min_stream_id_inclusive": min_stream_id,
            "max_stream_id_exclusive": max_stream_id + 1,
            "rows_inserted": 0,
        }
        progress_json = ujson.dumps(progress)

        cur.execute(
            "INSERT into background_updates (update_name, progress_json)"
            " VALUES (?, ?)", ("event_search", progress_json)
        )
<commit_msg>Fix param style to work on both sqlite and postgres<commit_after># Copyright 2015 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from synapse.storage.prepare_database import get_statements
from synapse.storage.engines import PostgresEngine, Sqlite3Engine

import ujson

logger = logging.getLogger(__name__)

POSTGRES_TABLE = """
CREATE TABLE IF NOT EXISTS event_search (
    event_id TEXT,
    room_id TEXT,
    sender TEXT,
    key TEXT,
    vector tsvector
);

CREATE INDEX event_search_fts_idx ON event_search USING gin(vector);
CREATE INDEX event_search_ev_idx ON event_search(event_id);
CREATE INDEX event_search_ev_ridx ON event_search(room_id);
"""

SQLITE_TABLE = (
    "CREATE VIRTUAL TABLE IF NOT EXISTS event_search"
    " USING fts4 ( event_id, room_id, sender, key, value )"
)

def run_upgrade(cur, database_engine, *args, **kwargs):
    if isinstance(database_engine, PostgresEngine):
        for statement in get_statements(POSTGRES_TABLE.splitlines()):
            cur.execute(statement)
    elif isinstance(database_engine, Sqlite3Engine):
        cur.execute(SQLITE_TABLE)
    else:
        raise Exception("Unrecognized database engine")

    cur.execute("SELECT MIN(stream_ordering) FROM events")
    rows = cur.fetchall()
    min_stream_id = rows[0][0]

    cur.execute("SELECT MAX(stream_ordering) FROM events")
    rows = cur.fetchall()
    max_stream_id = rows[0][0]

    if min_stream_id is not None and max_stream_id is not None:
        progress = {
            "target_min_stream_id_inclusive": min_stream_id,
            "max_stream_id_exclusive": max_stream_id + 1,
            "rows_inserted": 0,
        }
        progress_json = ujson.dumps(progress)

        sql = (
            "INSERT into background_updates (update_name, progress_json)"
            " VALUES (?, ?)"
        )

        sql = database_engine.convert_param_style(sql)

        cur.execute(sql, ("event_search", progress_json))
<|endoftext|>