<commit_before># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Imports for problem modules."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import importlib
from six.moves import range  # pylint: disable=redefined-builtin

MODULES = [
    "tensor2tensor.data_generators.algorithmic",
    "tensor2tensor.data_generators.algorithmic_math",
    "tensor2tensor.data_generators.algorithmic_math_deepmind",
    "tensor2tensor.data_generators.algorithmic_math_two_variables",
    "tensor2tensor.data_generators.allen_brain",
    "tensor2tensor.data_generators.audio",
    "tensor2tensor.data_generators.babi_qa",
    "tensor2tensor.data_generators.bair_robot_pushing",
    "tensor2tensor.data_generators.celeba",
    "tensor2tensor.data_generators.celebahq",
    "tensor2tensor.data_generators.cifar",
    "tensor2tensor.data_generators.cipher",
    "tensor2tensor.data_generators.cnn_dailymail",
    "tensor2tensor.data_generators.cola",
    "tensor2tensor.data_generators.common_voice",
    "tensor2tensor.data_generators.desc2code",
    "tensor2tensor.data_generators.fsns",
    "tensor2tensor.data_generators.function_docstring",
    "tensor2tensor.data_generators.gene_expression",
    "tensor2tensor.data_generators.google_robot_pushing",
    "tensor2tensor.data_generators.gym_env",
    "tensor2tensor.data_generators.ice_parsing",
    "tensor2tensor.data_generators.imagenet",
    "tensor2tensor.data_generators.image_lsun",
    "tensor2tensor.data_generators.imdb",
    "tensor2tensor.data_generators.lambada",
    "tensor2tensor.data_generators.librispeech",
    "tensor2tensor.data_generators.lm1b",
    "tensor2tensor.data_generators.lm1b_imdb",
    "tensor2tensor.data_generators.lm1b_mnli",
    "tensor2tensor.data_generators.mnist",
    "tensor2tensor.data_generators.mrpc",
    "tensor2tensor.data_generators.mscoco",
    "tensor2tensor.data_generators.multinli",
    "tensor2tensor.data_generators.paraphrase_ms_coco",
    "tensor2tensor.data_generators.program_search",
    "tensor2tensor.data_generators.ocr",
    "tensor2tensor.data_generators.pointer_generator_word",
    "tensor2tensor.data_generators.problem_hparams",
    "tensor2tensor.data_generators.ptb",
    "tensor2tensor.data_generators.qnli",
    "tensor2tensor.data_generators.quora_qpairs",
    "tensor2tensor.data_generators.rte",
    "tensor2tensor.data_generators.scitail",
    "tensor2tensor.data_generators.snli",
    "tensor2tensor.data_generators.stanford_nli",
    "tensor2tensor.data_generators.style_transfer",
    "tensor2tensor.data_generators.squad",
    "tensor2tensor.data_generators.sst_binary",
    "tensor2tensor.data_generators.subject_verb_agreement",
    "tensor2tensor.data_generators.timeseries",
    "tensor2tensor.data_generators.transduction_problems",
    "tensor2tensor.data_generators.translate_encs",
    "tensor2tensor.data_generators.translate_ende",
    "tensor2tensor.data_generators.translate_enet",
    "tensor2tensor.data_generators.translate_enfr",
    "tensor2tensor.data_generators.translate_enid",
    "tensor2tensor.data_generators.translate_enmk",
    "tensor2tensor.data_generators.translate_envi",
    "tensor2tensor.data_generators.translate_enzh",
    "tensor2tensor.data_generators.video_generated",
    "tensor2tensor.data_generators.vqa",
    "tensor2tensor.data_generators.wiki",
    "tensor2tensor.data_generators.wiki_lm",
    "tensor2tensor.data_generators.wiki_revision",
    "tensor2tensor.data_generators.wiki_multi_problems",
    "tensor2tensor.data_generators.wikisum.wikisum",
    "tensor2tensor.data_generators.wikitext103",
    "tensor2tensor.data_generators.wsj_parsing",
    "tensor2tensor.data_generators.wnli",
    "tensor2tensor.envs.mujoco_problems",
    "tensor2tensor.envs.tic_tac_toe_env_problem",
]
ALL_MODULES = list(MODULES)

def _is_import_err_msg(err_str, module):
  parts = module.split(".")
  suffixes = [".".join(parts[i:]) for i in range(len(parts))]
  return err_str in (
      ["No module named %s" % suffix for suffix in suffixes] +
      ["No module named '%s'" % suffix for suffix in suffixes])

def _handle_errors(errors):
  """Log out and possibly reraise errors during import."""
  if not errors:
    return
  log_all = True  # pylint: disable=unused-variable
  err_msg = "T2T: skipped importing {num_missing} data_generators modules."
  print(err_msg.format(num_missing=len(errors)))
  for module, err in errors:
    err_str = str(err)
    if not _is_import_err_msg(err_str, module):
      print("From module %s" % module)
      raise err
    if log_all:
      print("Did not import module: %s; Cause: %s" % (module, err_str))

def import_modules(modules):
  errors = []
  for module in modules:
    try:
      importlib.import_module(module)
    except ImportError as error:
      errors.append((module, error))
  _handle_errors(errors)
<commit_msg>Internal<commit_after># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Imports for problem modules."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import importlib
import six
from six.moves import range  # pylint: disable=redefined-builtin

MODULES = [
    "tensor2tensor.data_generators.algorithmic",
    "tensor2tensor.data_generators.algorithmic_math",
    "tensor2tensor.data_generators.algorithmic_math_deepmind",
    "tensor2tensor.data_generators.algorithmic_math_two_variables",
    "tensor2tensor.data_generators.allen_brain",
    "tensor2tensor.data_generators.audio",
    "tensor2tensor.data_generators.babi_qa",
    "tensor2tensor.data_generators.bair_robot_pushing",
    "tensor2tensor.data_generators.celeba",
    "tensor2tensor.data_generators.celebahq",
    "tensor2tensor.data_generators.cifar",
    "tensor2tensor.data_generators.cipher",
    "tensor2tensor.data_generators.cnn_dailymail",
    "tensor2tensor.data_generators.cola",
    "tensor2tensor.data_generators.common_voice",
    "tensor2tensor.data_generators.desc2code",
    "tensor2tensor.data_generators.fsns",
    "tensor2tensor.data_generators.function_docstring",
    "tensor2tensor.data_generators.gene_expression",
    "tensor2tensor.data_generators.google_robot_pushing",
    "tensor2tensor.data_generators.gym_env",
    "tensor2tensor.data_generators.ice_parsing",
    "tensor2tensor.data_generators.imagenet",
    "tensor2tensor.data_generators.image_lsun",
    "tensor2tensor.data_generators.imdb",
    "tensor2tensor.data_generators.lambada",
    "tensor2tensor.data_generators.librispeech",
    "tensor2tensor.data_generators.lm1b",
    "tensor2tensor.data_generators.lm1b_imdb",
    "tensor2tensor.data_generators.lm1b_mnli",
    "tensor2tensor.data_generators.mnist",
    "tensor2tensor.data_generators.mrpc",
    "tensor2tensor.data_generators.mscoco",
    "tensor2tensor.data_generators.multinli",
    "tensor2tensor.data_generators.paraphrase_ms_coco",
    "tensor2tensor.data_generators.program_search",
    "tensor2tensor.data_generators.ocr",
    "tensor2tensor.data_generators.pointer_generator_word",
    "tensor2tensor.data_generators.problem_hparams",
    "tensor2tensor.data_generators.ptb",
    "tensor2tensor.data_generators.qnli",
    "tensor2tensor.data_generators.quora_qpairs",
    "tensor2tensor.data_generators.rte",
    "tensor2tensor.data_generators.scitail",
    "tensor2tensor.data_generators.snli",
    "tensor2tensor.data_generators.stanford_nli",
    "tensor2tensor.data_generators.style_transfer",
    "tensor2tensor.data_generators.squad",
    "tensor2tensor.data_generators.sst_binary",
    "tensor2tensor.data_generators.subject_verb_agreement",
    "tensor2tensor.data_generators.timeseries",
    "tensor2tensor.data_generators.transduction_problems",
    "tensor2tensor.data_generators.translate_encs",
    "tensor2tensor.data_generators.translate_ende",
    "tensor2tensor.data_generators.translate_enet",
    "tensor2tensor.data_generators.translate_enfr",
    "tensor2tensor.data_generators.translate_enid",
    "tensor2tensor.data_generators.translate_enmk",
    "tensor2tensor.data_generators.translate_envi",
    "tensor2tensor.data_generators.translate_enzh",
    "tensor2tensor.data_generators.video_generated",
    "tensor2tensor.data_generators.vqa",
    "tensor2tensor.data_generators.wiki",
    "tensor2tensor.data_generators.wiki_lm",
    "tensor2tensor.data_generators.wiki_revision",
    "tensor2tensor.data_generators.wiki_multi_problems",
    "tensor2tensor.data_generators.wikisum.wikisum",
    "tensor2tensor.data_generators.wikitext103",
    "tensor2tensor.data_generators.wsj_parsing",
    "tensor2tensor.data_generators.wnli",
    "tensor2tensor.envs.mujoco_problems",
    "tensor2tensor.envs.tic_tac_toe_env_problem",
]
ALL_MODULES = list(MODULES)

def _is_import_err_msg(err_str, module):
  parts = module.split(".")
  suffixes = [".".join(parts[i:]) for i in range(len(parts))]
  return err_str in (
      ["No module named %s" % suffix for suffix in suffixes] +
      ["No module named '%s'" % suffix for suffix in suffixes])

def _handle_errors(errors):
  """Log out and possibly reraise errors during import."""
  if not errors:
    return
  log_all = True  # pylint: disable=unused-variable
  err_msg = "T2T: skipped importing {num_missing} data_generators modules."
  print(err_msg.format(num_missing=len(errors)))
  for module, err in errors:
    err_str = str(err)
    if not _is_import_err_msg(err_str, module):
      print("From module %s" % module)
      raise err
    if log_all:
      print("Did not import module: %s; Cause: %s" % (module, err_str))

def import_modules(modules):
  errors = []
  for module in modules:
    try:
      importlib.import_module(module)
    except ImportError as error:
      errors.append((module, error))
  _handle_errors(errors)
<|endoftext|>


<commit_before>import json

from kafka import KafkaConsumer

from get_omdb import GetOMDB

consumer = KafkaConsumer(group_id = 'omdb',
                         bootstrap_servers = ['212.111.42.214:9092'])

try:
    from GLOBALS import KAFKA_BROKER, OMDB_API
except ImportError:
    print('Get it somewhere else')

class CollectOMDB(object):

    def __init__(self):
        self.omdb = GetOMDB(api_key = OMDB_API)
        self.consumer = KafkaConsumer(group_id = 'omdb',
                                      bootstrap_servers = ['{0}'.format(KAFKA_BROKER)])
        self.consuer.subscribe(pattern = 'omdb')

    def run(self):
        '''
        Collects a message from the topic 'omdb'. This message is a json containing all the
        information collected from the apis further up the stream. We get the imdb_id from
        this data and pass it to the tmdb api. We append the information from the tmdb_api
        to the msg we collected, and then pass it to the next topic.
        '''
        for message in self.consumer:
            # message value and key are raw bytes -- decode if necessary!
            # e.g., for unicode: `message.value.decode('utf-8')
            msg_data  = json.loads(message.value)
            imdb_id = msg_data['imdb_id']
            omdb_data = self.omdb.get_info(imdb_id)
            msg_data.extend(omdb_data)

<commit_msg>Adding branch containing omdb kafka consumer<commit_after>import json

from kafka import KafkaConsumer, KafkaProducer

from get_omdb import GetOMDB

try:
    from GLOBALS import KAFKA_BROKER, OMDB_API
except ImportError:
    print('Get it somewhere else')

class CollectOMDB(object):

    def __init__(self):
        self.omdb = GetOMDB(api_key = OMDB_API)
        self.topic = 'omdb'
        self.consumer = KafkaConsumer(group_id = self.topic,
                                      bootstrap_servers=KAFKA_BROKER)
        self.consumer.subscribe(pattern = 'imdb_ids')
        self.producer = KafkaProducer(bootstrap_servers=KAFKA_BROKER)

    def run(self):
        '''
        Collects a message from the topic 'omdb'. This message is a json containing all the
        information collected from the apis further up the stream. We get the imdb_id from
        this data and pass it to the tmdb api. We append the information from the tmdb_api
        to the msg we collected, and then pass it to the next topic.
        '''
        for message in self.consumer:
            # message value and key are raw bytes -- decode if necessary!
            # e.g., for unicode: `message.value.decode('utf-8')
            msg_data  = json.loads(message.value)
            imdb_id = msg_data['imdb_id']
            omdb_data = self.omdb.get_info(imdb_id)
            msg_data.update(omdb_data)
            self.producer.send(self.topic, json.dumps(msg_data).encode())
            print('Processed: {}'.format(imdb_id))

if __name__ == '__main__':
    c = CollectOMDB()
    c.run()

<|endoftext|>


<commit_before>"""
    test collection and execution events
"""

import py
import time
from py.__.test.outcome import Skipped

class EventBus(object):
    """ General Event Bus for distributing events. """
    def __init__(self):
        self._subscribers = []

    def subscribe(self, callback):
        """ subscribe given callback to bus events. """
        self._subscribers.append(callback)

    def unsubscribe(self, callback):
        """ unsubscribe given callback from bus events. """
        self._subscribers.remove(callback)

    def notify(self, event):
        for subscriber in self._subscribers:
            subscriber(event)

class BaseEvent(object):
    def __repr__(self):
        l = ["%s=%s" %(key, value)
           for key, value in self.__dict__.items()]
        return "<%s %s>" %(self.__class__.__name__, " ".join(l),)

def timestamp():
    return time.time()

class NOP(BaseEvent):
    pass

# ----------------------------------------------------------------------
# Basic Live Reporting Events
# ----------------------------------------------------------------------

class TestrunStart(BaseEvent):
    def __init__(self):
        self.timestart = time.time()

class TestrunFinish(BaseEvent):
    def __init__(self, exitstatus=0):
        self.exitstatus = exitstatus
        self.timeend = time.time()

class InternalException(BaseEvent):
    def __init__(self, excinfo=None):
        if excinfo is None:
            excinfo = py.code.ExceptionInfo()
        self.repr = excinfo.getrepr(funcargs=True, showlocals=True)

# ----------------------------------------------------------------------
# Events related to collecting and executing test Items
# ----------------------------------------------------------------------

class ItemStart(BaseEvent):
    def __init__(self, item, host=None):
        self.item = item
        self.host = host
        self.time = timestamp()

class Deselected(BaseEvent):
    def __init__(self, items):
        self.items = items

class BaseReport(BaseEvent):
    failed = passed = skipped = None
    def __init__(self, colitem, **kwargs):
        self.colitem = colitem
        assert len(kwargs) == 1, kwargs
        name, value = kwargs.items()[0]
        setattr(self, name, True)
        self.outcome = value

    def toterminal(self, out):
        longrepr = self.outcome.longrepr
        if hasattr(longrepr, 'toterminal'):
            longrepr.toterminal(out)
        else:
            out.line(str(longrepr))

class ItemTestReport(BaseReport):
    """ Test Execution Report. """

class CollectionStart(BaseEvent):
    def __init__(self, collector):
        self.collector = collector

class CollectionReport(BaseReport):
    """ Collection Report. """
    def __init__(self, colitem, result, **kwargs):
        super(CollectionReport, self).__init__(colitem, **kwargs)
        self.result = result

class LooponfailingInfo(BaseEvent):
    def __init__(self, failreports, rootdirs):
        self.failreports = failreports
        self.rootdirs = rootdirs

# ----------------------------------------------------------------------
# Distributed Testing Events
# ----------------------------------------------------------------------

class RescheduleItems(BaseEvent):
    def __init__(self, items):
        self.items = items

class HostGatewayReady(BaseEvent):
    def __init__(self, host, roots):
        self.host = host
        self.roots = roots

class HostUp(BaseEvent):
    def __init__(self, host, platinfo):
        self.host = host
        self.platinfo = platinfo

class HostDown(BaseEvent):
    def __init__(self, host, error=None):
        self.host = host
        self.error = error

#
# events related to rsyncing
#

class HostRSyncing(BaseEvent):
    def __init__(self, host, root, remotepath, synced):
        self.host = host
        self.root = root
        self.remotepath = remotepath
        self.synced = synced

class RsyncFinished(BaseEvent):
    def __init__(self):
        self.time = timestamp()

class HostRSyncRootReady(BaseEvent):
    def __init__(self, host, root):
        self.host = host
        self.root = root

<commit_msg>[svn r57609] Keep comments in the same style accross the file<commit_after>"""
    test collection and execution events
"""

import py
import time
from py.__.test.outcome import Skipped

class EventBus(object):
    """ General Event Bus for distributing events. """
    def __init__(self):
        self._subscribers = []

    def subscribe(self, callback):
        """ subscribe given callback to bus events. """
        self._subscribers.append(callback)

    def unsubscribe(self, callback):
        """ unsubscribe given callback from bus events. """
        self._subscribers.remove(callback)

    def notify(self, event):
        for subscriber in self._subscribers:
            subscriber(event)

class BaseEvent(object):
    def __repr__(self):
        l = ["%s=%s" %(key, value)
           for key, value in self.__dict__.items()]
        return "<%s %s>" %(self.__class__.__name__, " ".join(l),)

def timestamp():
    return time.time()

class NOP(BaseEvent):
    pass

# ----------------------------------------------------------------------
# Basic Live Reporting Events
# ----------------------------------------------------------------------

class TestrunStart(BaseEvent):
    def __init__(self):
        self.timestart = time.time()

class TestrunFinish(BaseEvent):
    def __init__(self, exitstatus=0):
        self.exitstatus = exitstatus
        self.timeend = time.time()

class InternalException(BaseEvent):
    def __init__(self, excinfo=None):
        if excinfo is None:
            excinfo = py.code.ExceptionInfo()
        self.repr = excinfo.getrepr(funcargs=True, showlocals=True)

# ----------------------------------------------------------------------
# Events related to collecting and executing test Items
# ----------------------------------------------------------------------

class ItemStart(BaseEvent):
    def __init__(self, item, host=None):
        self.item = item
        self.host = host
        self.time = timestamp()

class Deselected(BaseEvent):
    def __init__(self, items):
        self.items = items

class BaseReport(BaseEvent):
    failed = passed = skipped = None
    def __init__(self, colitem, **kwargs):
        self.colitem = colitem
        assert len(kwargs) == 1, kwargs
        name, value = kwargs.items()[0]
        setattr(self, name, True)
        self.outcome = value

    def toterminal(self, out):
        longrepr = self.outcome.longrepr
        if hasattr(longrepr, 'toterminal'):
            longrepr.toterminal(out)
        else:
            out.line(str(longrepr))

class ItemTestReport(BaseReport):
    """ Test Execution Report. """

class CollectionStart(BaseEvent):
    def __init__(self, collector):
        self.collector = collector

class CollectionReport(BaseReport):
    """ Collection Report. """
    def __init__(self, colitem, result, **kwargs):
        super(CollectionReport, self).__init__(colitem, **kwargs)
        self.result = result

class LooponfailingInfo(BaseEvent):
    def __init__(self, failreports, rootdirs):
        self.failreports = failreports
        self.rootdirs = rootdirs

# ----------------------------------------------------------------------
# Distributed Testing Events
# ----------------------------------------------------------------------

class RescheduleItems(BaseEvent):
    def __init__(self, items):
        self.items = items

class HostGatewayReady(BaseEvent):
    def __init__(self, host, roots):
        self.host = host
        self.roots = roots

class HostUp(BaseEvent):
    def __init__(self, host, platinfo):
        self.host = host
        self.platinfo = platinfo

class HostDown(BaseEvent):
    def __init__(self, host, error=None):
        self.host = host
        self.error = error

# ---------------------------------------------------------------------
# Events related to rsyncing
# ---------------------------------------------------------------------

class HostRSyncing(BaseEvent):
    def __init__(self, host, root, remotepath, synced):
        self.host = host
        self.root = root
        self.remotepath = remotepath
        self.synced = synced

class RsyncFinished(BaseEvent):
    def __init__(self):
        self.time = timestamp()

class HostRSyncRootReady(BaseEvent):
    def __init__(self, host, root):
        self.host = host
        self.root = root

<|endoftext|>


<commit_before>#!/usr/bin/python3

import sys
import pykms

# draw test pattern via dmabuf?
dmabuf = False

# Use omap?
omap = False

if omap:
	card = pykms.OmapCard()
else:
	card = pykms.Card()

if len(sys.argv) > 1:
    conn_name = sys.argv[1]
else:
    conn_name = ""

res = pykms.ResourceManager(card)
conn = res.reserve_connector(conn_name)
crtc = res.reserve_crtc(conn)
plane = res.reserve_generic_plane(crtc)
mode = conn.get_default_mode()
modeb = mode.to_blob(card)

if omap:
	origfb = pykms.OmapFramebuffer(card, mode.hdisplay, mode.vdisplay, "XR24");
else:
	origfb = pykms.DumbFramebuffer(card, mode.hdisplay, mode.vdisplay, "XR24");

if dmabuf:
	fb = pykms.ExtFramebuffer(card, origfb.width, origfb.height, origfb.format,
		[origfb.fd(0)], [origfb.stride(0)], [origfb.offset(0)])
else:
	fb = origfb

pykms.draw_test_pattern(fb);

card.disable_planes()

req = pykms.AtomicReq(card)

req.add_connector(conn, crtc)
req.add_crtc(crtc, modeb)
req.add_plane(plane, fb, crtc, dst=(0, 0, mode.hdisplay, mode.vdisplay))

req.commit_sync(allow_modeset = True)

input("press enter to exit\n")
<commit_msg>test.py: cleanup<commit_after>#!/usr/bin/python3

import sys
import pykms
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--connector", default="")
parser.add_argument("--dmabuf", action="store_true", help="use dmabuf")
parser.add_argument("--omap", action="store_true", help="use omapcard")
args = parser.parse_args()

if args.omap:
	card = pykms.OmapCard()
else:
	card = pykms.Card()

res = pykms.ResourceManager(card)
conn = res.reserve_connector(args.connector)
crtc = res.reserve_crtc(conn)
plane = res.reserve_generic_plane(crtc)
mode = conn.get_default_mode()
modeb = mode.to_blob(card)

if args.omap:
	origfb = pykms.OmapFramebuffer(card, mode.hdisplay, mode.vdisplay, "XR24");
else:
	origfb = pykms.DumbFramebuffer(card, mode.hdisplay, mode.vdisplay, "XR24");

if args.dmabuf:
	fb = pykms.ExtFramebuffer(card, origfb.width, origfb.height, origfb.format,
		[origfb.fd(0)], [origfb.stride(0)], [origfb.offset(0)])
else:
	fb = origfb

pykms.draw_test_pattern(fb);

card.disable_planes()

req = pykms.AtomicReq(card)

req.add_connector(conn, crtc)
req.add_crtc(crtc, modeb)
req.add_plane(plane, fb, crtc, dst=(0, 0, mode.hdisplay, mode.vdisplay))

req.commit_sync(allow_modeset = True)

input("press enter to exit\n")
<|endoftext|>


<commit_before>"""
    A custom Storage interface for storing files to S3.
"""
from contextlib import closing
from datetime import datetime
import hashlib
import hmac
from http.client import HTTPConnection
import itertools
import os
import urllib.parse
from wsgiref.handlers import format_date_time

from .files import File
from .files import S3ContentFile
from .utils import b64_string
from .utils import ENCODING

class S3IOError(IOError):
    pass

class S3FileDoesNotExistError(S3IOError):
    def __init__(self, name=None, msg=None):
        total_msg = 'File does not exist: {}'.format(name)
        if msg:
            total_msg += ' {}'.format(msg)
        super().__init__(total_msg)

class Storage(object):
    """
    A base storage class, providing some default behaviors that all other
    storage systems can inherit or override, as necessary.
    """

    # The following methods represent a public interface to private methods.
    # These shouldn't be overridden by subclasses unless absolutely necessary.

    def open(self, name, mode='rb'):
        """
        Retrieves the specified file from storage.
        """
        return self._open(name, mode)

    def save(self, name, file):
        """
        Saves new content to the file specified by name. The content should be
        a proper File object or any python file-like object, ready to be read
        from the beginning.
        """
        # Get the proper name for the file, as it will actually be saved.
        if name is None:
            name = file.name

        if not hasattr(file, 'chunks'):
            file = File(file, name=name)

        name = self.get_available_name(name)
        name = self._save(name, file)

        # Store filenames with forward slashes, even on Windows
        return name.replace('\\', '/')

    # These methods are part of the public API, with default implementations.

    def get_valid_name(self, name):
        """
        Returns a filename, based on the provided filename, that's suitable for
        use in the target storage system.
        """
        return get_valid_filename(name)

    def get_available_name(self, name):
        """
        Returns a filename that's free on the target storage system, and
        available for new content to be written to.
        """
        dir_name, file_name = os.path.split(name)
        file_root, file_ext = os.path.splitext(file_name)
        # If the filename already exists, add an underscore and a number (before
        # the file extension, if one exists) to the filename until the generated
        # filename doesn't exist.
        count = itertools.count(1)
        while self.exists(name):
            # file_ext includes the dot.
            name = os.path.join(dir_name, "{}_{}{}".format(file_root, next(count), file_ext))

        return name

    def path(self, name):
        """
        Returns a local filesystem path where the file can be retrieved using
        Python's built-in open() function. Storage systems that can't be
        accessed using open() should *not* implement this method.
        """
        raise NotImplementedError("This backend doesn't support absolute paths.")

    # The following methods form the public API for storage systems, but with
    # no default implementations. Subclasses must implement *all* of these.

    def delete(self, name):
        """
        Deletes the specified file from the storage system.
        """
        raise NotImplementedError('subclasses of Storage must provide a delete() method')

    def exists(self, name):
        """
        Returns True if a file referened by the given name already exists in the
        storage system, or False if the name is available for a new file.
        """
        raise NotImplementedError('subclasses of Storage must provide a exists() method')

    def listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.
        """
        raise NotImplementedError('subclasses of Storage must provide a listdir() method')

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a size() method')

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a Web browser.
        """
        raise NotImplementedError('subclasses of Storage must provide a url() method')

    def accessed_time(self, name):
        """
        Returns the last accessed time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide an accessed_time() method')

    def created_time(self, name):
        """
        Returns the creation time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a created_time() method')

    def modified_time(self, name):
        """
        Returns the last modified time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a modified_time() method')

class S3Storage(Storage):
    """
    A custom storage implimentation for use with py3s3.

    An instance of this class can be used to move a py3s3 file
    up to or down from AWS.
    """
    def __init__(self, name_prefix, bucket, aws_access_key, aws_secret_key):
        self.name_prefix = name_prefix
        self.bucket = bucket
        self.access_key = aws_access_key
        self.secret_key = aws_secret_key
        self.netloc = '{}.s3.amazonaws.com'.format(bucket)

    @staticmethod
    def request_timestamp(timestamp=None):
        if timestamp is None:
            timestamp = datetime.now().timestamp()
        return format_date_time(timestamp)

    @staticmethod
    def datetime_from_aws_timestamp(timestamp):
        """
        Return datetime from parsed AWS header timestamp string.
        AWS Format:  Wed, 28 Oct 2009 22:32:00 GMT
        """
        fmt = '%a, %d %b %Y %X %Z'
        return datetime.strptime(timestamp, fmt)

    def _prepend_name_prefix(self, name):
        """Return file name (ie. path) with the prefix directory prepended"""
        if not self.name_prefix:
            return name
        base = self.name_prefix
        if base[0] != '/':
            base = '/' + base
        if name[0] != '/':
            name = '/' + name
        return base + name

    def request_signature(self, stringtosign):
        """
        Construct a signature by making an RFC2104 HMAC-SHA1
        of the following and converting it to Base64 UTF-8 encoded string.
        """
        digest = hmac.new(
            self.secret_key.encode(ENCODING),
            stringtosign.encode(ENCODING),
            hashlib.sha1
        ).digest()
        return b64_string(digest)

    def _put_file(self, file):
        """Send PUT request to S3 with file contents"""
        timestamp = self.request_timestamp()

        # build headers
        headers = dict()
        headers['Date'] = timestamp
        headers['Content-Length'] = file.size
        headers['Content-MD5'] = file.md5hash()
        if file.mimetype:
            headers['Content-Type'] = file.mimetype
        headers['x-amz-acl'] = 'public-read'

        stringtosign = '\n'.join([
            'PUT',
            file.md5hash(),
            file.mimetype,
            timestamp,
            'x-amz-acl:public-read',
            '/' + self.bucket + file.name
        ])
        signature = self.request_signature(stringtosign)

        headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])

        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('PUT', file.name, file.read(), headers=headers)
            response = conn.getresponse()

        if response.status not in (200,):
            raise IOError('py3s3 PUT error. Response status: {}'.format(response.status))

    def _save(self, name, file):
        prefixed_name = self._prepend_name_prefix(name)
        file.name = prefixed_name

        mimetype = file.mimetype if hasattr(file, 'mimetype') and file.mimetype else ''

        # Convert file to a S3ContentFile so file
        # can be pushed up to S3.
        if type(file) is not S3ContentFile:
            file = S3ContentFile(file.read(), file.name, mimetype)

        self._put_file(file)
        return name

    def _get_file(self, name):
        """
        Return a signature for use in GET requests
        """
        timestamp = self.request_timestamp()
        stringtosign = '\n'.join([
            'GET',
            '',
            '',
            timestamp,
            '/' + self.bucket + name
        ])
        signature = self.request_signature(stringtosign)
        headers = dict()
        headers['Date'] = timestamp
        if self.access_key and self.secret_key:
            headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])
        file = S3ContentFile('')
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('GET', name, headers=headers)
            response = conn.getresponse()
            if not response.status in (200,):
                if response.length is None:
                    # length == None seems to be returned from GET requests
                    # to non-existing files
                    raise S3FileDoesNotExistError(name)
                # catch all other cases
                raise S3IOError('py3s3 GET error. Response status: {}'.format(response.status))

            file = S3ContentFile(response.read())
        return file

    def _open(self, name, mode='rb'):
        prefixed_name = self._prepend_name_prefix(name)
        file = self._get_file(prefixed_name)
        file.name = name
        return file

    def delete(self, name):
        prefixed_name = self._prepend_name_prefix(name)
        timestamp = self.request_timestamp()
        stringtosign = '\n'.join([
            'DELETE',
            '',
            '',
            timestamp,
            '/' + self.bucket + prefixed_name
        ])
        signature = self.request_signature(stringtosign)
        headers = dict()
        headers['Date'] = timestamp
        if self.access_key and self.secret_key:
            headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('DELETE', prefixed_name, headers=headers)
            response = conn.getresponse()
            if not response.status in (204,):
                raise S3IOError('py3s3 DELETE error. Response status: {}'.format(response.status))

    def exists(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('HEAD', self.url(name))
            response = conn.getresponse()
            return response.status in (200,)

    def listdir(self, path):
        raise NotImplementedError()

    def size(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('GET', self.url(name))
            return conn.getresponse().length

    def url(self, name):
        """Return URL of resource"""
        scheme = 'http'
        path = self._prepend_name_prefix(name)
        query = ''
        fragment = ''
        url_tuple = (scheme, self.netloc, path, query, fragment)
        return urllib.parse.urlunsplit(url_tuple)

    def modified_time(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('HEAD', self.url(name))
            dt_header = conn.getresponse().getheader('Last-Modified', None)
        if dt_header is None:
            raise S3IOError('No modified time available for file: {}'.format(name))
        return self.datetime_from_aws_timestamp(dt_header)
<commit_msg>Reformat S3IOError messages. Added logic to storage exists method.<commit_after>"""
    A custom Storage interface for storing files to S3.
"""
from contextlib import closing
from datetime import datetime
import hashlib
import hmac
from http.client import HTTPConnection
import itertools
import os
import urllib.parse
from wsgiref.handlers import format_date_time

from .files import File
from .files import S3ContentFile
from .utils import b64_string
from .utils import ENCODING

class S3IOError(IOError):
    pass

class S3FileDoesNotExistError(S3IOError):
    def __init__(self, name=None, msg=None):
        total_msg = 'File does not exist: {}'.format(name)
        if msg:
            total_msg += ' {}'.format(msg)
        super().__init__(total_msg)

class Storage(object):
    """
    A base storage class, providing some default behaviors that all other
    storage systems can inherit or override, as necessary.
    """

    # The following methods represent a public interface to private methods.
    # These shouldn't be overridden by subclasses unless absolutely necessary.

    def open(self, name, mode='rb'):
        """
        Retrieves the specified file from storage.
        """
        return self._open(name, mode)

    def save(self, name, file):
        """
        Saves new content to the file specified by name. The content should be
        a proper File object or any python file-like object, ready to be read
        from the beginning.
        """
        # Get the proper name for the file, as it will actually be saved.
        if name is None:
            name = file.name

        if not hasattr(file, 'chunks'):
            file = File(file, name=name)

        name = self.get_available_name(name)
        name = self._save(name, file)

        # Store filenames with forward slashes, even on Windows
        return name.replace('\\', '/')

    # These methods are part of the public API, with default implementations.

    def get_valid_name(self, name):
        """
        Returns a filename, based on the provided filename, that's suitable for
        use in the target storage system.
        """
        return get_valid_filename(name)

    def get_available_name(self, name):
        """
        Returns a filename that's free on the target storage system, and
        available for new content to be written to.
        """
        dir_name, file_name = os.path.split(name)
        file_root, file_ext = os.path.splitext(file_name)
        # If the filename already exists, add an underscore and a number (before
        # the file extension, if one exists) to the filename until the generated
        # filename doesn't exist.
        count = itertools.count(1)
        while self.exists(name):
            # file_ext includes the dot.
            name = os.path.join(dir_name, "{}_{}{}".format(file_root, next(count), file_ext))

        return name

    def path(self, name):
        """
        Returns a local filesystem path where the file can be retrieved using
        Python's built-in open() function. Storage systems that can't be
        accessed using open() should *not* implement this method.
        """
        raise NotImplementedError("This backend doesn't support absolute paths.")

    # The following methods form the public API for storage systems, but with
    # no default implementations. Subclasses must implement *all* of these.

    def delete(self, name):
        """
        Deletes the specified file from the storage system.
        """
        raise NotImplementedError('subclasses of Storage must provide a delete() method')

    def exists(self, name):
        """
        Returns True if a file referened by the given name already exists in the
        storage system, or False if the name is available for a new file.
        """
        raise NotImplementedError('subclasses of Storage must provide a exists() method')

    def listdir(self, path):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.
        """
        raise NotImplementedError('subclasses of Storage must provide a listdir() method')

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a size() method')

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a Web browser.
        """
        raise NotImplementedError('subclasses of Storage must provide a url() method')

    def accessed_time(self, name):
        """
        Returns the last accessed time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide an accessed_time() method')

    def created_time(self, name):
        """
        Returns the creation time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a created_time() method')

    def modified_time(self, name):
        """
        Returns the last modified time (as datetime object) of the file
        specified by name.
        """
        raise NotImplementedError('subclasses of Storage must provide a modified_time() method')

class S3Storage(Storage):
    """
    A custom storage implimentation for use with py3s3.

    An instance of this class can be used to move a py3s3 file
    up to or down from AWS.
    """
    def __init__(self, name_prefix, bucket, aws_access_key, aws_secret_key):
        self.name_prefix = name_prefix
        self.bucket = bucket
        self.access_key = aws_access_key
        self.secret_key = aws_secret_key
        self.netloc = '{}.s3.amazonaws.com'.format(bucket)

    @staticmethod
    def request_timestamp(timestamp=None):
        if timestamp is None:
            timestamp = datetime.now().timestamp()
        return format_date_time(timestamp)

    @staticmethod
    def datetime_from_aws_timestamp(timestamp):
        """
        Return datetime from parsed AWS header timestamp string.
        AWS Format:  Wed, 28 Oct 2009 22:32:00 GMT
        """
        fmt = '%a, %d %b %Y %X %Z'
        return datetime.strptime(timestamp, fmt)

    def _prepend_name_prefix(self, name):
        """Return file name (ie. path) with the prefix directory prepended"""
        if not self.name_prefix:
            return name
        base = self.name_prefix
        if base[0] != '/':
            base = '/' + base
        if name[0] != '/':
            name = '/' + name
        return base + name

    def request_signature(self, stringtosign):
        """
        Construct a signature by making an RFC2104 HMAC-SHA1
        of the following and converting it to Base64 UTF-8 encoded string.
        """
        digest = hmac.new(
            self.secret_key.encode(ENCODING),
            stringtosign.encode(ENCODING),
            hashlib.sha1
        ).digest()
        return b64_string(digest)

    def _put_file(self, file):
        """Send PUT request to S3 with file contents"""
        timestamp = self.request_timestamp()

        # build headers
        headers = dict()
        headers['Date'] = timestamp
        headers['Content-Length'] = file.size
        headers['Content-MD5'] = file.md5hash()
        if file.mimetype:
            headers['Content-Type'] = file.mimetype
        headers['x-amz-acl'] = 'public-read'

        stringtosign = '\n'.join([
            'PUT',
            file.md5hash(),
            file.mimetype,
            timestamp,
            'x-amz-acl:public-read',
            '/' + self.bucket + file.name
        ])
        signature = self.request_signature(stringtosign)

        headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])

        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('PUT', file.name, file.read(), headers=headers)
            response = conn.getresponse()

        if response.status not in (200,):
            raise S3IOError('py3s3 PUT error. Response status: {}. Reason: {}'.format(
                            response.status, response.reason))

    def _save(self, name, file):
        prefixed_name = self._prepend_name_prefix(name)
        file.name = prefixed_name

        mimetype = file.mimetype if hasattr(file, 'mimetype') and file.mimetype else ''

        # Convert file to a S3ContentFile so file
        # can be pushed up to S3.
        if type(file) is not S3ContentFile:
            file = S3ContentFile(file.read(), file.name, mimetype)

        self._put_file(file)
        return name

    def _get_file(self, name):
        """
        Return a signature for use in GET requests
        """
        timestamp = self.request_timestamp()
        stringtosign = '\n'.join([
            'GET',
            '',
            '',
            timestamp,
            '/' + self.bucket + name
        ])
        signature = self.request_signature(stringtosign)
        headers = dict()
        headers['Date'] = timestamp
        if self.access_key and self.secret_key:
            headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])
        file = S3ContentFile('')
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('GET', name, headers=headers)
            response = conn.getresponse()
            if not response.status in (200,):
                if response.length is None:
                    # length == None seems to be returned from GET requests
                    # to non-existing files
                    raise S3FileDoesNotExistError(name)
                # catch all other cases
                raise S3IOError('py3s3 GET error. Response status: {}. Reason: {}.'.format(
                                response.status, response.reason))

            file = S3ContentFile(response.read())
        return file

    def _open(self, name, mode='rb'):
        prefixed_name = self._prepend_name_prefix(name)
        file = self._get_file(prefixed_name)
        file.name = name
        return file

    def delete(self, name):
        prefixed_name = self._prepend_name_prefix(name)
        timestamp = self.request_timestamp()
        stringtosign = '\n'.join([
            'DELETE',
            '',
            '',
            timestamp,
            '/' + self.bucket + prefixed_name
        ])
        signature = self.request_signature(stringtosign)
        headers = dict()
        headers['Date'] = timestamp
        if self.access_key and self.secret_key:
            headers['Authorization'] = ''.join(['AWS' + ' ', self.access_key, ':', signature])
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('DELETE', prefixed_name, headers=headers)
            response = conn.getresponse()
            if not response.status in (204,):
                raise S3IOError('py3s3 DELETE error. Response status: {}. Reason: {}.'.format(
                                response.status, response.reason))

    def exists(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('HEAD', self.url(name))
            response = conn.getresponse()
            if response.status in (200, 404):
                return response.status == 200
            else:
                raise S3IOError('py3s3 HEAD error. Reposne status: {}. Reason: {}'.format(
                                response.status, response.reason))

    def listdir(self, path):
        raise NotImplementedError()

    def size(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('GET', self.url(name))
            return conn.getresponse().length

    def url(self, name):
        """Return URL of resource"""
        scheme = 'http'
        path = self._prepend_name_prefix(name)
        query = ''
        fragment = ''
        url_tuple = (scheme, self.netloc, path, query, fragment)
        return urllib.parse.urlunsplit(url_tuple)

    def modified_time(self, name):
        with closing(HTTPConnection(self.netloc)) as conn:
            conn.request('HEAD', self.url(name))
            dt_header = conn.getresponse().getheader('Last-Modified', None)
        if dt_header is None:
            raise S3IOError('No modified time available for file: {}'.format(name))
        return self.datetime_from_aws_timestamp(dt_header)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

#import os
#from glob import iglob
import logging
import datetime
#import time
import json
import uuid

from werkzeug import generate_password_hash, check_password_hash
import flask.ext.login
from sqlalchemy import BigInteger, Integer, Boolean, Unicode,\
        Float, UnicodeText, Text, String
from sqlalchemy.schema import Table, MetaData, Column, ForeignKey
from sqlalchemy.orm import relationship, backref, class_mapper
from sqlalchemy.types import MutableType, TypeDecorator
from sqlalchemy import event, text
from sqlalchemy.engine import reflection
from sqlalchemy import create_engine
from sqlalchemy.schema import (
    MetaData,
    Table,
    DropTable,
    ForeignKeyConstraint,
    DropConstraint,
    )

from pybossa.core import db
from pybossa.util import pretty_date

log = logging.getLogger(__name__)

def make_timestamp():
    now = datetime.datetime.utcnow()
    return now.isoformat()

def make_uuid():
    return str(uuid.uuid4())

def rebuild_db():
    # See this SQLAlchemy recipe: http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
    inspector = reflection.Inspector.from_engine(db.engine)
    # gather all data first before dropping anything.
    # some DBs lock after things have been dropped in
    # a transaction.

    metadata = MetaData()

    tbs = []
    all_fks = []

    for table_name in inspector.get_table_names():
        fks = []
        for fk in inspector.get_foreign_keys(table_name):
            if not fk['name']: # pragma: no cover
                continue
            fks.append(
                ForeignKeyConstraint((),(),name=fk['name'])
                )
        t = Table(table_name,metadata,*fks)
        tbs.append(t)
        all_fks.extend(fks)

    for fkc in all_fks:
        db.engine.execute(DropConstraint(fkc))

    for table in tbs:
        db.engine.execute(DropTable(table))

    db.session.commit()
    db.create_all()

# =========================================
# Basics

class JSONType(MutableType, TypeDecorator):
    '''Additional Database Type for handling JSON values.
    '''
    impl = Text

    def __init__(self):
        super(JSONType, self).__init__()

    def process_bind_param(self, value, dialect):
        return json.dumps(value)

    def process_result_value(self, value, dialiect):
        return json.loads(value)

    def copy_value(self, value):
        return json.loads(json.dumps(value))

class StateEnum:
    '''When creating  a task, the task can have the following states::

       * ALL: First time created
       * IN_PROGRESS:  The task is being run by one user
       * PENDING: The task has been completed but need to be validated
       * VALID: The task has been completed and validated
       * INVALID: The task has been complete but it is invalid
       * ERROR: The task has an error
    '''
    ALL = u'all'
    IN_PROGRESS = u'in_progress'
    PENDING = u'pending'
    VALID = u'valid'
    INVALID = u'invalid'
    ERROR = u'error'

class DomainObject(object):

    def dictize(self):
        out = {}
        for col in self.__table__.c:
            out[col.name] = getattr(self, col.name)
        return out

    @classmethod
    def undictize(cls, dict_):
        raise NotImplementedError()

    def __str__(self):
        return self.__unicode__().encode('utf8')

    def __unicode__(self):
        repr = u'<%s' % self.__class__.__name__
        table = class_mapper(self.__class__).mapped_table
        for col in table.c:
            try:
                repr += u' %s=%s' % (col.name, getattr(self, col.name))
            except Exception, inst:
                repr += u' %s=%s' % (col.name, inst)

        repr += '>'
        return repr

# =========================================
# Domain Objects

class App(db.Model, DomainObject):
    '''A microtasking Application to which Tasks are associated.
    '''
    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.id)

    __tablename__ = 'app'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: Name / Title for this Application
    name = Column(Unicode(length=255), unique=True, nullable=False)
    #: slug used in urls etc
    short_name = Column(Unicode(length=255), unique=True, nullable=False)
    #: description
    description = Column(Unicode(length=255), nullable=False)
    #: long description
    long_description = Column(UnicodeText)
    #: Allow anonymous contributors to participate in the application tasks
    allow_anonymous_contributors = Column(Boolean, default=True)
    ## TODO: What is this?
    long_tasks = Column(Integer, default=0)
    #: Boolean integer (0,1) indicating that \
    #: this App should be hidden from everyone but Administrators
    hidden = Column(Integer, default=0)
    #: owner (id)
    owner_id = Column(Integer, ForeignKey('user.id'), nullable=False)
    ## Following may not be relevant ...
    ## TODO: ask about these
    #: estimate of time it should take for user
    time_estimate = Column(Integer, default=0)
    #: time limit for a task
    time_limit = Column(Integer, default=0)
    calibration_frac = Column(Float, default=0)
    bolt_course_id = Column(Integer, default=0)
    #: category(id)
    category_id = Column(Integer, ForeignKey('category.id'))
    #: Standard JSON blob for additional data. This field also
    #: stores information used by PyBossa, such as the app thumbnail,
    #: the schedule mode, and the task presenter.
    #:
    #:    {
    #:       task_presenter: [html/javascript],
    #:       thumbnail: [url to the thumbnail image]
    #:       sched: [scheduling mode]
    #:    }
    #:
    info = Column(JSONType, default=dict)

    ## Relationships
    #: `Task`s for this app.`
    tasks = relationship('Task', cascade='all, delete, delete-orphan', backref='app')
    #: `TaskRun`s for this app.`
    task_runs = relationship('TaskRun', backref='app',
                             cascade='all, delete-orphan',
                             order_by='TaskRun.finish_time.desc()')
    #: `Featured` or not for this app
    featured = relationship('Featured', cascade='all, delete, delete-orphan')
    #: `category` or not for this app
    category = relationship('Category')

    #: Percentage of completed tasks based on Task.state
    #: (0 not done, 1 completed)
    def completion_status(self):
        """Returns the percentage of submitted Tasks Runs done when a task is
        completed"""
        sql = text('''SELECT task.id, n_answers,
                   count(task_run.task_id) AS n_task_runs
                   FROM task LEFT OUTER JOIN task_run ON task.id=task_run.task_id
                   WHERE task.app_id=:app_id GROUP BY task.id''')
        results = db.engine.execute(sql, app_id=self.id)
        n_expected_task_runs = 0
        n_task_runs = 0
        for row in results:
            tmp = row[2]
            if row[2] > row[1]:
                tmp = row[1]
            n_expected_task_runs += row[1]
            n_task_runs += tmp
        pct = float(0)
        if n_expected_task_runs != 0:
           pct = float(n_task_runs) / float(n_expected_task_runs)
        return pct

    def n_completed_tasks(self):
        """Returns the number of Tasks that are completed"""
        completed = 0
        for t in self.tasks:
            if t.state == "completed":
                completed += 1
        return completed

    def last_activity(self):
        sql = text('''SELECT finish_time FROM task_run WHERE app_id=:app_id
                   ORDER BY finish_time DESC LIMIT 1''')
        results = db.engine.execute(sql, app_id=self.id)
        for row in results:
            if row is not None:
                return pretty_date(row[0])
            else:
                return None

class Featured(db.Model, DomainObject):
    '''A Table with Featured Apps.
    '''
    __tablename__ = 'featured'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: Name / Title for this Application
    app_id = Column(Integer, ForeignKey('app.id'))

class Category(db.Model, DomainObject):
    '''A Table with Categories for Applications.'''
    __tablename__ = 'category'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: Name / Title for this Application
    name = Column(Text, nullable=False, unique=True)
    #: slug / Category slug
    short_name = Column(Text, nullable=False, unique=True)
    #: description / Description for the category
    description = Column(Text, nullable=False)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)

class Task(db.Model, DomainObject):
    '''An individual Task which can be performed by a user. A Task is
    associated to an App.
    '''
    __tablename__ = 'task'
    #: unique id (automatically generated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: ForeignKey to App.id (NB: use task relationship rather than this field
    #: in normal use
    app_id = Column(Integer, ForeignKey('app.id', ondelete='CASCADE'), nullable=False)
    #: a StateEnum instance
    # TODO: state should be an integer?
    state = Column(UnicodeText, default=u'ongoing')
    #: Quorum (number of times this task should be done by different users)
    quorum = Column(Integer, default=0)
    #: Boolean indicating whether this is a calibration Task or not.
    calibration = Column(Integer, default=0)
    #: Value between 0 and 1 indicating priority of task within App
    #: (higher = more important)
    priority_0 = Column(Float, default=0)
    #: all configuration / details of the Task is stored in info which is
    #: an arbitrary JSON object. (Usually expected to be a hash/dict)
    #: For example for an image classification project this would be::
    #:
    #:    {
    #:       url: [image-url],
    #:       question: [is this a person]
    #:    }
    info = Column(JSONType, default=dict)
    #: Number of answers or TaskRuns per task
    n_answers = Column(Integer, default=30)

    ## Relationships
    #: `TaskRun`s for this task`
    task_runs = relationship('TaskRun', cascade='all, delete, delete-orphan', backref='task')

    def pct_status(self):
        """Returns the percentage of Tasks that are completed"""
        # DEPRECATED: self.info.n_answers will be removed
        # DEPRECATED: use self.t.n_answers instead
        if (self.info.get('n_answers')):
            self.n_answers = int(self.info['n_answers'])
        if self.n_answers != 0 and self.n_answers != None:
            return float(len(self.task_runs)) / self.n_answers
        else:
            return float(0)

class TaskRun(db.Model, DomainObject):
    '''A run of a given task by a specific user.
    '''
    __tablename__ = 'task_run'
    #: id
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: application id of this task run
    app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
    #: task id of this task run
    task_id = Column(Integer, ForeignKey('task.id', ondelete='CASCADE'),
                     nullable=False)
    #: user id of performer of this task
    user_id = Column(Integer, ForeignKey('user.id'))
    # ip address of this user (only if anonymous)
    user_ip = Column(Text)
    #: finish time (iso8601 formatted string)
    finish_time = Column(Text, default=make_timestamp)

    #: timeout for task
    timeout = Column(Integer)
    #: See same attribute in Task
    calibration = Column(Integer)
    info = Column(JSONType, default=dict)
    '''General writable field that should be used by clients to record results\
    of a TaskRun. Usually a template for this will be provided by Task
    For example::
        result: {
            whatever information shoudl be recorded -- up to task presenter
        }
    '''

class User(db.Model, DomainObject, flask.ext.login.UserMixin):
    __tablename__ = 'user'
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: email address ...
    email_addr = Column(Unicode(length=254), unique=True, nullable=False)
    #: user name
    name = Column(Unicode(length=254), unique=True, nullable=False)
    #: full name
    fullname = Column(Unicode(length=500), nullable=False)
    #: locale
    locale = Column(Unicode(length=254))
    #: api key
    api_key = Column(String(length=36), default=make_uuid, unique=True)
    passwd_hash = Column(Unicode(length=254), unique=True)
    #: Admin flag Boolean Integer (0,1)
    admin = Column(Boolean, default=False)
    #: TODO: find out ... bossa specific
    category = Column(Integer)
    #: TODO: find out ...
    flags = Column(Integer)
    # Twitter user_id field
    twitter_user_id = Column(BigInteger, unique=True)
    # Facebook user_id field
    facebook_user_id = Column(BigInteger, unique=True)
    # Google user_id field
    google_user_id = Column(String, unique=True)
    # CKAN API key field
    ckan_api = Column(String, unique=True)
    #: arbitrary additional information about the user in a JSON dict.
    info = Column(JSONType, default=dict)

    def get_id(self):
        '''id for login system. equates to name'''
        return self.name

    def set_password(self, password):
        self.passwd_hash = generate_password_hash(password)

    def check_password(self, password):
        # OAuth users do not have a password
        if self.passwd_hash:
            return check_password_hash(self.passwd_hash, password)
        else:
            return False

    @classmethod
    def by_name(cls, name):
        '''Lookup user by (user)name.'''
        return db.session.query(User).filter_by(name=name).first()

    ## Relationships
    #: `Task`s for this user
    task_runs = relationship('TaskRun', backref='user')
    apps = relationship('App', backref='owner')

@event.listens_for(User, 'before_insert')
def make_admin(mapper, conn, target):
    users = conn.scalar('select count(*) from "user"')
    if users == 0:
        target.admin = True
        #print "User %s is the first one, so we make it an admin" % target.name

@event.listens_for(App, 'before_update')
@event.listens_for(App, 'before_insert')
def empty_string_to_none(mapper, conn, target):
    if target.name == '':
        target.name = None
    if target.short_name == '':
        target.short_name = None
    if target.description == '':
        target.description = None
<commit_msg>Exclude from coverage<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

#import os
#from glob import iglob
import logging
import datetime
#import time
import json
import uuid

from werkzeug import generate_password_hash, check_password_hash
import flask.ext.login
from sqlalchemy import BigInteger, Integer, Boolean, Unicode,\
        Float, UnicodeText, Text, String
from sqlalchemy.schema import Table, MetaData, Column, ForeignKey
from sqlalchemy.orm import relationship, backref, class_mapper
from sqlalchemy.types import MutableType, TypeDecorator
from sqlalchemy import event, text
from sqlalchemy.engine import reflection
from sqlalchemy import create_engine
from sqlalchemy.schema import (
    MetaData,
    Table,
    DropTable,
    ForeignKeyConstraint,
    DropConstraint,
    )

from pybossa.core import db
from pybossa.util import pretty_date

log = logging.getLogger(__name__)

def make_timestamp():
    now = datetime.datetime.utcnow()
    return now.isoformat()

def make_uuid():
    return str(uuid.uuid4())

def rebuild_db():
    # See this SQLAlchemy recipe: http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DropEverything
    inspector = reflection.Inspector.from_engine(db.engine)
    # gather all data first before dropping anything.
    # some DBs lock after things have been dropped in
    # a transaction.

    metadata = MetaData()

    tbs = []
    all_fks = []

    for table_name in inspector.get_table_names():
        fks = []
        for fk in inspector.get_foreign_keys(table_name):
            if not fk['name']: # pragma: no cover
                continue
            fks.append(
                ForeignKeyConstraint((),(),name=fk['name'])
                )
        t = Table(table_name,metadata,*fks)
        tbs.append(t)
        all_fks.extend(fks)

    for fkc in all_fks:
        db.engine.execute(DropConstraint(fkc))

    for table in tbs:
        db.engine.execute(DropTable(table))

    db.session.commit()
    db.create_all()

# =========================================
# Basics

class JSONType(MutableType, TypeDecorator):
    '''Additional Database Type for handling JSON values.
    '''
    impl = Text

    def __init__(self):
        super(JSONType, self).__init__()

    def process_bind_param(self, value, dialect):
        return json.dumps(value)

    def process_result_value(self, value, dialiect):
        return json.loads(value)

    def copy_value(self, value):
        return json.loads(json.dumps(value))

class StateEnum:
    '''When creating  a task, the task can have the following states::

       * ALL: First time created
       * IN_PROGRESS:  The task is being run by one user
       * PENDING: The task has been completed but need to be validated
       * VALID: The task has been completed and validated
       * INVALID: The task has been complete but it is invalid
       * ERROR: The task has an error
    '''
    ALL = u'all'
    IN_PROGRESS = u'in_progress'
    PENDING = u'pending'
    VALID = u'valid'
    INVALID = u'invalid'
    ERROR = u'error'

class DomainObject(object):

    def dictize(self):
        out = {}
        for col in self.__table__.c:
            out[col.name] = getattr(self, col.name)
        return out

    @classmethod
    def undictize(cls, dict_):
        raise NotImplementedError()

    def __str__(self):
        return self.__unicode__().encode('utf8')

    def __unicode__(self): # pragma: no cover
        repr = u'<%s' % self.__class__.__name__
        table = class_mapper(self.__class__).mapped_table
        for col in table.c:
            try:
                repr += u' %s=%s' % (col.name, getattr(self, col.name))
            except Exception, inst:
                repr += u' %s=%s' % (col.name, inst)

        repr += '>'
        return repr

# =========================================
# Domain Objects

class App(db.Model, DomainObject):
    '''A microtasking Application to which Tasks are associated.
    '''
    def __repr__(self):
        return "%s(%s)" % (self.__class__.__name__, self.id)

    __tablename__ = 'app'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: Name / Title for this Application
    name = Column(Unicode(length=255), unique=True, nullable=False)
    #: slug used in urls etc
    short_name = Column(Unicode(length=255), unique=True, nullable=False)
    #: description
    description = Column(Unicode(length=255), nullable=False)
    #: long description
    long_description = Column(UnicodeText)
    #: Allow anonymous contributors to participate in the application tasks
    allow_anonymous_contributors = Column(Boolean, default=True)
    ## TODO: What is this?
    long_tasks = Column(Integer, default=0)
    #: Boolean integer (0,1) indicating that \
    #: this App should be hidden from everyone but Administrators
    hidden = Column(Integer, default=0)
    #: owner (id)
    owner_id = Column(Integer, ForeignKey('user.id'), nullable=False)
    ## Following may not be relevant ...
    ## TODO: ask about these
    #: estimate of time it should take for user
    time_estimate = Column(Integer, default=0)
    #: time limit for a task
    time_limit = Column(Integer, default=0)
    calibration_frac = Column(Float, default=0)
    bolt_course_id = Column(Integer, default=0)
    #: category(id)
    category_id = Column(Integer, ForeignKey('category.id'))
    #: Standard JSON blob for additional data. This field also
    #: stores information used by PyBossa, such as the app thumbnail,
    #: the schedule mode, and the task presenter.
    #:
    #:    {
    #:       task_presenter: [html/javascript],
    #:       thumbnail: [url to the thumbnail image]
    #:       sched: [scheduling mode]
    #:    }
    #:
    info = Column(JSONType, default=dict)

    ## Relationships
    #: `Task`s for this app.`
    tasks = relationship('Task', cascade='all, delete, delete-orphan', backref='app')
    #: `TaskRun`s for this app.`
    task_runs = relationship('TaskRun', backref='app',
                             cascade='all, delete-orphan',
                             order_by='TaskRun.finish_time.desc()')
    #: `Featured` or not for this app
    featured = relationship('Featured', cascade='all, delete, delete-orphan')
    #: `category` or not for this app
    category = relationship('Category')

    #: Percentage of completed tasks based on Task.state
    #: (0 not done, 1 completed)
    def completion_status(self):
        """Returns the percentage of submitted Tasks Runs done when a task is
        completed"""
        sql = text('''SELECT task.id, n_answers,
                   count(task_run.task_id) AS n_task_runs
                   FROM task LEFT OUTER JOIN task_run ON task.id=task_run.task_id
                   WHERE task.app_id=:app_id GROUP BY task.id''')
        results = db.engine.execute(sql, app_id=self.id)
        n_expected_task_runs = 0
        n_task_runs = 0
        for row in results:
            tmp = row[2]
            if row[2] > row[1]:
                tmp = row[1]
            n_expected_task_runs += row[1]
            n_task_runs += tmp
        pct = float(0)
        if n_expected_task_runs != 0:
           pct = float(n_task_runs) / float(n_expected_task_runs)
        return pct

    def n_completed_tasks(self):
        """Returns the number of Tasks that are completed"""
        completed = 0
        for t in self.tasks:
            if t.state == "completed":
                completed += 1
        return completed

    def last_activity(self):
        sql = text('''SELECT finish_time FROM task_run WHERE app_id=:app_id
                   ORDER BY finish_time DESC LIMIT 1''')
        results = db.engine.execute(sql, app_id=self.id)
        for row in results:
            if row is not None:
                return pretty_date(row[0])
            else:
                return None

class Featured(db.Model, DomainObject):
    '''A Table with Featured Apps.
    '''
    __tablename__ = 'featured'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: Name / Title for this Application
    app_id = Column(Integer, ForeignKey('app.id'))

class Category(db.Model, DomainObject):
    '''A Table with Categories for Applications.'''
    __tablename__ = 'category'
    #: Unique id for this app (autogenerated)
    id = Column(Integer, primary_key=True)
    #: Name / Title for this Application
    name = Column(Text, nullable=False, unique=True)
    #: slug / Category slug
    short_name = Column(Text, nullable=False, unique=True)
    #: description / Description for the category
    description = Column(Text, nullable=False)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)

class Task(db.Model, DomainObject):
    '''An individual Task which can be performed by a user. A Task is
    associated to an App.
    '''
    __tablename__ = 'task'
    #: unique id (automatically generated)
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: ForeignKey to App.id (NB: use task relationship rather than this field
    #: in normal use
    app_id = Column(Integer, ForeignKey('app.id', ondelete='CASCADE'), nullable=False)
    #: a StateEnum instance
    # TODO: state should be an integer?
    state = Column(UnicodeText, default=u'ongoing')
    #: Quorum (number of times this task should be done by different users)
    quorum = Column(Integer, default=0)
    #: Boolean indicating whether this is a calibration Task or not.
    calibration = Column(Integer, default=0)
    #: Value between 0 and 1 indicating priority of task within App
    #: (higher = more important)
    priority_0 = Column(Float, default=0)
    #: all configuration / details of the Task is stored in info which is
    #: an arbitrary JSON object. (Usually expected to be a hash/dict)
    #: For example for an image classification project this would be::
    #:
    #:    {
    #:       url: [image-url],
    #:       question: [is this a person]
    #:    }
    info = Column(JSONType, default=dict)
    #: Number of answers or TaskRuns per task
    n_answers = Column(Integer, default=30)

    ## Relationships
    #: `TaskRun`s for this task`
    task_runs = relationship('TaskRun', cascade='all, delete, delete-orphan', backref='task')

    def pct_status(self):
        """Returns the percentage of Tasks that are completed"""
        # DEPRECATED: self.info.n_answers will be removed
        # DEPRECATED: use self.t.n_answers instead
        if (self.info.get('n_answers')):
            self.n_answers = int(self.info['n_answers'])
        if self.n_answers != 0 and self.n_answers != None:
            return float(len(self.task_runs)) / self.n_answers
        else:
            return float(0)

class TaskRun(db.Model, DomainObject):
    '''A run of a given task by a specific user.
    '''
    __tablename__ = 'task_run'
    #: id
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: application id of this task run
    app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
    #: task id of this task run
    task_id = Column(Integer, ForeignKey('task.id', ondelete='CASCADE'),
                     nullable=False)
    #: user id of performer of this task
    user_id = Column(Integer, ForeignKey('user.id'))
    # ip address of this user (only if anonymous)
    user_ip = Column(Text)
    #: finish time (iso8601 formatted string)
    finish_time = Column(Text, default=make_timestamp)

    #: timeout for task
    timeout = Column(Integer)
    #: See same attribute in Task
    calibration = Column(Integer)
    info = Column(JSONType, default=dict)
    '''General writable field that should be used by clients to record results\
    of a TaskRun. Usually a template for this will be provided by Task
    For example::
        result: {
            whatever information shoudl be recorded -- up to task presenter
        }
    '''

class User(db.Model, DomainObject, flask.ext.login.UserMixin):
    __tablename__ = 'user'
    id = Column(Integer, primary_key=True)
    #: created timestamp (automatically set)
    created = Column(Text, default=make_timestamp)
    #: email address ...
    email_addr = Column(Unicode(length=254), unique=True, nullable=False)
    #: user name
    name = Column(Unicode(length=254), unique=True, nullable=False)
    #: full name
    fullname = Column(Unicode(length=500), nullable=False)
    #: locale
    locale = Column(Unicode(length=254))
    #: api key
    api_key = Column(String(length=36), default=make_uuid, unique=True)
    passwd_hash = Column(Unicode(length=254), unique=True)
    #: Admin flag Boolean Integer (0,1)
    admin = Column(Boolean, default=False)
    #: TODO: find out ... bossa specific
    category = Column(Integer)
    #: TODO: find out ...
    flags = Column(Integer)
    # Twitter user_id field
    twitter_user_id = Column(BigInteger, unique=True)
    # Facebook user_id field
    facebook_user_id = Column(BigInteger, unique=True)
    # Google user_id field
    google_user_id = Column(String, unique=True)
    # CKAN API key field
    ckan_api = Column(String, unique=True)
    #: arbitrary additional information about the user in a JSON dict.
    info = Column(JSONType, default=dict)

    def get_id(self):
        '''id for login system. equates to name'''
        return self.name

    def set_password(self, password):
        self.passwd_hash = generate_password_hash(password)

    def check_password(self, password):
        # OAuth users do not have a password
        if self.passwd_hash:
            return check_password_hash(self.passwd_hash, password)
        else:
            return False

    @classmethod
    def by_name(cls, name):
        '''Lookup user by (user)name.'''
        return db.session.query(User).filter_by(name=name).first()

    ## Relationships
    #: `Task`s for this user
    task_runs = relationship('TaskRun', backref='user')
    apps = relationship('App', backref='owner')

@event.listens_for(User, 'before_insert')
def make_admin(mapper, conn, target):
    users = conn.scalar('select count(*) from "user"')
    if users == 0:
        target.admin = True
        #print "User %s is the first one, so we make it an admin" % target.name

@event.listens_for(App, 'before_update')
@event.listens_for(App, 'before_insert')
def empty_string_to_none(mapper, conn, target):
    if target.name == '':
        target.name = None
    if target.short_name == '':
        target.short_name = None
    if target.description == '':
        target.description = None
<|endoftext|>