<commit_before>__all__ = ["SimpleHTTPRequestHandler"]

import os
import posixpath
import BaseHTTPServer
import urllib
import cgi
import shutil
import mimetypes
import re

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):

    """Simple HTTP request handler with GET/HEAD/POST commands.

    This serves files from the current directory and any of its
    subdirectories.  The MIME type for files is determined by
    calling the .guess_type() method. And can reveive file uploaded
    by client.

    The GET/HEAD/POST requests are identical except that the HEAD
    request omits the actual contents of the file.

    """

    server_version = "SimpleHTTPWithUpload/"

    def do_GET(self):
        f = self.send_head()
        if f:
            self.copyfile(f, self.wfile)
            f.close()

    def do_HEAD(self):
        f = self.send_head()
        if f:
            f.close()

    def do_POST(self):
        r, info = self.deal_post_data()
        print r, info, "by: ", self.client_address

        fileName = os.path.basename(info)
        fileName = fileName.split('.')[0]

        self.send_response(301)
        self.send_header('Location', 'http://localhost:8000/go.html?file=' + fileName)
        self.end_headers()

    def deal_post_data(self):
        boundary = self.headers.plisttext.split("=")[1]
        remainbytes = int(self.headers['content-length'])
        line = self.rfile.readline()
        remainbytes -= len(line)
        if not boundary in line:
            return (False, "Content NOT begin with boundary")
        line = self.rfile.readline()
        remainbytes -= len(line)
        fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"', line)
        if not fn:
            return (False, "Can't find out file name...")
        path = self.translate_path(self.path)
        if not os.path.exists(path):
            os.makedirs(path)
        fn = os.path.join(path, fn[0])
        line = self.rfile.readline()
        remainbytes -= len(line)
        line = self.rfile.readline()
        remainbytes -= len(line)
        try:
            out = open(fn, 'wb')
        except IOError:
            return (False, "Can't create file to write, do you have permission to write?")

        preline = self.rfile.readline()
        remainbytes -= len(preline)
        while remainbytes > 0:
            line = self.rfile.readline()
            remainbytes -= len(line)
            if boundary in line:
                preline = preline[0:-1]
                if preline.endswith('\r'):
                    preline = preline[0:-1]
                out.write(preline)
                out.close()
                return (True, "File '%s' upload success!" % fn)
            else:
                out.write(preline)
                preline = line
        return (False, "Unexpect Ends of data.")

    def send_head(self):
        """Common code for GET and HEAD commands.

        This sends the response code and MIME headers.

        Return value is either a file object (which has to be copied
        to the outputfile by the caller unless the command was HEAD,
        and must be closed by the caller under all circumstances), or
        None, in which case the caller has nothing further to do.

        """
        path = self.translate_path(self.path)
        f = None
        if os.path.isdir(path):
            if not self.path.endswith('/'):
                # redirect browser - doing basically what apache does
                self.send_response(301)
                self.send_header("Location", self.path + "/")
                self.end_headers()
                return None
            for index in "index.html", "index.htm":
                index = os.path.join(path, index)
                if os.path.exists(index):
                    path = index
                    break
            else:
                return self.list_directory(path)
        ctype = self.guess_type(path)
        try:
            # Always read in binary mode. Opening files in text mode may cause
            # newline translations, making the actual size of the content
            # transmitted *less* than the content-length!
            f = open(path, 'rb')
        except IOError:
            self.send_error(404, "File not found")
            return None
        self.send_response(200)
        self.send_header("Content-type", ctype)
        fs = os.fstat(f.fileno())
        self.send_header("Content-Length", str(fs[6]))
        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
        self.end_headers()
        return f

    def list_directory(self, path):
        """Helper to produce a directory listing (absent index.html).

        Return value is either a file object, or None (indicating an
        error).  In either case, the headers are sent, making the
        interface the same as for send_head().

        """
        try:
            list = os.listdir(path)
        except os.error:
            self.send_error(404, "No permission to list directory")
            return None

        list.sort(key=lambda a: a.lower())
        f = StringIO()
        displaypath = cgi.escape(urllib.unquote(self.path))

        f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
        f.write("\n<title>Directory listing for %s</title>\n" % displaypath)
        f.write("\n
Directory listing for %s
\n" % displaypath)
        f.write("
\n")
        f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
        f.write("<input name=\"file\" type=\"file\"/>")
        f.write("<input type=\"submit\" value=\"upload\"/></form>\n")
        f.write("
\n
\n")

        for name in list:
            fullname = os.path.join(path, name)
            displayname = linkname = name
            # Append / for directories or @ for symbolic links
            if os.path.isdir(fullname):
                displayname = name + "/"
                linkname = name + "/"
            if os.path.islink(fullname):
                displayname = name + "@"
                # Note: a link to a directory displays with @ and links with /
            f.write('
%s\n'
                    % (urllib.quote(linkname), cgi.escape(displayname)))
        f.write("
\n
\n\n\n")
        length = f.tell()
        f.seek(0)
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.send_header("Content-Length", str(length))
        self.end_headers()
        return f

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        # abandon query parameters
        path = path.split('?', 1)[0]
        path = path.split('#', 1)[0]
        path = posixpath.normpath(urllib.unquote(path))
        words = path.split('/')
        words = filter(None, words)
        path = os.getcwd()
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)
        return path

    def copyfile(self, source, outputfile):
        """Copy all data between two file objects.

        The SOURCE argument is a file object open for reading
        (or anything with a read() method) and the DESTINATION
        argument is a file object open for writing (or
        anything with a write() method).

        The only reason for overriding this would be to change
        the block size or perhaps to replace newlines by CRLF
        -- note however that this the default server uses this
        to copy binary data as well.

        """
        shutil.copyfileobj(source, outputfile)

    def guess_type(self, path):
        """Guess the type of a file.

        Argument is a PATH (a filename).

        Return value is a string of the form type/subtype,
        usable for a MIME Content-type header.

        The default implementation looks the file's extension
        up in the table self.extensions_map, using application/octet-stream
        as a default; however it would be permissible (if
        slow) to look inside the data to make a better guess.

        """

        base, ext = posixpath.splitext(path)
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        ext = ext.lower()
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        else:
            return self.extensions_map['']

    if not mimetypes.inited:
        mimetypes.init()  # try to read system mime.types
    extensions_map = mimetypes.types_map.copy()
    extensions_map.update({
        '': 'application/octet-stream',  # Default
        '.py': 'text/plain',
        '.c': 'text/plain',
        '.h': 'text/plain',
    })

def test(HandlerClass=SimpleHTTPRequestHandler,
         ServerClass=BaseHTTPServer.HTTPServer):
    BaseHTTPServer.test(HandlerClass, ServerClass)

if __name__ == '__main__':
    test()
<commit_msg>fixed E402 module level import not at top of file<commit_after>import os
import posixpath
import BaseHTTPServer
import urllib
import cgi
import shutil
import mimetypes
import re

__all__ = ["SimpleHTTPRequestHandler"]

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):

    """Simple HTTP request handler with GET/HEAD/POST commands.

    This serves files from the current directory and any of its
    subdirectories.  The MIME type for files is determined by
    calling the .guess_type() method. And can reveive file uploaded
    by client.

    The GET/HEAD/POST requests are identical except that the HEAD
    request omits the actual contents of the file.

    """

    server_version = "SimpleHTTPWithUpload/"

    def do_GET(self):
        f = self.send_head()
        if f:
            self.copyfile(f, self.wfile)
            f.close()

    def do_HEAD(self):
        f = self.send_head()
        if f:
            f.close()

    def do_POST(self):
        r, info = self.deal_post_data()
        print r, info, "by: ", self.client_address

        fileName = os.path.basename(info)
        fileName = fileName.split('.')[0]

        self.send_response(301)
        self.send_header('Location', 'http://localhost:8000/go.html?file=' + fileName)
        self.end_headers()

    def deal_post_data(self):
        boundary = self.headers.plisttext.split("=")[1]
        remainbytes = int(self.headers['content-length'])
        line = self.rfile.readline()
        remainbytes -= len(line)
        if not boundary in line:
            return (False, "Content NOT begin with boundary")
        line = self.rfile.readline()
        remainbytes -= len(line)
        fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"', line)
        if not fn:
            return (False, "Can't find out file name...")
        path = self.translate_path(self.path)
        if not os.path.exists(path):
            os.makedirs(path)
        fn = os.path.join(path, fn[0])
        line = self.rfile.readline()
        remainbytes -= len(line)
        line = self.rfile.readline()
        remainbytes -= len(line)
        try:
            out = open(fn, 'wb')
        except IOError:
            return (False, "Can't create file to write, do you have permission to write?")

        preline = self.rfile.readline()
        remainbytes -= len(preline)
        while remainbytes > 0:
            line = self.rfile.readline()
            remainbytes -= len(line)
            if boundary in line:
                preline = preline[0:-1]
                if preline.endswith('\r'):
                    preline = preline[0:-1]
                out.write(preline)
                out.close()
                return (True, "File '%s' upload success!" % fn)
            else:
                out.write(preline)
                preline = line
        return (False, "Unexpect Ends of data.")

    def send_head(self):
        """Common code for GET and HEAD commands.

        This sends the response code and MIME headers.

        Return value is either a file object (which has to be copied
        to the outputfile by the caller unless the command was HEAD,
        and must be closed by the caller under all circumstances), or
        None, in which case the caller has nothing further to do.

        """
        path = self.translate_path(self.path)
        f = None
        if os.path.isdir(path):
            if not self.path.endswith('/'):
                # redirect browser - doing basically what apache does
                self.send_response(301)
                self.send_header("Location", self.path + "/")
                self.end_headers()
                return None
            for index in "index.html", "index.htm":
                index = os.path.join(path, index)
                if os.path.exists(index):
                    path = index
                    break
            else:
                return self.list_directory(path)
        ctype = self.guess_type(path)
        try:
            # Always read in binary mode. Opening files in text mode may cause
            # newline translations, making the actual size of the content
            # transmitted *less* than the content-length!
            f = open(path, 'rb')
        except IOError:
            self.send_error(404, "File not found")
            return None
        self.send_response(200)
        self.send_header("Content-type", ctype)
        fs = os.fstat(f.fileno())
        self.send_header("Content-Length", str(fs[6]))
        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
        self.end_headers()
        return f

    def list_directory(self, path):
        """Helper to produce a directory listing (absent index.html).

        Return value is either a file object, or None (indicating an
        error).  In either case, the headers are sent, making the
        interface the same as for send_head().

        """
        try:
            list = os.listdir(path)
        except os.error:
            self.send_error(404, "No permission to list directory")
            return None

        list.sort(key=lambda a: a.lower())
        f = StringIO()
        displaypath = cgi.escape(urllib.unquote(self.path))

        f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
        f.write("\n<title>Directory listing for %s</title>\n" % displaypath)
        f.write("\n
Directory listing for %s
\n" % displaypath)
        f.write("
\n")
        f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
        f.write("<input name=\"file\" type=\"file\"/>")
        f.write("<input type=\"submit\" value=\"upload\"/></form>\n")
        f.write("
\n
\n")

        for name in list:
            fullname = os.path.join(path, name)
            displayname = linkname = name
            # Append / for directories or @ for symbolic links
            if os.path.isdir(fullname):
                displayname = name + "/"
                linkname = name + "/"
            if os.path.islink(fullname):
                displayname = name + "@"
                # Note: a link to a directory displays with @ and links with /
            f.write('
%s\n'
                    % (urllib.quote(linkname), cgi.escape(displayname)))
        f.write("
\n
\n\n\n")
        length = f.tell()
        f.seek(0)
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.send_header("Content-Length", str(length))
        self.end_headers()
        return f

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        # abandon query parameters
        path = path.split('?', 1)[0]
        path = path.split('#', 1)[0]
        path = posixpath.normpath(urllib.unquote(path))
        words = path.split('/')
        words = filter(None, words)
        path = os.getcwd()
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)
        return path

    def copyfile(self, source, outputfile):
        """Copy all data between two file objects.

        The SOURCE argument is a file object open for reading
        (or anything with a read() method) and the DESTINATION
        argument is a file object open for writing (or
        anything with a write() method).

        The only reason for overriding this would be to change
        the block size or perhaps to replace newlines by CRLF
        -- note however that this the default server uses this
        to copy binary data as well.

        """
        shutil.copyfileobj(source, outputfile)

    def guess_type(self, path):
        """Guess the type of a file.

        Argument is a PATH (a filename).

        Return value is a string of the form type/subtype,
        usable for a MIME Content-type header.

        The default implementation looks the file's extension
        up in the table self.extensions_map, using application/octet-stream
        as a default; however it would be permissible (if
        slow) to look inside the data to make a better guess.

        """

        base, ext = posixpath.splitext(path)
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        ext = ext.lower()
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        else:
            return self.extensions_map['']

    if not mimetypes.inited:
        mimetypes.init()  # try to read system mime.types
    extensions_map = mimetypes.types_map.copy()
    extensions_map.update({
        '': 'application/octet-stream',  # Default
        '.py': 'text/plain',
        '.c': 'text/plain',
        '.h': 'text/plain',
    })

def test(HandlerClass=SimpleHTTPRequestHandler,
         ServerClass=BaseHTTPServer.HTTPServer):
    BaseHTTPServer.test(HandlerClass, ServerClass)

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import torch
from ..lazy import DiagLazyTensor
from ..distributions import MultivariateNormal
from ._variational_distribution import _VariationalDistribution

class MeanFieldVariationalDistribution(_VariationalDistribution):
    """
    A :obj:`~gpytorch.variational._VariationalDistribution` that is defined to be a multivariate normal distribution
    with a diagonal covariance matrix. This will not be as flexible/expressive as a
    :obj:`~gpytorch.variational.CholeskyVariationalDistribution`.

    :param int num_inducing_points: Size of the variational distribution. This implies that the variational mean
        should be this size, and the variational covariance matrix should have this many rows and columns.
    :param torch.Size batch_shape: (Optional.) Specifies an optional batch size
        for the variational parameters. This is useful for example when doing additive variational inference.
    :param float mean_init_std: (default=1e-3) Standard deviation of gaussian noise to add to the mean initialization.
    """
    def __init__(self, num_inducing_points, batch_shape=torch.Size([]), mean_init_std=1e-3, **kwargs):
        super().__init__(num_inducing_points=num_inducing_points, batch_shape=batch_shape, mean_init_std=mean_init_std)
        mean_init = torch.zeros(num_inducing_points)
        covar_init = torch.ones(num_inducing_points)
        mean_init = mean_init.repeat(*batch_shape, 1)
        covar_init = covar_init.repeat(*batch_shape, 1)

        self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init))
        self.register_parameter(name="variational_stddev", parameter=torch.nn.Parameter(covar_init))

    def forward(self):
        variational_var = self.variational_stddev.pow(2)
        variational_covar = DiagLazyTensor(variational_var)
        return MultivariateNormal(self.variational_mean, variational_covar)

    def initialize_variational_distribution(self, prior_dist):
        self.variational_mean.data.copy_(prior_dist.mean)
        self.variational_mean.data.add_(self.mean_init_std, torch.randn_like(prior_dist.mean))
        self.variational_stddev.data.copy_(prior_dist.stddev)
<commit_msg>Ensure mean field dist has positive stddev property<commit_after>#!/usr/bin/env python3

import torch
from ..lazy import DiagLazyTensor
from ..distributions import MultivariateNormal
from ._variational_distribution import _VariationalDistribution

class MeanFieldVariationalDistribution(_VariationalDistribution):
    """
    A :obj:`~gpytorch.variational._VariationalDistribution` that is defined to be a multivariate normal distribution
    with a diagonal covariance matrix. This will not be as flexible/expressive as a
    :obj:`~gpytorch.variational.CholeskyVariationalDistribution`.

    :param int num_inducing_points: Size of the variational distribution. This implies that the variational mean
        should be this size, and the variational covariance matrix should have this many rows and columns.
    :param torch.Size batch_shape: (Optional.) Specifies an optional batch size
        for the variational parameters. This is useful for example when doing additive variational inference.
    :param float mean_init_std: (default=1e-3) Standard deviation of gaussian noise to add to the mean initialization.
    """
    def __init__(self, num_inducing_points, batch_shape=torch.Size([]), mean_init_std=1e-3, **kwargs):
        super().__init__(num_inducing_points=num_inducing_points, batch_shape=batch_shape, mean_init_std=mean_init_std)
        mean_init = torch.zeros(num_inducing_points)
        covar_init = torch.ones(num_inducing_points)
        mean_init = mean_init.repeat(*batch_shape, 1)
        covar_init = covar_init.repeat(*batch_shape, 1)

        self.register_parameter(name="variational_mean", parameter=torch.nn.Parameter(mean_init))
        self.register_parameter(name="_variational_stddev", parameter=torch.nn.Parameter(covar_init))

    @property
    def variational_stddev(self):
        return self._variational_stddev.abs().clamp_min(1e-8)

    def forward(self):
        variational_var = self.variational_stddev.pow(2)
        variational_covar = DiagLazyTensor(variational_var)
        return MultivariateNormal(self.variational_mean, variational_covar)

    def initialize_variational_distribution(self, prior_dist):
        self.variational_mean.data.copy_(prior_dist.mean)
        self.variational_mean.data.add_(self.mean_init_std, torch.randn_like(prior_dist.mean))
        self.variational_stddev.data.copy_(prior_dist.stddev)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
#
# This software's license gives you freedom; you can copy, convey,
# propagate, redistribute and/or modify this program under the terms of
# the GNU Affero General Public License (AGPL) as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version of the AGPL published by the FSF.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program in a file in the toplevel directory called
# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
#
"""
Instructor Tool: An XBlock for instructors to export student answers from a course.

All processing is done offline.
"""
import json
from django.core.paginator import Paginator
from xblock.core import XBlock
from xblock.exceptions import JsonHandlerError
from xblock.fields import Scope, String, Dict, List
from xblock.fragment import Fragment
from xblockutils.resources import ResourceLoader

loader = ResourceLoader(__name__)

PAGE_SIZE = 15

# Make '_' a no-op so we can scrape strings
def _(text):
    return text

@XBlock.needs("i18n")
@XBlock.wants('user')
class InstructorToolBlock(XBlock):
    """
    InstructorToolBlock: An XBlock for instructors to export student answers from a course.

    All processing is done offline.
    """
    display_name = String(
        display_name=_("Title (Display name)"),
        help=_("Title to display"),
        default=_("Instructor Tool"),
        scope=Scope.settings
    )
    active_export_task_id = String(
        # The UUID of the celery AsyncResult for the most recent export,
        # IF we are sill waiting for it to finish
        default="",
        scope=Scope.user_state,
    )
    last_export_result = Dict(
        # The info dict returned by the most recent successful export.
        # If the export failed, it will have an "error" key set.
        default=None,
        scope=Scope.user_state,
    )
    display_data = List(
        # The list of results associated with the most recent successful export.
        # Stored separately to avoid the overhead of sending it to the client.
        default=None,
        scope=Scope.user_state,
    )
    has_author_view = True

    @property
    def display_name_with_default(self):
        return "Instructor Tool"

    def author_view(self, context=None):
        """ Studio View """
        # Warn the user that this block will only work from the LMS. (Since the CMS uses
        # different celery queues; our task listener is waiting for tasks on the LMS queue)
        return Fragment(u'
Instructor Tool Block

This block only works from the LMS.
')

    def studio_view(self, context=None):
        """ View for editing Instructor Tool block in Studio. """
        # Display friendly message explaining that the block is not editable.
        return Fragment(u'
This is a preconfigured block. It is not editable.
')

    def check_pending_export(self):
        """
        If we're waiting for an export, see if it has finished, and if so, get the result.
        """
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        if self.active_export_task_id:
            async_result = export_data_task.AsyncResult(self.active_export_task_id)
            if async_result.ready():
                self._save_result(async_result)

    def _save_result(self, task_result):
        """ Given an AsyncResult or EagerResult, save it. """
        self.active_export_task_id = ''
        if task_result.successful():
            if isinstance(task_result.result, dict) and not task_result.result.get('error'):
                self.display_data = task_result.result['display_data']
                del task_result.result['display_data']
                self.last_export_result = task_result.result
            else:
                self.last_export_result = {'error': u'Unexpected result: {}'.format(repr(task_result.result))}
                self.display_data = None
        else:
            self.last_export_result = {'error': unicode(task_result.result)}
            self.display_data = None

    @XBlock.json_handler
    def get_result_page(self, data, suffix=''):
        """ Return requested page of `last_export_result`. """
        paginator = Paginator(self.display_data, PAGE_SIZE)
        page = data.get('page', None)
        return {
            'display_data': paginator.page(page).object_list,
            'num_results': len(self.display_data),
            'page_size': PAGE_SIZE
        }

    def student_view(self, context=None):
        """ Normal View """
        if not self.user_is_staff():
            return Fragment(u'
This interface can only be used by course staff.
')
        block_choices = {
            _('Multiple Choice Question'): 'MCQBlock',
            _('Rating Question'): 'RatingBlock',
            _('Long Answer'): 'AnswerBlock',
        }
        block_types = ('pb-mcq', 'pb-rating', 'pb-answer')
        flat_block_tree = []

        def get_block_id(block):
            """
            Return ID of `block`, taking into account needs of both LMS/CMS and workbench runtimes.
            """
            usage_id = block.scope_ids.usage_id
            # Try accessing block ID. If usage_id does not have it, return usage_id itself
            return unicode(getattr(usage_id, 'block_id', usage_id))

        def get_block_name(block):
            """
            Return name of `block`.

            - For MCQs, Ratings, Answer blocks this is one of:
              - block.question
              - block.name (fallback for old courses)
            - For other types of (non-eligible) blocks this is one of:
              - block.question
              - block.display_name
              - block ID (fallback if neither 'question' nor 'display_name' is suitable)
            """
            block_type = get_block_type(block)
            # Eligible block (question)
            if block_type in block_types:
                return getattr(block, 'question', block.name)
            # Non-eligible block (question or section/subsection/unit)
            # - Try "question" attribute:
            block_name = getattr(block, 'question', None)
            if not block_name:
                # - Try display_name:
                block_name = getattr(block, 'display_name', None)
            if not block_name:
                # - Default to ID:
                block_name = get_block_id(block)
            return block_name

        def get_block_type(block):
            """
            Return type of `block`, taking into account different key styles that might be in use.
            """
            try:
                block_type = block.runtime.id_reader.get_block_type(block.scope_ids.def_id)
            except AttributeError:
                block_type = block.runtime.id_reader.get_block_type(block.scope_ids.usage_id)
            return block_type

        def build_tree(block, ancestors):
            """
            Build up a tree of information about the XBlocks descending from root_block
            """
            block_id = get_block_id(block)
            block_name = get_block_name(block)
            block_type = get_block_type(block)
            eligible = block_type in block_types
            if eligible:
                # If this block is a question whose answers we can export,
                # we mark all of its ancestors as exportable too
                if ancestors and not ancestors[-1]["eligible"]:
                    for ancestor in ancestors:
                        ancestor["eligible"] = True
            new_entry = {
                "depth": len(ancestors),
                "id": block_id,
                "name": block_name,
                "eligible": eligible,
            }
            flat_block_tree.append(new_entry)
            if block.has_children and not block_type == 'pb-mcq' and not \
               getattr(block, "has_dynamic_children", lambda: False)():
                for child_id in block.children:
                    build_tree(block.runtime.get_block(child_id), ancestors=(ancestors + [new_entry]))

        root_block = self
        while root_block.parent:
            root_block = root_block.get_parent()
        root_block_id = get_block_id(root_block)
        root_entry = {
            "depth": 0,
            "id": root_block_id,
            "name": "All",
        }
        flat_block_tree.append(root_entry)

        for child_id in root_block.children:
            child_block = root_block.runtime.get_block(child_id)
            build_tree(child_block, [root_entry])

        html = loader.render_template(
            'templates/html/instructor_tool.html',
            {'block_choices': block_choices, 'block_tree': flat_block_tree}
        )
        fragment = Fragment(html)
        fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/instructor_tool.css'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/instructor_tool.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/underscore-min.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/backbone-min.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/backbone.paginator.min.js'))
        fragment.initialize_js('InstructorToolBlock')
        return fragment

    @property
    def download_url_for_last_report(self):
        """ Get the URL for the last report, if any """
        # Unfortunately this is a bit inefficient due to the ReportStore API
        if not self.last_export_result or self.last_export_result['error'] is not None:
            return None
        from instructor_task.models import ReportStore
        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
        course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
        return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])

    def _get_status(self):
        self.check_pending_export()
        return {
            'export_pending': bool(self.active_export_task_id),
            'last_export_result': self.last_export_result,
            'download_url': self.download_url_for_last_report,
        }

    def raise_error(self, code, message):
        """
        Raises an error and marks the block with a simulated failed task dict.
        """
        self.last_export_result = {
            'error': message,
        }
        self.display_data = None
        raise JsonHandlerError(code, message)

    @XBlock.json_handler
    def get_status(self, data, suffix=''):
        return self._get_status()

    @XBlock.json_handler
    def delete_export(self, data, suffix=''):
        self._delete_export()
        return self._get_status()

    def _delete_export(self):
        self.last_export_result = None
        self.display_data = None
        self.active_export_task_id = ''

    @XBlock.json_handler
    def start_export(self, data, suffix=''):
        """ Start a new asynchronous export """
        block_types = data.get('block_types', None)
        username = data.get('username', None)
        root_block_id = data.get('root_block_id', None)
        match_string = data.get('match_string', None)

        # Process user-submitted data
        if block_types == 'all':
            block_types = []
        else:
            block_types = [block_types]

        user_service = self.runtime.service(self, 'user')
        if not self.user_is_staff():
            return {'error': 'permission denied'}
        if not username:
            user_id = None
        else:
            user_id = user_service.get_anonymous_user_id(username, unicode(self.runtime.course_id))
            if user_id is None:
                self.raise_error(404, _("Could not find the specified username."))

        if not root_block_id:
            root_block_id = self.scope_ids.usage_id
            # Block ID not in workbench runtime.
            root_block_id = unicode(getattr(root_block_id, 'block_id', root_block_id))

        # Launch task
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        self._delete_export()
        # Make sure we nail down our state before sending off an asynchronous task.
        self.save()
        async_result = export_data_task.delay(
            # course_id not available in workbench.
            unicode(getattr(self.runtime, 'course_id', 'course_id')),
            root_block_id,
            block_types,
            user_id,
            match_string,
        )
        if async_result.ready():
            # In development mode, the task may have executed synchronously.
            # Store the result now, because we won't be able to retrieve it later :-/
            if async_result.successful():
                # Make sure the result can be represented as JSON, since the non-eager celery
                # requires that
                json.dumps(async_result.result)
            self._save_result(async_result)
        else:
            # The task is running asynchronously. Store the result ID so we can query its progress:
            self.active_export_task_id = async_result.id
        return self._get_status()

    @XBlock.json_handler
    def cancel_export(self, request, suffix=''):
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        if self.active_export_task_id:
            async_result = export_data_task.AsyncResult(self.active_export_task_id)
            async_result.revoke()
            self._delete_export()

    def _get_user_attr(self, attr):
        """Get an attribute of the current user."""
        user_service = self.runtime.service(self, 'user')
        if user_service:
            # May be None when creating bok choy test fixtures
            return user_service.get_current_user().opt_attrs.get(attr)
        return None

    def user_is_staff(self):
        """Return a Boolean value indicating whether the current user is a member of staff."""
        return self._get_user_attr('edx-platform.user_is_staff')
<commit_msg>Make sure dropdown excludes all choice fields (not just children of MRQs).<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
#
# This software's license gives you freedom; you can copy, convey,
# propagate, redistribute and/or modify this program under the terms of
# the GNU Affero General Public License (AGPL) as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version of the AGPL published by the FSF.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program in a file in the toplevel directory called
# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
#
"""
Instructor Tool: An XBlock for instructors to export student answers from a course.

All processing is done offline.
"""
import json
from django.core.paginator import Paginator
from xblock.core import XBlock
from xblock.exceptions import JsonHandlerError
from xblock.fields import Scope, String, Dict, List
from xblock.fragment import Fragment
from xblockutils.resources import ResourceLoader

loader = ResourceLoader(__name__)

PAGE_SIZE = 15

# Make '_' a no-op so we can scrape strings
def _(text):
    return text

@XBlock.needs("i18n")
@XBlock.wants('user')
class InstructorToolBlock(XBlock):
    """
    InstructorToolBlock: An XBlock for instructors to export student answers from a course.

    All processing is done offline.
    """
    display_name = String(
        display_name=_("Title (Display name)"),
        help=_("Title to display"),
        default=_("Instructor Tool"),
        scope=Scope.settings
    )
    active_export_task_id = String(
        # The UUID of the celery AsyncResult for the most recent export,
        # IF we are sill waiting for it to finish
        default="",
        scope=Scope.user_state,
    )
    last_export_result = Dict(
        # The info dict returned by the most recent successful export.
        # If the export failed, it will have an "error" key set.
        default=None,
        scope=Scope.user_state,
    )
    display_data = List(
        # The list of results associated with the most recent successful export.
        # Stored separately to avoid the overhead of sending it to the client.
        default=None,
        scope=Scope.user_state,
    )
    has_author_view = True

    @property
    def display_name_with_default(self):
        return "Instructor Tool"

    def author_view(self, context=None):
        """ Studio View """
        # Warn the user that this block will only work from the LMS. (Since the CMS uses
        # different celery queues; our task listener is waiting for tasks on the LMS queue)
        return Fragment(u'
Instructor Tool Block

This block only works from the LMS.
')

    def studio_view(self, context=None):
        """ View for editing Instructor Tool block in Studio. """
        # Display friendly message explaining that the block is not editable.
        return Fragment(u'
This is a preconfigured block. It is not editable.
')

    def check_pending_export(self):
        """
        If we're waiting for an export, see if it has finished, and if so, get the result.
        """
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        if self.active_export_task_id:
            async_result = export_data_task.AsyncResult(self.active_export_task_id)
            if async_result.ready():
                self._save_result(async_result)

    def _save_result(self, task_result):
        """ Given an AsyncResult or EagerResult, save it. """
        self.active_export_task_id = ''
        if task_result.successful():
            if isinstance(task_result.result, dict) and not task_result.result.get('error'):
                self.display_data = task_result.result['display_data']
                del task_result.result['display_data']
                self.last_export_result = task_result.result
            else:
                self.last_export_result = {'error': u'Unexpected result: {}'.format(repr(task_result.result))}
                self.display_data = None
        else:
            self.last_export_result = {'error': unicode(task_result.result)}
            self.display_data = None

    @XBlock.json_handler
    def get_result_page(self, data, suffix=''):
        """ Return requested page of `last_export_result`. """
        paginator = Paginator(self.display_data, PAGE_SIZE)
        page = data.get('page', None)
        return {
            'display_data': paginator.page(page).object_list,
            'num_results': len(self.display_data),
            'page_size': PAGE_SIZE
        }

    def student_view(self, context=None):
        """ Normal View """
        if not self.user_is_staff():
            return Fragment(u'
This interface can only be used by course staff.
')
        block_choices = {
            _('Multiple Choice Question'): 'MCQBlock',
            _('Rating Question'): 'RatingBlock',
            _('Long Answer'): 'AnswerBlock',
        }
        block_types = ('pb-mcq', 'pb-rating', 'pb-answer')
        flat_block_tree = []

        def get_block_id(block):
            """
            Return ID of `block`, taking into account needs of both LMS/CMS and workbench runtimes.
            """
            usage_id = block.scope_ids.usage_id
            # Try accessing block ID. If usage_id does not have it, return usage_id itself
            return unicode(getattr(usage_id, 'block_id', usage_id))

        def get_block_name(block):
            """
            Return name of `block`.

            - For MCQs, Ratings, Answer blocks this is one of:
              - block.question
              - block.name (fallback for old courses)
            - For other types of (non-eligible) blocks this is one of:
              - block.question
              - block.display_name
              - block ID (fallback if neither 'question' nor 'display_name' is suitable)
            """
            block_type = get_block_type(block)
            # Eligible block (question)
            if block_type in block_types:
                return getattr(block, 'question', block.name)
            # Non-eligible block (question or section/subsection/unit)
            # - Try "question" attribute:
            block_name = getattr(block, 'question', None)
            if not block_name:
                # - Try display_name:
                block_name = getattr(block, 'display_name', None)
            if not block_name:
                # - Default to ID:
                block_name = get_block_id(block)
            return block_name

        def get_block_type(block):
            """
            Return type of `block`, taking into account different key styles that might be in use.
            """
            try:
                block_type = block.runtime.id_reader.get_block_type(block.scope_ids.def_id)
            except AttributeError:
                block_type = block.runtime.id_reader.get_block_type(block.scope_ids.usage_id)
            return block_type

        def build_tree(block, ancestors):
            """
            Build up a tree of information about the XBlocks descending from root_block
            """
            block_id = get_block_id(block)
            block_name = get_block_name(block)
            block_type = get_block_type(block)
            if not block_type == 'pb-choice':
                eligible = block_type in block_types
                if eligible:
                    # If this block is a question whose answers we can export,
                    # we mark all of its ancestors as exportable too
                    if ancestors and not ancestors[-1]["eligible"]:
                        for ancestor in ancestors:
                            ancestor["eligible"] = True

                new_entry = {
                    "depth": len(ancestors),
                    "id": block_id,
                    "name": block_name,
                    "eligible": eligible,
                }
                flat_block_tree.append(new_entry)
                if block.has_children and not getattr(block, "has_dynamic_children", lambda: False)():
                    for child_id in block.children:
                        build_tree(block.runtime.get_block(child_id), ancestors=(ancestors + [new_entry]))

        root_block = self
        while root_block.parent:
            root_block = root_block.get_parent()
        root_block_id = get_block_id(root_block)
        root_entry = {
            "depth": 0,
            "id": root_block_id,
            "name": "All",
        }
        flat_block_tree.append(root_entry)

        for child_id in root_block.children:
            child_block = root_block.runtime.get_block(child_id)
            build_tree(child_block, [root_entry])

        html = loader.render_template(
            'templates/html/instructor_tool.html',
            {'block_choices': block_choices, 'block_tree': flat_block_tree}
        )
        fragment = Fragment(html)
        fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/instructor_tool.css'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/instructor_tool.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/underscore-min.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/backbone-min.js'))
        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/vendor/backbone.paginator.min.js'))
        fragment.initialize_js('InstructorToolBlock')
        return fragment

    @property
    def download_url_for_last_report(self):
        """ Get the URL for the last report, if any """
        # Unfortunately this is a bit inefficient due to the ReportStore API
        if not self.last_export_result or self.last_export_result['error'] is not None:
            return None
        from instructor_task.models import ReportStore
        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
        course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
        return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])

    def _get_status(self):
        self.check_pending_export()
        return {
            'export_pending': bool(self.active_export_task_id),
            'last_export_result': self.last_export_result,
            'download_url': self.download_url_for_last_report,
        }

    def raise_error(self, code, message):
        """
        Raises an error and marks the block with a simulated failed task dict.
        """
        self.last_export_result = {
            'error': message,
        }
        self.display_data = None
        raise JsonHandlerError(code, message)

    @XBlock.json_handler
    def get_status(self, data, suffix=''):
        return self._get_status()

    @XBlock.json_handler
    def delete_export(self, data, suffix=''):
        self._delete_export()
        return self._get_status()

    def _delete_export(self):
        self.last_export_result = None
        self.display_data = None
        self.active_export_task_id = ''

    @XBlock.json_handler
    def start_export(self, data, suffix=''):
        """ Start a new asynchronous export """
        block_types = data.get('block_types', None)
        username = data.get('username', None)
        root_block_id = data.get('root_block_id', None)
        match_string = data.get('match_string', None)

        # Process user-submitted data
        if block_types == 'all':
            block_types = []
        else:
            block_types = [block_types]

        user_service = self.runtime.service(self, 'user')
        if not self.user_is_staff():
            return {'error': 'permission denied'}
        if not username:
            user_id = None
        else:
            user_id = user_service.get_anonymous_user_id(username, unicode(self.runtime.course_id))
            if user_id is None:
                self.raise_error(404, _("Could not find the specified username."))

        if not root_block_id:
            root_block_id = self.scope_ids.usage_id
            # Block ID not in workbench runtime.
            root_block_id = unicode(getattr(root_block_id, 'block_id', root_block_id))

        # Launch task
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        self._delete_export()
        # Make sure we nail down our state before sending off an asynchronous task.
        self.save()
        async_result = export_data_task.delay(
            # course_id not available in workbench.
            unicode(getattr(self.runtime, 'course_id', 'course_id')),
            root_block_id,
            block_types,
            user_id,
            match_string,
        )
        if async_result.ready():
            # In development mode, the task may have executed synchronously.
            # Store the result now, because we won't be able to retrieve it later :-/
            if async_result.successful():
                # Make sure the result can be represented as JSON, since the non-eager celery
                # requires that
                json.dumps(async_result.result)
            self._save_result(async_result)
        else:
            # The task is running asynchronously. Store the result ID so we can query its progress:
            self.active_export_task_id = async_result.id
        return self._get_status()

    @XBlock.json_handler
    def cancel_export(self, request, suffix=''):
        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
        if self.active_export_task_id:
            async_result = export_data_task.AsyncResult(self.active_export_task_id)
            async_result.revoke()
            self._delete_export()

    def _get_user_attr(self, attr):
        """Get an attribute of the current user."""
        user_service = self.runtime.service(self, 'user')
        if user_service:
            # May be None when creating bok choy test fixtures
            return user_service.get_current_user().opt_attrs.get(attr)
        return None

    def user_is_staff(self):
        """Return a Boolean value indicating whether the current user is a member of staff."""
        return self._get_user_attr('edx-platform.user_is_staff')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import time
from threading import Thread

from six.moves.queue import Queue, Empty

from rqalpha.interface import AbstractEventSource
from rqalpha.environment import Environment
from rqalpha.utils.logger import system_log
from rqalpha.events import Event, EVENT
from rqalpha.utils import rq_json
from .utils import get_realtime_quotes, order_book_id_2_tushare_code, is_holiday_today, is_tradetime_now
from . import data_board

class RealtimeEventSource(AbstractEventSource):
    MARKET_DATA_EVENT = "RealtimeEventSource.MARKET_DATA_EVENT"

    def __init__(self, fps):
        self._env = Environment.get_instance()
        self.fps = fps
        self.event_queue = Queue()

        self.before_trading_fire_date = datetime.date(2000, 1, 1)
        self.after_trading_fire_date = datetime.date(2000, 1, 1)

        self.quotation_engine_thread = Thread(target=self.quotation_worker)
        self.quotation_engine_thread.daemon = True
        self.quotation_worker_started = False

        self.clock_engine_thread = Thread(target=self.clock_worker)
        self.clock_engine_thread.daemon = True

    def set_state(self, state):
        persist_dict = rq_json.convert_json_to_dict(state.decode('utf-8'))
        self.before_trading_fire_date = persist_dict['before_trading_fire_date']
        self.after_trading_fire_date = persist_dict['after_trading_fire_date']

    def get_state(self):
        return rq_json.convert_dict_to_json({
            "before_trading_fire_date": self.before_trading_fire_date,
            "after_trading_fire_date": self.after_trading_fire_date,
        }).encode('utf-8')

    def quotation_worker(self):
        while True:
            if True or not is_holiday_today() and is_tradetime_now():
                order_book_id_list = sorted(Environment.get_instance().data_proxy.all_instruments("CS").order_book_id.tolist())
                code_list = [order_book_id_2_tushare_code(code) for code in order_book_id_list]

                try:
                    realtime_quotes_df = get_realtime_quotes(code_list)
                    self.event_queue.put((None, self.MARKET_DATA_EVENT, realtime_quotes_df))
                    self.quotation_worker_started = True
                except Exception as e:
                    system_log.exception("get_realtime_quotes fail")
                    continue

            time.sleep(1)

    def clock_worker(self):
        while True:
            # wait for the first data ready
            if self.quotation_worker_started:
                break
            time.sleep(0.1)

        while True:
            time.sleep(self.fps)

            if is_holiday_today():
                time.sleep(60)
                continue

            dt = datetime.datetime.now()

            if dt.strftime("%H:%M:%S") >= "08:30:00" and dt.date() > self.before_trading_fire_date:
                self.event_queue.put((dt, EVENT.BEFORE_TRADING, None))
                self.before_trading_fire_date = dt.date()
            elif dt.strftime("%H:%M:%S") >= "15:10:00" and dt.date() > self.after_trading_fire_date:
                self.event_queue.put((dt, EVENT.AFTER_TRADING, None))
                self.after_trading_fire_date = dt.date()

            if True or is_tradetime_now():
                self.event_queue.put((dt, EVENT.BAR, None))

    def events(self, start_date, end_date, frequency):
        running = True

        self.clock_engine_thread.start()
        self.quotation_engine_thread.start()

        while running:
            real_dt = datetime.datetime.now()
            while True:
                try:
                    dt, event_type, data = self.event_queue.get(timeout=1)
                    if event_type == self.MARKET_DATA_EVENT:
                        data_board.realtime_quotes_df = data
                        continue
                    break
                except Empty:
                    continue

            system_log.debug("real_dt {}, dt {}, event {}", real_dt, dt, event_type)
            yield Event(event_type, calendar_dt=real_dt, trading_dt=dt)
<commit_msg>refactor event source<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import time
from threading import Thread

from six.moves.queue import Queue, Empty

from rqalpha.interface import AbstractEventSource
from rqalpha.environment import Environment
from rqalpha.utils.logger import system_log
from rqalpha.events import Event, EVENT
from rqalpha.utils import rq_json
from .utils import get_realtime_quotes, order_book_id_2_tushare_code, is_holiday_today, is_tradetime_now
from . import data_board

class RealtimeEventSource(AbstractEventSource):
    MARKET_DATA_EVENT = "RealtimeEventSource.MARKET_DATA_EVENT"

    def __init__(self, fps):
        self._env = Environment.get_instance()
        self.fps = fps
        self.event_queue = Queue()

        self.before_trading_fire_date = datetime.date(2000, 1, 1)
        self.after_trading_fire_date = datetime.date(2000, 1, 1)

        self.quotation_engine_thread = Thread(target=self.quotation_worker)
        self.quotation_engine_thread.daemon = True

        self.clock_engine_thread = Thread(target=self.clock_worker)
        self.clock_engine_thread.daemon = True

    def set_state(self, state):
        persist_dict = rq_json.convert_json_to_dict(state.decode('utf-8'))
        self.before_trading_fire_date = persist_dict['before_trading_fire_date']
        self.after_trading_fire_date = persist_dict['after_trading_fire_date']

    def get_state(self):
        return rq_json.convert_dict_to_json({
            "before_trading_fire_date": self.before_trading_fire_date,
            "after_trading_fire_date": self.after_trading_fire_date,
        }).encode('utf-8')

    def quotation_worker(self):
        while True:
            if not is_holiday_today() and is_tradetime_now():
                order_book_id_list = sorted(Environment.get_instance().data_proxy.all_instruments("CS").order_book_id.tolist())
                code_list = [order_book_id_2_tushare_code(code) for code in order_book_id_list]

                try:
                    data_board.realtime_quotes_df = get_realtime_quotes(code_list)
                except Exception as e:
                    system_log.exception("get_realtime_quotes fail")
                    continue

            time.sleep(1)

    def clock_worker(self):
        while True:
            # wait for the first data ready
            if not data_board.realtime_quotes_df.empty:
                break
            time.sleep(0.1)

        while True:
            time.sleep(self.fps)

            if is_holiday_today():
                time.sleep(60)
                continue

            dt = datetime.datetime.now()

            if dt.strftime("%H:%M:%S") >= "08:30:00" and dt.date() > self.before_trading_fire_date:
                self.event_queue.put((dt, EVENT.BEFORE_TRADING))
                self.before_trading_fire_date = dt.date()
            elif dt.strftime("%H:%M:%S") >= "15:10:00" and dt.date() > self.after_trading_fire_date:
                self.event_queue.put((dt, EVENT.AFTER_TRADING))
                self.after_trading_fire_date = dt.date()

            if is_tradetime_now():
                self.event_queue.put((dt, EVENT.BAR))

    def events(self, start_date, end_date, frequency):
        running = True

        self.clock_engine_thread.start()
        self.quotation_engine_thread.start()

        while running:
            real_dt = datetime.datetime.now()
            while True:
                try:
                    dt, event_type = self.event_queue.get(timeout=1)
                    break
                except Empty:
                    continue

            system_log.debug("real_dt {}, dt {}, event {}", real_dt, dt, event_type)
            yield Event(event_type, calendar_dt=real_dt, trading_dt=dt)
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing enhanced db.Model classes.

The classes in this module are intended to serve as base classes for all
Melange Datastore Models.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
]

from google.appengine.ext import db

from soc.views.helper import forms as forms_helper

class ModelWithFieldAttributes(db.Model):
  """A db.Model extension that provides access to Model properties attributes.

  Due to the way the Property class in Google App Engine implements __get__()
  and __set__(), it is not possible to access attributes of Model properties,
  such as verbose_name, from within a Django template.  This class works
  around that limitation by creating an inner Form class per Model class,
  since an unbound Form object contains (most of?) the property attributes
  attached to each corresponding Form field.

  Some are attributes are renamed during the conversion from a Model Property
  to a Form field; for example, verbose_name becomes label.  This is tolerable
  because any actual Form code refers to these new names, so they are should
  be familiar to view creators.
  """

  _fields_cache = None
  DICT_TYPES = (db.StringProperty, db.IntegerProperty)

  def toDict(self, field_names=None):
    """Returns a dict with all specified values of this entity.

    Args:
      field_names: the fields that should be included, defaults to
        all fields that are of a type that is in DICT_TYPES.
    """

    result = {}

    if not field_names:
      props = self.properties().iteritems()
      field_names = [k for k, v in props if isinstance(v, self.DICT_TYPES)]

    for key in field_names:
      # Skip everything that is not valid
      if not hasattr(self, key):
        continue

      value = getattr(self, key)

      if callable(value):
        value = value()

      result[key] = value

    return result

  @classmethod
  def fields(cls):
    """Called by the Django template engine during template instantiation.

    Since the attribute names use the Form fields naming instead of the
    Property attribute naming, accessing, for example:
      {{ entity.property.verbose_name }}
    is accomplished using:
      {{ entity.fields.property.label }}

    Args:
      cls: Model class, so that each Model class can create its own
        unbound Form the first time fields() is called by the Django
        template engine.

    Returns:
      A (created-on-first-use) unbound Form object that can be used to
      access Property attributes that are not accessible from the
      Property itself via the Model entity.
    """
    if not cls._fields_cache or (cls != cls._fields_cache.__class__.Meta.model):
      class FieldsProxy(forms_helper.BaseForm):
        """Form used as a proxy to access User model properties attributes.
        """

        class Meta:
          """Inner Meta class that pairs the User Model with this "form".
          """
          #: db.Model subclass for which to access model properties attributes
          model = cls

      cls._fields_cache = FieldsProxy()
    return cls._fields_cache
<commit_msg>Fixed whitespace in base model.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing enhanced db.Model classes.

The classes in this module are intended to serve as base classes for all
Melange Datastore Models.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
]

from google.appengine.ext import db

from soc.views.helper import forms as forms_helper

class ModelWithFieldAttributes(db.Model):
  """A db.Model extension that provides access to Model properties attributes.

  Due to the way the Property class in Google App Engine implements __get__()
  and __set__(), it is not possible to access attributes of Model properties,
  such as verbose_name, from within a Django template.  This class works
  around that limitation by creating an inner Form class per Model class,
  since an unbound Form object contains (most of?) the property attributes
  attached to each corresponding Form field.

  Some are attributes are renamed during the conversion from a Model Property
  to a Form field; for example, verbose_name becomes label.  This is tolerable
  because any actual Form code refers to these new names, so they are should
  be familiar to view creators.
  """

  _fields_cache = None
  DICT_TYPES = (db.StringProperty, db.IntegerProperty)

  def toDict(self, field_names=None):
    """Returns a dict with all specified values of this entity.

    Args:
      field_names: the fields that should be included, defaults to
        all fields that are of a type that is in DICT_TYPES.
    """

    result = {}

    if not field_names:
      props = self.properties().iteritems()
      field_names = [k for k, v in props if isinstance(v, self.DICT_TYPES)]

    for key in field_names:
      # Skip everything that is not valid
      if not hasattr(self, key):
        continue

      value = getattr(self, key)

      if callable(value):
        value = value()

      result[key] = value

    return result

  @classmethod
  def fields(cls):
    """Called by the Django template engine during template instantiation.

    Since the attribute names use the Form fields naming instead of the
    Property attribute naming, accessing, for example:
      {{ entity.property.verbose_name }}
    is accomplished using:
      {{ entity.fields.property.label }}

    Args:
      cls: Model class, so that each Model class can create its own
        unbound Form the first time fields() is called by the Django
        template engine.

    Returns:
      A (created-on-first-use) unbound Form object that can be used to
      access Property attributes that are not accessible from the
      Property itself via the Model entity.
    """
    if not cls._fields_cache or (cls != cls._fields_cache.__class__.Meta.model):
      class FieldsProxy(forms_helper.BaseForm):
        """Form used as a proxy to access User model properties attributes.
        """

        class Meta:
          """Inner Meta class that pairs the User Model with this "form".
          """
          #: db.Model subclass for which to access model properties attributes
          model = cls

      cls._fields_cache = FieldsProxy()
    return cls._fields_cache
<|endoftext|>


<commit_before>from collections import Counter
import math
import numpy

from nlp import nlp_utils
from utils.constants import Constants

__author__ = 'fpena'

def get_review_metrics(record):
    """
    Returns a list with the metrics of a review. This list is composed
    in the following way: [log(num_sentences + 1), log(num_words + 1),
    log(num_past_verbs + 1), log(num_verbs + 1),
    (log(num_past_verbs + 1) / log(num_verbs + 1))

    :type record: dict
    :param record: the review that wants to be analyzed, it should contain the
    text of the review
    :rtype: list[float]
    :return: a list with numeric metrics
    """
    review_text = record[Constants.TEXT_FIELD]
    log_sentences = math.log(len(nlp_utils.get_sentences(review_text)) + 1)
    log_words = math.log(len(nlp_utils.get_words(review_text)) + 1)
    # log_time_words = math.log(len(self.get_time_words(review.text)) + 1)
    tagged_words = record[Constants.POS_TAGS_FIELD]
    counts = Counter(tag for word, tag, lemma in tagged_words)
    # print(counts)
    log_past_verbs = math.log(counts['VBD'] + 1)
    log_verbs = math.log(nlp_utils.count_verbs(counts) + 1)
    log_personal_pronouns = math.log(counts['PRP'] + 1)
    # log_sentences = float(len(get_sentences(review.text)) + 1)
    # log_words = float(len(get_words(review.text)) + 1)
    # log_time_words = float(len(self.get_time_words(review.text)) + 1)
    # tagged_words = review.tagged_words
    # counts = Counter(tag for word, tag in tagged_words)
    # log_past_verbs = float(counts['VBD'] + 1)
    # log_verbs = float(count_verbs(counts) + 1)
    # log_personal_pronouns = float(counts['PRP'] + 1)

    # This ensures that when log_verbs = 0 the program won't crash
    if log_verbs == 0:
        past_verbs_ratio = 0
    else:
        past_verbs_ratio = log_past_verbs / log_verbs
    # This ensures that when log_verbs = 0 the program won't crash
    if log_words == 0:
        verbs_ratio = 0
        past_verbs_ratio2 = 0
        personal_pronouns_ratio = 0
    else:
        verbs_ratio = log_verbs / log_words
        past_verbs_ratio2 = log_past_verbs / log_words
        # time_words_ratio = log_time_words / log_words
        personal_pronouns_ratio = log_personal_pronouns / log_words

    result = [log_words, log_past_verbs, log_verbs, past_verbs_ratio, verbs_ratio, log_personal_pronouns, personal_pronouns_ratio, past_verbs_ratio2]#, log_time_words, time_words_ratio]
    # print(result)
    # result = [log_sentences]
    # result = [log_personal_pronouns, log_past_verbs, log_words]
    # result = [time_words_ratio, personal_pronouns_ratio]
    # result = [log_past_verbs]
    # result = [log_verbs]
    # result = [past_verbs_ratio]
    # result = [past_verbs_ratio2]
    # result = [log_personal_pronouns]
    # result = [personal_pronouns_ratio]
    # result = [log_time_words]
    # result = [time_words_ratio]

    return numpy.array(result)

def normalize_matrix_by_columns(matrix, min_values=None, max_values=None):
    """

    :type matrix: numpy.array
    """
    if max_values is None:
        max_values = matrix.max(axis=0)
    if min_values is None:
        min_values = matrix.min(axis=0)

    # print('max values', max_values)
    # print('min values', min_values)

    for index in range(matrix.shape[1]):
        matrix[:, index] -= min_values[index]
        matrix[:, index] /= (max_values[index] - min_values[index])
<commit_msg>Fixed small bug. Words in reviews where being counted wrong, they were including punctuation marks<commit_after>from collections import Counter
import math
import numpy

from nlp import nlp_utils
from utils.constants import Constants

__author__ = 'fpena'

def get_review_metrics(record):
    """
    Returns a list with the metrics of a review. This list is composed
    in the following way: [log(num_sentences + 1), log(num_words + 1),
    log(num_past_verbs + 1), log(num_verbs + 1),
    (log(num_past_verbs + 1) / log(num_verbs + 1))

    :type record: dict
    :param record: the review that wants to be analyzed, it should contain the
    text of the review
    :rtype: list[float]
    :return: a list with numeric metrics
    """
    review_text = record[Constants.TEXT_FIELD]
    log_sentences = math.log(len(nlp_utils.get_sentences(review_text)) + 1)
    # log_time_words = math.log(len(self.get_time_words(review.text)) + 1)
    tagged_words = record[Constants.POS_TAGS_FIELD]
    log_words = math.log(len(tagged_words) + 1)
    counts = Counter(tag for word, tag, lemma in tagged_words)
    # print(counts)
    log_past_verbs = math.log(counts['VBD'] + 1)
    log_verbs = math.log(nlp_utils.count_verbs(counts) + 1)
    log_personal_pronouns = math.log(counts['PRP'] + 1)
    # log_sentences = float(len(get_sentences(review.text)) + 1)
    # log_words = float(len(get_words(review.text)) + 1)
    # log_time_words = float(len(self.get_time_words(review.text)) + 1)
    # tagged_words = review.tagged_words
    # counts = Counter(tag for word, tag in tagged_words)
    # log_past_verbs = float(counts['VBD'] + 1)
    # log_verbs = float(count_verbs(counts) + 1)
    # log_personal_pronouns = float(counts['PRP'] + 1)

    # This ensures that when log_verbs = 0 the program won't crash
    if log_verbs == 0:
        past_verbs_ratio = 0
    else:
        past_verbs_ratio = log_past_verbs / log_verbs
    # This ensures that when log_verbs = 0 the program won't crash
    if log_words == 0:
        verbs_ratio = 0
        past_verbs_ratio2 = 0
        personal_pronouns_ratio = 0
    else:
        verbs_ratio = log_verbs / log_words
        past_verbs_ratio2 = log_past_verbs / log_words
        # time_words_ratio = log_time_words / log_words
        personal_pronouns_ratio = log_personal_pronouns / log_words

    result = [log_words, log_past_verbs, log_verbs, past_verbs_ratio, verbs_ratio, log_personal_pronouns, personal_pronouns_ratio, past_verbs_ratio2]#, log_time_words, time_words_ratio]
    # print(result)
    # result = [log_sentences]
    # result = [log_personal_pronouns, log_past_verbs, log_words]
    # result = [time_words_ratio, personal_pronouns_ratio]
    # result = [log_past_verbs]
    # result = [log_verbs]
    # result = [past_verbs_ratio]
    # result = [past_verbs_ratio2]
    # result = [log_personal_pronouns]
    # result = [personal_pronouns_ratio]
    # result = [log_time_words]
    # result = [time_words_ratio]

    return numpy.array(result)

def normalize_matrix_by_columns(matrix, min_values=None, max_values=None):
    """

    :type matrix: numpy.array
    """
    if max_values is None:
        max_values = matrix.max(axis=0)
    if min_values is None:
        min_values = matrix.min(axis=0)

    # print('max values', max_values)
    # print('min values', min_values)

    for index in range(matrix.shape[1]):
        matrix[:, index] -= min_values[index]
        matrix[:, index] /= (max_values[index] - min_values[index])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os

from django.db import models, migrations
import datetime
import tests.models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        # ('contenttypes', '0002_remove_content_type_name'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='All',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Brand',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='BrandT',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='CacheOnSaveModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=32)),
            ],
        ),
        migrations.CreateModel(
            name='Category',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Contained',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=30)),
            ],
        ),
        migrations.CreateModel(
            name='DbAgnostic',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='DbBinded',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='Extra',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(unique=True, db_column=b'custom_column_name')),
            ],
        ),
        migrations.CreateModel(
            name='GenericContainer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('object_id', models.PositiveIntegerField()),
                ('name', models.CharField(max_length=30)),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType')),
            ],
        ),
        migrations.CreateModel(
            name='Label',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('text', models.CharField(default=b'', max_length=127, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Labeling',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField()),
                ('brand', models.ForeignKey(to='tests.BrandT')),
            ],
        ),
        migrations.CreateModel(
            name='LabelT',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('text', models.CharField(default=b'', max_length=127, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Local',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Media',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Point',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('x', models.DecimalField(default=0.0, max_digits=8, decimal_places=6, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Post',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
                ('visible', models.BooleanField(default=True)),
                ('category', models.ForeignKey(related_name='posts', to='tests.Category')),
            ],
        ),
        migrations.CreateModel(
            name='Product',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=32)),
            ],
        ),
        migrations.CreateModel(
            name='ProductReview',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('status', models.IntegerField()),
                ('product', models.ForeignKey(related_name='reviews', to='tests.Product', null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Profile',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField()),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
        ),
        migrations.CreateModel(
            name='Video',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Weird',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('date_field', models.DateField(default=datetime.date(2000, 1, 1))),
                ('datetime_field', models.DateTimeField(default=datetime.datetime(2000, 1, 1, 10, 10))),
                ('time_field', models.TimeField(default=datetime.time(10, 10))),
                ('list_field', tests.models.IntegerArrayField(default=list)),
                ('custom_field', tests.models.CustomField(default=tests.models.custom_value_default)),
                ('binary_field', models.BinaryField()),
            ],
        ),
        migrations.CreateModel(
            name='Movie',
            fields=[
                ('media_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='tests.Media')),
                ('year', models.IntegerField()),
            ],
            bases=('tests.media',),
        ),
        migrations.CreateModel(
            name='PremiumBrand',
            fields=[
                ('brand_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='tests.Brand')),
                ('extra', models.CharField(default=b'', max_length=127, blank=True)),
            ],
            bases=('tests.brand',),
        ),
        migrations.AddField(
            model_name='labeling',
            name='label',
            field=models.ForeignKey(to='tests.LabelT'),
        ),
        migrations.AddField(
            model_name='extra',
            name='post',
            field=models.OneToOneField(to='tests.Post'),
        ),
        migrations.AddField(
            model_name='extra',
            name='to_tag',
            field=models.ForeignKey(to='tests.Extra', to_field=b'tag', null=True),
        ),
        migrations.AddField(
            model_name='brandt',
            name='labels',
            field=models.ManyToManyField(related_name='brands', through='tests.Labeling', to='tests.LabelT'),
        ),
        migrations.AddField(
            model_name='brand',
            name='labels',
            field=models.ManyToManyField(related_name='brands', to='tests.Label'),
        ),
        migrations.CreateModel(
            name='VideoProxy',
            fields=[
            ],
            options={
                'proxy': True,
            },
            bases=('tests.video',),
        ),
    ]

    from funcy import suppress

    with suppress(ImportError):
        import django.contrib.postgres.fields
        operations.append(
            migrations.CreateModel(
                name='TaggedPost',
                fields=[
                    ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                    ('name', models.CharField(max_length=200)),
                    ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None)),
                ],
            )
        )

    if os.environ.get('CACHEOPS_DB') == 'postgis':
        import django.contrib.gis.db.models.fields
        operations.append(
            migrations.CreateModel(
                name='Geometry',
                fields=[
                    ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                    ('point', django.contrib.gis.db.models.fields.PointField(blank=True, default=None, dim=3, geography=True, null=True, srid=4326)),
                ],
            )
        )
<commit_msg>Fix migrations for Python 3<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os

from django.db import models, migrations
import datetime
import tests.models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        # ('contenttypes', '0002_remove_content_type_name'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='All',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Brand',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='BrandT',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='CacheOnSaveModel',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=32)),
            ],
        ),
        migrations.CreateModel(
            name='Category',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Contained',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=30)),
            ],
        ),
        migrations.CreateModel(
            name='DbAgnostic',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='DbBinded',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
            ],
        ),
        migrations.CreateModel(
            name='Extra',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(unique=True, db_column='custom_column_name')),
            ],
        ),
        migrations.CreateModel(
            name='GenericContainer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('object_id', models.PositiveIntegerField()),
                ('name', models.CharField(max_length=30)),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType')),
            ],
        ),
        migrations.CreateModel(
            name='Label',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('text', models.CharField(default='', max_length=127, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Labeling',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField()),
                ('brand', models.ForeignKey(to='tests.BrandT')),
            ],
        ),
        migrations.CreateModel(
            name='LabelT',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('text', models.CharField(default='', max_length=127, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Local',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField(null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Media',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Point',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('x', models.DecimalField(default=0.0, max_digits=8, decimal_places=6, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='Post',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
                ('visible', models.BooleanField(default=True)),
                ('category', models.ForeignKey(related_name='posts', to='tests.Category')),
            ],
        ),
        migrations.CreateModel(
            name='Product',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=32)),
            ],
        ),
        migrations.CreateModel(
            name='ProductReview',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('status', models.IntegerField()),
                ('product', models.ForeignKey(related_name='reviews', to='tests.Product', null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Profile',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('tag', models.IntegerField()),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
        ),
        migrations.CreateModel(
            name='Video',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=128)),
            ],
        ),
        migrations.CreateModel(
            name='Weird',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('date_field', models.DateField(default=datetime.date(2000, 1, 1))),
                ('datetime_field', models.DateTimeField(default=datetime.datetime(2000, 1, 1, 10, 10))),
                ('time_field', models.TimeField(default=datetime.time(10, 10))),
                ('list_field', tests.models.IntegerArrayField(default=list)),
                ('custom_field', tests.models.CustomField(default=tests.models.custom_value_default)),
                ('binary_field', models.BinaryField()),
            ],
        ),
        migrations.CreateModel(
            name='Movie',
            fields=[
                ('media_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='tests.Media')),
                ('year', models.IntegerField()),
            ],
            bases=('tests.media',),
        ),
        migrations.CreateModel(
            name='PremiumBrand',
            fields=[
                ('brand_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='tests.Brand')),
                ('extra', models.CharField(default='', max_length=127, blank=True)),
            ],
            bases=('tests.brand',),
        ),
        migrations.AddField(
            model_name='labeling',
            name='label',
            field=models.ForeignKey(to='tests.LabelT'),
        ),
        migrations.AddField(
            model_name='extra',
            name='post',
            field=models.OneToOneField(to='tests.Post'),
        ),
        migrations.AddField(
            model_name='extra',
            name='to_tag',
            field=models.ForeignKey(to='tests.Extra', to_field='tag', null=True),
        ),
        migrations.AddField(
            model_name='brandt',
            name='labels',
            field=models.ManyToManyField(related_name='brands', through='tests.Labeling', to='tests.LabelT'),
        ),
        migrations.AddField(
            model_name='brand',
            name='labels',
            field=models.ManyToManyField(related_name='brands', to='tests.Label'),
        ),
        migrations.CreateModel(
            name='VideoProxy',
            fields=[
            ],
            options={
                'proxy': True,
            },
            bases=('tests.video',),
        ),
    ]

    from funcy import suppress

    with suppress(ImportError):
        import django.contrib.postgres.fields
        operations.append(
            migrations.CreateModel(
                name='TaggedPost',
                fields=[
                    ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                    ('name', models.CharField(max_length=200)),
                    ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None)),
                ],
            )
        )

    if os.environ.get('CACHEOPS_DB') == 'postgis':
        import django.contrib.gis.db.models.fields
        operations.append(
            migrations.CreateModel(
                name='Geometry',
                fields=[
                    ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                    ('point', django.contrib.gis.db.models.fields.PointField(blank=True, default=None, dim=3, geography=True, null=True, srid=4326)),
                ],
            )
        )
<|endoftext|>


<commit_before>"""Usage:
    octobot <msg> [options]

    Options:
        -u --incoming-webhook-url       Override incoming webhook url
        -n --username                   Override outgoing username
        -c --channel                    Override outgoing channel
        -i --icon-emoji                 Override outgoing emoji

"""

from __future__ import print_function
import sys
from pprint import pprint

from docopt import docopt
from octobot import __version__
from octobot.config import load_config
from octobot.utils import parse_arguments
from octobot.utils import config_data_to_incoming_webhook_args

def main():
    arguments = docopt(
        __doc__, version=__version__
    )

    print('\narguments:')
    print(arguments)

    print('\nparsed arguments:')
    print(parse_arguments(**arguments))

    config = load_config(**parse_arguments(**arguments))

    print('\nconfig data:')
    print(config.data.OCTOBOT_INCOMING_WEBHOOK_URL)
    print(config.data.OCTOBOT_MESSAGE)
    print(config.data.OCTOBOT_USERNAME)
    print(config.data.OCTOBOT_CHANNEL)
    print(config.data.OCTOBOT_ICON_EMOJI)

    print('\nconfig.data')
    pprint(config.data.__dict__)

    print('\nwebhook args')
    print(config_data_to_incoming_webhook_args(config.data))

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Fixes bad usage pattern options<commit_after>"""Usage:
    octobot <msg> [options]

    Options:
        -u URL, --incoming-webhook-url <url>     Override incoming webhook url
        -n NAME, --username <name>               Override outgoing username
        -c CHANNEL, --channel <channel>          Override outgoing channel
        -i ICON, --icon-emoji <icon>             Override outgoing emoji

"""

from __future__ import print_function
import sys
from pprint import pprint

from docopt import docopt
from octobot import __version__
from octobot.config import load_config
from octobot.utils import parse_arguments
from octobot.utils import config_data_to_incoming_webhook_args

def main():
    arguments = docopt(
        __doc__, version=__version__
    )

    print('\narguments:')
    print(arguments)

    print('\nparsed arguments:')
    print(parse_arguments(**arguments))

    config = load_config(**parse_arguments(**arguments))

    print('\nconfig data:')
    print(config.data.OCTOBOT_INCOMING_WEBHOOK_URL)
    print(config.data.OCTOBOT_MESSAGE)
    print(config.data.OCTOBOT_USERNAME)
    print(config.data.OCTOBOT_CHANNEL)
    print(config.data.OCTOBOT_ICON_EMOJI)

    print('\nconfig.data')
    pprint(config.data.__dict__)

    print('\nwebhook args')
    print(config_data_to_incoming_webhook_args(config.data))

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>