<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import io
import os
import numpy
import struct
import datetime

from . import base
from . import metadata
from .utils import b

#: Automatically detect whether the output is a TTY, if so, write ASCII
#: otherwise write BINARY
AUTOMATIC = 0
#: Force writing ASCII
ASCII = 1
#: Force writing BINARY
BINARY = 2

#: Amount of bytes to read while using buffered reading
BUFFER_SIZE = 4096
#: The amount of bytes in the header field
HEADER_SIZE = 80
#: The amount of bytes in the count field
COUNT_SIZE = 4
#: The maximum amount of triangles we can read from binary files
MAX_COUNT = 1e6

class BaseStl(base.BaseMesh):
    @classmethod
    def load(cls, fh, mode=AUTOMATIC):
        '''Load Mesh from STL file

        Automatically detects binary versus ascii STL files.

        :param file fh: The file handle to open
        :param int mode: Automatically detect the filetype or force binary
        '''
        header = fh.read(HEADER_SIZE).lower()
        if not header.strip():
            return

        if isinstance(header, str):  # pragma: no branch
            header = b(header)

        name = ''

        if mode in (AUTOMATIC, ASCII) and header.startswith(b('solid')):
            try:
                data = cls._load_ascii(fh, header)

                # Get the name from the first line
                name = header.split(b('\n'), 1)[0][5:].strip()
            except RuntimeError as exception:
                (recoverable, e) = exception.args
                if recoverable:  # Recoverable?
                    data = cls._load_binary(fh, header, check_size=False)
                else:
                    # Apparently we've read beyond the header. Let's try
                    # seeking :)
                    # Note that this fails when reading from stdin, we can't
                    # recover from that.
                    fh.seek(HEADER_SIZE)

                    # Since we know this is a seekable file now and we're not
                    # 100% certain it's binary, check the size while reading
                    data = cls._load_binary(fh, header, check_size=True)
        else:
            data = cls._load_binary(fh, header)

        return name, data

    @classmethod
    def _load_binary(cls, fh, header, check_size=False):
        # Read the triangle count
        count, = struct.unpack('@i', b(fh.read(COUNT_SIZE)))
        # raise RuntimeError()
        assert count < MAX_COUNT, ('File too large, got %d triangles which '
                                   'exceeds the maximum of %d') % (
                                       count, MAX_COUNT)

        if check_size:
            try:
                # Check the size of the file
                fh.seek(0, os.SEEK_END)
                raw_size = fh.tell() - HEADER_SIZE - COUNT_SIZE
                expected_count = int(raw_size / cls.dtype.itemsize)
                assert expected_count == count, ('Expected %d vectors but '
                                                 'header indicates %d') % (
                                                     expected_count, count)
                fh.seek(HEADER_SIZE + COUNT_SIZE)
            except IOError:  # pragma: no cover
                pass

        # Read the rest of the binary data
        return numpy.fromfile(fh, dtype=cls.dtype, count=count)

    @classmethod
    def _ascii_reader(cls, fh, header):
        lines = b(header).split(b('\n'))
        recoverable = [True]

        def get(prefix=''):
            prefix = b(prefix)

            if lines:
                line = lines.pop(0)
            else:
                raise RuntimeError(recoverable[0], 'Unable to find more lines')
            if not lines:
                recoverable[0] = False

                # Read more lines and make sure we prepend any old data
                lines[:] = b(fh.read(BUFFER_SIZE)).split(b('\n'))
                line += lines.pop(0)

            line = line.lower().strip()
            if prefix:
                if line.startswith(prefix):
                    values = line.replace(prefix, b(''), 1).strip().split()
                elif line.startswith(b('endsolid')):
                    raise StopIteration()
                else:
                    raise RuntimeError(recoverable[0],
                                       '%r should start with %r' % (line,
                                                                    prefix))

                if len(values) == 3:
                    return [float(v) for v in values]
                else:  # pragma: no cover
                    raise RuntimeError(recoverable[0],
                                       'Incorrect value %r' % line)
            else:
                return b(line)

        line = get()
        if not line.startswith(b('solid ')) and \
                line.startswith(b('solid')):
            cls.warning('ASCII STL files should start with solid <space>. '
                        'The application that produced this STL file may be '
                        'faulty, please report this error. The erroneous '
                        'line: %r', line)

        if not lines:
            raise RuntimeError(recoverable[0],
                               'No lines found, impossible to read')

        while True:
            # Read from the header lines first, until that point we can recover
            # and go to the binary option. After that we cannot due to
            # unseekable files such as sys.stdin
            #
            # Numpy doesn't support any non-file types so wrapping with a
            # buffer and/or StringIO does not work.
            try:
                normals = get('facet normal')
                assert get() == b('outer loop')
                v0 = get('vertex')
                v1 = get('vertex')
                v2 = get('vertex')
                assert get() == b('endloop')
                assert get() == b('endfacet')
                attrs = 0
                yield (normals, (v0, v1, v2), attrs)
            except AssertionError as e:
                raise RuntimeError(recoverable[0], e)
            except StopIteration:
                raise

    @classmethod
    def _load_ascii(cls, fh, header):
        return numpy.fromiter(cls._ascii_reader(fh, header), dtype=cls.dtype)

    def save(self, filename, fh=None, mode=AUTOMATIC, update_normals=True):
        '''Save the STL to a (binary) file

        If mode is :py:data:`AUTOMATIC` an :py:data:`ASCII` file will be
        written if the output is a TTY and a :py:data:`BINARY` file otherwise.

        :param str filename: The file to load
        :param file fh: The file handle to open
        :param int mode: The mode to write, default is :py:data:`AUTOMATIC`.
        :param bool update_normals: Whether to update the normals
        '''
        assert filename, 'Filename is required for the STL headers'
        if update_normals:
            self.update_normals()

        if mode is AUTOMATIC:
            if fh and os.isatty(fh.fileno()):  # pragma: no cover
                write = self._write_ascii
            else:
                write = self._write_binary
        elif mode is BINARY:
            write = self._write_binary
        elif mode is ASCII:
            write = self._write_ascii
        else:
            raise ValueError('Mode %r is invalid' % mode)

        name = os.path.split(filename)[-1]
        try:
            if fh:
                write(fh, name)
            else:
                with open(name, 'wb') as fh:
                    write(fh, filename)
        except IOError:  # pragma: no cover
            pass

    def _write_ascii(self, fh, name):
        print(fh)
        print(type(fh))

        def p(s, file):
            file.write(b('%s\n' % s))

        p('solid %s' % name, file=fh)

        for row in self.data:
            vectors = row['vectors']
            p('facet normal %f %f %f' % tuple(row['normals']), file=fh)
            p('  outer loop', file=fh)
            p('    vertex %f %f %f' % tuple(vectors[0]), file=fh)
            p('    vertex %f %f %f' % tuple(vectors[1]), file=fh)
            p('    vertex %f %f %f' % tuple(vectors[2]), file=fh)
            p('  endloop', file=fh)
            p('endfacet', file=fh)

        p('endsolid %s' % name, file=fh)

    def _write_binary(self, fh, name):
        # Create the header
        header = '%s (%s) %s %s' % (
            metadata.__package_name__,
            metadata.__version__,
            datetime.datetime.now(),
            name,
        )

        # Make it exactly 80 characters
        header = header[:80].ljust(80, ' ')
        packed = struct.pack('@i', self.data.size)

        if isinstance(fh, io.TextIOWrapper):  # pragma: no cover
            packed = str(packed)
        else:
            header = b(header)
            packed = b(packed)

        fh.write(header)
        fh.write(packed)
        self.data.tofile(fh)

    @classmethod
    def from_file(cls, filename, calculate_normals=True, fh=None,
                  mode=AUTOMATIC, **kwargs):
        '''Load a mesh from a STL file

        :param str filename: The file to load
        :param bool calculate_normals: Whether to update the normals
        :param file fh: The file handle to open
        :param dict \**kwargs: The same as for :py:class:`stl.mesh.Mesh`

        '''
        if fh:
            name, data = cls.load(fh, mode=mode)
        else:
            with open(filename, 'rb') as fh:
                name, data = cls.load(fh, mode=mode)

        return cls(data, calculate_normals, name=name, **kwargs)

    @classmethod
    def from_multi_file(cls, filename, calculate_normals=True, fh=None,
                        mode=ASCII, **kwargs):
        '''Load multiple meshes from a STL file

        :param str filename: The file to load
        :param bool calculate_normals: Whether to update the normals
        :param file fh: The file handle to open
        :param dict \**kwargs: The same as for :py:class:`stl.mesh.Mesh`
        '''
        if fh:
            close = False
        else:
            fh = open(filename, 'rb')
            close = True

        try:
            raw_data = cls.load(fh, mode=mode)
            while raw_data:
                name, data = raw_data
                yield cls(data, calculate_normals, name=name, **kwargs)
                raw_data = cls.load(fh, mode=mode)

        finally:
            if close:
                fh.close()

StlMesh = BaseStl.from_file

<commit_msg>fixed #10<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import io
import os
import numpy
import struct
import datetime

from . import base
from . import metadata
from .utils import b
from .utils import s

#: Automatically detect whether the output is a TTY, if so, write ASCII
#: otherwise write BINARY
AUTOMATIC = 0
#: Force writing ASCII
ASCII = 1
#: Force writing BINARY
BINARY = 2

#: Amount of bytes to read while using buffered reading
BUFFER_SIZE = 4096
#: The amount of bytes in the header field
HEADER_SIZE = 80
#: The amount of bytes in the count field
COUNT_SIZE = 4
#: The maximum amount of triangles we can read from binary files
MAX_COUNT = 1e6

class BaseStl(base.BaseMesh):
    @classmethod
    def load(cls, fh, mode=AUTOMATIC):
        '''Load Mesh from STL file

        Automatically detects binary versus ascii STL files.

        :param file fh: The file handle to open
        :param int mode: Automatically detect the filetype or force binary
        '''
        header = fh.read(HEADER_SIZE).lower()
        if not header.strip():
            return

        if isinstance(header, str):  # pragma: no branch
            header = b(header)

        name = ''

        if mode in (AUTOMATIC, ASCII) and header.startswith(b('solid')):
            try:
                data = cls._load_ascii(fh, header)

                # Get the name from the first line
                name = header.split(b('\n'), 1)[0][5:].strip()
            except RuntimeError as exception:
                (recoverable, e) = exception.args
                if recoverable:  # Recoverable?
                    data = cls._load_binary(fh, header, check_size=False)
                else:
                    # Apparently we've read beyond the header. Let's try
                    # seeking :)
                    # Note that this fails when reading from stdin, we can't
                    # recover from that.
                    fh.seek(HEADER_SIZE)

                    # Since we know this is a seekable file now and we're not
                    # 100% certain it's binary, check the size while reading
                    data = cls._load_binary(fh, header, check_size=True)
        else:
            data = cls._load_binary(fh, header)

        return name, data

    @classmethod
    def _load_binary(cls, fh, header, check_size=False):
        # Read the triangle count
        count, = struct.unpack(s('@i'), b(fh.read(COUNT_SIZE)))
        # raise RuntimeError()
        assert count < MAX_COUNT, ('File too large, got %d triangles which '
                                   'exceeds the maximum of %d') % (
                                       count, MAX_COUNT)

        if check_size:
            try:
                # Check the size of the file
                fh.seek(0, os.SEEK_END)
                raw_size = fh.tell() - HEADER_SIZE - COUNT_SIZE
                expected_count = int(raw_size / cls.dtype.itemsize)
                assert expected_count == count, ('Expected %d vectors but '
                                                 'header indicates %d') % (
                                                     expected_count, count)
                fh.seek(HEADER_SIZE + COUNT_SIZE)
            except IOError:  # pragma: no cover
                pass

        # Read the rest of the binary data
        return numpy.fromfile(fh, dtype=cls.dtype, count=count)

    @classmethod
    def _ascii_reader(cls, fh, header):
        lines = b(header).split(b('\n'))
        recoverable = [True]

        def get(prefix=''):
            prefix = b(prefix)

            if lines:
                line = lines.pop(0)
            else:
                raise RuntimeError(recoverable[0], 'Unable to find more lines')
            if not lines:
                recoverable[0] = False

                # Read more lines and make sure we prepend any old data
                lines[:] = b(fh.read(BUFFER_SIZE)).split(b('\n'))
                line += lines.pop(0)

            line = line.lower().strip()
            if prefix:
                if line.startswith(prefix):
                    values = line.replace(prefix, b(''), 1).strip().split()
                elif line.startswith(b('endsolid')):
                    raise StopIteration()
                else:
                    raise RuntimeError(recoverable[0],
                                       '%r should start with %r' % (line,
                                                                    prefix))

                if len(values) == 3:
                    return [float(v) for v in values]
                else:  # pragma: no cover
                    raise RuntimeError(recoverable[0],
                                       'Incorrect value %r' % line)
            else:
                return b(line)

        line = get()
        if not line.startswith(b('solid ')) and \
                line.startswith(b('solid')):
            cls.warning('ASCII STL files should start with solid <space>. '
                        'The application that produced this STL file may be '
                        'faulty, please report this error. The erroneous '
                        'line: %r', line)

        if not lines:
            raise RuntimeError(recoverable[0],
                               'No lines found, impossible to read')

        while True:
            # Read from the header lines first, until that point we can recover
            # and go to the binary option. After that we cannot due to
            # unseekable files such as sys.stdin
            #
            # Numpy doesn't support any non-file types so wrapping with a
            # buffer and/or StringIO does not work.
            try:
                normals = get('facet normal')
                assert get() == b('outer loop')
                v0 = get('vertex')
                v1 = get('vertex')
                v2 = get('vertex')
                assert get() == b('endloop')
                assert get() == b('endfacet')
                attrs = 0
                yield (normals, (v0, v1, v2), attrs)
            except AssertionError as e:
                raise RuntimeError(recoverable[0], e)
            except StopIteration:
                raise

    @classmethod
    def _load_ascii(cls, fh, header):
        return numpy.fromiter(cls._ascii_reader(fh, header), dtype=cls.dtype)

    def save(self, filename, fh=None, mode=AUTOMATIC, update_normals=True):
        '''Save the STL to a (binary) file

        If mode is :py:data:`AUTOMATIC` an :py:data:`ASCII` file will be
        written if the output is a TTY and a :py:data:`BINARY` file otherwise.

        :param str filename: The file to load
        :param file fh: The file handle to open
        :param int mode: The mode to write, default is :py:data:`AUTOMATIC`.
        :param bool update_normals: Whether to update the normals
        '''
        assert filename, 'Filename is required for the STL headers'
        if update_normals:
            self.update_normals()

        if mode is AUTOMATIC:
            if fh and os.isatty(fh.fileno()):  # pragma: no cover
                write = self._write_ascii
            else:
                write = self._write_binary
        elif mode is BINARY:
            write = self._write_binary
        elif mode is ASCII:
            write = self._write_ascii
        else:
            raise ValueError('Mode %r is invalid' % mode)

        name = os.path.split(filename)[-1]
        try:
            if fh:
                write(fh, name)
            else:
                with open(name, 'wb') as fh:
                    write(fh, filename)
        except IOError:  # pragma: no cover
            pass

    def _write_ascii(self, fh, name):
        print(fh)
        print(type(fh))

        def p(s, file):
            file.write(b('%s\n' % s))

        p('solid %s' % name, file=fh)

        for row in self.data:
            vectors = row['vectors']
            p('facet normal %f %f %f' % tuple(row['normals']), file=fh)
            p('  outer loop', file=fh)
            p('    vertex %f %f %f' % tuple(vectors[0]), file=fh)
            p('    vertex %f %f %f' % tuple(vectors[1]), file=fh)
            p('    vertex %f %f %f' % tuple(vectors[2]), file=fh)
            p('  endloop', file=fh)
            p('endfacet', file=fh)

        p('endsolid %s' % name, file=fh)

    def _write_binary(self, fh, name):
        # Create the header
        header = '%s (%s) %s %s' % (
            metadata.__package_name__,
            metadata.__version__,
            datetime.datetime.now(),
            name,
        )

        # Make it exactly 80 characters
        header = header[:80].ljust(80, ' ')
        packed = struct.pack('@i', self.data.size)

        if isinstance(fh, io.TextIOWrapper):  # pragma: no cover
            packed = str(packed)
        else:
            header = b(header)
            packed = b(packed)

        fh.write(header)
        fh.write(packed)
        self.data.tofile(fh)

    @classmethod
    def from_file(cls, filename, calculate_normals=True, fh=None,
                  mode=AUTOMATIC, **kwargs):
        '''Load a mesh from a STL file

        :param str filename: The file to load
        :param bool calculate_normals: Whether to update the normals
        :param file fh: The file handle to open
        :param dict \**kwargs: The same as for :py:class:`stl.mesh.Mesh`

        '''
        if fh:
            name, data = cls.load(fh, mode=mode)
        else:
            with open(filename, 'rb') as fh:
                name, data = cls.load(fh, mode=mode)

        return cls(data, calculate_normals, name=name, **kwargs)

    @classmethod
    def from_multi_file(cls, filename, calculate_normals=True, fh=None,
                        mode=ASCII, **kwargs):
        '''Load multiple meshes from a STL file

        :param str filename: The file to load
        :param bool calculate_normals: Whether to update the normals
        :param file fh: The file handle to open
        :param dict \**kwargs: The same as for :py:class:`stl.mesh.Mesh`
        '''
        if fh:
            close = False
        else:
            fh = open(filename, 'rb')
            close = True

        try:
            raw_data = cls.load(fh, mode=mode)
            while raw_data:
                name, data = raw_data
                yield cls(data, calculate_normals, name=name, **kwargs)
                raw_data = cls.load(fh, mode=mode)

        finally:
            if close:
                fh.close()

StlMesh = BaseStl.from_file

<|endoftext|>


<commit_before>"""
Data storage classes for Twitter Most Followed.
It should also perform aggregation for "most followed".
"""

class Storage(object):
	"Base storage class doin nothin."

	# Basic user functions
	def set_user_data(self, user_id, user_data):
		"Store user_data dictionary."
		print "Set user data:\t%s\n%s" % (user_id, user_data)

	# def get_user_data(self, user_id):
	# 	"Return user_id's data."
	# 	return None

	def is_protected(self, user_id):
		return False

	def mark_protected(self, user_id):
		print "Mark protected:\t%s" % user_id

	# Followers
	def set_followers(self, user_id, followers):
		"Store user_id's followers list."
		print "Set followers:\t%s\t%s" % (user_id, len(followers))

	def get_followers(self, user_id):
		"Return user_id's followers list."
		return None

	# Friends (aka following, or better yet leaders)
	def has_friends(self, user_id):
		"Return whether user_id's friends are stored."
		return False

	def set_friends(self, user_id, friends):
		"Store user_id's friends list."
		print "Set friends:\t%s\t%s" % (user_id, len(friends))

	def get_friends(self, user_id):
		"Return user_id's friends list."
		return None

	def most_followed(self):
		"""
		Aggregate most followed users from the store.
		Return top 5000.
		"""
		return None

class RedisStorage(Storage):
	"Redis storage "

	import redis
	r = redis.StrictRedis()

	def set_user_data(self, user_id, user_data):
		self.r.hmset('user_data:%s' % user_id, user_data)
		super(RedisStorage, self).set_user_data(user_id, user_data)

	def is_protected(self, user_id):
		return self.r.sismember('protected_users', user_id)

	def mark_protected(self, user_id):
		self.r.sadd('protected_users', user_id)
		super(RedisStorage, self).mark_protected(user_id)

	# Followers
	def set_followers(self, user_id, followers):
		"Store user_id's followers list."
		self.r.delete('user_followers:%s' % user_id)
		self.r.sadd('user_followers:%s' % user_id, *followers)
		super(RedisStorage, self).set_followers(user_id, followers)

	def get_followers(self, user_id):
		"Return user_id's followers list."
		return self.r.smembers('user_followers:%s' % user_id)

	# Friends (aka following, or better yet leaders)
	def has_friends(self, user_id):
		"Return whether user_id's friends are stored."
		return self.r.exists('user_friends:%s' % user_id)

	def set_friends(self, user_id, friends):
		"Store user_id's friends list."
		self.r.delete('user_friends:%s' % user_id)
		self.r.sadd('user_friends:%s' % user_id, *friends)
		super(RedisStorage, self).set_friends(user_id, friends)

	def get_friends(self, user_id):
		"Return user_id's friends list."
		return self.r.smembers('user_friends:%s' % user_id)

	def most_followed(self):
		keys = r.keys('user_friends:*')
		r.zunionstore('most_followed', keys)
		return r.zrevrange('most_followed', 0, 4999, withscores=True)

class FileStorage(Storage):
	pass

<commit_msg>Get/set most followed, and IoC for RedisStorage<commit_after>"""
Data storage classes for Twitter Most Followed.
It should also perform aggregation for "most followed".
"""

class Storage(object):
	"Base storage class doin nothin."

	# Basic user functions
	def set_user_data(self, user_id, user_data):
		"Store user_data dictionary."
		print "Set user data:\t%s\n%s" % (user_id, user_data)

	def get_user_data(self, user_id):
		"Return user_id's data."
		return None

	def is_protected(self, user_id):
		return False

	def mark_protected(self, user_id):
		print "Mark protected:\t%s" % user_id

	# Followers
	def set_followers(self, user_id, followers):
		"Store user_id's followers list."
		print "Set followers:\t%s\t%s" % (user_id, len(followers))

	def get_followers(self, user_id):
		"Return user_id's followers list."
		return None

	# Friends (aka following, or better yet leaders)
	def has_friends(self, user_id):
		"Return whether user_id's friends are stored."
		return False

	def set_friends(self, user_id, friends):
		"Store user_id's friends list."
		print "Set friends:\t%s\t%s" % (user_id, len(friends))

	def get_friends(self, user_id):
		"Return user_id's friends list."
		return None

	# Most Followed
	def set_most_followed(self):
		"Aggregate top most followed users from the store."
		print "Aggregate top most followed."

	def get_most_followed(self, count):
		"Return top `count` most followed."
		return None

class RedisStorage(Storage):
	"Redis storage "

	def __init__(self, r):
		"Pass in the redis object."
		self.r = r

	def set_user_data(self, user_id, user_data):
		"Store user_data dictionary."
		self.r.hmset('user_data:%s' % user_id, user_data)
		# super(RedisStorage, self).set_user_data(user_id, user_data)

	def get_user_data(self, user_id):
		"Return user_id's data."
		return self.r.hgetall('user_data:%s' % user_id)

	def is_protected(self, user_id):
		return self.r.sismember('protected_users', user_id)

	def mark_protected(self, user_id):
		self.r.sadd('protected_users', user_id)
		super(RedisStorage, self).mark_protected(user_id)

	# Followers
	def set_followers(self, user_id, followers):
		"Store user_id's followers list."
		self.r.delete('user_followers:%s' % user_id)
		self.r.sadd('user_followers:%s' % user_id, *followers)
		super(RedisStorage, self).set_followers(user_id, followers)

	def get_followers(self, user_id):
		"Return user_id's followers list."
		return self.r.smembers('user_followers:%s' % user_id)

	# Friends (aka following, or better yet leaders)
	def has_friends(self, user_id):
		"Return whether user_id's friends are stored."
		return self.r.exists('user_friends:%s' % user_id)

	def set_friends(self, user_id, friends):
		"Store user_id's friends list."
		self.r.delete('user_friends:%s' % user_id)
		self.r.sadd('user_friends:%s' % user_id, *friends)
		super(RedisStorage, self).set_friends(user_id, friends)

	def get_friends(self, user_id):
		"Return user_id's friends list."
		return self.r.smembers('user_friends:%s' % user_id)

	# Most Followed
	def set_most_followed(self):
		"Aggregate top most followed users from the store."
		keys = self.r.keys('user_friends:*')
		self.r.zunionstore('most_followed', keys)
		super(RedisStorage, self).set_most_followed()

	def get_most_followed(self, count):
		"Return top `count` most followed."
		return self.r.zrevrange('most_followed', 0, count-1, withscores=True)

class SQLiteStorage(Storage):
	pass

<|endoftext|>


<commit_before>import os.path
import json
import re
import logging
import urllib
import urllib2
import errno
from contextlib import contextmanager
import flask
from bson.objectid import ObjectId
import schema

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

class StorageError(Exception):
    pass

QUERY_ROWS = 1000

class FsStorage(object):

    def __init__(self, storage_path):
        self.storage_path = storage_path

    def _doc_path(self, doc_id):
        doc_id = int(doc_id)
        return os.path.join(self.storage_path, 'doc_%d.json' % doc_id)

    def save_document(self, doc_id, doc):
        data = doc.value
        if doc_id is None:
            doc_id = max([-1] + self.document_ids()) + 1
        else:
            doc_id = int(doc_id)
        log.info("saving document %r", doc_id)
        with open(self._doc_path(doc_id), 'wb') as f:
            json.dump(data, f, indent=2)
        return doc_id

    def load_document(self, doc_id):
        doc_id = int(doc_id)
        with open(self._doc_path(doc_id), 'rb') as f:
            return schema.SpaDoc(json.load(f))

    def document_ids(self):
        doc_id_list = []
        for name in os.listdir(self.storage_path):
            m = re.match(r'^doc_(?P<doc_id>\d+)\.json$', name)
            if m is None:
                continue

            doc_id_list.append(int(m.group('doc_id')))

        doc_id_list.sort()
        return doc_id_list

class MongoStorage(object):

    def __init__(self, db_name):
        self.connection = pymongo.Connection('localhost', 27017)
        self.db = self.connection[db_name]

    def save_document(self, doc_id, doc):
        data = doc.value
        if doc_id is not None:
            data = dict(data, _id=ObjectId(doc_id))
            log.info("saving new document %r")
        else:
            log.info("saving document %r", doc_id)

        doc_id = self.db['spadoc'].insert(data)
        return doc_id

    def load_document(self, doc_id):
        doc = self.db['spadoc'].find_one({'_id': ObjectId(doc_id)})
        if doc is None:
            return schema.SpaDoc()
        del doc['_id']
        return schema.SpaDoc(doc)

    def document_ids(self):
        doc_id_list = [doc['_id'] for doc in self.db['spadoc'].find()]
        doc_id_list.sort()
        return doc_id_list

try:
    import pymongo
except ImportError:
    MongoStorage = None

class SolrStorage(object):

    orig_field_name = 'orig_s'
    solr_base_url = 'http://localhost:8983/solr/'

    def _solr_doc(self, doc):
        data = doc.value
        full_text = (
            data['section1']['site_name'] or '' +
            data['section4']['quality'] or '' +
            data['section4']['vulnar'] or '' +
            data['section4']['docum'] or '')

        return {
            'id': data['section1']['sitecode'],
            'title': data['section1']['site_name'],
            'type_txt': data['section1']['type'],
            'text': full_text,
            self.orig_field_name: json.dumps(data),
        }

    @contextmanager
    def solr_http(self, request):
        if isinstance(request, urllib2.Request):
            url = request.get_full_url()
        else:
            url = request

        log.debug("Solr request to url %r", url)

        try:
            response = urllib2.urlopen(request)
        except urllib2.URLError, e:
            if e.reason.errno == errno.ECONNREFUSED:
                raise StorageError("Could not connect to Solr", e)
            else:
                raise

        try:
            yield response
        finally:
            response.close()

    def solr_query(self, query):
        url = self.solr_base_url + 'select?q=%s&wt=json&rows=%d' % (
                query, QUERY_ROWS)
        with self.solr_http(url) as http_response:
            answer = json.load(http_response)

        num_found = answer['response']['numFound']
        if num_found > QUERY_ROWS:
            log.warn("Found more results than expected: %d > %d",
                     num_found, QUERY_ROWS)

        return answer['response']['docs']

    def save_document(self, doc_id, doc):
        return self.save_document_batch([doc])[0]

    def save_document_batch(self, batch):
        url = self.solr_base_url + 'update/json?commit=true'
        request = urllib2.Request(url)
        request.add_header('Content-Type', 'application/json')
        request.add_data(json.dumps([self._solr_doc(doc) for doc in batch]))

        with self.solr_http(request) as response:
            response.read()

        return [doc['section1']['sitecode'].value for doc in batch]

    def load_document(self, doc_id):
        doc = self.solr_query('id:%s' % doc_id)[0]
        return schema.SpaDoc(json.loads(doc[self.orig_field_name]))

    def document_ids(self):
        return sorted([d['id'] for d in self.solr_query('*')])

def get_db(app=None):
    if app is None:
        app = flask.current_app
    config = app.config
    engine_name = config['STORAGE_ENGINE']

    if engine_name == 'solr':
        return SolrStorage()

    elif engine_name == 'filesystem':
        return FsStorage(config['STORAGE_FS_PATH'])

    else:
        raise ValueError('Unknown storage engine %r' % engine_name)
<commit_msg>send index values to Solr<commit_after>import os.path
import json
import re
import logging
import urllib
import urllib2
import errno
from contextlib import contextmanager
import flask
from bson.objectid import ObjectId
import schema

log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)

class StorageError(Exception):
    pass

QUERY_ROWS = 1000

class FsStorage(object):

    def __init__(self, storage_path):
        self.storage_path = storage_path

    def _doc_path(self, doc_id):
        doc_id = int(doc_id)
        return os.path.join(self.storage_path, 'doc_%d.json' % doc_id)

    def save_document(self, doc_id, doc):
        data = doc.value
        if doc_id is None:
            doc_id = max([-1] + self.document_ids()) + 1
        else:
            doc_id = int(doc_id)
        log.info("saving document %r", doc_id)
        with open(self._doc_path(doc_id), 'wb') as f:
            json.dump(data, f, indent=2)
        return doc_id

    def load_document(self, doc_id):
        doc_id = int(doc_id)
        with open(self._doc_path(doc_id), 'rb') as f:
            return schema.SpaDoc(json.load(f))

    def document_ids(self):
        doc_id_list = []
        for name in os.listdir(self.storage_path):
            m = re.match(r'^doc_(?P<doc_id>\d+)\.json$', name)
            if m is None:
                continue

            doc_id_list.append(int(m.group('doc_id')))

        doc_id_list.sort()
        return doc_id_list

class MongoStorage(object):

    def __init__(self, db_name):
        self.connection = pymongo.Connection('localhost', 27017)
        self.db = self.connection[db_name]

    def save_document(self, doc_id, doc):
        data = doc.value
        if doc_id is not None:
            data = dict(data, _id=ObjectId(doc_id))
            log.info("saving new document %r")
        else:
            log.info("saving document %r", doc_id)

        doc_id = self.db['spadoc'].insert(data)
        return doc_id

    def load_document(self, doc_id):
        doc = self.db['spadoc'].find_one({'_id': ObjectId(doc_id)})
        if doc is None:
            return schema.SpaDoc()
        del doc['_id']
        return schema.SpaDoc(doc)

    def document_ids(self):
        doc_id_list = [doc['_id'] for doc in self.db['spadoc'].find()]
        doc_id_list.sort()
        return doc_id_list

try:
    import pymongo
except ImportError:
    MongoStorage = None

class SolrStorage(object):

    orig_field_name = 'orig'
    solr_base_url = 'http://localhost:8983/solr/'

    def _solr_doc(self, doc):
        data = doc.value

        solr_doc = {
            'id': data['section1']['sitecode'],
            self.orig_field_name: json.dumps(data),
        }

        for element in schema.Search().all_children:
            value = element.properties['index'](doc)
            log.debug('index %s: %r', element.name, value)
            solr_doc[element.name] = value

        return solr_doc

    @contextmanager
    def solr_http(self, request):
        if isinstance(request, urllib2.Request):
            url = request.get_full_url()
        else:
            url = request

        log.debug("Solr request to url %r", url)

        try:
            response = urllib2.urlopen(request)
        except urllib2.URLError, e:
            if e.reason.errno == errno.ECONNREFUSED:
                raise StorageError("Could not connect to Solr", e)
            else:
                raise

        try:
            yield response
        finally:
            response.close()

    def solr_query(self, query):
        url = self.solr_base_url + 'select?q=%s&wt=json&rows=%d' % (
                query, QUERY_ROWS)
        with self.solr_http(url) as http_response:
            answer = json.load(http_response)

        num_found = answer['response']['numFound']
        if num_found > QUERY_ROWS:
            log.warn("Found more results than expected: %d > %d",
                     num_found, QUERY_ROWS)

        return answer['response']['docs']

    def save_document(self, doc_id, doc):
        return self.save_document_batch([doc])[0]

    def save_document_batch(self, batch):
        url = self.solr_base_url + 'update/json?commit=true'
        request = urllib2.Request(url)
        request.add_header('Content-Type', 'application/json')
        request.add_data(json.dumps([self._solr_doc(doc) for doc in batch]))

        with self.solr_http(request) as response:
            response.read()

        return [doc['section1']['sitecode'].value for doc in batch]

    def load_document(self, doc_id):
        doc = self.solr_query('id:%s' % doc_id)[0]
        return schema.SpaDoc(json.loads(doc[self.orig_field_name]))

    def document_ids(self):
        return sorted([d['id'] for d in self.solr_query('*')])

def get_db(app=None):
    if app is None:
        app = flask.current_app
    config = app.config
    engine_name = config['STORAGE_ENGINE']

    if engine_name == 'solr':
        return SolrStorage()

    elif engine_name == 'filesystem':
        return FsStorage(config['STORAGE_FS_PATH'])

    else:
        raise ValueError('Unknown storage engine %r' % engine_name)
<|endoftext|>


<commit_before>import os

class Streams():
    def __init__(self):
        ''' Constructor. '''
        self.root = "/home/pi/Music"
        self.predefinedStreams = [
            {
                "name": "Radio ZET",
                "stream": "http://n-1-11.dcs.redcdn.pl/sc/o2/Eurozet/live/audio.livx",
                "isRadio": True
            },
            {
                "name": "Polskie Radio 24",
                "stream": "http://stream3.polskieradio.pl:8080",
                "isRadio": True
            },
            {
                "name": "ZET Gold",
                "stream": "http://zgl01.cdn.eurozet.pl:8506/ZGLHIT.mp3",
                "isRadio": True
            },
            {
                "name": "Meloradio",
                "stream": "http://mel02.cdn.eurozet.pl:8800/mel-net.mp3",
                "isRadio": True
            },
            {
                "name": "Elton John Radio",
                "stream": "http://streaming.exclusive.radio/er/eltonjohn/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Madonna Radio",
                "stream": "http://streaming.exclusive.radio/er/madonna/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Elvis Presley Radio",
                "stream": "http://streaming.exclusive.radio/er/elvispresley/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Paul McCartney Radio",
                "stream": "http://streaming.exclusive.radio/er/paulmccartney/icecast.audio",
                "isRadio": True
            },
            {
                "name": "George Michael Radio",
                "stream": "http://streaming.exclusive.radio/er/georgemichael/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Abba Radio",
                "stream": "http://streaming.exclusive.radio/er/abba/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Coldplay Radio",
                "stream": "http://streaming.exclusive.radio/er/coldplay/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Ed Sheeran Radio",
                "stream": "http://streaming.exclusive.radio/er/edsheeran/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Johnny Cash Radio",
                "stream": "http://streaming.exclusive.radio/er/johnnycash/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Phil Collins Radio",
                "stream": "http://streaming.exclusive.radio/er/philcollins/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Scorpions Radio",
                "stream": "http://streaming.exclusive.radio/er/scorpions/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Shakira Radio",
                "stream": "http://streaming.exclusive.radio/er/shakira/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Whitney Houston Radio",
                "stream": "http://streaming.exclusive.radio/er/whitneyhouston/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Country Radio",
                "stream": "http://icepool.silvacast.com/COUNTRY108.mp3",
                "isRadio": True
            },
            {
                "name": "Cinemix",
                "stream": "http://94.23.51.96:8001/;",
                "isRadio": True
            },
            {
                "name": "Soundtracks",
                "stream": "http://hi5.streamingsoundtracks.com/;",
                "isRadio": True
            },
            {
                "name": "James Bond 007",
                "stream": "http://stream.laut.fm/007",
                "isRadio": True
            },
            {
                "name": "Radio Mambo",
                "stream": "http://178.32.139.184:8060/;",
                "isRadio": True
            },
            {
                "name": "Salsa Warriors",
                "stream": "http://192.99.17.12:6031/;stream/1",
                "isRadio": True
            },
            {
                "name": "Salsa AMS",
                "stream": "http://82.94.166.107:8067/;stream/1",
                "isRadio": True
            },
            {
                "name": "Fox News",
                "stream": "http://streaming-ent.shoutcast.com/foxnews",
                "isRadio": True
            },
            {
                "name": "RFI Monde",
                "stream": "http://live02.rfi.fr/rfimonde-96k.mp3",
                "isRadio": True
            },
            {
                "name": "BBC 1",
                "stream": "http://bbcmedia.ic.llnwd.net/stream/bbcmedia_radio1_mf_p",
                "isRadio": True
            },
            {
                "name": "BBC 2",
                "stream": "http://bbcmedia.ic.llnwd.net/stream/bbcmedia_radio2_mf_p",
                "isRadio": True
            },
            # {
            #     "name": "Christmas 1",
            #     "stream": "http://node-05.zeno.fm/aae9yc3ygnruv?rj-ttl=5&rj-tok=AAABdo8S4DcAoTqG00pxZjP_MQ",
            #     "isRadio": True
            # },
            # {
            #     "name": "Christmas 2",
            #     "stream": "http://str2b.openstream.co/1312?aw_0_1st.collectionid=4427&stationId=4427&publisherId=1336&k=1608717935",
            #     "isRadio": True
            # },
            # {
            #     "name": "Christmas 3",
            #     "stream": "http://tuner.m1.fm/M1-XMAS.mp3",
            #     "isRadio": True
            # },
            # {
            #     "name": "Koledy 1",
            #     "stream": "http://31.192.216.7/KOLEDY",
            #     "isRadio": True
            # },
            # {
            #     "name": "Koledy 2",
            #     "stream": "http://zt04.cdn.eurozet.pl/ZETKOL.mp3",
            #     "isRadio": True
            # }
        ]

    def get(self):
        for root, dirs, files in os.walk(self.root):
            if len(dirs):
                extraStreams = []
                for dir in dirs:
                    extraStreams.append({"name": dir, "stream": self.root + "/" + dir, "isRadio": False})
                allStreams = extraStreams + self.predefinedStreams
                allStreams.sort(key=lambda stream: (stream["isRadio"], stream["name"]), reverse=False)
                return allStreams
            else:
                return self.predefinedStreams
<commit_msg>more radios<commit_after>import os

class Streams():
    def __init__(self):
        ''' Constructor. '''
        self.root = "/home/pi/Music"
        self.predefinedStreams = [
            {
                "name": "Radio ZET",
                "stream": "http://n-1-11.dcs.redcdn.pl/sc/o2/Eurozet/live/audio.livx",
                "isRadio": True
            },
            {
                "name": "Polskie Radio 24",
                "stream": "http://stream3.polskieradio.pl:8080",
                "isRadio": True
            },
            {
                "name": "ZET Gold",
                "stream": "http://zgl01.cdn.eurozet.pl:8506/ZGLHIT.mp3",
                "isRadio": True
            },
            {
                "name": "Meloradio",
                "stream": "http://mel02.cdn.eurozet.pl:8800/mel-net.mp3",
                "isRadio": True
            },
            {
                "name": "Elton John Radio",
                "stream": "http://streaming.exclusive.radio/er/eltonjohn/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Madonna Radio",
                "stream": "http://streaming.exclusive.radio/er/madonna/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Elvis Presley Radio",
                "stream": "http://streaming.exclusive.radio/er/elvispresley/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Paul McCartney Radio",
                "stream": "http://streaming.exclusive.radio/er/paulmccartney/icecast.audio",
                "isRadio": True
            },
            {
                "name": "George Michael Radio",
                "stream": "http://streaming.exclusive.radio/er/georgemichael/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Abba Radio",
                "stream": "http://streaming.exclusive.radio/er/abba/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Coldplay Radio",
                "stream": "http://streaming.exclusive.radio/er/coldplay/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Ed Sheeran Radio",
                "stream": "http://streaming.exclusive.radio/er/edsheeran/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Johnny Cash Radio",
                "stream": "http://streaming.exclusive.radio/er/johnnycash/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Phil Collins Radio",
                "stream": "http://streaming.exclusive.radio/er/philcollins/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Scorpions Radio",
                "stream": "http://streaming.exclusive.radio/er/scorpions/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Beach Boys",
                "stream": "http://streaming.exclusive.radio/er/beachboys/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Gloria Estefan",
                "stream": "http://streaming.exclusive.radio/er/gloriaestefan/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Shakira Radio",
                "stream": "http://streaming.exclusive.radio/er/shakira/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Whitney Houston Radio",
                "stream": "http://streaming.exclusive.radio/er/whitneyhouston/icecast.audio",
                "isRadio": True
            },
            {
                "name": "Country Radio",
                "stream": "http://icepool.silvacast.com/COUNTRY108.mp3",
                "isRadio": True
            },
            {
                "name": "Cinemix",
                "stream": "http://94.23.51.96:8001/;",
                "isRadio": True
            },
            {
                "name": "Soundtracks",
                "stream": "http://hi5.streamingsoundtracks.com/;",
                "isRadio": True
            },
            {
                "name": "James Bond 007",
                "stream": "http://stream.laut.fm/007",
                "isRadio": True
            },
            {
                "name": "Radio Mambo",
                "stream": "http://178.32.139.184:8060/;",
                "isRadio": True
            },
            {
                "name": "Salsa Warriors",
                "stream": "http://192.99.17.12:6031/;stream/1",
                "isRadio": True
            },
            {
                "name": "Salsa AMS",
                "stream": "http://82.94.166.107:8067/;stream/1",
                "isRadio": True
            },
            {
                "name": "Fox News",
                "stream": "http://streaming-ent.shoutcast.com/foxnews",
                "isRadio": True
            },
            {
                "name": "RFI Monde",
                "stream": "http://live02.rfi.fr/rfimonde-96k.mp3",
                "isRadio": True
            },
            {
                "name": "BBC 1",
                "stream": "http://bbcmedia.ic.llnwd.net/stream/bbcmedia_radio1_mf_p",
                "isRadio": True
            },
            {
                "name": "BBC 2",
                "stream": "http://bbcmedia.ic.llnwd.net/stream/bbcmedia_radio2_mf_p",
                "isRadio": True
            },
            # {
            #     "name": "Christmas 1",
            #     "stream": "http://node-05.zeno.fm/aae9yc3ygnruv?rj-ttl=5&rj-tok=AAABdo8S4DcAoTqG00pxZjP_MQ",
            #     "isRadio": True
            # },
            # {
            #     "name": "Christmas 2",
            #     "stream": "http://str2b.openstream.co/1312?aw_0_1st.collectionid=4427&stationId=4427&publisherId=1336&k=1608717935",
            #     "isRadio": True
            # },
            # {
            #     "name": "Christmas 3",
            #     "stream": "http://tuner.m1.fm/M1-XMAS.mp3",
            #     "isRadio": True
            # },
            # {
            #     "name": "Koledy 1",
            #     "stream": "http://31.192.216.7/KOLEDY",
            #     "isRadio": True
            # },
            # {
            #     "name": "Koledy 2",
            #     "stream": "http://zt04.cdn.eurozet.pl/ZETKOL.mp3",
            #     "isRadio": True
            # }
        ]

    def get(self):
        for root, dirs, files in os.walk(self.root):
            if len(dirs):
                extraStreams = []
                for dir in dirs:
                    extraStreams.append({"name": dir, "stream": self.root + "/" + dir, "isRadio": False})
                allStreams = extraStreams + self.predefinedStreams
                allStreams.sort(key=lambda stream: (stream["isRadio"], stream["name"]), reverse=False)
                return allStreams
            else:
                return self.predefinedStreams
<|endoftext|>


<commit_before>import os

import json

from twisted.internet import reactor
from twisted.web.client import getPage, downloadPage
from twisted.internet.defer import DeferredList

class Submission(object):
    def __init__(self, child, subreddit):
        self.data = child['data']
        self.subreddit = self.data['subreddit']

    @property
    def directory(self):
        """Returns the directory without the filename"""
        return self.subreddit

    @property
    def url(self):
        """Returns the URL"""
        return self.data['url']

    @property
    def file_name(self):
        """Returns file name without path"""
        return self.url.split('/')[-1]

    @property
    def ext(self):
        """Returns file extension"""
        return self.file_name.split('.')[-1]

    @property
    def file_path(self):
        """Returns the complete path and filename"""
        return '%s/%s' % (self.directory, self.file_name)

    def checkCreateDir(self):
        """Checks if the directory exists, and if not, creates it"""
        if not os.path.exists(self.directory):
            os.makedirs(self.directory)

    def updateModifiedTime(self):
        """Changes the file date on a downloaded file to match that of the
        submission
        """
        os.utime(self.file_path, (0, self.data['created']))

    def process(self):
        """Returns a deferred(?) that will either:
            1. Download an image directly.
            2. Scrape the page of a link (ie a tumblr).
            3. Does nothing because it's not the kind of link we want.
        """

        def fileDateCallback(result):
            """wraps update modified time"""
            self.updateModifiedTime()

        def checkFileSize(result):
            """Fail on files lower than a set size"""
            print "checking file size", self.file_path
            if os.path.getsize(self.file_path) < 20*1024:
                raise TooSmallError

        def deleteFile(failure):
            """Delete the resulting file."""
            failure.trap(TooSmallError)
            print "deleting file", self.file_path
            os.remove(self.file_path)

        def writeError(failure):
            """If there's a problem writing the file for instance if the
            directory does not exist.
            """

            failure.trap(IOError)
            print failure.getErrorMessage()

        def finishChild(result):
            print "Finished child", self.subreddit, self.file_path, self.url

        if os.path.exists(self.file_path):
            raise FileExistsError

        self.checkCreateDir()

        if self.ext in ['jpg', 'png', 'gif']:
            d = downloadPage(str(self.url), self.file_path)
        else:
            raise UnknownLinkError(self.url)

        d.addCallback(checkFileSize)
        d.addCallbacks(fileDateCallback, deleteFile)
        d.addErrback(writeError)
        d.addBoth(finishChild)
        print "trying to download", str(self.url), self.file_path
        self.d = d
        return d

class UnknownLinkError(Exception):
    """This error is thrown when the program does not know what to do with a
    link.
    """
    pass

class FileExistsError(Exception):
    """This is an exception raised when the file that would be downloaded from
    a submission is already on the disk.
    """
    pass

class TooSmallError(Exception):
    """Raised if the file is under 20Kb (so probably not a useful image."""
    pass

def subredditCallback(page, subreddit, max_count, count=1, after=None):
    """Load a page of a subreddit, download links, and recursively call self
    on next page for as many pages as required."""
    data = json.loads(page)['data']

    previous_after = None  # no idea how to get this either

    dlist = []
    if data['children']:
        for child in data['children']:
            try:
                submission = Submission(child, subreddit)
                d = submission.process()
                dlist.append(d)
                print "Added ", submission.url
            except UnknownLinkError:
                print "Didn't know how to handle link.", submission.url
            except FileExistsError:
                submission.updateModifiedTime()
                print "File already exists.", submission.file_path

    if data['after'] != previous_after and count < max_count:
        new_count = count + 1

        d = getPage(str('http://www.reddit.com/r/%s/.json?count=%s&after=%s'
            % (subreddit, new_count, data['after'])))
        d.addCallback(subredditCallback,
            subreddit, max_count, new_count, data['after'])
        dlist.append(d)

    return DeferredList(dlist)

def finish(ign):
    print "Reached the finish!"
    reactor.stop()

def main():
    max_count = 10

    subreddits = [
        'bondage',
        'shinyporn',
        'wallpapers'
    ]

    dlist = []
    for subreddit in subreddits:
        d = getPage('http://www.reddit.com/r/%s/.json' % subreddit)
        d.addCallback(subredditCallback, subreddit, max_count)
        dlist.append(d)

    d = DeferredList(dlist)
    d.addCallback(finish)

if __name__ == '__main__':
    main()
    reactor.run()
<commit_msg>handle command line<commit_after>import os, sys

import json

from twisted.internet import reactor
from twisted.web.client import getPage, downloadPage
from twisted.internet.defer import DeferredList

class Submission(object):
    def __init__(self, child, subreddit):
        self.data = child['data']
        self.subreddit = self.data['subreddit']

    @property
    def directory(self):
        """Returns the directory without the filename"""
        return self.subreddit

    @property
    def url(self):
        """Returns the URL"""
        return self.data['url']

    @property
    def file_name(self):
        """Returns file name without path"""
        return self.url.split('/')[-1]

    @property
    def ext(self):
        """Returns file extension"""
        return self.file_name.split('.')[-1]

    @property
    def file_path(self):
        """Returns the complete path and filename"""
        return '%s/%s' % (self.directory, self.file_name)

    def checkCreateDir(self):
        """Checks if the directory exists, and if not, creates it"""
        if not os.path.exists(self.directory):
            os.makedirs(self.directory)

    def updateModifiedTime(self):
        """Changes the file date on a downloaded file to match that of the
        submission
        """
        os.utime(self.file_path, (0, self.data['created']))

    def process(self):
        """Returns a deferred(?) that will either:
            1. Download an image directly.
            2. Scrape the page of a link (ie a tumblr).
            3. Does nothing because it's not the kind of link we want.
        """

        def fileDateCallback(result):
            """wraps update modified time"""
            self.updateModifiedTime()

        def checkFileSize(result):
            """Fail on files lower than a set size"""
            print "checking file size", self.file_path
            if os.path.getsize(self.file_path) < 20*1024:
                raise TooSmallError

        def deleteFile(failure):
            """Delete the resulting file."""
            failure.trap(TooSmallError)
            print "deleting file", self.file_path
            os.remove(self.file_path)

        def writeError(failure):
            """If there's a problem writing the file for instance if the
            directory does not exist.
            """

            failure.trap(IOError)
            print failure.getErrorMessage()

        def finishChild(result):
            print "Finished child", self.subreddit, self.file_path, self.url

        if os.path.exists(self.file_path):
            raise FileExistsError

        self.checkCreateDir()

        if self.ext in ['jpg', 'png', 'gif']:
            d = downloadPage(str(self.url), self.file_path)
        else:
            raise UnknownLinkError(self.url)

        d.addCallback(checkFileSize)
        d.addCallbacks(fileDateCallback, deleteFile)
        d.addErrback(writeError)
        d.addBoth(finishChild)
        print "trying to download", str(self.url), self.file_path
        self.d = d
        return d

class UnknownLinkError(Exception):
    """This error is thrown when the program does not know what to do with a
    link.
    """
    pass

class FileExistsError(Exception):
    """This is an exception raised when the file that would be downloaded from
    a submission is already on the disk.
    """
    pass

class TooSmallError(Exception):
    """Raised if the file is under 20Kb (so probably not a useful image."""
    pass

class UsageError(Exception):
    """Raised if the command line is wrong"""
    pass

def subredditCallback(page, subreddit, max_count, count=1, after=None):
    """Load a page of a subreddit, download links, and recursively call self
    on next page for as many pages as required."""
    data = json.loads(page)['data']

    previous_after = None  # no idea how to get this either

    dlist = []
    if data['children']:
        for child in data['children']:
            try:
                submission = Submission(child, subreddit)
                d = submission.process()
                dlist.append(d)
                print "Added ", submission.url
            except UnknownLinkError:
                print "Didn't know how to handle link.", submission.url
            except FileExistsError:
                submission.updateModifiedTime()
                print "File already exists.", submission.file_path

    if data['after'] != previous_after and count < max_count:
        new_count = count + 1

        d = getPage(str('http://www.reddit.com/r/%s/.json?count=%s&after=%s'
            % (subreddit, new_count, data['after'])))
        d.addCallback(subredditCallback,
            subreddit, max_count, new_count, data['after'])
        dlist.append(d)

    return DeferredList(dlist)

def finish(ign):
    print "Reached the finish!"
    reactor.stop()

def main(subreddits, max_count):
    dlist = []
    for subreddit in subreddits:
        d = getPage('http://www.reddit.com/r/%s/.json' % subreddit)
        d.addCallback(subredditCallback, subreddit, max_count)
        dlist.append(d)

    d = DeferredList(dlist)
    d.addCallback(finish)

if __name__ == '__main__':
    try:
        subreddits = sys.argv[1].split(',')
        if subreddits[0] == '--help':
            raise UsageError()
    except (IndexError, UsageError):
        print "Usage: subdown.py <subreddit[,subreddit]> [pages]"
        exit()
    try:
        max_count = int(sys.argv[2])
    except IndexError:
        print "Pages not specified, defaulting to one."
        max_count = 1

    main(subreddits, max_count)
    reactor.run()
<|endoftext|>


<commit_before>"""
    Synthia
    ~~~~~~
    The Synthetic Intelligent Assistant for your home
"""
from flask import Flask
from playsound import playsound
import weather, requests, json, play_message

app = Flask(__name__)

# Calls a function to get a customer morning message, then plays it
def play_morning_message():
    message = get_morning_message()
    play_message.play_message(message)

# Gets a custom morning message with helpful tips to start your day
def get_morning_message():
    message = 'Good morning! '

    result = requests.get(weather.API_END_POINT);
    content = result.content
    data = json.loads(content)

    weather_data = data.get('weather')
    current_weather = weather_data[0].get('main')
    current_weather_desc = weather_data[0].get('description')

    if current_weather:
        message += 'The current weather is ' + current_weather_desc + " in Berlin. "
        current_weather_reminder_message = get_current_weather_reminder_message(current_weather)

        if current_weather_reminder_message:
            message += current_weather_reminder_message

    message += 'Have a nice day and don\'t forget your keys!'

    return message

# Returns a message based off the current weather
def get_current_weather_reminder_message(current_weather):
    return {
        weather.DRIZZLE or weather.RAINING: 'Don\'t forget your umbrella. ',
        weather.SUNNY: 'You should wear sunglasses. ',
    }.get(current_weather, '')

# Play a welcome home message
def play_welcome_home_message():
    message = get_welcome_home_message()
    play_message.play_message(message)

# Get a custom welcome home message
def get_welcome_home_message():
    return 'Welcome home.'

# Play a mp3 or m4a file
def play_song():
    audio_file = "songs/evening.m4a"
    playsound(audio_file)<commit_msg>refactored audio play code<commit_after>"""
    Synthia
    ~~~~~~
    The Synthetic Intelligent Assistant for your home
"""
from flask import Flask
import weather, requests, json, play_audio

app = Flask(__name__)

# Calls a function to get a customer morning message, then plays it
def play_morning_message():
    message = get_morning_message()
    play_audio.play_message(message)

# Gets a custom morning message with helpful tips to start your day
def get_morning_message():
    message = 'Good morning! '

    result = requests.get(weather.API_END_POINT);
    content = result.content
    data = json.loads(content)

    weather_data = data.get('weather')
    current_weather = weather_data[0].get('main')
    current_weather_desc = weather_data[0].get('description')

    if current_weather:
        message += 'The current weather is ' + current_weather_desc + " in Berlin. "
        current_weather_reminder_message = get_current_weather_reminder_message(current_weather)

        if current_weather_reminder_message:
            message += current_weather_reminder_message

    message += 'Have a nice day and don\'t forget your keys!'

    return message

# Returns a message based off the current weather
def get_current_weather_reminder_message(current_weather):
    return {
        weather.DRIZZLE or weather.RAINING: 'Don\'t forget your umbrella. ',
        weather.SUNNY: 'You should wear sunglasses. ',
    }.get(current_weather, '')

# Play a welcome home message
def play_welcome_home_message():
    message = get_welcome_home_message()
    play_audio.play_message(message)

# Get a custom welcome home message
def get_welcome_home_message():
    return 'Welcome home.'

# Play a mp3 or m4a file
def play_song():
    audio_file = "songs/evening.m4a"
    play_audio(audio_file)<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Module Contains a Function for Lexicographical Analysis"""

def lexicographics(to_analyze):
    splitsent = to_analyze.split('\n')
    splitwords = to_analyze.split()
    maximum = max(len(splitsent),len(splitwords))
    minimum = min(len(splitsent),len(splitsent))
    average = float(len(splitwords)/len(splitsent))
    print maximum, minimum, average

if __name__ == '__main__':
    lexicographics('''Don't stop believing,
    Hold on to that feeling.''')
<commit_msg>test3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Module Contains a Function for Lexicographical Analysis"""
import decimal

def lexicographics(to_analyze):
    list_of_sentences = to_analyze.split('\n')
    max_words = 0
    min_words = len(list_of_sentences[0].split())
    total_words = 0
    for sentence in list_of_sentences:
        numwords = len(sentence.split())
        if numwords > max_words:
            max_words = numwords

        if numwords <= min_words:
            min_words = numwords
        total_words += numwords
    average = total_words/decimal.Decimal(len(list_of_sentences))
    return max_words, min_words, average

if __name__ == '__main__':
    import data
    print lexicographics(data.SHAKESPEARE)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Contains expectations."""

import inquisition
<commit_msg>Wrote a string to replace instances of surprise<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Contains expectations."""

import inquisition

FISHY = inquisition.SPANISH

FISHY = FISHY.replace('surprise', 'haddock')
<|endoftext|>


<commit_before>CHARLIE = "Chocolate Factory"
VIOLET = "Gray"
PATRICIA = "Reichardt"
LINUS = "van Pelt"

<commit_msg>Fixed the file to give no failures.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Added all the required elements to this file """

CHARLIE = "Brown"
VIOLET = "Gray"
PATRICIA = "Reichardt"
LINUS = "van Pelt"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Provides Napoleon's great statement."""

NAPOLEON = "Able was I, ere I saw Elba."
<commit_msg>Reversed and lowercased<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Provides Napoleon's great statement."""

NAPOLEON = "Able was I, ere I saw Elba."

REVERSED = NAPOLEON[::-1].lower()
<|endoftext|>


<commit_before>import math
from fractions import Fraction
from itertools import count, product, chain, islice
from functools import reduce
from utils import sqrt, factorial

__all__ = ["Tchisla"]

MAX = 1 << 128
MAX_DIGITS = 128
MAX_CONCAT = 39
MAX_FACTORIAL = 34

class Tchisla:
    def __init__(self, n, target):
        self.n = n
        self.target = target
        self.solutions = {}
        self.visited = [[]]
        self.number_printed = set()

    def insert(self, x, depth, expression):
        self.solutions[x] = depth, expression
        self.visited[depth].append(x)
        if x == self.target:
            self.solution_found(self.target, True)
            return True

    def check(self, x, depth, expression):
        if x.numerator > MAX or x.denominator > MAX or x in self.solutions:
            return
        if self.insert(x, depth, expression):
            return True
        z = sqrt(x.denominator)
        if z is not None:
            y = sqrt(x.numerator)
            if y is not None and self.check(Fraction(y, z, False), depth, ("sqrt", x)):
                return True
        if x.denominator == 1 and x <= MAX_FACTORIAL:
            y = Fraction(factorial(int(x)))
            if self.check(y, depth, ("factorial", x)):
                return True

    def exponent(self, p, q, depth):
        if q.denominator != 1 or p == 1:
            return
        p_digits = math.log2(max(p.numerator, p.denominator))
        q_int = q.numerator
        exp = (("^", p, q), ("^", p, ("-", q)))
        q2 = q
        while p_digits * q_int > MAX_DIGITS:
            if q_int & 1 == 0:
                q_int >>= 1
                exp = (("sqrt", exp[0]), ("sqrt", exp[1]))
            else:
                return
        x = p ** q_int
        if self.check(x, depth, exp[0]):
            return True
        if self.check(x ** -1, depth, exp[1]):
            return True

    def binary(self, p, q, depth):
        if self.check(p + q, depth, ("+", p, q)):
            return True
        if p > q and self.check(p - q, depth, ("-", p, q)):
            return True
        elif p < q and self.check(q - p, depth, ("-", q, p)):
            return True
        if self.check(p * q, depth, ("*", p, q)):
            return True
        quotient = p / q
        if self.check(quotient, depth, ("/", p, q)):
            return True
        if self.check(quotient ** -1, depth, ("/", q, p)):
            return True
        if self.exponent(p, q, depth):
            return True
        if self.exponent(q, p, depth):
            return True

    def search(self, depth):
        self.visited.append([])
        if depth <= MAX_CONCAT:
            m = Fraction((10 ** depth - 1) // 9 * self.n)
            if self.check(m, depth, ()):
                return True
        for d1 in range(1, (depth + 1) >> 1):
            d2 = depth - d1
            for p, q in product(self.visited[d1], self.visited[d2]):
                if self.binary(p, q, depth):
                    return True
        if depth & 1 == 0:
            v = self.visited[depth >> 1]
            length = len(v)
            for i in range(length):
                for j in range(i, length):
                    if self.binary(v[i], v[j], depth):
                        return True

    def solve(self):
        for depth in count(1):
            if self.search(depth):
                return

    @staticmethod
    def number_printer(n):
        if type(n) is tuple:
            if len(n) == 2:
                if n[0] == "sqrt":
                    return "sqrt(" + Tchisla.number_printer(n[1]) + ")"
                elif n[0] == "-":
                    return "-" + Tchisla.number_printer(n[1])
            elif len(n) == 3:
                return Tchisla.number_printer(n[1]) + n[0] + Tchisla.number_printer(n[2])
        else:
            return str(n)

    def printer(self, n):
        depth, expression = self.solutions[n]
        string = str(depth) + ": " + str(n)
        if expression == ():
            return string
        string += " = "
        if len(expression) == 2:
            method, operator = expression
            if method == "sqrt":
                return string + "sqrt(" + Tchisla.number_printer(operator) + ")"
            elif method == "factorial":
                return string + str(operator) + "!"
        elif len(expression) == 3:
            method, op1, op2 = expression
            return string + Tchisla.number_printer(op1) + " " + method + " " + Tchisla.number_printer(op2)

    @staticmethod
    def number_dfs(expression):
        if type(expression) is tuple:
            return chain.from_iterable(map(Tchisla.number_dfs, islice(expression, 1, None)))
        else:
            return (expression,)

    def solution_found(self, n, force_print = False):
        if n in self.number_printed:
            return
        depth, expression = self.solutions[n]
        if expression or force_print:
            print(self.printer(n))
            self.number_printed.add(n)
            for x in Tchisla.number_dfs(expression):
                self.solution_found(x)
<commit_msg>improve code<commit_after>import math
from fractions import Fraction
from itertools import count, product, combinations_with_replacement, chain, islice
from functools import reduce
from utils import sqrt, factorial

__all__ = ["Tchisla"]

MAX = 1 << 128
MAX_DIGITS = 128
MAX_CONCAT = 39
MAX_FACTORIAL = 34

class Tchisla:
    def __init__(self, n, target):
        self.n = n
        self.target = target
        self.solutions = {}
        self.visited = [[]]
        self.number_printed = set()

    def insert(self, x, depth, expression):
        self.solutions[x] = depth, expression
        self.visited[depth].append(x)
        if x == self.target:
            self.solution_found(self.target, True)
            return True

    def check(self, x, depth, expression):
        if x.numerator > MAX or x.denominator > MAX or x in self.solutions:
            return
        if self.insert(x, depth, expression):
            return True
        z = sqrt(x.denominator)
        if z is not None:
            y = sqrt(x.numerator)
            if y is not None and self.check(Fraction(y, z, False), depth, ("sqrt", x)):
                return True
        if x.denominator == 1 and x <= MAX_FACTORIAL:
            y = Fraction(factorial(int(x)))
            if self.check(y, depth, ("factorial", x)):
                return True

    def exponent(self, p, q, depth):
        if q.denominator != 1 or p == 1:
            return
        p_digits = math.log2(max(p.numerator, p.denominator))
        q_int = q.numerator
        exp = (("^", p, q), ("^", p, ("-", q)))
        while p_digits * q_int > MAX_DIGITS:
            if q_int & 1 == 0:
                q_int >>= 1
                exp = (("sqrt", exp[0]), ("sqrt", exp[1]))
            else:
                return
        x = p ** q_int
        if self.check(x, depth, exp[0]):
            return True
        if self.check(x ** -1, depth, exp[1]):
            return True

    def binary(self, p, q, depth):
        if self.check(p + q, depth, ("+", p, q)):
            return True
        if p > q and self.check(p - q, depth, ("-", p, q)):
            return True
        elif p < q and self.check(q - p, depth, ("-", q, p)):
            return True
        if self.check(p * q, depth, ("*", p, q)):
            return True
        quotient = p / q
        if self.check(quotient, depth, ("/", p, q)):
            return True
        if self.check(quotient ** -1, depth, ("/", q, p)):
            return True
        if self.exponent(p, q, depth):
            return True
        if self.exponent(q, p, depth):
            return True

    def search(self, depth):
        self.visited.append([])
        if depth <= MAX_CONCAT:
            m = Fraction((10 ** depth - 1) // 9 * self.n)
            if self.check(m, depth, ()):
                return True
        for d1 in range(1, (depth + 1) >> 1):
            d2 = depth - d1
            for p, q in product(self.visited[d1], self.visited[d2]):
                if self.binary(p, q, depth):
                    return True
        if depth & 1 == 0:
            for p, q in combinations_with_replacement(self.visited[depth >> 1], 2):
                if self.binary(p, q, depth):
                    return True

    def solve(self):
        for depth in count(1):
            if self.search(depth):
                return

    @staticmethod
    def number_printer(n):
        if type(n) is tuple:
            if len(n) == 2:
                if n[0] == "sqrt":
                    return "sqrt(" + Tchisla.number_printer(n[1]) + ")"
                elif n[0] == "-":
                    return "-" + Tchisla.number_printer(n[1])
            elif len(n) == 3:
                return Tchisla.number_printer(n[1]) + n[0] + Tchisla.number_printer(n[2])
        else:
            return str(n)

    def printer(self, n):
        depth, expression = self.solutions[n]
        string = str(depth) + ": " + str(n)
        if expression == ():
            return string
        string += " = "
        if len(expression) == 2:
            method, operator = expression
            if method == "sqrt":
                return string + "sqrt(" + Tchisla.number_printer(operator) + ")"
            elif method == "factorial":
                return string + str(operator) + "!"
        elif len(expression) == 3:
            method, op1, op2 = expression
            return string + Tchisla.number_printer(op1) + " " + method + " " + Tchisla.number_printer(op2)

    @staticmethod
    def number_dfs(expression):
        if type(expression) is tuple:
            return chain.from_iterable(map(Tchisla.number_dfs, islice(expression, 1, None)))
        else:
            return (expression,)

    def solution_found(self, n, force_print = False):
        if n in self.number_printed:
            return
        depth, expression = self.solutions[n]
        if expression or force_print:
            print(self.printer(n))
            self.number_printed.add(n)
            for x in Tchisla.number_dfs(expression):
                self.solution_found(x)
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import urllib2
import csv
import dbconfig as cfg
origPage = 'http://steamsales.rhekua.com/'
page = urllib2.urlopen(origPage)
file = "C:\\Python27\\demo.html"
import sys
import re
import MySQLdb as mdb
reload(sys)
sys.setdefaultencoding('UTF8')

def pullGames():
 soup = BeautifulSoup (page, 'html.parser')
 #(page, 'html.parser')

 #Let's pull prices
 names = []
 pricing = []
 discounts = []
 discountPct = []
 #Push items to array for saving
 for name in soup.find_all('div', attrs={'class': 'tab_row'}):
  names.append(name.h4.text)
 for price in soup.find_all('div', attrs={'class': 'discount_original_price'}):
  pricing.append(price.text)
 for discount in soup.find_all('div', attrs={'class': 'discount_final_price'}):
  discounts.append(discount.text)
 for percent in soup.find_all('div', attrs={'class': 'discount_pct'}):
  discountPct.append(percent.text)

 con = mdb.connect(cfg.mysql['host'], cfg.mysql['user'], cfg.mysql['passwd'], cfg.mysql['db']);
 cur = con.cursor()
 con.set_character_set('utf8')
 cur.execute('SET NAMES utf8;')
 cur.execute('SET CHARACTER SET utf8;')
 cur.execute('SET character_set_connection=utf8;')

 for i in range(len(names)):
  sql = 'INSERT INTO game_pricing(game_name, original_price, discount_price) \
         VALUES("%s", "%s", "%s")' % \
         (names[i],pricing[i],discounts[i])
 #print sql
  cur.execute(sql)
  con.commit()

def checkDb():
 con = mdb.connect(cfg.mysql['host'], cfg.mysql['user'], cfg.mysql['passwd'], cfg.mysql['db']);
 cur = con.cursor()
 con.set_character_set('utf8')
 cur.execute('SET NAMES utf8;')
 cur.execute('SET CHARACTER SET utf8;')
 cur.execute('SET character_set_connection=utf8;')
 sql = 'SELECT game_name, original_price, discount_price FROM game_pricing'
 cur.execute(sql)
 results = cur.fetchall()
 for row in results:
  game_name = row[0]
  original_price = row[1]
  discount_price = row[2]
  print "game_name=%s, original_price=%s, discount_price=%s" % \
         (game_name, original_price, discount_price)
#For testing
checkDb()
#sql = "SELECT VERSION()"

##WORKING ----

##
## #Save to CSV
##ofile = open('output2.csv','wb')
##fieldname = ['name', 'discountPrice', 'originalPrice', 'percentOff']
##writer = csv.DictWriter(ofile, fieldnames = fieldname)
##writer.writeheader()
##for i in range(len(names)):
## writer.writerow({'name': names[i], 'discountPrice':pricing[i], 'originalPrice': discounts[i], 'percentOff': discountPct[i].strip('\n\r\t')})
##ofile.close()<commit_msg>Update CSV<commit_after>from bs4 import BeautifulSoup
import urllib2
import csv
import dbconfig as cfg
origPage = 'http://steamsales.rhekua.com/'
page = urllib2.urlopen(origPage)
file = "C:\\Python27\\demo.html"
import sys
import re
import MySQLdb as mdb
reload(sys)
sys.setdefaultencoding('UTF8')

def pullGames():
 soup = BeautifulSoup (page, 'html.parser')
 #(page, 'html.parser')

 #Let's pull prices
 names = []
 pricing = []
 discounts = []
 discountPct = []
 #Push items to array for saving
 for name in soup.find_all('div', attrs={'class': 'tab_row'}):
  names.append(name.h4.text)
 for price in soup.find_all('div', attrs={'class': 'discount_original_price'}):
  pricing.append(price.text)
 for discount in soup.find_all('div', attrs={'class': 'discount_final_price'}):
  discounts.append(discount.text)
 for percent in soup.find_all('div', attrs={'class': 'discount_pct'}):
  discountPct.append(percent.text)

 con = mdb.connect(cfg.mysql['host'], cfg.mysql['user'], cfg.mysql['passwd'], cfg.mysql['db']);
 cur = con.cursor()
 con.set_character_set('utf8')
 cur.execute('SET NAMES utf8;')
 cur.execute('SET CHARACTER SET utf8;')
 cur.execute('SET character_set_connection=utf8;')

 for i in range(len(names)):
  sql = 'INSERT INTO game_pricing(game_name, original_price, discount_price) \
         VALUES("%s", "%s", "%s")' % \
         (names[i],pricing[i],discounts[i])
 #print sql
  cur.execute(sql)
  con.commit()

def checkDb():
 con = mdb.connect(cfg.mysql['host'], cfg.mysql['user'], cfg.mysql['passwd'], cfg.mysql['db']);
 cur = con.cursor()
 con.set_character_set('utf8')
 cur.execute('SET NAMES utf8;')
 cur.execute('SET CHARACTER SET utf8;')
 cur.execute('SET character_set_connection=utf8;')
 sql = 'SELECT game_name, original_price, discount_price FROM game_pricing'
 cur.execute(sql)
 results = cur.fetchall()
 for row in results:
  game_name = row[0]
  original_price = row[1]
  discount_price = row[2]
  print "game_name=%s, original_price=%s, discount_price=%s" % \
         (game_name, original_price, discount_price)
#For testing
checkDb()
#sql = "SELECT VERSION()"

##WORKING ----

##
## #Save to CSV -- This will parse from Rhekua by default. If you need to output DB -> CSV, just implement this function into the checkDb() function
##ofile = open('output2.csv','wb')
##fieldname = ['name', 'discountPrice', 'originalPrice', 'percentOff']
##writer = csv.DictWriter(ofile, fieldnames = fieldname)
##writer.writeheader()
##for i in range(len(names)):
## writer.writerow({'name': names[i], 'discountPrice':pricing[i], 'originalPrice': discounts[i], 'percentOff': discountPct[i].strip('\n\r\t')}) # Rhekua use only. Excess linebreaks.
##ofile.close()<|endoftext|>


<commit_before>import sys
from PyQt5 import QtWidgets, QtCore, QtGui
#from _alt_elements import *

def startTest():
    print('Starting test...')

    app = QtWidgets.QApplication(sys.argv)
    window = TestWindow()

    sys.exit(app.exec_())
    pass

class TestWindow(QtWidgets.QMainWindow):
    def __init__(self, parent = None):
        super(TestWindow, self).__init__(parent)
        self.setupUi()

    def setupUi(self):
        self.setObjectName("MainWindow")
        self.resize(300,300)

        self.label1 = QtWidgets.QLabel("hi", self)
        self.label1.setGeometry(QtCore.QRect(10,0,30,20))
        self.label1.setObjectName("label1")

        self.textbox = QtWidgets.QLineEdit(self)
        self.textbox.setGeometry(QtCore.QRect(20,20,50,20))
        self.textbox.setObjectName("textbox")

        self.show()
        pass

    def closeEvent(self, event):
        print("Exiting...")
        super().closeEvent(event)
        pass

if __name__ == '__main__':
    startTest()<commit_msg>removed test_ui<commit_after><|endoftext|>


<commit_before>from psychopy import visual, core
from random import sample
from helpers import choose_pair, retrieve_subject_pairs, get_subject_info
import os

def step(window, transparency):

    # Image stuff
    img = visual.ImageStim(window,
                           image="Pairs/Pair 1/1.png",
                           color=(1,1,1),
                           size = [160,160],
                           pos = (-125,0),
                           opacity = transparency)

    frame_paths = ["Masks/" + file for file in os.listdir("Masks")]
    frames = map(lambda file_name: visual.ImageStim(window,
                                                    image = file_name,
                                                    color = (1,1,1),
                                                    size = [160, 160],
                                                    pos = (125,0)), frame_paths)

    for frameN in range(60):

        frame_num = frameN//6
        frame_path = "Masks/{}.JPG".format(frame_num)
        mask_frame = frames[frame_num]

        mask_frame.draw()
        img.draw()
        window.flip()

    return 0.025

def staircase(window, transparency):
    """
    Performs a single staircase to find the threshold of visibility for a
    subject

    returns the threshold
    """
    for i in range(40):
        transparency += step(window, transparency)
    return transparency

def main():
    #TODO: Do we need the list of names? Maybe for recording the results?
    #TODO: Figure how csv will be formatted. Info to save:
    #   * Subject number
    #   * Response time
    #   * Difficulty of name (Just record names instead and do in post?)

    mywin = visual.Window([1920,1080],
                          monitor = "testMonitor",
                          units = "pix",
                          rgb=(-1,-1,-1),
                          fullscr = True)
    staircase(mywin, 0)

if __name__ == "__main__":
    main()
<commit_msg>Finished basic staircase framework<commit_after>from psychopy import visual, core, event
from random import sample
from helpers import choose_pair, retrieve_subject_pairs, get_subject_info
import os

def step(window, transparency, img, frames):

    img.setOpacity(transparency)

    for frameN in range(60):

        frame_num = frameN//6
        mask_frame = frames[frame_num]

        mask_frame.draw()
        window.flip()

    keys = event.waitKeys(maxWait = 2)
    if keys:
        return -0.025
    else:
        return 0.025

def staircase(window, transparency):
    """
    Performs a single staircase to find the threshold of visibility for a
    subject

    returns the threshold
    """

    # Image stuff
    img = visual.ImageStim(window,
                           image="Pairs/Pair 1/1.png",
                           color=(1,1,1),
                           size = [160,160],
                           pos = (-125,0),
                           opacity = transparency)

    img.setAutoDraw(True)

    frame_paths = ["Masks/" + file for file in os.listdir("Masks")]
    frames = map(lambda file_name: visual.ImageStim(window,
                                                    image = file_name,
                                                    color = (1,1,1),
                                                    size = [160, 160],
                                                    pos = (125,0)), frame_paths)

    for i in range(10):
        transparency += step(window, transparency, img, frames)
    return transparency

def main():
    #TODO: Do we need the list of names? Maybe for recording the results?
    #TODO: Figure how csv will be formatted. Info to save:
    #   * Subject number
    #   * Response time
    #   * Difficulty of name (Just record names instead and do in post?)

    mywin = visual.Window([1920,1080],
                          monitor = "testMonitor",
                          units = "pix",
                          rgb=(-1,-1,-1),
                          fullscr = True)
    staircase(mywin, 0)

if __name__ == "__main__":
    main()
<|endoftext|>