<commit_before>#!/usr/bin/env python

"""minipkg.py - install pkgsrc

Usage: python minipkg.py [-h | --help] [-v | --version]
"""

from __future__ import print_function
import hashlib
import os
import subprocess
import sys
try:
    from urllib2 import (
        Request as url_request,
        urlopen as url_open,
    )
except ImportError:
    import urllib.request.Request as url_request
    import urllib.request.urlopen as url_open

__author__ = 'Mansour Moufid'
__copyright__ = 'Copyright 2015, Mansour Moufid'
__email__ = 'mansourmoufid@gmail.com'
__license__ = 'ISC'
__status__ = 'Development'
__version__ = '1.0'

supported_sys = ('Linux', 'Darwin')

supported_mach = {
    'i386': '32',
    'x86_64': '64',
}

archives = [
    'http://minipkg.eliteraspberries.com/pkgsrc-2015Q3.tar.gz',
    'http://minipkg.eliteraspberries.com/pkgsrc-eliteraspberries-0.3.tar.gz',
]

hash_algorithm = hashlib.sha256

archive_hashes = [
    'f56599dece253113f64d92c528989b7fcb899f3888c7c9fc40f70f08ac91fea6',
    'db8ebcd1c12229e9d2da92f888cdb8505f9c486381f3f9b2cd8948a14a04b671',
]

def uname():
    p = subprocess.Popen(['uname', '-sm'], stdout=subprocess.PIPE)
    p.wait()
    assert p.returncode == 0, 'uname'
    (sys, mach) = p.stdout.read().split()
    return (sys, mach)

def fetch(url, hash):
    filename = os.path.basename(url)
    if not os.path.exists(filename):
        req = url_request(url)
        res = url_open(req)
        dat = res.read()
        with open(filename, 'wb') as f:
            f.write(dat)
    with open(filename, 'r') as f:
        dat = f.read()
    h = hash_algorithm(dat)
    assert h.hexdigest() == hash

def extract(tgz, path):
    if not os.path.exists(path):
        os.mkdir(path)
    tar = tgz.rstrip('.gz')
    if not os.path.exists(tar):
        err = subprocess.call(['gunzip', tgz])
        assert err == 0, 'gunzip'
    err = subprocess.call(['tar', '-xf', tar, '-C', path])
    assert err == 0, 'tar'

recommended_packages = [
    'nbpatch-20100124.tgz',
    'nbsed-20120308.tgz',
    'nawk-20121220nb1.tgz',
    'libfetch-2.36nb1.tgz',
    'digest-20121220.tgz',
    'libtool-base-2.4.2nb10.tgz',
    'gzip-1.6.tgz',
    'bzip2-1.0.6nb1.tgz',
    'ncurses-6.0nb1.tgz',
    'readline-6.3nb3.tgz',
    'zlib-1.2.8nb3.tgz',
    'gettext-lib-0.19.4.tgz',
    'perl-5.22.0.tgz',
    'p5-Scalar-List-Utils-1.42nb1.tgz',
    'p5-CPAN-Meta-2.150005.tgz',
    'p5-inc-latest-0.500.tgz',
    'p5-Module-Build-0.42140.tgz',
    'p5-Perl4-CoreLibs-0.003nb5.tgz',
    'sqlite3-3.8.11.1.tgz',
    'openssl-1.0.2d.tgz',
    'xz-5.2.1.tgz',
    'libarchive-3.1.2nb1.tgz',
    'pkgin-0.9.3.tgz',
]

def install_binary_package(home, repo, pkg):
    pkg_url = '/'.join([repo, pkg])
    prefix = os.path.join(home, 'pkg')
    ret = subprocess.call(['pkg_add', '-p', prefix, pkg_url])
    assert ret == 0, 'pkg_add'

if __name__ == '__main__':

    assert len(sys.argv) in (1, 2)
    if len(sys.argv) == 1:
        pass
    elif len(sys.argv) == 2:
        if sys.argv[1] in ('-h', '--help'):
            print(__doc__)
            print('Supported systems:', supported_sys)
            print('Supported architectures:', list(supported_mach.keys()))
            sys.exit(os.EX_OK)
        elif sys.argv[1] in ('-v', '--version'):
            print('minipkg version', __version__)
            sys.exit(os.EX_OK)
        else:
            print(__doc__)
            sys.exit(os.EX_USAGE)

    print('minipkg: version', __version__)

    # Step 1:
    # Determine some information about the machine.
    HOME = os.environ['HOME']
    OPSYS, mach = uname()
    assert OPSYS in supported_sys, 'unsupported system'
    assert mach in supported_mach, 'unsupported architecture'
    ABI = supported_mach[mach]
    CC = os.environ.get('CC', 'clang')
    print('minipkg: HOME:', HOME)
    print('minipkg: OPSYS:', OPSYS)
    print('minipkg: ABI:', ABI)
    print('minipkg: CC:', CC)

    # Step 2:
    # Fetch the pkgsrc archive.
    for (archive, hash) in zip(archives, archive_hashes):
        print('minipkg: fetching', archive, '...')
        fetch(archive, hash)

    # Step 3:
    # Extract the pkgsrc archive.
    home_usr = os.path.join(HOME, 'usr')
    for tgz in map(os.path.basename, archives):
        print('minipkg: extracting', tgz, '...')
        extract(tgz, home_usr)

    # Step 4:
    # Bootstrap pkgsrc.
    print('minipkg: bootstrapping ...')
    sh = os.environ.get('SH', '/bin/bash')
    sh = sh.split(os.pathsep)[0]
    assert os.path.exists(sh), sh
    os.environ.update({'SH': sh})
    bootstrap_path = os.path.join(HOME, 'usr', 'pkgsrc', 'bootstrap')
    if not os.path.exists(os.path.join(bootstrap_path, 'work')):
        os.chdir(bootstrap_path)
        p = subprocess.Popen(
            [
                './bootstrap',
                '--unprivileged',
                '--abi', ABI,
                '--compiler', CC,
                '--make-jobs', '4',
                '--prefer-pkgsrc', 'no',
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        out, err = p.communicate()
        log = os.path.join(HOME, 'pkgsrc-bootstrap-log.txt')
        with open(log, 'w') as f:
            f.write(out)
            f.write(err)
        assert p.returncode == 0, 'bootstrap'

    # Step 5:
    # Set environment variables.
    print('minipkg: setting environment variables ...')
    vars = [
        ('PATH', os.path.join(HOME, 'pkg', 'bin')),
        ('PATH', os.path.join(HOME, 'pkg', 'sbin')),
        ('MANPATH', os.path.join(HOME, 'pkg', 'man')),
    ]
    for (key, val) in vars:
        os.environ.update({
            key: val + os.pathsep + os.environ.get(key, ''),
        })
    script = [
        'export %s="%s:$%s"' % (key, val, key)
        for (key, val) in vars
    ]
    dotfile = os.path.join(HOME, '.bash_profile')
    try:
        with open(dotfile, 'r') as f:
            profile = f.read()
    except IOError:
        profile = ''
    with open(dotfile, 'w') as f:
        print('# generated by minipkg', file=f)
        for line in script:
            print(line, file=f)
        print('export SH=%s' % sh, file=f)
        print(profile, file=f)

    # Step 6:
    # Install recommended binary packages.
    print('minipkg: installing packages ...')
    repo = '/'.join([
        'http://minipkg.eliteraspberries.com/packages',
        OPSYS,
        mach,
        'All',
    ])
    for pkg in recommended_packages:
        install_binary_package(HOME, repo, pkg)

    print('minipkg: done!')
<commit_msg>Check that the compiler exists.<commit_after>#!/usr/bin/env python

"""minipkg.py - install pkgsrc

Usage: python minipkg.py [-h | --help] [-v | --version]
"""

from __future__ import print_function
import hashlib
import os
import subprocess
import sys
try:
    from urllib2 import (
        Request as url_request,
        urlopen as url_open,
    )
except ImportError:
    import urllib.request.Request as url_request
    import urllib.request.urlopen as url_open

__author__ = 'Mansour Moufid'
__copyright__ = 'Copyright 2015, Mansour Moufid'
__email__ = 'mansourmoufid@gmail.com'
__license__ = 'ISC'
__status__ = 'Development'
__version__ = '1.0'

supported_sys = ('Linux', 'Darwin')

supported_mach = {
    'i386': '32',
    'x86_64': '64',
}

archives = [
    'http://minipkg.eliteraspberries.com/pkgsrc-2015Q3.tar.gz',
    'http://minipkg.eliteraspberries.com/pkgsrc-eliteraspberries-0.3.tar.gz',
]

hash_algorithm = hashlib.sha256

archive_hashes = [
    'f56599dece253113f64d92c528989b7fcb899f3888c7c9fc40f70f08ac91fea6',
    'db8ebcd1c12229e9d2da92f888cdb8505f9c486381f3f9b2cd8948a14a04b671',
]

def which(name):
    p = subprocess.Popen(['which', name], stdout=subprocess.PIPE)
    p.wait()
    if not p.returncode == 0:
        return None
    out = p.stdout.read()
    return out.rstrip('\n')

def uname():
    p = subprocess.Popen(['uname', '-sm'], stdout=subprocess.PIPE)
    p.wait()
    assert p.returncode == 0, 'uname'
    (sys, mach) = p.stdout.read().split()
    return (sys, mach)

def fetch(url, hash):
    filename = os.path.basename(url)
    if not os.path.exists(filename):
        req = url_request(url)
        res = url_open(req)
        dat = res.read()
        with open(filename, 'wb') as f:
            f.write(dat)
    with open(filename, 'r') as f:
        dat = f.read()
    h = hash_algorithm(dat)
    assert h.hexdigest() == hash

def extract(tgz, path):
    if not os.path.exists(path):
        os.mkdir(path)
    tar = tgz.rstrip('.gz')
    if not os.path.exists(tar):
        err = subprocess.call(['gunzip', tgz])
        assert err == 0, 'gunzip'
    err = subprocess.call(['tar', '-xf', tar, '-C', path])
    assert err == 0, 'tar'

recommended_packages = [
    'nbpatch-20100124.tgz',
    'nbsed-20120308.tgz',
    'nawk-20121220nb1.tgz',
    'libfetch-2.36nb1.tgz',
    'digest-20121220.tgz',
    'libtool-base-2.4.2nb10.tgz',
    'gzip-1.6.tgz',
    'bzip2-1.0.6nb1.tgz',
    'ncurses-6.0nb1.tgz',
    'readline-6.3nb3.tgz',
    'zlib-1.2.8nb3.tgz',
    'gettext-lib-0.19.4.tgz',
    'perl-5.22.0.tgz',
    'p5-Scalar-List-Utils-1.42nb1.tgz',
    'p5-CPAN-Meta-2.150005.tgz',
    'p5-inc-latest-0.500.tgz',
    'p5-Module-Build-0.42140.tgz',
    'p5-Perl4-CoreLibs-0.003nb5.tgz',
    'sqlite3-3.8.11.1.tgz',
    'openssl-1.0.2d.tgz',
    'xz-5.2.1.tgz',
    'libarchive-3.1.2nb1.tgz',
    'pkgin-0.9.3.tgz',
]

def install_binary_package(home, repo, pkg):
    pkg_url = '/'.join([repo, pkg])
    prefix = os.path.join(home, 'pkg')
    ret = subprocess.call(['pkg_add', '-p', prefix, pkg_url])
    assert ret == 0, 'pkg_add'

if __name__ == '__main__':

    assert len(sys.argv) in (1, 2)
    if len(sys.argv) == 1:
        pass
    elif len(sys.argv) == 2:
        if sys.argv[1] in ('-h', '--help'):
            print(__doc__)
            print('Supported systems:', supported_sys)
            print('Supported architectures:', list(supported_mach.keys()))
            sys.exit(os.EX_OK)
        elif sys.argv[1] in ('-v', '--version'):
            print('minipkg version', __version__)
            sys.exit(os.EX_OK)
        else:
            print(__doc__)
            sys.exit(os.EX_USAGE)

    print('minipkg: version', __version__)

    # Step 1:
    # Determine some information about the machine.
    HOME = os.environ['HOME']
    OPSYS, mach = uname()
    assert OPSYS in supported_sys, 'unsupported system'
    assert mach in supported_mach, 'unsupported architecture'
    ABI = supported_mach[mach]
    CC = os.environ.get('CC', 'clang')
    assert which(CC), CC
    print('minipkg: HOME:', HOME)
    print('minipkg: OPSYS:', OPSYS)
    print('minipkg: ABI:', ABI)
    print('minipkg: CC:', CC)

    # Step 2:
    # Fetch the pkgsrc archive.
    for (archive, hash) in zip(archives, archive_hashes):
        print('minipkg: fetching', archive, '...')
        fetch(archive, hash)

    # Step 3:
    # Extract the pkgsrc archive.
    home_usr = os.path.join(HOME, 'usr')
    for tgz in map(os.path.basename, archives):
        print('minipkg: extracting', tgz, '...')
        extract(tgz, home_usr)

    # Step 4:
    # Bootstrap pkgsrc.
    print('minipkg: bootstrapping ...')
    sh = os.environ.get('SH', '/bin/bash')
    sh = sh.split(os.pathsep)[0]
    assert os.path.exists(sh), sh
    os.environ.update({'SH': sh})
    bootstrap_path = os.path.join(HOME, 'usr', 'pkgsrc', 'bootstrap')
    if not os.path.exists(os.path.join(bootstrap_path, 'work')):
        os.chdir(bootstrap_path)
        p = subprocess.Popen(
            [
                './bootstrap',
                '--unprivileged',
                '--abi', ABI,
                '--compiler', CC,
                '--make-jobs', '4',
                '--prefer-pkgsrc', 'no',
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        out, err = p.communicate()
        log = os.path.join(HOME, 'pkgsrc-bootstrap-log.txt')
        with open(log, 'w') as f:
            f.write(out)
            f.write(err)
        assert p.returncode == 0, 'bootstrap'

    # Step 5:
    # Set environment variables.
    print('minipkg: setting environment variables ...')
    vars = [
        ('PATH', os.path.join(HOME, 'pkg', 'bin')),
        ('PATH', os.path.join(HOME, 'pkg', 'sbin')),
        ('MANPATH', os.path.join(HOME, 'pkg', 'man')),
    ]
    for (key, val) in vars:
        os.environ.update({
            key: val + os.pathsep + os.environ.get(key, ''),
        })
    script = [
        'export %s="%s:$%s"' % (key, val, key)
        for (key, val) in vars
    ]
    dotfile = os.path.join(HOME, '.bash_profile')
    try:
        with open(dotfile, 'r') as f:
            profile = f.read()
    except IOError:
        profile = ''
    with open(dotfile, 'w') as f:
        print('# generated by minipkg', file=f)
        for line in script:
            print(line, file=f)
        print('export SH=%s' % sh, file=f)
        print(profile, file=f)

    # Step 6:
    # Install recommended binary packages.
    print('minipkg: installing packages ...')
    repo = '/'.join([
        'http://minipkg.eliteraspberries.com/packages',
        OPSYS,
        mach,
        'All',
    ])
    for pkg in recommended_packages:
        install_binary_package(HOME, repo, pkg)

    print('minipkg: done!')
<|endoftext|>


<commit_before>import logging
import pickle
import shutil
import fileinput

import engine.util as util
from .abstract_runner import AbstractRunner
from ..simple_lxd import simple_lxd as lxd

logger = logging.getLogger(__name__)

class FilePushError(Exception):
    def __init__(self, message):
        super().__init__(message)

class EngineExecutionError(Exception):
    def __init__(self, message):
        super().__init__(message)

class FilePullError(Exception):
    def __init__(self, message):
        super().__init__(message)

class PythonRunner(AbstractRunner):
    def run(self, container_name, code_filename, function_name, input_tuple):
        logger.debug("Running {:s} with input {:}".format(code_filename, input_tuple))

        run_id = code_filename.split('.')[0]
        input_pickle = '{}.input.pickle'.format(run_id)
        with open(input_pickle, mode='wb') as f:
            logger.debug("Pickling input tuple in {:s}...".format(input_pickle))
            pickle.dump(input_tuple, file=f)

        runner_file = "{}.run.py".format(run_id)
        shutil.copy("run_it.py", runner_file)

        # Replace "$FUNCTION_NAME" in run_it.py with function name to execute from
        # the problem module.
        logger.info("Replacing $FUNCTION_NAME->{:s} in {:s}...".format(function_name, runner_file))
        with fileinput.FileInput(runner_file, inplace=True) as f:
            for line in f:
                print(line.replace("$FUNCTION_NAME", function_name), end='')

        for file_name in [code_filename, runner_file, input_pickle]:
            source_path = file_name
            target_path = "/root/{}".format(file_name)
            _, push_retval, push_stdout = lxd.file_push(container_name, source_path, target_path)

            if push_retval != 0:
                util.delete_file(code_filename)
                util.delete_file(runner_file)
                util.delete_file(input_pickle)
                raise FilePushError(push_stdout)

        runner_path = "/root/{}".format(runner_file)
        command = ['python3', runner_path]
        _, exec_retval, exec_stdout = lxd.execute(container_name, command)

        if exec_retval != 0:
            util.delete_file(code_filename)
            util.delete_file(runner_file)
            util.delete_file(input_pickle)
            raise EngineExecutionError(exec_stdout)

        p_info = {
            'return_value': exec_retval,
            'stdout': exec_stdout,
            'runtime': 0,
            'max_mem_usage': 0
        }

        output_pickle = '{}.output.pickle'.format(run_id)
        source_path = '/root/{}'.format(output_pickle)
        target_path = output_pickle

        _, pull_retval, pull_stdout = lxd.file_pull(container_name, source_path, target_path)

        if pull_retval != 0:
            util.delete_file(code_filename)
            util.delete_file(runner_file)
            util.delete_file(input_pickle)
            raise FilePullError(pull_stdout)

        with open(output_pickle, mode='rb') as f:
            output_dict = pickle.load(f)

        user_output = output_dict['user_output']
        p_info['runtime'] = output_dict['runtime']
        p_info['max_mem_usage'] = output_dict['max_mem_usage']

        logger.debug("Finished running user code.")
        logger.debug("runtime: {:g} s, max_mem_usage: {:g} kB".format(p_info['runtime'], p_info['max_mem_usage']))

        util.delete_file(input_pickle)
        util.delete_file(output_pickle)
        util.delete_file(runner_file)

        return user_output, p_info
<commit_msg>Execute test cases all at once in PythonRunner<commit_after>import pickle
import shutil
import logging
import fileinput

import engine.util as util
from .abstract_runner import AbstractRunner
from ..simple_lxd import simple_lxd as lxd

logger = logging.getLogger(__name__)

class FilePushError(Exception):
    def __init__(self, message):
        super().__init__(message)

class EngineExecutionError(Exception):
    def __init__(self, message):
        super().__init__(message)

class FilePullError(Exception):
    def __init__(self, message):
        super().__init__(message)

class PythonRunner(AbstractRunner):
    def run(self, container_name, code_filename, function_name, input_tuples):
        logger.info("Running {:s} with {:d} inputs...".format(code_filename, len(input_tuples)))

        run_id = code_filename.split('.')[0]
        input_pickle = "{:s}.input.pickle".format(run_id)
        with open(input_pickle, mode='wb') as f:
            logger.debug("Pickling input tuples in {:s}...".format(input_pickle))
            pickle.dump(input_tuples, file=f, protocol=pickle.HIGHEST_PROTOCOL)

        runner_file = "{:s}.run.py".format(run_id)
        shutil.copy("run_it.py", runner_file)

        # Replace "$FUNCTION_NAME" in run_it.py with function name to call from the problem module.
        logger.debug("Replacing $FUNCTION_NAME->{:s} in {:s}...".format(function_name, runner_file))
        with fileinput.FileInput(runner_file, inplace=True) as f:
            for line in f:
                print(line.replace("$FUNCTION_NAME", function_name), end='')

        for file_name in [code_filename, runner_file, input_pickle]:
            source_path = file_name
            target_path = "/root/{}".format(file_name)
            _, push_retval, push_stdout = lxd.file_push(container_name, source_path, target_path)

            if push_retval != 0:
                util.delete_file(code_filename)
                util.delete_file(runner_file)
                util.delete_file(input_pickle)
                raise FilePushError(push_stdout)

        runner_path = "/root/{}".format(runner_file)
        command = ["python3", runner_path]
        _, exec_retval, exec_stdout = lxd.execute(container_name, command)

        if exec_retval != 0:
            util.delete_file(code_filename)
            util.delete_file(runner_file)
            util.delete_file(input_pickle)
            raise EngineExecutionError(exec_stdout)

        user_outputs = []
        process_infos = []
        for i, _ in enumerate(input_tuples):
            output_pickle = "{:s}.output{:d}.pickle".format(run_id, i)
            source_path = "/root/{:s}".format(output_pickle)
            target_path = output_pickle

            _, pull_retval, pull_stdout = lxd.file_pull(container_name, source_path, target_path)

            if pull_retval != 0:
                util.delete_file(code_filename)
                util.delete_file(runner_file)
                util.delete_file(input_pickle)
                raise FilePullError(pull_stdout)

            with open(output_pickle, mode='rb') as f:
                output_dict = pickle.load(f)

            p_info = {
                'return_value': exec_retval,
                'stdout': exec_stdout,
                'runtime': output_dict['runtime'],
                'max_mem_usage': output_dict['max_mem_usage']
            }

            user_outputs.append(output_dict['user_output'])
            process_infos.append(p_info)

            logger.debug("runtime: {:g} s, max_mem_usage: {:g} kB".format(p_info['runtime'], p_info['max_mem_usage']))

            util.delete_file(output_pickle)

        logger.info("Finished running user code.")

        util.delete_file(input_pickle)
        util.delete_file(runner_file)

        return user_outputs, process_infos
<|endoftext|>


<commit_before>"""
MIT License
Copyright (c) 2016 Ashley Goldfarb

Code for scanning new emails in gmail account. Requires a client secret in client_secret.json from google to operate.
"""
import base64
import email
import os
import re
import logging
import copy
import time

import httplib2
import oauth2client
from apiclient import discovery, errors
from oauth2client import client
from oauth2client import tools

# from Tracking.Packages import Packages

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'

# logger = logging.getLogger('GMail')

class Gmail:
    """ A Class for interfacing with Gmail"""

    def __init__(self):
        """
        Constructor for Gmail
        Initilizes the conection to the Gmail servers.
        """

        self._log = logging.getLogger(self.__class__.__name__)

        self.credentials = self.get_credentials()
        self.http = self.credentials.authorize(httplib2.Http())
        self.service = discovery.build('gmail', 'v1', http=self.http)
        self.newest_history_id_retrieved = None

        self.email = []  # stores the last 100 downloaded emails.
        self._new_email = []
        self.new_email_callback = lambda x: None

    @property
    def new_email(self):
        return self._new_email

    @new_email.setter
    def new_email(self, value):
        self._new_email = value
        self.new_email_callback(self._new_email)

    def history_test(self):
        # message_ids = self.service.users().messages().list(userId="me", includeSpamTrash=False).execute()
        # message_response = self.service.users().messages().get(userId="me", id='1564d571a97815be', format="raw").execute()
        history = self.service.users().history().list(userId="me", startHistoryId='9945333').execute()
        message_ids = history['history']['messages']

    def get_recent_messages(self):
        filter_params = 'in: all -label:"Trash" -label:"Spam"'
        if self.newest_history_id_retrieved is None:
            # We have no record of the last message we have stored (this is probably the first run)
            # Preform a full sync

            if len(self.email) > 0:  # Sanity Check
                raise RuntimeError("No record of last history id, yet we have stored email!")

            message_ids = self.full_email_sync(number=50)['messages']
        else:
            try:
                message_ids, new_history_id = self.incremental_email_sync()
            except errors.HttpError:
                self._log.warning(
                    "Partial sync encountered a 404 error. The HistoryID may be out of date or invalid. \
                    Preforming full sync.")
                message_ids = self.full_email_sync()['messages']

            if message_ids is None:
                # no new emails to retrieve
                return False

        new_messages = []
        for message_id in message_ids:
            try:
                mime_msg, gmail_msg = self.get_mime_message(self.service, "me", message_id['id'])
            except TypeError as e:
                self._log.exception("Exception getting Messages. Forcing full sync")
                self.newest_history_id_retrieved = None
                return False

            unpacked_msg = self.unpack_email(mime_msg, gmail_msg, message_id['id'])
            new_messages.append(unpacked_msg)

        if len(new_messages) > 0:
            self.new_email = copy.deepcopy(new_messages)
            self.email = new_messages + self.email  # append new emails to the beginning of the list to preserve order

        try:
            self.newest_history_id_retrieved = new_history_id
        except NameError:
            self.newest_history_id_retrieved = self.email[0]['gmail_object']['historyId']

        return True

    def full_email_sync(self, number=100, call_count=0):
        """
        Preforms a full email sync and clears the email buffer if there are email present

        Returns:
            messages().list() response from google
        """

        if len(self.email) > 0:
            self.email = []
        try:
            response = self.service.users().messages().list(userId="me", maxResults=10, includeSpamTrash=False).execute()
        except httplib2.HttpLib2Error as e:

            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.full_email_sync(call_count=new_count)
            else:
                raise e

        except Exception as e:
            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        messages = response['messages']
        need_more_emails = True
        while need_more_emails:
            if len(messages) >= number:
                break
            if 'nextPageToken' in response:
                try:
                    response = self.service.users().messages().list(userId="me", pageToken=response['nextPageToken'],
                                                                    maxResults=10, includeSpamTrash=False).execute()
                except httplib2.HttpLib2Error as e:

                    if call_count < 5:
                        new_count = call_count + 1
                        self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                        time.sleep(new_count)
                        return self.full_email_sync(call_count=new_count)
                    else:
                        raise e

                except Exception as e:
                    if call_count < 5:
                        new_count = call_count + 1
                        self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                        time.sleep(new_count)
                        return self.incremental_email_sync(call_count=new_count)
                    else:
                        raise e

                messages.extend(response['messages'])
            else:
                break

        return {'messages': messages}

    def incremental_email_sync(self, call_count=0):
        """
        Preforms an incremental sync from the last history id recorded

        Returns: A list of message ids or None if there are no new messages

        """
        try:
            history = self.service.users().history().list(userId="me",
                                                          startHistoryId=self.newest_history_id_retrieved).execute()
        except httplib2.HttpLib2Error as e:

            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        except Exception as e:
            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        if 'history' not in history:
            # No new messages to download
            self.newest_history_id_retrieved = history['historyId']
            return None, None

        message_ids = []
        for event in history['history']:
            if 'messagesAdded' in event:
                message_ids.append({'id': event['messagesAdded'][0]['message']['id']})
                if len(event['messagesAdded']) > 1:
                    print("History message has more than 1 entry!!! WAT???!!!")

        if len(message_ids) < 1:
            # No added messages were in the response.
            self.newest_history_id_retrieved = history['historyId']
            return None, None

        return message_ids, history['historyId']

    def get_mime_message(self, service, user_id, msg_id, count=0):
        """Get a Message and use it to create a MIME Message.

        Args:
          service: Authorized Gmail API service instance.
          user_id: User's email address. The special value "me"
          can be used to indicate the authenticated user.
          msg_id: The ID of the Message required.

        Returns:
          A MIME Message, consisting of data from Message.
        """
        try:
            message = service.users().messages().get(userId=user_id, id=msg_id,
                                                     format='raw').execute()

            print('Message snippet: %s' % message['snippet'])

            msg_str = base64.urlsafe_b64decode(message['raw'].encode('ASCII'))
            decoded_str = msg_str.decode('utf-8', errors='ignore')
            mime_msg = email.message_from_string(decoded_str)

            return mime_msg, message
        except errors.HttpError as error:
            self._log.exception('An error occurred:')
            if count < 5:
                ncount = count + 1
                return self.get_mime_message(service, user_id, msg_id, ncount)
            else:
                self._log.critical("Exceeded max number of retries. Skipping message.")

    #region Gmail Interface Methods
    """ --- Gmail Interface Methods --- """
    @staticmethod
    def get_credentials():
        """Gets valid user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.

        Returns:
            Credentials, the obtained credential.
        """
        home_dir = os.path.expanduser('~')
        credential_dir = os.path.join(home_dir, '.credentials')
        if not os.path.exists(credential_dir):
            os.makedirs(credential_dir)
        credential_path = os.path.join(credential_dir,
                                       'gmail-python-quickstart.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
            flow.user_agent = APPLICATION_NAME
            if flags:
                credentials = tools.run_flow(flow, store, flags)
            else: # Needed only for compatibility with Python 2.6
                credentials = tools.run(flow, store)
            print('Storing credentials to ' + credential_path)
        return credentials

    #endregion

    #region Email Unpacking
    """ --- Gmail Email Depacking Methods --- """
    # def get_bodies(self, payload):
    #
    #     bodies = []
    #     if payload["mimeType"].startswith("multipart"):
    #         for part in payload["parts"]:
    #             new_body = self.get_bodies(part)
    #             if new_body is not None:
    #                 bodies.extend(new_body)
    #         return bodies
    #
    #     elif payload["mimeType"].startswith("text"):
    #         return [payload["body"]["data"]]
    #
    #     else:
    #         print("Unknown mimeType: " + payload["mimeType"])
    #         return None

    def unpack_body(self, _email):
        body_list = self.unpack_body_worker(_email)
        ret_body = ""
        for body in body_list:
            ret_body += (" " + body)

        return ret_body

    def unpack_body_worker(self, _email):

        bodies = []
        if _email.is_multipart():
            for part in _email.get_payload():
                new_body = self.unpack_body_worker(part)
                if new_body is not None:
                    bodies.extend(new_body)
            return bodies

        else:
            return [_email.get_payload()]

    def decode_international_header(self, _header_string):

        if _header_string is not None:
            decoded_subjects = email.header.decode_header(_header_string)

            tmp_subject = ""
            for decoded_subject in decoded_subjects:

                if decoded_subject[1] is not None:
                    tmp_subject += decoded_subject[0].decode(decoded_subject[1])
                else:
                    if isinstance(decoded_subject[0], bytes):
                        tmp_subject += decoded_subject[0].decode()
                    else:
                        tmp_subject += decoded_subject[0]
            return tmp_subject
        else:
            return None

    def parse_email_address(self, _address):
        """

        :param _address:
        :type _address:
        :return: (Sender Name, Sender Address)
        :rtype: (str, str)
        """
        decoded_address = self.decode_international_header(_address)
        search = re.search(r"(.*) <(.*)>", decoded_address)
        if search is not None:
            return search.groups()
        else:
            return (None, decoded_address)

    def unpack_email(self, _email, _gmail_obj, _id):
        p_email = {}
        p_email['email_object'] = _email
        p_email['gmail_object'] = _gmail_obj
        p_email['to'] = _email['to']
        p_email['from_name'], p_email['from_address'] = self.parse_email_address(_email['from'])
        p_email['subject'] = self.decode_international_header(_email['subject'])
        p_email['body'] = self.unpack_body(_email)
        p_email['id'] = _id

        return p_email

    #endregion

    ''' --- Search Code --- '''

def old_manual_method():
    pass
    # credentials = get_credentials()
    # http = credentials.authorize(httplib2.Http())
    # service = discovery.build('gmail', 'v1', http=http)
    #

    #
    # message_results = service.users().messages().list(userId="me", includeSpamTrash=False).execute()
    #
    # messages = []
    # for message_id in message_results["messages"]:
    #     # message_response = service.users().messages().get(userId="me", id=message_id['id'], format="full").execute()
    #
    #     mime_msg = get_mime_message(service, "me", message_id['id'])
    #     unpacked_msg = unpack_email(mime_msg, message_id['id'])
    #     messages.append(unpacked_msg)
    #
    # packages = Packages()
    # for message in messages:
    #     carrier, trackingNum = search_for_tracking(message["body"])
    #     if carrier is not None:
    #         print("Found {} Tracking number {} in email: {} from {}".format(carrier, trackingNum, message["subject"],
    #                                                                         message["from_address"]))
    #         packages.add_package(carrier, trackingNum, name=message["from_address"], )
    #
    # for package in packages.packages:
    #     """type:  Package"""
    #     package.get_tracking_data()

def test_callback(_email):
    print("{} New emails received".format(len(_email)))

def main():
    """Shows basic usage of the Gmail API.

    Creates a Gmail API service object and outputs a list of label names
    of the user's Gmail account.
    """
    import time
    gmail_interface = Gmail()
    gmail_interface.new_email_callback = test_callback

    count = 100
    for i in range(count):
        success = gmail_interface.get_recent_messages()

        if success:
            print("New emails received")
        else:
            print("No new emails")
        print("delaying for 1 minute check for new emails")
        time.sleep(20)

    print("Done")

if __name__ == '__main__':
    main()<commit_msg>Removed sanity check from get_recent_messages() as it can now be falsely tripped. Instead, we just preform a new full sync which should clear out the incongruity.<commit_after>"""
MIT License
Copyright (c) 2016 Ashley Goldfarb

Code for scanning new emails in gmail account. Requires a client secret in client_secret.json from google to operate.
"""
import base64
import email
import os
import re
import logging
import copy
import time

import httplib2
import oauth2client
from apiclient import discovery, errors
from oauth2client import client
from oauth2client import tools

# from Tracking.Packages import Packages

try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'

# logger = logging.getLogger('GMail')

class Gmail:
    """ A Class for interfacing with Gmail"""

    def __init__(self):
        """
        Constructor for Gmail
        Initilizes the conection to the Gmail servers.
        """

        self._log = logging.getLogger(self.__class__.__name__)

        self.credentials = self.get_credentials()
        self.http = self.credentials.authorize(httplib2.Http())
        self.service = discovery.build('gmail', 'v1', http=self.http)
        self.newest_history_id_retrieved = None

        self.email = []  # stores the last 100 downloaded emails.
        self._new_email = []
        self.new_email_callback = lambda x: None

    @property
    def new_email(self):
        return self._new_email

    @new_email.setter
    def new_email(self, value):
        self._new_email = value
        self.new_email_callback(self._new_email)

    def get_recent_messages(self):
        filter_params = 'in: all -label:"Trash" -label:"Spam"'
        if self.newest_history_id_retrieved is None:
            # We have no record of the last message we have stored (this is probably the first run)
            # Preform a full sync

            message_ids = self.full_email_sync(number=50)['messages']
        else:
            try:
                message_ids, new_history_id = self.incremental_email_sync()
            except errors.HttpError:
                self._log.warning(
                    "Partial sync encountered a 404 error. The HistoryID may be out of date or invalid. \
                    Preforming full sync.")
                message_ids = self.full_email_sync()['messages']

            if message_ids is None:
                # no new emails to retrieve
                return False

        new_messages = []
        for message_id in message_ids:
            try:
                mime_msg, gmail_msg = self.get_mime_message(self.service, "me", message_id['id'])
            except TypeError as e:
                self._log.exception("Exception getting Messages. Forcing full sync")
                self.newest_history_id_retrieved = None
                return False

            unpacked_msg = self.unpack_email(mime_msg, gmail_msg, message_id['id'])
            new_messages.append(unpacked_msg)

        if len(new_messages) > 0:
            self.new_email = copy.deepcopy(new_messages)
            self.email = new_messages + self.email  # append new emails to the beginning of the list to preserve order

        try:
            self.newest_history_id_retrieved = new_history_id
        except NameError:
            self.newest_history_id_retrieved = self.email[0]['gmail_object']['historyId']

        return True

    def full_email_sync(self, number=100, call_count=0):
        """
        Preforms a full email sync and clears the email buffer if there are email present

        Returns:
            messages().list() response from google
        """

        if len(self.email) > 0:
            # Clear email buffer when preforming a full sync
            self.email = []
        try:
            response = self.service.users().messages().list(userId="me", maxResults=10, includeSpamTrash=False).execute()
        except httplib2.HttpLib2Error as e:

            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.full_email_sync(call_count=new_count)
            else:
                raise e

        except Exception as e:
            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        messages = response['messages']
        need_more_emails = True
        while need_more_emails:
            if len(messages) >= number:
                break
            if 'nextPageToken' in response:
                try:
                    response = self.service.users().messages().list(userId="me", pageToken=response['nextPageToken'],
                                                                    maxResults=10, includeSpamTrash=False).execute()
                except httplib2.HttpLib2Error as e:

                    if call_count < 5:
                        new_count = call_count + 1
                        self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                        time.sleep(new_count)
                        return self.full_email_sync(call_count=new_count)
                    else:
                        raise e

                except Exception as e:
                    if call_count < 5:
                        new_count = call_count + 1
                        self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                        time.sleep(new_count)
                        return self.incremental_email_sync(call_count=new_count)
                    else:
                        raise e

                messages.extend(response['messages'])
            else:
                break

        return {'messages': messages}

    def incremental_email_sync(self, call_count=0):
        """
        Preforms an incremental sync from the last history id recorded

        Returns: A list of message ids or None if there are no new messages

        """
        try:
            history = self.service.users().history().list(userId="me",
                                                          startHistoryId=self.newest_history_id_retrieved).execute()
        except httplib2.HttpLib2Error as e:

            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("HttpLib2 Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        except Exception as e:
            if call_count < 5:
                new_count = call_count + 1
                self._log.exception("Unknown Error. Retrying in {} seconds.".format(new_count))
                time.sleep(new_count)
                return self.incremental_email_sync(call_count=new_count)
            else:
                raise e

        if 'history' not in history:
            # No new messages to download
            self.newest_history_id_retrieved = history['historyId']
            return None, None

        message_ids = []
        for event in history['history']:
            if 'messagesAdded' in event:
                message_ids.append({'id': event['messagesAdded'][0]['message']['id']})
                if len(event['messagesAdded']) > 1:
                    print("History message has more than 1 entry!!! WAT???!!!")

        if len(message_ids) < 1:
            # No added messages were in the response.
            self.newest_history_id_retrieved = history['historyId']
            return None, None

        return message_ids, history['historyId']

    def get_mime_message(self, service, user_id, msg_id, count=0):
        """Get a Message and use it to create a MIME Message.

        Args:
          service: Authorized Gmail API service instance.
          user_id: User's email address. The special value "me"
          can be used to indicate the authenticated user.
          msg_id: The ID of the Message required.

        Returns:
          A MIME Message, consisting of data from Message.
        """
        try:
            message = service.users().messages().get(userId=user_id, id=msg_id,
                                                     format='raw').execute()

            print('Message snippet: %s' % message['snippet'])

            msg_str = base64.urlsafe_b64decode(message['raw'].encode('ASCII'))
            decoded_str = msg_str.decode('utf-8', errors='ignore')
            mime_msg = email.message_from_string(decoded_str)

            return mime_msg, message
        except errors.HttpError as error:  # TODO: figure out how to detect Not Found
            self._log.exception('An error occurred:')
            if count < 5:
                ncount = count + 1
                time.sleep(ncount)
                return self.get_mime_message(service, user_id, msg_id, ncount)
            else:
                self._log.critical("Exceeded max number of retries. Skipping message.")

    #region Gmail Interface Methods
    """ --- Gmail Interface Methods --- """
    @staticmethod
    def get_credentials():
        """Gets valid user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.

        Returns:
            Credentials, the obtained credential.
        """
        home_dir = os.path.expanduser('~')
        credential_dir = os.path.join(home_dir, '.credentials')
        if not os.path.exists(credential_dir):
            os.makedirs(credential_dir)
        credential_path = os.path.join(credential_dir,
                                       'gmail-python-quickstart.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
            flow.user_agent = APPLICATION_NAME
            if flags:
                credentials = tools.run_flow(flow, store, flags)
            else: # Needed only for compatibility with Python 2.6
                credentials = tools.run(flow, store)
            print('Storing credentials to ' + credential_path)
        return credentials

    #endregion

    #region Email Unpacking
    """ --- Gmail Email Depacking Methods --- """
    # def get_bodies(self, payload):
    #
    #     bodies = []
    #     if payload["mimeType"].startswith("multipart"):
    #         for part in payload["parts"]:
    #             new_body = self.get_bodies(part)
    #             if new_body is not None:
    #                 bodies.extend(new_body)
    #         return bodies
    #
    #     elif payload["mimeType"].startswith("text"):
    #         return [payload["body"]["data"]]
    #
    #     else:
    #         print("Unknown mimeType: " + payload["mimeType"])
    #         return None

    def unpack_body(self, _email):
        body_list = self.unpack_body_worker(_email)
        ret_body = ""
        for body in body_list:
            ret_body += (" " + body)

        return ret_body

    def unpack_body_worker(self, _email):

        bodies = []
        if _email.is_multipart():
            for part in _email.get_payload():
                new_body = self.unpack_body_worker(part)
                if new_body is not None:
                    bodies.extend(new_body)
            return bodies

        else:
            return [_email.get_payload()]

    def decode_international_header(self, _header_string):

        if _header_string is not None:
            decoded_subjects = email.header.decode_header(_header_string)

            tmp_subject = ""
            for decoded_subject in decoded_subjects:

                if decoded_subject[1] is not None:
                    tmp_subject += decoded_subject[0].decode(decoded_subject[1])
                else:
                    if isinstance(decoded_subject[0], bytes):
                        tmp_subject += decoded_subject[0].decode()
                    else:
                        tmp_subject += decoded_subject[0]
            return tmp_subject
        else:
            return None

    def parse_email_address(self, _address):
        """

        :param _address:
        :type _address:
        :return: (Sender Name, Sender Address)
        :rtype: (str, str)
        """
        decoded_address = self.decode_international_header(_address)
        search = re.search(r"(.*) <(.*)>", decoded_address)
        if search is not None:
            return search.groups()
        else:
            return (None, decoded_address)

    def unpack_email(self, _email, _gmail_obj, _id):
        p_email = {}
        p_email['email_object'] = _email
        p_email['gmail_object'] = _gmail_obj
        p_email['to'] = _email['to']
        p_email['from_name'], p_email['from_address'] = self.parse_email_address(_email['from'])
        p_email['subject'] = self.decode_international_header(_email['subject'])
        p_email['body'] = self.unpack_body(_email)
        p_email['id'] = _id

        return p_email

    #endregion

    ''' --- Search Code --- '''

def old_manual_method():
    pass
    # credentials = get_credentials()
    # http = credentials.authorize(httplib2.Http())
    # service = discovery.build('gmail', 'v1', http=http)
    #

    #
    # message_results = service.users().messages().list(userId="me", includeSpamTrash=False).execute()
    #
    # messages = []
    # for message_id in message_results["messages"]:
    #     # message_response = service.users().messages().get(userId="me", id=message_id['id'], format="full").execute()
    #
    #     mime_msg = get_mime_message(service, "me", message_id['id'])
    #     unpacked_msg = unpack_email(mime_msg, message_id['id'])
    #     messages.append(unpacked_msg)
    #
    # packages = Packages()
    # for message in messages:
    #     carrier, trackingNum = search_for_tracking(message["body"])
    #     if carrier is not None:
    #         print("Found {} Tracking number {} in email: {} from {}".format(carrier, trackingNum, message["subject"],
    #                                                                         message["from_address"]))
    #         packages.add_package(carrier, trackingNum, name=message["from_address"], )
    #
    # for package in packages.packages:
    #     """type:  Package"""
    #     package.get_tracking_data()

def test_callback(_email):
    print("{} New emails received".format(len(_email)))

def main():
    """Shows basic usage of the Gmail API.

    Creates a Gmail API service object and outputs a list of label names
    of the user's Gmail account.
    """
    import time
    gmail_interface = Gmail()
    gmail_interface.new_email_callback = test_callback

    count = 100
    for i in range(count):
        success = gmail_interface.get_recent_messages()

        if success:
            print("New emails received")
        else:
            print("No new emails")
        print("delaying for 1 minute check for new emails")
        time.sleep(20)

    print("Done")

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This is -80 (minus80), a tool for long-term archival backup to Amazon S3 and Glacier.

If you're reading this on Amazon, this file both describes the data format and
contains code for restoring the backup to your local computer.  Keep reading.

Getting Started
===============
Sign up for Amazon Web Services (AWS) at http://aws.amazon.com/
You'll need to sign up for (at a minimum) S3, Glacier, and SNS.

Create a configuration file in JSON format that looks like this:

    {
        "aws_access_key": "XXX",
        "aws_secret_key": "XXX",
        "aws_s3_bucket": "XXX",
        "days_to_glacier": 30,
        "file_database": "~/.minus80.sqlite3",
    }

Replace the "XXX" with appropriate values from your new AWS acount.
The "days_to_glacier" is optional;  if omitted, Glacier will not be used.

Usage - Backup
==============
Storage costs can be reduced by setting up a bucket lifecycle rule that transfers
data from S3 storage to Glacier storage.  Since most of the bytes are in the data/
folder, I would suggest only archiving that folder, say, 30 days after upload.
See http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html

By default, a Glacier archive rule will be set on the bucket for you if your
config file includes a 'days_to_glacier' entry.  You can always delete or
disable it later from the AWS control panel.

Usage - Restore
===============
The restore function hasn't been written yet.  I mentioned this was for
disaster recovery, right?  See "Data Format", below.

Data Format
===========
All data is stored in an Amazon S3 bucket, using a simple content-addressable
storage scheme.  The contents of each file are stored in

    data/DATA_HASH

where DATA_HASH is the hexadecimal SHA-1 hash of the file contents.  This provides
automatic de-duplication -- if you have multiple copies of a single file, it is
only uploaded and stored once.

Metadata about each file is stored in JSON format in

    index/DATA_HASH/INFO_HASH.json

where INFO_HASH is the hexadecimal SHA-1 of the contents of the metadata JSON file.
The metadata includes a timestamp, so even if the contents of a particular file
change over time, one can recover previous versions from the backup if needed.
All the metadata for various instances of a particular piece of content are stored
under a common prefix (DATA_HASH) to make it feasible to purge all aliases to a
particular bit of content from the archive if e.g. it was included by mistake.

Once files have been backed up, this is recorded in a local SQLite database.
This is purely for efficiency -- if a file's size and mtime have not changed
since the last backup, it is assumed to already be in the archive, and is skipped.
This saves recalculating hashes for thousands of files.  However, if the local
database is deleted, the next backup attempt will verify that each file is in S3.

Philosophy
==========
Minus80 is a tool in the Unix tradition.  This means it tries to be simple and modular,
and not re-invent functionality that's available elsewhere. Thus:

- The only way to specify files is as a list of names, one per line, on stdin.
  Expected usage is to pipe the output of `find` to Minus80,
  but you could also have a manually curated backup list.
  It does not currently support null-separated output (-print0), because
  I assume you are a sane human and do not have newlines in your filenames.

- Minus80 does not have built-in bandwidth limits, because tools like
  `trickle` (http://monkey.org/~marius/pages/?page=trickle)
  and `throttled` (https://github.com/zquestz/throttled) are available.

- Minus80 does not encourage encryption, because it is designed for disaster recovery
  of files that are very important to you but of low importance to others, such as
  your personal photo library.  The risk of forgetting the encryption password
  is higher than the risk of the data being compromised in a damaging way.
  Of course, I can't stop you from using `gpg` to encrypt files before backing them up.

- The archive format is as simple as possible.  An average coder should be able to
  easily whip up a script to restore the data, given a description of the format.
  Hopefully, s/he could even reverse-engineer the format if necessary.

- Once backed up, data is never deleted.  This is not a backup tool for many small,
  frequently-changing files.  It is not for reconstructing a flawless image of your
  entire disk exactly as it was before the crash.  It is for worst-case-scenario
  recovery of your most precious digital assets if all your other backups have failed.

Minus80 was named for the -80 C freezers that scientists use for long-term cold storage.
Although it talks only to S3, the intention is to use bucket lifecycle rules to move
all data into Glacier for permanent storage.
"""
# MAKE SURE to increment this when code is updated,
# so a new copy gets stored in the archive!
VERSION = '0.1.0'

import argparse, hashlib, json, logging, os, sqlite3, sys, time
import os.path as osp
import boto

def init_db(dbpath):
    db = sqlite3.connect(osp.expanduser(dbpath))
    with db:
        db.executescript("""
            CREATE TABLE IF NOT EXISTS files (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                abspath TEXT NOT NULL,
                mtime NUMERIC NOT NULL,
                size INTEGER NOT NULL,
                infohash TEXT NOT NULL,
                datahash TEXT NOT NULL,
                updated INTEGER NOT NULL DEFAULT (STRFTIME('%s','now'))
            );
            CREATE UNIQUE INDEX IF NOT EXISTS idx_files_1 ON files (abspath, mtime, size);
            """)
    return db

def set_lifecycle(s3bucket, days_to_glacier):
    import boto.s3.lifecycle as lc
    RULE_ID = 'minus80_data_archive_rule'
    try:
        lifecycle = s3bucket.get_lifecycle_config()
    except boto.exception.S3ResponseError, ex:
        lifecycle = lc.Lifecycle()
    for rule in lifecycle:
        if rule.id == RULE_ID:
            logger.debug("HAS_LIFECYCLE not updating existing lifecycle rules on bucket")
            break
    else:
        lifecycle.append(lc.Rule(RULE_ID, 'data/', 'Enabled',
            transition=lc.Transition(days=days_to_glacier, storage_class='GLACIER')))
        s3bucket.configure_lifecycle(lifecycle)
        logger.info("SET_LIFECYCLE setting default transition-to-Glacier rule on bucket")

def hash_string(s, hashname='sha1'):
    hashfunc = hashlib.new(hashname)
    hashfunc.update(s)
    return hashfunc.hexdigest()

def hash_file_content(absfile, hashname='sha1', chunk=1024*1024):
    hashfunc = hashlib.new(hashname)
    infile = open(absfile, 'rb')
    while True:
        data = infile.read(chunk)
        if not data: break
        hashfunc.update(data)
    return hashfunc.hexdigest()

def upload_string(s3bucket, key_name, s, replace=False):
    """Returns True if data was transferred, False if it was already there."""
    # Uses HEAD to check if this key exists, or returns None
    key = s3bucket.get_key(key_name)
    if not replace and key is not None: return False
    key = s3bucket.new_key(key_name)
    key.set_contents_from_string(s, replace=replace)
    return True

def upload_file(s3bucket, key_name, filename, replace=False):
    """Returns True if data was transferred, False if it was already there."""
    # Uses HEAD to check if this key exists, or returns None
    key = s3bucket.get_key(key_name)
    if not replace and key is not None: return False
    key = s3bucket.new_key(key_name)
    key.set_contents_from_filename(filename, replace=replace)
    return True

def get_file_info(absfile, datahash):
    return json.dumps({
        'path':absfile,
        'size':osp.getsize(absfile),
        'mtime':osp.getmtime(absfile),
        'stored':time.time(),
        'data':datahash,
    }, separators=(',', ':'), sort_keys=True)

def do_backup(filename_iter, s3bucket, db):
    # Make sure current documentation exists in bucket.
    # We actually upload this whole file, to provide unambiguous info and a way to restore.
    upload_file(s3bucket, "README_%s.txt" % VERSION, __file__)
    for relfile in filename_iter:
        try:
            absfile = os.realpath(relfile) # eliminates symbolic links and does abspath()
            if osp.isdir(absfile):
                logger.info("SKIP_DIR %s" % absfile)
                continue
            if not osp.exists(absfile):
                logger.warning("DOES_NOT_EXIST file %s" % absfile)
            # If file of same name, modification date, and size is in database, we can skip it.
            mtime = osp.getmtime(absfile)
            fsize = osp.getsize(absfile)
            known_file = db.execute("SELECT * FROM files WHERE abspath = ? AND mtime = ? AND size = ?", (absfile, mtime, fsize)).fetchone() # or None
            if known_file is not None:
                logger.info("SKIP_KNOWN %s %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(known_file['updated'])), absfile))
                continue
            # Can't skip it.  Calculate the hashes!
            datahash = hash_file_contents(absfile)
            fileinfo = get_file_info(absfile, datahash)
            infohash = hash_string(fileinfo)
            indexkey = "index/%s/%s.json" % (datahash, infohash)
            datakey = "data/%s" % datahash
            # Upload the actual file contents, if needed.
            replace_index = False
            logger.debug("UPLOAD_READY %s %s" % (datakey, absfile))
            if upload_file(s3bucket, datakey, absfile):
                logger.info("UPLOAD_DONE %s %s" % (datakey, absfile))
                replace_index = True
            else:
                logger.info("UPLOAD_EXISTS %s %s" % (datakey, absfile))
            # Upload the metadata, if needed.
            logger.debug("INDEX_READY %s %s" % (indexkey, absfile))
            if upload_string(s3bucket, indexkey, fileinfo, replace=replace_index):
                logger.info("INDEX_DONE %s %s" % (indexkey, absfile))
            else:
                logger.info("INDEX_EXISTS %s %s" % (indexkey, absfile))
            # Note to self: we've now backed this file up.
            with db:
                db.execute("INSERT INTO files (abspath, mtime, size, infohash, datahash) VALUES (?, ?, ?, ?, ?)",
                    (absfile, mtime, fsize, infohash, datahash))
        except Exception, ex:
            logger.error("ERROR %s %s" % (relfile, ex))

def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument("config", metavar="CONFIG.json", nargs=1, type=argparse.FileType('r'))
    args = parser.parse_args(argv)
    config = json.load(args.config)

    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s\t%(levelname)s\t%(message)s')
    global logger
    logger = logging

    db = init_db(config['file_database'])

    s3conn = boto.connect_s3(config['aws_access_key'], config['aws_secret_key'])
    s3bucket = s3conn.create_bucket(config['aws_s3_bucket']) # just returns Bucket if exists
    if 'days_to_glacier' in config: set_lifecycle(s3bucket, config['days_to_glacier'])

    filename_iter = (line.rstrip("\r\n") for line in sys.stdin)
    do_backup(filename_iter, s3bucket)

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
<commit_msg>Lots of stupid typo bugs<commit_after>#!/usr/bin/env python
"""
This is -80 (minus80), a tool for long-term archival backup to Amazon S3 and Glacier.

If you're reading this on Amazon, this file both describes the data format and
contains code for restoring the backup to your local computer.  Keep reading.

Getting Started
===============
Sign up for Amazon Web Services (AWS) at http://aws.amazon.com/
You'll need to sign up for (at a minimum) S3, Glacier, and SNS.

Create a configuration file in JSON format that looks like this:

    {
        "aws_access_key": "XXX",
        "aws_secret_key": "XXX",
        "aws_s3_bucket": "XXX",
        "days_to_glacier": 30,
        "file_database": "~/.minus80.sqlite3",
    }

Replace the "XXX" with appropriate values from your new AWS acount.
The "days_to_glacier" is optional;  if omitted, Glacier will not be used.

Usage - Backup
==============
Storage costs can be reduced by setting up a bucket lifecycle rule that transfers
data from S3 storage to Glacier storage.  Since most of the bytes are in the data/
folder, I would suggest only archiving that folder, say, 30 days after upload.
See http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html

By default, a Glacier archive rule will be set on the bucket for you if your
config file includes a 'days_to_glacier' entry.  You can always delete or
disable it later from the AWS control panel.

Usage - Restore
===============
The restore function hasn't been written yet.  I mentioned this was for
disaster recovery, right?  See "Data Format", below.

Data Format
===========
All data is stored in an Amazon S3 bucket, using a simple content-addressable
storage scheme.  The contents of each file are stored in

    data/DATA_HASH

where DATA_HASH is the hexadecimal SHA-1 hash of the file contents.  This provides
automatic de-duplication -- if you have multiple copies of a single file, it is
only uploaded and stored once.

Metadata about each file is stored in JSON format in

    index/DATA_HASH/INFO_HASH.json

where INFO_HASH is the hexadecimal SHA-1 of the contents of the metadata JSON file.
The metadata includes a timestamp, so even if the contents of a particular file
change over time, one can recover previous versions from the backup if needed.
All the metadata for various instances of a particular piece of content are stored
under a common prefix (DATA_HASH) to make it feasible to purge all aliases to a
particular bit of content from the archive if e.g. it was included by mistake.

Once files have been backed up, this is recorded in a local SQLite database.
This is purely for efficiency -- if a file's size and mtime have not changed
since the last backup, it is assumed to already be in the archive, and is skipped.
This saves recalculating hashes for thousands of files.  However, if the local
database is deleted, the next backup attempt will verify that each file is in S3.

Philosophy
==========
Minus80 is a tool in the Unix tradition.  This means it tries to be simple and modular,
and not re-invent functionality that's available elsewhere. Thus:

- The only way to specify files is as a list of names, one per line, on stdin.
  Expected usage is to pipe the output of `find` to Minus80,
  but you could also have a manually curated backup list.
  It does not currently support null-separated output (-print0), because
  I assume you are a sane human and do not have newlines in your filenames.

- Minus80 does not have built-in bandwidth limits, because tools like
  `trickle` (http://monkey.org/~marius/pages/?page=trickle)
  and `throttled` (https://github.com/zquestz/throttled) are available.

- Minus80 does not encourage encryption, because it is designed for disaster recovery
  of files that are very important to you but of low importance to others, such as
  your personal photo library.  The risk of forgetting the encryption password
  is higher than the risk of the data being compromised in a damaging way.
  Of course, I can't stop you from using `gpg` to encrypt files before backing them up.

- The archive format is as simple as possible.  An average coder should be able to
  easily whip up a script to restore the data, given a description of the format.
  Hopefully, s/he could even reverse-engineer the format if necessary.

- Once backed up, data is never deleted.  This is not a backup tool for many small,
  frequently-changing files.  It is not for reconstructing a flawless image of your
  entire disk exactly as it was before the crash.  It is for worst-case-scenario
  recovery of your most precious digital assets if all your other backups have failed.

Minus80 was named for the -80 C freezers that scientists use for long-term cold storage.
Although it talks only to S3, the intention is to use bucket lifecycle rules to move
all data into Glacier for permanent storage.
"""
# MAKE SURE to increment this when code is updated,
# so a new copy gets stored in the archive!
VERSION = '0.1.0'

import argparse, hashlib, json, logging, os, sqlite3, sys, time
import os.path as osp
import boto

def init_db(dbpath):
    db = sqlite3.connect(osp.expanduser(dbpath))
    with db:
        db.executescript("""
            CREATE TABLE IF NOT EXISTS files (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                abspath TEXT NOT NULL,
                mtime NUMERIC NOT NULL,
                size INTEGER NOT NULL,
                infohash TEXT NOT NULL,
                datahash TEXT NOT NULL,
                updated INTEGER NOT NULL DEFAULT (STRFTIME('%s','now'))
            );
            CREATE UNIQUE INDEX IF NOT EXISTS idx_files_1 ON files (abspath, mtime, size);
            """)
    return db

def set_lifecycle(s3bucket, days_to_glacier):
    import boto.s3.lifecycle as lc
    RULE_ID = 'minus80_data_archive_rule'
    try:
        lifecycle = s3bucket.get_lifecycle_config()
    except boto.exception.S3ResponseError, ex:
        lifecycle = lc.Lifecycle()
    for rule in lifecycle:
        if rule.id == RULE_ID:
            logger.debug("HAS_LIFECYCLE not updating existing lifecycle rules on bucket")
            break
    else:
        lifecycle.append(lc.Rule(RULE_ID, 'data/', 'Enabled',
            transition=lc.Transition(days=days_to_glacier, storage_class='GLACIER')))
        s3bucket.configure_lifecycle(lifecycle)
        logger.info("SET_LIFECYCLE setting default transition-to-Glacier rule on bucket")

def hash_string(s, hashname='sha1'):
    hashfunc = hashlib.new(hashname)
    hashfunc.update(s)
    return hashfunc.hexdigest()

def hash_file_content(absfile, hashname='sha1', chunk=1024*1024):
    hashfunc = hashlib.new(hashname)
    infile = open(absfile, 'rb')
    while True:
        data = infile.read(chunk)
        if not data: break
        hashfunc.update(data)
    return hashfunc.hexdigest()

def upload_string(s3bucket, key_name, s, replace=False):
    """Returns True if data was transferred, False if it was already there."""
    # Uses HEAD to check if this key exists, or returns None
    key = s3bucket.get_key(key_name)
    if not replace and key is not None: return False
    key = s3bucket.new_key(key_name)
    key.set_contents_from_string(s, replace=replace)
    return True

def upload_file(s3bucket, key_name, filename, replace=False):
    """Returns True if data was transferred, False if it was already there."""
    # Uses HEAD to check if this key exists, or returns None
    key = s3bucket.get_key(key_name)
    if not replace and key is not None: return False
    key = s3bucket.new_key(key_name)
    key.set_contents_from_filename(filename, replace=replace)
    return True

def get_file_info(absfile, datahash):
    return json.dumps({
        'path':absfile,
        'size':osp.getsize(absfile),
        'mtime':osp.getmtime(absfile),
        'stored':time.time(),
        'data':datahash,
    }, separators=(',', ':'), sort_keys=True)

def do_backup(filename_iter, s3bucket, db):
    # Make sure current documentation exists in bucket.
    # We actually upload this whole file, to provide unambiguous info and a way to restore.
    upload_file(s3bucket, "README_%s.txt" % VERSION, __file__)
    for relfile in filename_iter:
        try:
            absfile = osp.realpath(relfile) # eliminates symbolic links and does abspath()
            if osp.isdir(absfile):
                logger.info("SKIP_DIR %s" % absfile)
                continue
            if not osp.exists(absfile):
                logger.warning("DOES_NOT_EXIST file %s" % absfile)
            # If file of same name, modification date, and size is in database, we can skip it.
            mtime = osp.getmtime(absfile)
            fsize = osp.getsize(absfile)
            known_file = db.execute("SELECT * FROM files WHERE abspath = ? AND mtime = ? AND size = ?", (absfile, mtime, fsize)).fetchone() # or None
            if known_file is not None:
                logger.info("SKIP_KNOWN %s %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(known_file['updated'])), absfile))
                continue
            # Can't skip it.  Calculate the hashes!
            datahash = hash_file_content(absfile)
            fileinfo = get_file_info(absfile, datahash)
            infohash = hash_string(fileinfo)
            indexkey = "index/%s/%s.json" % (datahash, infohash)
            datakey = "data/%s" % datahash
            # Upload the actual file contents, if needed.
            replace_index = False
            logger.debug("UPLOAD_READY %s %s" % (datakey, absfile))
            if upload_file(s3bucket, datakey, absfile):
                logger.info("UPLOAD_DONE %s %s" % (datakey, absfile))
                replace_index = True
            else:
                logger.info("UPLOAD_EXISTS %s %s" % (datakey, absfile))
            # Upload the metadata, if needed.
            logger.debug("INDEX_READY %s %s" % (indexkey, absfile))
            if upload_string(s3bucket, indexkey, fileinfo, replace=replace_index):
                logger.info("INDEX_DONE %s %s" % (indexkey, absfile))
            else:
                logger.info("INDEX_EXISTS %s %s" % (indexkey, absfile))
            # Note to self: we've now backed this file up.
            with db:
                db.execute("INSERT INTO files (abspath, mtime, size, infohash, datahash) VALUES (?, ?, ?, ?, ?)",
                    (absfile, mtime, fsize, infohash, datahash))
        except Exception, ex:
            logger.error("ERROR %s %s" % (relfile, ex))

def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument("config", metavar="CONFIG.json", type=argparse.FileType('r'))
    args = parser.parse_args(argv)
    config = json.load(args.config)

    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s\t%(levelname)s\t%(message)s')
    global logger
    logger = logging

    db = init_db(config['file_database'])

    s3conn = boto.connect_s3(config['aws_access_key'], config['aws_secret_key'])
    s3bucket = s3conn.create_bucket(config['aws_s3_bucket']) # just returns Bucket if exists
    if 'days_to_glacier' in config: set_lifecycle(s3bucket, config['days_to_glacier'])

    filename_iter = (line.rstrip("\r\n") for line in sys.stdin)
    do_backup(filename_iter, s3bucket, db)

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before># Andy Kotz final project: graphing calculator with regressions

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, TextAsset
from ggame import CircleAsset
from math import sin, cos, radians

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 1000

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0x9B30FF, 1.0)
grey = Color(0xd3d3d3, 0.7)
thinline = LineStyle(0, black)
yaxis = RectangleAsset(1, 1000, thinline, black)
xaxis = RectangleAsset(1900, 1, thinline, black)
ycursor =  RectangleAsset(1, 1000, thinline, grey)
xcursor = RectangleAsset(1900, 1, thinline, grey)
class Xcursorclass(Sprite):
    def __init__(self, position):
        super().__init__(xcursor, position)
class Ycursorclass(Sprite):
    def __init__(self, position):
        super().__init__(ycursor, position)
xcurse = Xcursorclass((0,0))
ycurse = Ycursorclass((0,0))
xaxisrulings = RectangleAsset(1, 7, thinline, black)
yaxisrulings = RectangleAsset(7, 1, thinline, black)
thinline = LineStyle(0, black)
mycircle = CircleAsset(3, thinline, blue)
mycirclebig = CircleAsset(6, thinline, blue)
Sprite (xaxis, (0, 500))
Sprite (yaxis, (950, 0))
yaxisrulingsprites = [Sprite(yaxisrulings, (947.5, y*20)) for y in range(-100, 100, 1)]
xaxisrulingsprites = [Sprite(xaxisrulings, (x*20+10, 497)) for x in range(-150, 150, 1)]

xcoordinates2 = range(-1500, 1500, 1)
xcoordinates = []
for x in xcoordinates2:
    x = x/32
    xcoordinates.append(x)

linetypelist = input("linear, quadratic, cubic, plot, function (l, q, c, p, f). Separate by commas: ")
linetypelist = linetypelist.split(",")
for linetype in linetypelist:
    if linetype == "l":
        m = float(input("linear m: "))
        b = float(input("linear b: "))
    if linetype == "q":
        a = float(input("quadratic a: "))
        b = float(input("quadratic b: "))
        c = float(input("quadratic c: "))
    if linetype == "c":
        a = float(input("cubic a: "))
        b = float(input("cubic b: "))
        c = float(input("cubic c: "))
        d = float(input("cubic d: "))
    if linetype == "f":
        newvar = 0
        function = list(input("y= "))
        goforfunction = 0
        while goforfunction <= len(function)-1:
            if function[goforfunction].isdigit():
                if function[goforfunction+1].isdigit():
                    function[goforfunction] = int(function[goforfunction])*10 + int(function[goforfunction])
                    function.remove(function[goforfunction+1])
    if linetype == "p":
        again = True
        ylistpts=[]
        xlistpts=[]
        while again == True:
            point = input("input point x,y. press q to quit, r to regress: ")
            if point == "q" or point == "r":
                again = False
            if again == True:
                point = point.split(",")
                Sprite(mycirclebig, (20*float(point[0])+950, -20*float(point[1])+500))
                xlistpts.append(float(point[0]))
                ylistpts.append(float(point[1]))
            if point == "r":
                xavg=0
                yavg=0
                for i in xlistpts:
                    xavg = xavg + i
                for b in ylistpts:
                    yavg = yavg + b
                xavg = xavg/(len(xlistpts))
                yavg = yavg/(len(ylistpts))
                slopelist = []
                goforslope = 0
                while goforslope <= len(xlistpts)-2:
                    x = xlistpts[goforslope+1]-xlistpts[goforslope]
                    y = ylistpts[goforslope+1]-ylistpts[goforslope]
                    slopelist.append(y/x)
                    goforslope += 1
                slopeavg = 0
                for h in slopelist:
                    slopeavg = slopeavg + h
                slopeavg = slopeavg/(len(slopelist))
                for x in xcoordinates:
                    yval = 20*(slopeavg*(x-xavg)+yavg)+500
                    if yval >= 0:
                        sprites = Sprite(mycircle, (20*x+950, yval))

    if linetype == "l":
        for x in xcoordinates:
            yval = 20*((-m)*x-b)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "q":
        for x in xcoordinates:
            yval = 20*(-a*x**2-b*x-c)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "c":
        for x in xcoordinates:
            yval = 20*(-a*x**3-b*x**2-c*x-d)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
coords = None
def mousePosition(event):
    global text
    global coords
    if coords != None:
        coords.destroy()
    xcurse.y = event.y-7
    ycurse.x = event.x-9
    text = TextAsset("(" + str(round((event.x-959)/20)) + "," + str(round((-(event.y-507))/20)) + ")", style = '10pt Arial')
    coords = Sprite(text, (event.x-7, event.y-22))

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
myapp.listenMouseEvent('mousemove', mousePosition)

# Andy Kotz final project: graphing calculator with regressions

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, TextAsset
from ggame import CircleAsset
from math import sin, cos, radians

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 1000

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0x9B30FF, 1.0)
grey = Color(0xd3d3d3, 0.7)
thinline = LineStyle(0, black)
yaxis = RectangleAsset(1, 1000, thinline, black)
xaxis = RectangleAsset(1900, 1, thinline, black)
ycursor =  RectangleAsset(1, 1000, thinline, grey)
xcursor = RectangleAsset(1900, 1, thinline, grey)
class Xcursorclass(Sprite):
    def __init__(self, position):
        super().__init__(xcursor, position)
class Ycursorclass(Sprite):
    def __init__(self, position):
        super().__init__(ycursor, position)
xcurse = Xcursorclass((0,0))
ycurse = Ycursorclass((0,0))
xaxisrulings = RectangleAsset(1, 7, thinline, black)
yaxisrulings = RectangleAsset(7, 1, thinline, black)
thinline = LineStyle(0, black)
mycircle = CircleAsset(3, thinline, blue)
mycirclebig = CircleAsset(6, thinline, blue)
Sprite (xaxis, (0, 500))
Sprite (yaxis, (950, 0))
yaxisrulingsprites = [Sprite(yaxisrulings, (947.5, y*20)) for y in range(-100, 100, 1)]
xaxisrulingsprites = [Sprite(xaxisrulings, (x*20+10, 497)) for x in range(-150, 150, 1)]

linetypelist = input("linear, quadratic, cubic, plot, function (l, q, c, p, f). Separate by commas: ")
linetypelist = linetypelist.split(",")
xcoordinates2 = range(-1500, 1500, 1)
xcoordinates = []
for x in xcoordinates2:
    x = x/32
    xcoordinates.append(x)
for linetype in linetypelist:
    if linetype == "l":
        m = float(input("linear m: "))
        b = float(input("linear b: "))
    if linetype == "q":
        a = float(input("quadratic a: "))
        b = float(input("quadratic b: "))
        c = float(input("quadratic c: "))
    if linetype == "c":
        a = float(input("cubic a: "))
        b = float(input("cubic b: "))
        c = float(input("cubic c: "))
        d = float(input("cubic d: "))
    if linetype == "f":
        function = list(str(input("y= ")))
        funcpos = 0
        while funcpos <= len(function)-1:

    if linetype == "p":
        again = True
        ylistpts=[]
        xlistpts=[]
        while again == True:
            point = input("input point x,y. press q to quit, r to regress: ")
            if point == "q" or point == "r":
                again = False
            if again == True:
                point = point.split(",")
                Sprite(mycirclebig, (20*float(point[0])+950, -20*float(point[1])+500))
                xlistpts.append(int(point[0]))
                ylistpts.append(int(point[1]))
                xlistpts = xlistpts.sort()
                ylistpts = ylistpts.sort()
            if point == "r":
                xavg=0
                yavg=0
                for i in xlistpts:
                    xavg = xavg + i
                for b in ylistpts:
                    yavg = yavg + b
                xavg = xavg/(len(xlistpts))
                yavg = yavg/(len(ylistpts))
                slopelist = []
                goforslope = 0
                while goforslope <= len(xlistpts)-2:
                    x = xlistpts[goforslope+1]-xlistpts[goforslope]
                    y = ylistpts[goforslope+1]-ylistpts[goforslope]
                    slopelist.append(y/x)
                    goforslope += 1
                slopeavg = 0
                for h in slopelist:
                    slopeavg = slopeavg + h
                slopeavg = -slopeavg/(len(slopelist))
                bval = slopeavg*xavg+yavg
                for x in xcoordinates:
                    yval = 20*((slopeavg)*x-bval)+500
                    if yval >= 0:
                        sprites = Sprite(mycircle, (20*x+950, yval))

    if linetype == "l":
        for x in xcoordinates:
            yval = 20*((-m)*x-b)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "q":
        for x in xcoordinates:
            yval = 20*(-a*x**2-b*x-c)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "c":
        for x in xcoordinates:
            yval = 20*(-a*x**3-b*x**2-c*x-d)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
coords = None
def mousePosition(event):
    global text
    global coords
    if coords != None:
        coords.destroy()
    xcurse.y = event.y-7
    ycurse.x = event.x-9
    text = TextAsset("(" + str(round((event.x-959)/20)) + "," + str(round((-(event.y-507))/20)) + ")", style = '10pt Arial')
    coords = Sprite(text, (event.x-7, event.y-22))

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
myapp.listenMouseEvent('mousemove', mousePosition)

<commit_msg>Updated from Brython Server: 12/16/2016 2:01:41 PM<commit_after># Andy Kotz final project: graphing calculator with regressions

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, TextAsset
from ggame import CircleAsset
from math import sin, cos, radians

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 1000

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0x9B30FF, 1.0)
grey = Color(0xd3d3d3, 0.7)
thinline = LineStyle(0, black)
yaxis = RectangleAsset(1, 1000, thinline, black)
xaxis = RectangleAsset(1900, 1, thinline, black)
ycursor =  RectangleAsset(1, 1000, thinline, grey)
xcursor = RectangleAsset(1900, 1, thinline, grey)
class Xcursorclass(Sprite):
    def __init__(self, position):
        super().__init__(xcursor, position)
class Ycursorclass(Sprite):
    def __init__(self, position):
        super().__init__(ycursor, position)
xcurse = Xcursorclass((0,0))
ycurse = Ycursorclass((0,0))
xaxisrulings = RectangleAsset(1, 7, thinline, black)
yaxisrulings = RectangleAsset(7, 1, thinline, black)
thinline = LineStyle(0, black)
mycircle = CircleAsset(3, thinline, blue)
mycirclebig = CircleAsset(6, thinline, blue)
Sprite (xaxis, (0, 500))
Sprite (yaxis, (950, 0))
yaxisrulingsprites = [Sprite(yaxisrulings, (947.5, y*20)) for y in range(-100, 100, 1)]
xaxisrulingsprites = [Sprite(xaxisrulings, (x*20+10, 497)) for x in range(-150, 150, 1)]

xcoordinates2 = range(-1500, 1500, 1)
xcoordinates = []
for x in xcoordinates2:
    x = x/32
    xcoordinates.append(x)

linetypelist = input("linear, quadratic, cubic, plot, function (l, q, c, p, f). Separate by commas: ")
linetypelist = linetypelist.split(",")
for linetype in linetypelist:
    if linetype == "l":
        m = float(input("linear m: "))
        b = float(input("linear b: "))
    if linetype == "q":
        a = float(input("quadratic a: "))
        b = float(input("quadratic b: "))
        c = float(input("quadratic c: "))
    if linetype == "c":
        a = float(input("cubic a: "))
        b = float(input("cubic b: "))
        c = float(input("cubic c: "))
        d = float(input("cubic d: "))
    if linetype == "f":
        newvar = 0
        function = list(input("y= "))
        goforfunction = 0
        while goforfunction <= len(function)-1:
            if function[goforfunction].isdigit():
                if function[goforfunction+1].isdigit():
                    function[goforfunction] = int(function[goforfunction])*10 + int(function[goforfunction])
                    function.remove(function[goforfunction+1])
    if linetype == "p":
        again = True
        ylistpts=[]
        xlistpts=[]
        while again == True:
            point = input("input point x,y. press q to quit, r to regress: ")
            if point == "q" or point == "r":
                again = False
            if again == True:
                point = point.split(",")
                Sprite(mycirclebig, (20*float(point[0])+950, -20*float(point[1])+500))
                xlistpts.append(float(point[0]))
                ylistpts.append(float(point[1]))
            if point == "r":
                xavg=0
                yavg=0
                for i in xlistpts:
                    xavg = xavg + i
                for b in ylistpts:
                    yavg = yavg + b
                xavg = xavg/(len(xlistpts))
                yavg = yavg/(len(ylistpts))
                slopelist = []
                goforslope = 0
                while goforslope <= len(xlistpts)-2:
                    x = xlistpts[goforslope+1]-xlistpts[goforslope]
                    y = ylistpts[goforslope+1]-ylistpts[goforslope]
                    slopelist.append(y/x)
                    goforslope += 1
                slopeavg = 0
                for h in slopelist:
                    slopeavg = slopeavg + h
                slopeavg = slopeavg/(len(slopelist))
                for x in xcoordinates:
                    yval = 20*(slopeavg*(x-xavg)+yavg)+500
                    if yval >= 0:
                        sprites = Sprite(mycircle, (20*x+950, yval))

    if linetype == "l":
        for x in xcoordinates:
            yval = 20*((-m)*x-b)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "q":
        for x in xcoordinates:
            yval = 20*(-a*x**2-b*x-c)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "c":
        for x in xcoordinates:
            yval = 20*(-a*x**3-b*x**2-c*x-d)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
coords = None
def mousePosition(event):
    global text
    global coords
    if coords != None:
        coords.destroy()
    xcurse.y = event.y-7
    ycurse.x = event.x-9
    text = TextAsset("(" + str(round((event.x-959)/20)) + "," + str(round((-(event.y-507))/20)) + ")", style = '10pt Arial')
    coords = Sprite(text, (event.x-7, event.y-22))

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
myapp.listenMouseEvent('mousemove', mousePosition)

# Andy Kotz final project: graphing calculator with regressions

from ggame import App, Color, LineStyle, Sprite, RectangleAsset, TextAsset
from ggame import CircleAsset
from math import sin, cos, radians

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 1000

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
purple = Color(0x9B30FF, 1.0)
grey = Color(0xd3d3d3, 0.7)
thinline = LineStyle(0, black)
yaxis = RectangleAsset(1, 1000, thinline, black)
xaxis = RectangleAsset(1900, 1, thinline, black)
ycursor =  RectangleAsset(1, 1000, thinline, grey)
xcursor = RectangleAsset(1900, 1, thinline, grey)
class Xcursorclass(Sprite):
    def __init__(self, position):
        super().__init__(xcursor, position)
class Ycursorclass(Sprite):
    def __init__(self, position):
        super().__init__(ycursor, position)
xcurse = Xcursorclass((0,0))
ycurse = Ycursorclass((0,0))
xaxisrulings = RectangleAsset(1, 7, thinline, black)
yaxisrulings = RectangleAsset(7, 1, thinline, black)
thinline = LineStyle(0, black)
mycircle = CircleAsset(3, thinline, blue)
mycirclebig = CircleAsset(6, thinline, blue)
Sprite (xaxis, (0, 500))
Sprite (yaxis, (950, 0))
yaxisrulingsprites = [Sprite(yaxisrulings, (947.5, y*20)) for y in range(-100, 100, 1)]
xaxisrulingsprites = [Sprite(xaxisrulings, (x*20+10, 497)) for x in range(-150, 150, 1)]

linetypelist = input("linear, quadratic, cubic, plot, function (l, q, c, p, f). Separate by commas: ")
linetypelist = linetypelist.split(",")
xcoordinates2 = range(-1500, 1500, 1)
xcoordinates = []
for x in xcoordinates2:
    x = x/32
    xcoordinates.append(x)
for linetype in linetypelist:
    if linetype == "l":
        m = float(input("linear m: "))
        b = float(input("linear b: "))
    if linetype == "q":
        a = float(input("quadratic a: "))
        b = float(input("quadratic b: "))
        c = float(input("quadratic c: "))
    if linetype == "c":
        a = float(input("cubic a: "))
        b = float(input("cubic b: "))
        c = float(input("cubic c: "))
        d = float(input("cubic d: "))
    if linetype == "f":
        function = list(str(input("y= ")))
        funcpos = 0
        while funcpos <= len(function)-1:
            if str(function[funcpos]).isdigit():
                if str(function[funcpos+1]).isdigit():

    if linetype == "p":
        again = True
        ylistpts=[]
        xlistpts=[]
        while again == True:
            point = input("input point x,y. press q to quit, r to regress: ")
            if point == "q" or point == "r":
                again = False
            if again == True:
                point = point.split(",")
                Sprite(mycirclebig, (20*float(point[0])+950, -20*float(point[1])+500))
                xlistpts.append(int(point[0]))
                ylistpts.append(int(point[1]))
                xlistpts = xlistpts.sort()
                ylistpts = ylistpts.sort()
            if point == "r":
                xavg=0
                yavg=0
                for i in xlistpts:
                    xavg = xavg + i
                for b in ylistpts:
                    yavg = yavg + b
                xavg = xavg/(len(xlistpts))
                yavg = yavg/(len(ylistpts))
                slopelist = []
                goforslope = 0
                while goforslope <= len(xlistpts)-2:
                    x = xlistpts[goforslope+1]-xlistpts[goforslope]
                    y = ylistpts[goforslope+1]-ylistpts[goforslope]
                    slopelist.append(y/x)
                    goforslope += 1
                slopeavg = 0
                for h in slopelist:
                    slopeavg = slopeavg + h
                slopeavg = -slopeavg/(len(slopelist))
                bval = slopeavg*xavg+yavg
                for x in xcoordinates:
                    yval = 20*((slopeavg)*x-bval)+500
                    if yval >= 0:
                        sprites = Sprite(mycircle, (20*x+950, yval))

    if linetype == "l":
        for x in xcoordinates:
            yval = 20*((-m)*x-b)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "q":
        for x in xcoordinates:
            yval = 20*(-a*x**2-b*x-c)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
    if linetype == "c":
        for x in xcoordinates:
            yval = 20*(-a*x**3-b*x**2-c*x-d)+500
            if yval >= 0:
                sprites = Sprite(mycircle, (20*x+950, yval))
coords = None
def mousePosition(event):
    global text
    global coords
    if coords != None:
        coords.destroy()
    xcurse.y = event.y-7
    ycurse.x = event.x-9
    text = TextAsset("(" + str(round((event.x-959)/20)) + "," + str(round((-(event.y-507))/20)) + ")", style = '10pt Arial')
    coords = Sprite(text, (event.x-7, event.y-22))

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
myapp.listenMouseEvent('mousemove', mousePosition)

<|endoftext|>