<commit_before># -*- coding: utf-8 -*-
"""
    jinja2.testsuite.loader
    ~~~~~~~~~~~~~~~~~~~~~~~

    Test the loaders.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD, see LICENSE for more details.
"""
import os
import sys
import tempfile
import shutil
import unittest

from jinja2.testsuite import JinjaTestCase, dict_loader, \
     package_loader, filesystem_loader, function_loader, \
     choice_loader, prefix_loader

from jinja2 import Environment, loaders
from jinja2._compat import PYPY, PY2
from jinja2.loaders import split_template_path
from jinja2.exceptions import TemplateNotFound

class LoaderTestCase(JinjaTestCase):

    def test_dict_loader(self):
        env = Environment(loader=dict_loader)
        tmpl = env.get_template('justdict.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_package_loader(self):
        env = Environment(loader=package_loader)
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_filesystem_loader(self):
        env = Environment(loader=filesystem_loader)
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        tmpl = env.get_template('foo/test.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_choice_loader(self):
        env = Environment(loader=choice_loader)
        tmpl = env.get_template('justdict.html')
        assert tmpl.render().strip() == 'FOO'
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_function_loader(self):
        env = Environment(loader=function_loader)
        tmpl = env.get_template('justfunction.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_prefix_loader(self):
        env = Environment(loader=prefix_loader)
        tmpl = env.get_template('a/test.html')
        assert tmpl.render().strip() == 'BAR'
        tmpl = env.get_template('b/justdict.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing')

    def test_caching(self):
        changed = False
        class TestLoader(loaders.BaseLoader):
            def get_source(self, environment, template):
                return u'foo', None, lambda: not changed
        env = Environment(loader=TestLoader(), cache_size=-1)
        tmpl = env.get_template('template')
        assert tmpl is env.get_template('template')
        changed = True
        assert tmpl is not env.get_template('template')
        changed = False

        env = Environment(loader=TestLoader(), cache_size=0)
        assert env.get_template('template') \
               is not env.get_template('template')

        env = Environment(loader=TestLoader(), cache_size=2)
        t1 = env.get_template('one')
        t2 = env.get_template('two')
        assert t2 is env.get_template('two')
        assert t1 is env.get_template('one')
        t3 = env.get_template('three')
        assert 'one' in env.cache
        assert 'two' not in env.cache
        assert 'three' in env.cache

    def test_dict_loader_cache_invalidates(self):
        mapping = {'foo': "one"}
        env = Environment(loader=loaders.DictLoader(mapping))
        assert env.get_template('foo').render() == "one"
        mapping['foo'] = "two"
        assert env.get_template('foo').render() == "two"

    def test_split_template_path(self):
        assert split_template_path('foo/bar') == ['foo', 'bar']
        assert split_template_path('./foo/bar') == ['foo', 'bar']
        self.assert_raises(TemplateNotFound, split_template_path, '../foo')

class ModuleLoaderTestCase(JinjaTestCase):
    archive = None

    def compile_down(self, zip='deflated', py_compile=False):
        super(ModuleLoaderTestCase, self).setup()
        log = []
        self.reg_env = Environment(loader=prefix_loader)
        if zip is not None:
            self.archive = tempfile.mkstemp(suffix='.zip')[1]
        else:
            self.archive = tempfile.mkdtemp()
        self.reg_env.compile_templates(self.archive, zip=zip,
                                       log_function=log.append,
                                       py_compile=py_compile)
        self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive))
        return ''.join(log)

    def teardown(self):
        super(ModuleLoaderTestCase, self).teardown()
        if hasattr(self, 'mod_env'):
            if os.path.isfile(self.archive):
                os.remove(self.archive)
            else:
                shutil.rmtree(self.archive)
            self.archive = None

    def test_log(self):
        log = self.compile_down()
        assert 'Compiled "a/foo/test.html" as ' \
               'tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a' in log
        assert 'Finished compiling templates' in log
        assert 'Could not compile "a/syntaxerror.html": ' \
               'Encountered unknown tag \'endif\'' in log

    def _test_common(self):
        tmpl1 = self.reg_env.get_template('a/test.html')
        tmpl2 = self.mod_env.get_template('a/test.html')
        assert tmpl1.render() == tmpl2.render()

        tmpl1 = self.reg_env.get_template('b/justdict.html')
        tmpl2 = self.mod_env.get_template('b/justdict.html')
        assert tmpl1.render() == tmpl2.render()

    def test_deflated_zip_compile(self):
        self.compile_down(zip='deflated')
        self._test_common()

    def test_stored_zip_compile(self):
        self.compile_down(zip='stored')
        self._test_common()

    def test_filesystem_compile(self):
        self.compile_down(zip=None)
        self._test_common()

    def test_weak_references(self):
        self.compile_down()
        tmpl = self.mod_env.get_template('a/test.html')
        key = loaders.ModuleLoader.get_template_key('a/test.html')
        name = self.mod_env.loader.module.__name__

        assert hasattr(self.mod_env.loader.module, key)
        assert name in sys.modules

        # unset all, ensure the module is gone from sys.modules
        self.mod_env = tmpl = None

        try:
            import gc
            gc.collect()
        except:
            pass

        assert name not in sys.modules

    # This test only makes sense on non-pypy python 2
    if PY2 and not PYPY:
        def test_byte_compilation(self):
            log = self.compile_down(py_compile=True)
            assert 'Byte-compiled "a/test.html"' in log
            tmpl1 = self.mod_env.get_template('a/test.html')
            mod = self.mod_env.loader.module. \
                tmpl_3c4ddf650c1a73df961a6d3d2ce2752f1b8fd490
            assert mod.__file__.endswith('.pyc')

    def test_choice_loader(self):
        log = self.compile_down()

        self.mod_env.loader = loaders.ChoiceLoader([
            self.mod_env.loader,
            loaders.DictLoader({'DICT_SOURCE': 'DICT_TEMPLATE'})
        ])

        tmpl1 = self.mod_env.get_template('a/test.html')
        self.assert_equal(tmpl1.render(), 'BAR')
        tmpl2 = self.mod_env.get_template('DICT_SOURCE')
        self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE')

    def test_prefix_loader(self):
        log = self.compile_down()

        self.mod_env.loader = loaders.PrefixLoader({
            'MOD':      self.mod_env.loader,
            'DICT':     loaders.DictLoader({'test.html': 'DICT_TEMPLATE'})
        })

        tmpl1 = self.mod_env.get_template('MOD/a/test.html')
        self.assert_equal(tmpl1.render(), 'BAR')
        tmpl2 = self.mod_env.get_template('DICT/test.html')
        self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE')

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(LoaderTestCase))
    suite.addTest(unittest.makeSuite(ModuleLoaderTestCase))
    return suite
<commit_msg>close temp file<commit_after># -*- coding: utf-8 -*-
"""
    jinja2.testsuite.loader
    ~~~~~~~~~~~~~~~~~~~~~~~

    Test the loaders.

    :copyright: (c) 2010 by the Jinja Team.
    :license: BSD, see LICENSE for more details.
"""
import os
import sys
import tempfile
import shutil
import unittest

from jinja2.testsuite import JinjaTestCase, dict_loader, \
     package_loader, filesystem_loader, function_loader, \
     choice_loader, prefix_loader

from jinja2 import Environment, loaders
from jinja2._compat import PYPY, PY2
from jinja2.loaders import split_template_path
from jinja2.exceptions import TemplateNotFound

class LoaderTestCase(JinjaTestCase):

    def test_dict_loader(self):
        env = Environment(loader=dict_loader)
        tmpl = env.get_template('justdict.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_package_loader(self):
        env = Environment(loader=package_loader)
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_filesystem_loader(self):
        env = Environment(loader=filesystem_loader)
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        tmpl = env.get_template('foo/test.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_choice_loader(self):
        env = Environment(loader=choice_loader)
        tmpl = env.get_template('justdict.html')
        assert tmpl.render().strip() == 'FOO'
        tmpl = env.get_template('test.html')
        assert tmpl.render().strip() == 'BAR'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_function_loader(self):
        env = Environment(loader=function_loader)
        tmpl = env.get_template('justfunction.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing.html')

    def test_prefix_loader(self):
        env = Environment(loader=prefix_loader)
        tmpl = env.get_template('a/test.html')
        assert tmpl.render().strip() == 'BAR'
        tmpl = env.get_template('b/justdict.html')
        assert tmpl.render().strip() == 'FOO'
        self.assert_raises(TemplateNotFound, env.get_template, 'missing')

    def test_caching(self):
        changed = False
        class TestLoader(loaders.BaseLoader):
            def get_source(self, environment, template):
                return u'foo', None, lambda: not changed
        env = Environment(loader=TestLoader(), cache_size=-1)
        tmpl = env.get_template('template')
        assert tmpl is env.get_template('template')
        changed = True
        assert tmpl is not env.get_template('template')
        changed = False

        env = Environment(loader=TestLoader(), cache_size=0)
        assert env.get_template('template') \
               is not env.get_template('template')

        env = Environment(loader=TestLoader(), cache_size=2)
        t1 = env.get_template('one')
        t2 = env.get_template('two')
        assert t2 is env.get_template('two')
        assert t1 is env.get_template('one')
        t3 = env.get_template('three')
        assert 'one' in env.cache
        assert 'two' not in env.cache
        assert 'three' in env.cache

    def test_dict_loader_cache_invalidates(self):
        mapping = {'foo': "one"}
        env = Environment(loader=loaders.DictLoader(mapping))
        assert env.get_template('foo').render() == "one"
        mapping['foo'] = "two"
        assert env.get_template('foo').render() == "two"

    def test_split_template_path(self):
        assert split_template_path('foo/bar') == ['foo', 'bar']
        assert split_template_path('./foo/bar') == ['foo', 'bar']
        self.assert_raises(TemplateNotFound, split_template_path, '../foo')

class ModuleLoaderTestCase(JinjaTestCase):
    archive = None

    def compile_down(self, zip='deflated', py_compile=False):
        super(ModuleLoaderTestCase, self).setup()
        log = []
        self.reg_env = Environment(loader=prefix_loader)
        if zip is not None:
            fd, self.archive = tempfile.mkstemp(suffix='.zip')
            os.close(fd)
        else:
            self.archive = tempfile.mkdtemp()
        self.reg_env.compile_templates(self.archive, zip=zip,
                                       log_function=log.append,
                                       py_compile=py_compile)
        self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive))
        return ''.join(log)

    def teardown(self):
        super(ModuleLoaderTestCase, self).teardown()
        if hasattr(self, 'mod_env'):
            if os.path.isfile(self.archive):
                os.remove(self.archive)
            else:
                shutil.rmtree(self.archive)
            self.archive = None

    def test_log(self):
        log = self.compile_down()
        assert 'Compiled "a/foo/test.html" as ' \
               'tmpl_a790caf9d669e39ea4d280d597ec891c4ef0404a' in log
        assert 'Finished compiling templates' in log
        assert 'Could not compile "a/syntaxerror.html": ' \
               'Encountered unknown tag \'endif\'' in log

    def _test_common(self):
        tmpl1 = self.reg_env.get_template('a/test.html')
        tmpl2 = self.mod_env.get_template('a/test.html')
        assert tmpl1.render() == tmpl2.render()

        tmpl1 = self.reg_env.get_template('b/justdict.html')
        tmpl2 = self.mod_env.get_template('b/justdict.html')
        assert tmpl1.render() == tmpl2.render()

    def test_deflated_zip_compile(self):
        self.compile_down(zip='deflated')
        self._test_common()

    def test_stored_zip_compile(self):
        self.compile_down(zip='stored')
        self._test_common()

    def test_filesystem_compile(self):
        self.compile_down(zip=None)
        self._test_common()

    def test_weak_references(self):
        self.compile_down()
        tmpl = self.mod_env.get_template('a/test.html')
        key = loaders.ModuleLoader.get_template_key('a/test.html')
        name = self.mod_env.loader.module.__name__

        assert hasattr(self.mod_env.loader.module, key)
        assert name in sys.modules

        # unset all, ensure the module is gone from sys.modules
        self.mod_env = tmpl = None

        try:
            import gc
            gc.collect()
        except:
            pass

        assert name not in sys.modules

    # This test only makes sense on non-pypy python 2
    if PY2 and not PYPY:
        def test_byte_compilation(self):
            log = self.compile_down(py_compile=True)
            assert 'Byte-compiled "a/test.html"' in log
            tmpl1 = self.mod_env.get_template('a/test.html')
            mod = self.mod_env.loader.module. \
                tmpl_3c4ddf650c1a73df961a6d3d2ce2752f1b8fd490
            assert mod.__file__.endswith('.pyc')

    def test_choice_loader(self):
        log = self.compile_down()

        self.mod_env.loader = loaders.ChoiceLoader([
            self.mod_env.loader,
            loaders.DictLoader({'DICT_SOURCE': 'DICT_TEMPLATE'})
        ])

        tmpl1 = self.mod_env.get_template('a/test.html')
        self.assert_equal(tmpl1.render(), 'BAR')
        tmpl2 = self.mod_env.get_template('DICT_SOURCE')
        self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE')

    def test_prefix_loader(self):
        log = self.compile_down()

        self.mod_env.loader = loaders.PrefixLoader({
            'MOD':      self.mod_env.loader,
            'DICT':     loaders.DictLoader({'test.html': 'DICT_TEMPLATE'})
        })

        tmpl1 = self.mod_env.get_template('MOD/a/test.html')
        self.assert_equal(tmpl1.render(), 'BAR')
        tmpl2 = self.mod_env.get_template('DICT/test.html')
        self.assert_equal(tmpl2.render(), 'DICT_TEMPLATE')

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(LoaderTestCase))
    suite.addTest(unittest.makeSuite(ModuleLoaderTestCase))
    return suite
<|endoftext|>


<commit_before>from .orm import create, attach, create_schema, touch, CreateView, engine_dispose
from .command_hash import command_hash
from sqlalchemy import MetaData, Table, and_, or_, func, select, exists
import os.path
import re
from docopt import DocoptExit

def is_subpath(column, subpath):
    subpathdir = subpath
    if subpath.find('/') != -1:
        if not subpath.endswith('/'):
            subpathdir = subpath + '/'
    elif subpath.find('\\') != -1:
        if not subpath.endswith('\\'):
            subpathdir = subpath + '\\'

    if subpath != subpathdir:
        return or_(
            column == subpath,
            func.SUBSTR(column, 1, len(subpathdir)) == subpathdir
        )
    else:
        return func.SUBSTR(column, 1, len(subpathdir)) == subpathdir

def attach_side(engine, side, dbpath, update, subpath):
    db = side + 'db'
    rw = side + 'rw'
    ro = side + 'ro'

    if dbpath:
        exists = os.path.exists(dbpath)

        if update or not exists:

            if not exists:
                touch(dbpath)

            attach(engine, rw, dbpath)

            rwFiles = Table('Files', MetaData(schema=rw), autoload=True, autoload_with=engine)

            sel = rwFiles.select()

            if subpath and exists:
                # rw = actual database
                # ro = view of lhsrw, restricted to LHSPATH
                sel = sel.where(is_subpath(rwFiles.c.path, subpath))
            else:
                # rw = actual database, created from scratch if necessary
                # ro = view of lhsrw
                pass

            if subpath:
                command_hash({'INPUTS': [subpath], '--quick': False, '--full': False, '--none': True }, engine=engine, schema=rw)

            engine.execute(CreateView(ro, sel))
            roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

            return rwFiles, roFiles
        else:
            attach(engine, db, dbpath)
            dbFiles = Table('Files', MetaData(schema=db), autoload=True, autoload_with=engine)

            sel = dbFiles.select()

            if subpath:
                # db = actual database
                # rw = None
                # ro = LHSPATH restricted view of lhsdb
                sel = sel.where(is_subpath(dbFiles.c.path, subpath))
            else:
                # db = actual database
                # rw = None
                # ro = view of lhsdb
                pass

            engine.execute(CreateView(ro, sel))
            roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

            return None, roFiles
    else:
        # rw = fresh memory database
        # ro = view f lhsrw

        attach(engine, rw)
        create_schema(engine, rw)
        rwFiles = Table('Files', MetaData(schema=rw), autoload=True, autoload_with=engine)

        engine.execute(CreateView(ro, rwFiles.select()))
        roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

        if subpath:
            command_hash({'INPUTS': [subpath], '--quick': False, '--full': False, '--none': True}, engine=engine, schema=rw)

        return rwFiles, roFiles

def command_comp(arguments):

    if not any(arguments[name] for name in ('--full', '--quick', '--none')):
        arguments['--none'] = True

    if not arguments['--none']:
        arguments['--size'] = True

    def match(sel, lhs, rhs, complete=False):
        if arguments['--size']:
            sel = sel.where(lhs.c.size == rhs.c.size)

        if arguments['--time']:
            sel = sel.where(lhs.c.time == rhs.c.time)

        if arguments['--extension']:
            sel = sel.where(lhs.c.extension == rhs.c.extension)

        if arguments['--basename']:
            sel = sel.where(lhs.c.basename == rhs.c.basename)

        if complete and arguments['--quick']:
            sel = sel.where(and_(
                lhs.c.hash_quick == rhs.c.hash_quick,
                lhs.c.hash_quick != None
            ))

        if complete and arguments['--full']:
            sel = sel.where(and_(
                lhs.c.hash_total == rhs.c.hash_total,
                lhs.c.hash_total != None
            ))

        # Not looking for the actual same file
        sel = sel.where(lhs.c.path != rhs.c.path)

        return sel

    def match_group(sel, lhs, rhs):
        group = []
        if arguments['--size']:
            group.append(lhs.c.size)

        if arguments['--time']:
            group.append(lhs.c.time)

        if arguments['--extension']:
            group.append(lhs.c.extension)

        if arguments['--basename']:
            group.append(lhs.c.basename)

        if arguments['--quick']:
            group.append(lhs.c.hash_quick)

        if arguments['--full']:
            group.append(lhs.c.hash_total)

        if len(group) != 0:
            sel = sel.group_by(group)

        return sel

    # ToDo: Add tests for each combination
    args = set(re.findall(r'\{[A-Z]+\}', arg) for arg in arguments['COMMAND'])
    if args == {'{LHS}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHS}', '{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS'), rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path, rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHS}', '{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS'), func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.group_by([lhs.c.path])
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path])
            return sel
    elif args == {'{LHSGROUP}', '{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP'), rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.group_by([rhs.c.path])
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSGROUP}', '{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP'), func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = match_group(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSONLY}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS')])
            sel = sel.where(not exists(
                match(select([rhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSONLYGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP')])
            sel = sel.where(not exists(
                match(select([rhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([lhs.c.path])
            return sel
    elif args == {'{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            return sel
    elif args == {'{RHSONLY}'}:
        def get_sel(lhs, rhs):
            sel = select([rhs.c.path.label('RHS')])
            sel = sel.where(not exists(
                match(select([lhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{RHSONLYGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(rhs.c.path).label('LHSGROUP')])
            sel = sel.where(not exists(
                match(select([lhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([rhs.c.path])
            return sel
    elif args == {'{DUPE}'}:
        # ToDo: Work out how to handle dupe/unique
        raise DocoptExit('DUPE not implemented yet')
    elif args == {'{DUPEGROUP}'}:
        raise DocoptExit('DUPEGROUP not implemented yet')
    elif args == {'{UNIQUE}'}:
        raise DocoptExit('UNIQUE not implemented yet')
    elif args == {'{UNIQUEGROUP}'}:
        raise DocoptExit('UNIQUEGROUP not implemented yet')
    else:
        raise DocoptExit('COMMAND does not contain a valid combination of special arguments')

    engine = create(None)
    with engine_dispose(engine):
        lhsrwFiles, lhsroFiles = attach_side(engine, 'lhs', arguments['--lhs-db'], arguments['--lhs-update'], arguments['--lhs-path'])
        rhsrwFiles, rhsroFiles = attach_side(engine, 'rhs', arguments['--rhs-db'], arguments['--rhs-update'], arguments['--rhs-path'])

        if not (lhsrwFiles is None and rhsrwFiles is None) and not arguments['--none']:
            # Do a preliminary comparison
            lhssel = match(select([lhsroFiles.c.path]), lhsroFiles, rhsroFiles)
            rhssel = match(select([rhsroFiles.c.path]), lhsroFiles, rhsroFiles)

            # Update rw table
            conn = engine.connect()
            try:
                for result in conn.execute(lhssel):
                    # ToDo: Update hash (and metadata?) for lhsrw
                    pass
                for result in conn.execute(rhssel):
                    # ToDo: Update hash (and metadata?) for rhsrw
                    pass
            finally:
                conn.close()

        # Do the full comparison

        sel = get_sel(lhsrwFiles or lhsroFiles, rhsrwFiles or rhsroFiles)
        for result in conn.execute(sel):
            cmd = [re.sub(r'\{([A-Z]+)\}', (lambda match: result[match.group(1)]), arg) for arg in arguments['COMMAND']]
            print(cmd)
            #ToDo: Execute command
<commit_msg>normalize path inputs and simplify check for pre-comparison requirement<commit_after>from .orm import create, attach, create_schema, touch, CreateView, engine_dispose
from .command_hash import command_hash
from sqlalchemy import MetaData, Table, and_, or_, func, select, exists
import os.path
import re
from docopt import DocoptExit

def is_subpath(column, subpath):
    subpathdir = subpath
    if subpath.find('/') != -1:
        if not subpath.endswith('/'):
            subpathdir = subpath + '/'
    elif subpath.find('\\') != -1:
        if not subpath.endswith('\\'):
            subpathdir = subpath + '\\'

    if subpath != subpathdir:
        return or_(
            column == subpath,
            func.SUBSTR(column, 1, len(subpathdir)) == subpathdir
        )
    else:
        return func.SUBSTR(column, 1, len(subpathdir)) == subpathdir

def attach_side(engine, side, dbpath, update, subpath):
    db = side + 'db'
    rw = side + 'rw'
    ro = side + 'ro'

    if dbpath:
        exists = os.path.exists(dbpath)

        if update or not exists:

            if not exists:
                touch(dbpath)

            attach(engine, rw, dbpath)

            rwFiles = Table('Files', MetaData(schema=rw), autoload=True, autoload_with=engine)

            sel = rwFiles.select()

            if subpath and exists:
                # rw = actual database
                # ro = view of lhsrw, restricted to LHSPATH
                sel = sel.where(is_subpath(rwFiles.c.path, subpath))
            else:
                # rw = actual database, created from scratch if necessary
                # ro = view of lhsrw
                pass

            if subpath:
                command_hash({'INPUTS': [subpath], '--quick': False, '--full': False, '--none': True }, engine=engine, schema=rw)

            engine.execute(CreateView(ro, sel))
            roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

            return rwFiles, roFiles
        else:
            attach(engine, db, dbpath)
            dbFiles = Table('Files', MetaData(schema=db), autoload=True, autoload_with=engine)

            sel = dbFiles.select()

            if subpath:
                # db = actual database
                # rw = None
                # ro = LHSPATH restricted view of lhsdb
                sel = sel.where(is_subpath(dbFiles.c.path, subpath))
            else:
                # db = actual database
                # rw = None
                # ro = view of lhsdb
                pass

            engine.execute(CreateView(ro, sel))
            roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

            return None, roFiles
    else:
        # rw = fresh memory database
        # ro = view f lhsrw

        attach(engine, rw)
        create_schema(engine, rw)
        rwFiles = Table('Files', MetaData(schema=rw), autoload=True, autoload_with=engine)

        engine.execute(CreateView(ro, rwFiles.select()))
        roFiles = Table(ro, MetaData(), autoload=True, autoload_with=engine)

        if subpath:
            command_hash({'INPUTS': [subpath], '--quick': False, '--full': False, '--none': True}, engine=engine, schema=rw)

        return rwFiles, roFiles

def command_comp(arguments):

    if not any(arguments[name] for name in ('--full', '--quick', '--none')):
        arguments['--none'] = True

    if not arguments['--none']:
        arguments['--size'] = True

    if arguments['--lhs-update'] and arguments['--lhs-path']:
        arguments['--lhs-path'] = os.path.realpath(arguments['--lhs-path'])

    if arguments['--rhs-update'] and arguments['--rhs-path']:
        arguments['--rhs-path'] = os.path.realpath(arguments['--rhs-path'])

    def match(sel, lhs, rhs, complete=False):
        if arguments['--size']:
            sel = sel.where(lhs.c.size == rhs.c.size)

        if arguments['--time']:
            sel = sel.where(lhs.c.time == rhs.c.time)

        if arguments['--extension']:
            sel = sel.where(lhs.c.extension == rhs.c.extension)

        if arguments['--basename']:
            sel = sel.where(lhs.c.basename == rhs.c.basename)

        if complete and arguments['--quick']:
            sel = sel.where(and_(
                lhs.c.hash_quick == rhs.c.hash_quick,
                lhs.c.hash_quick != None
            ))

        if complete and arguments['--full']:
            sel = sel.where(and_(
                lhs.c.hash_total == rhs.c.hash_total,
                lhs.c.hash_total != None
            ))

        # Not looking for the actual same file
        sel = sel.where(lhs.c.path != rhs.c.path)

        return sel

    def match_group(sel, lhs, rhs):
        group = []
        if arguments['--size']:
            group.append(lhs.c.size)

        if arguments['--time']:
            group.append(lhs.c.time)

        if arguments['--extension']:
            group.append(lhs.c.extension)

        if arguments['--basename']:
            group.append(lhs.c.basename)

        if arguments['--quick']:
            group.append(lhs.c.hash_quick)

        if arguments['--full']:
            group.append(lhs.c.hash_total)

        if len(group) != 0:
            sel = sel.group_by(group)

        return sel

    # ToDo: Add tests for each combination
    args = set(re.findall(r'\{[A-Z]+\}', arg) for arg in arguments['COMMAND'])
    if args == {'{LHS}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHS}', '{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS'), rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path, rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHS}', '{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS'), func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.group_by([lhs.c.path])
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([lhs.c.path])
            return sel
    elif args == {'{LHSGROUP}', '{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP'), rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.group_by([rhs.c.path])
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSGROUP}', '{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP'), func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = match_group(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSONLY}'}:
        def get_sel(lhs, rhs):
            sel = select([lhs.c.path.label('LHS')])
            sel = sel.where(not exists(
                match(select([rhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([lhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{LHSONLYGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(lhs.c.path).label('LHSGROUP')])
            sel = sel.where(not exists(
                match(select([rhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([lhs.c.path])
            return sel
    elif args == {'{RHS}'}:
        def get_sel(lhs, rhs):
            sel = select([rhs.c.path.label('RHS')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{RHSGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(rhs.c.path).label('RHSGROUP')])
            sel = match(sel, lhs, rhs)
            sel = sel.order_by([rhs.c.path])
            return sel
    elif args == {'{RHSONLY}'}:
        def get_sel(lhs, rhs):
            sel = select([rhs.c.path.label('RHS')])
            sel = sel.where(not exists(
                match(select([lhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([rhs.c.path])
            sel = sel.distinct()
            return sel
    elif args == {'{RHSONLYGROUP}'}:
        def get_sel(lhs, rhs):
            sel = select([func.group_concat(rhs.c.path).label('LHSGROUP')])
            sel = sel.where(not exists(
                match(select([lhs.c.path]), lhs, rhs)
            ))
            sel = sel.order_by([rhs.c.path])
            return sel
    elif args == {'{DUPE}'}:
        # ToDo: Work out how to handle dupe/unique
        raise DocoptExit('DUPE not implemented yet')
    elif args == {'{DUPEGROUP}'}:
        raise DocoptExit('DUPEGROUP not implemented yet')
    elif args == {'{UNIQUE}'}:
        raise DocoptExit('UNIQUE not implemented yet')
    elif args == {'{UNIQUEGROUP}'}:
        raise DocoptExit('UNIQUEGROUP not implemented yet')
    else:
        raise DocoptExit('COMMAND does not contain a valid combination of special arguments')

    engine = create(None)
    with engine_dispose(engine):
        lhsrwFiles, lhsroFiles = attach_side(engine, 'lhs', arguments['--lhs-db'], arguments['--lhs-update'], arguments['--lhs-path'])
        rhsrwFiles, rhsroFiles = attach_side(engine, 'rhs', arguments['--rhs-db'], arguments['--rhs-update'], arguments['--rhs-path'])

        if not arguments['--none'] and (lhsrwFiles != None or rhsrwFiles != None):
            # Do a preliminary comparison
            lhssel = match(select([lhsroFiles.c.path]), lhsroFiles, rhsroFiles)
            rhssel = match(select([rhsroFiles.c.path]), lhsroFiles, rhsroFiles)

            # Update rw table
            conn = engine.connect()
            try:
                for result in conn.execute(lhssel):
                    # ToDo: Update hash (and metadata?) for lhsrw
                    pass
                for result in conn.execute(rhssel):
                    # ToDo: Update hash (and metadata?) for rhsrw
                    pass
            finally:
                conn.close()

        # Do the full comparison

        sel = get_sel(lhsrwFiles or lhsroFiles, rhsrwFiles or rhsroFiles)
        for result in conn.execute(sel):
            cmd = [re.sub(r'\{([A-Z]+)\}', (lambda match: result[match.group(1)]), arg) for arg in arguments['COMMAND']]
            print(cmd)
            #ToDo: Execute command
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#=======================================================================
#
# sha1.py
# ---------
# Simple, pure Python model of the SHA-1 function. Used as a
# reference for the HW implementation. The code follows the structure
# of the HW implementation as much as possible.
#
#
# Author: Joachim Strömbergson
# (c) 2014 Secworks Sweden AB
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#=======================================================================

#-------------------------------------------------------------------
# Python module imports.
#-------------------------------------------------------------------
import sys

#-------------------------------------------------------------------
# Constants.
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# ChaCha()
#-------------------------------------------------------------------
class SHA1():
    def __init__(self, verbose = 0):
        self.verbose = verbose
        self.H = [0] * 5
        self.T = 0
        self.a = 0
        self.b = 0
        self.c = 0
        self.d = 0
        self.e = 0
        self.W = [0] * 80
        self.k = 0

    def init(self):
        self.H = [0x67452301, 0xefcdab89, 0x98badcfe,
                  0x10325476, 0xc3d2e1f0]

    def next(self, block):
        self._W_schedule(block)
        self._copy_digest()
        for i in range(80):
            self._sha1_round(i)
        self._update_digest()

    def get_digest(self):
        return self.H

    def _copy_digest(self):
        self.a = self.H[0]
        self.b = self.H[1]
        self.c = self.H[2]
        self.d = self.H[3]
        self.e = self.H[4]

    def _update_digest(self):
        self.H[0] = (self.H[0] + self.a) & 0xffffffff
        self.H[1] = (self.H[1] + self.b) & 0xffffffff
        self.H[2] = (self.H[2] + self.c) & 0xffffffff
        self.H[3] = (self.H[3] + self.d) & 0xffffffff
        self.H[4] = (self.H[4] + self.e) & 0xffffffff

    def _sha1_round(self, round):
        if round <= 19:
            self.k = 0x5a827999
            self.f = self._Ch(self.b, self.c, self.d)

        elif 20 <= round <= 39:
            self.k = 0x6ed9eba1
            self.f = self._Parity(self.b, self.c, self.d)

        elif 40 <= round <= 59:
            self.k = 0x8f1bbcdc
            self.f = self._Maj(self.b, self.c, self.d)

        elif 60 <= round <= 79:
            self.k = 0xca62c1d6
            self.f = self._Parity(self.b, self.c, self.d)

        if self.verbose:
            print("Round %0d" % round)
            print("Round input values:")
            print("a = 0x%08x, b = 0x%08x, c = 0x%08x" % (self.a, self.b, self.c))
            print("d = 0x%08x, e = 0x%08x" % (self.d, self.e))
            print("f = 0x%08x, k = 0x%08x, w = 0x%08x" % (self.f, self.k, self.W[round]))

        self.T = (self._rotl32(self.a, 5) + self.f + self.e + self.k + self.W[round]) & 0xffffffff
        self.e = self.d
        self.d = self.c
        self.c = self._rotl32(self.b, 30)
        self.b = self.a
        self.a = self.T

        if self.verbose:
            print("Round output values:")
            print("a = 0x%08x, b = 0x%08x, c = 0x%08x" % (self.a, self.b, self.c))
            print("d = 0x%08x, e = 0x%08x" % (self.d, self.e))
            print("")

    def _W_schedule(self, block):
        # Expand the block into 80 words before round operations.
        for i in range(80):
            if (i < 16):
                self.W[i] = block[i]
            else:
                self.W[i] = self._rotl32((self.W[(i - 3)] ^ self.W[(i - 8)] ^
                                          self.W[(i - 14)] ^ self.W[(i - 16)]), 1)
        if (self.verbose):
            print("W after schedule:")
            for i in range(80):
                print("W[%02d] = 0x%08x" % (i, self.W[i]))
            print("")

    def _Ch(self, x, y, z):
        return (x & y) ^ (~x & z)

    def _Maj(self, x, y, z):
        return (x & y) ^ (x & z) ^ (y & z)

    def _Parity(self, x, y, z):
        return (x ^ y ^ z)

    def _rotl32(self, n, r):
        return ((n << r) | (n >> (32 - r))) & 0xffffffff

def compare_digests(digest, expected):
    if (digest != expected):
        print("Error:")
        print("Got:")
        print(digest)
        print("Expected:")
        print(expected)
    else:
        print("Test case ok.")

#-------------------------------------------------------------------
# main()
#
# If executed tests the ChaCha class using known test vectors.
#-------------------------------------------------------------------
def main():
    print("Testing the SHA-1 Python model.")
    print("-------------------------------")
    print

    my_sha1 = SHA1(verbose=1);

    # TC1: NIST testcase with message "abc"
    TC1_block = [0x61626380, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000018]

    TC1_expected = [0xa9993e36, 0x4706816a, 0xba3e2571,
                    0x7850c26c, 0x9cd0d89d]

    my_sha1.init()
    my_sha1.next(TC1_block)
    my_digest = my_sha1.get_digest()
    compare_digests(my_digest, TC1_expected)

#-------------------------------------------------------------------
# __name__
# Python thingy which allows the file to be run standalone as
# well as parsed from within a Python interpreter.
#-------------------------------------------------------------------
if __name__=="__main__":
    # Run the main function.
    sys.exit(main())

#=======================================================================
# EOF sha1.py
#=======================================================================
<commit_msg>Adding structure for two block and multiple block message tests.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#=======================================================================
#
# sha1.py
# ---------
# Simple, pure Python model of the SHA-1 function. Used as a
# reference for the HW implementation. The code follows the structure
# of the HW implementation as much as possible.
#
#
# Author: Joachim Strömbergson
# (c) 2014 Secworks Sweden AB
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#=======================================================================

#-------------------------------------------------------------------
# Python module imports.
#-------------------------------------------------------------------
import sys

#-------------------------------------------------------------------
# Constants.
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# ChaCha()
#-------------------------------------------------------------------
class SHA1():
    def __init__(self, verbose = 0):
        self.verbose = verbose
        self.H = [0] * 5
        self.T = 0
        self.a = 0
        self.b = 0
        self.c = 0
        self.d = 0
        self.e = 0
        self.W = [0] * 80
        self.k = 0

    def init(self):
        self.H = [0x67452301, 0xefcdab89, 0x98badcfe,
                  0x10325476, 0xc3d2e1f0]

    def next(self, block):
        self._W_schedule(block)
        self._copy_digest()
        for i in range(80):
            self._sha1_round(i)
        self._update_digest()

    def get_digest(self):
        return self.H

    def _copy_digest(self):
        self.a = self.H[0]
        self.b = self.H[1]
        self.c = self.H[2]
        self.d = self.H[3]
        self.e = self.H[4]

    def _update_digest(self):
        self.H[0] = (self.H[0] + self.a) & 0xffffffff
        self.H[1] = (self.H[1] + self.b) & 0xffffffff
        self.H[2] = (self.H[2] + self.c) & 0xffffffff
        self.H[3] = (self.H[3] + self.d) & 0xffffffff
        self.H[4] = (self.H[4] + self.e) & 0xffffffff

    def _sha1_round(self, round):
        if round <= 19:
            self.k = 0x5a827999
            self.f = self._Ch(self.b, self.c, self.d)

        elif 20 <= round <= 39:
            self.k = 0x6ed9eba1
            self.f = self._Parity(self.b, self.c, self.d)

        elif 40 <= round <= 59:
            self.k = 0x8f1bbcdc
            self.f = self._Maj(self.b, self.c, self.d)

        elif 60 <= round <= 79:
            self.k = 0xca62c1d6
            self.f = self._Parity(self.b, self.c, self.d)

        if self.verbose:
            print("Round %0d" % round)
            print("Round input values:")
            print("a = 0x%08x, b = 0x%08x, c = 0x%08x" % (self.a, self.b, self.c))
            print("d = 0x%08x, e = 0x%08x" % (self.d, self.e))
            print("f = 0x%08x, k = 0x%08x, w = 0x%08x" % (self.f, self.k, self.W[round]))

        self.T = (self._rotl32(self.a, 5) + self.f + self.e + self.k + self.W[round]) & 0xffffffff
        self.e = self.d
        self.d = self.c
        self.c = self._rotl32(self.b, 30)
        self.b = self.a
        self.a = self.T

        if self.verbose:
            print("Round output values:")
            print("a = 0x%08x, b = 0x%08x, c = 0x%08x" % (self.a, self.b, self.c))
            print("d = 0x%08x, e = 0x%08x" % (self.d, self.e))
            print("")

    def _W_schedule(self, block):
        # Expand the block into 80 words before round operations.
        for i in range(80):
            if (i < 16):
                self.W[i] = block[i]
            else:
                self.W[i] = self._rotl32((self.W[(i - 3)] ^ self.W[(i - 8)] ^
                                          self.W[(i - 14)] ^ self.W[(i - 16)]), 1)
        if (self.verbose):
            print("W after schedule:")
            for i in range(80):
                print("W[%02d] = 0x%08x" % (i, self.W[i]))
            print("")

    def _Ch(self, x, y, z):
        return (x & y) ^ (~x & z)

    def _Maj(self, x, y, z):
        return (x & y) ^ (x & z) ^ (y & z)

    def _Parity(self, x, y, z):
        return (x ^ y ^ z)

    def _rotl32(self, n, r):
        return ((n << r) | (n >> (32 - r))) & 0xffffffff

def compare_digests(digest, expected):
    if (digest != expected):
        print("Error:")
        print("Got:")
        print(digest)
        print("Expected:")
        print(expected)
    else:
        print("Test case ok.")

#-------------------------------------------------------------------
# main()
#
# If executed tests the ChaCha class using known test vectors.
#-------------------------------------------------------------------
def main():
    print("Testing the SHA-1 Python model.")
    print("-------------------------------")
    print

    my_sha1 = SHA1(verbose=1);

    # TC1: NIST testcase with message "abc"
    TC1_block = [0x61626380, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000000,
                 0x00000000, 0x00000000, 0x00000000, 0x00000018]

    TC1_expected = [0xa9993e36, 0x4706816a, 0xba3e2571,
                    0x7850c26c, 0x9cd0d89d]
    my_sha1.init()
    my_sha1.next(TC1_block)
    my_digest = my_sha1.get_digest()
    compare_digests(my_digest, TC1_expected)

    # TC2: NIST testcase with message two block message.
    TC2_1_block = [0x61626380, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000018]

    TC2_2_block = [0x61626380, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000000,
                   0x00000000, 0x00000000, 0x00000000, 0x00000018]

    TC2_1_expected = [0xa9993e36, 0x4706816a, 0xba3e2571,
                      0x7850c26c, 0x9cd0d89d]
    TC2_2_expected = [0xa9993e36, 0x4706816a, 0xba3e2571,
                      0x7850c26c, 0x9cd0d89d]

    my_sha1.init()
    my_sha1.next(TC2_1_block)
    my_digest = my_sha1.get_digest()
    compare_digests(my_digest, TC2_1_expected)
    my_sha1.next(TC2_2_block)
    my_digest = my_sha1.get_digest()
    compare_digests(my_digest, TC2_2_expected)

    # TC3: Huge message with n blocks
    TC3_block = [0xaa55aa55, 0xdeadbeef, 0x55aa55aa, 0xf00ff00f,
                 0xaa55aa55, 0xdeadbeef, 0x55aa55aa, 0xf00ff00f,
                 0xaa55aa55, 0xdeadbeef, 0x55aa55aa, 0xf00ff00f,
                 0xaa55aa55, 0xdeadbeef, 0x55aa55aa, 0xf00ff00f]

    TC3_expected = [0xa9993e36, 0x4706816a, 0xba3e2571,
                    0x7850c26c, 0x9cd0d89d]
    n = 10000
    my_sha1.init()
    for i in range(n):
        my_sha1.next(TC3_block)
    my_digest = my_sha1.get_digest()
    compare_digests(my_digest, TC2_2_expected)

#-------------------------------------------------------------------
# __name__
# Python thingy which allows the file to be run standalone as
# well as parsed from within a Python interpreter.
#-------------------------------------------------------------------
if __name__=="__main__":
    # Run the main function.
    sys.exit(main())

#=======================================================================
# EOF sha1.py
#=======================================================================
<|endoftext|>


<commit_before>import argparse
import configparser
import getpass
import logging
import os
import re
import sys
if sys.version_info < (3, 4):
  raise "This requires Python 3.4 or later"

kDevPrj = '/tmp/.devprj_{0}'.format(getpass.getuser())

logger = None
args = None

class Project(object):
  """This class is a simple container for a project."""

  def __init__(self, fullpath, vcs):
    index = fullpath.rfind(os.sep)
    if index < 0:
      raise Exception('No separator?')
    self.name = fullpath[index+1:]
    self.path = fullpath
    self.vcs = vcs

  def __lt__(self, other):
    if self.name != other.name:
      return self.name < other.name
    else:
      return self.path < other.path

  def __str__(self):
    return '{0} [{1}] {2}'.format(self.name, self.vcs, self.path)

def main():
  # if user wants the last one, exit now
  if args.last:
    sys.exit(0)

  # parse the configuration file
  config = configparser.RawConfigParser()
  config.read(args.conf)

  # get, expand, and check project root directories
  roots = config.get('devsearch', 'root').split(os.pathsep)
  logger.debug('root={0}'.format(roots))
  roots = [full_expand(root) for root in roots]
  for root in roots:
    if not os.path.isdir(root):
      logger.error('\'{0}\' is not a valid directory'
                   .format(root))
      sys.exit(-1)

  # get and check VCSs
  vcss = config.get('devsearch', 'vcs').split(os.pathsep)
  logger.debug('vcs={0}'.format(vcss))
  for vcs in vcss:
    if not vcs in kSupportedVcss:
      logger.error('\'{0}\' is not a supported version control system'
                   .format(vcs))
      logger.error('options are: {0}'.format(','.join(kSupportedVcss)))
      sys.exit(-1)

  # compile regex
  try:
    logger.debug('regex: {0}'.args.project)
    regex = re.compile(args.project)
    args.project = regex
  except:
    logger.error('invalid regular expression: {0}'.format(args.project))
    sys.exit(-1)

  # search for all projects
  projects = set()
  for root in roots:
    projects.update(find_projects(root, vcss))
  projects = list(projects)
  projects.sort()

  # print all projects for debug mode
  logger.debug('All Projects:')
  for idx, project in enumerate(projects):
    logger.debug('{0}: {1}'.format(idx+1, project))

  # apply search function
  logger.debug('Filtering:')
  filtered = []
  for project in projects:
    if (args.project.search(project.path) or
        args.project.search(project.vcs)):
      filtered.append(project)
      logger.debug('{0} passed'.format(project))
    else:
      logger.debug('{0} failed'.format(project))

  # print all projects for debug mode
  logger.debug('Filtered Projects:')
  for index, project in enumerate(filtered):
    logging.debug('{0}: {1}'.format(index + 1, project))

  # if no matches were found, exit
  if len(filtered) == 0:
    print('no projects found')
    sys.exit(-1)

  # if one match was found, use it
  if len(filtered) == 1:
    used = use_project(filtered[0])
    sys.exit(0 if used else 1)

  # multiple matches were found
  # find widths in order to make nice columns
  max_index = 0
  max_name = 0
  max_vcs = 0
  for index, project in enumerate(filtered):
    index_str = str(index + 1)
    if len(index_str) > max_index:
      max_index = len(index_str)
    if len(project.name) > max_name:
      max_name = len(project.name)
    if len(project.vcs) > max_vcs:
      max_vcs = len(project.vcs)
  logger.debug('{0} {1} {2}'.format(max_index, max_name, max_vcs))

  # print the projects nicely
  for index, project in enumerate(filtered):
    index_str = str(index + 1)
    print('{0}{1} : '
          .format(index_str, ' ' * (max_index - len(index_str))),
          end='')
    print('{0}{1} '
          .format(project.name, ' ' * (max_name - len(project.name))),
          end='')
    print('[{0}]{1} '
          .format(project.vcs, ' ' * (max_vcs - len(project.vcs))),
          end='')
    print('{0}'.format(project.path))

  # if user only wants to view projects, exit now
  if args.show:
    sys.exit(1)

  # get user's project selection
  selection = input('select a project: ')
  logger.debug('selection = {0}'.format(selection))
  good = False
  try:
    selection_index = int(selection)
    good = selection_index >= 1 and selection_index <= len(filtered)
    if good:
      selection = selection_index
  except ValueError:
    pass

  # if specifier wasn't a valid index, try matching on project name
  if not good:
    for index, project in enumerate(filtered):
      if project.name == selection:
        selection = index + 1
        good = True
        break

  # display error message if needed
  if not good:
    print('invalid project name or index: {0}'.format(selection))
    sys.exit(-1)

  # use selected project
  used = use_project(filtered[selection - 1])
  sys.exit(0 if used else 1)

def find_projects(dir_path, supported_vcss):
  projects = set()
  vcs = vcs_type(dir_path, supported_vcss)
  if vcs:
    logging.debug('found {0} project at {1}'.format(vcs, dir_path))
    projects.add(Project(dir_path, vcs))
  else:
    try:
      for dir_item in os.listdir(dir_path):
        full = os.path.join(dir_path, dir_item)
        if os.path.isdir(full):
          projects.update(find_projects(full, supported_vcss))
    except OSError:
      pass
  return projects

kSupportedVcss = ['git', 'svn', 'hg', 'cvs']
def vcs_type(path, supported):
  is_cnt = 0
  is_git = is_svn = is_hg = is_cvs = False

  if ('git' in supported and
      os.path.isdir(os.path.join(path, '.git'))):
    is_cnt += 1
    is_git = True
  if ('svn' in supported and
      os.path.isdir(os.path.join(path, '.svn'))):
    is_cnt += 1
    is_svn = True
  if ('hg' in supported and
      os.path.isdir(os.path.join(path, '.hg'))):
    is_cnt += 1
    is_hg = True
  if ('cvs' in supported and
      os.path.isdir(os.path.join(path, 'CVS'))):
    is_cnt += 1
    is_cvs = True

  if is_cnt == 0:
    return None
  elif is_cnt == 1:
    if is_git:
      return 'git'
    elif is_svn:
      return 'svn'
    elif is_hg:
      return 'hg'
    elif is_cvs:
      return 'cvs'
    else:
      logger.error('devsearch programmer is an idiot')
      sys.exit(-1)
  else:
    logger.error('{0} appears to be multiple VCS types ?!?!'
                 .format(path))
    sys.exit(-1)

def full_expand(path):
  return os.path.abspath(os.path.expanduser(path))

def use_project(project):
  if args.show or logger.isEnabledFor(logging.DEBUG):
    print(project)
  if not args.show:
    with open(os.path.expanduser(kDevPrj), 'w') as fd:
      fd.write('{0}\n'.format(project.path))
      logger.debug('wrote path to {0}'.format(kDevPrj))
    return True
  else:
    return False

def check_conf_file(val):
  val = os.path.abspath(os.path.expanduser(val))
  if not os.path.isfile(val):
    raise argparse.ArgumentTypeError('{0} doesn\'t exist'.format(val))
  if not os.access(val, os.R_OK):
    raise argparse.ArgumentTypeError('{0} isn\'t readable'.format(val))
  return val

if __name__ == '__main__':
  desc = 'Go to your development project via intuitive search'
  parser = argparse.ArgumentParser(prog='devsearch', description=desc)
  parser.add_argument('-s', '--show', action='store_true',
                      help='show project(s) but don\'t change directory')
  parser.add_argument('-l', '--last', action='store_true',
                      help='loads the last project used')
  parser.add_argument('-c', '--conf', default='~/.devconf',
                      type=check_conf_file,
                      help='configuration file to use')
  parser.add_argument('-d', '--debug', action='store_true',
                      help='print lots of useless stuff to the console')
  parser.add_argument('project', default='', nargs='?',
                      help='project search text (regex)')
  try:
    args = parser.parse_args()
  except SystemExit:
    sys.exit(-1)

  # create a logger
  logger = logging.getLogger()
  logger.addHandler(logging.StreamHandler(stream=sys.stdout))
  if args.debug:
    logger.setLevel(logging.DEBUG)
  else:
    logger.setLevel(logging.INFO)
  logger.debug('args={0}'.format(args))

  try:
    sys.exit(main())
  except KeyboardInterrupt:
    print()
    sys.exit(-1)
<commit_msg>now prints regex as info<commit_after>import argparse
import configparser
import getpass
import logging
import os
import re
import sys
if sys.version_info < (3, 4):
  raise "This requires Python 3.4 or later"

kDevPrj = '/tmp/.devprj_{0}'.format(getpass.getuser())

logger = None
args = None

class Project(object):
  """This class is a simple container for a project."""

  def __init__(self, fullpath, vcs):
    index = fullpath.rfind(os.sep)
    if index < 0:
      raise Exception('No separator?')
    self.name = fullpath[index+1:]
    self.path = fullpath
    self.vcs = vcs

  def __lt__(self, other):
    if self.name != other.name:
      return self.name < other.name
    else:
      return self.path < other.path

  def __str__(self):
    return '{0} [{1}] {2}'.format(self.name, self.vcs, self.path)

def main():
  # if user wants the last one, exit now
  if args.last:
    sys.exit(0)

  # parse the configuration file
  config = configparser.RawConfigParser()
  config.read(args.conf)

  # get, expand, and check project root directories
  roots = config.get('devsearch', 'root').split(os.pathsep)
  logger.debug('root={0}'.format(roots))
  roots = [full_expand(root) for root in roots]
  for root in roots:
    if not os.path.isdir(root):
      logger.error('\'{0}\' is not a valid directory'
                   .format(root))
      sys.exit(-1)

  # get and check VCSs
  vcss = config.get('devsearch', 'vcs').split(os.pathsep)
  logger.debug('vcs={0}'.format(vcss))
  for vcs in vcss:
    if not vcs in kSupportedVcss:
      logger.error('\'{0}\' is not a supported version control system'
                   .format(vcs))
      logger.error('options are: {0}'.format(','.join(kSupportedVcss)))
      sys.exit(-1)

  # compile regex
  try:
    logger.info('project regex: {0}'.format(args.project))
    regex = re.compile(args.project)
    args.project = regex
  except:
    logger.error('invalid regular expression: {0}'.format(args.project))
    sys.exit(-1)

  # search for all projects
  projects = set()
  for root in roots:
    projects.update(find_projects(root, vcss))
  projects = list(projects)
  projects.sort()

  # print all projects for debug mode
  logger.debug('All Projects:')
  for idx, project in enumerate(projects):
    logger.debug('{0}: {1}'.format(idx+1, project))

  # apply search function
  logger.debug('Filtering:')
  filtered = []
  for project in projects:
    if (args.project.search(project.path) or
        args.project.search(project.vcs)):
      filtered.append(project)
      logger.debug('{0} passed'.format(project))
    else:
      logger.debug('{0} failed'.format(project))

  # print all projects for debug mode
  logger.debug('Filtered Projects:')
  for index, project in enumerate(filtered):
    logging.debug('{0}: {1}'.format(index + 1, project))

  # if no matches were found, exit
  if len(filtered) == 0:
    print('no projects found')
    sys.exit(-1)

  # if one match was found, use it
  if len(filtered) == 1:
    used = use_project(filtered[0])
    sys.exit(0 if used else 1)

  # multiple matches were found
  # find widths in order to make nice columns
  max_index = 0
  max_name = 0
  max_vcs = 0
  for index, project in enumerate(filtered):
    index_str = str(index + 1)
    if len(index_str) > max_index:
      max_index = len(index_str)
    if len(project.name) > max_name:
      max_name = len(project.name)
    if len(project.vcs) > max_vcs:
      max_vcs = len(project.vcs)
  logger.debug('column widths: {0} {1} {2}'.format(max_index, max_name, max_vcs))

  # print the projects nicely
  for index, project in enumerate(filtered):
    index_str = str(index + 1)
    print('{0}{1} : '
          .format(index_str, ' ' * (max_index - len(index_str))),
          end='')
    print('{0}{1} '
          .format(project.name, ' ' * (max_name - len(project.name))),
          end='')
    print('[{0}]{1} '
          .format(project.vcs, ' ' * (max_vcs - len(project.vcs))),
          end='')
    print('{0}'.format(project.path))

  # if user only wants to view projects, exit now
  if args.show:
    sys.exit(1)

  # get user's project selection
  selection = input('select a project: ')
  logger.debug('selection = {0}'.format(selection))
  good = False
  try:
    selection_index = int(selection)
    good = selection_index >= 1 and selection_index <= len(filtered)
    if good:
      selection = selection_index
  except ValueError:
    pass

  # if specifier wasn't a valid index, try matching on project name
  if not good:
    for index, project in enumerate(filtered):
      if project.name == selection:
        selection = index + 1
        good = True
        break

  # display error message if needed
  if not good:
    print('invalid project name or index: {0}'.format(selection))
    sys.exit(-1)

  # use selected project
  used = use_project(filtered[selection - 1])
  sys.exit(0 if used else 1)

def find_projects(dir_path, supported_vcss):
  projects = set()
  vcs = vcs_type(dir_path, supported_vcss)
  if vcs:
    logging.debug('found {0} project at {1}'.format(vcs, dir_path))
    projects.add(Project(dir_path, vcs))
  else:
    try:
      for dir_item in os.listdir(dir_path):
        full = os.path.join(dir_path, dir_item)
        if os.path.isdir(full):
          projects.update(find_projects(full, supported_vcss))
    except OSError:
      pass
  return projects

kSupportedVcss = ['git', 'svn', 'hg', 'cvs']
def vcs_type(path, supported):
  is_cnt = 0
  is_git = is_svn = is_hg = is_cvs = False

  if ('git' in supported and
      os.path.isdir(os.path.join(path, '.git'))):
    is_cnt += 1
    is_git = True
  if ('svn' in supported and
      os.path.isdir(os.path.join(path, '.svn'))):
    is_cnt += 1
    is_svn = True
  if ('hg' in supported and
      os.path.isdir(os.path.join(path, '.hg'))):
    is_cnt += 1
    is_hg = True
  if ('cvs' in supported and
      os.path.isdir(os.path.join(path, 'CVS'))):
    is_cnt += 1
    is_cvs = True

  if is_cnt == 0:
    return None
  elif is_cnt == 1:
    if is_git:
      return 'git'
    elif is_svn:
      return 'svn'
    elif is_hg:
      return 'hg'
    elif is_cvs:
      return 'cvs'
    else:
      logger.error('devsearch programmer is an idiot')
      sys.exit(-1)
  else:
    logger.error('{0} appears to be multiple VCS types ?!?!'
                 .format(path))
    sys.exit(-1)

def full_expand(path):
  return os.path.abspath(os.path.expanduser(path))

def use_project(project):
  if args.show or logger.isEnabledFor(logging.DEBUG):
    print(project)
  if not args.show:
    with open(os.path.expanduser(kDevPrj), 'w') as fd:
      fd.write('{0}\n'.format(project.path))
      logger.debug('wrote path to {0}'.format(kDevPrj))
    return True
  else:
    return False

def check_conf_file(val):
  val = os.path.abspath(os.path.expanduser(val))
  if not os.path.isfile(val):
    raise argparse.ArgumentTypeError('{0} doesn\'t exist'.format(val))
  if not os.access(val, os.R_OK):
    raise argparse.ArgumentTypeError('{0} isn\'t readable'.format(val))
  return val

if __name__ == '__main__':
  desc = 'Go to your development project via intuitive search'
  parser = argparse.ArgumentParser(prog='devsearch', description=desc)
  parser.add_argument('-s', '--show', action='store_true',
                      help='show project(s) but don\'t change directory')
  parser.add_argument('-l', '--last', action='store_true',
                      help='loads the last project used')
  parser.add_argument('-c', '--conf', default='~/.devconf',
                      type=check_conf_file,
                      help='configuration file to use')
  parser.add_argument('-d', '--debug', action='store_true',
                      help='print lots of useless stuff to the console')
  parser.add_argument('project', default='', nargs='?',
                      help='project search regex (quote with \'\')')
  try:
    args = parser.parse_args()
  except SystemExit:
    sys.exit(-1)

  # create a logger
  logger = logging.getLogger()
  logger.addHandler(logging.StreamHandler(stream=sys.stdout))
  if args.debug:
    logger.setLevel(logging.DEBUG)
  else:
    logger.setLevel(logging.INFO)
  logger.debug('args={0}'.format(args))

  try:
    sys.exit(main())
  except KeyboardInterrupt:
    print()
    sys.exit(-1)
<|endoftext|>


<commit_before># coding=utf-8
import logging
import urllib.parse

import tweepy

logger = logging.getLogger(__name__)

class TwitterAPI:
	def __init__(self):
		pass

class UserStream(tweepy.Stream):
	def user_stream(self):
		self.parameters = {
			"delimited": "length",
			"replies": "all",
			"filter_level": "none",
			"include_followings_activity": "True",
			"stall_warnings": "True",
			"with": "followings"
		}
		self.headers['Content-type'] = "application/x-www-form-urlencoded"
		self.scheme = "https"
		self.host = 'userstream.twitter.com'
		self.url = '/1.1/user.json'
		self.body = urllib.parse.urlencode(self.parameters)
		self.timeout = None
		self._start(False)
<commit_msg>Add: API Handler<commit_after># coding=utf-8
import logging
import urllib.parse

import tweepy

from TBFW.configparser import ConfigParser

logger = logging.getLogger(__name__)

class TwitterAPI:
	def __init__(self, accountId):
		Config = ConfigParser()
		accounts = Config.accounts

		self.auth = tweepy.OAuthHandler(accounts[accountId]["ck"], accounts[accountId]["cs"])
		self.auth.set_access_token(accounts[accountId]["at"], accounts[accountId]["ats"])
		self.API = tweepy.API(self.auth)

class UserStream(tweepy.Stream):
	def user_stream(self):
		self.parameters = {
			"delimited": "length",
			"replies": "all",
			"filter_level": "none",
			"include_followings_activity": "True",
			"stall_warnings": "True",
			"with": "followings"
		}
		self.headers['Content-type'] = "application/x-www-form-urlencoded"
		self.scheme = "https"
		self.host = 'userstream.twitter.com'
		self.url = '/1.1/user.json'
		self.body = urllib.parse.urlencode(self.parameters)
		self.timeout = None
		self._start(False)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 26-10-2017 9:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad / playground for the 4th chapter on the book
"""

import math
import time
import pylab
import pandas
import random
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix

# Constants
padding = 40
total_padding = padding * 2

# Seed pseudo-random number generator
random.seed(time.time())

# Helpers

def roc_curve(true_labels, predicted_probs, n_points=100, pos_class=1):
    # Initialization
    # Reference line (this one is easy)
    thr = pylab.linspace(0, 1, n_points)
    # I guess this is about true possitive rate
    tpr = pylab.zeros(n_points)
    # I guess this is about the false possitive rate
    fpr = pylab.zeros(n_points)
    # What the fuck is this, why the sample code is so fucking obscure?
    # Possitive and negative vectors?
    pos = true_labels == pos_class
    neg = np.logical_not(pos)
    # Count possitives and negatives
    n_pos = np.count_nonzero(pos)
    n_neg = np.count_nonzero(neg)
    # Calculate tpr and fpr for every position of the curve
    for index, value in enumerate(thr):
        tpr[index] = np.count_nonzero(np.logical_and(predicted_probs >= value, pos)) / n_pos
        fpr[index] = np.count_nonzero(np.logical_and(predicted_probs >= value, neg)) / n_neg
    return fpr, tpr, thr

def area_under_the_curve(true_labels, predicted_labels, pos_class=1):
    fpr, tpr, thr = roc_curve(true_labels, predicted_labels, pos_class=pos_class)
    area = -pylab.trapz(tpr, x=fpr)
    return area

def root_mean_square_error(true_values, predicted_values):
    n = len(true_values)
    residuals = 0
    for i in range(n):
        residuals += (true_values[i] - predicted_values[i]) ** 2
    return np.sqrt(residuals / n)

def r_square_error(true_values, predicted_values):
    n = len(true_values)
    mean = np.mean(true_values)
    residuals = 0
    total = 0
    for i in range(n):
        residuals += (true_values[i] - predicted_values[i]) ** 2
        total += (true_values[i] - mean) ** 2
    return 1.0 - (residuals / total)

print("+++> Create a random 100x5 Matrix")
features = pylab.rand(100, 5)
print("+++> Create the prediction target")
target = pylab.rand(100) > 0.5

# First we're going to try the holdout method
print("[{} Holdout Method {}]".format("-" * padding, "-" * padding))
n = features.shape[0]
n_train = math.floor(0.7 * n)
# Randomize index
# Note: sometimes you want to retain the order in the dataset and skip this step, e.g. in the case of time-based
# datasets where you want to test on 'later' instances
print("+++> Create a random permutation of the dataset")
idx = np.random.permutation(n)
# Split the index
print("+++> Split the dataset (train / test)")
idx_train = idx[:n_train]
idx_test = idx[n_train:]
# Break your data into training and testing subsets
print("+++> Break your data into training and testing subsets")
features_train = features[idx_train, :]
target_train = target[idx_train]
features_test = features[idx_test, :]
target_test = target[idx_test]
# Log the aspect of the data
print("---> Train dataset shape {}".format(features_train.shape))
print("---> Test dataset shape {}".format(features_test.shape))
print("---> Target data for the training dataset, shape {}".format(target_train.shape))
print("---> Target data for the test dataset, shape {}".format(target_test.shape))
# ---- And... that was it, according to the example
# print("-" * total_padding)

# K-fold cross-validation
# Number of items in the dataset
n = features.shape[0]
# Number of folds
k_folds = 10
# Un-initialized array with as many elements as elements in the dataset
preds_kfold = np.empty(n)
folds = np.random.randint(0, k_folds, size=n)

for idx in np.arange(k_folds):
    # For each fold, break your data into training and testing subsets
    features_train = features[folds != idx, :]
    target_train = target[folds != idx]
    features_test = features[folds == idx, :]
    # I don't really need to compute the target_test, I can do it on the fly later
    target_test = target[folds == idx]
    # Print the indices in each fold, for inspection
    print("Fold for index #{}: {}".format(idx, np.nonzero(folds == idx)[0]))
    # This part is about the model
    # Build and predict for CV fold (to be filled out)
    # model = train(features_train, target_train)
    # preds_kfold[folds == idx] = predict(model, features_test)
# Measure model accuracy
# accuracy = evaluate_acc(preds_kfold, target)
print("[{} ============= {}]\n\n".format("-" * padding, "-" * padding))

# The ROC Curve
print("[{} The ROC Curve {}]".format("-" * padding, "-" * padding))
# Randomly generated predictions should give us a diagonal ROC curve
preds = pylab.rand(len(target))
fpr, tpr, thr = roc_curve(target, preds, pos_class=True)
# pylab.plot(pylab.linspace(0, 1))
pylab.plot(fpr, tpr)
# Let's calculate the Area Under the Curve, for the ROC curve
print("+++> Area Under the Curve: {}".format(area_under_the_curve(target, preds, pos_class=True)))
print("[{} ============= {}]\n\n".format("-" * padding, "-" * padding))

# Multi-class Classification
print("[{} Multi-class Classification {}]".format("-" * padding, "-" * padding))
print("+++> Load the MNIST dataset")
mnist_dataset = pandas.read_csv("../book_code/data/mnist_small.csv")
# Hold-out method
print("+++> Holdout method for the dataset")
mnist_dataset_train = mnist_dataset[:int(0.8 * len(mnist_dataset))]
mnist_dataset_test = mnist_dataset[int(0.8 * len(mnist_dataset)):]
print("+++> Fit a Random Forest Classifier")
randomforest_classifier = RandomForestClassifier()
randomforest_classifier.fit(mnist_dataset_train.drop('label', axis=1), mnist_dataset_train['label'])
print("+++> Calculate the model predictions on the test holdout")
randomforest_classifier_predictions = randomforest_classifier.predict(mnist_dataset_test.drop('label', axis=1))
print("---> Sample predictions: {}".format(randomforest_classifier_predictions[:10]))
print("+++> Compute the Confusion Matrix")
randomforest_mnist_confusion_matrix = confusion_matrix(mnist_dataset_test['label'], randomforest_classifier_predictions)
pylab.matshow(randomforest_mnist_confusion_matrix, cmap='Greys')
pylab.colorbar()
pylab.savefig("figures/figure-4.19.eps", format='eps')
print("[{} ========================== {}]\n\n".format("-" * padding, "-" * padding))

# Root Mean Square Error
print("[{} Root Mean Square Error (RMSE) {}]".format("-" * padding, "-" * padding))
print("+++> Sample use of RMSE for random values and predictions, {}"
      .format(root_mean_square_error(pylab.rand(10), pylab.rand(10))))
print("[{} ============================= {}]\n\n".format("-" * padding, "-" * padding))

# R-Square Error
print("[{} R-Square Error (RSE) {}]".format("-" * padding, "-" * padding))
print("+++> Sample use of RSE for random values and predictions, {}"
      .format(r_square_error(np.arange(10) + pylab.rand(), np.arange(10) + pylab.rand())))
print("[{} ============================= {}]\n\n".format("-" * padding, "-" * padding))

# Grid search with Kernel-SVM model

# Show all plots
pylab.show()
<commit_msg>section boundaries in the logging subsystem<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : ml-playground
# Timestamp : 26-10-2017 9:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Scratchpad / playground for the 4th chapter on the book
"""

import math
import time
import pylab
import pandas
import random
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix

# Constants
padding = 40
total_padding = padding * 2

# Seed pseudo-random number generator
random.seed(time.time())

# Helpers

def roc_curve(true_labels, predicted_probs, n_points=100, pos_class=1):
    # Initialization
    # Reference line (this one is easy)
    thr = pylab.linspace(0, 1, n_points)
    # I guess this is about true possitive rate
    tpr = pylab.zeros(n_points)
    # I guess this is about the false possitive rate
    fpr = pylab.zeros(n_points)
    # What the fuck is this, why the sample code is so fucking obscure?
    # Possitive and negative vectors?
    pos = true_labels == pos_class
    neg = np.logical_not(pos)
    # Count possitives and negatives
    n_pos = np.count_nonzero(pos)
    n_neg = np.count_nonzero(neg)
    # Calculate tpr and fpr for every position of the curve
    for index, value in enumerate(thr):
        tpr[index] = np.count_nonzero(np.logical_and(predicted_probs >= value, pos)) / n_pos
        fpr[index] = np.count_nonzero(np.logical_and(predicted_probs >= value, neg)) / n_neg
    return fpr, tpr, thr

def area_under_the_curve(true_labels, predicted_labels, pos_class=1):
    fpr, tpr, thr = roc_curve(true_labels, predicted_labels, pos_class=pos_class)
    area = -pylab.trapz(tpr, x=fpr)
    return area

def root_mean_square_error(true_values, predicted_values):
    n = len(true_values)
    residuals = 0
    for i in range(n):
        residuals += (true_values[i] - predicted_values[i]) ** 2
    return np.sqrt(residuals / n)

def r_square_error(true_values, predicted_values):
    n = len(true_values)
    mean = np.mean(true_values)
    residuals = 0
    total = 0
    for i in range(n):
        residuals += (true_values[i] - predicted_values[i]) ** 2
        total += (true_values[i] - mean) ** 2
    return 1.0 - (residuals / total)

print("+++> Create a random 100x5 Matrix")
features = pylab.rand(100, 5)
print("+++> Create the prediction target")
target = pylab.rand(100) > 0.5

# First we're going to try the holdout method
print("[{} Holdout Method {}]".format("-" * padding, "-" * padding))
n = features.shape[0]
n_train = math.floor(0.7 * n)
# Randomize index
# Note: sometimes you want to retain the order in the dataset and skip this step, e.g. in the case of time-based
# datasets where you want to test on 'later' instances
print("+++> Create a random permutation of the dataset")
idx = np.random.permutation(n)
# Split the index
print("+++> Split the dataset (train / test)")
idx_train = idx[:n_train]
idx_test = idx[n_train:]
# Break your data into training and testing subsets
print("+++> Break your data into training and testing subsets")
features_train = features[idx_train, :]
target_train = target[idx_train]
features_test = features[idx_test, :]
target_test = target[idx_test]
# Log the aspect of the data
print("---> Train dataset shape {}".format(features_train.shape))
print("---> Test dataset shape {}".format(features_test.shape))
print("---> Target data for the training dataset, shape {}".format(target_train.shape))
print("---> Target data for the test dataset, shape {}".format(target_test.shape))
# ---- And... that was it, according to the example
# print("-" * total_padding)

# K-fold cross-validation
# Number of items in the dataset
n = features.shape[0]
# Number of folds
k_folds = 10
# Un-initialized array with as many elements as elements in the dataset
preds_kfold = np.empty(n)
folds = np.random.randint(0, k_folds, size=n)

for idx in np.arange(k_folds):
    # For each fold, break your data into training and testing subsets
    features_train = features[folds != idx, :]
    target_train = target[folds != idx]
    features_test = features[folds == idx, :]
    # I don't really need to compute the target_test, I can do it on the fly later
    target_test = target[folds == idx]
    # Print the indices in each fold, for inspection
    print("Fold for index #{}: {}".format(idx, np.nonzero(folds == idx)[0]))
    # This part is about the model
    # Build and predict for CV fold (to be filled out)
    # model = train(features_train, target_train)
    # preds_kfold[folds == idx] = predict(model, features_test)
# Measure model accuracy
# accuracy = evaluate_acc(preds_kfold, target)
print("[{} ============= {}]\n\n".format("-" * padding, "-" * padding))

# The ROC Curve
print("[{} The ROC Curve {}]".format("-" * padding, "-" * padding))
# Randomly generated predictions should give us a diagonal ROC curve
preds = pylab.rand(len(target))
fpr, tpr, thr = roc_curve(target, preds, pos_class=True)
# pylab.plot(pylab.linspace(0, 1))
pylab.plot(fpr, tpr)
# Let's calculate the Area Under the Curve, for the ROC curve
print("+++> Area Under the Curve: {}".format(area_under_the_curve(target, preds, pos_class=True)))
print("[{} ============= {}]\n\n".format("-" * padding, "-" * padding))

# Multi-class Classification
print("[{} Multi-class Classification {}]".format("-" * padding, "-" * padding))
print("+++> Load the MNIST dataset")
mnist_dataset = pandas.read_csv("../book_code/data/mnist_small.csv")
# Hold-out method
print("+++> Holdout method for the dataset")
mnist_dataset_train = mnist_dataset[:int(0.8 * len(mnist_dataset))]
mnist_dataset_test = mnist_dataset[int(0.8 * len(mnist_dataset)):]
print("+++> Fit a Random Forest Classifier")
randomforest_classifier = RandomForestClassifier()
randomforest_classifier.fit(mnist_dataset_train.drop('label', axis=1), mnist_dataset_train['label'])
print("+++> Calculate the model predictions on the test holdout")
randomforest_classifier_predictions = randomforest_classifier.predict(mnist_dataset_test.drop('label', axis=1))
print("---> Sample predictions: {}".format(randomforest_classifier_predictions[:10]))
print("+++> Compute the Confusion Matrix")
randomforest_mnist_confusion_matrix = confusion_matrix(mnist_dataset_test['label'], randomforest_classifier_predictions)
pylab.matshow(randomforest_mnist_confusion_matrix, cmap='Greys')
pylab.colorbar()
pylab.savefig("figures/figure-4.19.eps", format='eps')
print("[{} ========================== {}]\n\n".format("-" * padding, "-" * padding))

# Root Mean Square Error
print("[{} Root Mean Square Error (RMSE) {}]".format("-" * padding, "-" * padding))
print("+++> Sample use of RMSE for random values and predictions, {}"
      .format(root_mean_square_error(pylab.rand(10), pylab.rand(10))))
print("[{} ============================= {}]\n\n".format("-" * padding, "-" * padding))

# R-Square Error
print("[{} R-Square Error (RSE) {}]".format("-" * padding, "-" * padding))
print("+++> Sample use of RSE for random values and predictions, {}"
      .format(r_square_error(np.arange(10) + pylab.rand(), np.arange(10) + pylab.rand())))
print("[{} ============================= {}]\n\n".format("-" * padding, "-" * padding))

# Grid search with Kernel-SVM model
print("[{} Grid Search with Kernel-SVM Model {}]".format("-" * padding, "-" * padding))
print("[{} ================================= {}]\n\n".format("-" * padding, "-" * padding))

# Show all plots
pylab.show()
<|endoftext|>