<commit_before>"""
We deal (mostly) with remote hosts. To avoid special casing each different
commands (e.g. using `yum` as opposed to `apt`) we can make a one time call to
that remote host and set all the special cases for running commands depending
on the type of distribution/version we are dealing with.
"""
import logging
from ceph_deploy import exc
from ceph_deploy.util import versions
from ceph_deploy.hosts import debian, centos, fedora, suse, remotes, rhel, arch, alt
from ceph_deploy.connection import get_connection

logger = logging.getLogger()

def get(hostname,
        username=None,
        fallback=None,
        detect_sudo=True,
        use_rhceph=False,
        callbacks=None):
    """
    Retrieve the module that matches the distribution of a ``hostname``. This
    function will connect to that host and retrieve the distribution
    information, then return the appropriate module and slap a few attributes
    to that module defining the information it found from the hostname.

    For example, if host ``node1.example.com`` is an Ubuntu server, the
    ``debian`` module would be returned and the following would be set::

        module.name = 'ubuntu'
        module.release = '12.04'
        module.codename = 'precise'

    :param hostname: A hostname that is reachable/resolvable over the network
    :param fallback: Optional fallback to use if no supported distro is found
    :param use_rhceph: Whether or not to install RH Ceph on a RHEL machine or
                       the community distro.  Changes what host module is
                       returned for RHEL.
    :params callbacks: A list of callables that accept one argument (the actual
                       module that contains the connection) that will be
                       called, in order at the end of the instantiation of the
                       module.
    """
    conn = get_connection(
        hostname,
        username=username,
        logger=logging.getLogger(hostname),
        detect_sudo=detect_sudo
    )
    try:
        conn.import_module(remotes)
    except IOError as error:
        if 'already closed' in getattr(error, 'message', ''):
            raise RuntimeError('remote connection got closed, ensure ``requiretty`` is disabled for %s' % hostname)
    distro_name, release, codename = conn.remote_module.platform_information()
    if not codename or not _get_distro(distro_name):
        raise exc.UnsupportedPlatform(
            distro=distro_name,
            codename=codename,
            release=release)

    machine_type = conn.remote_module.machine_type()
    module = _get_distro(distro_name, use_rhceph=use_rhceph)
    module.name = distro_name
    module.normalized_name = _normalized_distro_name(distro_name)
    module.normalized_release = _normalized_release(release)
    module.distro = module.normalized_name
    module.is_el = module.normalized_name in ['redhat', 'centos', 'fedora', 'scientific', 'oracle', 'virtuozzo']
    module.is_rpm = module.normalized_name in ['redhat', 'centos',
                                               'fedora', 'scientific', 'suse', 'oracle', 'virtuozzo', 'alt']
    module.is_deb = module.normalized_name in ['debian', 'ubuntu']
    module.is_pkgtarxz = module.normalized_name in ['arch']
    module.release = release
    module.codename = codename
    module.conn = conn
    module.machine_type = machine_type
    module.init = module.choose_init(module)
    module.packager = module.get_packager(module)
    # execute each callback if any
    if callbacks:
        for c in callbacks:
            c(module)
    return module

def _get_distro(distro, fallback=None, use_rhceph=False):
    if not distro:
        return

    distro = _normalized_distro_name(distro)
    distributions = {
        'debian': debian,
        'ubuntu': debian,
        'centos': centos,
        'scientific': centos,
        'oracle': centos,
        'redhat': centos,
        'fedora': fedora,
        'suse': suse,
        'virtuozzo': centos,
        'arch': arch,
        'alt': alt
        }

    if distro == 'redhat' and use_rhceph:
        return rhel
    else:
        return distributions.get(distro) or _get_distro(fallback)

def _normalized_distro_name(distro):
    distro = distro.lower()
    if distro.startswith(('redhat', 'red hat')):
        return 'redhat'
    elif distro.startswith(('scientific', 'scientific linux')):
        return 'scientific'
    elif distro.startswith('oracle'):
        return 'oracle'
    elif distro.startswith(('suse', 'opensuse', 'sles')):
        return 'suse'
    elif distro.startswith('centos'):
        return 'centos'
    elif distro.startswith('linuxmint'):
        return 'ubuntu'
    elif distro.startswith('virtuozzo'):
        return 'virtuozzo'
    elif distro.startswith('arch'):
        return 'arch'
    elif distro.startswith(('alt', 'altlinux', 'basealt', 'alt linux')):
        return 'alt'
    return distro

def _normalized_release(release):
    """
    A normalizer function to make sense of distro
    release versions.

    Returns an object with: major, minor, patch, and garbage

    These attributes can be accessed as ints with prefixed "int"
    attribute names, for example:

        normalized_version.int_major
    """
    # TODO: at some point deprecate this function so that we just
    # use this class directly (and update every test that calls it
    return versions.NormalizedVersion(release)
<commit_msg>Add support for EulerOS<commit_after>"""
We deal (mostly) with remote hosts. To avoid special casing each different
commands (e.g. using `yum` as opposed to `apt`) we can make a one time call to
that remote host and set all the special cases for running commands depending
on the type of distribution/version we are dealing with.
"""
import logging
from ceph_deploy import exc
from ceph_deploy.util import versions
from ceph_deploy.hosts import debian, centos, fedora, suse, remotes, rhel, arch, alt
from ceph_deploy.connection import get_connection

logger = logging.getLogger()

def get(hostname,
        username=None,
        fallback=None,
        detect_sudo=True,
        use_rhceph=False,
        callbacks=None):
    """
    Retrieve the module that matches the distribution of a ``hostname``. This
    function will connect to that host and retrieve the distribution
    information, then return the appropriate module and slap a few attributes
    to that module defining the information it found from the hostname.

    For example, if host ``node1.example.com`` is an Ubuntu server, the
    ``debian`` module would be returned and the following would be set::

        module.name = 'ubuntu'
        module.release = '12.04'
        module.codename = 'precise'

    :param hostname: A hostname that is reachable/resolvable over the network
    :param fallback: Optional fallback to use if no supported distro is found
    :param use_rhceph: Whether or not to install RH Ceph on a RHEL machine or
                       the community distro.  Changes what host module is
                       returned for RHEL.
    :params callbacks: A list of callables that accept one argument (the actual
                       module that contains the connection) that will be
                       called, in order at the end of the instantiation of the
                       module.
    """
    conn = get_connection(
        hostname,
        username=username,
        logger=logging.getLogger(hostname),
        detect_sudo=detect_sudo
    )
    try:
        conn.import_module(remotes)
    except IOError as error:
        if 'already closed' in getattr(error, 'message', ''):
            raise RuntimeError('remote connection got closed, ensure ``requiretty`` is disabled for %s' % hostname)
    distro_name, release, codename = conn.remote_module.platform_information()
    if not codename or not _get_distro(distro_name):
        raise exc.UnsupportedPlatform(
            distro=distro_name,
            codename=codename,
            release=release)

    machine_type = conn.remote_module.machine_type()
    module = _get_distro(distro_name, use_rhceph=use_rhceph)
    module.name = distro_name
    module.normalized_name = _normalized_distro_name(distro_name)
    module.normalized_release = _normalized_release(release)
    module.distro = module.normalized_name
    module.is_el = module.normalized_name in ['redhat', 'centos', 'fedora', 'scientific', 'oracle', 'virtuozzo']
    module.is_rpm = module.normalized_name in ['redhat', 'centos',
                                               'fedora', 'scientific', 'suse', 'oracle', 'virtuozzo', 'alt']
    module.is_deb = module.normalized_name in ['debian', 'ubuntu']
    module.is_pkgtarxz = module.normalized_name in ['arch']
    module.release = release
    module.codename = codename
    module.conn = conn
    module.machine_type = machine_type
    module.init = module.choose_init(module)
    module.packager = module.get_packager(module)
    # execute each callback if any
    if callbacks:
        for c in callbacks:
            c(module)
    return module

def _get_distro(distro, fallback=None, use_rhceph=False):
    if not distro:
        return

    distro = _normalized_distro_name(distro)
    distributions = {
        'debian': debian,
        'ubuntu': debian,
        'centos': centos,
        'scientific': centos,
        'oracle': centos,
        'redhat': centos,
        'fedora': fedora,
        'suse': suse,
        'virtuozzo': centos,
        'arch': arch,
        'alt': alt
        }

    if distro == 'redhat' and use_rhceph:
        return rhel
    else:
        return distributions.get(distro) or _get_distro(fallback)

def _normalized_distro_name(distro):
    distro = distro.lower()
    if distro.startswith(('redhat', 'red hat')):
        return 'redhat'
    elif distro.startswith(('scientific', 'scientific linux')):
        return 'scientific'
    elif distro.startswith('oracle'):
        return 'oracle'
    elif distro.startswith(('suse', 'opensuse', 'sles')):
        return 'suse'
    elif distro.startswith(('centos', 'euleros')):
        return 'centos'
    elif distro.startswith('linuxmint'):
        return 'ubuntu'
    elif distro.startswith('virtuozzo'):
        return 'virtuozzo'
    elif distro.startswith('arch'):
        return 'arch'
    elif distro.startswith(('alt', 'altlinux', 'basealt', 'alt linux')):
        return 'alt'
    return distro

def _normalized_release(release):
    """
    A normalizer function to make sense of distro
    release versions.

    Returns an object with: major, minor, patch, and garbage

    These attributes can be accessed as ints with prefixed "int"
    attribute names, for example:

        normalized_version.int_major
    """
    # TODO: at some point deprecate this function so that we just
    # use this class directly (and update every test that calls it
    return versions.NormalizedVersion(release)
<|endoftext|>


<commit_before>"""
Peewee integration with APSW, "another python sqlite wrapper".

Project page: https://code.google.com/p/apsw/

APSW is a really neat library that provides a thin wrapper on top of SQLite's
C interface.

Here are just a few reasons to use APSW, taken from the documentation:

* APSW gives all functionality of SQLite, including virtual tables, virtual
  file system, blob i/o, backups and file control.
* Connections can be shared across threads without any additional locking.
* Transactions are managed explicitly by your code.
* APSW can handle nested transactions.
* Unicode is handled correctly.
* APSW is faster.
"""
import apsw
from peewee import *
from peewee import _sqlite_date_part
from peewee import _sqlite_date_trunc
from peewee import _sqlite_regexp
from peewee import BooleanField as _BooleanField
from peewee import DateField as _DateField
from peewee import DateTimeField as _DateTimeField
from peewee import DecimalField as _DecimalField
from peewee import logger
from peewee import savepoint
from peewee import TimeField as _TimeField
from peewee import transaction as _transaction
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqlite_ext import VirtualCharField
from playhouse.sqlite_ext import VirtualField
from playhouse.sqlite_ext import VirtualFloatField
from playhouse.sqlite_ext import VirtualIntegerField
from playhouse.sqlite_ext import VirtualModel

class transaction(_transaction):
    def __init__(self, db, lock_type='deferred'):
        self.db = db
        self.lock_type = lock_type

    def _begin(self):
        self.db.begin(self.lock_type)

class APSWDatabase(SqliteExtDatabase):
    def __init__(self, database, timeout=None, **kwargs):
        self.timeout = timeout
        self._modules = {}
        super(APSWDatabase, self).__init__(database, **kwargs)

    def register_module(self, mod_name, mod_inst):
        self._modules[mod_name] = mod_inst

    def unregister_module(self, mod_name):
        del(self._modules[mod_name])

    def _connect(self, database, **kwargs):
        conn = apsw.Connection(database, **kwargs)
        if self.timeout is not None:
            conn.setbusytimeout(self.timeout)
        try:
            self._add_conn_hooks(conn)
        except:
            conn.close()
            raise
        return conn

    def _add_conn_hooks(self, conn):
        super(APSWDatabase, self)._add_conn_hooks(conn)
        self._load_modules(conn)  # APSW-only.

    def _load_modules(self, conn):
        for mod_name, mod_inst in self._modules.items():
            conn.createmodule(mod_name, mod_inst)
        return conn

    def _load_aggregates(self, conn):
        for name, (klass, num_params) in self._aggregates.items():
            def make_aggregate():
                instance = klass()
                return (instance, instance.step, instance.finalize)
            conn.createaggregatefunction(name, make_aggregate)

    def _load_collations(self, conn):
        for name, fn in self._collations.items():
            conn.createcollation(name, fn)

    def _load_functions(self, conn):
        for name, (fn, num_params) in self._functions.items():
            conn.createscalarfunction(name, fn, num_params)

    def _load_extensions(self, conn):
        conn.enableloadextension(True)
        for extension in self._extensions:
            conn.loadextension(extension)

    def load_extension(self, extension):
        self._extensions.add(extension)
        if not self.is_closed():
            conn = self.get_conn()
            conn.enableloadextension(True)
            conn.loadextension(extension)

    def _execute_sql(self, cursor, sql, params):
        cursor.execute(sql, params or ())
        return cursor

    def execute_sql(self, sql, params=None, require_commit=True):
        logger.debug((sql, params))
        with self.exception_wrapper():
            cursor = self.get_cursor()
            self._execute_sql(cursor, sql, params)
        return cursor

    def last_insert_id(self, cursor, model):
        if model._meta.auto_increment:
            return cursor.getconnection().last_insert_rowid()

    def rows_affected(self, cursor):
        return cursor.getconnection().changes()

    def begin(self, lock_type='deferred'):
        self.get_cursor().execute('begin %s;' % lock_type)

    def commit(self):
        self.get_cursor().execute('commit;')

    def rollback(self):
        self.get_cursor().execute('rollback;')

    def transaction(self, lock_type='deferred'):
        return transaction(self, lock_type)

    def savepoint(self, sid=None):
        return savepoint(self, sid)

def nh(s, v):
    if v is not None:
        return str(v)

class BooleanField(_BooleanField):
    def db_value(self, v):
        v = super(BooleanField, self).db_value(v)
        if v is not None:
            return v and 1 or 0

class DateField(_DateField):
    db_value = nh

class TimeField(_TimeField):
    db_value = nh

class DateTimeField(_DateTimeField):
    db_value = nh

class DecimalField(_DecimalField):
    db_value = nh
<commit_msg>Update apsw URL<commit_after>"""
Peewee integration with APSW, "another python sqlite wrapper".

Project page: https://rogerbinns.github.io/apsw/

APSW is a really neat library that provides a thin wrapper on top of SQLite's
C interface.

Here are just a few reasons to use APSW, taken from the documentation:

* APSW gives all functionality of SQLite, including virtual tables, virtual
  file system, blob i/o, backups and file control.
* Connections can be shared across threads without any additional locking.
* Transactions are managed explicitly by your code.
* APSW can handle nested transactions.
* Unicode is handled correctly.
* APSW is faster.
"""
import apsw
from peewee import *
from peewee import _sqlite_date_part
from peewee import _sqlite_date_trunc
from peewee import _sqlite_regexp
from peewee import BooleanField as _BooleanField
from peewee import DateField as _DateField
from peewee import DateTimeField as _DateTimeField
from peewee import DecimalField as _DecimalField
from peewee import logger
from peewee import savepoint
from peewee import TimeField as _TimeField
from peewee import transaction as _transaction
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqlite_ext import VirtualCharField
from playhouse.sqlite_ext import VirtualField
from playhouse.sqlite_ext import VirtualFloatField
from playhouse.sqlite_ext import VirtualIntegerField
from playhouse.sqlite_ext import VirtualModel

class transaction(_transaction):
    def __init__(self, db, lock_type='deferred'):
        self.db = db
        self.lock_type = lock_type

    def _begin(self):
        self.db.begin(self.lock_type)

class APSWDatabase(SqliteExtDatabase):
    def __init__(self, database, timeout=None, **kwargs):
        self.timeout = timeout
        self._modules = {}
        super(APSWDatabase, self).__init__(database, **kwargs)

    def register_module(self, mod_name, mod_inst):
        self._modules[mod_name] = mod_inst

    def unregister_module(self, mod_name):
        del(self._modules[mod_name])

    def _connect(self, database, **kwargs):
        conn = apsw.Connection(database, **kwargs)
        if self.timeout is not None:
            conn.setbusytimeout(self.timeout)
        try:
            self._add_conn_hooks(conn)
        except:
            conn.close()
            raise
        return conn

    def _add_conn_hooks(self, conn):
        super(APSWDatabase, self)._add_conn_hooks(conn)
        self._load_modules(conn)  # APSW-only.

    def _load_modules(self, conn):
        for mod_name, mod_inst in self._modules.items():
            conn.createmodule(mod_name, mod_inst)
        return conn

    def _load_aggregates(self, conn):
        for name, (klass, num_params) in self._aggregates.items():
            def make_aggregate():
                instance = klass()
                return (instance, instance.step, instance.finalize)
            conn.createaggregatefunction(name, make_aggregate)

    def _load_collations(self, conn):
        for name, fn in self._collations.items():
            conn.createcollation(name, fn)

    def _load_functions(self, conn):
        for name, (fn, num_params) in self._functions.items():
            conn.createscalarfunction(name, fn, num_params)

    def _load_extensions(self, conn):
        conn.enableloadextension(True)
        for extension in self._extensions:
            conn.loadextension(extension)

    def load_extension(self, extension):
        self._extensions.add(extension)
        if not self.is_closed():
            conn = self.get_conn()
            conn.enableloadextension(True)
            conn.loadextension(extension)

    def _execute_sql(self, cursor, sql, params):
        cursor.execute(sql, params or ())
        return cursor

    def execute_sql(self, sql, params=None, require_commit=True):
        logger.debug((sql, params))
        with self.exception_wrapper():
            cursor = self.get_cursor()
            self._execute_sql(cursor, sql, params)
        return cursor

    def last_insert_id(self, cursor, model):
        if model._meta.auto_increment:
            return cursor.getconnection().last_insert_rowid()

    def rows_affected(self, cursor):
        return cursor.getconnection().changes()

    def begin(self, lock_type='deferred'):
        self.get_cursor().execute('begin %s;' % lock_type)

    def commit(self):
        self.get_cursor().execute('commit;')

    def rollback(self):
        self.get_cursor().execute('rollback;')

    def transaction(self, lock_type='deferred'):
        return transaction(self, lock_type)

    def savepoint(self, sid=None):
        return savepoint(self, sid)

def nh(s, v):
    if v is not None:
        return str(v)

class BooleanField(_BooleanField):
    def db_value(self, v):
        v = super(BooleanField, self).db_value(v)
        if v is not None:
            return v and 1 or 0

class DateField(_DateField):
    db_value = nh

class TimeField(_TimeField):
    db_value = nh

class DateTimeField(_DateTimeField):
    db_value = nh

class DecimalField(_DecimalField):
    db_value = nh
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.course_module import CourseDescriptor

from request_cache.middleware import RequestCache

from django.core.cache import get_cache, InvalidCacheBackendError

class Command(BaseCommand):
    help = '''Delete a MongoDB backed course'''

    def handle(self, *args, **options):
        if len(args) != 1 :
            raise CommandError("check_course requires one argument: <location>")

        loc_str = args[0]

        loc = CourseDescriptor.id_to_location(loc_str)

        ms = modulestore()

        # setup a request cache so we don't throttle the DB with all the metadata inheritance requests
        ms.request_cache = RequestCache.get_request_cache()

        course = ms.get_item(loc, depth=3)

        err_cnt = 0
        def _xlint_metadata(module):
            err_cnt = check_module_metadata_editability(module)
            for child in module.get_children():
                err_cnt = err_cnt + _xlint_metadata(child)
            return err_cnt

        _xlint_metadata(course)

        # we've had a bug where the xml_attributes field can we rewritten as a string rather than a dict
        def _check_xml_attributes_field(module):
            err_cnt = 0
            if hasattr(module, 'xml_attributes') and (isinstance(module, str) or isinstance(module, unicode)):
                print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location.url())
                err_cnt = err_cnt + 1
            for child in module.get_children():
                err_cnt = err_cnt + _check_xml_attributes_field(child)
            return err_cnt

        _check_xml_attributes_field(course)

        # check for dangling discussion items, this can cause errors in the forums
        def _get_discussion_items(module):
            discussion_items = []
            if module.location.category == 'discussion':
                discussion_items = discussion_items + [module.location.url()]

            for child in module.get_children():
                discussion_items = discussion_items + _get_discussion_items(child)

            return discussion_items

        discussion_items =_get_discussion_items(course)

        # now query all discussion items via get_items() and compare with the tree-traversal
        queried_discussion_items = ms.get_items(['i4x', course.location.org, course.location.course,
            'discussion', None, None])

        for item in queried_discussion_items:
            if item.location.url() not in discussion_items:
                print 'Found dangling discussion module = {0}'.format(item.location.url())

<commit_msg>add a command line utility to walk through the course and find common errors<commit_after>from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.course_module import CourseDescriptor

from request_cache.middleware import RequestCache

from django.core.cache import get_cache, InvalidCacheBackendError

class Command(BaseCommand):
    help = '''Delete a MongoDB backed course'''

    def handle(self, *args, **options):
        if len(args) != 1 :
            raise CommandError("check_course requires one argument: <location>")

        loc_str = args[0]

        loc = CourseDescriptor.id_to_location(loc_str)

        modulestore = modulestore()

        # setup a request cache so we don't throttle the DB with all the metadata inheritance requests
        modulestore.request_cache = RequestCache.get_request_cache()

        course = modulestore.get_item(loc, depth=3)

        err_cnt = 0
        def _xlint_metadata(module):
            err_cnt = check_module_metadata_editability(module)
            for child in module.get_children():
                err_cnt = err_cnt + _xlint_metadata(child)
            return err_cnt

        _xlint_metadata(course)

        # we've had a bug where the xml_attributes field can we rewritten as a string rather than a dict
        def _check_xml_attributes_field(module):
            err_cnt = 0
            if hasattr(module, 'xml_attributes') and (isinstance(module, str) or isinstance(module, unicode)):
                print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location.url())
                err_cnt = err_cnt + 1
            for child in module.get_children():
                err_cnt = err_cnt + _check_xml_attributes_field(child)
            return err_cnt

        _check_xml_attributes_field(course)

        # check for dangling discussion items, this can cause errors in the forums
        def _get_discussion_items(module):
            discussion_items = []
            if module.location.category == 'discussion':
                discussion_items = discussion_items + [module.location.url()]

            for child in module.get_children():
                discussion_items = discussion_items + _get_discussion_items(child)

            return discussion_items

        discussion_items =_get_discussion_items(course)

        # now query all discussion items via get_items() and compare with the tree-traversal
        queried_discussion_items = ms.get_items(['i4x', course.location.org, course.location.course,
            'discussion', None, None])

        for item in queried_discussion_items:
            if item.location.url() not in discussion_items:
                print 'Found dangling discussion module = {0}'.format(item.location.url())

<|endoftext|>


<commit_before>import logging
import socket
import datetime
import time
import os
import logging
import logging.handlers
import pprint
import threading
import anhost
import json

FORMAT = "[%(filename)s:%(lineno)s - %(threadName)s %(funcName)s] %(levelname)20s %(message)s"
logging.basicConfig(format=FORMAT)

logger = logging.getLogger("%s | %s | " % (os.getpid(), __file__) )
logger.setLevel(logging.DEBUG)
socketHandler = logging.handlers.SocketHandler('localhost',
                    logging.handlers.DEFAULT_TCP_LOGGING_PORT)
logger.addHandler(socketHandler)

## short simple code to just broadcast unicast message
def send_broadcast(local_ip,msg,port):
  logger.debug("\tSEND_BROADCAST")
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  prefix = '.'.join(local_ip.split(".")[0:3])+'.'
  suff = local_ip.split(".")[-1]
  logger.debug("\t\tbroadcasting to: %s*" % prefix)
  logger.debug("\t\tnot sending to: %s" % prefix+suff)
  for i in range(1,254):
    if i != int(suff):
      sock.sendto(msg, (prefix+str(i),port))
    else:
      logger.debug(i)
  logger.debug("broadcast done.")

def write_n_fi(n_fi, n_dict):
  x = open(n_fi,'w')
  for k,v in n_dict:
    x.write(k+','+v[0]+','+v[1])
  x.close()

def read_n_fi(n_fi):
  x = open(n_fi,'r')
  neighbor = {}
  for l in x:
    k = l.split(',')
    neighbor.update({k[0]:[k[1],k[2]]})
  x.close()
  return neighbor

def send_update(sock,n_fi,rip_port):
  logger.debug("sending update")
  n_dict = read_n_fi(n_fi)
  ip_self = sock.getsockname()[0]
  data_str = json.dumps(n_dict)
  send_broadcast(ip_self,data_str,rip_port)
  #FIXME, dont broadcast
  #for k in n_dict:
  #  if k != ip_self:
  #    sock.sendto(k,n_dict)
  logger.debug("update sent")

def send_handler(sock,n_fi,port):
  while True:
    logger.debug("Sending update")
    send_update(sock, n_fi,port)
    time.sleep(10)

def recv_update(neigh_fi,addr, update):
  add_list = []
  up_list = []
  dst_list = [x for x in up_list] # dst key list
  neighbors = read_n_fi(neigh_fi)
  logger.debug("\tRECV_UPDATE")
  logger.debug("\t\tupdate from: %s" % addr[0])
  logger.debug("\t\toriginal list: %s" % neighbors)
  logger.debug("\t\tneighbor's list: %s" % update)

  #add newly discovered nodes
  for node in update:
    cost = update[node][1] + 1 #add cost to us
    if node not in dst_list:
      add_list.append({node:[addr,cost]})
  #update old cost updates
    else:
      ## cost strictly less than, otherwise no upd
      if cost < neighbors[node][1]:
        up_list.append({node:[addr,cost]})

  logger.debug("added list: %s" % add_list)
  logger.debug("updated list: %s" % up_list)

  # need to update and return our modified dict
  dst_list = []
  if up_list:
    dst_list = [i.keys()[0] for i in up_list]
    #for i in up_list:
    #  dst_list.append(i.keys()[0])
  # new dict
  update_neighbors = {}
  # add new entries
  for k in add_list:
    update_nighbors.update(k)

  # add original entries
  for k in neighbors:
    # add original entries
    if k not in dst_list:
      update_neighbors.update(neighbors[k])
    # add updated entries
    else:
      for i in up_list:
        if k == i.keys()[0]:
           update_neighbors.update(up_list[i])

  logger.debug(pprint.pformat(update_neighbors))
  return update_neighbors

def recv_handler(rip_sock,n_fi):
  #try:
  while True:
    logger.debug("\taceepting messages...")
    msg, addr = rip_sock.recvfrom(4096)
    logger.debug("\tmessage: %s" % msg)
    logger.debug("\tsender's addr: (%s,%s)" % (addr[0],addr[1]))
    update = recv_update(n_fi,addr, json.loads(msg))
    write_n_fi(n_fi,update)
    logger.debug("\tupdate written out to file.")
    time.sleep(15)
  #except Exception,e:
  #  logger.error("Recver Error: %s" % str(e))

def rip_server(code, serv_port, rip_port,serv_fi):
  neigh = "%s.rip" % rip_port
  logger.debug("RIP SERVER:")
  logger.debug("PID: %s" % os.getpid())
  local_ip = anhost.get_ip_address("eth0")
  # dst : via, cost
  x = open(neigh,'w')
  x.close()
  write_n_fi(neigh,{local_ip:[local_ip,0]})

  logger.debug("\tinitial neighbor list: %s" % n_list)

  #set up rip server socket
  rip_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  rip_sock.bind((local_ip,rip_port))
  rip_sock.setblocking(1) # blocking recv

  ## start rip on neighbors
  send_broadcast(local_ip, code,serv_port)

  try:
    ## recver thread
    recv_thread = threading.Thread(target=recv_handler, args=(rip_sock,neigh,))
    recv_thread.start()
  except Exception,e:
    logger.error("Receving Thread Error")
    raise Exception(e)

  try:
    ## sender thread
    send_thread = threading.Thread(target=send_handler, args=(rip_sock,neigh,rip_port))
    send_thread.start()
  except Exception,e:
    logger.error("Sending Thread Error")
    raise Exception(e)

  try:
    recv_thread.join()
    send_thread.join()
  except KeyboardInterrupt:
    logging.info("\tServer killed by Ctrl-C")
    os.remove(serv_fi)
    os.remove(neigh)
    raise KeyboardInterrupt
  except Exception, e:
    logging.error("\tRIP Server Crash: %s" % e)
    os.remove(serv_fi)
    os.remove(neigh)
    raise Exception(e)
<commit_msg>edit for dict - bad code<commit_after>import logging
import socket
import datetime
import time
import os
import logging
import logging.handlers
import pprint
import threading
import anhost
import json

FORMAT = "[%(filename)s:%(lineno)s - %(threadName)s %(funcName)s] %(levelname)20s %(message)s"
logging.basicConfig(format=FORMAT)

logger = logging.getLogger("%s | %s | " % (os.getpid(), __file__) )
logger.setLevel(logging.DEBUG)
socketHandler = logging.handlers.SocketHandler('localhost',
                    logging.handlers.DEFAULT_TCP_LOGGING_PORT)
logger.addHandler(socketHandler)

## short simple code to just broadcast unicast message
def send_broadcast(local_ip,msg,port):
  logger.debug("\tSEND_BROADCAST")
  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  prefix = '.'.join(local_ip.split(".")[0:3])+'.'
  suff = local_ip.split(".")[-1]
  logger.debug("\t\tbroadcasting to: %s*" % prefix)
  logger.debug("\t\tnot sending to: %s" % prefix+suff)
  for i in range(1,254):
    if i != int(suff):
      sock.sendto(msg, (prefix+str(i),port))
    else:
      logger.debug(i)
  logger.debug("broadcast done.")

def write_n_fi(n_fi, n_dict):
  x = open(n_fi,'w')
  for k in n_dict:
    x.write(k+','+n_dict[k][0]+','+n_dict[k][1])
  x.close()

def read_n_fi(n_fi):
  x = open(n_fi,'r')
  neighbor = {}
  for l in x:
    k = l.split(',')
    neighbor.update({k[0]:[k[1],k[2]]})
  x.close()
  return neighbor

def send_update(sock,n_fi,rip_port):
  logger.debug("sending update")
  n_dict = read_n_fi(n_fi)
  ip_self = sock.getsockname()[0]
  data_str = json.dumps(n_dict)
  send_broadcast(ip_self,data_str,rip_port)
  #FIXME, dont broadcast
  #for k in n_dict:
  #  if k != ip_self:
  #    sock.sendto(k,n_dict)
  logger.debug("update sent")

def send_handler(sock,n_fi,port):
  while True:
    logger.debug("Sending update")
    send_update(sock, n_fi,port)
    time.sleep(10)

def recv_update(neigh_fi,addr, update):
  add_list = []
  up_list = []
  dst_list = [x for x in up_list] # dst key list
  neighbors = read_n_fi(neigh_fi)
  logger.debug("\tRECV_UPDATE")
  logger.debug("\t\tupdate from: %s" % addr[0])
  logger.debug("\t\toriginal list: %s" % neighbors)
  logger.debug("\t\tneighbor's list: %s" % update)

  #add newly discovered nodes
  for node in update:
    cost = update[node][1] + 1 #add cost to us
    if node not in dst_list:
      add_list.append({node:[addr,cost]})
  #update old cost updates
    else:
      ## cost strictly less than, otherwise no upd
      if cost < neighbors[node][1]:
        up_list.append({node:[addr,cost]})

  logger.debug("added list: %s" % add_list)
  logger.debug("updated list: %s" % up_list)

  # need to update and return our modified dict
  dst_list = []
  if up_list:
    dst_list = [i.keys()[0] for i in up_list]
    #for i in up_list:
    #  dst_list.append(i.keys()[0])
  # new dict
  update_neighbors = {}
  # add new entries
  for k in add_list:
    update_nighbors.update(k)

  # add original entries
  for k in neighbors:
    # add original entries
    if k not in dst_list:
      update_neighbors.update(neighbors[k])
    # add updated entries
    else:
      for i in up_list:
        if k == i.keys()[0]:
           update_neighbors.update(up_list[i])

  logger.debug(pprint.pformat(update_neighbors))
  return update_neighbors

def recv_handler(rip_sock,n_fi):
  #try:
  while True:
    logger.debug("\taceepting messages...")
    msg, addr = rip_sock.recvfrom(4096)
    logger.debug("\tmessage: %s" % msg)
    logger.debug("\tsender's addr: (%s,%s)" % (addr[0],addr[1]))
    update = recv_update(n_fi,addr, json.loads(msg))
    write_n_fi(n_fi,update)
    logger.debug("\tupdate written out to file.")
    time.sleep(15)
  #except Exception,e:
  #  logger.error("Recver Error: %s" % str(e))

def rip_server(code, serv_port, rip_port,serv_fi):
  neigh = "%s.rip" % rip_port
  logger.debug("RIP SERVER:")
  logger.debug("PID: %s" % os.getpid())
  local_ip = anhost.get_ip_address("eth0")
  # dst : via, cost
  x = open(neigh,'w')
  x.close()
  write_n_fi(neigh,{local_ip:[local_ip,0]})

  logger.debug("\tinitial neighbor list: %s" % n_list)

  #set up rip server socket
  rip_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
  rip_sock.bind((local_ip,rip_port))
  rip_sock.setblocking(1) # blocking recv

  ## start rip on neighbors
  send_broadcast(local_ip, code,serv_port)

  try:
    ## recver thread
    recv_thread = threading.Thread(target=recv_handler, args=(rip_sock,neigh,))
    recv_thread.start()
  except Exception,e:
    logger.error("Receving Thread Error")
    raise Exception(e)

  try:
    ## sender thread
    send_thread = threading.Thread(target=send_handler, args=(rip_sock,neigh,rip_port))
    send_thread.start()
  except Exception,e:
    logger.error("Sending Thread Error")
    raise Exception(e)

  try:
    recv_thread.join()
    send_thread.join()
  except KeyboardInterrupt:
    logging.info("\tServer killed by Ctrl-C")
    os.remove(serv_fi)
    os.remove(neigh)
    raise KeyboardInterrupt
  except Exception, e:
    logging.error("\tRIP Server Crash: %s" % e)
    os.remove(serv_fi)
    os.remove(neigh)
    raise Exception(e)
<|endoftext|>


<commit_before>from unittest import skipIf
from django.urls import reverse
from django.test import Client
from django.test.utils import override_settings
from myuw.models import VisitedLinkNew
from myuw.views.link import outbound_link
from myuw.test.api import missing_url, require_url, MyuwApiTest
from django.urls import reverse_lazy

@require_url('myuw_outbound_link')
@override_settings(LOGIN_URL=reverse_lazy('saml_login'))
class TestViewsLinkAdmin(MyuwApiTest):

    @skipIf(missing_url("myuw_outbound_link"),
            "myuw_outbound_link urls not configured")
    def test_outbound_link(self):
        link_url = "https%3A%2F%2Fhr.uw.edu%2F"
        label = ('.......................................................'
                 '.......................................................')
        url = "/out/?u={}&l={}".format(link_url, label)
        self.set_user('javerage')
        response = self.client.get(url)
        self.assertEquals(response.status_code, 302)
        vlinks = VisitedLinkNew.objects.all()
        self.assertEquals(len(vlinks), 1)
<commit_msg>more tests<commit_after>from unittest import skipIf
from django.urls import reverse
from django.test import Client
from django.test.utils import override_settings
from myuw.models import VisitedLinkNew
from myuw.views.link import outbound_link
from myuw.test.api import missing_url, require_url, MyuwApiTest
from django.urls import reverse_lazy

@require_url('myuw_outbound_link')
@override_settings(LOGIN_URL=reverse_lazy('saml_login'))
class TestViewsLinkAdmin(MyuwApiTest):

    @skipIf(missing_url("myuw_outbound_link"),
            "myuw_outbound_link urls not configured")
    def test_outbound_link(self):
        link_url = "https%3A%2F%2Fhr.uw.edu%2F"
        label = ('.......................................................'
                 '.......................................................')
        url = "/out/?u={}&l={}".format(link_url, label)
        self.set_user('javerage')
        response = self.client.get(url)
        self.assertEquals(response.status_code, 302)
        vlinks = VisitedLinkNew.objects.all()
        self.assertEquals(len(vlinks), 1)
        VisitedLinkNew.objects.all().delete()

        url = "/out?u={}&l={}".format(
            "https%3A%2F%2Fcoda.uw.edu%2F%232020-spring-TRAIN-100-A",
            "Course%20Dashboard%20for%20TRAIN%20100%20A")
        response = self.client.get(url)
        self.assertEquals(response.status_code, 302)
        vlinks = VisitedLinkNew.objects.all()
        self.assertEquals(len(vlinks), 1)
        self.assertEquals(vlinks[0].label, "Course Dashboard for TRAIN 100 A")
        VisitedLinkNew.objects.all().delete()

        url = "/out?u={}&l=".format(
            "https%3A%2F%2Fcoda.uw.edu%2F%232020-spring-TRAIN-100-A")
        response = self.client.get(url)
        self.assertEquals(response.status_code, 302)
        vlinks = VisitedLinkNew.objects.all()
        self.assertEquals(len(vlinks), 1)
        self.assertEquals(vlinks[0].label, "")
<|endoftext|>


<commit_before>from person import Fellow, Staff
from room import Office, LivingSpace

class Amity(object):
    ''' Amity Class

        Amity is a class that models a room allocation system for one of
        Andela's facilities called Amity. Amity creates rooms which can
        be offices or living spaces. An office can occupy a maximum of 6
        people. A living space can inhabit a maximum of 4 people. It also
        creates and allocates a person rooms. A person to be allocated
        could be a fellow or staff. Staff cannot be allocated living
        spaces. Fellows have a choice to choose a living space or not.
        This system will be used to automatically allocate spaces to
        people at random.

        Attributes:
            rooms (dict): This is a dict of all Room IDs of rooms in the
            system.
            persons (dict): This is a dict of all UUIDs of persons in the
            system.
            total_rooms (int): This is an integer representing all the
            rooms in the system.
            total_persons (int): This is an integer representing all the
            persons in the system.
            room_allocations (dict): This is a dictionary that holds all
            the data on room allocations in the system.

    '''
    rooms = {"Offices": [], "Living Spaces": []}
    persons = {"Fellows": [], "Staff": []}
    total_rooms = 0
    total_persons = 0
    room_allocations = {}

    def __init__(self):
        pass

    def create_room(self, name):
        """
        Create a room in Amity.

        This method creates a room or multiple of rooms depending on the
        amount of items in the list it receives. So, it receives a list.
        This method should also check that no duplicate rooms can be created.
        Another thing to be noted is that this method will create an Office by
        default unless the last string in the arguments is "-ls" which will
        create Living Spaces

        Parameters
        ----------
        name : list
            This is a list of names for the rooms to be created.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def add_person(self, name, role, allocate="N"):
        """
        Add Person and allocate random room.

        This method adds a person to the Amity System and immediately
        allocates the person a random room.

        Parameters
        ----------
        name : str
            This is a string representing the full name of the person.
        role : str
            This is a string of the person's role.
        allocate : str
            This is a boolean string that can either be "N" for "No"
            or "Y" for "Yes". It tells us if the person needs
            accommodation.

        Returns
        -------
        dict
            This is a dictionary containing the details of the newly
            added person or containing an error message when the
            operation is not successful.

        """
        pass

    def get_person_details(self, uuid):
        """
        Get Person details.

        This method fetches a Person's details using the provided id.

        Parameters
        ----------
        name : str
            This is a string representing the id of the person.

        Returns
        -------
        dict
            This is a dictionary containing the details of the
            a person.

        """
        pass

    def get_current_occupants(self, r_id):
        """
        Get specific Room occupants.

        This method fetches all the Persons in a particular room
        given the room ID.

        Parameters
        ----------
        r_id : str
            This is a string representing the id of the room.

        Returns
        -------
        dict
            This is a dictionary containing the details of
            current occupants.

        """
        pass

    def reallocate_person(self, uuid, room_name):
        """
        Reallocate a Person.

        This method reallocates a Person from a room to another room
        given the destination room name and the Unique User ID (uuid)
        of the user.

        Parameters
        ----------
        uuid : str
            This is a string representing the unique id of the Person.
        room_name : str
            This is a string representing name of the room the Person
            is to be reallocated.

        Returns
        -------
        dict
            This is a dictionary containing the details of the room
            the Person has been reallocated to together with all the
            occupants.

        """
        pass

    def get_empty_rooms(self):
        """
        Get all empty room in the system.

        This method goes through the entire system and looks for all
        rooms that have not been allocated anyone. In other words, all
        empty rooms in the system.

        Returns
        -------
        list
            This is a list of all the empty rooms in the system.

        """
        pass

    def print_allocations(self, param=None):
        """
        Print Room allocations.

        This method prints a list of allocations onto the screen.
        Specifying the optional -o option here outputs the registered
        allocations to a txt file.

        Parameters
        ----------
        param : str
            This is an optional string that specifies whether the
            output should be saved on a txt file.

        Returns
        -------
        dict
            This is a dictionary containing the details of all the
            allocation. The dictionary may be returned or saved in a
            file depending on the parameter passed.

        """
        pass

    def print_unallocated(self, param=None):
        """
        Print Room Unallocations.

        This method prints a list of unallocated people to the screen.
        Specifying the -o option here outputs the information to the
        txt file provided.

        Parameters
        ----------
        param : str
            This is an optional string that specifies whether the
            output should be saved on a txt file.

        Returns
        -------
        dict
            This is a dictionary containing the details of all the
            unallocated Persons. The dictionary may be returned or saved in a
            file depending on the parameter passed.

        """
        pass

    def print_room(self, r_id):
        """
        Print Room.

        This method prints all the details of a specific Room given
        the Room Id.

        Parameters
        ----------
        r_id : str
            This is a string representing the id of the Room to be
            printed.

        Returns
        -------
        dict
            This is a dictionary containing the details of the
            Room.

        """
        pass

    def load_people(self, infile):
        """
        Load People

        This method load people from a text file and adds them to the
        Amity System.

        Parameters
        ----------
        infile : str
            This is a string representing the name of the file that
            contains the people to be loaded.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass

    def save_state(self, outfile):
        """
        Persist State

        This method persists the state of the application by saving the
        current working data to a DB.

        Parameters
        ----------
        outfile : str
            This is a string representing the name of the DB to save the
            current state.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass

    def load_state(self, infile):
        """
        Load State

        This method loads a previously saved state for the application
        to resume fro that point the DB was created.

        Parameters
        ----------
        infile : str
            This is a string representing the name of the DB to load
            the state from.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass
<commit_msg>A few tweaks here and there :hammer:<commit_after>from person import Fellow, Staff
from room import Office, LivingSpace

class Amity(object):
    ''' Amity Class

        Amity is a class that models a room allocation system for one of
        Andela's facilities called Amity. Amity creates rooms which can
        be offices or living spaces. An office can occupy a maximum of 6
        people. A living space can inhabit a maximum of 4 people. It also
        creates and allocates a person rooms. A person to be allocated
        could be a fellow or staff. Staff cannot be allocated living
        spaces. Fellows have a choice to choose a living space or not.
        This system will be used to automatically allocate spaces to
        people at random.

        Attributes:
            rooms (dict): This is a dict of all Room IDs of rooms in the
            system.
            persons (dict): This is a dict of all UUIDs of persons in the
            system.
            total_rooms (int): This is an integer representing all the
            rooms in the system.
            total_persons (int): This is an integer representing all the
            persons in the system.
            room_allocations (dict): This is a dictionary that holds all
            the data on room allocations in the system.

    '''
    rooms = {"Offices": {}, "Living Spaces": {}}
    persons = {"Fellows": {}, "Staff": {}}
    total_rooms = 0
    total_persons = 0
    room_allocations = {}

    def __init__(self):
        pass

    def create_room(self, name):
        """
        Create a room in Amity.

        This method creates a room or multiple of rooms depending on the
        amount of items in the list it receives. So, it receives a list.
        This method should also check that no duplicate rooms can be created.
        Another thing to be noted is that this method will create an Office by
        default unless the last string in the arguments is "-ls" which will
        create Living Spaces

        Parameters
        ----------
        name : list
            This is a list of names for the rooms to be created.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def add_person(self, name, role, allocate="N"):
        """
        Add Person and allocate random room.

        This method adds a person to the Amity System and immediately
        allocates the person a random room.

        Parameters
        ----------
        name : str
            This is a string representing the full name of the person.
        role : str
            This is a string of the person's role.
        allocate : str
            This is a boolean string that can either be "N" for "No"
            or "Y" for "Yes". It tells us if the person needs
            accommodation.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def get_person_details(self, uuid):
        """
        Get Person details.

        This method fetches a Person's details using the provided id.

        Parameters
        ----------
        name : str
            This is a string representing the id of the person.

        Returns
        -------
        dict
            This is a dictionary containing the details of
            a person.

        """
        pass

    def get_current_occupants(self, r_id):
        """
        Get specific Room occupants.

        This method fetches all the Persons in a particular room
        given the room ID.

        Parameters
        ----------
        r_id : str
            This is a string representing the id of the room.

        Returns
        -------
        dict
            This is a dictionary containing the details of
            current occupants.

        """
        pass

    def reallocate_person(self, uuid, room_name):
        """
        Reallocate a Person.

        This method reallocates a Person from a room to another room
        given the destination room name and the Unique User ID (uuid)
        of the user.

        Parameters
        ----------
        uuid : str
            This is a string representing the unique id of the Person.
        room_name : str
            This is a string representing name of the room the Person
            is to be reallocated.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def get_empty_rooms(self):
        """
        Get all empty room in the system.

        This method goes through the entire system and looks for all
        rooms that have not been allocated anyone. In other words, all
        empty rooms in the system.

        Returns
        -------
        list
            This is a list of all the empty rooms in the system.

        """
        pass

    def print_allocations(self, param=None):
        """
        Print Room allocations.

        This method prints a list of allocations onto the screen.
        Specifying the optional -o option here outputs the registered
        allocations to a txt file.

        Parameters
        ----------
        param : str
            This is an optional string that specifies whether the
            output should be saved on a txt file.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def print_unallocated(self, param=None):
        """
        Print Room Unallocations.

        This method prints a list of unallocated people to the screen.
        Specifying the -o option here outputs the information to the
        txt file provided.

        Parameters
        ----------
        param : str
            This is an optional string that specifies whether the
            output should be saved on a txt file.

        Returns
        -------
        string
            This is a string showing success or failure of the operation.

        """
        pass

    def print_room(self, r_id):
        """
        Print Room.

        This method prints all the details of a specific Room given
        the Room Id.

        Parameters
        ----------
        r_id : str
            This is a string representing the id of the Room to be
            printed.

        Returns
        -------
        dict
            This is a dictionary containing the details of the
            Room.

        """
        pass

    def load_people(self, infile):
        """
        Load People

        This method load people from a text file and adds them to the
        Amity System.

        Parameters
        ----------
        infile : str
            This is a string representing the name of the file that
            contains the people to be loaded.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass

    def save_state(self, outfile):
        """
        Persist State

        This method persists the state of the application by saving the
        current working data to a DB.

        Parameters
        ----------
        outfile : str
            This is a string representing the name of the DB to save the
            current state.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass

    def load_state(self, infile):
        """
        Load State

        This method loads a previously saved state for the application
        to resume fro that point the DB was created.

        Parameters
        ----------
        infile : str
            This is a string representing the name of the DB to load
            the state from.

        Returns
        -------
        string
            This is a string representing a successful operation or
            a failure

        """
        pass
<|endoftext|>


<commit_before>"""pathscanner.py Scans paths and stores them in a sqlite3 database

You can use PathScanner to detect changes in a directory structure. For
efficiency, only creations, deletions and modifications are detected, not
moves.

Modified files are detected by looking at the mtime.

Instructions:
- Use initial_scan() to build the initial database.
- Use scan() afterwards, to get the changes.
- Use scan_tree() (which uses scan()) to get the changes in an entire
  directory structure.
- Use purge_path() to purge all the metadata for a path from the database.
- Use (add|update|remove)_files() to add/update/remove files manually (useful
  when your application has more/faster knowledge of changes)

TODO: unit tests (with *many* mock functions). Stable enough without them.
"""

__author__ = "Wim Leers (work@wimleers.com)"
__version__ = "$Rev$"
__date__ = "$Date$"
__license__ = "GPL"

import os
import stat
import sqlite3
from sets import Set

class PathScanner(object):
    """scan paths for changes, persistent storage using SQLite"""
    def __init__(self, dbcon, table="pathscanner", commit_interval=50):
        self.dbcon = dbcon
        self.dbcur = dbcon.cursor()
        self.table = table
        self.uncommitted_statements = 0
        self.commit_interval = commit_interval
        self.__prepare_db()

    def __prepare_db(self):
        """prepare the database (create the table structure)"""

        self.dbcur.execute("CREATE TABLE IF NOT EXISTS %s(path text, filename text, mtime integer)" % (self.table))
        self.dbcon.commit()

    def __walktree(self, path):
        rows = []
        for path, filename, mtime, is_dir in self.__listdir(path):
            rows.append((path, filename, mtime if not is_dir else -1))
            if is_dir:
                for childrows in self.__walktree(os.path.join(path, filename)):
                    yield childrows
        yield rows

    def __listdir(self, path):
        """list all the files in a directory

        Returns (path, filename, mtime, is_dir) tuples.
        """

        filenames = os.listdir(path)
        for filename in filenames:
            try:
                st = os.stat(os.path.join(path, filename))
                mtime = st[stat.ST_MTIME]
                is_dir = stat.S_ISDIR(st.st_mode)
                row = (path, filename, mtime, is_dir)
            except os.error:
                continue
            yield row

    def initial_scan(self, path):
        """perform the initial scan

        Returns False if there is already data available for this path.
        """

        # Check if there really isn't any data available for this path.
        self.dbcur.execute("SELECT COUNT(filename) FROM %s WHERE path=?" % (self.table), (path,))
        if self.dbcur.fetchone()[0] > 0:
            return False

        for files in self.__walktree(path):
            self.add_files(files)

    def purge_path(self, path):
        """purge the metadata for a given path and all its subdirectories"""

        self.dbcur.execute("DELETE FROM %s WHERE path LIKE ?" % (self.table), (path + "%",))
        self.dbcur.execute("VACUUM %s" % (self.table))
        self.dbcon.commit()

    def add_files(self, files):
        """add file metadata to the database

        Expected format: a set of (path, filename, mtime) tuples.
        """

        for row in files:
            self.dbcur.execute("INSERT INTO %s VALUES(?, ?, ?)" % (self.table), row)
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def update_files(self, files):
        """update file metadata in the database

        Expected format: a set of (path, filename, mtime) tuples.
        """

        for row in files:
            (path, filename, mtime) = row
            self.dbcur.execute("UPDATE %s SET mtime=? WHERE path=? AND filename=?" % (self.table), (mtime, path, filename))
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def delete_files(self, files):
        """delete file metadata from the database

        Expected format: a set of (path, filename) tuples.
        """

        for row in files:
            self.dbcur.execute("DELETE FROM %s WHERE path=? AND filename=?" % (self.table), row)
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def __db_batched_commit(self, force=False):
        """docstring for __db_commit"""
        # Commit to the database in batches, to reduce concurrency: collect
        # self.commit_interval rows, then commit.

        self.uncommitted_statements += 1
        if force == True or self.uncommitted_statements == self.commit_interval:
            self.dbcon.commit()
            self.uncommitted_rows = 0

    def scan(self, path):
        """scan a directory (without recursion!) for changes

        The database is also updated to reflect the new situation, of course.

        By design, so that this function can be used by scan_tree():
        - Cannot detect newly created directory trees.
        - Can detect deleted directory trees.
        """

        # Fetch the old metadata from the DB.
        self.dbcur.execute("SELECT filename, mtime FROM %s WHERE path=?" % (self.table), (path, ))
        old_files = {}
        for filename, mtime in self.dbcur.fetchall():
            old_files[filename] = (filename, mtime)

        # Get the current metadata.
        new_files = {}
        for path, filename, mtime, is_dir in self.__listdir(path):
            new_files[filename] = (filename, mtime if not is_dir else -1)

        scan_result = self.__scanhelper(path, old_files, new_files)

        # Add the created files to the DB.
        files = Set()
        for filename in scan_result["created"]:
            (filename, mtime) = new_files[filename]
            files.add((path, filename, mtime))
        self.add_files(files)
        # Update the modified files in the DB.
        files = Set()
        for filename in scan_result["modified"]:
            (filename, mtime) = new_files[filename]
            files.add((path, filename, mtime))
        self.update_files(files)
        # Remove the deleted files from the DB.
        files = Set()
        for filename in scan_result["deleted"]:
            if len(os.path.dirname(filename)):
                realpath = path + os.sep + os.path.dirname(filename)
            else:
                realpath = path
            realfilename = os.path.basename(filename)
            files.add((realpath, realfilename))
        self.delete_files(files)

        return scan_result

    def scan_tree(self, path):
        """scan a directory tree for changes"""

        # Scan the current directory for changes.
        result = self.scan(path)

        # Prepend the current path.
        for key in result.keys():
            tmp = Set()
            for filename in result[key]:
                tmp.add(path + os.sep + filename)
            result[key] = tmp
        yield (path, result)

        # Also scan each subdirectory.
        filenames = os.listdir(path)
        for filename in filenames:
            try:
                st = os.stat(os.path.join(path, filename))
                is_dir = stat.S_ISDIR(st.st_mode)
            except os.error:
                continue
            if is_dir:
                for subpath, subresult in self.scan_tree(os.path.join(path, filename)):
                    yield (subpath, subresult)

    def __scanhelper(self, path, old_files, new_files):
        """helper function for scan()

        old_files and new_files should be dictionaries of (filename, mtime)
        tuples, keyed by filename

        Returns a dictionary of sets of filenames with the keys "created",
        "deleted" and "modified".
        """

        # The dictionary that will be returned.
        result = {}
        result["created"] = Set()
        result["deleted"] = Set()
        result["modified"] = Set()

        # Create some sets that will make our work easier.
        old_filenames = Set(old_files.keys())
        new_filenames = Set(new_files.keys())

        # Step 1: find newly created files.
        result["created"] = new_filenames.difference(old_filenames)

        # Step 2: find deleted files.
        result["deleted"] = old_filenames.difference(new_filenames)

        # Step 3: find modified files.
        # Only files that are not created and not deleted can be modified!
        possibly_modified_files = new_filenames.union(old_filenames)
        possibly_modified_files = possibly_modified_files.symmetric_difference(result["created"])
        possibly_modified_files = possibly_modified_files.symmetric_difference(result["deleted"])
        for filename in possibly_modified_files:
            (filename, old_mtime) = old_files[filename]
            (filename, new_mtime) = new_files[filename]
            if old_mtime != new_mtime:
                result["modified"].add(filename)

        # Step 4
        # If a directory was deleted, we also need to retrieve the filenames
        # and paths of the files within that subtree.
        deleted_tree = Set()
        for deleted_file in result["deleted"]:
            (filename, mtime) = old_files[deleted_file]
            # An mtime of -1 means that this is a directory.
            if mtime == -1:
                dirpath = path + os.sep + filename
                self.dbcur.execute("SELECT * FROM %s WHERE path LIKE ?" % (self.table), (dirpath + "%",))
                files_in_dir = self.dbcur.fetchall()
                # Mark all files below the deleted directory also as deleted.
                for (subpath, subfilename, submtime) in files_in_dir:
                    deleted_tree.add(os.path.join(subpath, subfilename)[len(path) + 1:])
        result["deleted"] = result["deleted"].union(deleted_tree)

        return result

if __name__ == "__main__":
    # Sample usage
    path = "/Users/wimleers/Downloads"
    db = sqlite3.connect("pathscanner.db")
    scanner = PathScanner(db)
    # Force a rescan
    #scanner.purge_path(path)
    scanner.initial_scan(path)

    # Detect changes in a single directory
    #print scanner.scan(path)

    # Detect changes in the entire tree
    report = {}
    report["created"] = Set()
    report["deleted"] = Set()
    report["modified"] = Set()
    for path, result in scanner.scan_tree(path):
        report["created"] = report["created"].union(result["created"])
        report["deleted"] = report["deleted"].union(result["deleted"])
        report["modified"] = report["modified"].union(result["modified"])
    print report<commit_msg>Make pathscanner slightly more robust.<commit_after>"""pathscanner.py Scans paths and stores them in a sqlite3 database

You can use PathScanner to detect changes in a directory structure. For
efficiency, only creations, deletions and modifications are detected, not
moves.

Modified files are detected by looking at the mtime.

Instructions:
- Use initial_scan() to build the initial database.
- Use scan() afterwards, to get the changes.
- Use scan_tree() (which uses scan()) to get the changes in an entire
  directory structure.
- Use purge_path() to purge all the metadata for a path from the database.
- Use (add|update|remove)_files() to add/update/remove files manually (useful
  when your application has more/faster knowledge of changes)

TODO: unit tests (with *many* mock functions). Stable enough without them.
"""

__author__ = "Wim Leers (work@wimleers.com)"
__version__ = "$Rev$"
__date__ = "$Date$"
__license__ = "GPL"

import os
import stat
import sqlite3
from sets import Set

class PathScanner(object):
    """scan paths for changes, persistent storage using SQLite"""
    def __init__(self, dbcon, table="pathscanner", commit_interval=50):
        self.dbcon = dbcon
        self.dbcur = dbcon.cursor()
        self.table = table
        self.uncommitted_statements = 0
        self.commit_interval = commit_interval
        self.__prepare_db()

    def __prepare_db(self):
        """prepare the database (create the table structure)"""

        self.dbcur.execute("CREATE TABLE IF NOT EXISTS %s(path text, filename text, mtime integer)" % (self.table))
        self.dbcon.commit()

    def __walktree(self, path):
        rows = []
        for path, filename, mtime, is_dir in self.__listdir(path):
            rows.append((path, filename, mtime if not is_dir else -1))
            if is_dir:
                for childrows in self.__walktree(os.path.join(path, filename)):
                    yield childrows
        yield rows

    def __listdir(self, path):
        """list all the files in a directory

        Returns (path, filename, mtime, is_dir) tuples.
        """

        try:
            filenames = os.listdir(path)
        except os.error:
            return

        for filename in filenames:
            try:
                st = os.stat(os.path.join(path, filename))
                mtime = st[stat.ST_MTIME]
                is_dir = stat.S_ISDIR(st.st_mode)
                row = (path, filename, mtime, is_dir)
            except os.error:
                continue
            yield row

    def initial_scan(self, path):
        """perform the initial scan

        Returns False if there is already data available for this path.
        """

        # Check if there really isn't any data available for this path.
        self.dbcur.execute("SELECT COUNT(filename) FROM %s WHERE path=?" % (self.table), (path,))
        if self.dbcur.fetchone()[0] > 0:
            return False

        for files in self.__walktree(path):
            self.add_files(files)

    def purge_path(self, path):
        """purge the metadata for a given path and all its subdirectories"""

        self.dbcur.execute("DELETE FROM %s WHERE path LIKE ?" % (self.table), (path + "%",))
        self.dbcur.execute("VACUUM %s" % (self.table))
        self.dbcon.commit()

    def add_files(self, files):
        """add file metadata to the database

        Expected format: a set of (path, filename, mtime) tuples.
        """

        for row in files:
            self.dbcur.execute("INSERT INTO %s VALUES(?, ?, ?)" % (self.table), row)
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def update_files(self, files):
        """update file metadata in the database

        Expected format: a set of (path, filename, mtime) tuples.
        """

        for row in files:
            (path, filename, mtime) = row
            self.dbcur.execute("UPDATE %s SET mtime=? WHERE path=? AND filename=?" % (self.table), (mtime, path, filename))
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def delete_files(self, files):
        """delete file metadata from the database

        Expected format: a set of (path, filename) tuples.
        """

        for row in files:
            self.dbcur.execute("DELETE FROM %s WHERE path=? AND filename=?" % (self.table), row)
            self.__db_batched_commit()
        # Commit the remaining rows.
        self.__db_batched_commit(True)

    def __db_batched_commit(self, force=False):
        """docstring for __db_commit"""
        # Commit to the database in batches, to reduce concurrency: collect
        # self.commit_interval rows, then commit.

        self.uncommitted_statements += 1
        if force == True or self.uncommitted_statements == self.commit_interval:
            self.dbcon.commit()
            self.uncommitted_rows = 0

    def scan(self, path):
        """scan a directory (without recursion!) for changes

        The database is also updated to reflect the new situation, of course.

        By design, so that this function can be used by scan_tree():
        - Cannot detect newly created directory trees.
        - Can detect deleted directory trees.
        """

        # Fetch the old metadata from the DB.
        self.dbcur.execute("SELECT filename, mtime FROM %s WHERE path=?" % (self.table), (path, ))
        old_files = {}
        for filename, mtime in self.dbcur.fetchall():
            old_files[filename] = (filename, mtime)

        # Get the current metadata.
        new_files = {}
        for path, filename, mtime, is_dir in self.__listdir(path):
            new_files[filename] = (filename, mtime if not is_dir else -1)

        scan_result = self.__scanhelper(path, old_files, new_files)

        # Add the created files to the DB.
        files = Set()
        for filename in scan_result["created"]:
            (filename, mtime) = new_files[filename]
            files.add((path, filename, mtime))
        self.add_files(files)
        # Update the modified files in the DB.
        files = Set()
        for filename in scan_result["modified"]:
            (filename, mtime) = new_files[filename]
            files.add((path, filename, mtime))
        self.update_files(files)
        # Remove the deleted files from the DB.
        files = Set()
        for filename in scan_result["deleted"]:
            if len(os.path.dirname(filename)):
                realpath = path + os.sep + os.path.dirname(filename)
            else:
                realpath = path
            realfilename = os.path.basename(filename)
            files.add((realpath, realfilename))
        self.delete_files(files)

        return scan_result

    def scan_tree(self, path):
        """scan a directory tree for changes"""

        # Scan the current directory for changes.
        result = self.scan(path)

        # Prepend the current path.
        for key in result.keys():
            tmp = Set()
            for filename in result[key]:
                tmp.add(path + os.sep + filename)
            result[key] = tmp
        yield (path, result)

        # Also scan each subdirectory.
        filenames = os.listdir(path)
        for filename in filenames:
            try:
                st = os.stat(os.path.join(path, filename))
                is_dir = stat.S_ISDIR(st.st_mode)
            except os.error:
                continue
            if is_dir:
                for subpath, subresult in self.scan_tree(os.path.join(path, filename)):
                    yield (subpath, subresult)

    def __scanhelper(self, path, old_files, new_files):
        """helper function for scan()

        old_files and new_files should be dictionaries of (filename, mtime)
        tuples, keyed by filename

        Returns a dictionary of sets of filenames with the keys "created",
        "deleted" and "modified".
        """

        # The dictionary that will be returned.
        result = {}
        result["created"] = Set()
        result["deleted"] = Set()
        result["modified"] = Set()

        # Create some sets that will make our work easier.
        old_filenames = Set(old_files.keys())
        new_filenames = Set(new_files.keys())

        # Step 1: find newly created files.
        result["created"] = new_filenames.difference(old_filenames)

        # Step 2: find deleted files.
        result["deleted"] = old_filenames.difference(new_filenames)

        # Step 3: find modified files.
        # Only files that are not created and not deleted can be modified!
        possibly_modified_files = new_filenames.union(old_filenames)
        possibly_modified_files = possibly_modified_files.symmetric_difference(result["created"])
        possibly_modified_files = possibly_modified_files.symmetric_difference(result["deleted"])
        for filename in possibly_modified_files:
            (filename, old_mtime) = old_files[filename]
            (filename, new_mtime) = new_files[filename]
            if old_mtime != new_mtime:
                result["modified"].add(filename)

        # Step 4
        # If a directory was deleted, we also need to retrieve the filenames
        # and paths of the files within that subtree.
        deleted_tree = Set()
        for deleted_file in result["deleted"]:
            (filename, mtime) = old_files[deleted_file]
            # An mtime of -1 means that this is a directory.
            if mtime == -1:
                dirpath = path + os.sep + filename
                self.dbcur.execute("SELECT * FROM %s WHERE path LIKE ?" % (self.table), (dirpath + "%",))
                files_in_dir = self.dbcur.fetchall()
                # Mark all files below the deleted directory also as deleted.
                for (subpath, subfilename, submtime) in files_in_dir:
                    deleted_tree.add(os.path.join(subpath, subfilename)[len(path) + 1:])
        result["deleted"] = result["deleted"].union(deleted_tree)

        return result

if __name__ == "__main__":
    # Sample usage
    path = "/Users/wimleers/Downloads"
    db = sqlite3.connect("pathscanner.db")
    scanner = PathScanner(db)
    # Force a rescan
    #scanner.purge_path(path)
    scanner.initial_scan(path)

    # Detect changes in a single directory
    #print scanner.scan(path)

    # Detect changes in the entire tree
    report = {}
    report["created"] = Set()
    report["deleted"] = Set()
    report["modified"] = Set()
    for path, result in scanner.scan_tree(path):
        report["created"] = report["created"].union(result["created"])
        report["deleted"] = report["deleted"].union(result["deleted"])
        report["modified"] = report["modified"].union(result["modified"])
    print report<|endoftext|>