<commit_before>#!/usr/bin/env python

# by Al Nikolov <root@toor.fi.eu.org>

import bz2
import ConfigParser
import glob
import httplib
import imp
try:
    import json as serialize
except ImportError:
    import simplejson as serialize
import logging
import os
import pickle
import Queue
import signal
import StringIO
import subprocess
import sys
import threading
import time
import types

ini_files = (
    os.path.join('etc', 'nixstats.ini'),
    os.path.abspath('nixstats.ini'),
)

def run_agent():
    Agent().run()

def _plugin_name(plugin):
    if isinstance(plugin, basestring):
        basename = os.path.basename(plugin)
        return os.path.splitext(basename)[0]
    else:
        return plugin.__name__

class Agent:

    execute = Queue.Queue()
    metrics = Queue.Queue()
    data = Queue.Queue()
    shutdown = False

    def __init__(self):
        """
        Initialize internal strictures
        """
        self._config_init()
        self._logging_init()
        self._plugins_init()
        self._data_worker_init()
        self._dump_config()

    def _config_init(self):
        """
        Initialize configuration object
        """
        defaults = {
            'max_data_span': 60,
            'max_data_age': 60 * 10,
            'logging_level': logging.WARNING,
            'threads': 100,
            'ttl': 60,
            'interval': 60,
            'plugins': 'plugins',
            'enabled': 'no',
            'subprocess': 'no',
            'user': '',
            'server': '',
            'api_host': 'api.nixstats.com',
            'api_path': '/v2/server/poll',
        }
        sections = [
            'agent',
            'execution',
            'data',
        ]
        config = ConfigParser.RawConfigParser(defaults)
        config.read(ini_files)
        self.config = config
        for section in sections:
            self._config_section_create(section)

    def _config_section_create(self, section):
        """
        Create an addition section in the configuration object
        """
        if not self.config.has_section(section):
            self.config.add_section(section)

    def _logging_init(self):
        """
        Initialize logging faculty
        """
        level = self.config.getint('agent', 'logging_level')
        logging.basicConfig(level=level)
        logging.info('Agent logging_level %i', level)

    def _plugins_init(self):
        """
        Discover the plugins
        """
        logging.info('_plugins_init')
        path = self.config.get('agent', 'plugins')
        filenames = glob.glob(os.path.join(path, '*.py'))
        sys.path.insert(0, path)
        self.schedule = {}
        for filename in filenames:
            name = _plugin_name(filename)
            if name == 'plugins':
                continue
            self._config_section_create(name)
            if self.config.getboolean(name, 'enabled'):
                if self.config.getboolean(name, 'subprocess'):
                    self.schedule[filename] = 0
                else:
                    fp, pathname, description = \
                        imp.find_module(name)
                    module = None
                    try:
                        module = imp.load_module(
                                    name, fp, pathname, description)
                    finally:
                        # Since we may exit via an exception, close fp explicitly.
                        if fp:
                            fp.close()
                    if module:
                        self.schedule[module] = 0
                    else:
                        logging.error('import_plugin:%s:%s', name, sys.exc_type)

    def _subprocess_execution(self, task):
        """
        Execute /task/ in a subprocess
        """
        process = subprocess.Popen((sys.executable, task, ini_file),
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            universal_newlines=True)
        logging.info('%s:process:%i', threading.currentThread(), process.pid)
        interval = self.config.getint('execution', 'interval')
        name = _plugin_name(task)
        ttl = self.config.getint(name, 'ttl')
        ticks = ttl / interval or 1
        process.poll()
        while process.returncode is None and ticks > 0:
            logging.info('%s:tick:%i', threading.currentThread(), ticks)
            time.sleep(interval)
            ticks -= 1
            process.poll()
        if process.returncode is None:
            logging.error('%s:kill:%i', threading.currentThread(), process.pid)
            os.kill(process.pid, signal.SIGTERM)
        stdout, stderr = process.communicate()
        if process.returncode != 0 or stderr:
            logging.error('%s:%s:%s:%s', threading.currentThread(), task, process.returncode, stderr)
        if stdout:
            ret = pickle.loads(stdout)
        else:
            ret = None
        return ret

    def _execution(self):
        """
        Take queued execution requests, execute plugins and queue the results
        """
        while True:
            if self.shutdown:
                logging.info('%s:shutdown', threading.currentThread())
                break
            logging.info('%s:exec_queue:%i', threading.currentThread(), self.execute.qsize())
            try:
                task = self.execute.get_nowait()
            except Queue.Empty:
                break
            logging.info('%s:task:%s', threading.currentThread(), task)
            name = _plugin_name(task)
            ts = time.time()
            if isinstance(task, basestring):
                payload = self._subprocess_execution(task)
            else:
                try:
                    payload = task.Plugin().run(self.config)
                except:
                    logging.exception('plugin_exception')
                    payload = { 'exception': str(sys.exc_info()[0]) }
            self.metrics.put({
                'ts': ts,
                'task': task,
                'name': name,
                'payload': payload,
            })
            self.hire.release()

    def _data(self):
        """
        Take and collect data, send and clean if needed
        """
        logging.info('%s', threading.currentThread())
        collection = []
        while True:
            if self.shutdown:
                logging.info('%s:shutdown', threading.currentThread())
                break
            logging.info('%s:data_queue:%i:collection:%i', threading.currentThread(), self.data.qsize(), len(collection))
            while self.data.qsize():
                collection.append(self.data.get_nowait())
            if collection:
                first_ts = min((e['ts'] for e in collection))
                last_ts = max((e['ts'] for e in collection))
                now = time.time()
                max_age = self.config.getint('agent', 'max_data_age')
                max_span = self.config.getint('agent', 'max_data_span')
                send = False
                server = self.config.get('agent', 'server')
                user = self.config.get('agent', 'user')
                if last_ts - first_ts > max_span:
                    logging.info('Max data span')
                    send = True
                elif now - first_ts > max_age:
                    logging.info('Max data age')
                    send = True
                if send:
                    headers = {
                        "Content-type": "application/json",
                        "Authorization":
                            'ApiKey %s:%s' % (
                                self.config.get('agent', 'user'),
                                self.config.get('agent', 'server')),
                    }
                    clean = False
                    logging.info('collection:%s', collection)
                    if not server and not user:
                        logging.info('Empty server/user, but need to send: %s', serialize.dumps(collection))
                        clean = True
                    else:
                        connection = httplib.HTTPSConnection(self.config.get('data', 'api_host'))
                        connection.request('PUT', self.config.get('data', 'api_path'),
                             bz2.compress(str(serialize.dumps(collection)) + "\n"),
                             headers=headers)
                        response = connection.getresponse()
                        logging.info('%s', response)
                        connection.close()
                        if response.status == 200:
                            clean = True
                    if clean:
                        collection = []
            time.sleep(self.config.getint('data', 'interval'))

    def _data_worker_init(self):
        """
        Initialize data worker thread
        """
        logging.info('_data_worker_init')
        threading.Thread(target=self._data).start()

    def  _dump_config(self):
        """
        Dumps configuration object
        """
        buf = StringIO.StringIO()
        self.config.write(buf)
        logging.info('Config: %s', buf.getvalue())

    def run(self):
        """
        Start all the worker threads
        """
        logging.info('Agent main loop')
        interval = self.config.getint('agent', 'interval')
        self.hire = threading.Semaphore(
            self.config.getint('execution', 'threads'))
        try:
            while True:
                now = time.time()
                logging.info('%i threads', threading.activeCount())
                while self.metrics.qsize():
                    metrics = self.metrics.get_nowait()
                    name = metrics['name']
                    logging.info('metrics:%s', name)
                    plugin = metrics.get('task')
                    if plugin:
                        self.schedule[plugin] = \
                            now + self.config.getint(name, 'interval')
                        if isinstance(plugin, types.ModuleType):
                            metrics['task'] = plugin.__file__
                    self.data.put(metrics)
                execute = [what
                    for what, when in self.schedule.items()
                        if when <= now
                ]
                for name in execute:
                    logging.info('scheduling:%s', name)
                    del self.schedule[name]
                    self.execute.put(name)
                    if self.hire.acquire(False):
                        thread = threading.Thread(target=self._execution)
                        thread.start()
                        logging.info('new_execution_worker_thread:%s', thread)
                    else:
                        logging.warning('threads_capped')
                        self.metrics.put({
                            'ts': now,
                            'name': 'agent_internal',
                            'payload': {
                                'threads_capping':
                                    self.config.getint('execution', 'threads')}
                        })

                time.sleep(interval)
        except KeyboardInterrupt:
            logging.warning(sys.exc_info()[0])
            while True:
                wait_for = [thread
                    for thread in threading.enumerate()
                        if not thread.isDaemon() and
                           not isinstance(thread, threading._MainThread)]
                logging.info('Shutdown, waiting for %i threads to exit', len(wait_for))
                logging.info('Remaining threads: %s', threading.enumerate())
                if len(wait_for) == 0:
                    sys.exit(0)
                self.shutdown = True
                time.sleep(interval)

if __name__ == '__main__':
    run_agent()
<commit_msg>Update nixstatsagent.py<commit_after>#!/usr/bin/env python

# by Al Nikolov <root@toor.fi.eu.org>

import bz2
import ConfigParser
import glob
import httplib
import imp
try:
    import json as serialize
except ImportError:
    import simplejson as serialize
import logging
import os
import pickle
import Queue
import signal
import StringIO
import subprocess
import sys
import threading
import time
import types

ini_files = (
    os.path.join('/etc', 'nixstats.ini'),
    os.path.abspath('nixstats.ini'),
)

def run_agent():
    Agent().run()

def _plugin_name(plugin):
    if isinstance(plugin, basestring):
        basename = os.path.basename(plugin)
        return os.path.splitext(basename)[0]
    else:
        return plugin.__name__

class Agent:

    execute = Queue.Queue()
    metrics = Queue.Queue()
    data = Queue.Queue()
    shutdown = False

    def __init__(self):
        """
        Initialize internal strictures
        """
        self._config_init()
        self._logging_init()
        self._plugins_init()
        self._data_worker_init()
        self._dump_config()

    def _config_init(self):
        """
        Initialize configuration object
        """
        defaults = {
            'max_data_span': 60,
            'max_data_age': 60 * 10,
            'logging_level': logging.WARNING,
            'threads': 100,
            'ttl': 60,
            'interval': 60,
            'plugins': 'plugins',
            'enabled': 'no',
            'subprocess': 'no',
            'user': '',
            'server': '',
            'api_host': 'api.nixstats.com',
            'api_path': '/v2/server/poll',
        }
        sections = [
            'agent',
            'execution',
            'data',
        ]
        config = ConfigParser.RawConfigParser(defaults)
        config.read(ini_files)
        self.config = config
        for section in sections:
            self._config_section_create(section)

    def _config_section_create(self, section):
        """
        Create an addition section in the configuration object
        """
        if not self.config.has_section(section):
            self.config.add_section(section)

    def _logging_init(self):
        """
        Initialize logging faculty
        """
        level = self.config.getint('agent', 'logging_level')
        logging.basicConfig(level=level)
        logging.info('Agent logging_level %i', level)

    def _plugins_init(self):
        """
        Discover the plugins
        """
        logging.info('_plugins_init')
        path = self.config.get('agent', 'plugins')
        filenames = glob.glob(os.path.join(path, '*.py'))
        sys.path.insert(0, path)
        self.schedule = {}
        for filename in filenames:
            name = _plugin_name(filename)
            if name == 'plugins':
                continue
            self._config_section_create(name)
            if self.config.getboolean(name, 'enabled'):
                if self.config.getboolean(name, 'subprocess'):
                    self.schedule[filename] = 0
                else:
                    fp, pathname, description = \
                        imp.find_module(name)
                    module = None
                    try:
                        module = imp.load_module(
                                    name, fp, pathname, description)
                    finally:
                        # Since we may exit via an exception, close fp explicitly.
                        if fp:
                            fp.close()
                    if module:
                        self.schedule[module] = 0
                    else:
                        logging.error('import_plugin:%s:%s', name, sys.exc_type)

    def _subprocess_execution(self, task):
        """
        Execute /task/ in a subprocess
        """
        process = subprocess.Popen((sys.executable, task, ini_file),
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            universal_newlines=True)
        logging.info('%s:process:%i', threading.currentThread(), process.pid)
        interval = self.config.getint('execution', 'interval')
        name = _plugin_name(task)
        ttl = self.config.getint(name, 'ttl')
        ticks = ttl / interval or 1
        process.poll()
        while process.returncode is None and ticks > 0:
            logging.info('%s:tick:%i', threading.currentThread(), ticks)
            time.sleep(interval)
            ticks -= 1
            process.poll()
        if process.returncode is None:
            logging.error('%s:kill:%i', threading.currentThread(), process.pid)
            os.kill(process.pid, signal.SIGTERM)
        stdout, stderr = process.communicate()
        if process.returncode != 0 or stderr:
            logging.error('%s:%s:%s:%s', threading.currentThread(), task, process.returncode, stderr)
        if stdout:
            ret = pickle.loads(stdout)
        else:
            ret = None
        return ret

    def _execution(self):
        """
        Take queued execution requests, execute plugins and queue the results
        """
        while True:
            if self.shutdown:
                logging.info('%s:shutdown', threading.currentThread())
                break
            logging.info('%s:exec_queue:%i', threading.currentThread(), self.execute.qsize())
            try:
                task = self.execute.get_nowait()
            except Queue.Empty:
                break
            logging.info('%s:task:%s', threading.currentThread(), task)
            name = _plugin_name(task)
            ts = time.time()
            if isinstance(task, basestring):
                payload = self._subprocess_execution(task)
            else:
                try:
                    payload = task.Plugin().run(self.config)
                except:
                    logging.exception('plugin_exception')
                    payload = { 'exception': str(sys.exc_info()[0]) }
            self.metrics.put({
                'ts': ts,
                'task': task,
                'name': name,
                'payload': payload,
            })
            self.hire.release()

    def _data(self):
        """
        Take and collect data, send and clean if needed
        """
        logging.info('%s', threading.currentThread())
        collection = []
        while True:
            if self.shutdown:
                logging.info('%s:shutdown', threading.currentThread())
                break
            logging.info('%s:data_queue:%i:collection:%i', threading.currentThread(), self.data.qsize(), len(collection))
            while self.data.qsize():
                collection.append(self.data.get_nowait())
            if collection:
                first_ts = min((e['ts'] for e in collection))
                last_ts = max((e['ts'] for e in collection))
                now = time.time()
                max_age = self.config.getint('agent', 'max_data_age')
                max_span = self.config.getint('agent', 'max_data_span')
                send = False
                server = self.config.get('agent', 'server')
                user = self.config.get('agent', 'user')
                if last_ts - first_ts > max_span:
                    logging.info('Max data span')
                    send = True
                elif now - first_ts > max_age:
                    logging.info('Max data age')
                    send = True
                if send:
                    headers = {
                        "Content-type": "application/json",
                        "Authorization":
                            'ApiKey %s:%s' % (
                                self.config.get('agent', 'user'),
                                self.config.get('agent', 'server')),
                    }
                    clean = False
                    logging.info('collection:%s', collection)
                    if not server and not user:
                        logging.info('Empty server/user, but need to send: %s', serialize.dumps(collection))
                        clean = True
                    else:
                        connection = httplib.HTTPSConnection(self.config.get('data', 'api_host'))
                        connection.request('PUT', self.config.get('data', 'api_path'),
                             bz2.compress(str(serialize.dumps(collection)) + "\n"),
                             headers=headers)
                        response = connection.getresponse()
                        logging.info('%s', response)
                        connection.close()
                        if response.status == 200:
                            clean = True
                    if clean:
                        collection = []
            time.sleep(self.config.getint('data', 'interval'))

    def _data_worker_init(self):
        """
        Initialize data worker thread
        """
        logging.info('_data_worker_init')
        threading.Thread(target=self._data).start()

    def  _dump_config(self):
        """
        Dumps configuration object
        """
        buf = StringIO.StringIO()
        self.config.write(buf)
        logging.info('Config: %s', buf.getvalue())

    def run(self):
        """
        Start all the worker threads
        """
        logging.info('Agent main loop')
        interval = self.config.getint('agent', 'interval')
        self.hire = threading.Semaphore(
            self.config.getint('execution', 'threads'))
        try:
            while True:
                now = time.time()
                logging.info('%i threads', threading.activeCount())
                while self.metrics.qsize():
                    metrics = self.metrics.get_nowait()
                    name = metrics['name']
                    logging.info('metrics:%s', name)
                    plugin = metrics.get('task')
                    if plugin:
                        self.schedule[plugin] = \
                            now + self.config.getint(name, 'interval')
                        if isinstance(plugin, types.ModuleType):
                            metrics['task'] = plugin.__file__
                    self.data.put(metrics)
                execute = [what
                    for what, when in self.schedule.items()
                        if when <= now
                ]
                for name in execute:
                    logging.info('scheduling:%s', name)
                    del self.schedule[name]
                    self.execute.put(name)
                    if self.hire.acquire(False):
                        thread = threading.Thread(target=self._execution)
                        thread.start()
                        logging.info('new_execution_worker_thread:%s', thread)
                    else:
                        logging.warning('threads_capped')
                        self.metrics.put({
                            'ts': now,
                            'name': 'agent_internal',
                            'payload': {
                                'threads_capping':
                                    self.config.getint('execution', 'threads')}
                        })

                time.sleep(interval)
        except KeyboardInterrupt:
            logging.warning(sys.exc_info()[0])
            while True:
                wait_for = [thread
                    for thread in threading.enumerate()
                        if not thread.isDaemon() and
                           not isinstance(thread, threading._MainThread)]
                logging.info('Shutdown, waiting for %i threads to exit', len(wait_for))
                logging.info('Remaining threads: %s', threading.enumerate())
                if len(wait_for) == 0:
                    sys.exit(0)
                self.shutdown = True
                time.sleep(interval)

if __name__ == '__main__':
    run_agent()
<|endoftext|>


<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

from ..common import ReadOnlyDataStructure

class WorldcatData (ReadOnlyDataStructure):

    @property
    def title (self):
        return self.get("title")

    def __iter__ (self):
        return iter(self.get("libraries", ()))
<commit_msg>Use new property format for WorldcatData<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.

from ..common import ReadOnlyDataStructure

class WorldcatData (ReadOnlyDataStructure):

    auto_properties = ("title",)

    def __iter__ (self):
        return iter(self.get("libraries", ()))
<|endoftext|>


<commit_before>from django.utils import six

from nodeconductor.core.models import User
from nodeconductor.logging.log import EventLogger, event_logger
from nodeconductor.structure import models

class CustomerEventLogger(EventLogger):
    customer = models.Customer

    class Meta:
        event_types = ('customer_deletion_succeeded',
                       'customer_update_succeeded',
                       'customer_creation_succeeded')

class BalanceEventLogger(EventLogger):
    customer = models.Customer
    amount = float

    class Meta:
        event_types = ('customer_account_credited',
                       'customer_account_debited')

class ProjectEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup
    project_previous_name = six.text_type

    class Meta:
        nullable_fields = ['project_group', 'project_previous_name']
        event_types = ('project_deletion_succeeded',
                       'project_update_succeeded',
                       'project_creation_succeeded')

class ProjectGroupEventLogger(EventLogger):
    project_group = models.ProjectGroup

    class Meta:
        event_types = ('project_group_deletion_succeeded',
                       'project_group_update_succeeded',
                       'project_group_creation_succeeded')

class CustomerRoleEventLogger(EventLogger):
    customer = models.Customer
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        event_types = 'role_granted', 'role_revoked'

class ProjectRoleEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        nullable_fields = ['project_group']
        event_types = 'role_granted', 'role_revoked'

class ProjectGroupRoleEventLogger(EventLogger):
    project_group = models.ProjectGroup
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        event_types = 'role_granted', 'role_revoked'

class ProjectGroupMembershipEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup

    class Meta:
        event_types = 'project_added_to_project_group', 'project_removed_from_project_group'

class LicensesEventLogger(EventLogger):
    resource = models.Resource
    license_name = six.text_type
    license_type = six.text_type

    class Meta:
        event_types = 'resource_license_added',

class UserOrganizationEventLogger(EventLogger):
    affected_user = User
    affected_organization = six.text_type

    class Meta:
        event_types = ('user_organization_claimed',
                       'user_organization_approved',
                       'user_organization_rejected',
                       'user_organization_removed')

class ResourceEventLogger(EventLogger):
    resource = models.Resource

    class Meta:
        event_types = (
            'resource_start_scheduled',
            'resource_start_succeeded',
            'resource_start_failed',

            'resource_stop_scheduled',
            'resource_stop_succeeded',
            'resource_stop_failed',

            'resource_restart_scheduled',
            'resource_restart_succeeded',
            'resource_restart_failed',

            'resource_creation_scheduled',
            'resource_creation_succeeded',
            'resource_creation_failed',

            'resource_import_succeeded',
            'resource_update_succeeded',

            'resource_deletion_scheduled',
            'resource_deletion_succeeded',
            'resource_deletion_failed',
        )

class ServiceSettingsEventLogger(EventLogger):
    service_settings = models.ServiceSettings
    error_message = six.text_type

    class Meta:
        event_types = ('service_settings_sync_failed',
                       'service_settings_recovered')

class ServiceProjectLinkEventLogger(EventLogger):
    service_project_link = models.ServiceProjectLink
    error_message = six.text_type

    class Meta:
        event_types = ('service_project_link_creation_failed',
                       'service_project_link_sync_failed',
                       'service_project_link_recovered')

event_logger.register('customer_role', CustomerRoleEventLogger)
event_logger.register('project_role', ProjectRoleEventLogger)
event_logger.register('project_group_role', ProjectGroupRoleEventLogger)
event_logger.register('project_group_membership', ProjectGroupMembershipEventLogger)
event_logger.register('user_organization', UserOrganizationEventLogger)
event_logger.register('customer', CustomerEventLogger)
event_logger.register('project', ProjectEventLogger)
event_logger.register('project_group', ProjectGroupEventLogger)
event_logger.register('licenses', LicensesEventLogger)
event_logger.register('balance', BalanceEventLogger)
event_logger.register('resource', ResourceEventLogger)
event_logger.register('service_settings', ServiceSettingsEventLogger)
event_logger.register('service_project_link', ServiceProjectLinkEventLogger)
<commit_msg>Fix ServiceSettings event logger<commit_after>from django.utils import six

from nodeconductor.core.models import User
from nodeconductor.logging.log import EventLogger, event_logger
from nodeconductor.structure import models

class CustomerEventLogger(EventLogger):
    customer = models.Customer

    class Meta:
        event_types = ('customer_deletion_succeeded',
                       'customer_update_succeeded',
                       'customer_creation_succeeded')

class BalanceEventLogger(EventLogger):
    customer = models.Customer
    amount = float

    class Meta:
        event_types = ('customer_account_credited',
                       'customer_account_debited')

class ProjectEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup
    project_previous_name = six.text_type

    class Meta:
        nullable_fields = ['project_group', 'project_previous_name']
        event_types = ('project_deletion_succeeded',
                       'project_update_succeeded',
                       'project_creation_succeeded')

class ProjectGroupEventLogger(EventLogger):
    project_group = models.ProjectGroup

    class Meta:
        event_types = ('project_group_deletion_succeeded',
                       'project_group_update_succeeded',
                       'project_group_creation_succeeded')

class CustomerRoleEventLogger(EventLogger):
    customer = models.Customer
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        event_types = 'role_granted', 'role_revoked'

class ProjectRoleEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        nullable_fields = ['project_group']
        event_types = 'role_granted', 'role_revoked'

class ProjectGroupRoleEventLogger(EventLogger):
    project_group = models.ProjectGroup
    affected_user = User
    structure_type = six.text_type
    role_name = six.text_type

    class Meta:
        event_types = 'role_granted', 'role_revoked'

class ProjectGroupMembershipEventLogger(EventLogger):
    project = models.Project
    project_group = models.ProjectGroup

    class Meta:
        event_types = 'project_added_to_project_group', 'project_removed_from_project_group'

class LicensesEventLogger(EventLogger):
    resource = models.Resource
    license_name = six.text_type
    license_type = six.text_type

    class Meta:
        event_types = 'resource_license_added',

class UserOrganizationEventLogger(EventLogger):
    affected_user = User
    affected_organization = six.text_type

    class Meta:
        event_types = ('user_organization_claimed',
                       'user_organization_approved',
                       'user_organization_rejected',
                       'user_organization_removed')

class ResourceEventLogger(EventLogger):
    resource = models.Resource

    class Meta:
        event_types = (
            'resource_start_scheduled',
            'resource_start_succeeded',
            'resource_start_failed',

            'resource_stop_scheduled',
            'resource_stop_succeeded',
            'resource_stop_failed',

            'resource_restart_scheduled',
            'resource_restart_succeeded',
            'resource_restart_failed',

            'resource_creation_scheduled',
            'resource_creation_succeeded',
            'resource_creation_failed',

            'resource_import_succeeded',
            'resource_update_succeeded',

            'resource_deletion_scheduled',
            'resource_deletion_succeeded',
            'resource_deletion_failed',
        )

class ServiceSettingsEventLogger(EventLogger):
    service_settings = models.ServiceSettings
    error_message = six.text_type

    class Meta:
        nullable_fields = ['error_message']
        event_types = ('service_settings_sync_failed',
                       'service_settings_recovered')

class ServiceProjectLinkEventLogger(EventLogger):
    service_project_link = models.ServiceProjectLink
    error_message = six.text_type

    class Meta:
        event_types = ('service_project_link_creation_failed',
                       'service_project_link_sync_failed',
                       'service_project_link_recovered')

event_logger.register('customer_role', CustomerRoleEventLogger)
event_logger.register('project_role', ProjectRoleEventLogger)
event_logger.register('project_group_role', ProjectGroupRoleEventLogger)
event_logger.register('project_group_membership', ProjectGroupMembershipEventLogger)
event_logger.register('user_organization', UserOrganizationEventLogger)
event_logger.register('customer', CustomerEventLogger)
event_logger.register('project', ProjectEventLogger)
event_logger.register('project_group', ProjectGroupEventLogger)
event_logger.register('licenses', LicensesEventLogger)
event_logger.register('balance', BalanceEventLogger)
event_logger.register('resource', ResourceEventLogger)
event_logger.register('service_settings', ServiceSettingsEventLogger)
event_logger.register('service_project_link', ServiceProjectLinkEventLogger)
<|endoftext|>


<commit_before># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from re import compile as re_compile
import xml.etree.ElementTree as ET

INDEX_OF_FIRST_YEAR_SUBSTR = 7
INDEX_OF_SECOND_YEAR_SUBSTR = 11
NUMBER_OF_CHARS_IN_A_YEAR = 4
MARC_NULL_YEAR = "^^^^"

# I expect OCLC text values to look like `(OCoLC)ocm15015690`. It'll
# always start with `(OCoLC)`, and I'm always only interested in the
# decimal digits at the end.
RE_MARC_OCLC = re_compile(r"^\(OCoLC\).*?([0-9]+)$")

class MARCData:

    xmlns = "http://www.loc.gov/MARC21/slim"
    description = None # should be datafield(MDP, z)

    def __set_oclc (self):
        self.oclc = self.__get_oclc()

    def __get_oclc (self):
        for value in self.__find_all_datafields("035", "a"):
            match = RE_MARC_OCLC.match(value)

            if match is not None:
                return match.group(1)

        return None

    def __init__ (self, xml):
        self.__root = ET.fromstring(xml)
        self.bib = self.__get_controlfield("001")
        self.callno = self.__get_datafield("MDP", "h")
        self.author = self.__get_datafield("100", "a")
        self.title = self.__get_datafield("245", "a")

        self.__set_years()
        self.__set_oclc()

    def __get_controlfield (self, tag):
        return self.__get_text_if_not_none(
                ".//{{{xmlns}}}controlfield[@tag='{}']".format(
                        tag, xmlns=self.xmlns))

    def __get_datafield (self, tag, code):
        return self.__get_text_if_not_none(
                ".//{{{xmlns}}}datafield[@tag='{}']/"
                "{{{xmlns}}}subfield[@code='{}']".format(
                        tag, code, xmlns=self.xmlns))

    def __get_text_if_not_none (self, xpath):
        elt = self.__root.find(xpath)

        return getattr(elt, "text", None)

    def __find_all_datafields (self, tag, code):
        return self.__find_all_texts(
                ".//{{{xmlns}}}datafield[@tag='{}']/"
                "{{{xmlns}}}subfield[@code='{}']".format(
                        tag, code, xmlns=self.xmlns))

    def __find_all_texts (self, xpath):
        for elt in self.__root.findall(xpath):
            yield elt.text

    def __set_years (self):
        full_str = self.__get_controlfield("008")
        self.years = self.__get_first_year(full_str), \
                self.__get_second_year(full_str)

    def __get_first_year (self, full_str):
        return self.__get_year_substr(full_str,
                INDEX_OF_FIRST_YEAR_SUBSTR)

    def __get_second_year (self, full_str):
        return self.__get_year_substr(full_str,
                INDEX_OF_SECOND_YEAR_SUBSTR)

    def __get_year_substr (self, full_str, start_index):
        result = full_str[start_index:start_index +
                NUMBER_OF_CHARS_IN_A_YEAR]

        return None if result == MARC_NULL_YEAR else result
<commit_msg>Refactor MARCData<commit_after># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from re import compile as re_compile
import xml.etree.ElementTree as ET

INDEX_OF_FIRST_YEAR_SUBSTR = 7
INDEX_OF_SECOND_YEAR_SUBSTR = 11
NUMBER_OF_CHARS_IN_A_YEAR = 4
MARC_NULL_YEAR = "^^^^"

# I expect OCLC text values to look like `(OCoLC)ocm15015690`. It'll
# always start with `(OCoLC)`, and I'm always only interested in the
# decimal digits at the end.
RE_MARC_OCLC = re_compile(r"^\(OCoLC\).*?([0-9]+)$")

class MARCData:

    xmlns = "http://www.loc.gov/MARC21/slim"
    description = None # should be datafield(MDP, z)

    def __init__ (self, xml):
        self.__root = ET.fromstring(xml)
        self.bib = self.__get_controlfield("001")
        self.callno = self.__get_datafield("MDP", "h")
        self.author = self.__get_datafield("100", "a")
        self.title = self.__get_datafield("245", "a")

        self.__set_years()
        self.__set_oclc()

    def __get_controlfield (self, tag):
        xpath = self.__get_control_xpath(tag)
        return self.__get_text_if_not_none(xpath)

    def __get_control_xpath (self, tag):
        return ".//{{{xmlns}}}controlfield[@tag='{}']".format(
                        tag, xmlns=self.xmlns)

    def __get_datafield (self, tag, code):
        xpath = self.__get_data_xpath(tag, code)
        return self.__get_text_if_not_none(xpath)

    def __get_data_xpath (self, tag, code):
        return ".//{{{xmlns}}}datafield[@tag='{}']/" \
                "{{{xmlns}}}subfield[@code='{}']".format(
                        tag, code, xmlns=self.xmlns)

    def __get_text_if_not_none (self, xpath):
        elt = self.__root.find(xpath)

        return getattr(elt, "text", None)

    def __find_all_datafields (self, tag, code):
        xpath = self.__get_data_xpath(tag, code)
        return self.__find_all_texts(xpath)

    def __find_all_texts (self, xpath):
        for elt in self.__root.findall(xpath):
            yield elt.text

    def __set_years (self):
        full_str = self.__get_controlfield("008")
        self.years = self.__get_first_year(full_str), \
                self.__get_second_year(full_str)

    def __get_first_year (self, full_str):
        return self.__get_year_substr(full_str,
                INDEX_OF_FIRST_YEAR_SUBSTR)

    def __get_second_year (self, full_str):
        return self.__get_year_substr(full_str,
                INDEX_OF_SECOND_YEAR_SUBSTR)

    def __get_year_substr (self, full_str, start_index):
        result = full_str[start_index:start_index +
                NUMBER_OF_CHARS_IN_A_YEAR]

        return None if result == MARC_NULL_YEAR else result

    def __set_oclc (self):
        self.oclc = self.__get_oclc()

    def __get_oclc (self):
        for value in self.__find_all_datafields("035", "a"):
            match = RE_MARC_OCLC.match(value)

            if match is not None:
                return match.group(1)

        return None
<|endoftext|>


<commit_before>import logging
from os import path, mkdir
from datetime import datetime
from urllib.parse import urlsplit
from aiofiles import open as open_file
from aiohttp import ClientSession as Http
from asyncio import get_event_loop, gather

from goodline_iptv.m3u import create_m3u
from goodline_iptv.jtvfile import JtvFile
from goodline_iptv.xmltv import XmltvBuilder
from goodline_iptv.goodline_playlist import parse_playlist

EPG_URL = 'http://bolshoe.tv/tv.zip'
PLAYLIST_URL = 'http://playlist.bolshoe.tv/playlist'
PLAYLIST_FILENAME = 'playlist.m3u'
EPG_FILENAME = 'teleguide.xmltv'

logging.basicConfig(
    level=logging.DEBUG,
    format='%(asctime)s [%(levelname)-5.5s] [%(funcName)s] %(message)s',
    handlers=[
        logging.StreamHandler()
    ]
)

log = logging.getLogger()

async def download_playlist(http, url):
    log.info(f'Start downloading playlist from "{url}"')

    async with http.get(url) as response:

        if response.status != 200:
            log.error(f'Error downloading playlist. Response code {response.status}')
            return None

        log.info('Playlist successfully downloaded')

        return parse_playlist(await response.text(), log)

async def download_epg(http, url, out_dir):
    log.info(f'Start downloading epg from "{url}"')

    async with http.get(url) as response:
        if response.status != 200:
            log.error(f'Error downloading EPG. Response code {response.status}')
            return None

        data = await response.read()

        epg_path = path.join(out_dir, path.basename(urlsplit(url).path))

        log.debug(f'EPG successfully downloaded. Saving to "{epg_path}"')

        async with open_file(epg_path, mode='w+b') as f:
            await f.write(data)
            log.debug(f'EPG saved to "{epg_path}"')

    log.info('EPG successfully downloaded')

    return epg_path

async def download_icons(http, playlist, out_dir):

    icons_dir = path.join(out_dir, 'icons')

    if not path.exists(icons_dir):
        mkdir(icons_dir)

    async def download_icon(http, url, file_name):
        log.debug(f'Start downloading icon "{url}"')
        async with http.get(url) as response:
            if response.status != 200:
                log.error(f'Error downloading icon "{url}". Response code: {response.status}')
                return None

            data = await response.read()

            log.debug(f'Icon "{url}" successfully downloaded. Saving...')

            icon_path = path.join(icons_dir, file_name)
            async with open_file(icon_path, mode='w+b') as f:
                await f.write(data)

            log.debug(f'Icon "{url}" successfully saved to "{icon_path}"')

    log.info('Start downloading icons')

    await gather(*[download_icon(http, track['icon_url'], track['icon_name']) for track in playlist.values()])

    log.info('Icons downloading finished')

async def create_xmltv(out_dir, epg_path, playlist, encoding, timezone, pretty_xmltv=False):

    xmltv_builder = XmltvBuilder(timezone)

    for channel in JtvFile(epg_path, log, encoding).channels:

        if channel.track_id not in playlist:
            log.warning(f'{channel.track_id} not found in channels playlist. EPG ignored.')
            continue

        xmltv_builder.add_channel(
            channel.track_id,
            playlist[channel.track_id]['name'],
            playlist[channel.track_id]['icon_name']
        )

        prev_time = prev_name = None

        for time, name in channel.tracks:
            if prev_time:
                xmltv_builder.add_track(channel.track_id, prev_time, time, prev_name)
            prev_time, prev_name = time, name

    await xmltv_builder.save(path.join(out_dir, EPG_FILENAME), pretty_xmltv)

async def main(out_dir, encoding, timezone, pretty_xmltv):
    time_begin = datetime.now()

    log.info("Let's do it :)")

    if not path.exists(out_dir):
        log.debug(f'Directory {out_dir} does not exist. Creating.')
        mkdir(out_dir)

    async with Http() as http:

        playlist, epg_path = await gather(
            download_playlist(http, PLAYLIST_URL),
            download_epg(http, EPG_URL, out_dir)
        )

        await gather(
            download_icons(http, playlist, out_dir),
            create_m3u(playlist, path.join(out_dir, PLAYLIST_FILENAME), log)
        )

        await create_xmltv(out_dir, epg_path, playlist, encoding, timezone, pretty_xmltv)

    log.info(f'Finished due {datetime.now() - time_begin}')

def do_import(verbosity, out_dir, encoding, timezone, pretty_xmltv=False):

    log.setLevel((4 - verbosity) * 10)

    get_event_loop().run_until_complete(main(out_dir, encoding, timezone, pretty_xmltv))
<commit_msg>Regroup async operations<commit_after>import logging
from os import path, mkdir
from datetime import datetime
from urllib.parse import urlsplit
from aiofiles import open as open_file
from aiohttp import ClientSession as Http
from asyncio import get_event_loop, gather

from goodline_iptv.m3u import create_m3u
from goodline_iptv.jtvfile import JtvFile
from goodline_iptv.xmltv import XmltvBuilder
from goodline_iptv.goodline_playlist import parse_playlist

EPG_URL = 'http://bolshoe.tv/tv.zip'
PLAYLIST_URL = 'http://playlist.bolshoe.tv/playlist'
PLAYLIST_FILENAME = 'playlist.m3u'
EPG_FILENAME = 'teleguide.xmltv'

logging.basicConfig(
    level=logging.DEBUG,
    format='%(asctime)s [%(levelname)-5.5s] [%(funcName)s] %(message)s',
    handlers=[
        logging.StreamHandler()
    ]
)

log = logging.getLogger()

async def download_playlist(http, url):
    log.info(f'Start downloading playlist from "{url}"')

    async with http.get(url) as response:

        if response.status != 200:
            log.error(f'Error downloading playlist. Response code {response.status}')
            return None

        log.info('Playlist successfully downloaded')

        return parse_playlist(await response.text(), log)

async def download_epg(http, url, out_dir):
    log.info(f'Start downloading epg from "{url}"')

    async with http.get(url) as response:
        if response.status != 200:
            log.error(f'Error downloading EPG. Response code {response.status}')
            return None

        data = await response.read()

        epg_path = path.join(out_dir, path.basename(urlsplit(url).path))

        log.debug(f'EPG successfully downloaded. Saving to "{epg_path}"')

        async with open_file(epg_path, mode='w+b') as f:
            await f.write(data)
            log.debug(f'EPG saved to "{epg_path}"')

    log.info('EPG successfully downloaded')

    return epg_path

async def download_icons(http, playlist, out_dir):

    icons_dir = path.join(out_dir, 'icons')

    if not path.exists(icons_dir):
        mkdir(icons_dir)

    async def download_icon(http, url, file_name):
        log.debug(f'Start downloading icon "{url}"')
        async with http.get(url) as response:
            if response.status != 200:
                log.error(f'Error downloading icon "{url}". Response code: {response.status}')
                return None

            data = await response.read()

            log.debug(f'Icon "{url}" successfully downloaded. Saving...')

            icon_path = path.join(icons_dir, file_name)
            async with open_file(icon_path, mode='w+b') as f:
                await f.write(data)

            log.debug(f'Icon "{url}" successfully saved to "{icon_path}"')

    log.info('Start downloading icons')

    await gather(*[download_icon(http, track['icon_url'], track['icon_name']) for track in playlist.values()])

    log.info('Icons downloading finished')

async def create_xmltv(out_dir, epg_path, playlist, encoding, timezone, pretty_xmltv=False):

    xmltv_builder = XmltvBuilder(timezone)

    for channel in JtvFile(epg_path, log, encoding).channels:

        if channel.track_id not in playlist:
            log.warning(f'{channel.track_id} not found in channels playlist. EPG ignored.')
            continue

        xmltv_builder.add_channel(
            channel.track_id,
            playlist[channel.track_id]['name'],
            playlist[channel.track_id]['icon_name']
        )

        prev_time = prev_name = None

        for time, name in channel.tracks:
            if prev_time:
                xmltv_builder.add_track(channel.track_id, prev_time, time, prev_name)
            prev_time, prev_name = time, name

    await xmltv_builder.save(path.join(out_dir, EPG_FILENAME), pretty_xmltv)

async def main(out_dir, encoding, timezone, pretty_xmltv):
    time_begin = datetime.now()

    log.info("Let's do it :)")

    if not path.exists(out_dir):
        log.debug(f'Directory {out_dir} does not exist. Creating.')
        mkdir(out_dir)

    async with Http() as http:

        playlist, epg_path = await gather(
            download_playlist(http, PLAYLIST_URL),
            download_epg(http, EPG_URL, out_dir)
        )

        await download_icons(http, playlist, out_dir)

        await gather(
            create_xmltv(out_dir, epg_path, playlist, encoding, timezone, pretty_xmltv),
            create_m3u(playlist, path.join(out_dir, PLAYLIST_FILENAME), log)
        )

    log.info(f'Finished due {datetime.now() - time_begin}')

def do_import(verbosity, out_dir, encoding, timezone, pretty_xmltv=False):

    log.setLevel((4 - verbosity) * 10)

    get_event_loop().run_until_complete(main(out_dir, encoding, timezone, pretty_xmltv))
<|endoftext|>


<commit_before>from datetime import datetime
import falcon
import falcon.errors
import json
import sqlalchemy.exc
import sqlalchemy.orm.exc
from sqlalchemy.orm.properties import ColumnProperty
from sqlalchemy.inspection import inspect
import sqlalchemy.sql.sqltypes

try:
    import geoalchemy2.shape
    from geoalchemy2.elements import WKBElement
    from geoalchemy2.types import Geometry
    from shapely.geometry import Point
    support_geo = True
except ImportError:
    support_geo = False

class CollectionResource(object):
    """
    Provides CRUD facilities for a resource collection.
    """
    def __init__(self, db_session):
        self.db_session = db_session

    def deserialize(self, path_data, body_data):
        mapper      = inspect(self.model)
        attributes  = {}

        for key, value in path_data.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            if getattr(self.model, key, None) is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            attributes[key] = value

        for key, value in body_data.items():
            column = mapper.columns[key]
            if isinstance(column.type, sqlalchemy.sql.sqltypes.DateTime):
                attributes[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(column.type, Geometry) and column.type.geometry_type == 'POINT':
                point           = Point(value['x'], value['y'])
                # geoalchemy2.shape.from_shape uses buffer() which causes INSERT to fail
                attributes[key] = WKBElement(point.wkb, srid=4326)
            else:
                attributes[key] = value
        return attributes

    def serialize(self, resource):
        def _serialize_value(value):
            if isinstance(value, datetime):
                return value.strftime('%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(value, WKBElement):
                value = geoalchemy2.shape.to_shape(value)
                return {'x': value.x, 'y': value.y}
            else:
                return value
        attrs = inspect(self.model).attrs
        return {
            attr: _serialize_value(getattr(resource, attr)) for attr in attrs.keys() if isinstance(attrs[attr], ColumnProperty)
        }

    def on_get(self, req, resp, *args, **kwargs):
        """
        Return a collection of items.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)
        for filter_key, value in req.params.items():
            filter_parts = filter_key.split('__')
            key = filter_parts[0]
            if len(filter_parts) == 1:
                comparison = '='
            elif len(filter_parts) == 2:
                comparison = filter_parts[1]
            else:
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')

            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')
            if comparison == '=':
                resources = resources.filter(attr == value)
            elif comparison == 'startswith':
                resources = resources.filter(attr.like('{0}%'.format(value)))
            elif comparison == 'contains':
                resources = resources.filter(attr.like('%{0}%'.format(value)))
            elif comparison == 'lt':
                resources = resources.filter(attr < value)
            elif comparison == 'lte':
                resources = resources.filter(attr <= value)
            elif comparison == 'gt':
                resources = resources.filter(attr > value)
            elif comparison == 'gte':
                resources = resources.filter(attr >= value)
            else:
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': [
                self.serialize(resource) for resource in resources
            ],
        }

    def on_post(self, req, resp, *args, **kwargs):
        """
        Add an item to the collection.
        """
        attributes = self.deserialize(kwargs, req.context['doc'] if 'doc' in req.context else None)
        resource = self.model(**attributes)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_CREATED
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_patch(self, req, resp, *args, **kwargs):
        """
        Update a collection.

        For now, it only supports adding entities to the collection, like this:

        {
            'patches': [
                {'op': 'add', 'path': '/', 'value': {'name': 'Jim', 'age', 25}},
                {'op': 'add', 'path': '/', 'value': {'name': 'Bob', 'age', 28}}
            ]
        }

        """
        mapper  = inspect(self.model)
        patches = req.context['doc']['patches']

        for index, patch in enumerate(patches):
            # Only support adding entities in a collection patch, for now
            if 'op' not in patch or patch['op'] not in ['add']:
                raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid'.format(index))
            if patch['op'] == 'add':
                if 'path' not in patch or patch['path'] != '/':
                    raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid for op {1}'.format(index, patch['op']))
                try:
                    patch_value = patch['value']
                except KeyError:
                    raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid for op {1}'.format(index, patch['op']))
                args = {}
                for key, value in kwargs.items():
                    key = getattr(self, 'attr_map', {}).get(key, key)
                    if getattr(self.model, key, None) is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                        raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
                    args[key] = value
                for key, value in patch_value.items():
                    if isinstance(mapper.columns[key].type, sqlalchemy.sql.sqltypes.DateTime):
                        args[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
                    else:
                        args[key] = value
                resource = self.model(**args)
                self.db_session.add(resource)

        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {}

class SingleResource(object):
    """
    Provides CRUD facilities for a single resource.
    """
    def __init__(self, db_session):
        self.db_session = db_session

    def deserialize(self, data):
        mapper      = inspect(self.model)
        attributes  = {}

        for key, value in data.items():
            column = mapper.columns[key]
            if isinstance(column.type, sqlalchemy.sql.sqltypes.DateTime):
                attributes[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(column.type, Geometry) and column.type.geometry_type == 'POINT':
                point           = Point(value['x'], value['y'])
                # geoalchemy2.shape.from_shape uses buffer() which causes INSERT to fail
                attributes[key] = WKBElement(point.wkb, srid=4326)
            else:
                attributes[key] = value

        return attributes

    def serialize(self, resource):
        def _serialize_value(value):
            if isinstance(value, datetime):
                return value.strftime('%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(value, WKBElement):
                value = geoalchemy2.shape.to_shape(value)
                return {'x': value.x, 'y': value.y}
            else:
                return value
        attrs = inspect(self.model).attrs
        return {
            attr: _serialize_value(getattr(resource, attr)) for attr in attrs.keys() if isinstance(attrs[attr], ColumnProperty)
        }

    def on_get(self, req, resp, *args, **kwargs):
        """
        Return a single item.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError()

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_delete(self, req, resp, *args, **kwargs):
        """
        Delete a single item.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            deleted = resources.delete()
        except sqlalchemy.exc.IntegrityError as err:
            # As far we I know, this should only be caused by foreign key constraint being violated

            # No work has been done to rollback, hence no need for a transaction
            raise falcon.errors.HTTPConflict('Conflict', 'Other content links to this')

        if deleted == 0:
            raise falcon.errors.HTTPNotFound()
        elif deleted > 1:
            self.db_session.rollback()
            raise falcon.errors.HTTPInternalServerError()

        resp.status = falcon.HTTP_OK
        req.context['result'] = {}

    def on_put(self, req, resp, *args, **kwargs):
        """
        Update an item in the collection.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError()

        attributes = self.deserialize(req.context['doc'])
        for key, value in attributes.items():
            setattr(resource, key, value)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_patch(self, req, resp, *args, **kwargs):
        """
        Update part of an item in the collection.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError()

        attributes = self.deserialize(req.context['doc'])
        for key, value in attributes.items():
            setattr(resource, key, value)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }
<commit_msg>Add missing parameters to falcon.errors.HTTPInternalServerError calls<commit_after>from datetime import datetime
import falcon
import falcon.errors
import json
import sqlalchemy.exc
import sqlalchemy.orm.exc
from sqlalchemy.orm.properties import ColumnProperty
from sqlalchemy.inspection import inspect
import sqlalchemy.sql.sqltypes

try:
    import geoalchemy2.shape
    from geoalchemy2.elements import WKBElement
    from geoalchemy2.types import Geometry
    from shapely.geometry import Point
    support_geo = True
except ImportError:
    support_geo = False

class CollectionResource(object):
    """
    Provides CRUD facilities for a resource collection.
    """
    def __init__(self, db_session):
        self.db_session = db_session

    def deserialize(self, path_data, body_data):
        mapper      = inspect(self.model)
        attributes  = {}

        for key, value in path_data.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            if getattr(self.model, key, None) is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            attributes[key] = value

        for key, value in body_data.items():
            column = mapper.columns[key]
            if isinstance(column.type, sqlalchemy.sql.sqltypes.DateTime):
                attributes[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(column.type, Geometry) and column.type.geometry_type == 'POINT':
                point           = Point(value['x'], value['y'])
                # geoalchemy2.shape.from_shape uses buffer() which causes INSERT to fail
                attributes[key] = WKBElement(point.wkb, srid=4326)
            else:
                attributes[key] = value
        return attributes

    def serialize(self, resource):
        def _serialize_value(value):
            if isinstance(value, datetime):
                return value.strftime('%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(value, WKBElement):
                value = geoalchemy2.shape.to_shape(value)
                return {'x': value.x, 'y': value.y}
            else:
                return value
        attrs = inspect(self.model).attrs
        return {
            attr: _serialize_value(getattr(resource, attr)) for attr in attrs.keys() if isinstance(attrs[attr], ColumnProperty)
        }

    def on_get(self, req, resp, *args, **kwargs):
        """
        Return a collection of items.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)
        for filter_key, value in req.params.items():
            filter_parts = filter_key.split('__')
            key = filter_parts[0]
            if len(filter_parts) == 1:
                comparison = '='
            elif len(filter_parts) == 2:
                comparison = filter_parts[1]
            else:
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')

            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')
            if comparison == '=':
                resources = resources.filter(attr == value)
            elif comparison == 'startswith':
                resources = resources.filter(attr.like('{0}%'.format(value)))
            elif comparison == 'contains':
                resources = resources.filter(attr.like('%{0}%'.format(value)))
            elif comparison == 'lt':
                resources = resources.filter(attr < value)
            elif comparison == 'lte':
                resources = resources.filter(attr <= value)
            elif comparison == 'gt':
                resources = resources.filter(attr > value)
            elif comparison == 'gte':
                resources = resources.filter(attr >= value)
            else:
                raise falcon.errors.HTTPBadRequest('Invalid attribute', 'An attribute provided for filtering is invalid')

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': [
                self.serialize(resource) for resource in resources
            ],
        }

    def on_post(self, req, resp, *args, **kwargs):
        """
        Add an item to the collection.
        """
        attributes = self.deserialize(kwargs, req.context['doc'] if 'doc' in req.context else None)
        resource = self.model(**attributes)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_CREATED
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_patch(self, req, resp, *args, **kwargs):
        """
        Update a collection.

        For now, it only supports adding entities to the collection, like this:

        {
            'patches': [
                {'op': 'add', 'path': '/', 'value': {'name': 'Jim', 'age', 25}},
                {'op': 'add', 'path': '/', 'value': {'name': 'Bob', 'age', 28}}
            ]
        }

        """
        mapper  = inspect(self.model)
        patches = req.context['doc']['patches']

        for index, patch in enumerate(patches):
            # Only support adding entities in a collection patch, for now
            if 'op' not in patch or patch['op'] not in ['add']:
                raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid'.format(index))
            if patch['op'] == 'add':
                if 'path' not in patch or patch['path'] != '/':
                    raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid for op {1}'.format(index, patch['op']))
                try:
                    patch_value = patch['value']
                except KeyError:
                    raise falcon.errors.HTTPBadRequest('Invalid patch', 'Patch {0} is not valid for op {1}'.format(index, patch['op']))
                args = {}
                for key, value in kwargs.items():
                    key = getattr(self, 'attr_map', {}).get(key, key)
                    if getattr(self.model, key, None) is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                        raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
                    args[key] = value
                for key, value in patch_value.items():
                    if isinstance(mapper.columns[key].type, sqlalchemy.sql.sqltypes.DateTime):
                        args[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
                    else:
                        args[key] = value
                resource = self.model(**args)
                self.db_session.add(resource)

        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {}

class SingleResource(object):
    """
    Provides CRUD facilities for a single resource.
    """
    def __init__(self, db_session):
        self.db_session = db_session

    def deserialize(self, data):
        mapper      = inspect(self.model)
        attributes  = {}

        for key, value in data.items():
            column = mapper.columns[key]
            if isinstance(column.type, sqlalchemy.sql.sqltypes.DateTime):
                attributes[key] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(column.type, Geometry) and column.type.geometry_type == 'POINT':
                point           = Point(value['x'], value['y'])
                # geoalchemy2.shape.from_shape uses buffer() which causes INSERT to fail
                attributes[key] = WKBElement(point.wkb, srid=4326)
            else:
                attributes[key] = value

        return attributes

    def serialize(self, resource):
        def _serialize_value(value):
            if isinstance(value, datetime):
                return value.strftime('%Y-%m-%dT%H:%M:%SZ')
            elif support_geo and isinstance(value, WKBElement):
                value = geoalchemy2.shape.to_shape(value)
                return {'x': value.x, 'y': value.y}
            else:
                return value
        attrs = inspect(self.model).attrs
        return {
            attr: _serialize_value(getattr(resource, attr)) for attr in attrs.keys() if isinstance(attrs[attr], ColumnProperty)
        }

    def on_get(self, req, resp, *args, **kwargs):
        """
        Return a single item.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_delete(self, req, resp, *args, **kwargs):
        """
        Delete a single item.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            deleted = resources.delete()
        except sqlalchemy.exc.IntegrityError as err:
            # As far we I know, this should only be caused by foreign key constraint being violated

            # No work has been done to rollback, hence no need for a transaction
            raise falcon.errors.HTTPConflict('Conflict', 'Other content links to this')

        if deleted == 0:
            raise falcon.errors.HTTPNotFound()
        elif deleted > 1:
            self.db_session.rollback()
            raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')

        resp.status = falcon.HTTP_OK
        req.context['result'] = {}

    def on_put(self, req, resp, *args, **kwargs):
        """
        Update an item in the collection.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')

        attributes = self.deserialize(req.context['doc'])
        for key, value in attributes.items():
            setattr(resource, key, value)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }

    def on_patch(self, req, resp, *args, **kwargs):
        """
        Update part of an item in the collection.
        """
        resources = self.db_session.query(self.model)
        for key, value in kwargs.items():
            key = getattr(self, 'attr_map', {}).get(key, key)
            attr = getattr(self.model, key, None)
            if attr is None or not isinstance(inspect(self.model).attrs[key], ColumnProperty):
                raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')
            resources = resources.filter(attr == value)

        try:
            resource = resources.one()
        except sqlalchemy.orm.exc.NoResultFound:
            raise falcon.errors.HTTPNotFound()
        except sqlalchemy.orm.exc.MultipleResultsFound:
            raise falcon.errors.HTTPInternalServerError('Internal Server Error', 'An internal server error occurred')

        attributes = self.deserialize(req.context['doc'])
        for key, value in attributes.items():
            setattr(resource, key, value)

        self.db_session.add(resource)
        try:
            self.db_session.commit()
        except sqlalchemy.exc.IntegrityError as err:
            # Cases such as unallowed NULL value should have been checked
            # before we got here (e.g. validate against schema
            # using falconjsonio) - therefore assume this is a UNIQUE
            # constraint violation
            self.db_session.rollback()
            raise falcon.errors.HTTPConflict('Conflict', 'Unique constraint violated')
        except:
            self.db_session.rollback()
            raise

        resp.status = falcon.HTTP_OK
        req.context['result'] = {
            'data': self.serialize(resource),
        }
<|endoftext|>


<commit_before>#Searching and Downloading Google Images/Image Links

#Import Libraries

import time       #Importing the time library to check the time of code execution
import sys    #Importing the System Library

import urllib2

########### Edit From Here ###########

#This list is used to search keywords. You can edit this list to search for google images of your choice. You can simply add and remove elements of the list.
search_keyword = ['Australia', 'Pyramid of Giza']

#This list is used to further add suffix to your search term. Each element of the list will help you download 100 images. First element is blank which denotes that no suffix is added to the search keyword of the above list. You can edit the list by adding/deleting elements from it.So if the first element of the search_keyword is 'Australia' and the second element of keywords is 'high resolution', then it will search for 'Australia High Resolution'
keywords = ['',' high resolution',' paintings',' at night',' from top']

########### End of Editing ###########

#Downloading entire Web Document (Raw Page Content)
def download_page(url):
    version = (3,0)
    cur_version = sys.version_info
    if cur_version >= version:     #If the Current Version of Python is 3.0 or above
        import urllib.request    #urllib library for Extracting web pages
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib.request.Request(url, headers = headers)
            resp = urllib.request.urlopen(req)
            respData = str(resp.read())
            return respData
        except Exception as e:
            print(str(e))
    else:                        #If the Current Version of Python is 2.x
        import urllib2
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib2.Request(url, headers = headers)
            response = urllib2.urlopen(req)
            page = response.read()
            return page
        except:
            return"Page Not found"

#Finding 'Next Image' from the given raw page
def _images_get_next_item(s):
    start_line = s.find('rg_di')
    if start_line == -1:    #If no links are found then give an error!
        end_quote = 0
        link = "no_links"
        return link, end_quote
    else:
        start_line = s.find('"class="rg_di"')
        start_content = s.find('imgurl=',start_line+1)
        end_content = s.find('&',start_content+1)
        content_raw = str(s[start_content+7:end_content])
        return content_raw, end_content

#Getting all links with the help of '_images_get_next_image'
def _images_get_all_items(page):
    items = []
    while True:
        item, end_content = _images_get_next_item(page)
        if item == "no_links":
            break
        else:
            items.append(item)      #Append all the links in the list named 'Links'
            time.sleep(0.1)        #Timer could be used to slow down the request for image downloads
            page = page[end_content:]
    return items

############## Main Program ############
t0 = time.time()   #start the timer

#Download Image Links
i= 0
while i<len(search_keyword):
    items = []
    iteration = "Item no.: " + str(i+1) + " -->" + " Item name = " + str(search_keyword[i])
    print (iteration)
    print ("Evaluating...")
    search_keywords = search_keyword[i]
    search = search_keywords.replace(' ','%20')
    j = 0
    while j<len(keywords):
        pure_keyword = keywords[j].replace(' ','%20')
        url = 'https://www.google.com/search?q=' + search + pure_keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
        #print(url)
        raw_html =  (download_page(url))
        time.sleep(0.1)
        items = items + (_images_get_all_items(raw_html))
        j = j + 1
    #print ("Image Links = "+str(items))
    print ("Total Image Links = "+str(len(items)))
    print ("\n")
    i = i+1

    #This allows you to write all the links into a test file. This text file will be created in the same directory as your code. You can comment out the below 3 lines to stop writing the output to the text file.
    info = open('output.txt', 'a')        #Open the text file called database.txt
    info.write(str(i) + ': ' + str(search_keyword[i-1]) + ": " + str(items) + "\n\n\n")         #Write the title of the page
    info.close()                            #Close the file

t1 = time.time()    #stop the timer
total_time = t1-t0   #Calculating the total time required to crawl, find and download all the links of 60,000 images
print("Total time taken: "+str(total_time)+" Seconds")
print ("Starting Download...")

## To save imges to the same directory
# IN this saving process we are just skipping the URL if there is any error

k=0
errorCount=0
while(k<len(items)):
    from urllib2 import Request,urlopen
    from urllib2 import URLError, HTTPError

    try:
        req = Request(items[k], headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
        response = urlopen(req)
        output_file = open(str(k+1)+".jpg",'wb')
        data = response.read()
        output_file.write(data)
        response.close();

        print("completed ====> "+str(k+1))

        k=k+1;

    except IOError:   #If there is any IOError

        errorCount+=1
        print("IOError on image "+str(k+1))
        k=k+1;

    except HTTPError as e:  #If there is any HTTPError

        errorCount+=1
        print("HTTPError"+str(k))
        k=k+1;
    except URLError as e:

        errorCount+=1
        print("URLError "+str(k))
        k=k+1;

print("\n")
print("All are downloaded")
print("\n"+str(errorCount)+" ----> total Errors")

#----End of the main program ----#
<commit_msg>Updated scraping mechanism.<commit_after>#Searching and Downloading Google Images/Image Links

#Import Libraries

import time       #Importing the time library to check the time of code execution
import sys    #Importing the System Library

import urllib2

########### Edit From Here ###########

#This list is used to search keywords. You can edit this list to search for google images of your choice. You can simply add and remove elements of the list.
search_keyword = ['Scene Sketches']

#This list is used to further add suffix to your search term. Each element of the list will help you download 100 images. First element is blank which denotes that no suffix is added to the search keyword of the above list. You can edit the list by adding/deleting elements from it.So if the first element of the search_keyword is 'Australia' and the second element of keywords is 'high resolution', then it will search for 'Australia High Resolution'
keywords = ['']

########### End of Editing ###########

#Downloading entire Web Document (Raw Page Content)
def download_page(url):
    version = (3,0)
    cur_version = sys.version_info
    if cur_version >= version:     #If the Current Version of Python is 3.0 or above
        import urllib.request    #urllib library for Extracting web pages
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib.request.Request(url, headers = headers)
            resp = urllib.request.urlopen(req)
            respData = str(resp.read())
            return respData
        except Exception as e:
            print(str(e))
    else:                        #If the Current Version of Python is 2.x
        import urllib2
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib2.Request(url, headers = headers)
            response = urllib2.urlopen(req)
            print "Got a response";
            page = response.read()
            return page
        except:
            return"Page Not found"

#Finding 'Next Image' from the given raw page
def _images_get_next_item(s):
    start_line = s.find('rg_di')
    if start_line == -1:    #If no links are found then give an error!
        end_quote = 0
        link = "no_links"
        print "SAD!"
        return link, end_quote
    else:
        start_line = s.find('"oh":1000,"ou"')
        #start_content = s.find('imgurl=',start_line+1)
        start_content = s.find('"ou":', start_line+1)
        end_content = s.find('",',start_content+1)
        content_raw = str(s[start_content+6:end_content])
        #print "HERE:" + str(s[start_content-1:end_content]), start_line, end_content
        return content_raw, end_content

#Getting all links with the help of '_images_get_next_image'
def _images_get_all_items(page):
    items = []
    while True:
        item, end_content = _images_get_next_item(page)
        if item == "no_links":
            break
        else:
            items.append(item)      #Append all the links in the list named 'Links'
           # print items
            if len(items)==20:
                return items
            time.sleep(0.1)        #Timer could be used to slow down the request for image downloads
            page = page[end_content:]
    return items

############## Main Program ############
t0 = time.time()   #start the timer

#Download Image Links
i= 0
while i<len(search_keyword):
    items = []
    iteration = "Item no.: " + str(i+1) + " -->" + " Item name = " + str(search_keyword[i])
    print (iteration)
    print ("Evaluating...")
    search_keywords = search_keyword[i]
    search = search_keywords.replace(' ','%20')
    j = 0
    while j<len(keywords):
        pure_keyword = keywords[j].replace(' ','%20')
        url = 'https://www.google.com/search?q=' + search + pure_keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
        #print(url)
        raw_html =  (download_page(url))
        time.sleep(0.1)
        #print "ASD"
        items = items + (_images_get_all_items(raw_html))
        j = j + 1
    print ("Image Links = "+str(items))
    print ("Total Image Links = "+str(len(items)))
    print ("\n")
    i = i+1

    #This allows you to write all the links into a test file. This text file will be created in the same directory as your code. You can comment out the below 3 lines to stop writing the output to the text file.
    info = open('output.txt', 'a')        #Open the text file called database.txt
    info.write(str(i) + ': ' + str(search_keyword[i-1]) + ": " + str(items) + "\n\n\n")         #Write the title of the page
    info.close()                            #Close the file

t1 = time.time()    #stop the timer
total_time = t1-t0   #Calculating the total time required to crawl, find and download all the links of 60,000 images
print("Total time taken: "+str(total_time)+" Seconds")
print ("Starting Download...")

## To save imges to the same directory
# IN this saving process we are just skipping the URL if there is any error

k=0
errorCount=0
while(k<len(items)):
    from urllib2 import Request,urlopen
    from urllib2 import URLError, HTTPError

    try:
        req = Request(items[k], headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
        response = urlopen(items[k])
        output_file = open("../dataset/" + str(k+1)+".jpg",'wb')
        data = response.read()
        output_file.write(data)
        response.close();

        print("completed ====> "+str(k+1))

        k=k+1;

    except IOError:   #If there is any IOError

        errorCount+=1
        print("IOError on image "+str(k+1))
        k=k+1;

    except HTTPError as e:  #If there is any HTTPError

        errorCount+=1
        print("HTTPError"+str(k))
        k=k+1;
    except URLError as e:

        errorCount+=1
        print("URLError "+str(k))
        k=k+1;

print("\n")
print("All are downloaded")
print("\n"+str(errorCount)+" ----> total Errors")

#----End of the main program ----#
<|endoftext|>