<commit_before>from ckan.plugins.core import SingletonPlugin, implements
from ckanext.harvest.interfaces import IHarvester
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError
import ckan.plugins as p
from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents

import logging
import json
import requests
from requests.exceptions import ConnectionError
import os
import tempfile
from sqlalchemy import exists
from ckan import model

from ckan import logic
NotFound = logic.NotFound

log = logging.getLogger(__name__)

class XRoadHarvesterPlugin(HarvesterBase):

    def info(self):

        return {
            "name": "xroad",
            "title": "XRoad Rest Gateway",
            "description": "Server that provides Rest Gateway for XRoad"
        }

    def validate_config(self, config):

        return ""

    def gather_stage(self, harvest_job):
        log.debug('In xroad harvester gather_stage')

        last_finished_job = self._last_finished_job(harvest_job)
        if last_finished_job:
            last_time = last_finished_job.gather_started.isoformat()
        else:
            last_time = "2011-01-01"

        log.info('Searching for apis modified since: %s UTC',
                 last_time)
        try:
            catalog = self._get_xroad_catalog(harvest_job.source.url, last_time)
            members = self._parse_xroad_data(catalog)
        except ContentFetchError, e:
            self._save_gather_error('%r' % e.message, harvest_job)
            return False
        except KeyError, e:
            self._save_gather_error('Failed to parse response: %r' % e, harvest_job)
            return False
        #members = self.get_xroad_catalog("http://localhost:9090/rest-gateway-0.0.8-SNAPSHOT/Consumer/catalog", "2011-01-01")
        #file = open(os.path.join(os.path.dirname(__file__), '../tests/response.json'))
        #members = json.load(file)

        object_ids = []
        for member in members:

            # Create organization id
            org_id = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')))

            if member['subsystems'] and (type(member['subsystems']['subsystem']) is list):

                org = self._create_or_update_organization({'id': org_id, 'name': member['name'], 'created': member['created'], 'changed': member['changed'], 'removed': member.get('removed', None)}, harvest_job)
                if org is None:
                    continue
                for subsystem in member['subsystems']['subsystem']:

                    # Generate GUID
                    guid = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')) + '.' + unicode(subsystem.get('subsystemCode', '')))

                    # Create harvest object
                    obj = HarvestObject(guid=guid, job=harvest_job,
                                        content=json.dumps({
                                            'owner': org,
                                            'subsystem': subsystem
                                        }))

                    obj.save()
                    object_ids.append(obj.id)

            elif member['subsystems'] and (type(member['subsystems']['subsystem']) is dict):

                org = self._create_or_update_organization({'id': org_id, 'name': member['name'], 'created': member['created'], 'changed': member['changed'], 'removed': member.get('removed', None)}, harvest_job)
                if org is None:
                    continue
                # Generate GUID
                guid = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')) + '.' + unicode(member['subsystems']['subsystem'].get('subsystemCode', '')))
                # Create harvest object
                obj = HarvestObject(guid=guid, job=harvest_job,
                                    content=json.dumps({
                                    'owner': org,
                                     'subsystem': member['subsystems']['subsystem']
                                        }))

                obj.save()
                object_ids.append(obj.id)

        return object_ids

    def fetch_stage(self, harvest_object):
        log.info('In xroad harvester fetch_stage')
        try:
            dataset = json.loads(harvest_object.content)
        except ValueError:
            log.info('Could not parse content for object {0}'.format(harvest_object.id),
                     harvest_object, 'Import')
            self._save_object_error('Could not parse content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False

        services = dataset['subsystem'].get('services', None)
        try:
            if services:
                if type(services['service']) is dict:
                    services['service'] = [services['service']]
                for service in services['service']:
                    if 'wsdl' in service:
                        service['wsdl']['data']  = self._get_wsdl(harvest_object.source.url, service['wsdl']['externalId'])['GetWsdlResponse']['wsdl']
                harvest_object.content = json.dumps(dataset)
                harvest_object.save()
        except TypeError, ContentFetchError:
            self._save_object_error('Could not parse WSDL content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False
        # TODO: Should fetch WSDLs
        return True

    def import_stage(self, harvest_object):
        log.info('In xroad harvester import stage')

        try:
            dataset = json.loads(harvest_object.content)
        except ValueError:
            log.info('Could not parse content for object {0}'.format(harvest_object.id),
                harvest_object, 'Import')
            self._save_object_error('Could not parse content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False

        context = {
            'user': self._get_user_name(),
            'return_id_only': True,
            'ignore_auth': True,
        }

        removed = dataset['subsystem'].get('removed', None)

        try:
            package_dict = p.toolkit.get_action('package_show')(context, {'id': harvest_object.guid })
        except NotFound:
            if removed is not None:
                log.info("Subsystem has been removed, not creating..")
                return "unchanged"

            # Set id
            package_dict = {'id': harvest_object.guid }

        # Get rid of auth audit on the context otherwise we'll get an
        # exception
        context.pop('__auth_audit', None)

        # Local harvest source organization
        source_dataset = p.toolkit.get_action('package_show')(context, {'id': harvest_object.source.id})
        local_org = source_dataset.get('owner_org')

        # Create org
        log.info("Organization: " + dataset['owner']['name'] )

        context = {
            'model': model,
            'session': model.Session,
            'user': self._get_user_name(),
            'ignore_auth': True,
        }

        if removed is not None:
            log.info("Removing service %s", package_dict['name'])
            p.toolkit.get_action('package_delete')(context, {'id': package_dict['id']})
            harvest_object.current = False
            return True

        if dataset['owner'] is not None:
            local_org = dataset['owner']['name']
        package_dict['owner_org'] = local_org
        # Munge name

        package_dict['title'] = dataset['subsystem']['subsystemCode']
        package_dict['name'] = munge_title_to_name(dataset['subsystem']['subsystemCode'])
        #dataset['notes'] = {'fi': 'this is example'}
        package_dict['shared_resource'] = "no"

        result = self._create_or_update_package(package_dict, harvest_object, package_dict_form='package_show')
        apikey = self._get_api_key()
        if result is True or result is "unchanged":
                services = dataset['subsystem'].get('services', None)
                if services:
                    for service in services['service']:
                        if 'wsdl' in service and 'data' in service['wsdl']:

                            f = tempfile.NamedTemporaryFile(delete=False)
                            f.write(service['wsdl']['data'])
                            f.close()

                            service_code = service.get('serviceCode', None)
                            service_version = service.get('serviceVersion', None)

                            wsdl_exists = False
                            if service_code is not None and service_version is not None:

                                for resource in package_dict.get('resources', []):
                                    if resource['name'] == service_code + "." + service_version:
                                        wsdl_exists = True
                                        changed = service['wsdl'].get('changed', None)
                                        if changed and changed > resource['created']:
                                            log.info('WSDL changed after last harvest, replacing...')
                                            requests.post('https://0.0.0.0/api/action/resource_update',
                                                                     data={
                                                                         "package_id":package_dict['id'],
                                                                         "url": "",
                                                                         "name": service_code + "." + service_version,
                                                                         "id": resource['id']
                                                                     },
                                                                     headers={"X-CKAN-API-Key": apikey },
                                                                     files={'upload': ('service.wsdl',file(f.name))},
                                                                     verify=False)
                                            result = True

                                if wsdl_exists is False:
                                    requests.post('https://0.0.0.0/api/action/resource_create',
                                                     data={
                                                         "package_id":package_dict['id'],
                                                         "url": "",
                                                         "name": service_code + "." + service_version
                                                     },
                                                     headers={"X-CKAN-API-Key": apikey },
                                                     files={'upload': ('service.wsdl',file(f.name))},
                                                     verify=False)
                                    result = True

                            os.unlink(f.name)
                        else:
                            return False

        log.info('Created dataset %s', package_dict['name'])

        return result

    def _get_xroad_catalog(self, url, changed_after):
        try:
            r = requests.get(url + '/Consumer/ListMembers', params = {'changedAfter' : changed_after}, headers = {'Accept': 'application/json'})
        except ConnectionError:
            raise ContentFetchError("Calling XRoad service ListMembers failed!")
        if r.status_code != requests.codes.ok:
            raise ContentFetchError("Calling XRoad service ListMembers failed!")
        try:
            result = r.json()
        except ValueError:
            raise ContentFetchError("ListMembers JSON parse failed")
        return result

        #file = open(os.path.join(os.path.dirname(__file__), '../tests/response.json'))
        #return json.load(file)

    def _parse_xroad_data(self, res):
        #return res.json()['ListMembersResponse']['memberList']['members']
        if isinstance(res['ListMembersResponse']['memberList'], basestring):
            return []
        return res['ListMembersResponse']['memberList']['member']

    def _get_wsdl(self, url, external_id):
        try:
            r = requests.get(url + '/Consumer/GetWsdl', params = {'externalId' : external_id}, headers = {'Accept': 'application/json'})
        except ConnectionError:
            raise ContentFetchError("Calling XRoad service GetWsdl failed!")
        if r.status_code != requests.codes.ok:
            raise ContentFetchError("Calling XRoad service GetWsdl failed!")
        return r.json()

    @classmethod
    def _last_error_free_job(cls, harvest_job):
        # TODO weed out cancelled jobs somehow.
        # look for jobs with no gather errors
        jobs = \
            model.Session.query(HarvestJob) \
                .filter(HarvestJob.source == harvest_job.source) \
                .filter(HarvestJob.gather_started != None) \
                .filter(HarvestJob.status == 'Finished') \
                .filter(HarvestJob.id != harvest_job.id) \
                .filter(
                ~exists().where(
                    HarvestGatherError.harvest_job_id == HarvestJob.id)) \
                .order_by(HarvestJob.gather_started.desc())
        # now check them until we find one with no fetch/import errors
        # (looping rather than doing sql, in case there are lots of objects
        # and lots of jobs)
        for job in jobs:
            for obj in job.objects:
                if obj.current is False and \
                                obj.report_status != 'not modified':
                    # unsuccessful, so go onto the next job
                    break
            else:
                log.info("Returning job", job)
                return job

    @classmethod
    def _last_finished_job(self, harvest_job):
        job = model.Session.query(HarvestJob)\
            .filter(HarvestJob.source == harvest_job.source)\
            .filter(HarvestJob.finished != None)\
            .filter(HarvestJob.id != harvest_job.id)\
            .order_by(HarvestJob.finished.desc()).first()

        return job

    def _get_api_key(self):

        context = {'model': model,
           'ignore_auth': True,
        }

        site_user = p.toolkit.get_action('get_site_user')(context, {})

        return site_user['apikey']

    def _create_or_update_organization(self, data_dict, harvest_job):

        context = {
            'model': model,
            'session': model.Session,
            'user': self._get_user_name(),
            'ignore_auth': True,
        }

        try:
            org = p.toolkit.get_action('organization_show')(context, {'id': data_dict['id']})
            log.info("found %s", org)

            if data_dict['removed']:
                log.info("Organization was removed, removing from catalog..")
                p.toolkit.get_action('organization_delete')(context, org)
                return None

            last_finished_job = self._last_finished_job(harvest_job)
            log.info(last_finished_job)
            if last_finished_job and last_finished_job < data_dict['changed']:
                log.info("updating organization")
                org = p.toolkit.get_action('organization_update')(context, {'title': data_dict['name'],
                                                                      'name': munge_title_to_name(data_dict['name']),
                                                                      'id': data_dict['id']})
        except NotFound:
            if data_dict['removed']:
                log.info("Organization was removed, not creating..")
                return None
            log.info("Organization %s not found, creating...", data_dict['name'])

            # Get rid of auth audit on the context otherwise we'll get an
            # exception
            context.pop('__auth_audit', None)

            org = p.toolkit.get_action('organization_create')(context, {'title': data_dict['name'],
                                                                        'name': munge_title_to_name(data_dict['name']),
                                                                        'id': data_dict['id']})
            log.info(org)

        return org

class ContentFetchError(Exception):
    pass<commit_msg>LK-200: Harvested apis are private by default<commit_after>from ckan.plugins.core import SingletonPlugin, implements
from ckanext.harvest.interfaces import IHarvester
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.harvest.model import HarvestJob, HarvestObject, HarvestGatherError
import ckan.plugins as p
from ckan.lib.munge import munge_title_to_name, substitute_ascii_equivalents

import logging
import json
import requests
from requests.exceptions import ConnectionError
import os
import tempfile
from sqlalchemy import exists
from ckan import model

from ckan import logic
NotFound = logic.NotFound

log = logging.getLogger(__name__)

class XRoadHarvesterPlugin(HarvesterBase):

    def info(self):

        return {
            "name": "xroad",
            "title": "XRoad Rest Gateway",
            "description": "Server that provides Rest Gateway for XRoad"
        }

    def validate_config(self, config):

        return ""

    def gather_stage(self, harvest_job):
        log.debug('In xroad harvester gather_stage')

        last_finished_job = self._last_finished_job(harvest_job)
        if last_finished_job:
            last_time = last_finished_job.gather_started.isoformat()
        else:
            last_time = "2011-01-01"

        log.info('Searching for apis modified since: %s UTC',
                 last_time)
        try:
            catalog = self._get_xroad_catalog(harvest_job.source.url, last_time)
            members = self._parse_xroad_data(catalog)
        except ContentFetchError, e:
            self._save_gather_error('%r' % e.message, harvest_job)
            return False
        except KeyError, e:
            self._save_gather_error('Failed to parse response: %r' % e, harvest_job)
            return False
        #members = self.get_xroad_catalog("http://localhost:9090/rest-gateway-0.0.8-SNAPSHOT/Consumer/catalog", "2011-01-01")
        #file = open(os.path.join(os.path.dirname(__file__), '../tests/response.json'))
        #members = json.load(file)

        object_ids = []
        for member in members:

            # Create organization id
            org_id = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')))

            if member['subsystems'] and (type(member['subsystems']['subsystem']) is list):

                org = self._create_or_update_organization({'id': org_id, 'name': member['name'], 'created': member['created'], 'changed': member['changed'], 'removed': member.get('removed', None)}, harvest_job)
                if org is None:
                    continue
                for subsystem in member['subsystems']['subsystem']:

                    # Generate GUID
                    guid = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')) + '.' + unicode(subsystem.get('subsystemCode', '')))

                    # Create harvest object
                    obj = HarvestObject(guid=guid, job=harvest_job,
                                        content=json.dumps({
                                            'owner': org,
                                            'subsystem': subsystem
                                        }))

                    obj.save()
                    object_ids.append(obj.id)

            elif member['subsystems'] and (type(member['subsystems']['subsystem']) is dict):

                org = self._create_or_update_organization({'id': org_id, 'name': member['name'], 'created': member['created'], 'changed': member['changed'], 'removed': member.get('removed', None)}, harvest_job)
                if org is None:
                    continue
                # Generate GUID
                guid = substitute_ascii_equivalents(unicode(member.get('xRoadInstance', '')) + '.' + unicode(member.get('memberClass', '')) + '.' + unicode(member.get('memberCode', '')) + '.' + unicode(member['subsystems']['subsystem'].get('subsystemCode', '')))
                # Create harvest object
                obj = HarvestObject(guid=guid, job=harvest_job,
                                    content=json.dumps({
                                    'owner': org,
                                     'subsystem': member['subsystems']['subsystem']
                                        }))

                obj.save()
                object_ids.append(obj.id)

        return object_ids

    def fetch_stage(self, harvest_object):
        log.info('In xroad harvester fetch_stage')
        try:
            dataset = json.loads(harvest_object.content)
        except ValueError:
            log.info('Could not parse content for object {0}'.format(harvest_object.id),
                     harvest_object, 'Import')
            self._save_object_error('Could not parse content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False

        services = dataset['subsystem'].get('services', None)
        try:
            if services:
                if type(services['service']) is dict:
                    services['service'] = [services['service']]
                for service in services['service']:
                    if 'wsdl' in service:
                        service['wsdl']['data']  = self._get_wsdl(harvest_object.source.url, service['wsdl']['externalId'])['GetWsdlResponse']['wsdl']
                harvest_object.content = json.dumps(dataset)
                harvest_object.save()
        except TypeError, ContentFetchError:
            self._save_object_error('Could not parse WSDL content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False
        # TODO: Should fetch WSDLs
        return True

    def import_stage(self, harvest_object):
        log.info('In xroad harvester import stage')

        try:
            dataset = json.loads(harvest_object.content)
        except ValueError:
            log.info('Could not parse content for object {0}'.format(harvest_object.id),
                harvest_object, 'Import')
            self._save_object_error('Could not parse content for object {0}'.format(harvest_object.id),
                                    harvest_object, 'Import')
            return False

        context = {
            'user': self._get_user_name(),
            'return_id_only': True,
            'ignore_auth': True,
        }

        removed = dataset['subsystem'].get('removed', None)

        try:
            package_dict = p.toolkit.get_action('package_show')(context, {'id': harvest_object.guid })
        except NotFound:
            if removed is not None:
                log.info("Subsystem has been removed, not creating..")
                return "unchanged"

            # Set id
            package_dict = {'id': harvest_object.guid }

        # Get rid of auth audit on the context otherwise we'll get an
        # exception
        context.pop('__auth_audit', None)

        # Local harvest source organization
        source_dataset = p.toolkit.get_action('package_show')(context, {'id': harvest_object.source.id})
        local_org = source_dataset.get('owner_org')

        # Create org
        log.info("Organization: " + dataset['owner']['name'] )

        context = {
            'model': model,
            'session': model.Session,
            'user': self._get_user_name(),
            'ignore_auth': True,
        }

        if removed is not None:
            log.info("Removing service %s", package_dict['name'])
            p.toolkit.get_action('package_delete')(context, {'id': package_dict['id']})
            harvest_object.current = False
            return True

        if dataset['owner'] is not None:
            local_org = dataset['owner']['name']
        package_dict['owner_org'] = local_org
        # Munge name

        package_dict['title'] = dataset['subsystem']['subsystemCode']
        package_dict['name'] = munge_title_to_name(dataset['subsystem']['subsystemCode'])
        package_dict['shared_resource'] = "no"
        package_dict['private'] = True

        result = self._create_or_update_package(package_dict, harvest_object, package_dict_form='package_show')
        apikey = self._get_api_key()
        if result is True or result is "unchanged":
                services = dataset['subsystem'].get('services', None)
                if services:
                    for service in services['service']:
                        if 'wsdl' in service and 'data' in service['wsdl']:

                            f = tempfile.NamedTemporaryFile(delete=False)
                            f.write(service['wsdl']['data'])
                            f.close()

                            service_code = service.get('serviceCode', None)
                            service_version = service.get('serviceVersion', None)

                            wsdl_exists = False
                            if service_code is not None and service_version is not None:

                                for resource in package_dict.get('resources', []):
                                    if resource['name'] == service_code + "." + service_version:
                                        wsdl_exists = True
                                        changed = service['wsdl'].get('changed', None)
                                        if changed and changed > resource['created']:
                                            log.info('WSDL changed after last harvest, replacing...')
                                            requests.post('https://0.0.0.0/api/action/resource_update',
                                                                     data={
                                                                         "package_id":package_dict['id'],
                                                                         "url": "",
                                                                         "name": service_code + "." + service_version,
                                                                         "id": resource['id']
                                                                     },
                                                                     headers={"X-CKAN-API-Key": apikey },
                                                                     files={'upload': ('service.wsdl',file(f.name))},
                                                                     verify=False)
                                            result = True

                                if wsdl_exists is False:
                                    requests.post('https://0.0.0.0/api/action/resource_create',
                                                     data={
                                                         "package_id":package_dict['id'],
                                                         "url": "",
                                                         "name": service_code + "." + service_version
                                                     },
                                                     headers={"X-CKAN-API-Key": apikey },
                                                     files={'upload': ('service.wsdl',file(f.name))},
                                                     verify=False)
                                    result = True

                            os.unlink(f.name)
                        else:
                            return False

        log.info('Created dataset %s', package_dict['name'])

        return result

    def _get_xroad_catalog(self, url, changed_after):
        try:
            r = requests.get(url + '/Consumer/ListMembers', params = {'changedAfter' : changed_after}, headers = {'Accept': 'application/json'})
        except ConnectionError:
            raise ContentFetchError("Calling XRoad service ListMembers failed!")
        if r.status_code != requests.codes.ok:
            raise ContentFetchError("Calling XRoad service ListMembers failed!")
        try:
            result = r.json()
        except ValueError:
            raise ContentFetchError("ListMembers JSON parse failed")
        return result

        #file = open(os.path.join(os.path.dirname(__file__), '../tests/response.json'))
        #return json.load(file)

    def _parse_xroad_data(self, res):
        #return res.json()['ListMembersResponse']['memberList']['members']
        if isinstance(res['ListMembersResponse']['memberList'], basestring):
            return []
        return res['ListMembersResponse']['memberList']['member']

    def _get_wsdl(self, url, external_id):
        try:
            r = requests.get(url + '/Consumer/GetWsdl', params = {'externalId' : external_id}, headers = {'Accept': 'application/json'})
        except ConnectionError:
            raise ContentFetchError("Calling XRoad service GetWsdl failed!")
        if r.status_code != requests.codes.ok:
            raise ContentFetchError("Calling XRoad service GetWsdl failed!")
        return r.json()

    @classmethod
    def _last_error_free_job(cls, harvest_job):
        # TODO weed out cancelled jobs somehow.
        # look for jobs with no gather errors
        jobs = \
            model.Session.query(HarvestJob) \
                .filter(HarvestJob.source == harvest_job.source) \
                .filter(HarvestJob.gather_started != None) \
                .filter(HarvestJob.status == 'Finished') \
                .filter(HarvestJob.id != harvest_job.id) \
                .filter(
                ~exists().where(
                    HarvestGatherError.harvest_job_id == HarvestJob.id)) \
                .order_by(HarvestJob.gather_started.desc())
        # now check them until we find one with no fetch/import errors
        # (looping rather than doing sql, in case there are lots of objects
        # and lots of jobs)
        for job in jobs:
            for obj in job.objects:
                if obj.current is False and \
                                obj.report_status != 'not modified':
                    # unsuccessful, so go onto the next job
                    break
            else:
                log.info("Returning job", job)
                return job

    @classmethod
    def _last_finished_job(self, harvest_job):
        job = model.Session.query(HarvestJob)\
            .filter(HarvestJob.source == harvest_job.source)\
            .filter(HarvestJob.finished != None)\
            .filter(HarvestJob.id != harvest_job.id)\
            .order_by(HarvestJob.finished.desc()).first()

        return job

    def _get_api_key(self):

        context = {'model': model,
           'ignore_auth': True,
        }

        site_user = p.toolkit.get_action('get_site_user')(context, {})

        return site_user['apikey']

    def _create_or_update_organization(self, data_dict, harvest_job):

        context = {
            'model': model,
            'session': model.Session,
            'user': self._get_user_name(),
            'ignore_auth': True,
        }

        try:
            org = p.toolkit.get_action('organization_show')(context, {'id': data_dict['id']})
            log.info("found %s", org)

            if data_dict['removed']:
                log.info("Organization was removed, removing from catalog..")
                p.toolkit.get_action('organization_delete')(context, org)
                return None

            last_finished_job = self._last_finished_job(harvest_job)
            log.info(last_finished_job)
            if last_finished_job and last_finished_job < data_dict['changed']:
                log.info("updating organization")
                org = p.toolkit.get_action('organization_update')(context, {'title': data_dict['name'],
                                                                      'name': munge_title_to_name(data_dict['name']),
                                                                      'id': data_dict['id']})
        except NotFound:
            if data_dict['removed']:
                log.info("Organization was removed, not creating..")
                return None
            log.info("Organization %s not found, creating...", data_dict['name'])

            # Get rid of auth audit on the context otherwise we'll get an
            # exception
            context.pop('__auth_audit', None)

            org = p.toolkit.get_action('organization_create')(context, {'title': data_dict['name'],
                                                                        'name': munge_title_to_name(data_dict['name']),
                                                                        'id': data_dict['id']})
            log.info(org)

        return org

class ContentFetchError(Exception):
    pass<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import cgi
import cgitb
import datetime
from configparser import SafeConfigParser

from dnsupdate import doUpdate, isValidV4Addr, isValidV6Addr

cgitb.enable()

configuration_file = "ddns.ini"

def read_data(domain):
    valid_options = ["password", "dns-server", "nsupdate-key", "origin"]

    cp = SafeConfigParser()
    cp.read(configuration_file)
    if cp.has_section(domain):
        data = dict()
        for option, value in cp.items(domain):
            if option in valid_options:
                data[option] = value
            else:
                print("Error: Found a invalid option for domain",
                      domain, ":", option)
                exit()
        return data

    return None

def print_header():
    print("Content-Type: text/html")     # HTML is following
    print()                              # blank line, end of headers

ip_arguments = ["ip4addr", "ip6addr"]

def read_arguments():
    required_arguments = ["domain", "password"]

    ip_check = {"ip4addr": isValidV4Addr, "ip6addr": isValidV6Addr}

    valid_arguments = required_arguments + ip_arguments
    args = dict()
    # Read the arguments
    form = cgi.FieldStorage()
    for arg in valid_arguments:
        if arg in form:
            args[arg] = form[arg].value

    # Verify that they are complete
    error = False
    for reqarg in required_arguments:
        if reqarg not in args:
            print("Error:", reqarg, "was not passed
")
            error = True

    foundAny = False
    for iparg in ip_arguments:
        if iparg in args:
            if not ip_check[iparg](args[iparg]):
                print("Error:", args[iparg], "is not a valid", iparg)
                error = True
            else:
                foundAny = True

    if not error and not foundAny:
        # Get IP from REMOTE_ADDR anc check whether it is v4 or v6
        ip = os.environ["REMOTE_ADDR"]
        for iparg in ip_arguments:
            if ip_check[iparg](ip):
                args[iparg] = ip

    if error is True:
        print("Error: There were errors while parsing the arguments.
")
        exit()

    return args

def generate_nsupdate_key_string(ddata):
    origin = ddata["origin"]
    if not origin.endswith("."):
        origin = origin + "."

    return {origin: ddata["nsupdate-key"]}

def generate_action_string(domain, ddata, ip, ipaddr_type, action="update"):
    ip_type = {"ip4addr": "A", "ip6addr": "AAAA"}

    TTL = 60
    TYPE = ip_type[ipaddr_type]
    return [action, domain, str(TTL), TYPE, ip]

def main():
    print_header()

    # Parse the passed arguments
    arguments = read_arguments()

    domain = arguments["domain"]

    # Read the 'configuration_file'
    data = read_data(domain)

    # Is the request valid and the domain configured?
    if not (data is not None and arguments["password"] == data["password"]):
        print("Error: Invalid domain/password combination")
        exit()

    nsu_str = generate_nsupdate_key_string(data)

    def date_update_str(): return ["update", domain, "60", "TXT",
                                   "last update: %s Europe/Berlin" % datetime.datetime.now()]

    # add date record
    doUpdate(data["dns-server"], nsu_str,
             data["origin"], False, date_update_str())

    # Update each requested ipaddr_type, or delete if not passed
    for ipaddr_type in ip_arguments:
        if ipaddr_type in arguments:
            a = "update"
        else:
            a = "delete"

        action = generate_action_string(
            domain, data, arguments[ipaddr_type], ipaddr_type, action=a)
        doUpdate(data["dns-server"], nsu_str,
                 data["origin"], False, action)

if __name__ == "__main__":
    main()
<commit_msg>catch dns exceptions<commit_after>#!/usr/bin/env python3

import os
import cgi
import cgitb
import datetime
from configparser import SafeConfigParser

from dnsupdate import doUpdate, isValidV4Addr, isValidV6Addr
import dns.exception

cgitb.enable()

configuration_file = "ddns.ini"

def read_data(domain):
    valid_options = ["password", "dns-server", "nsupdate-key", "origin"]

    cp = SafeConfigParser()
    cp.read(configuration_file)
    if cp.has_section(domain):
        data = dict()
        for option, value in cp.items(domain):
            if option in valid_options:
                data[option] = value
            else:
                print("Error: Found a invalid option for domain",
                      domain, ":", option)
                exit()
        return data

    return None

def print_header():
    print("Content-Type: text/html")     # HTML is following
    print()                              # blank line, end of headers

ip_arguments = ["ip4addr", "ip6addr"]

def read_arguments():
    required_arguments = ["domain", "password"]

    ip_check = {"ip4addr": isValidV4Addr, "ip6addr": isValidV6Addr}

    valid_arguments = required_arguments + ip_arguments
    args = dict()
    # Read the arguments
    form = cgi.FieldStorage()
    for arg in valid_arguments:
        if arg in form:
            args[arg] = form[arg].value

    # Verify that they are complete
    error = False
    for reqarg in required_arguments:
        if reqarg not in args:
            print("Error:", reqarg, "was not passed
")
            error = True

    foundAny = False
    for iparg in ip_arguments:
        if iparg in args:
            try:
                ip_check[iparg](args[iparg])
                foundAny = True
            except dns.exception.SyntaxError as e:
                print("Error:", args[iparg], "is not a valid", iparg)
                error = True

    if not error and not foundAny:
        # Get IP from REMOTE_ADDR anc check whether it is v4 or v6
        ip = os.environ["REMOTE_ADDR"]
        for iparg in ip_arguments:
            try:
                if ip_check[iparg](ip):
                    args[iparg] = ip

            except dns.exception.SyntaxError as e:
                pass

    if error is True:
        print("Error: There were errors while parsing the arguments.
")
        exit()

    return args

def generate_nsupdate_key_string(ddata):
    origin = ddata["origin"]
    if not origin.endswith("."):
        origin = origin + "."

    return {origin: ddata["nsupdate-key"]}

def generate_action_string(domain, ddata, ip, ipaddr_type, action="update"):
    ip_type = {"ip4addr": "A", "ip6addr": "AAAA"}

    TTL = 60
    TYPE = ip_type[ipaddr_type]
    return [action, domain, str(TTL), TYPE, ip]

def main():
    print_header()

    # Parse the passed arguments
    arguments = read_arguments()

    domain = arguments["domain"]

    # Read the 'configuration_file'
    data = read_data(domain)

    # Is the request valid and the domain configured?
    if not (data is not None and arguments["password"] == data["password"]):
        print("Error: Invalid domain/password combination")
        exit()

    nsu_str = generate_nsupdate_key_string(data)

    def date_update_str(): return ["update", domain, "60", "TXT",
                                   "last update: %s Europe/Berlin" % datetime.datetime.now()]

    # add date record
    doUpdate(data["dns-server"], nsu_str,
             data["origin"], False, date_update_str())

    # Update each requested ipaddr_type, or delete if not passed
    for ipaddr_type in ip_arguments:
        if ipaddr_type in arguments:
            a = "update"
        else:
            a = "delete"

        action = generate_action_string(
            domain, data, arguments[ipaddr_type], ipaddr_type, action=a)
        doUpdate(data["dns-server"], nsu_str,
                 data["origin"], False, action)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>## @file
# This file is used to define class objects of INF file [UserExtension] section.
# It will consumed by InfParser.
#
# Copyright (c) 2011, Intel Corporation. All rights reserved.

#
# This program and the accompanying materials are licensed and made available
# under the terms and conditions of the BSD License which accompanies this
# distribution. The full text of the license may be found at
# http://opensource.org/licenses/bsd-license.php
#
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.

'''
InfUserExtensionsObject
'''

from Logger import StringTable as ST
from Logger import ToolError
import Logger.Log as Logger
from Library import GlobalData

from Library.Misc import Sdict

class InfUserExtensionItem():
    def __init__(self,
                 Content = '',
                 UserId = '',
                 IdString = ''):
        self.Content  = Content
        self.UserId   = UserId
        self.IdString = IdString
        self.SupArchList = []

    def SetContent(self, Content):
        self.Content = Content
    def GetContent(self):
        return self.Content

    def SetUserId(self, UserId):
        self.UserId = UserId
    def GetUserId(self):
        return self.UserId

    def SetIdString(self, IdString):
        self.IdString = IdString
    def GetIdString(self):
        return self.IdString

    def SetSupArchList(self, SupArchList):
        self.SupArchList = SupArchList
    def GetSupArchList(self):
        return self.SupArchList

##
#
#
#
class InfUserExtensionObject():
    def __init__(self):
        self.UserExtension = Sdict()

    def SetUserExtension(self, UserExtensionCont, IdContent=None, LineNo=None):
        if not UserExtensionCont or UserExtensionCont == '':
            return True
        #
        # IdContent is a list contain UserId and IdString
        # For this call the general section header  parser, if no definition of
        # IdString/UserId, it will return 'COMMON'
        #
        for IdContentItem in IdContent:
            InfUserExtensionItemObj = InfUserExtensionItem()
            if IdContentItem[0] == 'COMMON':
                UserId = ''
            else:
                UserId = IdContentItem[0]

            if IdContentItem[1] == 'COMMON':
                IdString = ''
            else:
                IdString = IdContentItem[1]

            #
            # Fill UserExtensionObj members.
            #
            InfUserExtensionItemObj.SetUserId(UserId)
            InfUserExtensionItemObj.SetIdString(IdString)
            InfUserExtensionItemObj.SetContent(UserExtensionCont)
            InfUserExtensionItemObj.SetSupArchList(IdContentItem[2])

            for CheckItem in self.UserExtension:
                if IdContentItem[0] == CheckItem[0] and IdContentItem[1] == CheckItem[1]:
                    if IdContentItem[2].upper() == 'COMMON' or CheckItem[2].upper() == 'COMMON':
                        #
                        # For COMMON ARCH type, do special check.
                        #
                        Logger.Error('InfParser',
                            ToolError.FORMAT_INVALID,
                            ST.ERR_INF_PARSER_UE_SECTION_DUPLICATE_ERROR%\
                            (IdContentItem[0] + '.' + IdContentItem[1] + '.' + IdContentItem[2]),
                            File=GlobalData.gINF_MODULE_NAME,
                            Line=LineNo,
                            ExtraData=None)

            if self.UserExtension.has_key(IdContentItem):
                #
                # Each UserExtensions section header must have a unique set
                # of UserId, IdString and Arch values.
                # This means that the same UserId can be used in more than one
                # section header, provided the IdString or Arch values are
                # different. The same IdString values can be used in more than
                # one section header if the UserId or Arch values are
                # different. The same UserId and the same IdString can be used
                # in a section header if the Arch values are different in each
                # of the section headers.
                #
                Logger.Error('InfParser',
                             ToolError.FORMAT_INVALID,
                             ST.ERR_INF_PARSER_UE_SECTION_DUPLICATE_ERROR%\
                             (IdContentItem[0] + '.' + IdContentItem[1] + '.' + IdContentItem[2]),
                             File=GlobalData.gINF_MODULE_NAME,
                             Line=LineNo,
                             ExtraData=None)
            else:
                UserExtensionList = []
                UserExtensionList.append(InfUserExtensionItemObj)
                self.UserExtension[IdContentItem] = UserExtensionList

        return True

    def GetUserExtension(self):
        return self.UserExtension<commit_msg>1. Remove the check for duplicated UserExtension section.<commit_after>## @file
# This file is used to define class objects of INF file [UserExtension] section.
# It will consumed by InfParser.
#
# Copyright (c) 2011, Intel Corporation. All rights reserved.

#
# This program and the accompanying materials are licensed and made available
# under the terms and conditions of the BSD License which accompanies this
# distribution. The full text of the license may be found at
# http://opensource.org/licenses/bsd-license.php
#
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.

'''
InfUserExtensionsObject
'''

from Logger import StringTable as ST
from Logger import ToolError
import Logger.Log as Logger
from Library import GlobalData

from Library.Misc import Sdict

class InfUserExtensionItem():
    def __init__(self,
                 Content = '',
                 UserId = '',
                 IdString = ''):
        self.Content  = Content
        self.UserId   = UserId
        self.IdString = IdString
        self.SupArchList = []

    def SetContent(self, Content):
        self.Content = Content
    def GetContent(self):
        return self.Content

    def SetUserId(self, UserId):
        self.UserId = UserId
    def GetUserId(self):
        return self.UserId

    def SetIdString(self, IdString):
        self.IdString = IdString
    def GetIdString(self):
        return self.IdString

    def SetSupArchList(self, SupArchList):
        self.SupArchList = SupArchList
    def GetSupArchList(self):
        return self.SupArchList

##
#
#
#
class InfUserExtensionObject():
    def __init__(self):
        self.UserExtension = Sdict()

    def SetUserExtension(self, UserExtensionCont, IdContent=None, LineNo=None):
        if not UserExtensionCont or UserExtensionCont == '':
            return True
        #
        # IdContent is a list contain UserId and IdString
        # For this call the general section header  parser, if no definition of
        # IdString/UserId, it will return 'COMMON'
        #
        for IdContentItem in IdContent:
            InfUserExtensionItemObj = InfUserExtensionItem()
            if IdContentItem[0] == 'COMMON':
                UserId = ''
            else:
                UserId = IdContentItem[0]

            if IdContentItem[1] == 'COMMON':
                IdString = ''
            else:
                IdString = IdContentItem[1]

            #
            # Fill UserExtensionObj members.
            #
            InfUserExtensionItemObj.SetUserId(UserId)
            InfUserExtensionItemObj.SetIdString(IdString)
            InfUserExtensionItemObj.SetContent(UserExtensionCont)
            InfUserExtensionItemObj.SetSupArchList(IdContentItem[2])

#            for CheckItem in self.UserExtension:
#                if IdContentItem[0] == CheckItem[0] and IdContentItem[1] == CheckItem[1]:
#                    if IdContentItem[2].upper() == 'COMMON' or CheckItem[2].upper() == 'COMMON':
#                        #
#                        # For COMMON ARCH type, do special check.
#                        #
#                        Logger.Error('InfParser',
#                            ToolError.FORMAT_INVALID,
#                            ST.ERR_INF_PARSER_UE_SECTION_DUPLICATE_ERROR%\
#                            (IdContentItem[0] + '.' + IdContentItem[1] + '.' + IdContentItem[2]),
#                            File=GlobalData.gINF_MODULE_NAME,
#                            Line=LineNo,
#                            ExtraData=None)

            if self.UserExtension.has_key(IdContentItem):
                #
                # Each UserExtensions section header must have a unique set
                # of UserId, IdString and Arch values.
                # This means that the same UserId can be used in more than one
                # section header, provided the IdString or Arch values are
                # different. The same IdString values can be used in more than
                # one section header if the UserId or Arch values are
                # different. The same UserId and the same IdString can be used
                # in a section header if the Arch values are different in each
                # of the section headers.
                #
                Logger.Error('InfParser',
                             ToolError.FORMAT_INVALID,
                             ST.ERR_INF_PARSER_UE_SECTION_DUPLICATE_ERROR%\
                             (IdContentItem[0] + '.' + IdContentItem[1] + '.' + IdContentItem[2]),
                             File=GlobalData.gINF_MODULE_NAME,
                             Line=LineNo,
                             ExtraData=None)
            else:
                UserExtensionList = []
                UserExtensionList.append(InfUserExtensionItemObj)
                self.UserExtension[IdContentItem] = UserExtensionList

        return True

    def GetUserExtension(self):
        return self.UserExtension<|endoftext|>


<commit_before># Copyright 2015 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
from urwid import Text

from subiquitycore.ui.buttons import done_btn, cancel_btn, menu_btn, other_btn
from subiquitycore.ui.container import Columns, ListBox, Pile
from subiquitycore.ui.utils import button_pile, Padding
from subiquitycore.view import BaseView

from subiquity.models.filesystem import humanize_size

log = logging.getLogger('subiquity.ui.filesystem.disk_partition')

class DiskPartitionView(BaseView):
    footer = (_("Partition the disk, or format the entire device "
              "without partitions"))

    def __init__(self, model, controller, disk):
        self.model = model
        self.controller = controller
        self.disk = disk
        self.title = _("Partition, format, and mount {}").format(disk.label)

        self.body = Pile([
            ('pack', Text("")),
            Padding.center_79(ListBox(
                self._build_model_inputs() + [
                Text(""),
                self.show_disk_info_w(),
                ])),
            ('pack', Text("")),
            ('pack', self._build_buttons()),
            ('pack', Text("")),
            ])
        super().__init__(self.body)

    def _build_buttons(self):
        cancel = cancel_btn(_("Cancel"), on_press=self.cancel)
        done = done_btn(_("Done"), on_press=self.done)
        return button_pile([done, cancel])

    def _build_model_inputs(self):
        partitioned_disks = []

        if self.disk.free > 0:
            if len(self.disk.partitions()) > 0:
                final_label = _("Add another partition")
            else:
                final_label = _("Add first partition")
            label_width = max(25, len(final_label) + 4)
        else:
            label_width = 25

        def format_volume(label, part):
            size = humanize_size(part.size)
            if part.fs() is None:
                 fstype = '-'
                 mountpoint = '-'
            elif part.fs().mount() is None:
                fstype = part.fs().fstype
                mountpoint = '-'
            else:
                fstype = part.fs().fstype
                mountpoint = part.fs().mount().path
            if part.type == 'disk':
                part_btn = menu_btn(label, on_press=self._click_disk)
            else:
                part_btn = menu_btn(label, on_press=self._click_part, user_arg=part)
            return Columns([
                (label_width, part_btn),
                (9, Text(size, align="right")),
                Text(fstype),
                Text(mountpoint),
            ], 2)
        if self.disk.fs() is not None:
            partitioned_disks.append(format_volume(_("entire disk"), self.disk))
        else:
            for part in self.disk.partitions():
                partitioned_disks.append(format_volume(_("Partition {}").format(part._number), part))
        if self.disk.free > 0:
            free_space = humanize_size(self.disk.free)
            add_btn = menu_btn(final_label, on_press=self.add_partition)
            partitioned_disks.append(Columns([
                (label_width, add_btn),
                (9, Text(free_space, align="right")),
                Text(_("free space")),
            ], 2))
        if self.model.bootable():
            for p in self.disk.partitions():
                if p.flag in ('bios_grub', 'boot'):
                    break
            else:
                partitioned_disks.append(Text(""))
                partitioned_disks.append(
                    button_pile([other_btn(label=_("Select as boot disk"), on_press=self.make_boot_disk)]))
        if len(self.disk.partitions()) == 0 and \
           self.disk.available:
            text = _("Format or create swap on entire device (unusual, advanced)")
            partitioned_disks.append(Text(""))
            partitioned_disks.append(
                menu_btn(label=text, on_press=self.format_entire))

        return partitioned_disks

    def _click_part(self, sender, part):
        self.controller.edit_partition(self.disk, part)

    def _click_disk(self, sender):
        self.controller.format_entire(self.disk)

    def show_disk_info_w(self):
        """ Runs hdparm against device and displays its output
        """
        text = _("Show disk information")
        return menu_btn(
                label=text,
                on_press=self.show_disk_info)

    def show_disk_info(self, result):
        self.controller.show_disk_information(self.disk)

    def add_partition(self, result):
        self.controller.add_disk_partition(self.disk)

    def format_entire(self, result):
        self.controller.format_entire(self.disk)

    def make_boot_disk(self, sender):
        self.controller.make_boot_disk(self.disk)

    def done(self, result):
        ''' Return to FilesystemView '''
        self.controller.manual()

    def cancel(self, button=None):
        self.controller.manual()
<commit_msg>pep8 fixes for subiquity/ui/views/filesystem/disk_partition.py<commit_after># Copyright 2015 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging
from urwid import Text

from subiquitycore.ui.buttons import done_btn, cancel_btn, menu_btn, other_btn
from subiquitycore.ui.container import Columns, ListBox, Pile
from subiquitycore.ui.utils import button_pile, Padding
from subiquitycore.view import BaseView

from subiquity.models.filesystem import humanize_size

log = logging.getLogger('subiquity.ui.filesystem.disk_partition')

class DiskPartitionView(BaseView):
    footer = (_("Partition the disk, or format the entire device "
              "without partitions"))

    def __init__(self, model, controller, disk):
        self.model = model
        self.controller = controller
        self.disk = disk
        self.title = _("Partition, format, and mount {}").format(disk.label)

        self.body = Pile([
            ('pack', Text("")),
            Padding.center_79(ListBox(
                self._build_model_inputs() + [
                    Text(""),
                    self.show_disk_info_w(),
                ])),
            ('pack', Text("")),
            ('pack', self._build_buttons()),
            ('pack', Text("")),
            ])
        super().__init__(self.body)

    def _build_buttons(self):
        cancel = cancel_btn(_("Cancel"), on_press=self.cancel)
        done = done_btn(_("Done"), on_press=self.done)
        return button_pile([done, cancel])

    def _build_model_inputs(self):
        partitioned_disks = []

        if self.disk.free > 0:
            if len(self.disk.partitions()) > 0:
                final_label = _("Add another partition")
            else:
                final_label = _("Add first partition")
            label_width = max(25, len(final_label) + 4)
        else:
            label_width = 25

        def format_volume(label, part):
            size = humanize_size(part.size)
            if part.fs() is None:
                fstype = '-'
                mountpoint = '-'
            elif part.fs().mount() is None:
                fstype = part.fs().fstype
                mountpoint = '-'
            else:
                fstype = part.fs().fstype
                mountpoint = part.fs().mount().path
            if part.type == 'disk':
                part_btn = menu_btn(label, on_press=self._click_disk)
            else:
                part_btn = menu_btn(label, on_press=self._click_part,
                                    user_arg=part)
            return Columns([
                (label_width, part_btn),
                (9, Text(size, align="right")),
                Text(fstype),
                Text(mountpoint),
            ], 2)
        if self.disk.fs() is not None:
            partitioned_disks.append(
                format_volume(_("entire disk"), self.disk))
        else:
            for part in self.disk.partitions():
                partitioned_disks.append(
                    format_volume(_("Partition {}").format(part._number),
                                  part))
        if self.disk.free > 0:
            free_space = humanize_size(self.disk.free)
            add_btn = menu_btn(final_label, on_press=self.add_partition)
            partitioned_disks.append(Columns([
                (label_width, add_btn),
                (9, Text(free_space, align="right")),
                Text(_("free space")),
            ], 2))
        if self.model.bootable():
            for p in self.disk.partitions():
                if p.flag in ('bios_grub', 'boot'):
                    break
            else:
                partitioned_disks.append(Text(""))
                partitioned_disks.append(
                    button_pile([other_btn(label=_("Select as boot disk"),
                                           on_press=self.make_boot_disk)]))
        if len(self.disk.partitions()) == 0 and self.disk.available:
            text = _("Format or create swap on entire device "
                     "(unusual, advanced)")
            partitioned_disks.append(Text(""))
            partitioned_disks.append(
                menu_btn(label=text, on_press=self.format_entire))

        return partitioned_disks

    def _click_part(self, sender, part):
        self.controller.edit_partition(self.disk, part)

    def _click_disk(self, sender):
        self.controller.format_entire(self.disk)

    def show_disk_info_w(self):
        """ Runs hdparm against device and displays its output
        """
        text = _("Show disk information")
        return menu_btn(
                label=text,
                on_press=self.show_disk_info)

    def show_disk_info(self, result):
        self.controller.show_disk_information(self.disk)

    def add_partition(self, result):
        self.controller.add_disk_partition(self.disk)

    def format_entire(self, result):
        self.controller.format_entire(self.disk)

    def make_boot_disk(self, sender):
        self.controller.make_boot_disk(self.disk)

    def done(self, result):
        ''' Return to FilesystemView '''
        self.controller.manual()

    def cancel(self, button=None):
        self.controller.manual()
<|endoftext|>


<commit_before>from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.PreprintList.as_view(), name=views.PreprintList.view_name),
    url(r'^(?P<node_id>\w+)/$', views.PreprintDetail.as_view(), name=views.PreprintDetail.view_name),
    url(r'^(?P<node_id>\w+)/contributors/$', views.PreprintContributorsList.as_view(), name=views.PreprintContributorsList.view_name),
    url(r'^(?P<node_id>\w+)/relationships/preprint_provider/$', views.PreprintToPreprintProviderRelationship.as_view(), name=views.PreprintToPreprintProviderRelationship.view_name),
]
<commit_msg>Add preprint_providers list to preprints urls<commit_after>from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.PreprintList.as_view(), name=views.PreprintList.view_name),
    url(r'^(?P<node_id>\w+)/$', views.PreprintDetail.as_view(), name=views.PreprintDetail.view_name),
    url(r'^(?P<node_id>\w+)/contributors/$', views.PreprintContributorsList.as_view(), name=views.PreprintContributorsList.view_name),
    url(r'^(?P<node_id>\w+)/relationships/preprint_providers/$', views.PreprintToPreprintProviderRelationship.as_view(), name=views.PreprintToPreprintProviderRelationship.view_name),
    url(r'^(?P<node_id>\w+)/preprint_providers/$', views.PreprintPreprintProvidersList.as_view(), name=views.PreprintPreprintProvidersList.view_name),
]
<|endoftext|>


<commit_before>import json
import asyncio
from urllib.parse import urlparse, parse_qsl

import aiohttp
import aiohttp.server
from aiohttp import MultiDict

import router
import rule
import service

class HttpRequestHandler(aiohttp.server.ServerHttpProtocol):

    async def handle_request(self, message, payload):
        api, params, context = await self.parser_request(message, payload)

        result, err = await rule.ruleManager.check_api_rule(api, params, context)
        if result == False and err is not None:
            await self.process_response(message, None, err)
        else:
            raw, err = await router.serviceRouter.async_call_api(api, params['args'], context, timeout = 5)
            await self.process_response(message, raw, err)

    async def parser_request(self, message, payload):
        url_info = urlparse(message.path)
        path = url_info.path

        args = {}
        if message.method == 'GET':
            pairs = parse_qsl(url_info.query)
        elif message.method == 'POST':
            raw = await payload.read()
            pairs = parse_qsl(raw)

        for (k,v) in pairs:
            args[k] = v

        params = dict(
            sign = args.get('sign', None),
            e = args.get('e', 0),
        )
        if args.get('args', None) is not None:
            params['args'] = json.loads(args['args'])
        else:
            params['args'] = dict()

        context = dict(
            User_Id = message.headers.get('HTTP_X_FIVEMILES_USER_ID', None),
            User_Token = message.headers.get('HTTP_X_FIVEMILES_USER_TOKEN', None)
        )
        return path, params, context

    async def process_response(self, message, raw, err):
        response = aiohttp.Response(
            self.writer, 200, http_version = message.version
        )
        if err is None:
            response.add_header('HTTP_X_FIVEMILES_CODE', '500')
        else:
            response.add_header('HTTP_X_FIVEMILES_CODE', '0')
        response.send_headers()
        if err is None:
            response.write(json.dumps(raw).encode('utf-8'))
        else:
            response.write(err.encode('utf-8'))
        await response.write_eof()<commit_msg>add gzip deflate support<commit_after>import json
import asyncio
from urllib.parse import urlparse, parse_qsl

import aiohttp
import aiohttp.server
from aiohttp import MultiDict

import router
import rule
import service

class HttpRequestHandler(aiohttp.server.ServerHttpProtocol):

    async def handle_request(self, message, payload):
        api, params, context = await self.parser_request(message, payload)

        result, err = await rule.ruleManager.check_api_rule(api, params, context)
        if result == False and err is not None:
            await self.process_response(message, None, err)
        else:
            raw, err = await router.serviceRouter.async_call_api(api, params['args'], context, timeout = 5)
            await self.process_response(message, raw, err)

    async def parser_request(self, message, payload):
        url_info = urlparse(message.path)
        path = url_info.path

        args = {}
        if message.method == 'GET':
            pairs = parse_qsl(url_info.query)
        elif message.method == 'POST':
            raw = await payload.read()
            pairs = parse_qsl(raw)

        for (k,v) in pairs:
            args[k] = v

        params = dict(
            sign = args.get('sign', None),
            e = args.get('e', 0),
        )
        if args.get('args', None) is not None:
            params['args'] = json.loads(args['args'])
        else:
            params['args'] = dict()

        context = dict(
            User_Id = message.headers.get('HTTP_X_FIVEMILES_USER_ID', None),
            User_Token = message.headers.get('HTTP_X_FIVEMILES_USER_TOKEN', None)
        )
        return path, params, context

    async def process_response(self, message, raw, err):
        response = aiohttp.Response(
            self.writer, 200, http_version = message.version
        )
        response.add_header('Transfer-Encoding', 'chunked')

        accept_encoding = message.headers.get('accept-encoding', '').lower()
        if 'gzip' in accept_encoding:
            response.add_header('Content-Encoding', 'gzip')
            response.add_compression_filter('gzip')
        elif 'deflate' in accept_encoding:
            response.add_header('Content-Encoding', 'deflate')
            response.add_compression_filter('deflate')

        response.add_chunking_filter(1024)

        output_raw = None
        if err is None:
            response.add_header('HTTP_X_FIVEMILES_CODE', '0')
            response.add_header('Content-type', 'text/json')
            output_raw = json.dumps(raw).encode('utf-8')
        else:
            response.add_header('HTTP_X_FIVEMILES_CODE', '500')
            response.add_header('Content-type', 'text/plain')
            output_raw = err.encode('utf-8')
        response.send_headers()

        response.write(output_raw)
        await response.write_eof()
        if response.keep_alive():
            self.keep_alive(True)<|endoftext|>


<commit_before>from collections import Iterable
import json

from django.db.models import ForeignKey
from django.utils.functional import cached_property
from rest_framework.serializers import ModelSerializer, ListSerializer

class PropertySerializerOptions(object):
    """Meta class options for PropertySerializer"""
    def __init__(self, meta):
        self.model = getattr(meta, 'model', None)
        self.read_only_props = getattr(meta, 'read_only_props', [])
        self.json_props = getattr(meta, 'json_props', [])
        self.exclude = ('id',)
        self.list_serializer_class = ListToDictSerializer

    @cached_property
    def parent_obj_field(self):
        # Automagically get the name of field containing relation to parent
        for field in self.model._meta.fields:
            if isinstance(field, ForeignKey):
                return field.name

        raise ValueError(
            '{0} misses relation to parent model'.format(self.model))

class ListToDictSerializer(ListSerializer):
    default_error_messages = []

    @property
    def data(self):
        return super(ListSerializer, self).data

    def to_representation(self, value):
        # Since this is ListSerializer, it makes everything a list. We need
        # dict (<name>: <value>}, so convert it to dictionary.
        result_list = super(
            ListToDictSerializer, self).to_representation(value)

        result_dict = {}
        for pair in result_list:
            result_dict.update(pair)

        return result_dict

    def to_internal_value(self, data):
        data_list = []
        for (prop_name, prop_value) in data.items():
            data_list.append({'name': prop_name, 'value': prop_value})
        return super(ListToDictSerializer, self).to_internal_value(data_list)

    def save(self):
        return [
            self.child.save(property_data) for data in self.validated_data
        ]

class PropertySerializer(ModelSerializer):
    """Allows to operate on properties of certain resource as dictionary

    Intended to be used as a base class for serializer for property resource
    exposed via separate endpoint.

    We consider Property as a model of the following structure:

    class Property(model):
        parent_model: ForeignKey(...)
        name = CharField()
        value = CharField()

    After converting collection of properties to representation we'll get the
    following dict:

    {
        prop1.name: prop1.value,
        prop2.name: prop2.value,
        ....
    }

    And an input dictionary of the above structure will be converted to
    the collection of Property instances
    """

    _options_class = PropertySerializerOptions

    def __init__(self, *args, **kwargs):
        super(PropertySerializer, self).__init__(*args, **kwargs)
        self.opts = self._options_class(self.Meta)
        self.Meta.list_serializer_class = ListToDictSerializer

    def to_representation(self, data):
        if isinstance(data, dict):
            return {data['name']: self._get_value(data)}
        else:
            return {data.name: self._get_value({'name': data.name, 'value': data.value})}

    def _get_value(self, data):
        if data['name'] in self.opts.json_props:
            return json.loads(data['value'])
        return data['value']

    def save(self, property_data=None):
        property_data = property_data or self.validated_data
        prop_name = property_data['name']

        if prop_name in self.opts.read_only_props:
            return prop

        # Try to find property by it's name.
        parent_obj_field = self.opts.parent_obj_field
        filters = {
            parent_obj_field: property_data[parent_obj_field],
            'name': prop_name
        }

        # If it already exists - update it's value. Otherwise - create a new
        # property.
        prop = self.Meta.model.objects.filter(**filters).first()
        if prop:
            prop.value=property_data['value']
            prop.save()
        else:
            prop = self.Meta.model.objects.create(**property_data)

        return prop

    def to_internal_value(self, data):
        # Data can be in two formats here depending on many=True/False:
        #   - {<prop_name>: <prop_value>}
        #   - {'name': <prop_name>, 'value': <prop_value>}
        # Convert to the format accepted by standard DRF serializers.
        if not('name' in data and 'value' in data):
            data = self._to_extended_dict(data)

        # Handle JSON fields (value encoded as JSON).
        if data['name'] in self.opts.json_props:
            data['value'] = json.dumps(data['value'])

        data = self._add_parent_obj_field(data)
        return super(PropertySerializer, self).to_internal_value(data)

    def _to_extended_dict(self, data):
        """ Convert dictionary of properties:
        {<prop_name>: <prop_value>} ->
            {'name': <prop_name>, 'value': <prop_value>}
        """
        prop_name, prop_value = data.items()[0]
        return {'name': prop_name, 'value': prop_value}

    def _add_parent_obj_field(self, data):
        # Property requires ID of parent object (properties owner, e.g. User
        # for user's properties case). Take it from context, which is passed
        # from view.
        parent_obj_field = self.opts.parent_obj_field
        parent_id_field = parent_obj_field + '_id'
        data[parent_obj_field] = self.context.get('parent_id',
            self.context['view'].kwargs[parent_id_field])
        return data

class PropertiesOwnerSerializer(ModelSerializer):
    def update(self, instance, validated_data):
        for attr, value in validated_data.items():
            if isinstance(self.fields[attr], ListToDictSerializer):
                self._save_properties(instance, attr)
                continue
            setattr(instance, attr, value)
        instance.save()

        return instance

    def _save_properties(self, instance, field_name):
        data_dict = self._data_list_to_dict(self.validated_data[field_name])
        properties_serializer = self.fields[field_name].child.__class__
        serializer = properties_serializer(data=data_dict, many=True)
        serializer.is_valid()
        serializer.save()

    def _data_list_to_dict(self, properties_list):
        properties_dict = {}
        for property in properties_list:
            properties_dict[property['name']] = property['value']

        return properties_dict
<commit_msg>Fixed issues found using Flake8.<commit_after>import json

from django.db.models import ForeignKey
from django.utils.functional import cached_property
from rest_framework.serializers import ModelSerializer, ListSerializer

class PropertySerializerOptions(object):
    """Meta class options for PropertySerializer"""
    def __init__(self, meta):
        self.model = getattr(meta, 'model', None)
        self.read_only_props = getattr(meta, 'read_only_props', [])
        self.json_props = getattr(meta, 'json_props', [])
        self.exclude = ('id',)
        self.list_serializer_class = ListToDictSerializer

    @cached_property
    def parent_obj_field(self):
        # Automagically get the name of field containing relation to parent
        for field in self.model._meta.fields:
            if isinstance(field, ForeignKey):
                return field.name

        raise ValueError(
            '{0} misses relation to parent model'.format(self.model))

class ListToDictSerializer(ListSerializer):
    default_error_messages = []

    @property
    def data(self):
        return super(ListSerializer, self).data

    def to_representation(self, value):
        # Since this is ListSerializer, it makes everything a list. We need
        # dict (<name>: <value>}, so convert it to dictionary.
        result_list = super(
            ListToDictSerializer, self).to_representation(value)

        result_dict = {}
        for pair in result_list:
            result_dict.update(pair)

        return result_dict

    def to_internal_value(self, data):
        data_list = []
        for (prop_name, prop_value) in data.items():
            data_list.append({'name': prop_name, 'value': prop_value})
        return super(ListToDictSerializer, self).to_internal_value(data_list)

    def save(self):
        return [
            self.child.save(property_data)
            for property_data in self.validated_data
        ]

class PropertySerializer(ModelSerializer):
    """Allows to operate on properties of certain resource as dictionary

    Intended to be used as a base class for serializer for property resource
    exposed via separate endpoint.

    We consider Property as a model of the following structure:

    class Property(model):
        parent_model: ForeignKey(...)
        name = CharField()
        value = CharField()

    After converting collection of properties to representation we'll get the
    following dict:

    {
        prop1.name: prop1.value,
        prop2.name: prop2.value,
        ....
    }

    And an input dictionary of the above structure will be converted to
    the collection of Property instances
    """

    _options_class = PropertySerializerOptions

    def __init__(self, *args, **kwargs):
        super(PropertySerializer, self).__init__(*args, **kwargs)
        self.opts = self._options_class(self.Meta)
        self.Meta.list_serializer_class = ListToDictSerializer

    def to_representation(self, data):
        if isinstance(data, dict):
            return {data['name']: self._get_value(data)}
        else:
            return {data.name: self._get_value(
                {'name': data.name, 'value': data.value})}

    def _get_value(self, data):
        if data['name'] in self.opts.json_props:
            return json.loads(data['value'])
        return data['value']

    def save(self, property_data=None):
        property_data = property_data or self.validated_data
        prop_name = property_data['name']

        # Try to find property by it's name.
        parent_obj_field = self.opts.parent_obj_field
        filters = {
            parent_obj_field: property_data[parent_obj_field],
            'name': prop_name
        }

        # If it already exists - update it's value. Otherwise - create a new
        # property.
        prop = self.Meta.model.objects.filter(**filters).first()

        if prop_name in self.opts.read_only_props:
            return prop

        if prop:
            prop.value = property_data['value']
            prop.save()
        else:
            prop = self.Meta.model.objects.create(**property_data)

        return prop

    def to_internal_value(self, data):
        # Data can be in two formats here depending on many=True/False:
        #   - {<prop_name>: <prop_value>}
        #   - {'name': <prop_name>, 'value': <prop_value>}
        # Convert to the format accepted by standard DRF serializers.
        if not('name' in data and 'value' in data):
            data = self._to_extended_dict(data)

        # Handle JSON fields (value encoded as JSON).
        if data['name'] in self.opts.json_props:
            data['value'] = json.dumps(data['value'])

        data = self._add_parent_obj_field(data)
        return super(PropertySerializer, self).to_internal_value(data)

    def _to_extended_dict(self, data):
        """ Convert dictionary of properties:
        {<prop_name>: <prop_value>} ->
            {'name': <prop_name>, 'value': <prop_value>}
        """
        prop_name, prop_value = data.items()[0]
        return {'name': prop_name, 'value': prop_value}

    def _add_parent_obj_field(self, data):
        # Property requires ID of parent object (properties owner, e.g. User
        # for user's properties case). Take it from context, which is passed
        # from view.
        parent_obj_field = self.opts.parent_obj_field
        parent_id_field = parent_obj_field + '_id'
        data[parent_obj_field] = self.context.get(
            'parent_id', self.context['view'].kwargs[parent_id_field])
        return data

class PropertiesOwnerSerializer(ModelSerializer):
    def update(self, instance, validated_data):
        for attr, value in validated_data.items():
            if isinstance(self.fields[attr], ListToDictSerializer):
                self._save_properties(instance, attr)
                continue
            setattr(instance, attr, value)
        instance.save()

        return instance

    def _save_properties(self, instance, field_name):
        data_dict = self._data_list_to_dict(self.validated_data[field_name])
        properties_serializer = self.fields[field_name].child.__class__
        serializer = properties_serializer(data=data_dict, many=True)
        serializer.is_valid()
        serializer.save()

    def _data_list_to_dict(self, properties_list):
        properties_dict = {}
        for property in properties_list:
            properties_dict[property['name']] = property['value']

        return properties_dict
<|endoftext|>


<commit_before>import os
from datetime import datetime
import tarfile
import tempfile
from optparse import make_option
import re

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError

from ... import utils
from ...storage.base import BaseStorage
from ...storage.base import StorageError

DATE_FORMAT = getattr(settings, 'DBBACKUP_DATE_FORMAT', '%Y-%m-%d-%H%M%S')
CLEANUP_KEEP = getattr(settings, 'DBBACKUP_CLEANUP_KEEP', 10)

class Command(BaseCommand):
    help = "backup_media [--encrypt]"
    option_list = BaseCommand.option_list + (
        make_option("-c", "--clean", help="Clean up old backup files", action="store_true", default=False),
        make_option("-s", "--servername", help="Specify server name to include in backup filename"),
        make_option("-e", "--encrypt", help="Encrypt the backup files", action="store_true", default=False),
    )

    @utils.email_uncaught_exception
    def handle(self, *args, **options):
        try:
            self.servername = options.get('servername')
            self.storage = BaseStorage.storage_factory()

            self.backup_mediafiles(options.get('encrypt'))

            if options.get('clean'):
                self.cleanup_old_backups()

        except StorageError, err:
            raise CommandError(err)

    def backup_mediafiles(self, encrypt):
        print "Backing up media files"
        output_file = self.create_backup_file(self.get_source_dir(), self.get_backup_basename())

        if encrypt:
            encrypted_file = utils.encrypt_file(output_file)
            output_file = encrypted_file

        print "  Backup tempfile created: %s (%s)" % (output_file.name, utils.handle_size(output_file))
        print "  Writing file to %s: %s" % (self.storage.name, self.storage.backup_dir())
        self.storage.write_file(output_file)

    def get_backup_basename(self):
        # todo: use DBBACKUP_FILENAME_TEMPLATE
        server_name = self.get_servername()
        if server_name:
            server_name = '-%s' % server_name

        return '%s%s-%s.media.tar.gz' % (
            self.get_databasename(),
            server_name,
            datetime.now().strftime(DATE_FORMAT)
        )

    def get_databasename(self):
        return settings.DATABASES['default']['NAME']

    def create_backup_file(self, source_dir, backup_basename):
        temp_dir = tempfile.mkdtemp()
        try:
            backup_filename = os.path.join(temp_dir, backup_basename)
            try:
                tar_file = tarfile.open(backup_filename, 'w|gz')
                try:
                    tar_file.add(source_dir)
                finally:
                    tar_file.close()

                return utils.create_spooled_temporary_file(backup_filename, backup_basename)
            finally:
                if os.path.exists(backup_filename):
                    os.remove(backup_filename)
        finally:
            os.rmdir(temp_dir)

    def get_source_dir(self):
        return getattr(settings, 'DBBACKUP_MEDIA_PATH') or settings.MEDIA_ROOT

    def cleanup_old_backups(self):
        """ Cleanup old backups, keeping the number of backups specified by
        DBBACKUP_CLEANUP_KEEP and any backups that occur on first of the month.
        """
        print "Cleaning Old Backups for media files"

        file_list = self.get_backup_file_list()

        for backup_date, filename in file_list[0:-CLEANUP_KEEP]:
            if int(backup_date.strftime("%d")) != 1:
                print "  Deleting: %s" % filename
                self.storage.delete_file(filename)

    def get_backup_file_list(self):
        """ Return a list of backup files including the backup date. The result is a list of tuples (datetime, filename).
            The list is sorted by date.
        """
        server_name = self.get_servername()
        if server_name:
            server_name = '-%s' % server_name

        media_re = re.compile(r'%s%s-(.*)\.media\.tar\.gz' % (self.get_databasename(), server_name))

        def is_media_backup(filename):
            return media_re.search(filename)

        def get_datetime_from_filename(filename):
            datestr = re.findall(media_re, filename)[0]
            return datetime.strptime(datestr, DATE_FORMAT)

        file_list = [
            (get_datetime_from_filename(f), f)
            for f in self.storage.list_directory()
            if is_media_backup(f)
        ]
        return sorted(file_list, key=lambda v: v[0])

    def get_servername(self):
        return self.servername or getattr(settings, 'DBBACKUP_SERVER_NAME', '')<commit_msg>Allows fallback to MEDIA_ROOT if DBBACKUP_MEDIA_PATH isn't defined.<commit_after>import os
from datetime import datetime
import tarfile
import tempfile
from optparse import make_option
import re

from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.management.base import CommandError

from ... import utils
from ...storage.base import BaseStorage
from ...storage.base import StorageError

DATE_FORMAT = getattr(settings, 'DBBACKUP_DATE_FORMAT', '%Y-%m-%d-%H%M%S')
CLEANUP_KEEP = getattr(settings, 'DBBACKUP_CLEANUP_KEEP', 10)

class Command(BaseCommand):
    help = "backup_media [--encrypt]"
    option_list = BaseCommand.option_list + (
        make_option("-c", "--clean", help="Clean up old backup files", action="store_true", default=False),
        make_option("-s", "--servername", help="Specify server name to include in backup filename"),
        make_option("-e", "--encrypt", help="Encrypt the backup files", action="store_true", default=False),
    )

    @utils.email_uncaught_exception
    def handle(self, *args, **options):
        try:
            self.servername = options.get('servername')
            self.storage = BaseStorage.storage_factory()

            self.backup_mediafiles(options.get('encrypt'))

            if options.get('clean'):
                self.cleanup_old_backups()

        except StorageError, err:
            raise CommandError(err)

    def backup_mediafiles(self, encrypt):
        print "Backing up media files"
        output_file = self.create_backup_file(self.get_source_dir(), self.get_backup_basename())

        if encrypt:
            encrypted_file = utils.encrypt_file(output_file)
            output_file = encrypted_file

        print "  Backup tempfile created: %s (%s)" % (output_file.name, utils.handle_size(output_file))
        print "  Writing file to %s: %s" % (self.storage.name, self.storage.backup_dir())
        self.storage.write_file(output_file)

    def get_backup_basename(self):
        # todo: use DBBACKUP_FILENAME_TEMPLATE
        server_name = self.get_servername()
        if server_name:
            server_name = '-%s' % server_name

        return '%s%s-%s.media.tar.gz' % (
            self.get_databasename(),
            server_name,
            datetime.now().strftime(DATE_FORMAT)
        )

    def get_databasename(self):
        return settings.DATABASES['default']['NAME']

    def create_backup_file(self, source_dir, backup_basename):
        temp_dir = tempfile.mkdtemp()
        try:
            backup_filename = os.path.join(temp_dir, backup_basename)
            try:
                tar_file = tarfile.open(backup_filename, 'w|gz')
                try:
                    tar_file.add(source_dir)
                finally:
                    tar_file.close()

                return utils.create_spooled_temporary_file(backup_filename, backup_basename)
            finally:
                if os.path.exists(backup_filename):
                    os.remove(backup_filename)
        finally:
            os.rmdir(temp_dir)

    def get_source_dir(self):
        return getattr(settings, 'DBBACKUP_MEDIA_PATH', None) or settings.MEDIA_ROOT

    def cleanup_old_backups(self):
        """ Cleanup old backups, keeping the number of backups specified by
        DBBACKUP_CLEANUP_KEEP and any backups that occur on first of the month.
        """
        print "Cleaning Old Backups for media files"

        file_list = self.get_backup_file_list()

        for backup_date, filename in file_list[0:-CLEANUP_KEEP]:
            if int(backup_date.strftime("%d")) != 1:
                print "  Deleting: %s" % filename
                self.storage.delete_file(filename)

    def get_backup_file_list(self):
        """ Return a list of backup files including the backup date. The result is a list of tuples (datetime, filename).
            The list is sorted by date.
        """
        server_name = self.get_servername()
        if server_name:
            server_name = '-%s' % server_name

        media_re = re.compile(r'%s%s-(.*)\.media\.tar\.gz' % (self.get_databasename(), server_name))

        def is_media_backup(filename):
            return media_re.search(filename)

        def get_datetime_from_filename(filename):
            datestr = re.findall(media_re, filename)[0]
            return datetime.strptime(datestr, DATE_FORMAT)

        file_list = [
            (get_datetime_from_filename(f), f)
            for f in self.storage.list_directory()
            if is_media_backup(f)
        ]
        return sorted(file_list, key=lambda v: v[0])

    def get_servername(self):
        return self.servername or getattr(settings, 'DBBACKUP_SERVER_NAME', '')<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import sys
import shutil
import argparse
import subprocess

skeleton_repo = "https://github.com/mciucu/stemjs-demo"

global_requirements = ['babel-cli', 'rollup']

def colorize(text):
    COLOR_CODE = '\033[95m'
    END_CODE = '\033[0m'
    return COLOR_CODE + text + END_CODE

def main():
    parser = argparse.ArgumentParser(description='Create simple stemjs app')
    parser.add_argument('project_dir', metavar='<project-directory>')

    backend = parser.add_mutually_exclusive_group()
    backend.add_argument('-e', '--express', help='Use Express as server backend', action='store_true')
    backend.add_argument('-d', '--django', help='Use Django as server backend', action='store_true')
    backend.add_argument('-s', '--simple', help='Use SimpleHTTPServer as server backend (default)', action='store_true')

    args = parser.parse_args()
    project_dir = args.project_dir

    if os.path.exists(project_dir):
        print("Directory {} already exists!".format(project_dir))
        return -1

    if args.express:
        skeleton_branch = "express"
        skeleton_type = "Express"
    elif args.django:
        skeleton_branch = "django"
        skeleton_type = "Django"
    else:
        skeleton_branch = "master"
        skeleton_type = "SimpleHTTPServer"

    print("Importing skeleton with {}\n".format(skeleton_type))

    subprocess.check_call(['git', 'clone', skeleton_repo, project_dir])
    subprocess.check_call(['git', 'checkout', skeleton_branch], cwd=project_dir)

    shutil.rmtree('{}/.git'.format(project_dir))

    print("Globally installing {}\n".format(", ".join(map(colorize, global_requirements))))
    subprocess.check_call(['sudo', 'npm', 'install', '-g', *global_requirements])

    print("Installing requirements with npm\n")
    subprocess.check_call(['npm', 'update'], cwd=project_dir)

    print("Compiling using rollup... ", end='', flush=True)
    subprocess.check_call(['rollup', '-c'], cwd="{}/src".format(project_dir), stdout=open(os.devnull, "w"))
    print("OK")

    print("\nSuccessfully created project {} at {}".format(colorize(project_dir), colorize(os.path.abspath(project_dir))))
    print("Check out the README located there")
    print("\nInside that directory, you can run\n")
    print("\tnpm start")
    print("\t\tto start a simple development server\n")

    print("\tcd src; rollup -c --watch")
    print("\t\tto start rollup\n")

    print("Also try {} --help for more options (such as express or django backend)".format(sys.argv[0]))

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>* Update script to reflect changes to building<commit_after>#!/usr/bin/env python3
import os
import sys
import shutil
import argparse
import subprocess

skeleton_repo = "https://github.com/mciucu/stemjs-demo"

global_requirements = ['babel-cli', 'rollup']

def colorize(text):
    COLOR_CODE = '\033[95m'
    END_CODE = '\033[0m'
    return COLOR_CODE + text + END_CODE

def main():
    parser = argparse.ArgumentParser(description='Create simple stemjs app')
    parser.add_argument('project_dir', metavar='<project-directory>')

    backend = parser.add_mutually_exclusive_group()
    backend.add_argument('-e', '--express', help='Use Express as server backend', action='store_true')
    backend.add_argument('-d', '--django', help='Use Django as server backend', action='store_true')
    backend.add_argument('-s', '--simple', help='Use SimpleHTTPServer as server backend (default)', action='store_true')

    args = parser.parse_args()
    project_dir = args.project_dir

    if os.path.exists(project_dir):
        print("Directory {} already exists!".format(project_dir))
        return -1

    if args.express:
        skeleton_branch = "express"
        skeleton_type = "Express"
    elif args.django:
        skeleton_branch = "django"
        skeleton_type = "Django"
    else:
        skeleton_branch = "master"
        skeleton_type = "SimpleHTTPServer"

    print("Importing skeleton with {}\n".format(skeleton_type))

    subprocess.check_call(['git', 'clone', skeleton_repo, project_dir])
    subprocess.check_call(['git', 'checkout', skeleton_branch], cwd=project_dir)

    shutil.rmtree('{}/.git'.format(project_dir))

    print("Globally installing {}\n".format(", ".join(map(colorize, global_requirements))))
    subprocess.check_call(['sudo', 'npm', 'install', '-g', *global_requirements])

    print("Installing requirements with npm\n")
    subprocess.check_call(['npm', 'update'], cwd=project_dir)

    print("Compiling\n", end='', flush=True)
    subprocess.check_call(['npm', 'run-script', 'build'], cwd=project_dir)

    print("\nSuccessfully created project {} at {}".format(colorize(project_dir), colorize(os.path.abspath(project_dir))))
    print("Check out the README located there")
    print("\nInside that directory, you can run\n")
    print("\tnpm start")
    print("\t\tto start a simple development server\n")

    print("\tnpm run-script watch")
    print("\t\tto watch for changes and recompile\n")

    print("Also try {} --help for more options (such as express or django backend)".format(sys.argv[0]))

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>