<commit_before>from buildbot.process.buildstep import BuildStep, ShellMixin
from buildbot.process.buildstep import LogLineObserver
from buildbot.process.results import FAILURE
from buildbot.process.results import SUCCESS
from buildbot.process.results import WARNINGS
from buildbot import config
from os import path
import re
from twisted.internet import defer

class VSLogLineObserver(LogLineObserver):

    stdoutDelimiter = "\r\n"
    stderrDelimiter = "\r\n"

    _re_delimiter = re.compile(r'^(\d+>)?-{5}.+-{5}$')
    _re_file = re.compile(r'^(\d+>)?[^ ]+\.(cpp|c)$')
    _re_warning = re.compile(r' ?: warning [A-Z]+[0-9]+:')
    _re_error = re.compile(r' ?error ([A-Z]+[0-9]+)?\s?: ')

    nbFiles = 0
    nbProjects = 0
    nbWarnings = 0
    nbErrors = 0

    logwarnings = None
    logerrors = None

    def __init__(self, **kwargs):
        LogLineObserver.__init__(self, **kwargs)

    @defer.inlineCallbacks
    def getOrCreateLog(self, logName):
        try:
            currentLog = self.step.getLog(logName)
        except KeyError:
            pass
        if currentLog is None:
            currentLog = yield self.step.addLog(logName)
        defer.returnValue(currentLog)

    @defer.inlineCallbacks
    def outLineReceived(self, line):
        if self._re_delimiter.search(line):
            self.nbProjects += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout("%s\n" % line)
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStdout("%s\n" % line)
            self.step.setProgress('projects', self.nbProjects)
        elif self._re_file.search(line):
            self.nbFiles += 1
            self.step.setProgress('files', self.nbFiles)
        elif self._re_warning.search(line):
            self.nbWarnings += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout("%s\n" % line)
            self.step.setProgress('warnings', self.nbWarnings)
        elif self._re_error.search("%s\n" % line):
            # error has no progress indication
            self.nbErrors += 1
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStderr("%s\n" % line)

class UnrealLogLineObserver(VSLogLineObserver):

    _re_file = re.compile(r'^\[\d+/\d+\].*\.(cpp|c)$')
    _re_ubt_error = re.compile(r' ?[Ee]rror\s*: ')
    _re_clang_warning = re.compile(r':\s*warning\s*:')
    _re_clang_error = re.compile(r':\s*error\s*: ')

    @defer.inlineCallbacks
    def parseLine(self, line):
        if (self._re_ubt_error.search(line) or
                self._re_clang_error.search(line)):
            self.nbErrors += 1
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStderr(u"{0}\n".format(line))
            self.step.updateSummary()
            defer.returnValue(True)
        elif self._re_clang_warning.search(line):
            self.nbWarnings += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout(u"{0}\n".format(line))
            self.step.setProgress('warnings', self.nbWarnings)
            self.step.updateSummary()
            defer.returnValue(True)
        defer.returnValue(False)

    @defer.inlineCallbacks
    def outLineReceived(self, line):
        if(self._re_file.search(line)):
            self.nbFiles += 1
        result = yield self.parseLine(line)
        if result is False:
            super(UnrealLogLineObserver, self).outLineReceived(line)

    def errLineReceived(self, line):
        if self.parseLine(line) is False:
            super(UnrealLogLineObserver, self).errLineReceived(line)

class BaseUnrealCommand(ShellMixin, BuildStep):

    """
    Base class for all unreal related commands, holds basic parameters:
    - engine_path: Path to the root folder of the used engine
    - project_path: Path to the uproject file of the project used in the build

    In addition some utility functions are provided
    to help ensure the build parameters are valid
    """

    renderables = [
        "project_path",
        "engine_path",
    ]

    supported_target_platforms = [
        "Win32",
        "Win64",
        "Mac",
        "XboxOne",
        "PS4",
        "IOS",
        "Android",
        "HTML5",
        "Linux",
        "TVOS"
    ]
    supported_target_config = [
        "Debug",
        "DebugGame",
        "Development",
        "Test",
        "Shipping"
    ]
    supported_build_platforms = ["Windows", "Linux", "Mac"]
    supported_engine_types = ["Source", "Installed", "Rocket"]

    name = "BaseUnrealCommand"

    def __init__(
            self,
            engine_path,
            project_path,
            do_sanity_checks=True,
            engine_type="Rocket",
            build_platform="Windows",
            **kwargs):
        self.engine_path = engine_path
        self.project_path = project_path
        self.build_platform = build_platform
        self.do_sanity_checks = do_sanity_checks
        self.engine_type = engine_type
        kwargs = self.setupShellMixin(kwargs)
        self.runSanityChecks()
        super(BaseUnrealCommand, self).__init__(**kwargs)

    def getPlatformScriptExtension(self, inside_platform_dir=False):
        if self.build_platform == "Windows":
            return "bat"
        # Mac scripts use shell for scripts other than
        # UAT and the Project level scripts
        elif self.build_platform == "Linux" or inside_platform_dir:
            return "sh"
        elif self.build_platform == "Mac":
            return "command"

    def runSanityChecks(self):
        if self.do_sanity_checks:
            self.doSanityChecks()

    def getEngineBatchFilesPath(self, script, inside_platform_dir=False):
        platform_dir = ""
        # Windows is the main platform and has no platform specific dir
        if inside_platform_dir and self.build_platform != "Windows":
            platform_dir = self.build_platform
        return path.join(
            self.engine_path,
            "Engine",
            "Build",
            "BatchFiles",
            platform_dir,
            "{0}.{1}".format(
                script, self.getPlatformScriptExtension(inside_platform_dir)))

    def getProjectFileName(self):
        projectName = self.project_path
        projectName = projectName.replace("\\", "/")
        splittedName = projectName.split("/")
        if len(splittedName) >= 2:
            projectName = splittedName[-1]
        return projectName

    def doSanityChecks(self):
        if (isinstance(self.build_platform, str) and
                self.build_platform not in self.supported_build_platforms):
            config.error("build_platform '{0}' is not supported".format(
                self.build_platform))
        if (isinstance(self.engine_type, str) and
                self.engine_type not in self.supported_engine_types):
            config.error(
                "engine_type '{0}' is not supported".format(self.engine_type))

    def setupLogfiles(self):
        self.logobserver = UnrealLogLineObserver()
        self.addLogObserver('stdio', self.logobserver)

    def getDescriptionDetails(self):
        details = []
        files = self.getStatistic('files', 0)
        if files > 0:
            details.append('{0} files'.format(files))
        warnings = self.getStatistic('warnings', 0)
        if warnings > 0:
            details.append('{0} warnings'.format(warnings))
        errors = self.getStatistic('errors', 0)
        if errors > 0:
            details.append('{0} errors'.format(errors))
        return details

    def evaluateCommand(self, cmd):
        if cmd.didFail():
            return FAILURE
        if self.logobserver.nbErrors > 0:
            return FAILURE
        if self.logobserver.nbWarnings > 0:
            return WARNINGS
        else:
            return SUCCESS
<commit_msg>initialized currentLog<commit_after>from buildbot.process.buildstep import BuildStep, ShellMixin
from buildbot.process.buildstep import LogLineObserver
from buildbot.process.results import FAILURE
from buildbot.process.results import SUCCESS
from buildbot.process.results import WARNINGS
from buildbot import config
from os import path
import re
from twisted.internet import defer

class VSLogLineObserver(LogLineObserver):

    stdoutDelimiter = "\r\n"
    stderrDelimiter = "\r\n"

    _re_delimiter = re.compile(r'^(\d+>)?-{5}.+-{5}$')
    _re_file = re.compile(r'^(\d+>)?[^ ]+\.(cpp|c)$')
    _re_warning = re.compile(r' ?: warning [A-Z]+[0-9]+:')
    _re_error = re.compile(r' ?error ([A-Z]+[0-9]+)?\s?: ')

    nbFiles = 0
    nbProjects = 0
    nbWarnings = 0
    nbErrors = 0

    logwarnings = None
    logerrors = None

    def __init__(self, **kwargs):
        LogLineObserver.__init__(self, **kwargs)

    @defer.inlineCallbacks
    def getOrCreateLog(self, logName):
        try:
            currentLog = self.step.getLog(logName)
        except KeyError:
            currentLog = None
        if currentLog is None:
            currentLog = yield self.step.addLog(logName)
        defer.returnValue(currentLog)

    @defer.inlineCallbacks
    def outLineReceived(self, line):
        if self._re_delimiter.search(line):
            self.nbProjects += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout("%s\n" % line)
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStdout("%s\n" % line)
            self.step.setProgress('projects', self.nbProjects)
        elif self._re_file.search(line):
            self.nbFiles += 1
            self.step.setProgress('files', self.nbFiles)
        elif self._re_warning.search(line):
            self.nbWarnings += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout("%s\n" % line)
            self.step.setProgress('warnings', self.nbWarnings)
        elif self._re_error.search("%s\n" % line):
            # error has no progress indication
            self.nbErrors += 1
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStderr("%s\n" % line)

class UnrealLogLineObserver(VSLogLineObserver):

    _re_file = re.compile(r'^\[\d+/\d+\].*\.(cpp|c)$')
    _re_ubt_error = re.compile(r' ?[Ee]rror\s*: ')
    _re_clang_warning = re.compile(r':\s*warning\s*:')
    _re_clang_error = re.compile(r':\s*error\s*: ')

    @defer.inlineCallbacks
    def parseLine(self, line):
        if (self._re_ubt_error.search(line) or
                self._re_clang_error.search(line)):
            self.nbErrors += 1
            self.logerrors = yield self.getOrCreateLog("errors")
            self.logerrors.addStderr(u"{0}\n".format(line))
            self.step.updateSummary()
            defer.returnValue(True)
        elif self._re_clang_warning.search(line):
            self.nbWarnings += 1
            self.logwarnings = yield self.getOrCreateLog("warnings")
            self.logwarnings.addStdout(u"{0}\n".format(line))
            self.step.setProgress('warnings', self.nbWarnings)
            self.step.updateSummary()
            defer.returnValue(True)
        defer.returnValue(False)

    @defer.inlineCallbacks
    def outLineReceived(self, line):
        if(self._re_file.search(line)):
            self.nbFiles += 1
        result = yield self.parseLine(line)
        if result is False:
            super(UnrealLogLineObserver, self).outLineReceived(line)

    def errLineReceived(self, line):
        if self.parseLine(line) is False:
            super(UnrealLogLineObserver, self).errLineReceived(line)

class BaseUnrealCommand(ShellMixin, BuildStep):

    """
    Base class for all unreal related commands, holds basic parameters:
    - engine_path: Path to the root folder of the used engine
    - project_path: Path to the uproject file of the project used in the build

    In addition some utility functions are provided
    to help ensure the build parameters are valid
    """

    renderables = [
        "project_path",
        "engine_path",
    ]

    supported_target_platforms = [
        "Win32",
        "Win64",
        "Mac",
        "XboxOne",
        "PS4",
        "IOS",
        "Android",
        "HTML5",
        "Linux",
        "TVOS"
    ]
    supported_target_config = [
        "Debug",
        "DebugGame",
        "Development",
        "Test",
        "Shipping"
    ]
    supported_build_platforms = ["Windows", "Linux", "Mac"]
    supported_engine_types = ["Source", "Installed", "Rocket"]

    name = "BaseUnrealCommand"

    def __init__(
            self,
            engine_path,
            project_path,
            do_sanity_checks=True,
            engine_type="Rocket",
            build_platform="Windows",
            **kwargs):
        self.engine_path = engine_path
        self.project_path = project_path
        self.build_platform = build_platform
        self.do_sanity_checks = do_sanity_checks
        self.engine_type = engine_type
        kwargs = self.setupShellMixin(kwargs)
        self.runSanityChecks()
        super(BaseUnrealCommand, self).__init__(**kwargs)

    def getPlatformScriptExtension(self, inside_platform_dir=False):
        if self.build_platform == "Windows":
            return "bat"
        # Mac scripts use shell for scripts other than
        # UAT and the Project level scripts
        elif self.build_platform == "Linux" or inside_platform_dir:
            return "sh"
        elif self.build_platform == "Mac":
            return "command"

    def runSanityChecks(self):
        if self.do_sanity_checks:
            self.doSanityChecks()

    def getEngineBatchFilesPath(self, script, inside_platform_dir=False):
        platform_dir = ""
        # Windows is the main platform and has no platform specific dir
        if inside_platform_dir and self.build_platform != "Windows":
            platform_dir = self.build_platform
        return path.join(
            self.engine_path,
            "Engine",
            "Build",
            "BatchFiles",
            platform_dir,
            "{0}.{1}".format(
                script, self.getPlatformScriptExtension(inside_platform_dir)))

    def getProjectFileName(self):
        projectName = self.project_path
        projectName = projectName.replace("\\", "/")
        splittedName = projectName.split("/")
        if len(splittedName) >= 2:
            projectName = splittedName[-1]
        return projectName

    def doSanityChecks(self):
        if (isinstance(self.build_platform, str) and
                self.build_platform not in self.supported_build_platforms):
            config.error("build_platform '{0}' is not supported".format(
                self.build_platform))
        if (isinstance(self.engine_type, str) and
                self.engine_type not in self.supported_engine_types):
            config.error(
                "engine_type '{0}' is not supported".format(self.engine_type))

    def setupLogfiles(self):
        self.logobserver = UnrealLogLineObserver()
        self.addLogObserver('stdio', self.logobserver)

    def getDescriptionDetails(self):
        details = []
        files = self.getStatistic('files', 0)
        if files > 0:
            details.append('{0} files'.format(files))
        warnings = self.getStatistic('warnings', 0)
        if warnings > 0:
            details.append('{0} warnings'.format(warnings))
        errors = self.getStatistic('errors', 0)
        if errors > 0:
            details.append('{0} errors'.format(errors))
        return details

    def evaluateCommand(self, cmd):
        if cmd.didFail():
            return FAILURE
        if self.logobserver.nbErrors > 0:
            return FAILURE
        if self.logobserver.nbWarnings > 0:
            return WARNINGS
        else:
            return SUCCESS
<|endoftext|>


<commit_before><commit_msg>Removed empty story file glenn.py<commit_after><|endoftext|>


<commit_before># Take the scraped PPC data and try to import it into PopIt.
#
# It looks for someone with the same name by searching PopIt, and, if
# matches are found, asks the user whether they're the same person.
# This decision is recorded in a JSON file so you should only need to
# make that choice once.
#
# TODO: add a --noinput option, which won't ask for input, but instead
# output text and return an error; this will be useful for running
# this job from cron - if you get an email from cron, you'll need to
# run the script manually to make those decisions.

import os
from os.path import join
import re
import json

from django.core.management.base import BaseCommand
from django.conf import settings

import requests

from candidates.update import PersonParseMixin, PersonUpdateMixin
from candidates.static_data import MapItData
from candidates.views import CandidacyMixin

party_slug_to_popit_party = {
    'labour': {
        'name': 'Labour Party',
        'id': 'party:53',
    },
    'conservatives': {
        'name': 'Conservative Party',
        'id': 'party:52',
    },
    'libdem': {
        'name': 'Liberal Democrats',
        'id': 'party:90',
    }
}

constituency_corrections = {
    'Berwick': 'Berwick-upon-Tweed',
    'Chester': 'City of Chester',
    'Dover and Deal': 'Dover',
    'Hull North': 'Kingston upon Hull North',
    'Hull West and Hessle': 'Kingston upon Hull West and Hessle',
    'Isle of Wright': 'Isle of Wight',
    'Richmond': 'Richmond (Yorks)',
    'Stratford-upon-Avon': 'Stratford-on-Avon',
    'West Cornwall and Scilly': 'St Ives',
}

def cons_key(s):
    return re.sub(r',', '', s).lower()

constituency_lookup = {
    cons_key(k): v for k, v in
    MapItData.constituencies_2010_name_map.items()
}

def get_constituency_from_name(constituency_name):
    name = re.sub(r'\xA0', ' ', constituency_name)
    name = re.sub(r' & ', ' and ', name)
    name = constituency_corrections.get(name, name)
    name = re.sub(r'(?i)candidate for ', '', name)
    mapit_data = constituency_lookup[cons_key(name)]
    post_id = str(mapit_data['id'])
    return {
        'post_id': post_id,
        'name': mapit_data['name'],
        'mapit_url': 'http://mapit.mysociety.org/area/{0}'.format(post_id),
    }

ppc_data_directory = join(
    settings.BASE_DIR,
    'data',
    'UK-Political-Parties',
    'ppcs',
    'json'
)

human_decisions_filename = join(
    settings.BASE_DIR,
    'data',
    'human-ppc-decisions.json'
)

def get_human_decisions():
    with open(human_decisions_filename) as f:
        return json.load(f)

def write_human_decisions(human_decisions):
    output = json.dumps(human_decisions, indent=4, sort_keys=True)
    output = re.sub(r'(?ms)\s*$', '', output)
    with open(human_decisions_filename, 'w') as f:
        f.write(output)

def record_human_decision(ppc_dict, popit_person, same):
    human_decisions = get_human_decisions()
    human_decisions.append({
        'same_person': same,
        'ppc': ppc_dict,
        'popit_person': popit_person,
    })
    write_human_decisions(human_decisions)

def get_human_decision(ppc_data):
    fields_to_match = ['party_slug', 'constituency', 'name']
    matching_decisions = []
    for decision in get_human_decisions():
        if all(decision['ppc'][k] == ppc_data[k] for k in fields_to_match):
            matching_decisions.append(decision)
    return matching_decisions

def merge_person_data(old_data, new_data):
    data = old_data.copy()
    standing_in = data.get('standing_in', {})
    party_memberships = data.get('party_memberships', {})
    standing_in.update(new_data.get('standing_in', {}))
    party_memberships.update(new_data.get('party_memberships', {}))
    data.update(new_data)
    data['standing_in'] = standing_in
    data['party_memberships'] = party_memberships
    return data

def get_ppc_url_from_popit_result(popit_result):
    links = popit_result.get('links', [])
    for link in links:
        if link.get('note') == 'party PPC page':
            return link.get['url']
    return None

class Command(CandidacyMixin, PersonParseMixin, PersonUpdateMixin, BaseCommand):
    help = "Import scraped PPC data"

    # Currently unused:
    #
    #     "region",
    #     "full_url",
    #     "path",
    #     "phone",
    #     "slug",
    #     "address",
    #     "current_mp",
    #     "blog_url",
    #     "campaign_url",
    #     "biography_url",

    def get_person_data_from_ppc(self, ppc_data):
        return {
            'name': ppc_data.get('name', ''),
            'email': ppc_data.get('email', ''),
            'birth_date': None,
            'twitter_username': ppc_data.get('twitter_username', ''),
            'facebook_personal_url': ppc_data.get('facebook_url', ''),
            'homepage_url': ppc_data.get('homepage_url', ''),
            'party_ppc_page_url': ppc_data.get('full_url', ''),
            'standing_in': {
                '2015': ppc_data['constituency_object'],
            },
            'party_memberships': {
                '2015': ppc_data['party_object'],
            }
        }

    def update_popit_person(self, popit_person_id, ppc_data):
        # Get the existing data first:
        person_data = self.get_person(popit_person_id)
        previous_versions = person_data.pop('versions')
        new_person_data = self.get_person_data_from_ppc(ppc_data)
        # Remove any empty keys, we don't want to overwrite exiting
        # data with nothing:
        keys = new_person_data.keys()
        for key in keys:
            if not new_person_data[key]:
                del new_person_data[key]
        merged_person_data = merge_person_data(person_data, new_person_data)
        change_metadata = self.get_change_metadata(
            None,
            'Updated candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        return self.update_person(
            merged_person_data,
            change_metadata,
            previous_versions,
        )

    def add_popit_person(self, ppc_data):
        change_metadata = self.get_change_metadata(
            None,
            'Created new candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        person_data = self.get_person_data_from_ppc(ppc_data)
        return self.create_person(person_data, change_metadata)

    def already_added_for_2015(self, ppc_data, popit_result):
        popit_constituency_name_2015 = popit_result['standing_in'].get('2015', {}).get('name')
        popit_party_id_2015 = popit_result['party_memberships'].get('2015', {}).get('id')
        return (ppc_data['name'] == popit_result['name'] and
                ppc_data['party_object']['id'] == popit_party_id_2015 and
                ppc_data['constituency_object']['name'] == popit_constituency_name_2015)

    def match_based_on_2010(self, decision_popit, popit_result):
        # We assume that the name is already a match (or a close
        # match) because this came from a search result...
        dsi2010 = decision_popit['standing_in']
        psi2010 = popit_result['standing_in']
        dpm2010 = decision_popit['party_memberships']
        ppm2010 = popit_result['party_memberships']
        for d in (dsi2010, psi2010, dpm2010, ppm2010):
            if '2010' not in d:
                return False
        return (
            dsi2010['2010']['name'] == psi2010['2010']['name'] and
            dpm2010['2010']['name'] == ppm2010['2010']['name']
        )

    def already_matched_to_2010_person(self, ppc_data, popit_result):
        decisions = get_human_decision(ppc_data)
        for decision in decisions:
            popit_person = decision['popit_person']
            if self.match_based_on_2010(popit_person, popit_result):
                return decision['same_person']
        return None

    def decision_from_user(self, ppc_data, popit_result):
        # Ask whether the PPC is the same as this person from the
        # PopIt search results:
        print "--------------------------------------------------------"
        print "Is this PPC ..."
        print "        Name:", ppc_data['name'].encode('utf-8')
        if 'email' in ppc_data:
            print "        Email:", ppc_data['email']
        print "        Party:", ppc_data['party_slug']
        print "        Constituency:", ppc_data['constituency']
        if 'full_url' in ppc_data:
            print "        PPC URL:", ppc_data['full_url']
        print "... the same as this person from PopIt:"
        print "        PopIt URL:", popit_result['html_url']
        print "        Name:", popit_result['name'].encode('utf-8')
        if 'email' in popit_result:
            print "        Email:", popit_result['email']
        for year in popit_result['party_memberships']:
            print "        Party in {0}: {1}".format(
                year, popit_result['party_memberships'][year]['name']
            )
        for year in popit_result['standing_in']:
            print "        Constituency in {0}: {1}".format(
                year, popit_result['standing_in'][year]['name']
            )
        response = ''
        while response.lower() not in ('y', 'n'):
            response = raw_input('Are they the same? (y/Y/n/N) ')
            response = response.lower().strip()
            if response == 'y':
                return True
            elif response == 'n':
                return False

    def handle_person(self, ppc_data):
        print u"PPC ({party_slug}): {name}".format(**ppc_data).encode('utf-8')
        # Search PopIt for anyone with the same name. (FIXME: we
        # should make this a bit fuzzier when the PopIt API
        # supports that.)
        person_search_url = self.get_search_url(
            'persons', '"' + ppc_data['name'] + '"'
        )
        r = requests.get(
            person_search_url
        )
        for result in r.json()['result']:
            # FIXME: With the current code, updating Elasticsearch
            # fails because it rejects the empty string as a null date
            # value. To just get this working, GET the person directly
            # so the information's defintely up to date.
            result = self.api.persons(result['id']).get()['result']
            # We don't need past versions of the person or their
            # memberships, and it's unnecessary verbose to record them
            # with decisions.
            del result['versions']
            del result['memberships']
            if self.already_added_for_2015(ppc_data, result):
                # Then just update that person with the possibly
                # updated scraped data:
                self.update_popit_person(result['id'], ppc_data)
                return
            # Do we already have a decision about whether this PPC is
            # the same as another from 2010?
            decision = self.already_matched_to_2010_person(ppc_data, result)
            if decision is not None:
                if decision:
                    self.update_popit_person(result['id'], ppc_data)
                    return
            else:
                # We have to ask the user for a decision:
                if self.decision_from_user(ppc_data, result):
                    self.update_popit_person(result['id'], ppc_data)
                    record_human_decision(ppc_data, result, True)
                    return
                else:
                    record_human_decision(ppc_data, result, False)
        # If we haven't returned from the function by this stage, we
        # need to create a new candidate in PopIt:
        self.add_popit_person(ppc_data)

    def handle(self, **options):

        for party_slug in sorted(os.listdir(ppc_data_directory)):
            json_directory = join(
                ppc_data_directory,
                party_slug
            )
            for leafname in sorted(os.listdir(json_directory)):
                filename = join(json_directory, leafname)
                with open(filename) as f:
                    ppc_data = json.load(f)
                    ppc_data['party_slug'] = party_slug
                    ppc_data['party_object'] = party_slug_to_popit_party[party_slug]
                    ppc_data['constituency_object'] = get_constituency_from_name(
                        ppc_data['constituency']
                    )
                    self.handle_person(ppc_data)
<commit_msg>Make sure the PPC importer doesn't overwrite existing data<commit_after># Take the scraped PPC data and try to import it into PopIt.
#
# It looks for someone with the same name by searching PopIt, and, if
# matches are found, asks the user whether they're the same person.
# This decision is recorded in a JSON file so you should only need to
# make that choice once.
#
# TODO: add a --noinput option, which won't ask for input, but instead
# output text and return an error; this will be useful for running
# this job from cron - if you get an email from cron, you'll need to
# run the script manually to make those decisions.

import os
from os.path import join
import re
import json

from django.core.management.base import BaseCommand
from django.conf import settings

import requests

from candidates.update import PersonParseMixin, PersonUpdateMixin
from candidates.static_data import MapItData
from candidates.views import CandidacyMixin

party_slug_to_popit_party = {
    'labour': {
        'name': 'Labour Party',
        'id': 'party:53',
    },
    'conservatives': {
        'name': 'Conservative Party',
        'id': 'party:52',
    },
    'libdem': {
        'name': 'Liberal Democrats',
        'id': 'party:90',
    }
}

constituency_corrections = {
    'Berwick': 'Berwick-upon-Tweed',
    'Chester': 'City of Chester',
    'Dover and Deal': 'Dover',
    'Hull North': 'Kingston upon Hull North',
    'Hull West and Hessle': 'Kingston upon Hull West and Hessle',
    'Isle of Wright': 'Isle of Wight',
    'Richmond': 'Richmond (Yorks)',
    'Stratford-upon-Avon': 'Stratford-on-Avon',
    'West Cornwall and Scilly': 'St Ives',
}

def cons_key(s):
    return re.sub(r',', '', s).lower()

constituency_lookup = {
    cons_key(k): v for k, v in
    MapItData.constituencies_2010_name_map.items()
}

def get_constituency_from_name(constituency_name):
    name = re.sub(r'\xA0', ' ', constituency_name)
    name = re.sub(r' & ', ' and ', name)
    name = constituency_corrections.get(name, name)
    name = re.sub(r'(?i)candidate for ', '', name)
    mapit_data = constituency_lookup[cons_key(name)]
    post_id = str(mapit_data['id'])
    return {
        'post_id': post_id,
        'name': mapit_data['name'],
        'mapit_url': 'http://mapit.mysociety.org/area/{0}'.format(post_id),
    }

ppc_data_directory = join(
    settings.BASE_DIR,
    'data',
    'UK-Political-Parties',
    'ppcs',
    'json'
)

human_decisions_filename = join(
    settings.BASE_DIR,
    'data',
    'human-ppc-decisions.json'
)

def get_human_decisions():
    with open(human_decisions_filename) as f:
        return json.load(f)

def write_human_decisions(human_decisions):
    output = json.dumps(human_decisions, indent=4, sort_keys=True)
    output = re.sub(r'(?ms)\s*$', '', output)
    with open(human_decisions_filename, 'w') as f:
        f.write(output)

def record_human_decision(ppc_dict, popit_person, same):
    human_decisions = get_human_decisions()
    human_decisions.append({
        'same_person': same,
        'ppc': ppc_dict,
        'popit_person': popit_person,
    })
    write_human_decisions(human_decisions)

def get_human_decision(ppc_data):
    fields_to_match = ['party_slug', 'constituency', 'name']
    matching_decisions = []
    for decision in get_human_decisions():
        if all(decision['ppc'][k] == ppc_data[k] for k in fields_to_match):
            matching_decisions.append(decision)
    return matching_decisions

def merge_person_data(old_data, new_data):
    data = old_data.copy()
    standing_in = data.get('standing_in', {})
    party_memberships = data.get('party_memberships', {})
    standing_in.update(new_data.get('standing_in', {}))
    party_memberships.update(new_data.get('party_memberships', {}))
    data.update(new_data)
    data['standing_in'] = standing_in
    data['party_memberships'] = party_memberships
    return data

def get_ppc_url_from_popit_result(popit_result):
    links = popit_result.get('links', [])
    for link in links:
        if link.get('note') == 'party PPC page':
            return link.get['url']
    return None

class Command(CandidacyMixin, PersonParseMixin, PersonUpdateMixin, BaseCommand):
    help = "Import scraped PPC data"

    # Currently unused:
    #
    #     "region",
    #     "full_url",
    #     "path",
    #     "phone",
    #     "slug",
    #     "address",
    #     "current_mp",
    #     "blog_url",
    #     "campaign_url",
    #     "biography_url",

    def get_person_data_from_ppc(self, ppc_data):
        return {
            'name': ppc_data.get('name', ''),
            'email': ppc_data.get('email', ''),
            'birth_date': None,
            'twitter_username': ppc_data.get('twitter_username', ''),
            'facebook_personal_url': ppc_data.get('facebook_url', ''),
            'homepage_url': ppc_data.get('homepage_url', ''),
            'party_ppc_page_url': ppc_data.get('full_url', ''),
            'standing_in': {
                '2015': ppc_data['constituency_object'],
            },
            'party_memberships': {
                '2015': ppc_data['party_object'],
            }
        }

    def update_popit_person(self, popit_person_id, ppc_data):
        # Get the existing data first:
        person_data = self.get_person(popit_person_id)
        previous_versions = person_data.pop('versions')
        new_person_data = self.get_person_data_from_ppc(ppc_data)
        # Remove any empty keys, we don't want to overwrite exiting
        # data with nothing:
        keys = new_person_data.keys()
        warnings = []
        for key in keys:
            if not new_person_data[key]:
                del new_person_data[key]
            # Also make sure that we don't overwrite any existing
            # fields that are filled in with different values:
            if key not in ('standing_in', 'party_memberships'):
                new_person_data_value = new_person_data.get(key)
                person_data_value = person_data.get(key)
                if person_data_value and new_person_data_value and new_person_data_value != person_data_value:
                    warnings.append(u"not overwriting {0}".format(person_data_value))
                    warnings.append(u"with the new value {0}".format(new_person_data_value))
                    del new_person_data[key]
        if warnings:
            print "Warnings for person/{0} {1}".format(
                popit_person_id, person_data['name']
            )
            for warning in warnings:
                print "  ...", warning
        merged_person_data = merge_person_data(person_data, new_person_data)
        change_metadata = self.get_change_metadata(
            None,
            'Updated candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        return self.update_person(
            merged_person_data,
            change_metadata,
            previous_versions,
        )

    def add_popit_person(self, ppc_data):
        change_metadata = self.get_change_metadata(
            None,
            'Created new candidate from official PPC data ({0})'.format(ppc_data['party_slug']),
        )
        person_data = self.get_person_data_from_ppc(ppc_data)
        return self.create_person(person_data, change_metadata)

    def already_added_for_2015(self, ppc_data, popit_result):
        popit_constituency_name_2015 = popit_result['standing_in'].get('2015', {}).get('name')
        popit_party_id_2015 = popit_result['party_memberships'].get('2015', {}).get('id')
        return (ppc_data['name'] == popit_result['name'] and
                ppc_data['party_object']['id'] == popit_party_id_2015 and
                ppc_data['constituency_object']['name'] == popit_constituency_name_2015)

    def match_based_on_2010(self, decision_popit, popit_result):
        # We assume that the name is already a match (or a close
        # match) because this came from a search result...
        dsi2010 = decision_popit['standing_in']
        psi2010 = popit_result['standing_in']
        dpm2010 = decision_popit['party_memberships']
        ppm2010 = popit_result['party_memberships']
        for d in (dsi2010, psi2010, dpm2010, ppm2010):
            if '2010' not in d:
                return False
        return (
            dsi2010['2010']['name'] == psi2010['2010']['name'] and
            dpm2010['2010']['name'] == ppm2010['2010']['name']
        )

    def already_matched_to_2010_person(self, ppc_data, popit_result):
        decisions = get_human_decision(ppc_data)
        for decision in decisions:
            popit_person = decision['popit_person']
            if self.match_based_on_2010(popit_person, popit_result):
                return decision['same_person']
        return None

    def decision_from_user(self, ppc_data, popit_result):
        # Ask whether the PPC is the same as this person from the
        # PopIt search results:
        print "--------------------------------------------------------"
        print "Is this PPC ..."
        print "        Name:", ppc_data['name'].encode('utf-8')
        if 'email' in ppc_data:
            print "        Email:", ppc_data['email']
        print "        Party:", ppc_data['party_slug']
        print "        Constituency:", ppc_data['constituency']
        if 'full_url' in ppc_data:
            print "        PPC URL:", ppc_data['full_url']
        print "... the same as this person from PopIt:"
        print "        PopIt URL:", popit_result['html_url']
        print "        Name:", popit_result['name'].encode('utf-8')
        if 'email' in popit_result:
            print "        Email:", popit_result['email']
        for year in popit_result['party_memberships']:
            print "        Party in {0}: {1}".format(
                year, popit_result['party_memberships'][year]['name']
            )
        for year in popit_result['standing_in']:
            print "        Constituency in {0}: {1}".format(
                year, popit_result['standing_in'][year]['name']
            )
        response = ''
        while response.lower() not in ('y', 'n'):
            response = raw_input('Are they the same? (y/Y/n/N) ')
            response = response.lower().strip()
            if response == 'y':
                return True
            elif response == 'n':
                return False

    def handle_person(self, ppc_data):
        print u"PPC ({party_slug}): {name}".format(**ppc_data).encode('utf-8')
        # Search PopIt for anyone with the same name. (FIXME: we
        # should make this a bit fuzzier when the PopIt API
        # supports that.)
        person_search_url = self.get_search_url(
            'persons', '"' + ppc_data['name'] + '"'
        )
        r = requests.get(
            person_search_url
        )
        for result in r.json()['result']:
            # FIXME: With the current code, updating Elasticsearch
            # fails because it rejects the empty string as a null date
            # value. To just get this working, GET the person directly
            # so the information's defintely up to date.
            result = self.api.persons(result['id']).get()['result']
            # We don't need past versions of the person or their
            # memberships, and it's unnecessary verbose to record them
            # with decisions.
            del result['versions']
            del result['memberships']
            if self.already_added_for_2015(ppc_data, result):
                # Then just update that person with the possibly
                # updated scraped data:
                self.update_popit_person(result['id'], ppc_data)
                return
            # Do we already have a decision about whether this PPC is
            # the same as another from 2010?
            decision = self.already_matched_to_2010_person(ppc_data, result)
            if decision is not None:
                if decision:
                    self.update_popit_person(result['id'], ppc_data)
                    return
            else:
                # We have to ask the user for a decision:
                if self.decision_from_user(ppc_data, result):
                    self.update_popit_person(result['id'], ppc_data)
                    record_human_decision(ppc_data, result, True)
                    return
                else:
                    record_human_decision(ppc_data, result, False)
        # If we haven't returned from the function by this stage, we
        # need to create a new candidate in PopIt:
        self.add_popit_person(ppc_data)

    def handle(self, **options):

        for party_slug in sorted(os.listdir(ppc_data_directory)):
            json_directory = join(
                ppc_data_directory,
                party_slug
            )
            for leafname in sorted(os.listdir(json_directory)):
                filename = join(json_directory, leafname)
                with open(filename) as f:
                    ppc_data = json.load(f)
                    ppc_data['party_slug'] = party_slug
                    ppc_data['party_object'] = party_slug_to_popit_party[party_slug]
                    ppc_data['constituency_object'] = get_constituency_from_name(
                        ppc_data['constituency']
                    )
                    self.handle_person(ppc_data)
<|endoftext|>


<commit_before>import os

from catalogService import errors
from catalogService.rest import baseDriver
from catalogService.rest.models import images
from catalogService.rest.models import instances

try:
    import novaclient
    from glance.client import Client
except ImportError:
    pass

class OpenStack_Image(images.BaseImage):
    "OpenStack Image"

# This is provided by the nova api
#class OpenStack_InstanceTypes(instances.InstanceTypes):
#    "OpenStack Instance Types"
#
#    idMap = [
#        ('xenent.small', "Small"),
#        ('xenent.medium', "Medium"),
#    ]

# Nova address
# Nova port
# Glance address (until apis are integrated)
# Glance port
_configurationDescriptorXmlData = """<?xml version='1.0' encoding='UTF-8'?>
<descriptor xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.rpath.org/permanent/descriptor-1.0.xsd descriptor-1.0.xsd">
  <metadata>
    <displayName>OpenStack Configuration</displayName>
    <descriptions>
      <desc>Configure OpenStack</desc>
    </descriptions>
  </metadata>
  <dataFields>
    <field>
      <name>nova_server</name>
      <descriptions>
        <desc>Nova Server</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/novaServerName.html'/>
    </field>
    <field>
      <name>nova_port</name>
      <descriptions>
        <desc>Nova Port</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/novaPortNumber.html'/>
    </field>
    <field>
      <name>glance_server</name>
      <descriptions>
        <desc>Glance Server</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/glanceServerName.html'/>
    </field>
    <field>
      <name>glance_port</name>
      <descriptions>
        <desc>Glance Port</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/glancePortNumber.html'/>
    </field>
    <field>
      <name>alias</name>
      <descriptions>
        <desc>Name</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/alias.html'/>
    </field>
    <field>
      <name>description</name>
      <descriptions>
        <desc>Full Description</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/description.html'/>
    </field>
  </dataFields>
</descriptor>"""

# User Name
# Auth Token
_credentialsDescriptorXmlData = """<?xml version='1.0' encoding='UTF-8'?>
<descriptor xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.rpath.org/permanent/descriptor-1.0.xsd descriptor-1.0.xsd">
  <metadata>
    <displayName>OpenStack User Credentials</displayName>
    <descriptions>
      <desc>User Credentials for OpenStack</desc>
    </descriptions>
  </metadata>
  <dataFields>
    <field>
      <name>username</name>
      <descriptions>
        <desc>User Name</desc>
      </descriptions>
      <type>str</type>
      <constraints>
        <descriptions>
          <desc>Field must contain between 1 and 32 characters</desc>
        </descriptions>
        <length>255</length>
      </constraints>
      <required>true</required>
    </field>
    <field>
      <name>auth_token</name>
      <descriptions>
        <desc>Auth Token</desc>
      </descriptions>
      <type>str</type>
      <constraints>
        <descriptions>
          <desc>Field must contain between 1 and 40 characters</desc>
        </descriptions>
        <length>40</length>
      </constraints>
      <required>true</required>
      <password>true</password>
    </field>    @classmethod
    def getCloudNameFromDescriptorData(cls, descriptorData):
        return descriptorData.getField('name')

  </dataFields>
</descriptor>
"""

# http://glance.openstack.org/client.html
# http://pypi.python.org/pypi/python-novaclient
class ConsolidatedClient(object):
    def __inst__(self, nova_client, glance_client):
        self.nova = nova_client
        self.glance = glance_client

class OpenStackClient(baseDriver.BaseDriver):
    Image = OpenStack_Image
    cloudType = 'openstack'

    configurationDescriptorXmlData = _configurationDescriptorXmlData
    credentialsDescriptorXmlData = _credentialsDescriptorXmlData

    RBUILDER_BUILD_TYPE = 'XEN_OVA'  # TODO: Determine appropriate rBuilder image type
    # This should probably be the KVM Raw....

    @classmethod
    def getCloudNameFromDescriptorData(cls, descriptorData):
        return descriptorData.getField('alias')

    @classmethod
    def isDriverFunctional(cls):
        if not novaclient or not Client: #nova and glance clients
            return False
        return True

    # Right now 1.1 is the only version that is supported
    def _openstack_api_version(self):
        return 'v1.1'

    def drvCreateCloudClient(self, credentials):
        cloudConfig = self.getTargetConfiguration()
        server = cloudConfig['nova_server']
        port = cloudConfig['nova_port']
        glance_server = cloudConfig['glance_server']
        glance_port = cloudConfig['glance_port']
        api_version = self._openstack_api_version()
        try:
            # password is a ProtectedString, we have to convert to string
            nova_client = novaclient.OpenStack( credentials['username'],
                                            credentials['auth_token'],
                                            "http://%s:%s/%s/" %
                                            (server, port, api_version) )
            glance_client = Client(glance_server, glance_port)
            clients = ConsolidatedClient(nova_client, glance_client)
        # TODO: determine more specific list of exceptions to catch
        except:
            raise errors.PermissionDenied(message = \
                    "There was an error initializing Nova and Glance clients")
        return clients

    def drvVerifyCloudConfiguration(self, config):
        return

    def terminateInstances(self, instanceIds):
        running_instances = self.getInstances(instanceIds)
        for server in running_instances:
            server.delete() # there is no terminate method in novaclient

        insts = instances.BaseInstances()
        insts.extend(running_instances)
        # Set state
        for inst in insts:
            inst.setState("Terminating")
        return insts

    def terminateInstance(self, instanceId):
        return self.terminateInstances([instanceId])

    def _get_flavors(self):
        client = self.client.nova
        return client.flavors.list()

    # TODO: figure this out.  It is an option launch param.
    def _get_ipgroups(self):
        client = self.client.nova
        return client.ipgroup.list()

    # This also takes optional "ipgroup", "meta", and "files",
    # see novaclient/servers.py, ignoring for now.  TODO: really ignore?
    def drvPopulateLaunchDescriptor(self, descr):
        descr.setDisplayName("OpenStack Launch Parameters")
        descr.addDescription("OpenStack Launch Parameters")
        self.drvLaunchDescriptorCommonFields(descr)
        flavors = self._get_flavors()
        if not flavors:
            raise errors.CatalogError("No instance flavors defined")
        flavor_id_name_map = dict( (f.id, f.name) for f in flavors)
        flavor_id_name_map.keys().sort()
        descr.addDataField("flavor",
            descriptions = "Flavor",
            required = True,
            help = [
                ("launch/flavors.html", None)
            ],
            type = descr.EnumeratedType(
                descr.ValueWithDescription(id, descriptions = flavor_id_name_map[id])
                for x in flavor_id_name_map.keys()),
            default = flavor_id_name_map.keys()[0],
            )

        return descr

    # TODO: remove when novaclient has caught up to v1.1.
    # This pulls a resource id from from a resource ref url
    def _get_id_from_ref(self, resource_ref):
        return resource_ref.split('/')[-1]

    def drvGetInstances(self, instanceIds):
        client = self.client.nova
        cloudAlias = self.getCloudAlias()
        instanceList = instances.BaseInstances()
        for server in client.servers.list():

            instanceId = server.id
            image_id = self._get_id_from_ref(server.imageRef)

            inst = self._nodeFactory.newInstance(id = instanceId,
                imageId = image_id or 'UNKNOWN',
                instanceId = instanceId,
                instanceName = server.name,
                instanceDescription = 'UNDEFINED',
                reservationId = server.id, # Does this have an openstack equivalent
                dnsName = 'UNKNOWN',
                publicDnsName = ",".join(server.addresses['public']), # valid?
                privateDnsName = ",".join(server.addresses['private']), # valid?
                state = server.status,
                launchTime = server.created if hasattr(server, 'created') else 'UNKNOWN',
                cloudName = self.cloudName,
                cloudAlias = cloudAlias)

            instanceList.append(inst)
        instanceList.sort(key = lambda x: (x.getState(), x.getInstanceId()))
        return self.filterInstances(instanceIds, instanceList)

    def getLaunchInstanceParameters(self, image, descriptorData):
        params = baseDriver.BaseDriver.getLaunchInstanceParameters(self,
            image, descriptorData)
        getField = descriptorData.getField
        srUuid = getField('storageRepository')
        params['srUuid'] = srUuid
        return params

    def launchInstanceProcess(self, job, image, auth, **launchParams):
        ppop = launchParams.pop
        srUuid = ppop('srUuid')
        instanceName = ppop('instanceName')
        instanceDescription = ppop('instanceDescription')

        cloudConfig = self.getTargetConfiguration()
        nameLabel = image.getLongName()
        nameDescription = image.getBuildDescription()

        if not image.getIsDeployed():
            self._deployImage(job, image, auth, srUuid)

        imageId = image.getInternalTargetId()

        job.addHistoryEntry('Cloning template')
        realId = self.cloneTemplate(job, imageId, instanceName,
            instanceDescription)
        job.addHistoryEntry('Attaching credentials')
        try:
            self._attachCredentials(realId, srUuid)
        except Exception, e:
            self.log_exception("Exception attaching credentials: %s" % e)
        job.addHistoryEntry('Launching')
        self.startVm(realId)
        return self.client.xenapi.VM.get_uuid(realId)

    def getImagesFromTarget(self, imageIdsFilter):
        cloudAlias = self.getCloudAlias()

        imageList = images.BaseImages()

        client = self.client.glance

        for image in client.get_images_detailed():
            #if image. - do we want to restrict this to only bootable image?
            #EG, machine/raw type?  it appears that glance isn't storing
            # type anyway; so perhaps don't bother.
            pass
            imgChecksum = vm['other_config'].get('cloud-catalog-checksum')
            if imgChecksum:
                is_rBuilderImage = True
                imageId = imgChecksum
            else:
                is_rBuilderImage = False
                imageId = vm['uuid']

            if imageIdsFilter is not None and imageId not in imageIdsFilter:
                continue

            image = self._nodeFactory.newImage(id = imageId,
                    imageId = imageId, isDeployed = True,
                    is_rBuilderImage = is_rBuilderImage,
                    longName = vm['name_label'],
                    buildDescription = vm['name_description'],
                    cloudName = self.cloudName,
                    internalTargetId = vm['uuid'],
                    cloudAlias = cloudAlias)
            imageList.append(image)
        return imageList

    def startVm(self, vmRef):
        client.nova.create("name", imageRef, flavorRef) # ipGroup, etc
<commit_msg>Pyflakes<commit_after>import os

from catalogService import errors
from catalogService.rest import baseDriver
from catalogService.rest.models import images
from catalogService.rest.models import instances

try:
    import novaclient
    from glance.client import Client
except ImportError:
    pass

class OpenStack_Image(images.BaseImage):
    "OpenStack Image"

# This is provided by the nova api
#class OpenStack_InstanceTypes(instances.InstanceTypes):
#    "OpenStack Instance Types"
#
#    idMap = [
#        ('xenent.small', "Small"),
#        ('xenent.medium', "Medium"),
#    ]

# Nova address
# Nova port
# Glance address (until apis are integrated)
# Glance port
_configurationDescriptorXmlData = """<?xml version='1.0' encoding='UTF-8'?>
<descriptor xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.rpath.org/permanent/descriptor-1.0.xsd descriptor-1.0.xsd">
  <metadata>
    <displayName>OpenStack Configuration</displayName>
    <descriptions>
      <desc>Configure OpenStack</desc>
    </descriptions>
  </metadata>
  <dataFields>
    <field>
      <name>nova_server</name>
      <descriptions>
        <desc>Nova Server</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/novaServerName.html'/>
    </field>
    <field>
      <name>nova_port</name>
      <descriptions>
        <desc>Nova Port</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/novaPortNumber.html'/>
    </field>
    <field>
      <name>glance_server</name>
      <descriptions>
        <desc>Glance Server</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/glanceServerName.html'/>
    </field>
    <field>
      <name>glance_port</name>
      <descriptions>
        <desc>Glance Port</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/glancePortNumber.html'/>
    </field>
    <field>
      <name>alias</name>
      <descriptions>
        <desc>Name</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/alias.html'/>
    </field>
    <field>
      <name>description</name>
      <descriptions>
        <desc>Full Description</desc>
      </descriptions>
      <type>str</type>
      <required>true</required>
      <help href='configuration/description.html'/>
    </field>
  </dataFields>
</descriptor>"""

# User Name
# Auth Token
_credentialsDescriptorXmlData = """<?xml version='1.0' encoding='UTF-8'?>
<descriptor xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.rpath.org/permanent/descriptor-1.0.xsd descriptor-1.0.xsd">
  <metadata>
    <displayName>OpenStack User Credentials</displayName>
    <descriptions>
      <desc>User Credentials for OpenStack</desc>
    </descriptions>
  </metadata>
  <dataFields>
    <field>
      <name>username</name>
      <descriptions>
        <desc>User Name</desc>
      </descriptions>
      <type>str</type>
      <constraints>
        <descriptions>
          <desc>Field must contain between 1 and 32 characters</desc>
        </descriptions>
        <length>255</length>
      </constraints>
      <required>true</required>
    </field>
    <field>
      <name>auth_token</name>
      <descriptions>
        <desc>Auth Token</desc>
      </descriptions>
      <type>str</type>
      <constraints>
        <descriptions>
          <desc>Field must contain between 1 and 40 characters</desc>
        </descriptions>
        <length>40</length>
      </constraints>
      <required>true</required>
      <password>true</password>
    </field>    @classmethod
    def getCloudNameFromDescriptorData(cls, descriptorData):
        return descriptorData.getField('name')

  </dataFields>
</descriptor>
"""

# http://glance.openstack.org/client.html
# http://pypi.python.org/pypi/python-novaclient
class ConsolidatedClient(object):
    def __inst__(self, nova_client, glance_client):
        self.nova = nova_client
        self.glance = glance_client

class OpenStackClient(baseDriver.BaseDriver):
    Image = OpenStack_Image
    cloudType = 'openstack'

    configurationDescriptorXmlData = _configurationDescriptorXmlData
    credentialsDescriptorXmlData = _credentialsDescriptorXmlData

    RBUILDER_BUILD_TYPE = 'XEN_OVA'  # TODO: Determine appropriate rBuilder image type
    # This should probably be the KVM Raw....

    @classmethod
    def getCloudNameFromDescriptorData(cls, descriptorData):
        return descriptorData.getField('alias')

    @classmethod
    def isDriverFunctional(cls):
        if not novaclient or not Client: #nova and glance clients
            return False
        return True

    # Right now 1.1 is the only version that is supported
    def _openstack_api_version(self):
        return 'v1.1'

    def drvCreateCloudClient(self, credentials):
        cloudConfig = self.getTargetConfiguration()
        server = cloudConfig['nova_server']
        port = cloudConfig['nova_port']
        glance_server = cloudConfig['glance_server']
        glance_port = cloudConfig['glance_port']
        api_version = self._openstack_api_version()
        try:
            # password is a ProtectedString, we have to convert to string
            nova_client = novaclient.OpenStack( credentials['username'],
                                            credentials['auth_token'],
                                            "http://%s:%s/%s/" %
                                            (server, port, api_version) )
            glance_client = Client(glance_server, glance_port)
            clients = ConsolidatedClient(nova_client, glance_client)
        # TODO: determine more specific list of exceptions to catch
        except:
            raise errors.PermissionDenied(message = \
                    "There was an error initializing Nova and Glance clients")
        return clients

    def drvVerifyCloudConfiguration(self, config):
        return

    def terminateInstances(self, instanceIds):
        running_instances = self.getInstances(instanceIds)
        for server in running_instances:
            server.delete() # there is no terminate method in novaclient

        insts = instances.BaseInstances()
        insts.extend(running_instances)
        # Set state
        for inst in insts:
            inst.setState("Terminating")
        return insts

    def terminateInstance(self, instanceId):
        return self.terminateInstances([instanceId])

    def _get_flavors(self):
        client = self.client.nova
        return client.flavors.list()

    # TODO: figure this out.  It is an option launch param.
    def _get_ipgroups(self):
        client = self.client.nova
        return client.ipgroup.list()

    # This also takes optional "ipgroup", "meta", and "files",
    # see novaclient/servers.py, ignoring for now.  TODO: really ignore?
    def drvPopulateLaunchDescriptor(self, descr):
        descr.setDisplayName("OpenStack Launch Parameters")
        descr.addDescription("OpenStack Launch Parameters")
        self.drvLaunchDescriptorCommonFields(descr)
        flavors = self._get_flavors()
        if not flavors:
            raise errors.CatalogError("No instance flavors defined")
        flavor_id_name_map = dict( (f.id, f.name) for f in flavors)
        flavor_id_name_map.keys().sort()
        descr.addDataField("flavor",
            descriptions = "Flavor",
            required = True,
            help = [
                ("launch/flavors.html", None)
            ],
            type = descr.EnumeratedType(
                descr.ValueWithDescription(id, descriptions = flavor_id_name_map[id])
                for x in flavor_id_name_map.keys()),
            default = flavor_id_name_map.keys()[0],
            )

        return descr

    # TODO: remove when novaclient has caught up to v1.1.
    # This pulls a resource id from from a resource ref url
    def _get_id_from_ref(self, resource_ref):
        return resource_ref.split('/')[-1]

    def drvGetInstances(self, instanceIds):
        client = self.client.nova
        cloudAlias = self.getCloudAlias()
        instanceList = instances.BaseInstances()
        for server in client.servers.list():

            instanceId = server.id
            image_id = self._get_id_from_ref(server.imageRef)

            inst = self._nodeFactory.newInstance(id = instanceId,
                imageId = image_id or 'UNKNOWN',
                instanceId = instanceId,
                instanceName = server.name,
                instanceDescription = 'UNDEFINED',
                reservationId = server.id, # Does this have an openstack equivalent
                dnsName = 'UNKNOWN',
                publicDnsName = ",".join(server.addresses['public']), # valid?
                privateDnsName = ",".join(server.addresses['private']), # valid?
                state = server.status,
                launchTime = server.created if hasattr(server, 'created') else 'UNKNOWN',
                cloudName = self.cloudName,
                cloudAlias = cloudAlias)

            instanceList.append(inst)
        instanceList.sort(key = lambda x: (x.getState(), x.getInstanceId()))
        return self.filterInstances(instanceIds, instanceList)

    def getLaunchInstanceParameters(self, image, descriptorData):
        params = baseDriver.BaseDriver.getLaunchInstanceParameters(self,
            image, descriptorData)
        getField = descriptorData.getField
        srUuid = getField('storageRepository')
        params['srUuid'] = srUuid
        return params

    def launchInstanceProcess(self, job, image, auth, **launchParams):
        ppop = launchParams.pop
        instanceName = ppop('instanceName')
        instanceDescription = ppop('instanceDescription')

        cloudConfig = self.getTargetConfiguration()
        nameLabel = image.getLongName()
        nameDescription = image.getBuildDescription()

        self._deployImage(job, image, auth)

        imageId = image.getInternalTargetId()

        job.addHistoryEntry('Cloning template')
        realId = self.cloneTemplate(job, imageId, instanceName,
            instanceDescription)
        job.addHistoryEntry('Attaching credentials')
        try:
            self._attachCredentials(realId)
        except Exception, e:
            self.log_exception("Exception attaching credentials: %s" % e)
        job.addHistoryEntry('Launching')
        self.startVm(realId)
        return self.client.xenapi.VM.get_uuid(realId)

    def getImagesFromTarget(self, imageIdsFilter):
        cloudAlias = self.getCloudAlias()

        imageList = images.BaseImages()

        client = self.client.glance

        for image in client.get_images_detailed():
            #if image. - do we want to restrict this to only bootable image?
            #EG, machine/raw type?  it appears that glance isn't storing
            # type anyway; so perhaps don't bother.
            pass
        return imageList

    def startVm(self, name, imageRef, flavorRef):
        client = self.client.nova
        server = client.create(name, imageRef, flavorRef) # ipGroup, etc
<|endoftext|>


<commit_before>def bubble_sort(input_list):
  input_list_length = len(input_list)
  for passed in range(input_list_length):
    for index in range(passed, input_list_length-1):
      if (input_list[index] > input_list[index+1])
        input_list[index], input_list[index+1] =  input_list[index+1], input_list[index]

  return input_list

def insert_sort(input_list):

  return input_list
<commit_msg>Add insert sort<commit_after>def bubble_sort(input_list):
  input_list_length = len(input_list)
  for passed in range(input_list_length):
    for index in range(passed, input_list_length-1):
      if (input_list[index] > input_list[index+1])
        input_list[index], input_list[index+1] =  input_list[index+1], input_list[index]

  return input_list

def insert_sort(input_list):
  input_list_length = len(input_list)
  for i in range(1,input_list_length):
    result_list = insert_sort_single_sort(input_list[:i], input_list[i])
    if (i != input_list_length):
      input_list = result_list + input_list[i+1:]
  return input_list

def insert_sort_single_sort(input_list, new_element):
  is_insert = false
  for i in range(len(input_list)):
    if input_list[i] > new_element:
      input_list.insert(i,new_element)
      is_insert = true
  if is_isert == false:
    input_list.append(new_element)
  return input_list

<|endoftext|>


<commit_before>"""UDP Clients for sending OSC messages to an OSC server"""

try:
    from collections.abc import Iterable
except ImportError: # python 3.5
    from collections import Iterable

import socket

from .osc_message_builder import OscMessageBuilder
from pythonosc.osc_message import OscMessage
from pythonosc.osc_bundle import OscBundle

from typing import Union

class UDPClient(object):
    """OSC client to send :class:`OscMessage` or :class:`OscBundle` via UDP"""

    def __init__(self, address: str, port: int, allow_broadcast: bool = False) -> None:
        """Initialize client

        As this is UDP it will not actually make any attempt to connect to the
        given server at ip:port until the send() method is called.

        Args:
            address: IP address of server
            port: Port of server
            allow_broadcast: Allow for broadcast transmissions
        """
        for addr in socket.getaddrinfo(address, port, type=socket.SOCK_DGRAM):
            af, socktype, protocol, canonname, sa = addr

            try:
                self._sock = socket.socket(af, socktype)
            except OSError:
                continue
            break

        self._sock.setblocking(False)
        if allow_broadcast:
            self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
        self._address = address
        self._port = port

    def send(self, content: Union[OscMessage, OscBundle]) -> None:
        """Sends an :class:`OscMessage` or :class:`OscBundle` via UDP

        Args:
            content: Message or bundle to be sent
        """
        self._sock.sendto(content.dgram, (self._address, self._port))

class SimpleUDPClient(UDPClient):
    """Simple OSC client that automatically builds :class:`OscMessage` from arguments"""

    def send_message(self, address: str, value: Union[int, float, bytes, str, bool, tuple, list]) -> None:
        """Build :class:`OscMessage` from arguments and send to server

        Args:
            address: OSC address the message shall go to
            value: One or more arguments to be added to the message
        """
        builder = OscMessageBuilder(address=address)
        if value is None:
            values = []
        elif not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
            values = [value]
        else:
            values = value
        for val in values:
            builder.add_arg(val)
        msg = builder.build()
        self.send(msg)
<commit_msg>Use more precise annotation now we have one<commit_after>"""UDP Clients for sending OSC messages to an OSC server"""

try:
    from collections.abc import Iterable
except ImportError: # python 3.5
    from collections import Iterable

import socket

from .osc_message_builder import OscMessageBuilder, ArgValue
from pythonosc.osc_message import OscMessage
from pythonosc.osc_bundle import OscBundle

from typing import Union

class UDPClient(object):
    """OSC client to send :class:`OscMessage` or :class:`OscBundle` via UDP"""

    def __init__(self, address: str, port: int, allow_broadcast: bool = False) -> None:
        """Initialize client

        As this is UDP it will not actually make any attempt to connect to the
        given server at ip:port until the send() method is called.

        Args:
            address: IP address of server
            port: Port of server
            allow_broadcast: Allow for broadcast transmissions
        """
        for addr in socket.getaddrinfo(address, port, type=socket.SOCK_DGRAM):
            af, socktype, protocol, canonname, sa = addr

            try:
                self._sock = socket.socket(af, socktype)
            except OSError:
                continue
            break

        self._sock.setblocking(False)
        if allow_broadcast:
            self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
        self._address = address
        self._port = port

    def send(self, content: Union[OscMessage, OscBundle]) -> None:
        """Sends an :class:`OscMessage` or :class:`OscBundle` via UDP

        Args:
            content: Message or bundle to be sent
        """
        self._sock.sendto(content.dgram, (self._address, self._port))

class SimpleUDPClient(UDPClient):
    """Simple OSC client that automatically builds :class:`OscMessage` from arguments"""

    def send_message(self, address: str, value: ArgValue) -> None:
        """Build :class:`OscMessage` from arguments and send to server

        Args:
            address: OSC address the message shall go to
            value: One or more arguments to be added to the message
        """
        builder = OscMessageBuilder(address=address)
        if value is None:
            values = []
        elif not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
            values = [value]
        else:
            values = value
        for val in values:
            builder.add_arg(val)
        msg = builder.build()
        self.send(msg)
<|endoftext|>


<commit_before>import os, random, cv2
import numpy as np
from PIL import Image
from keras.utils import np_utils
from collections import Counter

'''
    file name: dataset.py
    author/s: Keiron O'Shea
    description: simple toolkit to load images, with their adjacent labels
    into arrays - suitable for use with keras/theano.
'''

def more_data(directory, image_list, image_height, image_width):
    response = raw_input("Do you want to produce more data? (Y/N): ")
    if response in ('Y', 'y', 'Yes', 'yes'):
        print "Creating new data, this may take awhile depending on the number of images...\n"
        rotation_matrix_1 = cv2.getRotationMatrix2D((image_height/2,image_width/2), 4, 1)
        rotation_matrix_2 = cv2.getRotationMatrix2D((image_height/2,image_width/2), 7, 1)
        rotation_matrix_3 = cv2.getRotationMatrix2D((image_height/2,image_width/2), -7, 1)
        rotation_matrix_4 = cv2.getRotationMatrix2D((image_height/2,image_width/2), -4, 1)
        #I want this to go into memory.
        for images in image_list:
            image = cv2.imread(directory+"/"+images[0]+"/"+images[1])
            rotated_image_1 = cv2.warpAffine(image, rotation_matrix_1, (image_height, image_width))
            rotated_image_2 = cv2.warpAffine(image, rotation_matrix_2, (image_height, image_width))
            rotated_image_3 = cv2.warpAffine(image, rotation_matrix_3, (image_height, image_width))
            rotated_image_4 = cv2.warpAffine(image, rotation_matrix_4, (image_height, image_width))
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".1.jpg",rotated_image_1)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".2.jpg",rotated_image_2)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".3.jpg",rotated_image_3)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".4.jpg",rotated_image_4)
    else:
        pass

def get_class_size(image_list):
    list_of_classes = Counter(labels[0] for labels in image_list).iteritems()
    return list_of_classes

'''
get_labels()

    Allows the correlation of labels to its image for pre-labelling purposes.

'''

def get_labels(directory):
    imglist = []
    for dirname,dirnames,filenames in os.walk(directory):
        for filename in filenames:
            label = os.path.basename(os.path.normpath(dirname))
            imglist.append([label, filename])
    class_sizes = get_class_size(imglist)
    return imglist, class_sizes

'''
load_images()

    Loads the preprocessed images from a folder, and correlates them
    with a correct label for use in keras/theano.

'''
def load_images(directory, image_height, image_width):
    # Retrieves a list of images.
    image_list, class_sizes = get_labels(directory)
    number_files = len(image_list)

    origlist = []
    labels = []
    list_of_classes = get_class_size(image_list)
    for list1 in list_of_classes:
        origlist.append(list1[1])
        labels.append(list1[0])
    for i in range(len(origlist)):
        list2 = origlist[:]
        labels2 = labels[:]
        list2.pop(i)
        labels2.pop(i)
    #print labels2
    for j in range(len(list2)):
        perIncrease = float((origlist[i]-list2[j]))/float(origlist[i])
        print "percentage increase ",perIncrease
        if ( perIncrease  > 0.3):
            # load data of  class of list2[j]
            print "add more data of class ",labels2[j]
        #more_data(directory, image_list, image_hieght, image_width)

    # Creates an array ready for the images to go into vectors.
    train_data = np.empty((number_files, 3, image_height, image_width), dtype="float32")
    # Flattens it.
    train_data.flatten()
    # Creates an array, ready for the labels to go into vectors to correlate with training_data
    train_label = np.empty((number_files,), dtype="uint8")
    for i, image_name in enumerate(image_list):
        # Open the files.
        images = Image.open(directory+"/"+image_name[0]+"/"+image_name[1])
        images = images.resize((image_height, image_width))
        # Converts the images into float32 representation
        vectored_image = np.asarray(images, dtype="float32")
        # 3 shape vector..
        train_data[i,:,:,:] = [vectored_image[:,:,0],vectored_image[:,:,1],vectored_image[:,:,2]]
        string = image_name[0]
        #Assigns label to the image.
        train_label[i] = int(string)
    get_class_size(image_list)
    return train_data, train_label

'''
splitTrainValidationAndTest()

    Uses the split parameter to split the dataset into two sets.
    One for training (split) and another for validation.

    Standard call from vectorise().

'''

def splitTrainAndValdidation(split, number_images, train_data, label, height, width):
    # Calculating the number of training data.
    number_training_data = number_images * split

    # Calculating the percentage of data used for validation data.
    number_validation_data = number_images * (1.0 - split)

    X_train = train_data[: number_training_data]
    Y_train = label[: number_training_data]

    X_val = train_data[number_training_data :][0 : number_validation_data]
    Y_val = label[number_training_data :][0 : number_validation_data]

    # Setting the shapes for optimised vectorisation.
    X_train = X_train.reshape(X_train.shape[0], 3, height, width)/255
    X_val = X_val.reshape(X_val.shape[0], 3, height, width)/255

    # Setting the datasets to float32. (Single-precision floating-point format) - Suitable for NNs!
    X_train = X_train.astype("float32")
    X_val = X_val.astype("float32")

    # Returning the data (X = data, Y = labels)
    return X_train, Y_train, X_val, Y_val

'''
splitTrainValidationAndTest()

    Uses the split parameter to split the dataset into three sets.
    One for training (split), one for validation (5% of split) and
    another for tesint (5% of split)

    Only used when called with a final boolean value from vectorise()

'''

def splitTrainValidationAndTest(split, number_images, data, label, height, width):

    # Calculating the number of training data.
    number_training_data = number_images * split

    number_validation_data = number_images * ((1-split)/2)
    number_test_data = number_images * ((1-split)/2)

    print 'total number of images: ',number_images
    #print 'number of data ',data.shape()
    print 'number of training data: ',number_training_data
    print 'number of validation data: ',number_validation_data
    print 'number of testing data: ',number_test_data

    X_train = data[: number_training_data]
    Y_train = label[: number_training_data]

    X_val = data[number_training_data :][0 : number_validation_data]
    Y_val = label[number_training_data :][0 : number_validation_data]

    X_test = data[number_training_data::][number_validation_data::]
    Y_test = label[number_training_data::][number_validation_data::]

    # Setting the shapes for optimised vectorisation.
    X_train = X_train.reshape(X_train.shape[0], 3, height, width)/255
    X_val = X_val.reshape(X_val.shape[0], 3, height, width)/255
    X_test = X_test.reshape(X_test.shape[0], 3, height, width)/255

    # Setting the datasets to float32. (Single-precision floating-point format) - Suitable for NNs!
    X_train = X_train.astype("float32")
    X_val = X_val.astype("float32")
    X_test = X_test.astype("float32")

    # Returning the data (X = data, Y = labels)
    return X_train, Y_train, X_val, Y_val, X_test, Y_test

def vectorise(directory, nb_classes, height, width, split, with_test=False): # Get train + val by default.
    # Nasty-ass unoptimised image vectors with labels.
    train_data, train_label= load_images(directory, height, width)

    number_images = len(train_label)
    index = [i for i in range(number_images)]

    # Shuffling the dataset
    random.shuffle(index)

    train_data = train_data[index]
    train_label = train_label[index]

    # Convert class vector to binary class matrices for categorial_crossentropy.
    label = np_utils.to_categorical(train_label, nb_classes)

    # Just get train + validation
    if with_test == False:
        X_train, Y_train, X_val, Y_val = splitTrainAndValdidation(split, number_images, train_data, label, height, width)
        return X_train, Y_train, X_val, Y_val

    # Get train, validation and testing data.
    if with_test == True:
        X_train, Y_train, X_val, Y_val, X_test, Y_test = splitTrainValidationAndTest(split, number_images, train_data, label, height, width)
        return X_train, Y_train, X_val, Y_val, X_test, Y_test

<commit_msg>Added another function<commit_after>import os, random, cv2
import numpy as np
from PIL import Image
from keras.utils import np_utils
from collections import Counter

'''
    file name: dataset.py
    author/s: Keiron O'Shea
    description: simple toolkit to load images, with their adjacent labels
    into arrays - suitable for use with keras/theano.
'''

def more_data(directory, image_list, image_height, image_width):
    response = raw_input("Do you want to produce more data? (Y/N): ")
    if response in ('Y', 'y', 'Yes', 'yes'):
        print "Creating new data, this may take awhile depending on the number of images...\n"
        rotation_matrix_1 = cv2.getRotationMatrix2D((image_height/2,image_width/2), 4, 1)
        rotation_matrix_2 = cv2.getRotationMatrix2D((image_height/2,image_width/2), 7, 1)
        rotation_matrix_3 = cv2.getRotationMatrix2D((image_height/2,image_width/2), -7, 1)
        rotation_matrix_4 = cv2.getRotationMatrix2D((image_height/2,image_width/2), -4, 1)
        #I want this to go into memory.
        for images in image_list:
            image = cv2.imread(directory+"/"+images[0]+"/"+images[1])
            rotated_image_1 = cv2.warpAffine(image, rotation_matrix_1, (image_height, image_width))
            rotated_image_2 = cv2.warpAffine(image, rotation_matrix_2, (image_height, image_width))
            rotated_image_3 = cv2.warpAffine(image, rotation_matrix_3, (image_height, image_width))
            rotated_image_4 = cv2.warpAffine(image, rotation_matrix_4, (image_height, image_width))
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".1.jpg",rotated_image_1)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".2.jpg",rotated_image_2)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".3.jpg",rotated_image_3)
            cv2.imwrite(directory+"/"+images[0]+"/" + str(images[1]).split('.')[0]+".4.jpg",rotated_image_4)
    else:
        pass

def get_class_size(image_list):
    list_of_classes = Counter(labels[0] for labels in image_list).iteritems()
    return list_of_classes

'''
get_labels()

    Allows the correlation of labels to its image for pre-labelling purposes.

'''

def get_labels(directory):
    imglist = []
    for dirname,dirnames,filenames in os.walk(directory):
        for filename in filenames:
            label = os.path.basename(os.path.normpath(dirname))
            imglist.append([label, filename])
    class_sizes = get_class_size(imglist)
    return imglist, class_sizes

def check_class_stabilitiy(image_list):
    origlist = []
    labels = []
    list_of_classes = get_class_size(image_list)
    for list1 in list_of_classes:
        origlist.append(list1[1])
        labels.append(list1[0])
    for i in range(len(origlist)):
        list2 = origlist[:]
        labels2 = labels[:]
        list2.pop(i)
        labels2.pop(i)
    for j in range(len(list2)):
        perIncrease = float((origlist[i]-list2[j]))/float(origlist[i])
        print "percentage increase ",perIncrease
        if ( perIncrease  > 0.3):
            print "add more data of class ",labels2[j]

'''
load_images()

    Loads the preprocessed images from a folder, and correlates them
    with a correct label for use in keras/theano.

'''
def load_images(directory, image_height, image_width):
    # Retrieves a list of images.
    image_list, class_sizes = get_labels(directory)
    number_files = len(image_list)
    check_class_stabilitiy(image_list)
    # more_data(directory, image_list, image_height, image_width)
    # Creates an array ready for the images to go into vectors.
    train_data = np.empty((number_files, 3, image_height, image_width), dtype="float32")
    # Flattens it.
    train_data.flatten()
    # Creates an array, ready for the labels to go into vectors to correlate with training_data
    train_label = np.empty((number_files,), dtype="uint8")
    for i, image_name in enumerate(image_list):
        # Open the files.
        images = Image.open(directory+"/"+image_name[0]+"/"+image_name[1])
        images = images.resize((image_height, image_width))
        # Converts the images into float32 representation
        vectored_image = np.asarray(images, dtype="float32")
        # 3 shape vector..
        train_data[i,:,:,:] = [vectored_image[:,:,0],vectored_image[:,:,1],vectored_image[:,:,2]]
        string = image_name[0]
        #Assigns label to the image.
        train_label[i] = int(string)
    get_class_size(image_list)
    return train_data, train_label

'''
splitTrainValidationAndTest()

    Uses the split parameter to split the dataset into two sets.
    One for training (split) and another for validation.

    Standard call from vectorise().

'''

def splitTrainAndValdidation(split, number_images, train_data, label, height, width):
    # Calculating the number of training data.
    number_training_data = number_images * split

    # Calculating the percentage of data used for validation data.
    number_validation_data = number_images * (1.0 - split)

    X_train = train_data[: number_training_data]
    Y_train = label[: number_training_data]

    X_val = train_data[number_training_data :][0 : number_validation_data]
    Y_val = label[number_training_data :][0 : number_validation_data]

    # Setting the shapes for optimised vectorisation.
    X_train = X_train.reshape(X_train.shape[0], 3, height, width)/255
    X_val = X_val.reshape(X_val.shape[0], 3, height, width)/255

    # Setting the datasets to float32. (Single-precision floating-point format) - Suitable for NNs!
    X_train = X_train.astype("float32")
    X_val = X_val.astype("float32")

    # Returning the data (X = data, Y = labels)
    return X_train, Y_train, X_val, Y_val

'''
splitTrainValidationAndTest()

    Uses the split parameter to split the dataset into three sets.
    One for training (split), one for validation (5% of split) and
    another for tesint (5% of split)

    Only used when called with a final boolean value from vectorise()

'''

def splitTrainValidationAndTest(split, number_images, data, label, height, width):

    # Calculating the number of training data.
    number_training_data = number_images * split

    number_validation_data = number_images * ((1-split)/2)
    number_test_data = number_images * ((1-split)/2)

    print 'total number of images: ',number_images
    #print 'number of data ',data.shape()
    print 'number of training data: ',number_training_data
    print 'number of validation data: ',number_validation_data
    print 'number of testing data: ',number_test_data

    X_train = data[: number_training_data]
    Y_train = label[: number_training_data]

    X_val = data[number_training_data :][0 : number_validation_data]
    Y_val = label[number_training_data :][0 : number_validation_data]

    X_test = data[number_training_data::][number_validation_data::]
    Y_test = label[number_training_data::][number_validation_data::]

    # Setting the shapes for optimised vectorisation.
    X_train = X_train.reshape(X_train.shape[0], 3, height, width)/255
    X_val = X_val.reshape(X_val.shape[0], 3, height, width)/255
    X_test = X_test.reshape(X_test.shape[0], 3, height, width)/255

    # Setting the datasets to float32. (Single-precision floating-point format) - Suitable for NNs!
    X_train = X_train.astype("float32")
    X_val = X_val.astype("float32")
    X_test = X_test.astype("float32")

    # Returning the data (X = data, Y = labels)
    return X_train, Y_train, X_val, Y_val, X_test, Y_test

def vectorise(directory, nb_classes, height, width, split, with_test=False): # Get train + val by default.

    # Nasty-ass unoptimised image vectors with labels.
    train_data, train_label= load_images(directory, height, width)

    number_images = len(train_label)
    index = [i for i in range(number_images)]

    # Shuffling the dataset
    random.shuffle(index)

    train_data = train_data[index]
    train_label = train_label[index]

    # Convert class vector to binary class matrices for categorial_crossentropy.
    label = np_utils.to_categorical(train_label, nb_classes)

    # Just get train + validation
    if with_test == False:
        X_train, Y_train, X_val, Y_val = splitTrainAndValdidation(split, number_images, train_data, label, height, width)
        return X_train, Y_train, X_val, Y_val

    # Get train, validation and testing data.
    if with_test == True:
        X_train, Y_train, X_val, Y_val, X_test, Y_test = splitTrainValidationAndTest(split, number_images, train_data, label, height, width)
        return X_train, Y_train, X_val, Y_val, X_test, Y_test

<|endoftext|>


<commit_before>#!/usr/bin/env python2
# Copyright (c) 2014 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

# Exercise the wallet keypool, and interaction with wallet encryption/locking

# Add python-bitcoinrpc to module search path:
import os
import sys

import json
import shutil
import subprocess
import tempfile
import traceback

from test_framework.util import *

def check_array_result(object_array, to_match, expected):
    """
    Pass in array of JSON objects, a dictionary with key/value pairs
    to match against, and another dictionary with expected key/value
    pairs.
    """
    num_matched = 0
    for item in object_array:
        all_match = True
        for key,value in to_match.items():
            if item[key] != value:
                all_match = False
        if not all_match:
            continue
        for key,value in expected.items():
            if item[key] != value:
                raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value)))
            num_matched = num_matched+1
    if num_matched == 0:
        raise AssertionError("No objects matched %s"%(str(to_match)))

def run_test(nodes, tmpdir):
    # Encrypt wallet and wait to terminate
    nodes[0].encryptwallet('test')
    bitcoind_processes[0].wait()
    # Restart node 0
    nodes[0] = start_node(0, tmpdir)
    # Keep creating keys
    addr = nodes[0].getnewaddress()
    try:
        addr = nodes[0].getnewaddress()
        raise AssertionError('Keypool should be exhausted after one address')
    except JSONRPCException,e:
        assert(e.error['code']==-12)

    # put three new keys in the keypool
    nodes[0].walletpassphrase('test', 12000)
    nodes[0].keypoolrefill(3)
    nodes[0].walletlock()

    # drain the keys
    addr = set()
    addr.add(nodes[0].getrawchangeaddress())
    addr.add(nodes[0].getrawchangeaddress())
    addr.add(nodes[0].getrawchangeaddress())
    addr.add(nodes[0].getrawchangeaddress())
    # assert that four unique addresses were returned
    assert(len(addr) == 4)
    # the next one should fail
    try:
        addr = nodes[0].getrawchangeaddress()
        raise AssertionError('Keypool should be exhausted after three addresses')
    except JSONRPCException,e:
        assert(e.error['code']==-12)

    # refill keypool with three new addresses
    nodes[0].walletpassphrase('test', 12000)
    nodes[0].keypoolrefill(3)
    nodes[0].walletlock()

    # drain them by mining
    nodes[0].generate(1)
    nodes[0].generate(1)
    nodes[0].generate(1)
    nodes[0].generate(1)
    try:
        nodes[0].generate(1)
        raise AssertionError('Keypool should be exhausted after three addesses')
    except JSONRPCException,e:
        assert(e.error['code']==-12)

def main():
    import optparse

    parser = optparse.OptionParser(usage="%prog [options]")
    parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
                      help="Leave bitcoinds and test.* datadir on exit or error")
    parser.add_option("--srcdir", dest="srcdir", default="../../src",
                      help="Source directory containing bitcoind/bitcoin-cli (default: %default%)")
    parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
                      help="Root directory for datadirs")
    (options, args) = parser.parse_args()

    os.environ['PATH'] = options.srcdir+":"+os.environ['PATH']

    check_json_precision()

    success = False
    nodes = []
    try:
        print("Initializing test directory "+options.tmpdir)
        if not os.path.isdir(options.tmpdir):
            os.makedirs(options.tmpdir)
        initialize_chain(options.tmpdir)

        nodes = start_nodes(1, options.tmpdir)

        run_test(nodes, options.tmpdir)

        success = True

    except AssertionError as e:
        print("Assertion failed: "+e.message)
    except JSONRPCException as e:
        print("JSONRPC error: "+e.error['message'])
        traceback.print_tb(sys.exc_info()[2])
    except Exception as e:
        print("Unexpected exception caught during testing: "+str(sys.exc_info()[0]))
        traceback.print_tb(sys.exc_info()[2])

    if not options.nocleanup:
        print("Cleaning up")
        stop_nodes(nodes)
        wait_bitcoinds()
        shutil.rmtree(options.tmpdir)

    if success:
        print("Tests successful")
        sys.exit(0)
    else:
        print("Failed")
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>[qa] keypool: Fix white space to prepare transition to test framework<commit_after>#!/usr/bin/env python2
# Copyright (c) 2014 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

# Exercise the wallet keypool, and interaction with wallet encryption/locking

# Add python-bitcoinrpc to module search path:
import os
import sys

import json
import shutil
import subprocess
import tempfile
import traceback

from test_framework.util import *

def check_array_result(object_array, to_match, expected):
    """
    Pass in array of JSON objects, a dictionary with key/value pairs
    to match against, and another dictionary with expected key/value
    pairs.
    """
    num_matched = 0
    for item in object_array:
        all_match = True
        for key,value in to_match.items():
            if item[key] != value:
                all_match = False
        if not all_match:
            continue
        for key,value in expected.items():
            if item[key] != value:
                raise AssertionError("%s : expected %s=%s"%(str(item), str(key), str(value)))
            num_matched = num_matched+1
    if num_matched == 0:
        raise AssertionError("No objects matched %s"%(str(to_match)))

    def run_test(nodes, tmpdir):
        # Encrypt wallet and wait to terminate
        nodes[0].encryptwallet('test')
        bitcoind_processes[0].wait()
        # Restart node 0
        nodes[0] = start_node(0, tmpdir)
        # Keep creating keys
        addr = nodes[0].getnewaddress()
        try:
            addr = nodes[0].getnewaddress()
            raise AssertionError('Keypool should be exhausted after one address')
        except JSONRPCException,e:
            assert(e.error['code']==-12)

        # put three new keys in the keypool
        nodes[0].walletpassphrase('test', 12000)
        nodes[0].keypoolrefill(3)
        nodes[0].walletlock()

        # drain the keys
        addr = set()
        addr.add(nodes[0].getrawchangeaddress())
        addr.add(nodes[0].getrawchangeaddress())
        addr.add(nodes[0].getrawchangeaddress())
        addr.add(nodes[0].getrawchangeaddress())
        # assert that four unique addresses were returned
        assert(len(addr) == 4)
        # the next one should fail
        try:
            addr = nodes[0].getrawchangeaddress()
            raise AssertionError('Keypool should be exhausted after three addresses')
        except JSONRPCException,e:
            assert(e.error['code']==-12)

        # refill keypool with three new addresses
        nodes[0].walletpassphrase('test', 12000)
        nodes[0].keypoolrefill(3)
        nodes[0].walletlock()

        # drain them by mining
        nodes[0].generate(1)
        nodes[0].generate(1)
        nodes[0].generate(1)
        nodes[0].generate(1)
        try:
            nodes[0].generate(1)
            raise AssertionError('Keypool should be exhausted after three addesses')
        except JSONRPCException,e:
            assert(e.error['code']==-12)

    def main():
        import optparse

        parser = optparse.OptionParser(usage="%prog [options]")
        parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
                          help="Leave bitcoinds and test.* datadir on exit or error")
        parser.add_option("--srcdir", dest="srcdir", default="../../src",
                          help="Source directory containing bitcoind/bitcoin-cli (default: %default%)")
        parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
                          help="Root directory for datadirs")
        (options, args) = parser.parse_args()

        os.environ['PATH'] = options.srcdir+":"+os.environ['PATH']

        check_json_precision()

        success = False
        nodes = []
        try:
            print("Initializing test directory "+options.tmpdir)
            if not os.path.isdir(options.tmpdir):
                os.makedirs(options.tmpdir)
            initialize_chain(options.tmpdir)

            nodes = start_nodes(1, options.tmpdir)

            run_test(nodes, options.tmpdir)

            success = True

        except AssertionError as e:
            print("Assertion failed: "+e.message)
        except JSONRPCException as e:
            print("JSONRPC error: "+e.error['message'])
            traceback.print_tb(sys.exc_info()[2])
        except Exception as e:
            print("Unexpected exception caught during testing: "+str(sys.exc_info()[0]))
            traceback.print_tb(sys.exc_info()[2])

        if not options.nocleanup:
            print("Cleaning up")
            stop_nodes(nodes)
            wait_bitcoinds()
            shutil.rmtree(options.tmpdir)

        if success:
            print("Tests successful")
            sys.exit(0)
        else:
            print("Failed")
            sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>