<commit_before>#!/usr/bin/env python
"""Module to run Phylogenetic Analysis by Maximum Likelihood (PAML)."""

from Bio import SeqIO
from Bio.Alphabet import generic_dna
from divergence import create_directory, extract_archive_of_files, create_archive_of_files
from divergence.select_taxa import select_genomes_from_file
from subprocess import check_call, STDOUT
import getopt
import logging as log
import os.path
import sys
import tempfile

def run_paml(genomes_a, genomes_b, sico_files):
    """Run PAML for representatives of clades A and B in each of the SICO files, to calculate dN/dS."""
    #Trash existing paml dir first
    create_directory('paml', delete_first = True)

    representative_a = genomes_a[0]['RefSeq project ID']
    representative_b = genomes_b[0]['RefSeq project ID']

    log.info('Running PAML for {0} aligned and trimmed SICOs'.format(len(sico_files)))
    return [_run_yn00(representative_a, representative_b, sico_file) for sico_file in sico_files]

YN00 = '/projects/divergence/software/paml44/bin/yn00'

def _run_yn00(repr_id_a, repr_id_b, sico_file):
    """Run yn00 from PAML for selected sequence records from sico_file, returning main nexus output file."""
    #Find sequences from above chosen clade representatives in each SICO file
    seqr_a = None
    seqr_b = None
    for seqrecord in SeqIO.parse(sico_file, 'fasta', alphabet = generic_dna):
        refseq_id = seqrecord.id.split('|')[0]
        if refseq_id == repr_id_a:
            seqr_a = seqrecord
        elif refseq_id == repr_id_b:
            seqr_b = seqrecord
        if seqr_a and seqr_b:
            break
    else:
        assert False, '{0}: Both seqr_a & seqr_b should have been set\n{1}\n{2}'.format(sico_file, seqr_a, seqr_b)

    #Write the representative sequence records out to file in yn00 compatible format
    sico = os.path.splitext(os.path.split(sico_file)[1])[0]
    yn00_dir = create_directory('paml/' + sico)
    nexus_file = os.path.join(yn00_dir, sico + '.nexus')
    _write_nexus_file(seqr_a, seqr_b, nexus_file)

    #Generate yn00 configuration file
    output_file = os.path.join(yn00_dir, 'yn' + sico)
    config_file = os.path.join(yn00_dir, 'yn00.ctl')
    _write_config_file(nexus_file, output_file, config_file)

    #Run yn00
    command = [YN00, os.path.split(config_file)[1]]
    check_call(command, cwd = yn00_dir, stdout = open('/dev/null', mode = 'w'), stderr = STDOUT)

    assert os.path.isfile(output_file) and os.path.getsize(output_file), 'Expected some content in ' + output_file
    return output_file

def _write_nexus_file(seqr_a, seqr_b, nexus_file):
    """Write representative sequences out to a file in the PAML compatible nexus format."""
    nexus_contents = """
#NEXUS
begin data;
   dimensions ntax={ntax} nchar={nchar};
   format datatype=dna missing=? gap=-;
matrix
clade_a  {seqa}
clade_b  {seqb}
;
end;
""".format(ntax = 2, nchar = len(seqr_a), seqa = str(seqr_a.seq), seqb = str(seqr_b.seq))
    with open(nexus_file, mode = 'w') as write_handle:
        #SeqIO.write([seqr_a, seqr_b], write_handle, 'nexus')
        write_handle.write(nexus_contents)

def _write_config_file(nexus_file, output_file, config_file):
    """Write a yn00 configuration file using relative paths to the nexus file and output file."""
    config_contents = """
   seqfile = {0} * sequence data file name
   outfile = {1} * main result file
   verbose = 0  * 1: detailed output (list sequences), 0: concise output

     icode = 0  * 0:universal code; 1:mammalian mt; 2-10:see below

 weighting = 0  * weighting pathways between codons (0/1)?
commonf3x4 = 0  * use one set of codon freqs for all pairs (0/1)?
*    ndata = 1

* Genetic codes: 0:universal, 1:mammalian mt., 2:yeast mt., 3:mold mt.,
* 4: invertebrate mt., 5: ciliate nuclear, 6: echinoderm mt.,
* 7: euplotid mt., 8: alternative yeast nu. 9: ascidian mt.,
* 10: blepharisma nu.
* These codes correspond to transl_table 1 to 11 of GENEBANK.
""".format(os.path.split(nexus_file)[1], os.path.split(output_file)[1])
    with open(config_file, mode = 'w') as write_handle:
        write_handle.write(config_contents)

def main(args):
    """Main function called when run from command line or as part of pipeline."""

    def _parse_options(args):
        """Use getopt to parse command line argument options"""

        def _usage():
            """Print _usage information"""
            print """
Usage: run_paml.py
--genomes-a=FILE    file with RefSeq id from complete genomes table on each line for clade A
--genomes-b=FILE    file with RefSeq id from complete genomes table on each line for clade B
--sico-zip=FILE     archive of aligned & trimmed single copy orthologous (SICO) genes
--paml-zip=FILE     destination file path for archive of PAML output per SICO gene
"""

        options = ['genomes-a', 'genomes-b', 'sico-zip', 'paml-zip']
        try:
            #postfix '=' to indicate options require an argument
            long_options = [opt + '=' for opt in options]
            tuples = getopt.getopt(args, '', long_options)[0]
            arguments = dict((opt[2:], value) for opt, value in tuples)
        except getopt.GetoptError as err:
            print str(err)
            _usage()
            sys.exit(1)

        #Ensure all arguments were provided
        for opt in options:
            if opt not in arguments:
                print 'Mandatory argument {0} not provided'.format(opt)
                _usage()
                sys.exit(1)

        #Retrieve & return file paths from dictionary
        return [arguments[option] for option in options]

    genome_a_ids_file, genome_b_ids_file, sico_zip, paml_zip = _parse_options(args)

    #Parse file containing RefSeq project IDs & retrieve associated genome dictionaries from complete genomes table
    genomes_a = select_genomes_from_file(genome_a_ids_file)
    genomes_b = select_genomes_from_file(genome_b_ids_file)

    #Extract files from zip archive
    temp_dir = tempfile.mkdtemp()
    sico_files = extract_archive_of_files(sico_zip, temp_dir)

    #Actually run cleanup
    paml_files = run_paml(genomes_a, genomes_b, sico_files)

    #Write the produced files to command line argument filenames
    create_archive_of_files(paml_zip, paml_files)

    #Exit after a comforting log message
    log.info("Produced: \n%s", paml_zip)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Use temporary directory for PAML files<commit_after>#!/usr/bin/env python
"""Module to run Phylogenetic Analysis by Maximum Likelihood (PAML)."""

from Bio import SeqIO
from Bio.Alphabet import generic_dna
from divergence import create_directory, extract_archive_of_files, create_archive_of_files
from divergence.select_taxa import select_genomes_from_file
from subprocess import check_call, STDOUT
import getopt
import logging as log
import os.path
import sys
import tempfile

def run_paml(genomes_a, genomes_b, sico_files):
    """Run PAML for representatives of clades A and B in each of the SICO files, to calculate dN/dS."""
    #PAML runs inside a temporary folder, to prevent interference with simultaneous runs
    paml_dir = tempfile.mkdtemp(prefix = 'paml_run_')

    #Pick the first genomes as representatives for each clade
    representative_a = genomes_a[0]['RefSeq project ID']
    representative_b = genomes_b[0]['RefSeq project ID']

    log.info('Running PAML for {0} aligned and trimmed SICOs'.format(len(sico_files)))
    return [_run_yn00(paml_dir, representative_a, representative_b, sico_file) for sico_file in sico_files]

YN00 = '/projects/divergence/software/paml44/bin/yn00'

def _run_yn00(paml_dir, repr_id_a, repr_id_b, sico_file):
    """Run yn00 from PAML for selected sequence records from sico_file, returning main nexus output file."""
    #Find sequences from above chosen clade representatives in each SICO file
    seqr_a = None
    seqr_b = None
    for seqrecord in SeqIO.parse(sico_file, 'fasta', alphabet = generic_dna):
        refseq_id = seqrecord.id.split('|')[0]
        if refseq_id == repr_id_a:
            seqr_a = seqrecord
        elif refseq_id == repr_id_b:
            seqr_b = seqrecord
        if seqr_a and seqr_b:
            break
    else:
        assert False, '{0}: Both seqr_a & seqr_b should have been set\n{1}\n{2}'.format(sico_file, seqr_a, seqr_b)

    #Write the representative sequence records out to file in yn00 compatible format
    sico = os.path.splitext(os.path.split(sico_file)[1])[0]
    yn00_dir = create_directory(sico, inside_dir = paml_dir)
    nexus_file = os.path.join(yn00_dir, sico + '.nexus')
    _write_nexus_file(seqr_a, seqr_b, nexus_file)

    #Generate yn00 configuration file
    output_file = os.path.join(yn00_dir, 'yn' + sico)
    config_file = os.path.join(yn00_dir, 'yn00.ctl')
    _write_config_file(nexus_file, output_file, config_file)

    #Run yn00
    command = [YN00, os.path.split(config_file)[1]]
    check_call(command, cwd = yn00_dir, stdout = open('/dev/null', mode = 'w'), stderr = STDOUT)

    assert os.path.isfile(output_file) and os.path.getsize(output_file), 'Expected some content in ' + output_file
    return output_file

def _write_nexus_file(seqr_a, seqr_b, nexus_file):
    """Write representative sequences out to a file in the PAML compatible nexus format."""
    nexus_contents = """
#NEXUS
begin data;
   dimensions ntax={ntax} nchar={nchar};
   format datatype=dna missing=? gap=-;
matrix
clade_a  {seqa}
clade_b  {seqb}
;
end;
""".format(ntax = 2, nchar = len(seqr_a), seqa = str(seqr_a.seq), seqb = str(seqr_b.seq))
    with open(nexus_file, mode = 'w') as write_handle:
        #SeqIO.write([seqr_a, seqr_b], write_handle, 'nexus')
        write_handle.write(nexus_contents)

def _write_config_file(nexus_file, output_file, config_file):
    """Write a yn00 configuration file using relative paths to the nexus file and output file."""
    config_contents = """
   seqfile = {0} * sequence data file name
   outfile = {1} * main result file
   verbose = 0  * 1: detailed output (list sequences), 0: concise output

     icode = 0  * 0:universal code; 1:mammalian mt; 2-10:see below

 weighting = 0  * weighting pathways between codons (0/1)?
commonf3x4 = 0  * use one set of codon freqs for all pairs (0/1)?
*    ndata = 1

* Genetic codes: 0:universal, 1:mammalian mt., 2:yeast mt., 3:mold mt.,
* 4: invertebrate mt., 5: ciliate nuclear, 6: echinoderm mt.,
* 7: euplotid mt., 8: alternative yeast nu. 9: ascidian mt.,
* 10: blepharisma nu.
* These codes correspond to transl_table 1 to 11 of GENEBANK.
""".format(os.path.split(nexus_file)[1], os.path.split(output_file)[1])
    with open(config_file, mode = 'w') as write_handle:
        write_handle.write(config_contents)

def main(args):
    """Main function called when run from command line or as part of pipeline."""

    def _parse_options(args):
        """Use getopt to parse command line argument options"""

        def _usage():
            """Print _usage information"""
            print """
Usage: run_paml.py
--genomes-a=FILE    file with RefSeq id from complete genomes table on each line for clade A
--genomes-b=FILE    file with RefSeq id from complete genomes table on each line for clade B
--sico-zip=FILE     archive of aligned & trimmed single copy orthologous (SICO) genes
--paml-zip=FILE     destination file path for archive of PAML output per SICO gene
"""

        options = ['genomes-a', 'genomes-b', 'sico-zip', 'paml-zip']
        try:
            #postfix '=' to indicate options require an argument
            long_options = [opt + '=' for opt in options]
            tuples = getopt.getopt(args, '', long_options)[0]
            arguments = dict((opt[2:], value) for opt, value in tuples)
        except getopt.GetoptError as err:
            print str(err)
            _usage()
            sys.exit(1)

        #Ensure all arguments were provided
        for opt in options:
            if opt not in arguments:
                print 'Mandatory argument {0} not provided'.format(opt)
                _usage()
                sys.exit(1)

        #Retrieve & return file paths from dictionary
        return [arguments[option] for option in options]

    genome_a_ids_file, genome_b_ids_file, sico_zip, paml_zip = _parse_options(args)

    #Parse file containing RefSeq project IDs & retrieve associated genome dictionaries from complete genomes table
    genomes_a = select_genomes_from_file(genome_a_ids_file)
    genomes_b = select_genomes_from_file(genome_b_ids_file)

    #Extract files from zip archive
    temp_dir = tempfile.mkdtemp()
    sico_files = extract_archive_of_files(sico_zip, temp_dir)

    #Actually run cleanup
    paml_files = run_paml(genomes_a, genomes_b, sico_files)

    #Write the produced files to command line argument filenames
    create_archive_of_files(paml_zip, paml_files)

    #Exit after a comforting log message
    log.info("Produced: \n%s", paml_zip)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8
from __future__ import unicode_literals, absolute_import

import os
import sys

import django
from django.conf import settings
from django.test.utils import get_runner

def run_tests(*test_args):
    if not test_args:
        test_args = ['tests']

    os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings'
    django.setup()
    TestRunner = get_runner(settings)
    test_runner = TestRunner()
    failures = test_runner.run_tests(test_args)
    sys.exit(bool(failures))

if __name__ == '__main__':
    run_tests(*sys.argv[1:])
<commit_msg>#2 no longer needed<commit_after><|endoftext|>


<commit_before>from copy import copy
from typing import Dict
from unittest import TestCase

from eventsourcing.application.snapshotting import SnapshottingApplication
from eventsourcing.application.sqlalchemy import SQLAlchemyApplication
from eventsourcing.domain.model.aggregate import BaseAggregateRoot
from eventsourcing.domain.model.events import DomainEvent
from eventsourcing.whitehead import TEntity

class TestUpcastable(TestCase):
    """
    This test case checks that event state can be updated.
    """

    def test_upcast_recorded_state(self):
        # Upcast from original version to original version.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = {"a": 1}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 1)

        # Upcast from original version to version 1.
        UpcastableEventFixture.__class_version__ = 1
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 10000)

        # Upcast from original version to version 2.
        UpcastableEventFixture.__class_version__ = 2
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 1000)

        # Upcast from version 1 to version 2.
        state_v1 = {"a": 10000, "__class_version__": 1}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v1)
        self.assertEqual(obj_state["a"], 1000)

        # Upcast from version 2 to version 2.
        state_v2 = {"a": 1000, "__class_version__": 2}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v2)
        self.assertEqual(obj_state["a"], 1000)

    def test_event_state_has_class_version(self):
        # Construct original version.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = UpcastableEventFixture(a=1).__dict__
        # Check state doesn't have '__class_version__' attribute.
        self.assertEqual(state_v0, {"a": 1})

        # Construct version 1.
        UpcastableEventFixture.__class_version__ = 1
        state_v1 = UpcastableEventFixture(a=1).__dict__
        # Check state does have '__class_version__' attribute.
        self.assertEqual(state_v1, {"a": 1, "__class_version__": 1})

        # Construct version 2.
        UpcastableEventFixture.__class_version__ = 2
        state_v2 = UpcastableEventFixture(a=1).__dict__
        # Check state does have '__class_version__' attribute.
        self.assertEqual(state_v2, {"a": 1, "__class_version__": 2})

    def test_upcast_event_state(self):
        # Construct state with original version of the event class.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = UpcastableEventFixture(a=1).__dict__
        self.assertEqual(state_v0["a"], 1)

        # Construct state with version 1 of the event class.
        UpcastableEventFixture.__class_version__ = 1
        state_v1 = UpcastableEventFixture(a=10000).__dict__
        self.assertEqual(state_v1["a"], 10000)

        # Check original version is correctly upcast to version 1.
        state_v1_from_v0 = UpcastableEventFixture.__upcast_state__(copy(state_v0))
        self.assertEqual(state_v1, state_v1_from_v0)
        self.assertEqual(state_v0["a"], 1)

        # Check version 1 is correctly upcast to version 1.
        state_v1_from_v1 = UpcastableEventFixture.__upcast_state__(copy(state_v1))
        self.assertEqual(state_v1, state_v1_from_v1)
        self.assertEqual(state_v1["a"], 10000)

        # Construct state with version 2 of the event class.
        UpcastableEventFixture.__class_version__ = 2
        state_v2 = UpcastableEventFixture(a=1000).__dict__

        # Check original version is correctly upcast to version 2.
        state_v2_from_v0 = UpcastableEventFixture.__upcast_state__(copy(state_v0))
        self.assertEqual(state_v2, state_v2_from_v0)

        # Check version 1 is correctly upcast to version 2.
        state_v2_from_v1 = UpcastableEventFixture.__upcast_state__(copy(state_v1))
        self.assertEqual(state_v2, state_v2_from_v1)
        self.assertEqual(state_v1["a"], 10000)
        self.assertEqual(state_v0["a"], 1)

        # Check version 2 is correctly upcast to version 2.
        state_v2_from_v2 = UpcastableEventFixture.__upcast_state__(copy(state_v2))
        self.assertEqual(state_v2, state_v2_from_v2)
        self.assertEqual(state_v2["a"], 1000)
        self.assertEqual(state_v1["a"], 10000)
        self.assertEqual(state_v0["a"], 1)

class UpcastableEventFixture(DomainEvent):
    @classmethod
    def __upcast__(cls, obj_state: Dict, class_version: int):
        if class_version == 0:
            obj_state["a"] *= 10000
        elif class_version == 1:
            obj_state["a"] /= 10
        return obj_state

class TestUpcastingActiveWhenStoringAndRetrievingEvents(TestCase):
    def test_reconstructing_aggregate_state_from_versioned_events(self):
        """
        Mimick evolution of an event class, from original to version 2.

        Could/should just test reconstruct_object() calls __upcast_state__,
        but this test is decoupled from implementation, and simply checks
        upcasting is active when storing and retrieving aggregate state.
        """
        app = SQLAlchemyApplication(persist_event_type=BaseAggregateRoot.Event)
        with app:
            my_aggregate = MultiVersionAggregateFixture.__create__()
            # Trigger without any kwargs (original behaviour).
            my_aggregate.trigger()
            my_aggregate.__save__()

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)  # still uninitialised
            self.assertEqual(copy.units, None)  # still uninitialised

            # Increase the version of the event class (introduce 'value' attribute).
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )

            # Check v1 code can read original event version (backwards compatibility).
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 0)  # gets default
            self.assertEqual(copy.units, None)  # still uninitialised

            # Trigger with value kwarg (first version behaviour).
            my_aggregate.trigger(value=10)
            my_aggregate.__save__()

            # Check v1 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)  # gets set value
            self.assertEqual(copy.units, None)  # still uninitialised

            # Increase the version of the event class.
            MultiVersionAggregateFixture.Triggered.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v2
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v2
            )

            # Check v2 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, "")

            # Trigger with value and units kwargs (second version behaviour).
            my_aggregate.trigger(value=20, units="mm")
            my_aggregate.__save__()

            # Check v2 code can read original, first and second event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 20)
            self.assertEqual(copy.units, "mm")

            # Check original code reads later event versions (forwards compatibility).
            MultiVersionAggregateFixture.Triggered.__class_version__ = 0
            del MultiVersionAggregateFixture.Triggered.__upcast__
            del MultiVersionAggregateFixture.Triggered.mutate
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)  # still uninitialised
            self.assertEqual(copy.units, None)  # still uninitialised

            # Check v1 code reads later versions.
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 20)  # observes event attribute
            self.assertEqual(copy.units, None)  # still uninitialised

    def test_reconstructing_aggregate_state_from_snapshot(self):
        """
        Mimick evolution of an event class, from original to version 2.

        Could/should just test reconstruct_object() calls __upcast_state__,
        but this test is decoupled from implementation, and simply checks
        upcasting is active when storing and retrieving aggregate state.
        """
        app = SnapshottingApplication.mixin(SQLAlchemyApplication)(
            persist_event_type=BaseAggregateRoot.Event,

        )
        with app:
            my_aggregate = MultiVersionAggregateFixture.__create__()
            # Trigger without any kwargs (original behaviour).
            my_aggregate.trigger()
            my_aggregate.__save__()

            # Take a snapshot.
            app.repository.take_snapshot(my_aggregate.id)

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)
            self.assertEqual(copy.units, None)

            # Increase the version of the aggregate and event class.
            MultiVersionAggregateFixture.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )

            # Check v1 code upcasts snapshot (which doesnt have 'value').
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 0)  # gets default
            self.assertEqual(copy.units, None)

            # Trigger with value kwarg (first version behaviour).
            my_aggregate.trigger(value=10)
            my_aggregate.__save__()

            # Check v1 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, None)

            # Take a snapshot.
            app.repository.take_snapshot(my_aggregate.id)

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, None)

            # Increase the version of the aggregate and event class.
            MultiVersionAggregateFixture.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v2
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v2
            )

            # Check v2 code upcasts snapshot (which doesnt have 'units').
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, '') # gets default

class MultiVersionAggregateFixture(BaseAggregateRoot):
    DEFAULT_VALUE = 0
    DEFAULT_UNITS = ""

    @classmethod
    def __upcast__(cls, obj_state: Dict, class_version: int) -> Dict:
        if class_version == 0:
            obj_state['value'] = cls.DEFAULT_VALUE
        elif class_version == 1:
            obj_state['units'] = cls.DEFAULT_UNITS

        return obj_state

    def __init__(self, **kwargs):
        super(MultiVersionAggregateFixture, self).__init__(**kwargs)
        self.value = None
        self.units = None

    def trigger(self, **kwargs):
        self.__trigger_event__(self.Triggered, **kwargs)

    class Triggered(BaseAggregateRoot.Event):
        @property
        def value(self):
            return self.__dict__["value"]

        @property
        def units(self):
            return self.__dict__["units"]

        def mutate_v1(self, obj: TEntity) -> None:
            obj.value = self.value

        def mutate_v2(self, obj: TEntity) -> None:
            obj.value = self.value
            obj.units = self.units

        @classmethod
        def upcast_v1(cls, obj_state, class_version) -> None:
            if class_version == 0:
                obj_state["value"] = MultiVersionAggregateFixture.DEFAULT_VALUE
            return obj_state

        @classmethod
        def upcast_v2(cls, obj_state, class_version) -> None:
            if class_version == 0:
                obj_state["value"] = MultiVersionAggregateFixture.DEFAULT_VALUE
            elif class_version == 1:
                obj_state["units"] = MultiVersionAggregateFixture.DEFAULT_UNITS
            return obj_state
<commit_msg>Fixed event versioning test (need to reset state of the fixture class).<commit_after>from copy import copy
from typing import Dict
from unittest import TestCase

from eventsourcing.application.snapshotting import SnapshottingApplication
from eventsourcing.application.sqlalchemy import SQLAlchemyApplication
from eventsourcing.domain.model.aggregate import BaseAggregateRoot
from eventsourcing.domain.model.events import DomainEvent
from eventsourcing.whitehead import TEntity

class TestUpcastable(TestCase):
    """
    This test case checks that event state can be updated.
    """

    def test_upcast_recorded_state(self):
        # Upcast from original version to original version.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = {"a": 1}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 1)

        # Upcast from original version to version 1.
        UpcastableEventFixture.__class_version__ = 1
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 10000)

        # Upcast from original version to version 2.
        UpcastableEventFixture.__class_version__ = 2
        obj_state = UpcastableEventFixture.__upcast_state__(state_v0)
        self.assertEqual(obj_state["a"], 1000)

        # Upcast from version 1 to version 2.
        state_v1 = {"a": 10000, "__class_version__": 1}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v1)
        self.assertEqual(obj_state["a"], 1000)

        # Upcast from version 2 to version 2.
        state_v2 = {"a": 1000, "__class_version__": 2}
        obj_state = UpcastableEventFixture.__upcast_state__(state_v2)
        self.assertEqual(obj_state["a"], 1000)

    def test_event_state_has_class_version(self):
        # Construct original version.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = UpcastableEventFixture(a=1).__dict__
        # Check state doesn't have '__class_version__' attribute.
        self.assertEqual(state_v0, {"a": 1})

        # Construct version 1.
        UpcastableEventFixture.__class_version__ = 1
        state_v1 = UpcastableEventFixture(a=1).__dict__
        # Check state does have '__class_version__' attribute.
        self.assertEqual(state_v1, {"a": 1, "__class_version__": 1})

        # Construct version 2.
        UpcastableEventFixture.__class_version__ = 2
        state_v2 = UpcastableEventFixture(a=1).__dict__
        # Check state does have '__class_version__' attribute.
        self.assertEqual(state_v2, {"a": 1, "__class_version__": 2})

    def test_upcast_event_state(self):
        # Construct state with original version of the event class.
        UpcastableEventFixture.__class_version__ = 0
        state_v0 = UpcastableEventFixture(a=1).__dict__
        self.assertEqual(state_v0["a"], 1)

        # Construct state with version 1 of the event class.
        UpcastableEventFixture.__class_version__ = 1
        state_v1 = UpcastableEventFixture(a=10000).__dict__
        self.assertEqual(state_v1["a"], 10000)

        # Check original version is correctly upcast to version 1.
        state_v1_from_v0 = UpcastableEventFixture.__upcast_state__(copy(state_v0))
        self.assertEqual(state_v1, state_v1_from_v0)
        self.assertEqual(state_v0["a"], 1)

        # Check version 1 is correctly upcast to version 1.
        state_v1_from_v1 = UpcastableEventFixture.__upcast_state__(copy(state_v1))
        self.assertEqual(state_v1, state_v1_from_v1)
        self.assertEqual(state_v1["a"], 10000)

        # Construct state with version 2 of the event class.
        UpcastableEventFixture.__class_version__ = 2
        state_v2 = UpcastableEventFixture(a=1000).__dict__

        # Check original version is correctly upcast to version 2.
        state_v2_from_v0 = UpcastableEventFixture.__upcast_state__(copy(state_v0))
        self.assertEqual(state_v2, state_v2_from_v0)

        # Check version 1 is correctly upcast to version 2.
        state_v2_from_v1 = UpcastableEventFixture.__upcast_state__(copy(state_v1))
        self.assertEqual(state_v2, state_v2_from_v1)
        self.assertEqual(state_v1["a"], 10000)
        self.assertEqual(state_v0["a"], 1)

        # Check version 2 is correctly upcast to version 2.
        state_v2_from_v2 = UpcastableEventFixture.__upcast_state__(copy(state_v2))
        self.assertEqual(state_v2, state_v2_from_v2)
        self.assertEqual(state_v2["a"], 1000)
        self.assertEqual(state_v1["a"], 10000)
        self.assertEqual(state_v0["a"], 1)

class UpcastableEventFixture(DomainEvent):
    @classmethod
    def __upcast__(cls, obj_state: Dict, class_version: int):
        if class_version == 0:
            obj_state["a"] *= 10000
        elif class_version == 1:
            obj_state["a"] /= 10
        return obj_state

class TestUpcastingActiveWhenStoringAndRetrievingEvents(TestCase):

    def tearDown(self) -> None:
        # Reset class versions (in case running in suite).
        MultiVersionAggregateFixture.Triggered.__class_version__ = 0
        MultiVersionAggregateFixture.__class_version__ = 0
        del(MultiVersionAggregateFixture.Triggered.__upcast__)
        del(MultiVersionAggregateFixture.Triggered.mutate)

    def test_reconstructing_aggregate_state_from_versioned_events(self):
        """
        Mimick evolution of an event class, from original to version 2.

        Could/should just test reconstruct_object() calls __upcast_state__,
        but this test is decoupled from implementation, and simply checks
        upcasting is active when storing and retrieving aggregate state.
        """
        app = SQLAlchemyApplication(persist_event_type=BaseAggregateRoot.Event)
        with app:
            my_aggregate = MultiVersionAggregateFixture.__create__()
            # Trigger without any kwargs (original behaviour).
            my_aggregate.trigger()
            my_aggregate.__save__()

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)  # still uninitialised
            self.assertEqual(copy.units, None)  # still uninitialised

            # Increase the version of the event class (introduce 'value' attribute).
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )

            # Check v1 code can read original event version (backwards compatibility).
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 0)  # gets default
            self.assertEqual(copy.units, None)  # still uninitialised

            # Trigger with value kwarg (first version behaviour).
            my_aggregate.trigger(value=10)
            my_aggregate.__save__()

            # Check v1 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)  # gets set value
            self.assertEqual(copy.units, None)  # still uninitialised

            # Increase the version of the event class.
            MultiVersionAggregateFixture.Triggered.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v2
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v2
            )

            # Check v2 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, "")

            # Trigger with value and units kwargs (second version behaviour).
            my_aggregate.trigger(value=20, units="mm")
            my_aggregate.__save__()

            # Check v2 code can read original, first and second event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 20)
            self.assertEqual(copy.units, "mm")

            # Check original code reads later event versions (forwards compatibility).
            MultiVersionAggregateFixture.Triggered.__class_version__ = 0
            del MultiVersionAggregateFixture.Triggered.__upcast__
            del MultiVersionAggregateFixture.Triggered.mutate
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)  # still uninitialised
            self.assertEqual(copy.units, None)  # still uninitialised

            # Check v1 code reads later versions.
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 20)  # observes event attribute
            self.assertEqual(copy.units, None)  # still uninitialised

    def test_reconstructing_aggregate_state_from_snapshot(self):
        """
        Mimick evolution of an event class, from original to version 2.

        Could/should just test reconstruct_object() calls __upcast_state__,
        but this test is decoupled from implementation, and simply checks
        upcasting is active when storing and retrieving aggregate state.
        """
        app = SnapshottingApplication.mixin(SQLAlchemyApplication)(
            persist_event_type=BaseAggregateRoot.Event,

        )
        with app:
            my_aggregate = MultiVersionAggregateFixture.__create__()
            # Trigger without any kwargs (original behaviour).
            my_aggregate.trigger()
            my_aggregate.__save__()

            # Take a snapshot.
            app.repository.take_snapshot(my_aggregate.id)

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, None)
            self.assertEqual(copy.units, None)

            # Increase the version of the aggregate and event class.
            MultiVersionAggregateFixture.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__class_version__ = 1
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v1
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v1
            )

            # Check v1 code upcasts snapshot (which doesnt have 'value').
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 0)  # gets default
            self.assertEqual(copy.units, None)

            # Trigger with value kwarg (first version behaviour).
            my_aggregate.trigger(value=10)
            my_aggregate.__save__()

            # Check v1 code can read original and first event versions.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, None)

            # Take a snapshot.
            app.repository.take_snapshot(my_aggregate.id)

            # Check the default values are used for value and units.
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, None)

            # Increase the version of the aggregate and event class.
            MultiVersionAggregateFixture.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__class_version__ = 2
            MultiVersionAggregateFixture.Triggered.__upcast__ = (
                MultiVersionAggregateFixture.Triggered.upcast_v2
            )
            MultiVersionAggregateFixture.Triggered.mutate = (
                MultiVersionAggregateFixture.Triggered.mutate_v2
            )

            # Check v2 code upcasts snapshot (which doesnt have 'units').
            copy = app.repository[my_aggregate.id]
            assert isinstance(copy, MultiVersionAggregateFixture)
            self.assertEqual(copy.value, 10)
            self.assertEqual(copy.units, '') # gets default

class MultiVersionAggregateFixture(BaseAggregateRoot):
    DEFAULT_VALUE = 0
    DEFAULT_UNITS = ""

    @classmethod
    def __upcast__(cls, obj_state: Dict, class_version: int) -> Dict:
        if class_version == 0:
            obj_state['value'] = cls.DEFAULT_VALUE
        elif class_version == 1:
            obj_state['units'] = cls.DEFAULT_UNITS

        return obj_state

    def __init__(self, **kwargs):
        super(MultiVersionAggregateFixture, self).__init__(**kwargs)
        self.value = None
        self.units = None

    def trigger(self, **kwargs):
        self.__trigger_event__(self.Triggered, **kwargs)

    class Triggered(BaseAggregateRoot.Event):
        @property
        def value(self):
            return self.__dict__["value"]

        @property
        def units(self):
            return self.__dict__["units"]

        def mutate_v1(self, obj: TEntity) -> None:
            obj.value = self.value

        def mutate_v2(self, obj: TEntity) -> None:
            obj.value = self.value
            obj.units = self.units

        @classmethod
        def upcast_v1(cls, obj_state, class_version) -> None:
            if class_version == 0:
                obj_state["value"] = MultiVersionAggregateFixture.DEFAULT_VALUE
            return obj_state

        @classmethod
        def upcast_v2(cls, obj_state, class_version) -> None:
            if class_version == 0:
                obj_state["value"] = MultiVersionAggregateFixture.DEFAULT_VALUE
            elif class_version == 1:
                obj_state["units"] = MultiVersionAggregateFixture.DEFAULT_UNITS
            return obj_state
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import unittest

from huey import tests

def _requirements_installed():
    try:
        import django
        return True
    except Exception:
        return False

def run_tests(*test_args):
    suite = unittest.TestLoader().loadTestsFromModule(tests)
    result = unittest.TextTestRunner(verbosity=1).run(suite)
    if os.path.exists('huey.db'):
        os.unlink('huey.db')
    if result.failures:
        sys.exit(1)
    elif result.errors:
        sys.exit(2)

def run_django_tests(*test_args):
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "huey.contrib.djhuey.tests.settings")
    from django.core.management import execute_from_command_line
    args = sys.argv
    args.insert(1, "test")
    args.append('huey.contrib.djhuey.tests')
    execute_from_command_line(args)

if __name__ == '__main__':
    if not _requirements_installed():
        print('Requirements are not installed. Run "pip install -r test_requirements.txt" to install all dependencies.')
        sys.exit(2)
    run_tests(*sys.argv[1:])
    run_django_tests(*sys.argv[1:])
    sys.exit(0)

<commit_msg>Allow running tests without Django.<commit_after>#!/usr/bin/env python
import os
import sys
import unittest

from huey import tests

def _requirements_installed():
    try:
        import django
        return True
    except Exception:
        return False

def run_tests(*test_args):
    suite = unittest.TestLoader().loadTestsFromModule(tests)
    result = unittest.TextTestRunner(verbosity=1).run(suite)
    if os.path.exists('huey.db'):
        os.unlink('huey.db')
    if result.failures:
        sys.exit(1)
    elif result.errors:
        sys.exit(2)

def run_django_tests(*test_args):
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "huey.contrib.djhuey.tests.settings")
    from django.core.management import execute_from_command_line
    args = sys.argv
    args.insert(1, "test")
    args.append('huey.contrib.djhuey.tests')
    execute_from_command_line(args)

if __name__ == '__main__':
    run_tests(*sys.argv[1:])
    if _requirements_installed():
        run_django_tests(*sys.argv[1:])
    else:
        print('Django not installed, skipping Django tests.')
    sys.exit(0)

<|endoftext|>


<commit_before># imports
import json
import time
import pickle
import scipy.misc
import skimage.io
import caffe

import numpy as np
import os.path as osp

from xml.dom import minidom
from random import shuffle
from threading import Thread
from PIL import Image

from tools import SimpleTransformer

class PascalMultilabelDataLayerSync(caffe.Layer):

    """
    This is a simple syncronous datalayer for training a multilabel model on
    PASCAL.
    """

    def setup(self, bottom, top):

        self.top_names = ['data', 'label']

        # === Read input parameters ===

        # params is a python dictionary with layer parameters.
        params = eval(self.param_str)

        # Check the paramameters for validity.
        check_params(params)

        # store input as class variables
        self.batch_size = params['batch_size']

        # Create a batch loader to load the images.
        self.batch_loader = BatchLoader(params, None)

        # === reshape tops ===
        # since we use a fixed input image size, we can shape the data layer
        # once. Else, we'd have to do it in the reshape call.
        top[0].reshape(
            self.batch_size, 3, params['im_shape'][0], params['im_shape'][1])
        # Note the 20 channels (because PASCAL has 20 classes.)
        top[1].reshape(self.batch_size, 20)

        print_info("PascalMultilabelDataLayerSync", params)

    def forward(self, bottom, top):
        """
        Load data.
        """
        for itt in range(self.batch_size):
            # Use the batch loader to load the next image.
            im, multilabel = self.batch_loader.load_next_image()

            # Add directly to the caffe data layer
            top[0].data[itt, ...] = im
            top[1].data[itt, ...] = multilabel

    def reshape(self, bottom, top):
        """
        There is no need to reshape the data, since the input is of fixed size
        (rows and columns)
        """
        pass

    def backward(self, top, propagate_down, bottom):
        """
        These layers does not back propagate
        """
        pass

class BatchLoader(object):

    """
    This class abstracts away the loading of images.
    Images can either be loaded singly, or in a batch. The latter is used for
    the asyncronous data layer to preload batches while other processing is
    performed.
    """

    def __init__(self, params, result):
        self.result = result
        self.batch_size = params['batch_size']
        self.pascal_root = params['pascal_root']
        self.im_shape = params['im_shape']
        # get list of image indexes.
        list_file = params['split'] + '.txt'
        self.indexlist = [line.rstrip('\n') for line in open(
            osp.join(self.pascal_root, 'ImageSets/Main', list_file))]
        self._cur = 0  # current image
        # this class does some simple data-manipulations
        self.transformer = SimpleTransformer()

        print "BatchLoader initialized with {} images".format(
            len(self.indexlist))

    def load_next_image(self):
        """
        Load the next image in a batch.
        """
        # Did we finish an epoch?
        if self._cur == len(self.indexlist):
            self._cur = 0
            shuffle(self.indexlist)

        # Load an image
        index = self.indexlist[self._cur]  # Get the image index
        image_file_name = index + '.jpg'
        im = np.asarray(Image.open(
            osp.join(self.pascal_root, 'JPEGImages', image_file_name)))
        im = scipy.misc.imresize(im, self.im_shape)  # resize

        # do a simple horizontal flip as data augmentation
        flip = np.random.choice(2)*2-1
        im = im[:, ::flip, :]

        # Load and prepare ground truth
        multilabel = np.zeros(20).astype(np.float32)
        anns = load_pascal_annotation(index, self.pascal_root)
        for label in anns['gt_classes']:
            # in the multilabel problem we don't care how MANY instances
            # there are of each class. Only if they are present.
            # The "-1" is b/c we are not interested in the background
            # class.
            multilabel[label - 1] = 1

        self._cur += 1
        return self.transformer.preprocess(im), multilabel

def load_pascal_annotation(index, pascal_root):
    """
    This code is borrowed from Ross Girshick's FAST-RCNN code
    (https://github.com/rbgirshick/fast-rcnn).
    It parses the PASCAL .xml metadata files.
    See publication for further details: (http://arxiv.org/abs/1504.08083).

    Thanks Ross!

    """
    classes = ('__background__',  # always index 0
               'aeroplane', 'bicycle', 'bird', 'boat',
               'bottle', 'bus', 'car', 'cat', 'chair',
                         'cow', 'diningtable', 'dog', 'horse',
                         'motorbike', 'person', 'pottedplant',
                         'sheep', 'sofa', 'train', 'tvmonitor')
    class_to_ind = dict(zip(classes, xrange(21)))

    filename = osp.join(pascal_root, 'Annotations', index + '.xml')
    # print 'Loading: {}'.format(filename)

    def get_data_from_tag(node, tag):
        return node.getElementsByTagName(tag)[0].childNodes[0].data

    with open(filename) as f:
        data = minidom.parseString(f.read())

    objs = data.getElementsByTagName('object')
    num_objs = len(objs)

    boxes = np.zeros((num_objs, 4), dtype=np.uint16)
    gt_classes = np.zeros((num_objs), dtype=np.int32)
    overlaps = np.zeros((num_objs, 21), dtype=np.float32)

    # Load object bounding boxes into a data frame.
    for ix, obj in enumerate(objs):
        # Make pixel indexes 0-based
        x1 = float(get_data_from_tag(obj, 'xmin')) - 1
        y1 = float(get_data_from_tag(obj, 'ymin')) - 1
        x2 = float(get_data_from_tag(obj, 'xmax')) - 1
        y2 = float(get_data_from_tag(obj, 'ymax')) - 1
        cls = class_to_ind[
            str(get_data_from_tag(obj, "name")).lower().strip()]
        boxes[ix, :] = [x1, y1, x2, y2]
        gt_classes[ix] = cls
        overlaps[ix, cls] = 1.0

    overlaps = scipy.sparse.csr_matrix(overlaps)

    return {'boxes': boxes,
            'gt_classes': gt_classes,
            'gt_overlaps': overlaps,
            'flipped': False,
            'index': index}

def check_params(params):
    """
    A utility function to check the parameters for the data layers.
    """
    assert 'split' in params.keys(
    ), 'Params must include split (train, val, or test).'

    required = ['batch_size', 'pascal_root', 'im_shape']
    for r in required:
        assert r in params.keys(), 'Params must include {}'.format(r)

def print_info(name, params):
    """
    Ouput some info regarding the class
    """
    print "{} initialized for split: {}, with bs: {}, im_shape: {}.".format(
        name,
        params['split'],
        params['batch_size'],
        params['im_shape'])
<commit_msg>fix typo in pascal_multilabel_datalayers.py<commit_after># imports
import json
import time
import pickle
import scipy.misc
import skimage.io
import caffe

import numpy as np
import os.path as osp

from xml.dom import minidom
from random import shuffle
from threading import Thread
from PIL import Image

from tools import SimpleTransformer

class PascalMultilabelDataLayerSync(caffe.Layer):

    """
    This is a simple synchronous datalayer for training a multilabel model on
    PASCAL.
    """

    def setup(self, bottom, top):

        self.top_names = ['data', 'label']

        # === Read input parameters ===

        # params is a python dictionary with layer parameters.
        params = eval(self.param_str)

        # Check the parameters for validity.
        check_params(params)

        # store input as class variables
        self.batch_size = params['batch_size']

        # Create a batch loader to load the images.
        self.batch_loader = BatchLoader(params, None)

        # === reshape tops ===
        # since we use a fixed input image size, we can shape the data layer
        # once. Else, we'd have to do it in the reshape call.
        top[0].reshape(
            self.batch_size, 3, params['im_shape'][0], params['im_shape'][1])
        # Note the 20 channels (because PASCAL has 20 classes.)
        top[1].reshape(self.batch_size, 20)

        print_info("PascalMultilabelDataLayerSync", params)

    def forward(self, bottom, top):
        """
        Load data.
        """
        for itt in range(self.batch_size):
            # Use the batch loader to load the next image.
            im, multilabel = self.batch_loader.load_next_image()

            # Add directly to the caffe data layer
            top[0].data[itt, ...] = im
            top[1].data[itt, ...] = multilabel

    def reshape(self, bottom, top):
        """
        There is no need to reshape the data, since the input is of fixed size
        (rows and columns)
        """
        pass

    def backward(self, top, propagate_down, bottom):
        """
        These layers does not back propagate
        """
        pass

class BatchLoader(object):

    """
    This class abstracts away the loading of images.
    Images can either be loaded singly, or in a batch. The latter is used for
    the asyncronous data layer to preload batches while other processing is
    performed.
    """

    def __init__(self, params, result):
        self.result = result
        self.batch_size = params['batch_size']
        self.pascal_root = params['pascal_root']
        self.im_shape = params['im_shape']
        # get list of image indexes.
        list_file = params['split'] + '.txt'
        self.indexlist = [line.rstrip('\n') for line in open(
            osp.join(self.pascal_root, 'ImageSets/Main', list_file))]
        self._cur = 0  # current image
        # this class does some simple data-manipulations
        self.transformer = SimpleTransformer()

        print "BatchLoader initialized with {} images".format(
            len(self.indexlist))

    def load_next_image(self):
        """
        Load the next image in a batch.
        """
        # Did we finish an epoch?
        if self._cur == len(self.indexlist):
            self._cur = 0
            shuffle(self.indexlist)

        # Load an image
        index = self.indexlist[self._cur]  # Get the image index
        image_file_name = index + '.jpg'
        im = np.asarray(Image.open(
            osp.join(self.pascal_root, 'JPEGImages', image_file_name)))
        im = scipy.misc.imresize(im, self.im_shape)  # resize

        # do a simple horizontal flip as data augmentation
        flip = np.random.choice(2)*2-1
        im = im[:, ::flip, :]

        # Load and prepare ground truth
        multilabel = np.zeros(20).astype(np.float32)
        anns = load_pascal_annotation(index, self.pascal_root)
        for label in anns['gt_classes']:
            # in the multilabel problem we don't care how MANY instances
            # there are of each class. Only if they are present.
            # The "-1" is b/c we are not interested in the background
            # class.
            multilabel[label - 1] = 1

        self._cur += 1
        return self.transformer.preprocess(im), multilabel

def load_pascal_annotation(index, pascal_root):
    """
    This code is borrowed from Ross Girshick's FAST-RCNN code
    (https://github.com/rbgirshick/fast-rcnn).
    It parses the PASCAL .xml metadata files.
    See publication for further details: (http://arxiv.org/abs/1504.08083).

    Thanks Ross!

    """
    classes = ('__background__',  # always index 0
               'aeroplane', 'bicycle', 'bird', 'boat',
               'bottle', 'bus', 'car', 'cat', 'chair',
                         'cow', 'diningtable', 'dog', 'horse',
                         'motorbike', 'person', 'pottedplant',
                         'sheep', 'sofa', 'train', 'tvmonitor')
    class_to_ind = dict(zip(classes, xrange(21)))

    filename = osp.join(pascal_root, 'Annotations', index + '.xml')
    # print 'Loading: {}'.format(filename)

    def get_data_from_tag(node, tag):
        return node.getElementsByTagName(tag)[0].childNodes[0].data

    with open(filename) as f:
        data = minidom.parseString(f.read())

    objs = data.getElementsByTagName('object')
    num_objs = len(objs)

    boxes = np.zeros((num_objs, 4), dtype=np.uint16)
    gt_classes = np.zeros((num_objs), dtype=np.int32)
    overlaps = np.zeros((num_objs, 21), dtype=np.float32)

    # Load object bounding boxes into a data frame.
    for ix, obj in enumerate(objs):
        # Make pixel indexes 0-based
        x1 = float(get_data_from_tag(obj, 'xmin')) - 1
        y1 = float(get_data_from_tag(obj, 'ymin')) - 1
        x2 = float(get_data_from_tag(obj, 'xmax')) - 1
        y2 = float(get_data_from_tag(obj, 'ymax')) - 1
        cls = class_to_ind[
            str(get_data_from_tag(obj, "name")).lower().strip()]
        boxes[ix, :] = [x1, y1, x2, y2]
        gt_classes[ix] = cls
        overlaps[ix, cls] = 1.0

    overlaps = scipy.sparse.csr_matrix(overlaps)

    return {'boxes': boxes,
            'gt_classes': gt_classes,
            'gt_overlaps': overlaps,
            'flipped': False,
            'index': index}

def check_params(params):
    """
    A utility function to check the parameters for the data layers.
    """
    assert 'split' in params.keys(
    ), 'Params must include split (train, val, or test).'

    required = ['batch_size', 'pascal_root', 'im_shape']
    for r in required:
        assert r in params.keys(), 'Params must include {}'.format(r)

def print_info(name, params):
    """
    Output some info regarding the class
    """
    print "{} initialized for split: {}, with bs: {}, im_shape: {}.".format(
        name,
        params['split'],
        params['batch_size'],
        params['im_shape'])
<|endoftext|>


<commit_before>import numpy as np
import pytest

from devito.logger import info
from devito import norm
from examples.seismic.viscoacoustic import ViscoacousticWaveSolver
from examples.seismic import demo_model, setup_geometry, seismic_args

def viscoacoustic_setup(shape=(50, 50), spacing=(15.0, 15.0), tn=500., space_order=4,
                        nbl=40, preset='layers-viscoacoustic', kernel='sls',
                        time_order=2, **kwargs):
    model = demo_model(preset, space_order=space_order, shape=shape, nbl=nbl,
                       dtype=kwargs.pop('dtype', np.float32), spacing=spacing)

    # Source and receiver geometries
    geometry = setup_geometry(model, tn)

    # Create solver object to provide relevant operators
    solver = ViscoacousticWaveSolver(model, geometry, space_order=space_order,
                                     kernel=kernel, time_order=time_order, **kwargs)
    return solver

def run(shape=(50, 50), spacing=(20.0, 20.0), tn=1000.0,
        space_order=4, nbl=40, autotune=False, preset='layers-viscoacoustic',
        kernel='sls', time_order=2, **kwargs):

    solver = viscoacoustic_setup(shape=shape, spacing=spacing, nbl=nbl, tn=tn,
                                 space_order=space_order, preset=preset,
                                 kernel=kernel, time_order=time_order, **kwargs)
    info("Applying Forward")

    rec, p, summary = solver.forward(autotune=autotune)

    return (summary.gflopss, summary.oi, summary.timings, [rec, p.data])

@pytest.mark.parametrize('kernel, time_order, normrec, atol', [
    ('sls', 2, 684.4527, 1e-2),
    ('sls', 1, 133.8698, 1e-2),
    ('ren', 2, 677.7255, 1e-2),
    ('ren', 1, 129.3433, 1e-2),
    ('deng_mcmechan', 2, 673.0798, 1e-2),
    ('deng_mcmechan', 1, 131.8282, 1e-2),
])
def test_viscoacoustic(kernel, time_order, normrec, atol):
    _, _, _, [rec, _] = run(kernel=kernel, time_order=time_order)
    assert np.isclose(norm(rec), normrec, atol=atol, rtol=0)

@pytest.mark.parametrize('ndim', [1, 2, 3])
@pytest.mark.parametrize('kernel', ['sls', 'ren', 'deng_mcmechan'])
@pytest.mark.parametrize('time_order', [1, 2])
def test_viscoacoustic_stability(ndim, kernel, time_order):
    shape = tuple([11]*ndim)
    spacing = tuple([20]*ndim)
    _, _, _, [rec, _] = run(shape=shape, spacing=spacing, tn=20000.0, nbl=0,
                            kernel=kernel, time_order=time_order)
    assert np.isfinite(norm(rec))

if __name__ == "__main__":
    description = ("Example script for a set of viscoacoustic operators.")
    parser = seismic_args(description)
    parser.add_argument("-k", dest="kernel", default='sls',
                        choices=['sls', 'ren', 'deng_mcmechan'],
                        help="Choice of finite-difference kernel")
    parser.add_argument("-to", "--time_order", default=2,
                        type=int, help="Time order of the equation")
    args = parser.parse_args()
    # Preset parameters
    ndim = args.ndim
    shape = args.shape[:args.ndim]
    spacing = tuple(ndim * [10.0])
    tn = args.tn if args.tn > 0 else (750. if ndim < 3 else 1250.)
    preset = 'constant-viscoacoustic' if args.constant else 'layers-viscoacoustic'

    run(shape=shape, spacing=spacing, nbl=args.nbl, tn=tn, opt=args.opt,
        space_order=args.space_order, autotune=args.autotune, preset=preset,
        kernel=args.kernel, time_order=args.time_order)
<commit_msg>Examples: Fix assert values<commit_after>import numpy as np
import pytest

from devito.logger import info
from devito import norm
from examples.seismic.viscoacoustic import ViscoacousticWaveSolver
from examples.seismic import demo_model, setup_geometry, seismic_args

def viscoacoustic_setup(shape=(50, 50), spacing=(15.0, 15.0), tn=500., space_order=4,
                        nbl=40, preset='layers-viscoacoustic', kernel='sls',
                        time_order=2, **kwargs):
    model = demo_model(preset, space_order=space_order, shape=shape, nbl=nbl,
                       dtype=kwargs.pop('dtype', np.float32), spacing=spacing)

    # Source and receiver geometries
    geometry = setup_geometry(model, tn)

    # Create solver object to provide relevant operators
    solver = ViscoacousticWaveSolver(model, geometry, space_order=space_order,
                                     kernel=kernel, time_order=time_order, **kwargs)
    return solver

def run(shape=(50, 50), spacing=(20.0, 20.0), tn=1000.0,
        space_order=4, nbl=40, autotune=False, preset='layers-viscoacoustic',
        kernel='sls', time_order=2, **kwargs):

    solver = viscoacoustic_setup(shape=shape, spacing=spacing, nbl=nbl, tn=tn,
                                 space_order=space_order, preset=preset,
                                 kernel=kernel, time_order=time_order, **kwargs)
    info("Applying Forward")

    rec, p, summary = solver.forward(autotune=autotune)

    return (summary.gflopss, summary.oi, summary.timings, [rec, p.data])

@pytest.mark.parametrize('kernel, time_order, normrec, atol', [
    ('sls', 2, 684.4526, 1e-1),
    ('sls', 1, 133.8698, 1e-1),
    ('ren', 2, 677.7254, 1e-1),
    ('ren', 1, 129.3433, 1e-1),
    ('deng_mcmechan', 2, 673.0799, 1e-1),
    ('deng_mcmechan', 1, 131.8282, 1e-1),
])
def test_viscoacoustic(kernel, time_order, normrec, atol):
    _, _, _, [rec, _] = run(kernel=kernel, time_order=time_order)
    assert np.isclose(norm(rec), normrec, atol=atol, rtol=0)

@pytest.mark.parametrize('ndim', [2, 3])
@pytest.mark.parametrize('kernel', ['sls', 'ren', 'deng_mcmechan'])
@pytest.mark.parametrize('time_order', [1, 2])
def test_viscoacoustic_stability(ndim, kernel, time_order):
    shape = tuple([11]*ndim)
    spacing = tuple([20]*ndim)
    _, _, _, [rec, _] = run(shape=shape, spacing=spacing, tn=20000.0, nbl=0,
                            kernel=kernel, time_order=time_order)
    assert np.isfinite(norm(rec))

if __name__ == "__main__":
    description = ("Example script for a set of viscoacoustic operators.")
    parser = seismic_args(description)
    parser.add_argument("-k", dest="kernel", default='sls',
                        choices=['sls', 'ren', 'deng_mcmechan'],
                        help="Choice of finite-difference kernel")
    parser.add_argument("-to", "--time_order", default=2,
                        type=int, help="Time order of the equation")
    args = parser.parse_args()
    # Preset parameters
    ndim = args.ndim
    shape = args.shape[:args.ndim]
    spacing = tuple(ndim * [10.0])
    tn = args.tn if args.tn > 0 else (750. if ndim < 3 else 1250.)
    preset = 'constant-viscoacoustic' if args.constant else 'layers-viscoacoustic'

    run(shape=shape, spacing=spacing, nbl=args.nbl, tn=tn, opt=args.opt,
        space_order=args.space_order, autotune=args.autotune, preset=preset,
        kernel=args.kernel, time_order=args.time_order)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Standard library imports
import os
from os.path import dirname
from os.path import realpath
import sys

# Third party imports
import django

# Local application / specific library imports
from tests.settings import configure

def run(verbosity, *args):
    from django_nose import NoseTestSuiteRunner
    runner = NoseTestSuiteRunner(verbosity=verbosity)

    if not args:
        args = ['tests']
    num_failures = runner.run_tests(args)
    if num_failures:
        sys.exit(num_failures)

if __name__ == '__main__':
    args = sys.argv[1:]

    verbosity = 1
    if args:
        # If some args were specified, try to see if any nose options have
        # been specified. If they have, then don't set any.
        has_options = any(map(lambda x: x.startswith('--'), args))
        if has_options:
            # Remove options as nose will pick these up from sys.argv
            for arg in args:
                if arg.startswith('--verbosity'):
                    verbosity = int(arg[-1])
            args = [arg for arg in args if not arg.startswith('-')]

    # Give feedback on used versions
    sys.stderr.write('Using Python version {0} from {1}\n'.format(sys.version[:5], sys.executable))
    sys.stderr.write('Using Django version {0} from {1}\n'.format(
        django.get_version(),
        os.path.dirname(os.path.abspath(django.__file__)))
    )

    # Detect location and available modules
    module_root = dirname(realpath(__file__))

    # Configure Django and machina
    configure()
    if hasattr(django, 'setup'):
        # As outlined in the Django 1.7 release notes, it is now required
        # to explicitly initialize Django at the beginning if any standalone
        # script in order to properly support the Django "app registry".
        django.setup()

    # To infinity... and beyond!
    run(verbosity, *args)
<commit_msg>Fixed test runner<commit_after>#!/usr/bin/env python

# Standard library imports
import os
from os.path import dirname
from os.path import realpath
import sys

# Third party imports
import django

# Local application / specific library imports
from tests.settings import configure

def run(verbosity, *args):
    if django.VERSION < (1, 8):
        from django_nose import NoseTestSuiteRunner
        runner = NoseTestSuiteRunner(verbosity=verbosity)
    else:
        from django.test.runner import DiscoverRunner
        runner = DiscoverRunner(verbosity=verbosity)

    if not args:
        args = ['tests']
    num_failures = runner.run_tests(args)
    if num_failures:
        sys.exit(num_failures)

if __name__ == '__main__':
    args = sys.argv[1:]

    verbosity = 1
    if args:
        # If some args were specified, try to see if any nose options have
        # been specified. If they have, then don't set any.
        has_options = any(map(lambda x: x.startswith('--'), args))
        if has_options:
            # Remove options as nose will pick these up from sys.argv
            for arg in args:
                if arg.startswith('--verbosity'):
                    verbosity = int(arg[-1])
            args = [arg for arg in args if not arg.startswith('-')]

    # Give feedback on used versions
    sys.stderr.write('Using Python version {0} from {1}\n'.format(sys.version[:5], sys.executable))
    sys.stderr.write('Using Django version {0} from {1}\n'.format(
        django.get_version(),
        os.path.dirname(os.path.abspath(django.__file__)))
    )

    # Detect location and available modules
    module_root = dirname(realpath(__file__))

    # Configure Django and machina
    configure()
    if hasattr(django, 'setup'):
        # As outlined in the Django 1.7 release notes, it is now required
        # to explicitly initialize Django at the beginning if any standalone
        # script in order to properly support the Django "app registry".
        django.setup()

    # To infinity... and beyond!
    run(verbosity, *args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys

import django

from django.conf import settings

DEFAULT_SETTINGS = dict(
    INSTALLED_APPS=[
        "django.contrib.auth",
        "django.contrib.contenttypes",
        "django.contrib.sites",
        "account",
        "kaleo",
        "kaleo.tests"
    ],
    DATABASES={
        "default": {
            "ENGINE": "django.db.backends.sqlite3",
            "NAME": ":memory:",
        }
    },
    SITE_ID=1,
    ROOT_URLCONF="kaleo.tests.urls",
    SECRET_KEY="notasecret",
)

def runtests(*test_args):
    if not settings.configured:
        settings.configure(**DEFAULT_SETTINGS)

    # Compatibility with Django 1.7's stricter initialization
    if hasattr(django, "setup"):
        django.setup()

    if not test_args:
        test_args = ["tests"]

    parent = os.path.dirname(os.path.abspath(__file__))
    sys.path.insert(0, parent)

    from django.test.simple import DjangoTestSuiteRunner
    failures = DjangoTestSuiteRunner(
        verbosity=1, interactive=True, failfast=False).run_tests(test_args)
    sys.exit(failures)

if __name__ == "__main__":
    runtests(*sys.argv[1:])
<commit_msg>Update runtests<commit_after>#!/usr/bin/env python
import os
import sys

import django

from django.conf import settings

DEFAULT_SETTINGS = dict(
    INSTALLED_APPS=[
        "django.contrib.auth",
        "django.contrib.contenttypes",
        "django.contrib.sites",
        "account",
        "kaleo",
        "kaleo.tests"
    ],
    DATABASES={
        "default": {
            "ENGINE": "django.db.backends.sqlite3",
            "NAME": ":memory:",
        }
    },
    SITE_ID=1,
    ROOT_URLCONF="kaleo.tests.urls",
    SECRET_KEY="notasecret",
)

def runtests(*test_args):
    if not settings.configured:
        settings.configure(**DEFAULT_SETTINGS)

    # Compatibility with Django 1.7's stricter initialization
    if hasattr(django, "setup"):
        django.setup()

    parent = os.path.dirname(os.path.abspath(__file__))
    sys.path.insert(0, parent)

    try:
        from django.test.runner import DiscoverRunner
        runner_class = DiscoverRunner
        test_args = ["biblion.tests"]
    except ImportError:
        from django.test.simple import DjangoTestSuiteRunner
        runner_class = DjangoTestSuiteRunner
        test_args = ["tests"]

    failures = runner_class(
        verbosity=1, interactive=True, failfast=False).run_tests(test_args)
    sys.exit(failures)

if __name__ == "__main__":
    runtests(*sys.argv[1:])
<|endoftext|>


<commit_before>import urllib, httplib, time, hashlib, base64, os, mimetypes
from BeautifulSoup import BeautifulStoneSoup
from properties import loadcredentials

CHUNK_SIZE = 4096

DEFAULT_HOST = "s3.amazonaws.com"
INSECURE_PROTOCOL = "http"
SECURE_PROTOCOL = "https"

DATE_HEADER = "date"
CONTENT_MD5_HEADER = "content-md5"
CONTENT_TYPE_HEADER = "content-type"
ALTERNATIVE_DATE_HEADER = "x-amz-date"
AMAZON_HEADER_PREFIX = "x-amz-"

VIDEO_CONTENT_TYPES = {'.ogv': 'video/ogg', '.mp4': 'video/mp4', '.webm': 'video/webm'}

class Parameters:
    def __init__(self, method, bucket_name=None, object_key=None, headers={}, sub_resource=None, expires=None):
        self.method = method
        self.bucket_name = bucket_name
        self.object_key = object_key
        self.headers = headers
        self.sub_resource = sub_resource
        self.expires = expires

    def setAuthHeader(self, credentials):
        value = "AWS " + credentials.access_key_id + ":" + credentials.sign(self.textToSign())
        self.headers['Authorization'] = value

    def textToSign(self):
        if not 'Date' in self.headers:
            self.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())

        canonical_headers = self.canonicalHeaders()
        canonical_header_keys = canonical_headers.keys()
        canonical_header_keys.sort()

        result = self.method + '\n'

        for header in canonical_header_keys:
            if header.startswith(AMAZON_HEADER_PREFIX):
                result += header + ':' + canonical_headers[header]
            else:
                result += canonical_headers[header]
            result += '\n'

        return result + self.createPath()

    def canonicalHeaders(self):
        result = {}

        for key, value in self.headers.iteritems():
            header_name = key.lower()
            if self.isCanonicalHeader(header_name):
                result[header_name] = value

        if ALTERNATIVE_DATE_HEADER in result:
            result[DATE_HEADER] = ''

        if self.expires:
            result[DATE_HEADER] = str(self.expires)

        if not CONTENT_TYPE_HEADER in result:
            result[CONTENT_TYPE_HEADER] = ''

        if not CONTENT_MD5_HEADER in result:
            result[CONTENT_MD5_HEADER] = ''

        if not DATE_HEADER in result:
            raise Exception, "Must have a date header"

        return result

    def isCanonicalHeader(self, name):
        return name == CONTENT_TYPE_HEADER or\
               name == CONTENT_MD5_HEADER or\
               name == DATE_HEADER or\
               name.startswith(AMAZON_HEADER_PREFIX)

    def not_empty(self, value):
        return value and len(value) > 0

    def createPath(self, use_vhost=False):
        path = '/'
        if not use_vhost and self.not_empty(self.bucket_name):
            path += self.bucket_name + "/"
        if self.not_empty(self.object_key):
            path += urllib.quote_plus(self.object_key, '/')
        if self.not_empty(self.sub_resource):
            path += '?' + self.sub_resource
        return path

    def hostname(self, use_vhost=False):
        if use_vhost and self.not_empty(self.bucket_name):
            return self.bucket_name + '.' + DEFAULT_HOST
        else:
            return DEFAULT_HOST

class S3Client:
    def __init__(self, credentials, use_https=True, use_vhost=True):
        self.credentials = credentials
        self.use_https = use_https
        self.use_vhost = use_vhost

    def listBuckets(self):
        params = Parameters('GET')
        return self.process(params, lambda conn: self.parseBuckets(conn.getresponse()))

    def bucketExists(self, bucket_name):
        params = Parameters('HEAD', bucket_name)
        return self.process(params, lambda conn: conn.getresponse().status != httplib.NOT_FOUND)

    def createBucket(self, bucket_name):
        params = Parameters('PUT', bucket_name)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.OK))

    def deleteBucket(self, bucket_name):
        params = Parameters('DELETE', bucket_name)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.NO_CONTENT))

    def listObjects(self, bucket_name):
        params = Parameters('GET', bucket_name)
        return self.process(params, lambda conn: self.parseObjects(conn.getresponse()))

    def objectExists(self, bucket_name, object_key):
        params = Parameters('HEAD', bucket_name, object_key)
        return self.process(params, lambda conn: conn.getresponse().status == httplib.OK)

    def createObject(self, bucket_name, object_key, file_path):
        headers = {
            'Content-Length': str(os.path.getsize(file_path)),
            'Content-Type': self.getMimeType(file_path),
            'Content-MD5': self.computeMD5(file_path)
        }
        params = Parameters('PUT', bucket_name, object_key, headers)
        self.process(params, lambda conn: self.sendFile(conn, file_path))

    def downloadObject(self, bucket_name, object_key, file_path):
        params = Parameters('GET', bucket_name, object_key)
        self.process(params, lambda conn: self.receiveFile(conn.getresponse(), file_path))

    def publicURL(self, bucket_name, object_key, expires_datetime):
        expires = int(time.mktime(expires_datetime.timetuple()))
        params = Parameters('GET', bucket_name, object_key, {}, None, expires)
        signature = self.credentials.sign(params.textToSign())
        query = [
            ('AWSAccessKeyId', self.credentials.access_key_id),
            ('Expires', str(expires)),
            ('Signature', signature)
        ]
        if self.use_https:
            protocol = SECURE_PROTOCOL
        else:
            protocol = INSECURE_PROTOCOL
        return protocol + '://' +\
               params.hostname(self.use_vhost) +\
               params.createPath(self.use_vhost) +\
               "?" + urllib.urlencode(query)

    def deleteObject(self, bucket_name, object_key):
        params = Parameters('DELETE', bucket_name, object_key)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.NO_CONTENT))

    def process(self, params, callback):
        params.setAuthHeader(self.credentials)
        if self.use_https:
            conn = httplib.HTTPSConnection(params.hostname(self.use_vhost))
        else:
            conn = httplib.HTTPConnection(params.hostname(self.use_vhost))
        try:
            conn.putrequest(params.method, params.createPath(self.use_vhost))
            for name, value in params.headers.iteritems():
                conn.putheader(name, value)
            conn.endheaders()
            return callback(conn)
        finally:
            conn.close()

    def sendFile(self, conn, file_path):
        src = open(file_path, 'rb')
        try:
            self.copy(src.read, conn.send)
        finally:
            src.close()
        response = conn.getresponse()
        self.expect(response, httplib.OK)

    def receiveFile(self, response, file_path):
        self.expect(response, httplib.OK)
        dest = open(file_path, 'wb')
        try:
            self.copy(response.read, dest.write)
        finally:
            dest.close()

    def parseBuckets(self, response):
        self.expect(response, httplib.OK)
        result = []
        dom = BeautifulStoneSoup(response.read())
        for element in dom.findAll('bucket'):
            result.append(element.find('name').string)
        return result

    def parseObjects(self, response):
        self.expect(response, httplib.OK)
        result = []
        dom = BeautifulStoneSoup(response.read())
        for element in dom.findAll('contents'):
            name = element.find('key').string
            size = element.find('size').string
            last_modified = element.find('lastmodified').string
            result.append(S3Object(name, size, last_modified))
        return result

    def expect(self, response, expected):
        if response.status != expected:
            message = "Expected [%s], actual [%s]" % (str(expected), str(response.status))
            raise BadHttpResponse, message

    def getMimeType(self, file_path):
        basename = os.path.basename(file_path)
        name, ext = os.path.splitext(basename)
        if ext in VIDEO_CONTENT_TYPES:
            return VIDEO_CONTENT_TYPES[ext]
        else:
            result = mimetypes.guess_type(basename)
            return result[0] if result[0] else 'application/octet-stream'

    def computeMD5(self, file_path):
        md5 = hashlib.md5()
        src = open(file_path, 'rb')
        try:
            self.copy(src.read, md5.update)
        finally:
            src.close()
        digest = md5.digest()
        return base64.b64encode(digest)

    def copy(self, input, output):
        buf = input(CHUNK_SIZE)
        while len(buf) > 0:
            output(buf)
            buf = input(CHUNK_SIZE)

class BadHttpResponse(Exception):
    """Raised when an unexpected status code is returned"""
    pass

class S3Object:
    def __init__(self, name, size, last_modified):
        self.last_modified = last_modified
        self.size = size
        self.name = name

    def describe(self):
        return self.name, self.size, self.last_modified

    def __str__(self):
        return '%s -- %s bytes -- %s' % self.describe()

if __name__ == '__main__':
    client = S3Client(loadcredentials())
    for bucket_name in client.listBuckets():
        print "\nBucket:", bucket_name
        for object in client.listObjects(bucket_name):
            print "Object:", object
<commit_msg>recognise the common MIME types too<commit_after>import urllib, httplib, time, hashlib, base64, os, mimetypes
from BeautifulSoup import BeautifulStoneSoup
from properties import loadcredentials

CHUNK_SIZE = 4096

DEFAULT_HOST = "s3.amazonaws.com"
INSECURE_PROTOCOL = "http"
SECURE_PROTOCOL = "https"

DATE_HEADER = "date"
CONTENT_MD5_HEADER = "content-md5"
CONTENT_TYPE_HEADER = "content-type"
ALTERNATIVE_DATE_HEADER = "x-amz-date"
AMAZON_HEADER_PREFIX = "x-amz-"

VIDEO_CONTENT_TYPES = {'.ogv': 'video/ogg', '.mp4': 'video/mp4', '.webm': 'video/webm'}

class Parameters:
    def __init__(self, method, bucket_name=None, object_key=None, headers={}, sub_resource=None, expires=None):
        self.method = method
        self.bucket_name = bucket_name
        self.object_key = object_key
        self.headers = headers
        self.sub_resource = sub_resource
        self.expires = expires

    def setAuthHeader(self, credentials):
        value = "AWS " + credentials.access_key_id + ":" + credentials.sign(self.textToSign())
        self.headers['Authorization'] = value

    def textToSign(self):
        if not 'Date' in self.headers:
            self.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())

        canonical_headers = self.canonicalHeaders()
        canonical_header_keys = canonical_headers.keys()
        canonical_header_keys.sort()

        result = self.method + '\n'

        for header in canonical_header_keys:
            if header.startswith(AMAZON_HEADER_PREFIX):
                result += header + ':' + canonical_headers[header]
            else:
                result += canonical_headers[header]
            result += '\n'

        return result + self.createPath()

    def canonicalHeaders(self):
        result = {}

        for key, value in self.headers.iteritems():
            header_name = key.lower()
            if self.isCanonicalHeader(header_name):
                result[header_name] = value

        if ALTERNATIVE_DATE_HEADER in result:
            result[DATE_HEADER] = ''

        if self.expires:
            result[DATE_HEADER] = str(self.expires)

        if not CONTENT_TYPE_HEADER in result:
            result[CONTENT_TYPE_HEADER] = ''

        if not CONTENT_MD5_HEADER in result:
            result[CONTENT_MD5_HEADER] = ''

        if not DATE_HEADER in result:
            raise Exception, "Must have a date header"

        return result

    def isCanonicalHeader(self, name):
        return name == CONTENT_TYPE_HEADER or\
               name == CONTENT_MD5_HEADER or\
               name == DATE_HEADER or\
               name.startswith(AMAZON_HEADER_PREFIX)

    def not_empty(self, value):
        return value and len(value) > 0

    def createPath(self, use_vhost=False):
        path = '/'
        if not use_vhost and self.not_empty(self.bucket_name):
            path += self.bucket_name + "/"
        if self.not_empty(self.object_key):
            path += urllib.quote_plus(self.object_key, '/')
        if self.not_empty(self.sub_resource):
            path += '?' + self.sub_resource
        return path

    def hostname(self, use_vhost=False):
        if use_vhost and self.not_empty(self.bucket_name):
            return self.bucket_name + '.' + DEFAULT_HOST
        else:
            return DEFAULT_HOST

class S3Client:
    def __init__(self, credentials, use_https=True, use_vhost=True):
        self.credentials = credentials
        self.use_https = use_https
        self.use_vhost = use_vhost

    def listBuckets(self):
        params = Parameters('GET')
        return self.process(params, lambda conn: self.parseBuckets(conn.getresponse()))

    def bucketExists(self, bucket_name):
        params = Parameters('HEAD', bucket_name)
        return self.process(params, lambda conn: conn.getresponse().status != httplib.NOT_FOUND)

    def createBucket(self, bucket_name):
        params = Parameters('PUT', bucket_name)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.OK))

    def deleteBucket(self, bucket_name):
        params = Parameters('DELETE', bucket_name)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.NO_CONTENT))

    def listObjects(self, bucket_name):
        params = Parameters('GET', bucket_name)
        return self.process(params, lambda conn: self.parseObjects(conn.getresponse()))

    def objectExists(self, bucket_name, object_key):
        params = Parameters('HEAD', bucket_name, object_key)
        return self.process(params, lambda conn: conn.getresponse().status == httplib.OK)

    def createObject(self, bucket_name, object_key, file_path):
        headers = {
            'Content-Length': str(os.path.getsize(file_path)),
            'Content-Type': self.getMimeType(file_path),
            'Content-MD5': self.computeMD5(file_path)
        }
        params = Parameters('PUT', bucket_name, object_key, headers)
        self.process(params, lambda conn: self.sendFile(conn, file_path))

    def downloadObject(self, bucket_name, object_key, file_path):
        params = Parameters('GET', bucket_name, object_key)
        self.process(params, lambda conn: self.receiveFile(conn.getresponse(), file_path))

    def publicURL(self, bucket_name, object_key, expires_datetime):
        expires = int(time.mktime(expires_datetime.timetuple()))
        params = Parameters('GET', bucket_name, object_key, {}, None, expires)
        signature = self.credentials.sign(params.textToSign())
        query = [
            ('AWSAccessKeyId', self.credentials.access_key_id),
            ('Expires', str(expires)),
            ('Signature', signature)
        ]
        if self.use_https:
            protocol = SECURE_PROTOCOL
        else:
            protocol = INSECURE_PROTOCOL
        return protocol + '://' +\
               params.hostname(self.use_vhost) +\
               params.createPath(self.use_vhost) +\
               "?" + urllib.urlencode(query)

    def deleteObject(self, bucket_name, object_key):
        params = Parameters('DELETE', bucket_name, object_key)
        self.process(params, lambda conn: self.expect(conn.getresponse(), httplib.NO_CONTENT))

    def process(self, params, callback):
        params.setAuthHeader(self.credentials)
        if self.use_https:
            conn = httplib.HTTPSConnection(params.hostname(self.use_vhost))
        else:
            conn = httplib.HTTPConnection(params.hostname(self.use_vhost))
        try:
            conn.putrequest(params.method, params.createPath(self.use_vhost))
            for name, value in params.headers.iteritems():
                conn.putheader(name, value)
            conn.endheaders()
            return callback(conn)
        finally:
            conn.close()

    def sendFile(self, conn, file_path):
        src = open(file_path, 'rb')
        try:
            self.copy(src.read, conn.send)
        finally:
            src.close()
        response = conn.getresponse()
        self.expect(response, httplib.OK)

    def receiveFile(self, response, file_path):
        self.expect(response, httplib.OK)
        dest = open(file_path, 'wb')
        try:
            self.copy(response.read, dest.write)
        finally:
            dest.close()

    def parseBuckets(self, response):
        self.expect(response, httplib.OK)
        result = []
        dom = BeautifulStoneSoup(response.read())
        for element in dom.findAll('bucket'):
            result.append(element.find('name').string)
        return result

    def parseObjects(self, response):
        self.expect(response, httplib.OK)
        result = []
        dom = BeautifulStoneSoup(response.read())
        for element in dom.findAll('contents'):
            name = element.find('key').string
            size = element.find('size').string
            last_modified = element.find('lastmodified').string
            result.append(S3Object(name, size, last_modified))
        return result

    def expect(self, response, expected):
        if response.status != expected:
            message = "Expected [%s], actual [%s]" % (str(expected), str(response.status))
            raise BadHttpResponse, message

    def getMimeType(self, file_path):
        basename = os.path.basename(file_path)
        name, ext = os.path.splitext(basename)
        if ext in VIDEO_CONTENT_TYPES:
            return VIDEO_CONTENT_TYPES[ext]
        else:
            type, encoding = mimetypes.guess_type(basename, False)
            return type if type else 'application/octet-stream'

    def computeMD5(self, file_path):
        md5 = hashlib.md5()
        src = open(file_path, 'rb')
        try:
            self.copy(src.read, md5.update)
        finally:
            src.close()
        digest = md5.digest()
        return base64.b64encode(digest)

    def copy(self, input, output):
        buf = input(CHUNK_SIZE)
        while len(buf) > 0:
            output(buf)
            buf = input(CHUNK_SIZE)

class BadHttpResponse(Exception):
    """Raised when an unexpected status code is returned"""
    pass

class S3Object:
    def __init__(self, name, size, last_modified):
        self.last_modified = last_modified
        self.size = size
        self.name = name

    def describe(self):
        return self.name, self.size, self.last_modified

    def __str__(self):
        return '%s -- %s bytes -- %s' % self.describe()

if __name__ == '__main__':
    client = S3Client(loadcredentials())
    for bucket_name in client.listBuckets():
        print "\nBucket:", bucket_name
        for object in client.listObjects(bucket_name):
            print "Object:", object
<|endoftext|>