<commit_before># -*- mode: Python; coding: utf-8 -*-
"""Functions for computing Krippendorff's alpha (α), a measure of
inter-annotator agreement between two or more annotators."""

import csv
import sys
import math
import types
from itertools import permutations

import numpy as np

NUMBERS = (int, float, long, complex)

class DataType():
    """Base class for defining data types"""
    def __init__(self, t):
        self.type = t

    def __getitem__(self, arg):
        return self.type(arg)

    def name(self):
        return self.__class__.__name__

    def get(self, arg):
        return self[arg]

    def load(self, datafile):
        """Load data from TSV
        (rows = subjects; columns = annotators)"""
        with open(datafile) as f:
            reader = csv.reader(f, delimiter='\t')
            raw_data = []
            for row in reader:
                raw_data.append(map(self.get, row))
            rows = len(raw_data)
            columns = max(map(len, raw_data))
            data = np.zeros((rows, columns), dtype=self.type)
            for i in range(rows):
                row = raw_data[i]
                for j in range(columns):
                    data[i,j] = row[j] if j < len(row) else None
            return data

class Numeric(DataType):
    """A numeric data type with a metric difference function"""
    def __init__(self):
        self.type = float

    def __getitem__(self, arg):
        try:
            number = self.type(arg)
        except ValueError:
            return None
        if any(isinstance(number, t) for t in NUMBERS):
            if not math.isnan(number):
                return number
            else:
                return None

    def difference(self, v1, v2, values):
        """Return the metric difference between v1 and v2 in the range [0.0,1.0]

        The difference is normalized by the maximum value in the data set"""
        maximum_value = np.max(filter(None, map(self.get, values)))
        return np.divide(abs(v1 - v2), maximum_value)

class Nominal(DataType):
    """A nominal data type with a nominal difference function"""
    def __init__(self):
        self.type = basestring

    def __getitem__(self, arg):
        if isinstance(arg, self.type):
            if arg == '':
                return None
            else:
                return arg
        elif any(isinstance(arg, t) for t in NUMBERS):
            return str(arg)
        else:
            return None

    def difference(self, v1, v2, _):
        """Return 0.0 if v1 and v2 are the same value, 1.0 otherwise"""
        return float(v1 != v2)

def get_coincidence_matrix(data, codebook, data_type):
    """Given an N x M matrix D (data) with N subjects and M annotators/coders,
    produce an L x L coincidence matrix C where L is the number of labels/values
    assigned in the data such that cell C[i,j] is the frequency that the
    annotators assigned labels i and j to a subject."""
    labels = set(codebook.keys())
    shape = (len(labels), len(labels))
    matrix = np.zeros(shape, dtype=float)
    for row in data:
        unit = filter(None, map(data_type.get, row))
        if len(unit) > 1:
            for v1, v2 in permutations(unit, 2):
                i, j = codebook[v1], codebook[v2]
                matrix[i,j] += 1.0 / (len(unit) - 1.0)
    return matrix.astype(int)

def delta(coincidence_matrix, inverse_codebook, difference):
    """Compute a delta vector from a coincidence matrix, an inverse codebook,
    and a difference function

    The inverse codebook allows for looking up values from the row/column
    indices of the coincidence matrix"""
    delta = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            v1, v2 = inverse_codebook[i], inverse_codebook[j]
            delta.append(difference(v1, v2, inverse_codebook.values()))
    return np.array(delta)

def observation(coincidence_matrix, codebook, d):
    """Compute the observed agreement D(o)"""
    o = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            o.append(coincidence_matrix[i,j])
    observation = np.dot(o, d)
    return observation

def expectation(coincidence_matrix, d):
    """Compute the expected agreement D(e)"""
    n = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            cm_i = np.sum(coincidence_matrix[i])
            cm_j = np.sum(coincidence_matrix[j])
            n.append(cm_i * cm_j)
    expectation = np.divide(
        np.dot(n, d),
        sum(coincidence_matrix.sum(axis=1)) - 1
    )
    return expectation

def krippendorff(data, data_type):
    """Compute Krippendorff's α given a matrix D (data) of annotations
    and a difference function δ.

    The data matrix D must be an N x M matrix where N is the number of subjects
    and M is the number of annotators/coders."""
    if not type(data) == np.ndarray:
        raise TypeError, 'expected a numpy array'
    if len(data.shape) != 2:
        raise ValueError, 'input must be 2-dimensional array'
    values = set(filter(None, map(data_type.get, data.flatten())))
    codebook = dict((v,i) for (i,v) in enumerate(values))
    inverse_codebook = dict(enumerate(values))
    cm = get_coincidence_matrix(data, codebook, data_type)
    d = delta(cm, inverse_codebook, data_type.difference)
    observed = observation(cm, codebook, d)
    expected = expectation(cm, d)
    perfection = 1.0
    a = perfection - np.divide(observed, expected)
    return a

if __name__ == '__main__':
    import argparse

    DATA_TYPES = dict((dt().name().lower(), dt()) for dt in (Nominal, Numeric))
    parser = argparse.ArgumentParser(
        description=__doc__
    )
    parser.add_argument(
        'filename',
        metavar='data.csv',
        help='a CSV data file (rows=subjects, columns=annotators)'
    )
    parser.add_argument(
        '-t',
        '--type',
        default='nominal',
        choices=DATA_TYPES.keys(),
        help='how to treat the data labels (default="nominal")'
    )
    args = parser.parse_args()
    data_type = DATA_TYPES[args.type.lower()]
    data = data_type.load(args.filename)
    print 'δ: {} difference'.format(data_type.name())
    print 'α: {}'.format(krippendorff(data, data_type))
<commit_msg>remove unecessary argument<commit_after># -*- mode: Python; coding: utf-8 -*-
"""Functions for computing Krippendorff's alpha (α), a measure of
inter-annotator agreement between two or more annotators."""

import csv
import sys
import math
import types
from itertools import permutations

import numpy as np

NUMBERS = (int, float, long, complex)

class DataType():
    """Base class for defining data types"""
    def __init__(self, t):
        self.type = t

    def __getitem__(self, arg):
        return self.type(arg)

    def name(self):
        return self.__class__.__name__

    def get(self, arg):
        return self[arg]

    def load(self, datafile):
        """Load data from TSV
        (rows = subjects; columns = annotators)"""
        with open(datafile) as f:
            reader = csv.reader(f, delimiter='\t')
            raw_data = []
            for row in reader:
                raw_data.append(map(self.get, row))
            rows = len(raw_data)
            columns = max(map(len, raw_data))
            data = np.zeros((rows, columns), dtype=self.type)
            for i in range(rows):
                row = raw_data[i]
                for j in range(columns):
                    data[i,j] = row[j] if j < len(row) else None
            return data

class Numeric(DataType):
    """A numeric data type with a metric difference function"""
    def __init__(self):
        self.type = float

    def __getitem__(self, arg):
        try:
            number = self.type(arg)
        except ValueError:
            return None
        if any(isinstance(number, t) for t in NUMBERS):
            if not math.isnan(number):
                return number
            else:
                return None

    def difference(self, v1, v2, values):
        """Return the metric difference between v1 and v2 in the range [0.0,1.0]

        The difference is normalized by the maximum value in the data set"""
        maximum_value = np.max(filter(None, map(self.get, values)))
        return np.divide(abs(v1 - v2), maximum_value)

class Nominal(DataType):
    """A nominal data type with a nominal difference function"""
    def __init__(self):
        self.type = basestring

    def __getitem__(self, arg):
        if isinstance(arg, self.type):
            if arg == '':
                return None
            else:
                return arg
        elif any(isinstance(arg, t) for t in NUMBERS):
            return str(arg)
        else:
            return None

    def difference(self, v1, v2, _):
        """Return 0.0 if v1 and v2 are the same value, 1.0 otherwise"""
        return float(v1 != v2)

def get_coincidence_matrix(data, codebook, data_type):
    """Given an N x M matrix D (data) with N subjects and M annotators/coders,
    produce an L x L coincidence matrix C where L is the number of labels/values
    assigned in the data such that cell C[i,j] is the frequency that the
    annotators assigned labels i and j to a subject."""
    labels = set(codebook.keys())
    shape = (len(labels), len(labels))
    matrix = np.zeros(shape, dtype=float)
    for row in data:
        unit = filter(None, map(data_type.get, row))
        if len(unit) > 1:
            for v1, v2 in permutations(unit, 2):
                i, j = codebook[v1], codebook[v2]
                matrix[i,j] += 1.0 / (len(unit) - 1.0)
    return matrix.astype(int)

def delta(coincidence_matrix, inverse_codebook, difference):
    """Compute a delta vector from a coincidence matrix, an inverse codebook,
    and a difference function

    The inverse codebook allows for looking up values from the row/column
    indices of the coincidence matrix"""
    delta = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            v1, v2 = inverse_codebook[i], inverse_codebook[j]
            delta.append(difference(v1, v2, inverse_codebook.values()))
    return np.array(delta)

def observation(coincidence_matrix, d):
    """Compute the observed agreement D(o)"""
    o = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            o.append(coincidence_matrix[i,j])
    observation = np.dot(o, d)
    return observation

def expectation(coincidence_matrix, d):
    """Compute the expected agreement D(e)"""
    n = []
    for i in range(len(coincidence_matrix)):
        for j in range(i+1, len(coincidence_matrix)):
            cm_i = np.sum(coincidence_matrix[i])
            cm_j = np.sum(coincidence_matrix[j])
            n.append(cm_i * cm_j)
    expectation = np.divide(
        np.dot(n, d),
        sum(coincidence_matrix.sum(axis=1)) - 1
    )
    return expectation

def krippendorff(data, data_type):
    """Compute Krippendorff's α given a matrix D (data) of annotations
    and a difference function δ.

    The data matrix D must be an N x M matrix where N is the number of subjects
    and M is the number of annotators/coders."""
    if not type(data) == np.ndarray:
        raise TypeError, 'expected a numpy array'
    if len(data.shape) != 2:
        raise ValueError, 'input must be 2-dimensional array'
    values = set(filter(None, map(data_type.get, data.flatten())))
    codebook = dict((v,i) for (i,v) in enumerate(values))
    inverse_codebook = dict(enumerate(values))
    cm = get_coincidence_matrix(data, codebook, data_type)
    d = delta(cm, inverse_codebook, data_type.difference)
    observed = observation(cm, d)
    expected = expectation(cm, d)
    perfection = 1.0
    a = perfection - np.divide(observed, expected)
    return a

if __name__ == '__main__':
    import argparse

    DATA_TYPES = dict((dt().name().lower(), dt()) for dt in (Nominal, Numeric))
    parser = argparse.ArgumentParser(
        description=__doc__
    )
    parser.add_argument(
        'filename',
        metavar='data.csv',
        help='a CSV data file (rows=subjects, columns=annotators)'
    )
    parser.add_argument(
        '-t',
        '--type',
        default='nominal',
        choices=DATA_TYPES.keys(),
        help='how to treat the data labels (default="nominal")'
    )
    args = parser.parse_args()
    data_type = DATA_TYPES[args.type.lower()]
    data = data_type.load(args.filename)
    print 'δ: {} difference'.format(data_type.name())
    print 'α: {}'.format(krippendorff(data, data_type))
<|endoftext|>


<commit_before>"""
Migration that addresses the problem reported in GH#3006
On core.migrations.0061, a couple of settings got switched from the general
group to the styling group. The codebase was also change to lookup the settings
using the styling group.
However, the journal_defaults.json was never changed, which has lead to 2
possible wrong states.

Case A: pre-existing installation ran migration core.0061:
    - In this state the settings where swapped correctly to styling. however,
      due to the setting group not being updated, a duplicate setting with the
      group general was re-introduced. This setting will not have any relevant
      values attached, since the codebase has been using the correct setting
      since
    - Action: Delete the setting associated with the general group
Case B: New installations since core.0061 was introduced:
    - In this state, migration 0061 would have run before the journal_defaults
      would've have been loaded. As a result, the styling group exists but
      the settings have always been on the general group. It is safe to delete
      these settings, because the codebase would have never allowed their values
      to be changed.
    - Action: Swap the group setting of the existing setting
      (essentially run 0061 again)

"""
from __future__ import unicode_literals

from django.db import migrations, models

STYLING_SETTINGS = [
    "enable_editorial_images",
    "multi_page_editorial",
]

def update_settings_group(apps, setting_names, from_group, to_group):
    Setting = apps.get_model("core", "Setting")
    SettingGroup = apps.get_model("core", "SettingGroup")
    setting_group, _ = SettingGroup.objects.get_or_create(name=to_group)
    for setting_name in STYLING_SETTINGS:
        import pdb;pdb.set_trace()
        settings = Setting.objects.filter(
            name=setting_name,
        )
        settings_with_from = settings.filter(group__name=from_group)
        settings_with_to = settings.filter(group__name=to_group)
        if settings_with_from.exists() and settings_with_to.exists():
            # Case A
            settings_with_from.delete()
        elif settings_with_from.exists() and not settings_with_to.exists():
            # Case B
            setting = settings_with_from.first()
            setting.group = setting_group
            setting.save()
            settings_with_from.delete()

def general_to_styling(apps, schema_editor):
    update_settings_group(apps, STYLING_SETTINGS, "general", "styling")

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0073_auto_20220630_1608'),
    ]

    operations = [
        migrations.RunPython(
            general_to_styling, reverse_code=migrations.RunPython.noop),
    ]
<commit_msg>Remove PDB<commit_after>"""
Migration that addresses the problem reported in GH#3006
On core.migrations.0061, a couple of settings got switched from the general
group to the styling group. The codebase was also change to lookup the settings
using the styling group.
However, the journal_defaults.json was never changed, which has lead to 2
possible wrong states.

Case A: pre-existing installation ran migration core.0061:
    - In this state the settings where swapped correctly to styling. however,
      due to the setting group not being updated, a duplicate setting with the
      group general was re-introduced. This setting will not have any relevant
      values attached, since the codebase has been using the correct setting
      since
    - Action: Delete the setting associated with the general group
Case B: New installations since core.0061 was introduced:
    - In this state, migration 0061 would have run before the journal_defaults
      would've have been loaded. As a result, the styling group exists but
      the settings have always been on the general group. It is safe to delete
      these settings, because the codebase would have never allowed their values
      to be changed.
    - Action: Swap the group setting of the existing setting
      (essentially run 0061 again)

"""
from __future__ import unicode_literals

from django.db import migrations, models

STYLING_SETTINGS = [
    "enable_editorial_images",
    "multi_page_editorial",
]

def update_settings_group(apps, setting_names, from_group, to_group):
    Setting = apps.get_model("core", "Setting")
    SettingGroup = apps.get_model("core", "SettingGroup")
    setting_group, _ = SettingGroup.objects.get_or_create(name=to_group)
    for setting_name in STYLING_SETTINGS:
        settings = Setting.objects.filter(
            name=setting_name,
        )
        settings_with_from = settings.filter(group__name=from_group)
        settings_with_to = settings.filter(group__name=to_group)
        if settings_with_from.exists() and settings_with_to.exists():
            # Case A
            settings_with_from.delete()
        elif settings_with_from.exists() and not settings_with_to.exists():
            # Case B
            setting = settings_with_from.first()
            setting.group = setting_group
            setting.save()
            settings_with_from.delete()

def general_to_styling(apps, schema_editor):
    update_settings_group(apps, STYLING_SETTINGS, "general", "styling")

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0073_auto_20220630_1608'),
    ]

    operations = [
        migrations.RunPython(
            general_to_styling, reverse_code=migrations.RunPython.noop),
    ]
<|endoftext|>


<commit_before>import functools
import unittest

import numpy

from chainer.functions.normalization._standardize import _standardize
from chainer import testing

def _skip_if(cond, reason):
    """Skip test if cond(self) is True"""
    def decorator(impl):
        @functools.wraps(impl)
        def wrapper(self, *args, **kwargs):
            if cond(self):
                raise unittest.SkipTest(reason)
            else:
                impl(self, *args, **kwargs)
        return wrapper
    return decorator

@testing.parameterize(*testing.product([
    [
        {'ch_out': 1},
        {'ch_out': 5},
    ],
    [
        {'size': 10},
        {'size': 20},
    ],
    [
        {'dtype': numpy.float32},
        {'dtype': numpy.float16},
    ],
    [
        # same (str): flag whether input elems are same values.
        #   'no'   : all elems are randamly-chosen,
        #   'equal': all elems are equal,
        #   'near' : all elems are (randomly-chosen small values
        #            + same value).
        {'eps': 1e-5, 'same': 'no'},
        {'eps': 1e-1, 'same': 'no'},
        {'eps': 1e-1, 'same': 'equal'},
        {'eps': 1e-1, 'same': 'near'},
    ],
]))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    testing.product({
        'use_cuda': [False],
        'use_ideep': ['never', 'always'],
    })
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'use_cudnn': ['never', 'always'],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestStandardize(testing.FunctionTestCase):

    def setUp(self):
        self.skip_double_backward_test = self.same in ('equal', 'near')
        if self.dtype == numpy.float16:
            self.check_forward_options.update({'atol': 5e-3, 'rtol': 1e-2})
            self.check_backward_options.update(
                {'atol': 5e-3, 'rtol': 1e-2, 'eps': self.eps})
            self.check_double_backward_options.update(
                {'atol': 5e-3, 'rtol': 1e-2, 'eps': self.eps})

    def generate_inputs(self):
        shape = self.ch_out, self.size
        if self.same in ('equal', 'near'):
            # Make self.x have same values
            x = numpy.ones(shape, self.dtype)
            x *= numpy.random.uniform(-1, 1)
            if self.same == 'near':
                # Make self.x have slightly different values
                zero_scale = 10. ** numpy.random.randint(-40, -3)
                x += numpy.random.uniform(-zero_scale, zero_scale, shape)
        else:
            x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return _standardize(x, self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        mu = numpy.mean(x, axis=1, keepdims=True)
        x_mu = x - mu
        var = numpy.mean(numpy.square(x_mu), axis=1, keepdims=True)
        std = numpy.sqrt(var, dtype=x.dtype) + self.eps
        inv_std = 1. / std
        return x_mu * inv_std,

testing.run_module(__name__, __file__)
<commit_msg>fix `check_backward_options` to update eps correctly<commit_after>import functools
import unittest

import numpy

from chainer.functions.normalization._standardize import _standardize
from chainer import testing

def _skip_if(cond, reason):
    """Skip test if cond(self) is True"""
    def decorator(impl):
        @functools.wraps(impl)
        def wrapper(self, *args, **kwargs):
            if cond(self):
                raise unittest.SkipTest(reason)
            else:
                impl(self, *args, **kwargs)
        return wrapper
    return decorator

@testing.parameterize(*testing.product([
    [
        {'ch_out': 1},
        {'ch_out': 5},
    ],
    [
        {'size': 10},
        {'size': 20},
    ],
    [
        {'dtype': numpy.float32},
        {'dtype': numpy.float16},
    ],
    [
        # same (str): flag whether input elems are same values.
        #   'no'   : all elems are randamly-chosen,
        #   'equal': all elems are equal,
        #   'near' : all elems are (randomly-chosen small values
        #            + same value).
        {'eps': 1e-5, 'same': 'no'},
        {'eps': 1e-1, 'same': 'no'},
        {'eps': 1e-1, 'same': 'equal'},
        {'eps': 1e-1, 'same': 'near'},
    ],
]))
@testing.backend.inject_backend_tests(
    None,
    # CPU tests
    testing.product({
        'use_cuda': [False],
        'use_ideep': ['never', 'always'],
    })
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'use_cudnn': ['never', 'always'],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0'],
    })
)
class TestStandardize(testing.FunctionTestCase):

    def setUp(self):
        self.skip_double_backward_test = self.same in ('equal', 'near')
        self.check_backward_options.update({'eps': self.eps})
        self.check_double_backward_options.update({'eps': self.eps})
        if self.dtype == numpy.float16:
            self.check_forward_options.update({'atol': 5e-3, 'rtol': 1e-2})
            self.check_backward_options.update(
                {'atol': 5e-3, 'rtol': 1e-2})
            self.check_double_backward_options.update(
                {'atol': 5e-3, 'rtol': 1e-2})

    def generate_inputs(self):
        shape = self.ch_out, self.size
        if self.same in ('equal', 'near'):
            # Make self.x have same values
            x = numpy.ones(shape, self.dtype)
            x *= numpy.random.uniform(-1, 1)
            if self.same == 'near':
                # Make self.x have slightly different values
                zero_scale = 10. ** numpy.random.randint(-40, -3)
                x += numpy.random.uniform(-zero_scale, zero_scale, shape)
        else:
            x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        return x,

    def forward(self, inputs, device):
        x, = inputs
        return _standardize(x, self.eps),

    def forward_expected(self, inputs):
        x, = inputs
        mu = numpy.mean(x, axis=1, keepdims=True)
        x_mu = x - mu
        var = numpy.mean(numpy.square(x_mu), axis=1, keepdims=True)
        std = numpy.sqrt(var, dtype=x.dtype) + self.eps
        inv_std = 1. / std
        return x_mu * inv_std,

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>#
# Dalton Burke
#

# Test correct functionality of the retrieval of level0 files

import subprocess
import datetime
import shutil

local_path = 'l0_test_ingest'

# Remove data from old tests
shutil.rmtree(local_path, ignore_errors=True)

current_time = datetime.datetime.utcnow()

ten_hours_ago = str(current_time - datetime.timedelta(hours=10)).replace(' ', '_')
five_hours_ago = str(current_time - datetime.timedelta(hours=5)).replace(' ', '_')
current_time = str(current_time).replace(' ', '_')

source_types = ['MODIS_AQUA', 'MODIS_TERRA', 'VIIRS_NPP']

# -----------------------------------------------------------------------
# Download all data sources from the last 5 hours

print "TESTING SOURCES FOR FILES IN LAST 5 HOURS\n"

for t in source_types:
    print "\nRETRIEVING %s FILES FROM THE LAST 5 HOURS WITH CALL:" % t
    print './level0_retr.sh %s %s %s %s \n' %
           (t, five_hours_ago, current_time, local_path)

    subprocess.call(['./level0_retr.sh', t, five_hours_ago, current_time, local_path])

print "\nDONE RETRIEVING FILES FROM LAST 5 HOURS \n\n"

# -----------------------------------------------------------------------
# Download all data sources from the last 10 hours
# (some data we should already have, so those should be skipped)

print "TESTING SOURCES FOR FILES IN LAST 10 HOURS\n"

for t in source_types:
    print "\nRETRIEVING %s FILES FROM THE LAST 10 HOURS WITH CALL:" % t
    print './level0_retr.sh %s %s %s %s \n' %
           (t, ten_hours_ago, current_time, local_path)

    subprocess.call(['./level0_retr.sh', t, ten_hours_ago, current_time, local_path])

print "\nDONE RETRIEVING FILES FROM LAST 10 HOURS"
<commit_msg>fixed print statement across 2 lines<commit_after>#
# Dalton Burke
#

# Test correct functionality of the retrieval of level0 files

import subprocess
import datetime
import shutil

local_path = 'l0_test_ingest'

# Remove data from old tests
shutil.rmtree(local_path, ignore_errors=True)

current_time = datetime.datetime.utcnow()

ten_hours_ago = str(current_time - datetime.timedelta(hours=10)).replace(' ', '_')
five_hours_ago = str(current_time - datetime.timedelta(hours=5)).replace(' ', '_')
current_time = str(current_time).replace(' ', '_')

source_types = ['MODIS_AQUA', 'MODIS_TERRA', 'VIIRS_NPP']

# -----------------------------------------------------------------------
# Download all data sources from the last 5 hours

print "TESTING SOURCES FOR FILES IN LAST 5 HOURS\n"

for t in source_types:
    print "\nRETRIEVING %s FILES FROM THE LAST 5 HOURS WITH CALL:" % t
    print './level0_retr.sh %s %s %s %s \n' % (t, five_hours_ago, current_time, local_path)

    subprocess.call(['./level0_retr.sh', t, five_hours_ago, current_time, local_path])

print "\nDONE RETRIEVING FILES FROM LAST 5 HOURS \n\n"

# -----------------------------------------------------------------------
# Download all data sources from the last 10 hours
# (some data we should already have, so those should be skipped)

print "TESTING SOURCES FOR FILES IN LAST 10 HOURS\n"

for t in source_types:
    print "\nRETRIEVING %s FILES FROM THE LAST 10 HOURS WITH CALL:" % t
    print './level0_retr.sh %s %s %s %s \n' % (t, ten_hours_ago, current_time, local_path)

    subprocess.call(['./level0_retr.sh', t, ten_hours_ago, current_time, local_path])

print "\nDONE RETRIEVING FILES FROM LAST 10 HOURS"
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi. Copyright Camptocamp SA
#    Donors: Hasa Sàrl, Open Net Sàrl and Prisme Solutions Informatique SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from tools.translate import _
from osv import fields, osv
import re

class Bank(osv.osv):
    """Inherit res.bank class in order to add swiss specific field"""
    _inherit = 'res.bank'
    _columns = {
        ### Internal reference
        'code': fields.char('Code', size=64),
        ###Swiss unik bank identifier also use in IBAN number
        'clearing': fields.char('Clearing number', size=64),
        ### city of the bank
        'city': fields.char('City', size=128, select=1),
    }

Bank()

class ResPartnerBank(osv.osv):
    _inherit = "res.partner.bank"

    _columns = {
        'name': fields.char('Description', size=128, required=True),
        'post_number': fields.char('Post number', size=64, help="Postal number 0x-xxxxxx-x or xxxxx"),
        'bvr_adherent_num': fields.char('Bank BVR adherent number', size=11, help="Your Bank adherent number to be printed in references of your BVR. This is not a postal account number."),
        'dta_code': fields.char('DTA code', size=5),
        'print_bank': fields.boolean('Print Bank on BVR'),
        'print_account': fields.boolean('Print Account Number on BVR'),
        'acc_number': fields.char('Account/IBAN Number', size=64),
        'my_bank': fields.boolean('Use my account to print BVR ?', help="Check to print BVR invoices"),
    }

    def name_get(self, cursor, uid, ids, context=None):
        if not len(ids):
            return []
        bank_type_obj = self.pool.get('res.partner.bank.type')

        type_ids = bank_type_obj.search(cursor, uid, [])
        bank_type_names = {}
        for bank_type in bank_type_obj.browse(cursor, uid, type_ids,
                context=context):
            bank_type_names[bank_type.code] = bank_type.name
        res = []
        for r in self.read(cursor, uid, ids, ['name','state'], context):
            res.append((r['id'], r['name']+' : '+bank_type_names.get(r['state'], '')))
        return res

    def _prepare_name(self, bank):
        "Hook to get bank number of bank account"
        res = u''
        if bank.acc_number:
            res = super(ResPartnerBank, self)._prepare_name(bank) or u''
        if bank.post_number:
            if res:
                res =  u"%s - %s" % (res, bank.post_number)
            else:
                res = bank.post_number
        return res

    def _check_9_pos_postal_num(self, number):
        """
        check if a postal number in format xx-xxxxxx-x is correct,
        return true if it matches the pattern
        and if check sum mod10 is ok
        """
        from tools import mod10r
        pattern = r'^[0-9]{2}-[0-9]{1,6}-[0-9]$'
        if not re.search(pattern, number):
            return False
        num, checksum = (number.replace('-','')[:-1], number[-1:])
        return mod10r(num)[-1:] == checksum

    def _check_5_pos_postal_num(self, number):
        """
        check if a postal number on 5 positions is correct
        """
        pattern = r'^[0-9]{1,5}$'
        if not re.search(pattern, number):
            return False
        return True

    def _check_postal_num(self, cursor, uid, ids):
        banks = self.browse(cursor, uid, ids)
        for b in banks:
            if not b.post_number:
                return True
            return self._check_9_pos_postal_num(b.post_number) or \
                   self._check_5_pos_postal_num(b.post_number)

    _constraints = [(_check_postal_num,
                    'Please enter a correct postal number. (01-23456-5 or 12345)',
                    ['post_number'])]

    _sql_constraints = [('bvr_adherent_uniq', 'unique (bvr_adherent_num)',
        'The BVR adherent number must be unique !')]

ResPartnerBank()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX] l10n_ch - postal number check<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi. Copyright Camptocamp SA
#    Donors: Hasa Sàrl, Open Net Sàrl and Prisme Solutions Informatique SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from tools.translate import _
from osv import fields, osv
import re

class Bank(osv.osv):
    """Inherit res.bank class in order to add swiss specific field"""
    _inherit = 'res.bank'
    _columns = {
        ### Internal reference
        'code': fields.char('Code', size=64),
        ###Swiss unik bank identifier also use in IBAN number
        'clearing': fields.char('Clearing number', size=64),
        ### city of the bank
        'city': fields.char('City', size=128, select=1),
    }

Bank()

class ResPartnerBank(osv.osv):
    _inherit = "res.partner.bank"

    _columns = {
        'name': fields.char('Description', size=128, required=True),
        'post_number': fields.char('Post number', size=64, help="Postal number 0x-xxxxxx-x or xxxxx"),
        'bvr_adherent_num': fields.char('Bank BVR adherent number', size=11, help="Your Bank adherent number to be printed in references of your BVR. This is not a postal account number."),
        'dta_code': fields.char('DTA code', size=5),
        'print_bank': fields.boolean('Print Bank on BVR'),
        'print_account': fields.boolean('Print Account Number on BVR'),
        'acc_number': fields.char('Account/IBAN Number', size=64),
        'my_bank': fields.boolean('Use my account to print BVR ?', help="Check to print BVR invoices"),
    }

    def name_get(self, cursor, uid, ids, context=None):
        if not len(ids):
            return []
        bank_type_obj = self.pool.get('res.partner.bank.type')

        type_ids = bank_type_obj.search(cursor, uid, [])
        bank_type_names = {}
        for bank_type in bank_type_obj.browse(cursor, uid, type_ids,
                context=context):
            bank_type_names[bank_type.code] = bank_type.name
        res = []
        for r in self.read(cursor, uid, ids, ['name','state'], context):
            res.append((r['id'], r['name']+' : '+bank_type_names.get(r['state'], '')))
        return res

    def _prepare_name(self, bank):
        "Hook to get bank number of bank account"
        res = u''
        if bank.acc_number:
            res = super(ResPartnerBank, self)._prepare_name(bank) or u''
        if bank.post_number:
            if res:
                res =  u"%s - %s" % (res, bank.post_number)
            else:
                res = bank.post_number
        return res

    def _check_9_pos_postal_num(self, number):
        """
        check if a postal number in format xx-xxxxxx-x is correct,
        return true if it matches the pattern
        and if check sum mod10 is ok
        """
        from tools import mod10r
        pattern = r'^[0-9]{2}-[0-9]{1,6}-[0-9]$'
        if not re.search(pattern, number):
            return False
        nums = number.split('-')
        prefix = nums[0]
        num = nums[1].rjust(6,'0')
        checksum = nums[2]
        return mod10r(prefix + num)[-1:] == checksum

    def _check_5_pos_postal_num(self, number):
        """
        check if a postal number on 5 positions is correct
        """
        pattern = r'^[0-9]{1,5}$'
        if not re.search(pattern, number):
            return False
        return True

    def _check_postal_num(self, cursor, uid, ids):
        banks = self.browse(cursor, uid, ids)
        for b in banks:
            if not b.post_number:
                return True
            return self._check_9_pos_postal_num(b.post_number) or \
                   self._check_5_pos_postal_num(b.post_number)

    _constraints = [(_check_postal_num,
                    'Please enter a correct postal number. (01-23456-5 or 12345)',
                    ['post_number'])]

    _sql_constraints = [('bvr_adherent_uniq', 'unique (bvr_adherent_num)',
        'The BVR adherent number must be unique !')]

ResPartnerBank()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before># coding=utf-8

from heapq import nlargest
from operator import itemgetter
from time import time

from metric import levenshtein_metric
from utils import corpus_stats, error_stats, get_corrections

__author__ = "Michał Ciołczyk"

CORPUS = [
    "data/dramat.iso.utf8",
    "data/popul.iso.utf8",
    "data/proza.iso.utf8",
    "data/publ.iso.utf8",
    "data/wp.iso.utf8",
]

ERROR_FILE = "data/bledy.txt"
FORMS_FILE = "data/formy.txt"
FORMS_ENCODING = "iso-8859-2"
MAX_LENGTH_DIFFERENCE = 3
LAMBDA = 5
ALPHABET = 'aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż'

def correct_word(word, corpus_stats, errors_stats, corrections):
    corrections = {
        correction: errors_stats[levenshtein_metric(word, correction, False)] * corpus_stats[correction] ** LAMBDA for
        correction in corrections if abs(len(word) - len(correction)) <= MAX_LENGTH_DIFFERENCE
        }
    return nlargest(10, corrections.items(), key=itemgetter(1))

def compute_possible_corrections(word, forms):
    return known([word], forms) or known(edits1(word), forms) or known_edits2(word, forms) or [word]

def known(words, forms):
    return set(w for w in words if w in forms)

def edits1(word):
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [a + b[1:] for a, b in splits if b]
    transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b) > 1]
    replaces = [a + c + b[1:] for a, b in splits for c in ALPHABET if b]
    inserts = [a + c + b for a, b in splits for c in ALPHABET]
    return set(deletes + transposes + replaces + inserts)

def known_edits2(word, forms):
    return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in forms)

if __name__ == "__main__":
    print("Please wait, reading data files...")

    forms = get_corrections(FORMS_FILE, FORMS_ENCODING)
    corpus_statistics = corpus_stats(CORPUS, len(forms))
    error_statistics = error_stats(ERROR_FILE)

    print("Please enter words to correct them.")

    while True:
        try:
            word = input("> ")
            time1 = time()
            corrections = correct_word(
                word, corpus_statistics, error_statistics, compute_possible_corrections(word, forms)
            )
            time2 = time()
            print(corrections)
            print("Corrections: %s" % ', '.join(list(map(itemgetter(0), corrections))))
            print("Run for %s seconds." % (time2 - time1))
        except EOFError:
            break
<commit_msg>[lab3] Done<commit_after># coding=utf-8

from heapq import nlargest
from operator import itemgetter
from time import time

from metric import levenshtein_metric
from utils import corpus_stats, error_stats, get_corrections

__author__ = "Michał Ciołczyk"

CORPUS = [
    "data/dramat.iso.utf8",
    "data/popul.iso.utf8",
    "data/proza.iso.utf8",
    "data/publ.iso.utf8",
    "data/wp.iso.utf8",
]

ERROR_FILE = "data/bledy.txt"
FORMS_FILE = "data/formy.txt"
FORMS_ENCODING = "iso-8859-2"
MAX_LENGTH_DIFFERENCE = 3
LAMBDA = 5
ALPHABET = 'aąbcćdeęfghijklłmnńoópqrsśtuvwxyzźż'

def correct_word(word, corpus_stats, errors_stats, corrections):
    corrections = {
        correction: errors_stats[levenshtein_metric(word, correction, False)] * corpus_stats[correction] ** LAMBDA for
        correction in corrections if abs(len(word) - len(correction)) <= MAX_LENGTH_DIFFERENCE
        }
    return nlargest(10, corrections.items(), key=itemgetter(1))

def compute_possible_corrections(word, forms):
    return known([word], forms) or known(edits1(word), forms) or known_edits2(word, forms) or [word]

def known(words, forms):
    return set(w for w in words if w in forms)

def edits1(word):
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [a + b[1:] for a, b in splits if b]
    transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b) > 1]
    replaces = [a + c + b[1:] for a, b in splits for c in ALPHABET if b]
    inserts = [a + c + b for a, b in splits for c in ALPHABET]
    return set(deletes + transposes + replaces + inserts)

def known_edits2(word, forms):
    return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in forms)

if __name__ == "__main__":
    print("Please wait, reading data files...")

    forms = get_corrections(FORMS_FILE, FORMS_ENCODING)
    corpus_statistics = corpus_stats(CORPUS, len(forms))
    error_statistics = error_stats(ERROR_FILE)

    print("Please enter words to correct them.")

    while True:
        try:
            word = input("> ")
            time1 = time()
            # noinspection PyTypeChecker
            corrections = correct_word(
                word, corpus_statistics, error_statistics, compute_possible_corrections(word, forms)
            )
            time2 = time()
            print(corrections)
            print("Corrections: %s" % ', '.join(list(map(itemgetter(0), corrections))))
            print("Run for %s seconds." % (time2 - time1))
        except EOFError:
            break
<|endoftext|>


<commit_before>import glob
from Bio import AlignIO
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
__author__ = 'lcoghill'

alignment_files = glob.glob('alignments/*.alignment.fasta')
fasta_files = glob.glob('expanded-fasta/*.fasta')

count = 1
for f in alignment_files:
    new_alignment = []
    print "Correcting file %s / %s..." % (count, len(alignment_files))
    clean_name = f.split("/")
    clean_name = clean_name[1].replace(".alignment.fasta", "")
    fasta_file = "expanded-fasta/"+clean_name+".fasta"
    alignment = AlignIO.read(open(f), "fasta")
    seq_list = SeqIO.parse(open(fasta_file), "fasta")
    seq_names = []
    for s in seq_list:
        seq_names.append(s.name)
    for a in alignment:
        for s in seq_names:
            if a.id.replace("gi|", "") in s:
                a.id = a.id.replace(a.id, s)
                dummy = SeqRecord(Seq(str(a.seq)), id=a.id)
                new_alignment.append(dummy)

    out_file = "new-alignments/"+clean_name+".alignment.fasta"
    output_handle = open(out_file, "w")
    SeqIO.write(new_alignment, output_handle, "fasta")
    output_handle.close()
    count += 1
<commit_msg>Delete label-change.py<commit_after><|endoftext|>


<commit_before>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.test import TestCase
from labels.models import MuseumObject, Portal, TextLabel

class SimpleTest(TestCase):

    def test_basic_addition(self):
        """
        Tests that 1 + 1 always equals 2.
        """
        self.assertEqual(1 + 1, 2)

class LabelTest(TestCase):

    def setUp(self):
        """
        Tests that we can create a copy of the API data in the
        MuseumObject model
        """
        mo, cr = MuseumObject.objects.get_or_create(object_number='O7351')

    def test_label_download(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)

        # test the data fields
        self.assertTrue(len(mo.name) > 0)
        self.assertTrue(len(mo.museum_number) > 0)
        self.assertTrue(len(mo.artist_maker) > 0)

        # test the labels
        self.assertTrue(mo.cmslabel_set.count() > 0)

    def test_missing_object(self):
        mo, cr = MuseumObject.objects.get_or_create(object_number='OMISSING')

        self.assertTrue(mo.name.find('UNABLE') > -1)
        # test the labels
        self.assertTrue(mo.cmslabel_set.count() == 0)

    def test_thumbnail_url(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)
        self.assertTrue(mo.thumbnail_url.endswith('jpg'))
        self.assertTrue(mo.thumbnail_tag().find('cache') > -1)

    def test_redownload(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)
        original_name = mo.name
        replaced_name = 'Foo Bar Baz'
        self.assertNotEqual(mo.name, replaced_name)

        # change the name
        mo.name = replaced_name
        mo.save()
        self.assertEqual(mo.name, replaced_name)

        mo.redownload = True
        mo.save()

        # ensure original name was redownloaded
        self.assertEqual(mo.name, original_name)
        self.assertFalse(mo.redownload)

    def test_download_image(self):

        # count image
        mo = MuseumObject.objects.get(id=1)

        ims = mo.image_set.all()
        ic = ims.count()
        self.assertTrue(ic > 0)

        test_image = ims[0]

        self.assertEquals(
            unicode(test_image.image_file).find(test_image.image_id), 14)

        # check primary image position
        self.assertEquals(mo.image_set.filter(position=0).count(), 1)
        self.assertTrue(mo.image_set.filter(position=1).count() > 0)

    def test_new_object(self):

        mo = MuseumObject()
        mo.name = 'Name'
        mo.date_text = 'Date text'
        mo.artist_maker = 'Artist maker'
        mo.place = 'Place'
        mo.materials_techniques = 'Materials techniques'
        mo.museum_number = 'Museum number'
        mo.object_number = '012345'
        mo.credit_line = 'Credit line'
        mo.main_text = """Main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text."""
        mo.save()

    def test_new_loanobject(self):
        """
        Loan objects do not have O numbers but don't punish them
        """
        mo = MuseumObject()
        mo.name = 'Name'
        mo.date_text = 'Date text'
        mo.artist_maker = 'Artist maker'
        mo.place = 'Place'
        mo.materials_techniques = 'Materials techniques'
        mo.museum_number = 'Museum number'
        mo.credit_line = 'Credit line'
        mo.main_text = """Main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text."""
        mo.save()

class PortalTest(TestCase):

    def setUp(self):
        #create a portal
        pt = Portal()
        pt.name = 'Frank Lloyd Wright'
        pt.save()

    def test_textlabel(self):
        #create a Text Label
        tl = TextLabel()
        tl.title = 'Frank Lloyd Wright (Biography)'
        tl.portal_id = 1
        tl.biography = True
        tl.main_text = """Frank Lloyd Wright (born Frank Lincoln Wright,
                            June 8, 1867 - April 9, 1959) was an American architect,
                            interior designer, writer and educator, who designed
                            more than 1,000 structures and completed 500 works.
                            Wright believed in designing structures which were
                            in harmony with humanity and its environment, a
                            philosophy he called organic architecture. This
                            philosophy was best exemplified by his design for
                            Fallingwater (1935), which has been called "the best
                            all-time work of American architecture".[1] Wright was
                            a leader of the Prairie School movement of architecture
                            and developed the concept of the Usonian home, his unique
                            vision for urban planning in the United States."""
        tl.save()

        #test the text label appear in the HTML of the portal page
        response = self.client.get('/portal/1/')
        self.assertContains(response, """

Frank Lloyd Wright (Biography)

""", 1, 200)

    def test_museumobject(self):
        #create a Museum Object
        mo = MuseumObject()
        mo.name = 'Armchair'
        mo.portal = Portal.objects.get(id=1)
        mo.date_text = '1904 (made)'
        mo.artist_maker = 'Wright, Frank Lloyd'
        mo.place = 'America'
        mo.materials_techniques = """Frame: painted steel, with cast-iron base and rubber
                                        castersUpholstery: slip seat with horsehair(?)
                                        stuffing, and leather cover (probably original)"""
        mo.museum_number = 'W.43-1981'
        mo.object_number = 'O112088'
        mo.credit_line = 'Lorem ipsum'
        mo.main_text = """Wright designed a variety of metal chairs and desks for the
                            headquarters of this mail-order soap company. The client's
                            requirement that the building be fireproof provided the
                            impetus for Wright's use of metal. The form of the chair
                            and the decoration of perforated squares on the back indicate
                            Wright's likely awareness of contemporary Viennese design."""
        mo.save()

        #test the object appear in the Portal
        response = self.client.get('/portal/1/')
        self.assertContains(response, '

Armchair

', 1, 200)

    def test_edit_TextLabel(self):
        #create a new Text Label
        tl = TextLabel()
        tl.title = 'Historical Context 2'
        tl.portal = Portal.objects.get(id=1)
        tl.biography = False
        tl.main_text = """The massive stained oak table combines forms derived from the
                            Gothic Revival and Arts and Crafts movements of the later
                            19th century. The colour and shape of the table, and of other
                            pieces of furniture, were echoed in interior details throughout
                            the house. Wright believed that wood should be cut simply and
                            stained (never varnished) to reveal the 'nature' of the material."""
        tl.save()

        #get the Text Label and set the new title
        tl = TextLabel.objects.get(id=1)
        replacing_title = 'The stained oak table'
        self.assertNotEqual(tl.title, replacing_title)

        # change the title
        tl.title = replacing_title
        tl.save()
        self.assertEqual(tl.title, replacing_title)

        #test the title has changed in the portal view
        response = self.client.get('/portal/1/')
        self.assertContains(response, """

The stained oak table

""", 1, 200)
<commit_msg>writing more tests for the Digital Labels<commit_after>"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.test import TestCase
from labels.models import MuseumObject, DigitalLabel, Portal, TextLabel, Image

class SimpleTest(TestCase):

    def test_basic_addition(self):
        """
        Tests that 1 + 1 always equals 2.
        """
        self.assertEqual(1 + 1, 2)

class ObjectTest(TestCase):

    def setUp(self):
        """
        Tests that we can create a copy of the API data in the
        MuseumObject model
        """
        mo, cr = MuseumObject.objects.get_or_create(object_number='O7351')

    def test_label_download(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)

        # test the data fields
        self.assertTrue(len(mo.name) > 0)
        self.assertTrue(len(mo.museum_number) > 0)
        self.assertTrue(len(mo.artist_maker) > 0)

        # test the labels
        self.assertTrue(mo.cmslabel_set.count() > 0)

    def test_missing_object(self):
        mo, cr = MuseumObject.objects.get_or_create(object_number='OMISSING')

        self.assertTrue(mo.name.find('UNABLE') > -1)
        # test the labels
        self.assertTrue(mo.cmslabel_set.count() == 0)

    def test_thumbnail_url(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)
        self.assertTrue(mo.thumbnail_url.endswith('jpg'))
        self.assertTrue(mo.thumbnail_tag().find('cache') > -1)

    def test_redownload(self):

        # get our label
        mo = MuseumObject.objects.get(id=1)
        original_name = mo.name
        replaced_name = 'Foo Bar Baz'
        self.assertNotEqual(mo.name, replaced_name)

        # change the name
        mo.name = replaced_name
        mo.save()
        self.assertEqual(mo.name, replaced_name)

        mo.redownload = True
        mo.save()

        # ensure original name was redownloaded
        self.assertEqual(mo.name, original_name)
        self.assertFalse(mo.redownload)

    def test_download_image(self):

        # count image
        mo = MuseumObject.objects.get(id=1)

        ims = mo.image_set.all()
        ic = ims.count()
        self.assertTrue(ic > 0)

        test_image = ims[0]

        self.assertEquals(
            unicode(test_image.image_file).find(test_image.image_id), 14)

        # check primary image position
        self.assertEquals(mo.image_set.filter(position=0).count(), 1)
        self.assertTrue(mo.image_set.filter(position=1).count() > 0)

    def test_new_object(self):

        mo = MuseumObject()
        mo.name = 'Name'
        mo.date_text = 'Date text'
        mo.artist_maker = 'Artist maker'
        mo.place = 'Place'
        mo.materials_techniques = 'Materials techniques'
        mo.museum_number = 'Museum number'
        mo.object_number = '012345'
        mo.credit_line = 'Credit line'
        mo.main_text = """Main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text."""
        mo.save()

    def test_new_loanobject(self):
        """
        Loan objects do not have O numbers but don't punish them
        """
        mo = MuseumObject()
        mo.name = 'Name'
        mo.date_text = 'Date text'
        mo.artist_maker = 'Artist maker'
        mo.place = 'Place'
        mo.materials_techniques = 'Materials techniques'
        mo.museum_number = 'Museum number'
        mo.credit_line = 'Credit line'
        mo.main_text = """Main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text,
                            main text, main text, main text, main text."""
        mo.save()

class DLabelTest(TestCase):

    def setUp(self):
        #create a portal
        dl = DigitalLabel()
        dl.name = 'Turning'
        dl.save()

    def test_label_object(self):
        mo = MuseumObject()
        mo.object_number = 'O73708'
        mo.digitallabel = DigitalLabel.objects.get(id=1)
        mo.save()

        response = self.client.get('/digitallabel/1/')
        self.assertContains(response, '

Washstand

', 1, 200)

    def test_gateway_object(self):
        mo = MuseumObject()
        mo.object_number = 'O59319'
        mo.digitallabel = DigitalLabel.objects.get(id=1)
        mo.gateway_object = True
        mo.save()

        response = self.client.get('/digitallabel/1/')
        self.assertContains(response, """

Armchair

""", 1, 200)

    def test_timeout_images(self):
        mo = MuseumObject()
        mo.object_number = 'O321535'
        mo.digitallabel = DigitalLabel.objects.get(id=1)
        mo.save()

        response = self.client.get('/digitallabel/1/')
        self.assertNotContains(response, '<img class="timeout"', 200)

        dl = DigitalLabel.objects.get(id=1)
        dl.timeout_images.add(Image.objects.get(id=1))

        response = self.client.get('/digitallabel/1/')
        self.assertContains(response, '<img class="timeout"', 1, 200)

class PortalTest(TestCase):

    def setUp(self):
        #create a portal
        pt = Portal()
        pt.name = 'Frank Lloyd Wright'
        pt.save()

    def test_textlabel(self):
        #create a Text Label
        tl = TextLabel()
        tl.title = 'Frank Lloyd Wright (Biography)'
        tl.portal_id = 1
        tl.biography = True
        tl.main_text = """Frank Lloyd Wright (born Frank Lincoln Wright,
                            June 8, 1867 - April 9, 1959) was an American architect,
                            interior designer, writer and educator, who designed
                            more than 1,000 structures and completed 500 works.
                            Wright believed in designing structures which were
                            in harmony with humanity and its environment, a
                            philosophy he called organic architecture. This
                            philosophy was best exemplified by his design for
                            Fallingwater (1935), which has been called "the best
                            all-time work of American architecture".[1] Wright was
                            a leader of the Prairie School movement of architecture
                            and developed the concept of the Usonian home, his unique
                            vision for urban planning in the United States."""
        tl.save()

        #test the text label appear in the HTML of the portal page
        response = self.client.get('/portal/1/')
        self.assertContains(response, """

Frank Lloyd Wright (Biography)

""", 1, 200)

    def test_museumobject(self):
        #create a Museum Object
        mo = MuseumObject()
        mo.name = 'Armchair'
        mo.portal = Portal.objects.get(id=1)
        mo.date_text = '1904 (made)'
        mo.artist_maker = 'Wright, Frank Lloyd'
        mo.place = 'America'
        mo.materials_techniques = """Frame: painted steel, with cast-iron base and rubber
                                        castersUpholstery: slip seat with horsehair(?)
                                        stuffing, and leather cover (probably original)"""
        mo.museum_number = 'W.43-1981'
        mo.object_number = 'O112088'
        mo.credit_line = 'Lorem ipsum'
        mo.main_text = """Wright designed a variety of metal chairs and desks for the
                            headquarters of this mail-order soap company. The client's
                            requirement that the building be fireproof provided the
                            impetus for Wright's use of metal. The form of the chair
                            and the decoration of perforated squares on the back indicate
                            Wright's likely awareness of contemporary Viennese design."""
        mo.save()

        #test the object appear in the Portal
        response = self.client.get('/portal/1/')
        self.assertContains(response, '

Armchair

', 1, 200)

    def test_edit_TextLabel(self):
        #create a new Text Label
        tl = TextLabel()
        tl.title = 'Historical Context 2'
        tl.portal = Portal.objects.get(id=1)
        tl.biography = False
        tl.main_text = """The massive stained oak table combines forms derived from the
                            Gothic Revival and Arts and Crafts movements of the later
                            19th century. The colour and shape of the table, and of other
                            pieces of furniture, were echoed in interior details throughout
                            the house. Wright believed that wood should be cut simply and
                            stained (never varnished) to reveal the 'nature' of the material."""
        tl.save()

        #get the Text Label and set the new title
        tl = TextLabel.objects.get(id=1)
        replacing_title = 'The stained oak table'
        self.assertNotEqual(tl.title, replacing_title)

        # change the title
        tl.title = replacing_title
        tl.save()
        self.assertEqual(tl.title, replacing_title)

        #test the title has changed in the portal view
        response = self.client.get('/portal/1/')
        self.assertContains(response, """

The stained oak table

""", 1, 200)
<|endoftext|>


<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2012 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.shortcuts import render
from django.http import HttpResponse
from django.utils import simplejson
from components.archival_storage import forms
from main import models
from components.filesystem_ajax.views import send_file
from components import helpers
import os
import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")
import pyes
import httplib

AIPSTOREPATH = '/var/archivematica/sharedDirectory/www/AIPsStore'

def archival_storage(request):
    return archival_storage_sip_display(request)

def archival_storage_page(request, page=None):
    return archival_storage_sip_display(request, page)

def archival_storage_search(request):
    queries = request.GET.getlist('query')
    ops     = request.GET.getlist('op')

    # if no op arg provided, add default
    try:
        ops[index]
    except:
        ops = ['']

    if queries[0] == '':
        queries[0] = '*'

    # set pagination-related variables
    items_per_page = 20

    page = request.GET.get('page', 0)
    if page == '':
        page = 0
    page = int(page)

    start = page * items_per_page + 1

    conn = pyes.ES('127.0.0.1:9200')

    # do fulltext search
    must_haves     = []
    should_haves   = []
    must_not_haves = []

    index = 0
    for query in queries:
        if ops[index] == '' or ops[index] == 'and':
            must_haves.append(pyes.StringQuery(query))
        elif ops[index] == 'or':
            should_haves.append(pyes.StringQuery(query))
        else:
            must_not_haves.append(pyes.StringQuery(query))

        index = index + 1

    #queries.append(pyes.TermQuery('fileExtension', 'wma'))
    q = pyes.BoolQuery(must=must_haves, should=should_haves, must_not=must_not_haves).search()
    q.facet.add_term_facet('fileExtension')

    try:
        results = conn.search_raw(query=q, indices='aips', type='aip', start=start - 1, size=items_per_page)
    except:
        return HttpResponse('Error accessing index.')

    # augment result data
    modifiedResults = []

    for item in results.hits.hits:
        clone = item._source.copy()

        # try to find AIP details in database
        try:
            aip = models.AIP.objects.get(sipuuid=clone['AIPUUID'])
            clone['sipname'] = aip.sipname
            clone['href']    = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        except:
            aip = None
            clone['sipname'] = False

        clone['filename'] = os.path.basename(clone['filePath'])
        clone['document_id'] = item['_id']
        clone['document_id_no_hyphens'] = item['_id'].replace('-', '____')

        modifiedResults.append(clone)

    file_extension_usage = results['facets']['fileExtension']['terms']
    number_of_results = results.hits.total

    # use augmented result data
    results = modifiedResults

    # limit end by total hits
    end = start + items_per_page - 1
    if end > number_of_results:
        end = number_of_results

    # determine the previous page, if any
    previous_page = False
    if page > 0:
        previous_page = page - 1

    # determine the next page, if any
    next_page = False
    if (items_per_page * (page + 1)) < number_of_results:
        next_page = page + 1

    # make sure results is set
    try:
        if results:
            pass
    except:
        results = False

    form = forms.StorageSearchForm(initial={'query': queries[0]})
    return render(request, 'archival_storage/archival_storage_search.html', locals())

def archival_storage_indexed_count(index):
    aip_indexed_file_count = 0
    try:
        conn = pyes.ES('127.0.0.1:9200')
        count_data = conn.count(indices=index)
        aip_indexed_file_count = count_data.count
    except:
        pass
    return aip_indexed_file_count

def archival_storage_sip_download(request, path):
    full_path = os.path.join(os.path.dirname(AIPSTOREPATH), path)
    return send_file(request, full_path)

def archival_storage_sip_display(request, current_page_number=None):
    form = forms.StorageSearchForm()

    total_size = 0

    # get ElasticSearch stats
    aip_indexed_file_count = archival_storage_indexed_count('aips')

    # get AIPs from DB
    aips = models.AIP.objects.all()

    # handle pagination
    page = helpers.pager(aips, 10, current_page_number)

    sips = []
    for aip in page['objects']:
        sip = {}
        sip['href'] = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        sip['name'] = aip.sipname
        sip['uuid'] = aip.sipuuid

        sip['date'] = aip.sipdate

        try:
            size = os.path.getsize(aip.filepath) / float(1024) / float(1024)
            total_size = total_size + size
            sip['size'] = '{0:.2f} MB'.format(size)
        except:
            sip['size'] = 'Removed'

        sips.append(sip)

    order_by = request.GET.get('order_by', 'name');
    sort_by  = request.GET.get('sort_by', 'up');

    def sort_aips(sip):
        value = 0
        if 'name' == order_by:
            value = sip['name'].lower()
        else:
            value = sip[order_by]
        return value
    sips = sorted(sips, key = sort_aips)

    if sort_by == 'down':
        sips.reverse()

    total_size = '{0:.2f}'.format(total_size)

    return render(request, 'archival_storage/archival_storage.html', locals())

def archival_storage_file_json(request, document_id_modified):
    document_id = document_id_modified.replace('____', '-')
    conn = httplib.HTTPConnection("127.0.0.1:9200")
    conn.request("GET", "/aips/aip/" + document_id)
    response = conn.getresponse()
    data = response.read()
    pretty_json = simplejson.dumps(simplejson.loads(data), sort_keys=True, indent=2)
    return HttpResponse(pretty_json, content_type='application/json')
<commit_msg>Fixed bug with archival storage search.<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2012 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.shortcuts import render
from django.http import HttpResponse
from django.utils import simplejson
from components.archival_storage import forms
from main import models
from components.filesystem_ajax.views import send_file
from components import helpers
import os
import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")
import pyes
import httplib

AIPSTOREPATH = '/var/archivematica/sharedDirectory/www/AIPsStore'

def archival_storage(request):
    return archival_storage_sip_display(request)

def archival_storage_page(request, page=None):
    return archival_storage_sip_display(request, page)

def archival_storage_search(request):
    queries = request.GET.getlist('query')
    ops     = request.GET.getlist('op')

    # if no op arg provided, add default
    try:
        ops[0]
    except:
        ops = ['']

    if queries[0] == '':
        queries[0] = '*'

    # set pagination-related variables
    items_per_page = 20

    page = request.GET.get('page', 0)
    if page == '':
        page = 0
    page = int(page)

    start = page * items_per_page + 1

    conn = pyes.ES('127.0.0.1:9200')

    # do fulltext search
    must_haves     = []
    should_haves   = []
    must_not_haves = []

    index = 0
    for query in queries:
        try:
            if ops[index] == '' or ops[index] == 'and':
                must_haves.append(pyes.StringQuery(query))
            elif ops[index] == 'or':
                should_haves.append(pyes.StringQuery(query))
            else:
                must_not_haves.append(pyes.StringQuery(query))

            index = index + 1
        except:
            return HttpResponse(index)

    #queries.append(pyes.TermQuery('fileExtension', 'wma'))
    q = pyes.BoolQuery(must=must_haves, should=should_haves, must_not=must_not_haves).search()
    q.facet.add_term_facet('fileExtension')

    try:
        results = conn.search_raw(query=q, indices='aips', type='aip', start=start - 1, size=items_per_page)
    except:
        return HttpResponse('Error accessing index.')

    # augment result data
    modifiedResults = []

    for item in results.hits.hits:
        clone = item._source.copy()

        # try to find AIP details in database
        try:
            aip = models.AIP.objects.get(sipuuid=clone['AIPUUID'])
            clone['sipname'] = aip.sipname
            clone['href']    = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        except:
            aip = None
            clone['sipname'] = False

        clone['filename'] = os.path.basename(clone['filePath'])
        clone['document_id'] = item['_id']
        clone['document_id_no_hyphens'] = item['_id'].replace('-', '____')

        modifiedResults.append(clone)

    file_extension_usage = results['facets']['fileExtension']['terms']
    number_of_results = results.hits.total

    # use augmented result data
    results = modifiedResults

    # limit end by total hits
    end = start + items_per_page - 1
    if end > number_of_results:
        end = number_of_results

    # determine the previous page, if any
    previous_page = False
    if page > 0:
        previous_page = page - 1

    # determine the next page, if any
    next_page = False
    if (items_per_page * (page + 1)) < number_of_results:
        next_page = page + 1

    # make sure results is set
    try:
        if results:
            pass
    except:
        results = False

    form = forms.StorageSearchForm(initial={'query': queries[0]})
    return render(request, 'archival_storage/archival_storage_search.html', locals())

def archival_storage_indexed_count(index):
    aip_indexed_file_count = 0
    try:
        conn = pyes.ES('127.0.0.1:9200')
        count_data = conn.count(indices=index)
        aip_indexed_file_count = count_data.count
    except:
        pass
    return aip_indexed_file_count

def archival_storage_sip_download(request, path):
    full_path = os.path.join(os.path.dirname(AIPSTOREPATH), path)
    return send_file(request, full_path)

def archival_storage_sip_display(request, current_page_number=None):
    form = forms.StorageSearchForm()

    total_size = 0

    # get ElasticSearch stats
    aip_indexed_file_count = archival_storage_indexed_count('aips')

    # get AIPs from DB
    aips = models.AIP.objects.all()

    # handle pagination
    page = helpers.pager(aips, 10, current_page_number)

    sips = []
    for aip in page['objects']:
        sip = {}
        sip['href'] = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        sip['name'] = aip.sipname
        sip['uuid'] = aip.sipuuid

        sip['date'] = aip.sipdate

        try:
            size = os.path.getsize(aip.filepath) / float(1024) / float(1024)
            total_size = total_size + size
            sip['size'] = '{0:.2f} MB'.format(size)
        except:
            sip['size'] = 'Removed'

        sips.append(sip)

    order_by = request.GET.get('order_by', 'name');
    sort_by  = request.GET.get('sort_by', 'up');

    def sort_aips(sip):
        value = 0
        if 'name' == order_by:
            value = sip['name'].lower()
        else:
            value = sip[order_by]
        return value
    sips = sorted(sips, key = sort_aips)

    if sort_by == 'down':
        sips.reverse()

    total_size = '{0:.2f}'.format(total_size)

    return render(request, 'archival_storage/archival_storage.html', locals())

def archival_storage_file_json(request, document_id_modified):
    document_id = document_id_modified.replace('____', '-')
    conn = httplib.HTTPConnection("127.0.0.1:9200")
    conn.request("GET", "/aips/aip/" + document_id)
    response = conn.getresponse()
    data = response.read()
    pretty_json = simplejson.dumps(simplejson.loads(data), sort_keys=True, indent=2)
    return HttpResponse(pretty_json, content_type='application/json')
<|endoftext|>


<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2012 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.shortcuts import render
from django.http import HttpResponse
from django.utils import simplejson
from django.views.static import serve
from components.archival_storage import forms
from main import models
import os
import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")
import pyes
import httplib

AIPSTOREPATH = '/var/archivematica/sharedDirectory/www/AIPsStore'

def archival_storage(request):
    return archival_storage_sip_display(request)

def archival_storage_page(request, page=None):
    return archival_storage_sip_display(request, page)

def archival_storage_search(request):
    query = request.GET.get('query', '')

    if query == '':
        query = '*'

    # set pagination-related variables
    items_per_page = 20

    page = request.GET.get('page', 0)
    if page == '':
        page = 0
    page = int(page)

    start = page * items_per_page + 1

    conn = pyes.ES('127.0.0.1:9200')

    # do fulltext search
    q = pyes.StringQuery(query)

    try:
        results = conn.search_raw(query=q, indices='aips', type='aip', start=start - 1, size=items_per_page)
    except:
        return HttpResponse('Error accessing index.')

    # augment result data
    modifiedResults = []

    for item in results.hits.hits:
        clone = item._source.copy()

        # try to find AIP details in database
        try:
            aip = models.AIP.objects.get(sipuuid=clone['AIPUUID'])
            clone['sipname'] = aip.sipname
            clone['href']    = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        except:
            aip = None
            clone['sipname'] = False

        clone['filename'] = os.path.basename(clone['filePath'])
        clone['document_id'] = item['_id']
        clone['document_id_no_hyphens'] = item['_id'].replace('-', '____')

        modifiedResults.append(clone)

    number_of_results = results.hits.total

    # use augmented result data
    results = modifiedResults

    # limit end by total hits
    end = start + items_per_page - 1
    if end > number_of_results:
        end = number_of_results

    # determine the previous page, if any
    previous_page = False
    if page > 0:
        previous_page = page - 1

    # determine the next page, if any
    next_page = False
    if (items_per_page * (page + 1)) < number_of_results:
        next_page = page + 1

    # make sure results is set
    try:
        if results:
            pass
    except:
        results = False

    form = forms.StorageSearchForm(initial={'query': query})
    return render(request, 'archival_storage/archival_storage_search.html', locals())

def archival_storage_indexed_count(index):
    aip_indexed_file_count = 0
    try:
        conn = pyes.ES('127.0.0.1:9200')
        count_data = conn.count(indices=index)
        aip_indexed_file_count = count_data.count
    except:
        pass
    return aip_indexed_file_count

def archival_storage_sip_download(request, path):
    # TODO: use AIPSTOREPATH instead of hardcoded path
    return serve(request, path, document_root='/var/archivematica/sharedDirectory/www')

def archival_storage_sip_display(request, current_page_number=None):
    form = forms.StorageSearchForm()

    total_size = 0

    # get ElasticSearch stats
    aip_indexed_file_count = archival_storage_indexed_count('aips')

    # get AIPs from DB
    aips = models.AIP.objects.all()

    # handle pagination
    p                   = Paginator(aips, 20)
    current_page_number = 1 if current_page_number == None else int(current_page_number)
    page                = p.page(current_page_number)
    has_next            = page.has_next()
    next_page           = current_page_number + 1
    has_previous        = page.has_previous()
    previous_page       = current_page_number - 1
    has_other_pages     = page.has_other_pages()

    sips = []
    for aip in page.object_list:
        sip = {}
        sip['href'] = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        sip['name'] = aip.sipname
        sip['uuid'] = aip.sipuuid

        sip['date'] = aip.sipdate

        try:
            size = os.path.getsize(aip.filepath) / float(1024) / float(1024)
            total_size = total_size + size
            sip['size'] = '{0:.2f} MB'.format(size)
        except:
            sip['size'] = 'Removed'

        sips.append(sip)

    order_by = request.GET.get('order_by', 'name');
    sort_by  = request.GET.get('sort_by', 'up');

    def sort_aips(sip):
        value = 0
        if 'name' == order_by:
            value = sip['name'].lower()
        else:
            value = sip[order_by]
        return value
    sips = sorted(sips, key = sort_aips)

    if sort_by == 'down':
        sips.reverse()

    total_size = '{0:.2f}'.format(total_size)

    return render(request, 'archival_storage/archival_storage.html', locals())

def archival_storage_file_json(request, document_id_modified):
    document_id = document_id_modified.replace('____', '-')
    conn = httplib.HTTPConnection("127.0.0.1:9200")
    conn.request("GET", "/aips/aip/" + document_id)
    response = conn.getresponse()
    data = response.read()
    pretty_json = simplejson.dumps(simplejson.loads(data), sort_keys=True, indent=2)
    return HttpResponse(pretty_json, content_type='application/json')
<commit_msg>Fixed issue of large AIP downloads failing.<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2012 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.shortcuts import render
from django.http import HttpResponse
from django.utils import simplejson
from components.archival_storage import forms
from main import models
from main import filesystem
import os
import sys
sys.path.append("/usr/lib/archivematica/archivematicaCommon/externals")
import pyes
import httplib

AIPSTOREPATH = '/var/archivematica/sharedDirectory/www/AIPsStore'

def archival_storage(request):
    return archival_storage_sip_display(request)

def archival_storage_page(request, page=None):
    return archival_storage_sip_display(request, page)

def archival_storage_search(request):
    query = request.GET.get('query', '')

    if query == '':
        query = '*'

    # set pagination-related variables
    items_per_page = 20

    page = request.GET.get('page', 0)
    if page == '':
        page = 0
    page = int(page)

    start = page * items_per_page + 1

    conn = pyes.ES('127.0.0.1:9200')

    # do fulltext search
    q = pyes.StringQuery(query)

    try:
        results = conn.search_raw(query=q, indices='aips', type='aip', start=start - 1, size=items_per_page)
    except:
        return HttpResponse('Error accessing index.')

    # augment result data
    modifiedResults = []

    for item in results.hits.hits:
        clone = item._source.copy()

        # try to find AIP details in database
        try:
            aip = models.AIP.objects.get(sipuuid=clone['AIPUUID'])
            clone['sipname'] = aip.sipname
            clone['href']    = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        except:
            aip = None
            clone['sipname'] = False

        clone['filename'] = os.path.basename(clone['filePath'])
        clone['document_id'] = item['_id']
        clone['document_id_no_hyphens'] = item['_id'].replace('-', '____')

        modifiedResults.append(clone)

    number_of_results = results.hits.total

    # use augmented result data
    results = modifiedResults

    # limit end by total hits
    end = start + items_per_page - 1
    if end > number_of_results:
        end = number_of_results

    # determine the previous page, if any
    previous_page = False
    if page > 0:
        previous_page = page - 1

    # determine the next page, if any
    next_page = False
    if (items_per_page * (page + 1)) < number_of_results:
        next_page = page + 1

    # make sure results is set
    try:
        if results:
            pass
    except:
        results = False

    form = forms.StorageSearchForm(initial={'query': query})
    return render(request, 'archival_storage/archival_storage_search.html', locals())

def archival_storage_indexed_count(index):
    aip_indexed_file_count = 0
    try:
        conn = pyes.ES('127.0.0.1:9200')
        count_data = conn.count(indices=index)
        aip_indexed_file_count = count_data.count
    except:
        pass
    return aip_indexed_file_count

def archival_storage_sip_download(request, path):
    full_path = os.path.join(os.path.dirname(AIPSTOREPATH), path)
    return filesystem.send_file(request, full_path)

def archival_storage_sip_display(request, current_page_number=None):
    form = forms.StorageSearchForm()

    total_size = 0

    # get ElasticSearch stats
    aip_indexed_file_count = archival_storage_indexed_count('aips')

    # get AIPs from DB
    aips = models.AIP.objects.all()

    # handle pagination
    p                   = Paginator(aips, 20)
    current_page_number = 1 if current_page_number == None else int(current_page_number)
    page                = p.page(current_page_number)
    has_next            = page.has_next()
    next_page           = current_page_number + 1
    has_previous        = page.has_previous()
    previous_page       = current_page_number - 1
    has_other_pages     = page.has_other_pages()

    sips = []
    for aip in page.object_list:
        sip = {}
        sip['href'] = aip.filepath.replace(AIPSTOREPATH + '/', "AIPsStore/")
        sip['name'] = aip.sipname
        sip['uuid'] = aip.sipuuid

        sip['date'] = aip.sipdate

        try:
            size = os.path.getsize(aip.filepath) / float(1024) / float(1024)
            total_size = total_size + size
            sip['size'] = '{0:.2f} MB'.format(size)
        except:
            sip['size'] = 'Removed'

        sips.append(sip)

    order_by = request.GET.get('order_by', 'name');
    sort_by  = request.GET.get('sort_by', 'up');

    def sort_aips(sip):
        value = 0
        if 'name' == order_by:
            value = sip['name'].lower()
        else:
            value = sip[order_by]
        return value
    sips = sorted(sips, key = sort_aips)

    if sort_by == 'down':
        sips.reverse()

    total_size = '{0:.2f}'.format(total_size)

    return render(request, 'archival_storage/archival_storage.html', locals())

def archival_storage_file_json(request, document_id_modified):
    document_id = document_id_modified.replace('____', '-')
    conn = httplib.HTTPConnection("127.0.0.1:9200")
    conn.request("GET", "/aips/aip/" + document_id)
    response = conn.getresponse()
    data = response.read()
    pretty_json = simplejson.dumps(simplejson.loads(data), sort_keys=True, indent=2)
    return HttpResponse(pretty_json, content_type='application/json')
<|endoftext|>


<commit_before># https://flashair-developers.com/en/documents/api/

from __future__ import unicode_literals, print_function

import attr
from attr.validators import and_, instance_of

from constantly import FlagConstant, Flags

from twisted.python.url import URL
from twisted.python.filepath import FilePath

from twisted.internet.defer import inlineCallbacks

from twisted.web.http import OK
from twisted.web.client import PartialDownloadError, readBody

class FileAttributes(Flags):
    READONLY = FlagConstant()
    HIDDEN = FlagConstant()
    SYSTEM = FlagConstant()
    VOLUME = FlagConstant()
    DIRECTLY = FlagConstant()
    ARCHIVE = FlagConstant()

@attr.s(frozen=True)
class File(object):
    name = attr.ib(validator=instance_of(FilePath))
    size = attr.ib(validator=instance_of(int))
    attributes = attr.ib(
        validator=lambda self, attr, value: (
            value in FileAttributes.iterconstants()
        ),
    )
    date = attr.ib(validator=instance_of(int))
    time = attr.ib(validator=instance_of(int))

def has_attribute(which):
    def validator(self, attr, file):
        value = file.attributes
        if value & which:
            return
        raise ValueError(
            "{} missing required attribute {}".format(value, which)
        )
    return validator

@attr.s(frozen=True)
class DeleteFile(object):
    file = attr.ib(
        validator=and_(
            instance_of(File),
            has_attribute(FileAttributes.DIRECTLY),
        ),
    )

    def uri(self):
        return URL(
            path=["upload.cgi"],
            query=[
                ("DEL", self.file.name.path),
            ],
        )

    def headers(self):
        return None

    def body(self):
        return None

    def process_response(self, response):
        if response.code != OK:
            raise Exception(
                "Unexpected response code {}:\n{}".format(response.code)
            )
        return None

@attr.s(frozen=True)
class GetFileList(object):
    opcode = 100
    minimum_version = "1.00.03"
    directory = attr.ib(validator=instance_of(FilePath))

    def uri(self):
        return URL(
            path=["command.cgi"],
            query=[
                ("op", "{}".format(self.opcode)),
                ("DIR", self.directory.path),
            ],
        )

    def headers(self):
        return None

    def body(self):
        return None

    def process_response(self, response):
        d = readBody(response)
        def read(body):
            if response.code != OK:
                raise Exception(
                    "Unexpected response code {}:\n{}".format(response.code, body)
                )
            lines = body.decode("utf-8").split("\r\n")
            if lines[0] == "WLANSD_FILELIST": # XXX???
                lines = lines[1:-1]
            else:
                raise Exception("Whuat? {}".format(lines))
            for line in lines:
                parts = line.split(",")
                if len(parts) != 6:
                    raise Exception("Uauaua {}".format(parts))
                yield File(
                    name=FilePath(parts[0] or "/").child(parts[1]),
                    size=int(parts[2]),
                    attributes=lookupByValue(FileAttributes, int(parts[3])),
                    date=int(parts[4]),
                    time=int(parts[5]),
                )
        d.addErrback(lambda reason: reason.check(PartialDownloadError) and reason.value.args[2])
        d.addCallback(read)
        return d

def lookupByValue(constants, flags):
    result = None
    for flag in constants.iterconstants():
        if flag.value & flags:
            if result is None:
                result = flag
            else:
                result |= flag
    if result is None:
        raise ValueError("File attribute unknown: {}".format(flags))
    return result

def get_file_list(treq, root, path):
    return execute(treq, root, GetFileList(directory=path))

def download_file(treq, root, path):
    uri = root.replace(path=path.segmentsFrom(FilePath(b"/")))
    url = uri.asURI().asText().encode("ascii")
    return treq.get(url)

def remove_file(treq, root, file):
    return execute(treq, root, DeleteFile(file=file))

def execute(treq, root, operation):
    uri = operation.uri().replace(
        scheme=root.scheme,
        host=root.host,
        port=root.port,
    )
    headers = operation.headers()

    url = uri.asURI().asText().encode("ascii")
    print("Getting", url)
    d = treq.get(
        url,
        headers,
    )
    d.addCallback(operation.process_response)
    return d

@inlineCallbacks
def visit(treq, root_uri, root_directory, visitor):
    work = [root_directory]

    while work:
        path = work.pop()
        files = yield get_file_list(treq,  root_uri, path)
        for f in files:
            if f.attributes & FileAttributes.DIRECTLY:
                work.append(f.name)
            else:
                yield visitor(f)
<commit_msg>we _don't_ want directories<commit_after># https://flashair-developers.com/en/documents/api/

from __future__ import unicode_literals, print_function

import attr
from attr.validators import and_, instance_of

from constantly import FlagConstant, Flags

from twisted.python.url import URL
from twisted.python.filepath import FilePath

from twisted.internet.defer import inlineCallbacks

from twisted.web.http import OK
from twisted.web.client import PartialDownloadError, readBody

class FileAttributes(Flags):
    READONLY = FlagConstant()
    HIDDEN = FlagConstant()
    SYSTEM = FlagConstant()
    VOLUME = FlagConstant()
    DIRECTLY = FlagConstant()
    ARCHIVE = FlagConstant()

@attr.s(frozen=True)
class File(object):
    name = attr.ib(validator=instance_of(FilePath))
    size = attr.ib(validator=instance_of(int))
    attributes = attr.ib(
        validator=lambda self, attr, value: (
            value in FileAttributes.iterconstants()
        ),
    )
    date = attr.ib(validator=instance_of(int))
    time = attr.ib(validator=instance_of(int))

def has_attribute(which):
    def validator(self, attr, file):
        value = file.attributes
        if value & which:
            return
        raise ValueError(
            "{} missing required attribute {}".format(value, which)
        )
    return validator

@attr.s(frozen=True)
class DeleteFile(object):
    file = attr.ib(
        validator=and_(
            instance_of(File),
            has_attribute(FileAttributes.ARCHIVE),
        ),
    )

    def uri(self):
        return URL(
            path=["upload.cgi"],
            query=[
                ("DEL", self.file.name.path),
            ],
        )

    def headers(self):
        return None

    def body(self):
        return None

    def process_response(self, response):
        if response.code != OK:
            raise Exception(
                "Unexpected response code {}:\n{}".format(response.code)
            )
        return None

@attr.s(frozen=True)
class GetFileList(object):
    opcode = 100
    minimum_version = "1.00.03"
    directory = attr.ib(validator=instance_of(FilePath))

    def uri(self):
        return URL(
            path=["command.cgi"],
            query=[
                ("op", "{}".format(self.opcode)),
                ("DIR", self.directory.path),
            ],
        )

    def headers(self):
        return None

    def body(self):
        return None

    def process_response(self, response):
        d = readBody(response)
        def read(body):
            if response.code != OK:
                raise Exception(
                    "Unexpected response code {}:\n{}".format(response.code, body)
                )
            lines = body.decode("utf-8").split("\r\n")
            if lines[0] == "WLANSD_FILELIST": # XXX???
                lines = lines[1:-1]
            else:
                raise Exception("Whuat? {}".format(lines))
            for line in lines:
                parts = line.split(",")
                if len(parts) != 6:
                    raise Exception("Uauaua {}".format(parts))
                yield File(
                    name=FilePath(parts[0] or "/").child(parts[1]),
                    size=int(parts[2]),
                    attributes=lookupByValue(FileAttributes, int(parts[3])),
                    date=int(parts[4]),
                    time=int(parts[5]),
                )
        d.addErrback(lambda reason: reason.check(PartialDownloadError) and reason.value.args[2])
        d.addCallback(read)
        return d

def lookupByValue(constants, flags):
    result = None
    for flag in constants.iterconstants():
        if flag.value & flags:
            if result is None:
                result = flag
            else:
                result |= flag
    if result is None:
        raise ValueError("File attribute unknown: {}".format(flags))
    return result

def get_file_list(treq, root, path):
    return execute(treq, root, GetFileList(directory=path))

def download_file(treq, root, path):
    uri = root.replace(path=path.segmentsFrom(FilePath(b"/")))
    url = uri.asURI().asText().encode("ascii")
    return treq.get(url)

def remove_file(treq, root, file):
    return execute(treq, root, DeleteFile(file=file))

def execute(treq, root, operation):
    uri = operation.uri().replace(
        scheme=root.scheme,
        host=root.host,
        port=root.port,
    )
    headers = operation.headers()

    url = uri.asURI().asText().encode("ascii")
    print("Getting", url)
    d = treq.get(
        url,
        headers,
    )
    d.addCallback(operation.process_response)
    return d

@inlineCallbacks
def visit(treq, root_uri, root_directory, visitor):
    work = [root_directory]

    while work:
        path = work.pop()
        files = yield get_file_list(treq,  root_uri, path)
        for f in files:
            if f.attributes & FileAttributes.DIRECTLY:
                work.append(f.name)
            else:
                yield visitor(f)
<|endoftext|>