<commit_before>import argparse
import os
import shutil
import time

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models

parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
parser.add_argument('data', metavar='DIR',
                    help='path to dataset')
parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18',
                    choices=model_names,
                    help='model architecture: ' +
                        ' | '.join(model_names) +
                        ' (default: resnet18)')
parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--epochs', default=90, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('-b', '--batch-size', default=256, type=int,
                    metavar='N', help='mini-batch size (default: 256)')
parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print-freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                    help='evaluate model on validation set')
parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                    help='use pre-trained model')
parser.add_argument('--world-size', default=1, type=int,
                    help='number of distributed processes')
parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str,
                    help='url used to set up distributed training')
parser.add_argument('--dist-backend', default='gloo', type=str,
                    help='distributed backend')

best_prec1 = 0

def main():
    global args, best_prec1
    args = parser.parse_args()

    args.distributed = args.world_size > 1

    if args.distributed:
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                                world_size=args.world_size)

    # create model
    if args.pretrained:
        print("=> using pre-trained model '{}'".format(args.arch))
        model = models.__dict__[args.arch](pretrained=True)
    else:
        print("=> creating model '{}'".format(args.arch))
        model = models.__dict__[args.arch]()

    if not args.distributed:
        if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
            model.features = torch.nn.DataParallel(model.features)
            model.cuda()
        else:
            model = torch.nn.DataParallel(model).cuda()
    else:
        model.cuda()
        model = torch.nn.parallel.DistributedDataParallel(model)

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda()

    optimizer = torch.optim.SGD(model.parameters(), args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            best_prec1 = checkpoint['best_prec1']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.resume, checkpoint['epoch']))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    cudnn.benchmark = True

    # Data loading code
    traindir = os.path.join(args.data, 'train')
    valdir = os.path.join(args.data, 'val')
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

    train_dataset = datasets.ImageFolder(
        traindir,
        transforms.Compose([
            transforms.RandomSizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))

    if args.distributed:
        train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
    else:
        train_sampler = None

    train_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
        num_workers=args.workers, pin_memory=True, sampler=train_sampler)

    val_loader = torch.utils.data.DataLoader(
        datasets.ImageFolder(valdir, transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers, pin_memory=True)

    if args.evaluate:
        validate(val_loader, model, criterion)
        return

    for epoch in range(args.start_epoch, args.epochs):
        if args.distributed:
            train_sampler.set_epoch(epoch)
        adjust_learning_rate(optimizer, epoch)

        # train for one epoch
        train(train_loader, model, criterion, optimizer, epoch)

        # evaluate on validation set
        prec1 = validate(val_loader, model, criterion)

        # remember best prec@1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint({
            'epoch': epoch + 1,
            'arch': args.arch,
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
            'optimizer' : optimizer.state_dict(),
        }, is_best)

def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    for i, (input, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input)
        target_var = torch.autograd.Variable(target)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   epoch, i, len(train_loader), batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1, top5=top5))

def validate(val_loader, model, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    for i, (input, target) in enumerate(val_loader):
        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input, volatile=True)
        target_var = torch.autograd.Variable(target, volatile=True)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   i, len(val_loader), batch_time=batch_time, loss=losses,
                   top1=top1, top5=top5))

    print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
          .format(top1=top1, top5=top5))

    return top1.avg

def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = args.lr * (0.1 ** (epoch // 30))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

if __name__ == '__main__':
    main()
<commit_msg>Add model_names back. Fixes #195<commit_after>import argparse
import os
import shutil
import time

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models

model_names = sorted(name for name in models.__dict__
    if name.islower() and not name.startswith("__")
    and callable(models.__dict__[name]))

parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
parser.add_argument('data', metavar='DIR',
                    help='path to dataset')
parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18',
                    choices=model_names,
                    help='model architecture: ' +
                        ' | '.join(model_names) +
                        ' (default: resnet18)')
parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                    help='number of data loading workers (default: 4)')
parser.add_argument('--epochs', default=90, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('-b', '--batch-size', default=256, type=int,
                    metavar='N', help='mini-batch size (default: 256)')
parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print-freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                    help='evaluate model on validation set')
parser.add_argument('--pretrained', dest='pretrained', action='store_true',
                    help='use pre-trained model')
parser.add_argument('--world-size', default=1, type=int,
                    help='number of distributed processes')
parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str,
                    help='url used to set up distributed training')
parser.add_argument('--dist-backend', default='gloo', type=str,
                    help='distributed backend')

best_prec1 = 0

def main():
    global args, best_prec1
    args = parser.parse_args()

    args.distributed = args.world_size > 1

    if args.distributed:
        dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                                world_size=args.world_size)

    # create model
    if args.pretrained:
        print("=> using pre-trained model '{}'".format(args.arch))
        model = models.__dict__[args.arch](pretrained=True)
    else:
        print("=> creating model '{}'".format(args.arch))
        model = models.__dict__[args.arch]()

    if not args.distributed:
        if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
            model.features = torch.nn.DataParallel(model.features)
            model.cuda()
        else:
            model = torch.nn.DataParallel(model).cuda()
    else:
        model.cuda()
        model = torch.nn.parallel.DistributedDataParallel(model)

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda()

    optimizer = torch.optim.SGD(model.parameters(), args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            best_prec1 = checkpoint['best_prec1']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(args.resume, checkpoint['epoch']))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    cudnn.benchmark = True

    # Data loading code
    traindir = os.path.join(args.data, 'train')
    valdir = os.path.join(args.data, 'val')
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

    train_dataset = datasets.ImageFolder(
        traindir,
        transforms.Compose([
            transforms.RandomSizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))

    if args.distributed:
        train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
    else:
        train_sampler = None

    train_loader = torch.utils.data.DataLoader(
        train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
        num_workers=args.workers, pin_memory=True, sampler=train_sampler)

    val_loader = torch.utils.data.DataLoader(
        datasets.ImageFolder(valdir, transforms.Compose([
            transforms.Scale(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers, pin_memory=True)

    if args.evaluate:
        validate(val_loader, model, criterion)
        return

    for epoch in range(args.start_epoch, args.epochs):
        if args.distributed:
            train_sampler.set_epoch(epoch)
        adjust_learning_rate(optimizer, epoch)

        # train for one epoch
        train(train_loader, model, criterion, optimizer, epoch)

        # evaluate on validation set
        prec1 = validate(val_loader, model, criterion)

        # remember best prec@1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint({
            'epoch': epoch + 1,
            'arch': args.arch,
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
            'optimizer' : optimizer.state_dict(),
        }, is_best)

def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    for i, (input, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input)
        target_var = torch.autograd.Variable(target)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   epoch, i, len(train_loader), batch_time=batch_time,
                   data_time=data_time, loss=losses, top1=top1, top5=top5))

def validate(val_loader, model, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    for i, (input, target) in enumerate(val_loader):
        target = target.cuda(async=True)
        input_var = torch.autograd.Variable(input, volatile=True)
        target_var = torch.autograd.Variable(target, volatile=True)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data[0], input.size(0))
        top1.update(prec1[0], input.size(0))
        top5.update(prec5[0], input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   i, len(val_loader), batch_time=batch_time, loss=losses,
                   top1=top1, top5=top5))

    print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
          .format(top1=top1, top5=top5))

    return top1.avg

def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
    torch.save(state, filename)
    if is_best:
        shutil.copyfile(filename, 'model_best.pth.tar')

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = args.lr * (0.1 ** (epoch // 30))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Amara, universalsubtitles.org
#
# Copyright (C) 2013 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program.  If not, see http://www.gnu.org/licenses/agpl-3.0.html.

import simplejson as json

import babelsubs
from django.contrib.auth.decorators import login_required
from django.conf import settings
from django.http import HttpResponse, Http404
from django.db.models import Count
from django.conf import settings
from django.contrib import messages
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.utils.decorators import method_decorator
from django.utils.http import urlencode
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django.views.generic import View
from django.shortcuts import render, get_object_or_404, redirect
from django.template.defaultfilters import urlize, linebreaks, force_escape
from django.views.decorators.clickjacking import xframe_options_exempt

from subtitles import shims
from subtitles.workflows import get_workflow
from subtitles.models import SubtitleLanguage, SubtitleVersion
from subtitles.templatetags.new_subtitles_tags import visibility
from subtitles.forms import SubtitlesUploadForm
from teams.models import Task
from teams.permissions import can_add_version, can_assign_task
from utils.text import fmt
from videos.models import Video

def _version_data(version):
    '''
    Creates a dict with version info, suitable for encoding
    into json and bootstrapping the editor.
    '''
    return {
        'metadata': version.get_metadata(),
        'subtitles': version.get_subtitles().to_xml(),
        'title': version.title,
        'description': version.description,
    }

def _language_data(language, editing_version, translated_from_version,
                   base_language):
    '''
    Creates a dict with language info, suitable for encoding
    into json and bootstrapping the editor. Includes
    the version data for the version being edited and the
    original translation source, if any.
    '''
    versions_data = []

    versions = list(language.subtitleversion_set.full())
    for i, version in enumerate(versions):
        version_data = {
            'version_no':version.version_number,
            'visibility': visibility(version),
        }
        if editing_version == version:
            version_data.update(_version_data(version))
        elif translated_from_version == version:
            version_data.update(_version_data(version))
        elif (language.language_code == base_language and
              i == len(versions) - 1):
            version_data.update(_version_data(version))

        versions_data.append(version_data)

    subtitle_language = editing_version.subtitle_language if editing_version else ''

    return {
        'translatedFrom': translated_from_version and {
            'language_code': translated_from_version.subtitle_language.language_code,
            'version_number': translated_from_version.version_number,
        },
        'editingLanguage': language == subtitle_language,
        'language_code': language.language_code,
        'name': language.get_language_code_display(),
        'pk': language.pk,
        'numVersions': language.num_versions,
        'versions': versions_data,
        'subtitles_complete': language.subtitles_complete,
        'is_rtl': language.is_rtl(),
        'is_original': language.is_primary_audio_language()
    }

def regain_lock(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = video.subtitle_language(language_code)

    if not language.can_writelock(request.browser_id):
        return HttpResponse(json.dumps({'ok': False}))

    language.writelock(request.user, request.browser_id, save=True)
    return HttpResponse(json.dumps({'ok': True}))

@login_required
@require_POST
def release_lock(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = video.subtitle_language(language_code)

    if language.can_writelock(request.browser_id):
        language.release_writelock()

    return HttpResponse(json.dumps({'url': reverse('videos:video', args=(video_id,))}))

def assign_task_for_editor(video, language_code, user):
    team_video = video.get_team_video()
    if team_video is None:
        return None
    task_set = team_video.task_set.incomplete().filter(language=language_code)
    tasks = list(task_set[:1])
    if tasks:
        task = tasks[0]
        if task.assignee is None and can_assign_task(task, user):
            task.assignee = user
            task.set_expiration()
            task.save()

        if task.assignee != user:
            return fmt(_("Another user is currently performing "
                         "the %(task_type)s task for these subtitles"),
                       task_type=task.get_type_display())

def get_team_attributes_for_editor(video):
    team_video = video.get_team_video()
    if team_video:
        team = team_video.team
        return dict([('teamName', team.name), ('guidelines', dict(
            [(s.key_name.split('_', 1)[-1],
              linebreaks(urlize(force_escape(s.data))))
             for s in team.settings.guidelines()
             if s.data.strip()]))])
    else:
        return None

def old_editor(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = get_object_or_404(SubtitleLanguage, video=video,
                                 language_code=language_code)
    url_path = shims.get_widget_url(language,
                                    request.GET.get('mode'),
                                    request.GET.get('task_id'))
    return redirect("http://%s%s" % (request.get_host(), url_path))

class SubtitleEditorBase(View):
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(SubtitleEditorBase, self).dispatch(
            request, *args, **kwargs)

    def get(self, request, video_id, language_code):
        video = get_object_or_404(Video, video_id=video_id)

        if (video.primary_audio_language_code and
            SubtitleVersion.objects.extant().filter(
                video=video, language_code=video.primary_audio_language_code)
            .exists()):
            base_language = video.primary_audio_language_code
        else:
            base_language = None

        try:
            editing_language = video.newsubtitlelanguage_set.get(language_code=language_code)
        except SubtitleLanguage.DoesNotExist:
            editing_language = SubtitleLanguage(video=video,language_code=language_code)

        if not editing_language.can_writelock(request.browser_id):
            messages.error(request, _("You can't edit this subtitle because it's locked"))
            return redirect(video)

        error_message = assign_task_for_editor(video, language_code, request.user)
        if error_message:
            messages.error(request, error_message)
            return redirect(video)
        team_video = video.get_team_video()
        if team_video is not None:
            task = team_video.get_task_for_editor(language_code)
        else:
            task = None
        check_result = can_add_version(request.user, video, language_code)
        if not check_result:
            messages.error(request, check_result.message)
            return redirect(video)

        editing_language.writelock(request.user, request.browser_id, save=True)

        # if this language is a translation, show both
        editing_version = editing_language.get_tip(public=False)
        # we ignore forking because even if it *is* a fork, we still want to show
        # the user the rererence languages:
        translated_from_version = editing_language.\
            get_translation_source_version(ignore_forking=True)

        languages = video.newsubtitlelanguage_set.annotate(
            num_versions=Count('subtitleversion'))

        workflow = get_workflow(video)
        if 'video_url' in request.GET:
            video_urls = [request.GET['video_url']]
        else:
            video_urls = workflow.editor_video_urls(language_code)

        editor_data = {
            'canSync': bool(request.GET.get('canSync', True)),
            'canAddAndRemove': bool(request.GET.get('canAddAndRemove', True)),
            # front end needs this to be able to set the correct
            # api headers for saving subs
            'authHeaders': {
                'x-api-username': request.user.username,
                'x-apikey': request.user.get_api_key()
            },
            'username': request.user.username,
            'video': {
                'id': video.video_id,
                'title': video.title,
                'description': video.description,
                'primaryVideoURL': video.get_video_url(),
                'videoURLs': video_urls,
                'metadata': video.get_metadata(),
            },
            'editingVersion': {
                'languageCode': editing_language.language_code,
                'versionNumber': (editing_version.version_number
                                  if editing_version else None),
            },
            'baseLanguage': base_language,
            'languages': [_language_data(lang, editing_version,
                                         translated_from_version, base_language)
                          for lang in languages],
            'languageCode': request.LANGUAGE_CODE,
            'oldEditorURL': reverse('subtitles:old-editor', kwargs={
                'video_id': video.video_id,
                'language_code': editing_language.language_code,
            }),
            'staticURL': settings.STATIC_URL,
            'notesHeading': 'Editor Notes',
        }

        editor_data.update(workflow.editor_data(request.user, language_code))

        if task:
            editor_data['task_id'] = task.id
            editor_data['savedNotes'] = task.body
            editor_data['task_needs_pane'] = task.get_type_display() in ('Review', 'Approve')
            editor_data['team_slug'] = task.team.slug
            editor_data['oldEditorURL'] += '?' + urlencode({
                'mode': Task.TYPE_NAMES[task.type].lower(),
                'task_id': task.id,
            })

        team_attributes = get_team_attributes_for_editor(video)
        if team_attributes:
            editor_data['teamAttributes'] = team_attributes

        return render(request, "editor/editor.html", {
            'video': video,
            'DEBUG': settings.DEBUG,
            'language': editing_language,
            'other_languages': languages,
            'version': editing_version,
            'translated_from_version': translated_from_version,
            'task': task,
            'editor_data': json.dumps(editor_data, indent=4),
            'upload_subtitles_form': SubtitlesUploadForm(
                request.user, video,
                initial={'language_code': editing_language.language_code})
        })

class SubtitleEditor(SubtitleEditorBase):
    @method_decorator(xframe_options_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(SubtitleEditor, self).dispatch(
            request, *args, **kwargs)

def download(request, video_id, language_code, filename, format,
             version_number=None):

    video = get_object_or_404(Video, video_id=video_id)

    language = video.subtitle_language(language_code)
    if language is None:
        raise Http404()

    team_video = video.get_team_video()

    if team_video and not team_video.team.user_is_member(request.user):
        # Non-members can only see public versions
        version = language.version(public_only=True,
                                   version_number=version_number)
    else:
        version = language.version(public_only=False,
                                   version_number=version_number)

    if not version:
        raise Http404()
    if not format in babelsubs.get_available_formats():
        raise HttpResponseServerError("Format not found")

    subs_text = babelsubs.to(version.get_subtitles(), format,
                             language=version.language_code)
    # since this is a downlaod, we can afford not to escape tags, specially
    # true since speaker change is denoted by '>>' and that would get entirely
    # stripped out
    response = HttpResponse(subs_text, mimetype="text/plain")
    response['Content-Disposition'] = 'attachment'
    return response

def download_all(request, video_id, filename):
    video = get_object_or_404(Video, video_id=video_id)
    merged_dfxp = video.get_merged_dfxp()

    if merged_dfxp is None:
        raise Http404()

    response = HttpResponse(merged_dfxp, mimetype="text/plain")
    response['Content-Disposition'] = 'attachment'
    return response
<commit_msg>Refactored subtitle editor view some more<commit_after># -*- coding: utf-8 -*-
# Amara, universalsubtitles.org
#
# Copyright (C) 2013 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program.  If not, see http://www.gnu.org/licenses/agpl-3.0.html.

import simplejson as json

import babelsubs
from django.contrib.auth.decorators import login_required
from django.conf import settings
from django.http import HttpResponse, Http404
from django.db.models import Count
from django.conf import settings
from django.contrib import messages
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.utils.decorators import method_decorator
from django.utils.http import urlencode
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django.views.generic import View
from django.shortcuts import render, get_object_or_404, redirect
from django.template.defaultfilters import urlize, linebreaks, force_escape
from django.views.decorators.clickjacking import xframe_options_exempt

from subtitles import shims
from subtitles.workflows import get_workflow
from subtitles.models import SubtitleLanguage, SubtitleVersion
from subtitles.templatetags.new_subtitles_tags import visibility
from subtitles.forms import SubtitlesUploadForm
from teams.models import Task
from teams.permissions import can_add_version, can_assign_task
from utils.text import fmt
from videos.models import Video

def _version_data(version):
    '''
    Creates a dict with version info, suitable for encoding
    into json and bootstrapping the editor.
    '''
    return {
        'metadata': version.get_metadata(),
        'subtitles': version.get_subtitles().to_xml(),
        'title': version.title,
        'description': version.description,
    }

def regain_lock(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = video.subtitle_language(language_code)

    if not language.can_writelock(request.browser_id):
        return HttpResponse(json.dumps({'ok': False}))

    language.writelock(request.user, request.browser_id, save=True)
    return HttpResponse(json.dumps({'ok': True}))

@login_required
@require_POST
def release_lock(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = video.subtitle_language(language_code)

    if language.can_writelock(request.browser_id):
        language.release_writelock()

    return HttpResponse(json.dumps({'url': reverse('videos:video', args=(video_id,))}))

def old_editor(request, video_id, language_code):
    video = get_object_or_404(Video, video_id=video_id)
    language = get_object_or_404(SubtitleLanguage, video=video,
                                 language_code=language_code)
    url_path = shims.get_widget_url(language,
                                    request.GET.get('mode'),
                                    request.GET.get('task_id'))
    return redirect("http://%s%s" % (request.get_host(), url_path))

class SubtitleEditorBase(View):
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(SubtitleEditorBase, self).dispatch(
            request, *args, **kwargs)

    def get_video_urls(self):
        """Get video URLs to send to the editor."""
        return self.workflow.editor_video_urls(self.language_code)

    def calc_base_language(self):
        if (self.video.primary_audio_language_code and
            SubtitleVersion.objects.extant().filter(
                video=self.video,
                language_code=self.video.primary_audio_language_code)
            .exists()):
            self.base_language = self.video.primary_audio_language_code
        else:
            self.base_language = None

    def calc_editing_language(self):
        self.editing_language = self.video.subtitle_language(self.language_code)
        if self.editing_language is None:
            self.editing_language = SubtitleLanguage(
                video=self.video, language_code=self.language_code)

    def check_can_writelock(self):
        if not self.editing_language.can_writelock(self.request.browser_id):
            msg = _("You can't edit this subtitle because it's locked")
            messages.error(self.request, msg)
            return False
        else:
            return True

    def get_editor_data(self):
        editor_data = {
            'canSync': bool(self.request.GET.get('canSync', True)),
            'canAddAndRemove': bool(self.request.GET.get('canAddAndRemove', True)),
            # front end needs this to be able to set the correct
            # api headers for saving subs
            'authHeaders': {
                'x-api-username': self.request.user.username,
                'x-apikey': self.request.user.get_api_key()
            },
            'username': self.request.user.username,
            'video': {
                'id': self.video.video_id,
                'title': self.video.title,
                'description': self.video.description,
                'primaryVideoURL': self.video.get_video_url(),
                'videoURLs': self.get_video_urls(),
                'metadata': self.video.get_metadata(),
            },
            'editingVersion': {
                'languageCode': self.editing_language.language_code,
                'versionNumber': (self.editing_version.version_number
                                  if self.editing_version else None),
            },
            'baseLanguage': self.base_language,
            'languages': [self.editor_data_for_language(lang)
                          for lang in self.languages],
            'languageCode': self.request.LANGUAGE_CODE,
            'oldEditorURL': reverse('subtitles:old-editor', kwargs={
                'video_id': self.video.video_id,
                'language_code': self.editing_language.language_code,
            }),
            'staticURL': settings.STATIC_URL,
            'notesHeading': 'Editor Notes',
        }

        editor_data.update(self.workflow.editor_data(
            self.user, self.language_code))

        team_attributes = self.get_team_editor_data()
        if team_attributes:
            editor_data['teamAttributes'] = team_attributes

        return editor_data

    def editor_data_for_language(self, language):
        versions_data = []

        versions = list(language.subtitleversion_set.full())
        for i, version in enumerate(versions):
            version_data = {
                'version_no':version.version_number,
                'visibility': visibility(version),
            }
            if self.editing_version == version:
                version_data.update(_version_data(version))
            elif self.translated_from_version == version:
                version_data.update(_version_data(version))
            elif (language.language_code == self.base_language and
                  i == len(versions) - 1):
                version_data.update(_version_data(version))

            versions_data.append(version_data)

        return {
            'translatedFrom': self.translated_from_version and {
                'language_code': self.translated_from_version.subtitle_language.language_code,
                'version_number': self.translated_from_version.version_number,
            },
            'editingLanguage': language == self.editing_language,
            'language_code': language.language_code,
            'name': language.get_language_code_display(),
            'pk': language.pk,
            'numVersions': language.num_versions,
            'versions': versions_data,
            'subtitles_complete': language.subtitles_complete,
            'is_rtl': language.is_rtl(),
            'is_original': language.is_primary_audio_language()
        }

    def get_team_editor_data(self):
        if self.team_video:
            team = self.team_video.team
            return dict([('teamName', team.name), ('guidelines', dict(
                [(s.key_name.split('_', 1)[-1],
                  linebreaks(urlize(force_escape(s.data))))
                 for s in team.settings.guidelines()
                 if s.data.strip()]))])
        else:
            return None

    def assign_task_for_editor(self):
        """Try to assign any unassigned tasks to our user.

        If we can't assign the task, return False.
        """
        if self.team_video is None:
            return True
        task_set = self.team_video.task_set.incomplete().filter(
            language=self.language_code)
        tasks = list(task_set[:1])
        if tasks:
            task = tasks[0]
            if task.assignee is None and can_assign_task(task, self.user):
                task.assignee = self.user
                task.set_expiration()
                task.save()

            if task.assignee != self.user:
                msg = fmt(_("Another user is currently performing "
                            "the %(task_type)s task for these subtitles"),
                          task_type=task.get_type_display())
                messages.error(self.request, msg)
                return False
        return True

    def handle_task(self, context, editor_data):
        """Does most of the dirty-work to handle tasks.  """
        context['task'] = None
        if self.team_video is None:
            return

        task = self.team_video.get_task_for_editor(self.language_code)
        if not task:
            return
        context['task'] = task
        editor_data['task_id'] = task.id
        editor_data['savedNotes'] = task.body
        editor_data['task_needs_pane'] = task.get_type_display() in ('Review', 'Approve')
        editor_data['team_slug'] = task.team.slug
        editor_data['oldEditorURL'] += '?' + urlencode({
            'mode': Task.TYPE_NAMES[task.type].lower(),
            'task_id': task.id,
        })

    def get(self, request, video_id, language_code):
        self.video = get_object_or_404(Video, video_id=video_id)
        self.team_video = self.video.get_team_video()
        self.language_code = language_code
        self.user = request.user
        self.calc_base_language()
        self.calc_editing_language()

        if not self.check_can_writelock():
            return redirect(self.video)
        if not self.assign_task_for_editor():
            return redirect(self.video)
        check_result = can_add_version(request.user, self.video, language_code)
        if not check_result:
            messages.error(request, check_result.message)
            return redirect(self.video)

        self.editing_language.writelock(self.user, self.request.browser_id,
                                        save=True)
        self.editing_version = self.editing_language.get_tip(public=False)
        # we ignore forking because even if it *is* a fork, we still want to
        # show the user the rererence languages:
        self.translated_from_version = self.editing_language.\
            get_translation_source_version(ignore_forking=True)
        self.languages = self.video.newsubtitlelanguage_set.annotate(
            num_versions=Count('subtitleversion'))
        self.workflow = get_workflow(self.video)
        editor_data = self.get_editor_data()

        context = {
            'video': self.video,
            'DEBUG': settings.DEBUG,
            'language': self.editing_language,
            'other_languages': self.languages,
            'version': self.editing_version,
            'translated_from_version': self.translated_from_version,
            'upload_subtitles_form': SubtitlesUploadForm(
                request.user, self.video,
                initial={'language_code': self.editing_language.language_code})
        }
        self.handle_task(context, editor_data)
        context['editor_data'] = json.dumps(editor_data, indent=4)

        return render(request, "editor/editor.html", context)

class SubtitleEditor(SubtitleEditorBase):
    @method_decorator(xframe_options_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(SubtitleEditor, self).dispatch(
            request, *args, **kwargs)

def download(request, video_id, language_code, filename, format,
             version_number=None):

    video = get_object_or_404(Video, video_id=video_id)

    language = video.subtitle_language(language_code)
    if language is None:
        raise Http404()

    team_video = video.get_team_video()

    if team_video and not team_video.team.user_is_member(request.user):
        # Non-members can only see public versions
        version = language.version(public_only=True,
                                   version_number=version_number)
    else:
        version = language.version(public_only=False,
                                   version_number=version_number)

    if not version:
        raise Http404()
    if not format in babelsubs.get_available_formats():
        raise HttpResponseServerError("Format not found")

    subs_text = babelsubs.to(version.get_subtitles(), format,
                             language=version.language_code)
    # since this is a downlaod, we can afford not to escape tags, specially
    # true since speaker change is denoted by '>>' and that would get entirely
    # stripped out
    response = HttpResponse(subs_text, mimetype="text/plain")
    response['Content-Disposition'] = 'attachment'
    return response

def download_all(request, video_id, filename):
    video = get_object_or_404(Video, video_id=video_id)
    merged_dfxp = video.get_merged_dfxp()

    if merged_dfxp is None:
        raise Http404()

    response = HttpResponse(merged_dfxp, mimetype="text/plain")
    response['Content-Disposition'] = 'attachment'
    return response
<|endoftext|>


<commit_before>"""
Imgur CLI
"""

import argparse
import logging
import os
import sys

from collections import namedtuple

import imgurpython

from . import __version__
from imgur_cli.exceptions import CommandError
from imgur_cli.utils import cli_arg

try:
    from imgur_cli.config import config
except ImportError:
    config = None

logger = logging.getLogger(__name__)

def imgur_credentials():
    ImgurCredentials = namedtuple('ImgurCredentials',
                                  ['client_id', 'client_secret', 'access_token',
                                   'refresh_token', 'mashape_key'])

    if config:
        client_id = config.get('IMGUR_CLIENT_ID')
        client_secret = config.get('IMGUR_CLIENT_SECRET')
        access_token = config.get('IMGUR_ACCESS_TOKEN')
        refresh_token = config.get('IMGUR_REFRESH_TOKEN')
        mashape_key = config.get('IMGUR_MASHAPE_KEY')
    else:
        client_id = os.environ.get('IMGUR_CLIENT_ID')
        client_secret = os.environ.get('IMGUR_CLIENT_SECRET')
        access_token = os.environ.get('IMGUR_ACCESS_TOKEN')
        refresh_token = os.environ.get('IMGUR_REFRESH_TOKEN')
        mashape_key = os.environ.get('IMGUR_MASHAPE_KEY')

    if not client_id or not client_secret:
        raise imgurpython.client.ImgurClientError('Client credentials not found. '
                                                  'Ensure you have both client id '
                                                  'and client secret')

    return ImgurCredentials(client_id, client_secret, access_token,
                            refresh_token, mashape_key)

class ImgurCli():

    @property
    def base_parser(self):
        parser = argparse.ArgumentParser(prog='imgur', description=__doc__.strip(),
                                         epilog='See "imgur help COMMAND" for '
                                         'help on a specific command')

        # Global arguments
        parser.add_argument('-v', '--version', action='version',
                            version='%(prog)s {0}'.format(__version__))

        parser.add_argument('--debug', default=False, action='store_true',
                            help='Print debugging output')

        return parser

    def setup_debugging(self):
        streamformat = "%(levelname)s (%(module)s:%(lineno)d) %(message)s"
        # Set up the root logger to debug so that the submodules can print
        # debug messages
        logging.basicConfig(level=logging.DEBUG, format=streamformat)

    @property
    def subcommand_parser(self):
        pass

    def _find_actions(self, subparsers, actions_module):
        pass

    def _add_base_completion_subparser(self, subparsers):
        pass

    def do_bash_completion(self):
        """Prints arguments for bash-completion"""
        pass

    def do_help(self, args):
        """Display help about this program or one of its subcommands"""
        pass

    def main(self, argv):
        pass

def main():
    try:
        imgur_cli = ImgurCli()
        imgur_cli.main(sys.argv[1:])
    except Exception as e:
        logger.debug(e, exc_info=1)
        print(e, file=sys.stderr)
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>Define method to add help for subcommand<commit_after>"""
Imgur CLI
"""

import argparse
import logging
import os
import sys

from collections import namedtuple

import imgurpython

from . import __version__
from imgur_cli.exceptions import CommandError
from imgur_cli.utils import cli_arg

try:
    from imgur_cli.config import config
except ImportError:
    config = None

logger = logging.getLogger(__name__)

def imgur_credentials():
    ImgurCredentials = namedtuple('ImgurCredentials',
                                  ['client_id', 'client_secret', 'access_token',
                                   'refresh_token', 'mashape_key'])

    if config:
        client_id = config.get('IMGUR_CLIENT_ID')
        client_secret = config.get('IMGUR_CLIENT_SECRET')
        access_token = config.get('IMGUR_ACCESS_TOKEN')
        refresh_token = config.get('IMGUR_REFRESH_TOKEN')
        mashape_key = config.get('IMGUR_MASHAPE_KEY')
    else:
        client_id = os.environ.get('IMGUR_CLIENT_ID')
        client_secret = os.environ.get('IMGUR_CLIENT_SECRET')
        access_token = os.environ.get('IMGUR_ACCESS_TOKEN')
        refresh_token = os.environ.get('IMGUR_REFRESH_TOKEN')
        mashape_key = os.environ.get('IMGUR_MASHAPE_KEY')

    if not client_id or not client_secret:
        raise imgurpython.client.ImgurClientError('Client credentials not found. '
                                                  'Ensure you have both client id '
                                                  'and client secret')

    return ImgurCredentials(client_id, client_secret, access_token,
                            refresh_token, mashape_key)

class ImgurCli():

    @property
    def base_parser(self):
        parser = argparse.ArgumentParser(prog='imgur', description=__doc__.strip(),
                                         epilog='See "imgur help COMMAND" for '
                                         'help on a specific command')

        # Global arguments
        parser.add_argument('-v', '--version', action='version',
                            version='%(prog)s {0}'.format(__version__))

        parser.add_argument('--debug', default=False, action='store_true',
                            help='Print debugging output')

        return parser

    def setup_debugging(self):
        streamformat = "%(levelname)s (%(module)s:%(lineno)d) %(message)s"
        # Set up the root logger to debug so that the submodules can print
        # debug messages
        logging.basicConfig(level=logging.DEBUG, format=streamformat)

    @property
    def subcommand_parser(self):
        pass

    def _find_actions(self, subparsers, actions_module):
        pass

    def _add_base_completion_subparser(self, subparsers):
        pass

    def do_bash_completion(self):
        """Prints arguments for bash-completion"""
        pass

    @cli_arg('command', metavar='<subcommand', nargs='?',
             help='Display help for <subcommand>')
    def do_help(self, args):
        """Display help about this program or one of its subcommands"""
        print('do_help', args.command)
        if args.command:
            if args.command in self.subcommands:
                self.subcommands[args.command].print_help()
            else:
                raise CommandError('{0} is not valid subcommand'
                                   .format(args.command))
        else:
            self.parser.print_help()

    def main(self, argv):
        pass

def main():
    try:
        imgur_cli = ImgurCli()
        imgur_cli.main(sys.argv[1:])
    except Exception as e:
        logger.debug(e, exc_info=1)
        print(e, file=sys.stderr)
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright 2017 Camptocamp SA
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)

{
    "name": "Components",
    "summary": "Add capabilities to register and use decoupled components,"
    " as an alternative to model classes",
    "version": "14.0.1.0.2",
    "author": "Camptocamp," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/connector",
    "license": "LGPL-3",
    "category": "Generic Modules",
    "depends": ["base"],
    "external_dependencies": {
        "python": [
            "cachetools",
        ]
    },
    "installable": True,
    "development_status": "Production/Stable",
    "maintainers": ["guewen"],
}
<commit_msg>component 14.0.1.1.0<commit_after># Copyright 2017 Camptocamp SA
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)

{
    "name": "Components",
    "summary": "Add capabilities to register and use decoupled components,"
    " as an alternative to model classes",
    "version": "14.0.1.1.0",
    "author": "Camptocamp," "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/connector",
    "license": "LGPL-3",
    "category": "Generic Modules",
    "depends": ["base"],
    "external_dependencies": {
        "python": [
            "cachetools",
        ]
    },
    "installable": True,
    "development_status": "Production/Stable",
    "maintainers": ["guewen"],
}
<|endoftext|>


<commit_before>import csv
import itertools
import os

from demands import HTTPServiceError
from django.core.management import BaseCommand, CommandError
from django.core.validators import URLValidator
from requests import exceptions
from structlog import get_logger

def mk_validator(validator_class):
    def validator_callback(input_str):
        validator = validator_class()
        validator(input_str)
        return input_str
    return validator_callback

class Command(BaseCommand):

    help = ("Make HTTP requests to a ndoh-hub instance to initiate opt-out "
            "Changes for MSISDNs read from a CSV file (expected format "
            "<id>;<msisdn>;<count>)")

    def add_arguments(self, parser):
        parser.add_argument(
            '--csv', type=file,
            help='CSV file to parse for MSISDNs')
        parser.add_argument(
            '--hub-token', type=str,
            default=os.environ.get('HUB_TOKEN'),
            help=('The authorization token for the NDoH Hub'))
        parser.add_argument(
            '--hub-url',
            type=mk_validator(URLValidator),
            default=os.environ.get('HUB_URL'),
            help=('The URL for the NDoH Hub API'))
        parser.add_argument(
            '--identity-store-token', type=str,
            default=os.environ.get('IDENTITY_STORE_TOKEN'),
            help='The token for the ID Store')
        parser.add_argument(
            '--identity-store-url',
            type=mk_validator(URLValidator),
            default=os.environ.get('IDENTITY_STORE_URL'),
            help='The Identity Store API URL')
        parser.add_argument(
            '--start', type=int,
            help='The row number to start from')
        parser.add_argument(
            '--end', type=int,
            help='The row number to end on (non-inclusive). Requires --start')

    def handle(self, *args, **options):
        identity_store_token = options['identity_store_token']
        identity_store_url = options['identity_store_url']
        hub_token = options['hub_token']
        hub_url = options['hub_url']
        csv_file = options['csv']
        start = options['start']
        end = options['end']
        log = get_logger()

        if not csv_file:
            raise CommandError('--csv is a required parameter')

        if not hub_url:
            raise CommandError('--hub-url is a required parameter')

        if not hub_token:
            raise CommandError('--hub-token is a required parameter')

        if not identity_store_url:
            raise CommandError('--identity-store-url is a required parameter')

        if not identity_store_token:
            raise CommandError('--identity-store-token is a required '
                               'parameter')

        from seed_services_client import IdentityStoreApiClient, HubApiClient

        ids_client = IdentityStoreApiClient(identity_store_token,
                                            identity_store_url)
        hub_client = HubApiClient(hub_token, hub_url)

        csv_reader = csv.reader(csv_file, delimiter=';')

        if start and end:
            rows = itertools.islice(csv_reader, start, end)
            row_count = start
        elif start and not end:
            rows = itertools.islice(csv_reader, start)
            row_count = start
        elif end and not start:
            raise CommandError('--start is a required parameter when '
                               'specifying --end')
        else:
            rows = csv_reader
            row_count = 1
            # skip the header row
            next(csv_reader)

        for idx, row in enumerate(rows, start=row_count):
            loop_log = log.bind(row=idx)
            msisdn = '+{0}'.format(row[1])
            loop_log = loop_log.bind(msisdn=msisdn)
            result = ids_client.get_identity_by_address('msisdn', msisdn)
            if not result or 'count' not in result or result['count'] == 0:
                loop_log.error('Could not load identity for msisdn.')
                continue

            if result['count'] > 1:
                msg = 'Warning: Found {0} identities'
                msg = msg.format(result['count'])
                loop_log.warn(msg)
            identity = result['results'][0]
            # Use this logger only for this iteration of the loop
            id_log = loop_log.bind(identity=identity['id'])
            change = {
                'registrant_id': identity['id'],
                'action': 'momconnect_nonloss_optout',
                'data': {
                    'reason': 'sms_failure'
                }
            }
            try:
                result = hub_client.create_change(change)
            except exceptions.ConnectionError as exc:
                id_log.error('Connection error to Hub API: {}'
                             .format(exc.message))
                break
            except HTTPServiceError as exc:
                id_log.error('Invalid Hub API response', url=exc.response.url,
                             status_code=exc.response.status_code)
                break

            if result:
                id_log.info('Successfully submitted changed.')
            else:
                id_log.error('Change failed', response=result)
<commit_msg>Handle file opening more gracefully<commit_after>import csv
import itertools
import os

from demands import HTTPServiceError
from django.core.management import BaseCommand, CommandError
from django.core.validators import URLValidator
from requests import exceptions
from seed_services_client import IdentityStoreApiClient, HubApiClient
from structlog import get_logger

def mk_validator(validator_class):
    def validator_callback(input_str):
        validator = validator_class()
        validator(input_str)
        return input_str
    return validator_callback

class Command(BaseCommand):

    help = ("Make HTTP requests to a ndoh-hub instance to initiate opt-out "
            "Changes for MSISDNs read from a CSV file (expected format "
            "<id>;<msisdn>;<count>)")

    def add_arguments(self, parser):
        parser.add_argument(
            '--csv', type=str,
            help='CSV file to parse for MSISDNs')
        parser.add_argument(
            '--hub-token', type=str,
            default=os.environ.get('HUB_TOKEN'),
            help=('The authorization token for the NDoH Hub'))
        parser.add_argument(
            '--hub-url',
            type=mk_validator(URLValidator),
            default=os.environ.get('HUB_URL'),
            help=('The URL for the NDoH Hub API'))
        parser.add_argument(
            '--identity-store-token', type=str,
            default=os.environ.get('IDENTITY_STORE_TOKEN'),
            help='The token for the ID Store')
        parser.add_argument(
            '--identity-store-url',
            type=mk_validator(URLValidator),
            default=os.environ.get('IDENTITY_STORE_URL'),
            help='The Identity Store API URL')
        parser.add_argument(
            '--start', type=int,
            help='The row number to start from')
        parser.add_argument(
            '--end', type=int,
            help='The row number to end on (non-inclusive). Requires --start')

    def handle(self, *args, **options):
        identity_store_token = options['identity_store_token']
        identity_store_url = options['identity_store_url']
        hub_token = options['hub_token']
        hub_url = options['hub_url']
        file_name = options['csv']
        start = options['start']
        end = options['end']
        log = get_logger()

        if not file_name:
            raise CommandError('--csv is a required parameter')

        if not os.path.isfile(file_name):
            raise CommandError('--csv must be a valid path to a file')

        if not hub_url:
            raise CommandError('--hub-url is a required parameter')

        if not hub_token:
            raise CommandError('--hub-token is a required parameter')

        if not identity_store_url:
            raise CommandError('--identity-store-url is a required parameter')

        if not identity_store_token:
            raise CommandError('--identity-store-token is a required '
                               'parameter')

        ids_client = IdentityStoreApiClient(identity_store_token,
                                            identity_store_url)
        hub_client = HubApiClient(hub_token, hub_url)

        with open(file_name) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=';')

            if start and end:
                rows = itertools.islice(csv_reader, start, end)
                row_count = start
            elif start and not end:
                rows = itertools.islice(csv_reader, start)
                row_count = start
            elif end and not start:
                raise CommandError('--start is a required parameter when '
                                   'specifying --end')
            else:
                rows = csv_reader
                row_count = 1
                # skip the header row
                next(csv_reader)

            for idx, row in enumerate(rows, start=row_count):
                loop_log = log.bind(row=idx)
                msisdn = '+{0}'.format(row[1])
                loop_log = loop_log.bind(msisdn=msisdn)
                result = ids_client.get_identity_by_address('msisdn', msisdn)
                if not result or 'count' not in result or result['count'] == 0:
                    loop_log.error('Could not load identity for msisdn.')
                    continue

                if result['count'] > 1:
                    msg = 'Warning: Found {0} identities'
                    msg = msg.format(result['count'])
                    loop_log.warn(msg)
                identity = result['results'][0]
                # Use this logger only for this iteration of the loop
                id_log = loop_log.bind(identity=identity['id'])
                change = {
                    'registrant_id': identity['id'],
                    'action': 'momconnect_nonloss_optout',
                    'data': {
                        'reason': 'sms_failure'
                    }
                }
                try:
                    result = hub_client.create_change(change)
                except exceptions.ConnectionError as exc:
                    id_log.error('Connection error to Hub API: {}'
                                 .format(exc.message))
                    break
                except HTTPServiceError as exc:
                    id_log.error('Invalid Hub API response',
                                 url=exc.response.url,
                                 status_code=exc.response.status_code)
                    break

                if result:
                    id_log.info('Successfully submitted changed.')
                else:
                    id_log.error('Change failed', response=result)
<|endoftext|>


<commit_before>from xml.etree import ElementTree as ET
from datetime import datetime, timedelta

tp_ns = '{http://www.peaksware.com/PWX/1/0}'

def summary_attrib(summary, attrib, type):
    try:
        summarydata = summary.find(tp_ns + type)
        value = summarydata.attrib[attrib]
    except AttributeError:
        value = 0
    return value

def sample_value(sample, type):
    try:
        value = sample.find(tp_ns + type).text
    except AttributeError:
        value = 0
    return value

class PWXEntry(object):
    def __init__(self, time, hr, speed, cadence, power, torque, temp, altitude, lat, lon):
        self.time = time
        self.hr = hr
        self.speed = speed
        self.cadence = cadence
        self.power = power
        self.torque = torque
        self.temp = temp
        self.altitude = altitude
        self.lon = lon
        self.lat = lat

    def __str__(self):
        return '[%s] hr: %s spd: %s cad %s pwr %s torque: %s temp: %s alt: %s' % (self.time, self.hr, self.speed, self.cadence, self.power, self.torque, self.temp, self.altitude)

class PWXParser(object):
    def __init__(self, gps_distance=False):
        self.entries = []
        self.distance = 0.0
        self.ascent = 0.0
        self.descent = 0.0
        self.max_speed = 0
        self.avg_speed = 0
        self.max_hr = 0
        self.avg_hr = 0
        self.avg_cadence = 0
        self.max_cadence = 0
        self.avg_power = 0
        self.max_power = 0
        self.avg_torque = 0.0
        self.max_torque = 0.0
        self.pedaling_cad_seconds = 0
        self.pedaling_cad = 0
        self.pedaling_power = 0
        self.pedaling_power_seconds = 0
        self.avg_temp = 0.0
        self.max_temp = 0.0
        self.kcal_sum = 0

    def parse_uploaded_file(self, f):
        doc = ET.parse(f)
        workout = doc.find(tp_ns + "workout")

        start = datetime.strptime(workout.find(tp_ns + "time").text,"%Y-%m-%dT%H:%M:%S")

        self.start_time = start.time()
        self.date = start.date()

        summary = workout.find(tp_ns + "summarydata")
        self.duration = '%ss' % (int(summary.find(tp_ns + "duration").text))
        try:
            self.distance = float(summary.find(tp_ns + "dist").text)
        except AttributeError:
            self.distance = 0.0
        self.avg_hr      = int(summary_attrib(summary, "avg", "hr"))
        self.max_hr      = int(summary_attrib(summary, "max", "hr"))
        self.avg_speed   = float(summary_attrib(summary, "avg", "spd"))*3.6
        self.max_speed   = float(summary_attrib(summary, "max", "spd"))*3.6
        self.avg_cadence = int(summary_attrib(summary, "avg", "cad"))
        self.max_cadence = int(summary_attrib(summary, "max", "cad"))
        self.avg_power   = int(summary_attrib(summary, "avg", "pwr"))
        self.max_power   = int(summary_attrib(summary, "max", "pwr"))
        self.avg_torque  = float(summary_attrib(summary, "avg", "torq"))
        self.max_torque  = float(summary_attrib(summary, "max", "torq"))
        self.avg_temp    = float(summary_attrib(summary, "avg", "temp"))
        self.max_temp    = float(summary_attrib(summary, "max", "temp"))
        self.ascent = float(sample_value(summary, "climbingelevation"))
        self.kcal_sum = int(float(sample_value(summary, "work"))*0.239005736)

        samples = workout.getiterator(tp_ns + "sample")
        for sample in samples:
            time = start + timedelta(0, int(sample.find(tp_ns + "timeoffset").text))

            hr     = int(sample_value(sample, "hr"))
            spd    = float(sample_value(sample, "spd"))*3.6
            cad    = int(sample_value(sample, "cad"))
            pwr    = int(sample_value(sample, "pwr"))
            torque = float(sample_value(sample, "torq"))
            temp   = float(sample_value(sample, "temp"))
            alt    = float(sample_value(sample, "alt"))
            lat    = float(sample_value(sample, "lat"))
            lon    = float(sample_value(sample, "lon"))

            self.entries.append(PWXEntry(time,hr,spd,cad,pwr,torque,temp,alt, lat, lon))

        last = self.entries[0].time
        for e in self. entries:
            interval = (e.time - last).seconds
            if e.cadence > 0:
                self.pedaling_cad += e.cadence*interval
                self.pedaling_cad_seconds += interval
            if e.power > 0:
                self.pedaling_power += e.power*interval
                self.pedaling_power_seconds += interval
            last = e.time
        if self.pedaling_cad and self.pedaling_cad_seconds:
            self.avg_pedaling_cad = self.pedaling_cad/self.pedaling_cad_seconds
        if self.pedaling_power and self.pedaling_power_seconds:
            self.avg_pedaling_power = self.pedaling_power/self.pedaling_power_seconds

if __name__ == '__main__':
    import pprint
    import sys
    t = PWXParser(0)
    t.parse_uploaded_file(file(sys.argv[1]))

    for pwx_e in t.entries:
        print pwx_e

    print 'start: %s %s - duration: %s' % (t.date, t.start_time, t.duration)
    print 'HR - avg: %s - max: %s' % (t.avg_hr, t.max_hr)
    print 'SPEED - avg: %s - max: %s' % (t.avg_speed, t.max_speed)
    print 'CADENCE - avg: %s - max: %s - pedal: %s' % (t.avg_cadence, t.max_cadence, t.avg_pedaling_cad)
    print 'POWER - avg: %s - max: %s - pedal: %s' % (t.avg_power, t.max_power, t.avg_pedaling_power)
    print 'TORQUE - avg: %s - max: %s' % (t.avg_torque, t.max_torque)
    print 'TEMP - avg: %s - max: %s' % (t.avg_temp, t.max_temp)
<commit_msg>Re-introduced support for absolute time on sample, with fallback to offset from start.<commit_after>from xml.etree import ElementTree as ET
from datetime import datetime, timedelta

tp_ns = '{http://www.peaksware.com/PWX/1/0}'

def summary_attrib(summary, attrib, type):
    try:
        summarydata = summary.find(tp_ns + type)
        value = summarydata.attrib[attrib]
    except AttributeError:
        value = 0
    return value

def sample_value(sample, type):
    try:
        value = sample.find(tp_ns + type).text
    except AttributeError:
        value = 0
    return value

class PWXEntry(object):
    def __init__(self, time, hr, speed, cadence, power, torque, temp, altitude, lat, lon):
        self.time = time
        self.hr = hr
        self.speed = speed
        self.cadence = cadence
        self.power = power
        self.torque = torque
        self.temp = temp
        self.altitude = altitude
        self.lon = lon
        self.lat = lat

    def __str__(self):
        return '[%s] hr: %s spd: %s cad %s pwr %s torque: %s temp: %s alt: %s' % (self.time, self.hr, self.speed, self.cadence, self.power, self.torque, self.temp, self.altitude)

class PWXParser(object):
    def __init__(self, gps_distance=False):
        self.entries = []
        self.distance = 0.0
        self.ascent = 0.0
        self.descent = 0.0
        self.max_speed = 0
        self.avg_speed = 0
        self.max_hr = 0
        self.avg_hr = 0
        self.avg_cadence = 0
        self.max_cadence = 0
        self.avg_power = 0
        self.max_power = 0
        self.avg_torque = 0.0
        self.max_torque = 0.0
        self.pedaling_cad_seconds = 0
        self.pedaling_cad = 0
        self.pedaling_power = 0
        self.pedaling_power_seconds = 0
        self.avg_temp = 0.0
        self.max_temp = 0.0
        self.kcal_sum = 0

    def parse_uploaded_file(self, f):
        doc = ET.parse(f)
        workout = doc.find(tp_ns + "workout")

        start = datetime.strptime(workout.find(tp_ns + "time").text,"%Y-%m-%dT%H:%M:%S")

        self.start_time = start.time()
        self.date = start.date()

        summary = workout.find(tp_ns + "summarydata")
        self.duration = '%ss' % (int(summary.find(tp_ns + "duration").text))
        try:
            self.distance = float(summary.find(tp_ns + "dist").text)
        except AttributeError:
            self.distance = 0.0
        self.avg_hr      = int(summary_attrib(summary, "avg", "hr"))
        self.max_hr      = int(summary_attrib(summary, "max", "hr"))
        self.avg_speed   = float(summary_attrib(summary, "avg", "spd"))*3.6
        self.max_speed   = float(summary_attrib(summary, "max", "spd"))*3.6
        self.avg_cadence = int(summary_attrib(summary, "avg", "cad"))
        self.max_cadence = int(summary_attrib(summary, "max", "cad"))
        self.avg_power   = int(summary_attrib(summary, "avg", "pwr"))
        self.max_power   = int(summary_attrib(summary, "max", "pwr"))
        self.avg_torque  = float(summary_attrib(summary, "avg", "torq"))
        self.max_torque  = float(summary_attrib(summary, "max", "torq"))
        self.avg_temp    = float(summary_attrib(summary, "avg", "temp"))
        self.max_temp    = float(summary_attrib(summary, "max", "temp"))
        self.ascent = float(sample_value(summary, "climbingelevation"))
        self.kcal_sum = int(float(sample_value(summary, "work"))*0.239005736)

        samples = workout.getiterator(tp_ns + "sample")
        for sample in samples:
            try:
                time = datetime.strptime(sample.find(tp_ns + "time").text,"%Y-%m-%dT%H:%M:%S")
            except AttributeError:
                time = start + timedelta(0, int(sample.find(tp_ns + "timeoffset").text))

            hr     = int(sample_value(sample, "hr"))
            spd    = float(sample_value(sample, "spd"))*3.6
            cad    = int(sample_value(sample, "cad"))
            pwr    = int(sample_value(sample, "pwr"))
            torque = float(sample_value(sample, "torq"))
            temp   = float(sample_value(sample, "temp"))
            alt    = float(sample_value(sample, "alt"))
            lat    = float(sample_value(sample, "lat"))
            lon    = float(sample_value(sample, "lon"))

            self.entries.append(PWXEntry(time,hr,spd,cad,pwr,torque,temp,alt, lat, lon))

        last = self.entries[0].time
        for e in self. entries:
            interval = (e.time - last).seconds
            if e.cadence > 0:
                self.pedaling_cad += e.cadence*interval
                self.pedaling_cad_seconds += interval
            if e.power > 0:
                self.pedaling_power += e.power*interval
                self.pedaling_power_seconds += interval
            last = e.time
        if self.pedaling_cad and self.pedaling_cad_seconds:
            self.avg_pedaling_cad = self.pedaling_cad/self.pedaling_cad_seconds
        if self.pedaling_power and self.pedaling_power_seconds:
            self.avg_pedaling_power = self.pedaling_power/self.pedaling_power_seconds

if __name__ == '__main__':
    import pprint
    import sys
    t = PWXParser(0)
    t.parse_uploaded_file(file(sys.argv[1]))

    for pwx_e in t.entries:
        print pwx_e

    print 'start: %s %s - duration: %s' % (t.date, t.start_time, t.duration)
    print 'HR - avg: %s - max: %s' % (t.avg_hr, t.max_hr)
    print 'SPEED - avg: %s - max: %s' % (t.avg_speed, t.max_speed)
    print 'CADENCE - avg: %s - max: %s - pedal: %s' % (t.avg_cadence, t.max_cadence, t.avg_pedaling_cad)
    print 'POWER - avg: %s - max: %s - pedal: %s' % (t.avg_power, t.max_power, t.avg_pedaling_power)
    print 'TORQUE - avg: %s - max: %s' % (t.avg_torque, t.max_torque)
    print 'TEMP - avg: %s - max: %s' % (t.avg_temp, t.max_temp)
<|endoftext|>