<commit_before>#!/usr/bin/python
''' remcom test module '''
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import argparse
import os
import time
from select import select
import multiprocessing as mp
from nose.tools import nottest

from .remove_commercials import remove_commercials
from .roku_utils import (make_thumbnails, make_audio_analysis_plots_wrapper,
                         make_time_series_plot_wrapper,
                         publish_transcode_job_to_queue, get_random_hex_string)
from .util import (run_command, send_command, OpenUnixSocketServer,
                   OpenSocketConnection, HOMEDIR)

REMCOM_SOCKET_FILE = '/tmp/.remcom_test_socket'
INPUT_DIR = '%s/Documents/movies' % HOMEDIR
OUTPUT_DIR = '/media/dileptonnas'

def remove_commercials_wrapper(input_file='', output_dir='', begin_time=0,
                               end_time=0):
    '''
        cut and splice recorded file to remove undesired sections
        (commercials), use temp not test here...
    '''
    input_string = '%d,%d' % (begin_time, end_time)
    output_file = '%s/%s' % (output_dir, os.path.basename(input_file))
    remove_commercials(input_file, output_file,
                       timing_string=input_string, do_async=True)
    return output_file

def make_video_script(input_file='', begin_time=0):
    ''' write script to create small test video file '''
    if not os.path.exists(input_file):
        return -1
    input_string = '%d,%d' % (begin_time, begin_time+10)
    tmpfile = '%s/dvdrip/tmp/test_%06x' % (HOMEDIR, get_random_hex_string(3))
    avifile = '%s.avi' % tmpfile
    mp4file = '%s.mp4' % tmpfile
    wavfile = '%s.wav' % tmpfile
    remove_commercials(infile=input_file, outfile=avifile,
                       timing_string=input_string)
    run_command('time HandBrakeCLI -i %s -f mp4 -e x264 ' % avifile +
                '-b 600 -o %s > /dev/null 2>&1\n' % mp4file)
    run_command('mpv --ao=pcm:file=%s --no-video ' % wavfile +
                '%s > /dev/null 2>&1\n' % avifile)
    run_command('mv %s %s/public_html/videos/temp.mp4\n' % (mp4file, HOMEDIR))
    os.remove(avifile)
    retval = make_audio_analysis_plots_wrapper(wavfile, prefix='temp')
    os.remove(wavfile)
    return retval

def make_transcode_script(input_file):
    """
        write out transcode file at the end of the recording,
        will be used to convert mpg to avi / mp4 later on
    """
    prefix = os.path.basename(input_file)
    for suffix in '.avi', '.mp4', '.mkv':
        prefix = prefix.replace(suffix, '')
    output_file = '%s/dvdrip/avi/%s.mp4' % (HOMEDIR, prefix)
    with open('%s/dvdrip/tmp/%s.sh' % (HOMEDIR, prefix), 'w') as outfile:
        outfile.write('#!/bin/bash\n')
        outfile.write('nice -n 19 HandBrakeCLI ')
        outfile.write('-i %s -o %s ' % (input_file, output_file))
        outfile.write('--preset "Android" ')
        outfile.write('> ~/dvdrip/log/%s_mp4.out 2>&1\n' % prefix)
        outfile.write('mv %s ~/Documents/movies/\n' % output_file)
        outfile.write('mv ~/dvdrip/log/%s_mp4.out ~/tmp_avi/\n' % prefix)

    os.rename('%s/dvdrip/tmp/%s.sh' % (HOMEDIR, prefix),
              '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix))
    return '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)

def remcom(movie_filename, output_dir, begin_time, end_time,
           msg_q=None, socketfile=REMCOM_SOCKET_FILE):
    '''
        main function, takes filename, timing file as options
        communication via socket:
            q: quit remcom
            f: move forward, number of minutes can be included, i.e.
                f5 moves forward 5 minues
            b: move backwards... b5 moves backwards 5 minutes
            s: select
            v: test video
            t: thumbnail
    '''
    with OpenUnixSocketServer(socketfile) as sock:
        while True:
            outstring = []
            with OpenSocketConnection(sock) as conn:
                args = conn.recv(1024)
                if hasattr(args, 'decode'):
                    args = args.decode()
                args = args.split()

                original_end_time = end_time

                for option in args:
                    if option == 'q':
                        conn.send(b'q')
                        msg_q.put('q')
                        return 0
                    elif option[0] == 'f':
                        try:
                            end_time += int(option[1:]) * 60
                        except ValueError:
                            end_time += 60
                    elif option[0] == 'b':
                        try:
                            end_time -= int(option[1:]) * 60
                        except ValueError:
                            end_time -= 60
                    elif option == 's':
                        output_file = remove_commercials_wrapper(
                            movie_filename, output_dir, begin_time, end_time)
                        make_transcode_script(output_file)
                        conn.send(b'done')
                        msg_q.put('q')
                        return 0
                    elif option == 'v':
                        outstring.append('%s' % make_video_script(
                            movie_filename, begin_time=end_time))
                    elif option == 't':
                        outstring.append(
                            make_time_series_plot_wrapper(movie_filename))
                    else:
                        try:
                            end_time = int(option) * 60
                            print('end_time: %s' % option)
                        except ValueError:
                            continue
                outstring.append(
                    '%s %s %s %s' % (os.path.basename(movie_filename),
                                     '/'.join(output_dir.split('/')[-2:]),
                                     begin_time, end_time))
                tmp_begin = make_thumbnails(input_file=movie_filename,
                                            begin_time=end_time,
                                            output_dir=HOMEDIR +
                                            '/public_html/videos/'
                                            'thumbnails_tv')
                outstring.append('%d, s/f/b/q/t:' % (tmp_begin / 60))
                outstring = '\n'.join(outstring)
                conn.send(outstring.encode())

        if original_end_time > end_time:
            print('original %s new %s args %s' % (original_end_time, end_time,
                                                  args))
    return 0

def keyboard_input():
    ''' keyboard input from stdin using select function '''
    while True:
        in_, _, _ = select([os.sys.stdin], [], [], 0.1)
        if not in_:
            yield None
        else:
            for s__ in in_:
                if s__ == os.sys.stdin:
                    yield os.sys.stdin.readline()

def remcom_main(movie_filename, output_dir, begin_time, end_time):
    ''' main recom_test function '''
    msg_q = mp.Queue()

    net = mp.Process(target=remcom, args=(movie_filename, output_dir,
                                          begin_time, end_time, msg_q))
    net.start()
    time.sleep(5)

    for option in keyboard_input():
        if option:
            option = send_command(option, socketfile=REMCOM_SOCKET_FILE)
            if option:
                print(option,)
                os.sys.stdout.flush()
            option = False
        while not msg_q.empty():
            option = msg_q.get()
            print('got message %s' % option)
        if option == 'q':
            run_command('kill -9 %d 2>&1 > /dev/null' % (net.pid,))
            print('killing %d' % (net.pid,))
            break
        time.sleep(0.1)
    net.join(10)

@nottest
def remcom_test_main():
    parser = argparse.ArgumentParser(description='remcom_test script')
    parser.add_argument('files', nargs='*', help='remcom_test files')
    parser.add_argument('-d', '--directory', type=str,
                        help='specify directory (must be in Documents/movies)')
    parser.add_argument('-u', '--unwatched', action='store_true',
                        help='copy to television/unwatched')
    args = parser.parse_args()

    directory = getattr(args, 'directory', None)
    unwatched = getattr(args, 'unwatched', False)
    files = getattr(args, 'files', [])
    if directory:
        if not os.path.exists(
                '%s/Documents/movies/%s' % (OUTPUT_DIR, directory)):
            raise Exception('bad directory')
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            input_filename = '%s/%s' % (INPUT_DIR, fname)
            mp4_filename = fname.replace('.avi', '.mp4')
            prefix = os.path.basename(input_filename)
            for suffix in '.avi', '.mp4', '.mkv':
                prefix = prefix.replace(suffix, '')
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)
            output_directory = '%s/Documents/movies/' % (OUTPUT_DIR, directory)
            remcom_main(input_filename, output_directory, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % input_filename)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('cp %s/Documents/movies/%s %s/Documents/movies/'
                           '%s/\n' % (HOMEDIR, mp4_filename, OUTPUT_DIR,
                                      directory))
                inpf.write('mv %s/Documents/movies/%s/%s %s/'
                           'Documents/movies/\n' % (OUTPUT_DIR, directory,
                                                    fname, HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s/Documents/movies/%s/%s\n'
                           % (HOMEDIR, OUTPUT_DIR, directory, mp4_filename))
            publish_transcode_job_to_queue(mp4_script)
    elif unwatched:
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            input_filename = '%s/%s' % (INPUT_DIR, fname)
            mp4_filename = fname.replace('.avi', '.mp4')
            prefix = input_filename.split('/')[-1]
            for suffix in '.avi', '.mp4', '.mkv':
                prefix = prefix.replace(suffix, '')
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)
            output_dir = '%s/television/unwatched' % OUTPUT_DIR
            remcom_main(input_filename, output_dir, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % input_filename)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('cp %s/Documents/movies/%s %s/'
                           'television/unwatched/\n' % (HOMEDIR, mp4_filename,
                                                        OUTPUT_DIR))
                inpf.write('mv %s/television/unwatched/%s %s/'
                           'Documents/movies/\n' % (OUTPUT_DIR, fname,
                                                    HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s/%s\n'
                           % (HOMEDIR, output_dir, mp4_filename))
            publish_transcode_job_to_queue(mp4_script)
    else:
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            tmp = fname.split('.')[0].split('_')
            show = '_'.join(tmp[:-2])
            season = int(tmp[-2].strip('s'))
            episode = int(tmp[-1].strip('ep'))

            prefix = '%s_s%02d_ep%02d' % (show, season, episode)
            output_dir = '%s/Documents/television/%s/season%d' % (
                OUTPUT_DIR, show, season)
            inavifile = '%s/Documents/movies/%s.avi' % (
                HOMEDIR, prefix)
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (
                HOMEDIR, prefix)
            avifile = '%s/%s.avi' % (
                output_dir, prefix)
            mp4file = '%s/Documents/movies/%s.mp4' % (
                HOMEDIR, prefix)
            mp4file_final = '%s/%s.mp4' % (
                output_dir, prefix)
            remcom_main(inavifile, output_dir, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % input_filename)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('mkdir -p %s\n' % output_dir)
                inpf.write('cp %s %s\n' % (mp4file, mp4file_final))
                inpf.write('mv %s %s/Documents/movies/\n' % (avifile, HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s\n' % (HOMEDIR,
                                                           mp4file_final))
            publish_transcode_job_to_queue(mp4_script)
<commit_msg>fix error message<commit_after>#!/usr/bin/python
''' remcom test module '''
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import argparse
import os
import time
from select import select
import multiprocessing as mp
from nose.tools import nottest

from .remove_commercials import remove_commercials
from .roku_utils import (make_thumbnails, make_audio_analysis_plots_wrapper,
                         make_time_series_plot_wrapper,
                         publish_transcode_job_to_queue, get_random_hex_string)
from .util import (run_command, send_command, OpenUnixSocketServer,
                   OpenSocketConnection, HOMEDIR)

REMCOM_SOCKET_FILE = '/tmp/.remcom_test_socket'
INPUT_DIR = '%s/Documents/movies' % HOMEDIR
OUTPUT_DIR = '/media/dileptonnas'

def remove_commercials_wrapper(input_file='', output_dir='', begin_time=0,
                               end_time=0):
    '''
        cut and splice recorded file to remove undesired sections
        (commercials), use temp not test here...
    '''
    input_string = '%d,%d' % (begin_time, end_time)
    output_file = '%s/%s' % (output_dir, os.path.basename(input_file))
    remove_commercials(input_file, output_file,
                       timing_string=input_string, do_async=True)
    return output_file

def make_video_script(input_file='', begin_time=0):
    ''' write script to create small test video file '''
    if not os.path.exists(input_file):
        return -1
    input_string = '%d,%d' % (begin_time, begin_time+10)
    tmpfile = '%s/dvdrip/tmp/test_%06x' % (HOMEDIR, get_random_hex_string(3))
    avifile = '%s.avi' % tmpfile
    mp4file = '%s.mp4' % tmpfile
    wavfile = '%s.wav' % tmpfile
    remove_commercials(infile=input_file, outfile=avifile,
                       timing_string=input_string)
    run_command('time HandBrakeCLI -i %s -f mp4 -e x264 ' % avifile +
                '-b 600 -o %s > /dev/null 2>&1\n' % mp4file)
    run_command('mpv --ao=pcm:file=%s --no-video ' % wavfile +
                '%s > /dev/null 2>&1\n' % avifile)
    run_command('mv %s %s/public_html/videos/temp.mp4\n' % (mp4file, HOMEDIR))
    os.remove(avifile)
    retval = make_audio_analysis_plots_wrapper(wavfile, prefix='temp')
    os.remove(wavfile)
    return retval

def make_transcode_script(input_file):
    """
        write out transcode file at the end of the recording,
        will be used to convert mpg to avi / mp4 later on
    """
    prefix = os.path.basename(input_file)
    for suffix in '.avi', '.mp4', '.mkv':
        prefix = prefix.replace(suffix, '')
    output_file = '%s/dvdrip/avi/%s.mp4' % (HOMEDIR, prefix)
    with open('%s/dvdrip/tmp/%s.sh' % (HOMEDIR, prefix), 'w') as outfile:
        outfile.write('#!/bin/bash\n')
        outfile.write('nice -n 19 HandBrakeCLI ')
        outfile.write('-i %s -o %s ' % (input_file, output_file))
        outfile.write('--preset "Android" ')
        outfile.write('> ~/dvdrip/log/%s_mp4.out 2>&1\n' % prefix)
        outfile.write('mv %s ~/Documents/movies/\n' % output_file)
        outfile.write('mv ~/dvdrip/log/%s_mp4.out ~/tmp_avi/\n' % prefix)

    os.rename('%s/dvdrip/tmp/%s.sh' % (HOMEDIR, prefix),
              '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix))
    return '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)

def remcom(movie_filename, output_dir, begin_time, end_time,
           msg_q=None, socketfile=REMCOM_SOCKET_FILE):
    '''
        main function, takes filename, timing file as options
        communication via socket:
            q: quit remcom
            f: move forward, number of minutes can be included, i.e.
                f5 moves forward 5 minues
            b: move backwards... b5 moves backwards 5 minutes
            s: select
            v: test video
            t: thumbnail
    '''
    with OpenUnixSocketServer(socketfile) as sock:
        while True:
            outstring = []
            with OpenSocketConnection(sock) as conn:
                args = conn.recv(1024)
                if hasattr(args, 'decode'):
                    args = args.decode()
                args = args.split()

                original_end_time = end_time

                for option in args:
                    if option == 'q':
                        conn.send(b'q')
                        msg_q.put('q')
                        return 0
                    elif option[0] == 'f':
                        try:
                            end_time += int(option[1:]) * 60
                        except ValueError:
                            end_time += 60
                    elif option[0] == 'b':
                        try:
                            end_time -= int(option[1:]) * 60
                        except ValueError:
                            end_time -= 60
                    elif option == 's':
                        output_file = remove_commercials_wrapper(
                            movie_filename, output_dir, begin_time, end_time)
                        make_transcode_script(output_file)
                        conn.send(b'done')
                        msg_q.put('q')
                        return 0
                    elif option == 'v':
                        outstring.append('%s' % make_video_script(
                            movie_filename, begin_time=end_time))
                    elif option == 't':
                        outstring.append(
                            make_time_series_plot_wrapper(movie_filename))
                    else:
                        try:
                            end_time = int(option) * 60
                            print('end_time: %s' % option)
                        except ValueError:
                            continue
                outstring.append(
                    '%s %s %s %s' % (os.path.basename(movie_filename),
                                     '/'.join(output_dir.split('/')[-2:]),
                                     begin_time, end_time))
                tmp_begin = make_thumbnails(input_file=movie_filename,
                                            begin_time=end_time,
                                            output_dir=HOMEDIR +
                                            '/public_html/videos/'
                                            'thumbnails_tv')
                outstring.append('%d, s/f/b/q/t:' % (tmp_begin / 60))
                outstring = '\n'.join(outstring)
                conn.send(outstring.encode())

        if original_end_time > end_time:
            print('original %s new %s args %s' % (original_end_time, end_time,
                                                  args))
    return 0

def keyboard_input():
    ''' keyboard input from stdin using select function '''
    while True:
        in_, _, _ = select([os.sys.stdin], [], [], 0.1)
        if not in_:
            yield None
        else:
            for s__ in in_:
                if s__ == os.sys.stdin:
                    yield os.sys.stdin.readline()

def remcom_main(movie_filename, output_dir, begin_time, end_time):
    ''' main recom_test function '''
    msg_q = mp.Queue()

    net = mp.Process(target=remcom, args=(movie_filename, output_dir,
                                          begin_time, end_time, msg_q))
    net.start()
    time.sleep(5)

    for option in keyboard_input():
        if option:
            option = send_command(option, socketfile=REMCOM_SOCKET_FILE)
            if option:
                print(option,)
                os.sys.stdout.flush()
            option = False
        while not msg_q.empty():
            option = msg_q.get()
            print('got message %s' % option)
        if option == 'q':
            run_command('kill -9 %d 2>&1 > /dev/null' % (net.pid,))
            print('killing %d' % (net.pid,))
            break
        time.sleep(0.1)
    net.join(10)

@nottest
def remcom_test_main():
    parser = argparse.ArgumentParser(description='remcom_test script')
    parser.add_argument('files', nargs='*', help='remcom_test files')
    parser.add_argument('-d', '--directory', type=str,
                        help='specify directory (must be in Documents/movies)')
    parser.add_argument('-u', '--unwatched', action='store_true',
                        help='copy to television/unwatched')
    args = parser.parse_args()

    directory = getattr(args, 'directory', None)
    unwatched = getattr(args, 'unwatched', False)
    files = getattr(args, 'files', [])
    if directory:
        if not os.path.exists(
                '%s/Documents/movies/%s' % (OUTPUT_DIR, directory)):
            raise Exception('bad directory')
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            input_filename = '%s/%s' % (INPUT_DIR, fname)
            mp4_filename = fname.replace('.avi', '.mp4')
            prefix = os.path.basename(input_filename)
            for suffix in '.avi', '.mp4', '.mkv':
                prefix = prefix.replace(suffix, '')
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)
            output_directory = '%s/Documents/movies/' % (OUTPUT_DIR, directory)
            remcom_main(input_filename, output_directory, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % input_filename)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('cp %s/Documents/movies/%s %s/Documents/movies/'
                           '%s/\n' % (HOMEDIR, mp4_filename, OUTPUT_DIR,
                                      directory))
                inpf.write('mv %s/Documents/movies/%s/%s %s/'
                           'Documents/movies/\n' % (OUTPUT_DIR, directory,
                                                    fname, HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s/Documents/movies/%s/%s\n'
                           % (HOMEDIR, OUTPUT_DIR, directory, mp4_filename))
            publish_transcode_job_to_queue(mp4_script)
    elif unwatched:
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            input_filename = '%s/%s' % (INPUT_DIR, fname)
            mp4_filename = fname.replace('.avi', '.mp4')
            prefix = input_filename.split('/')[-1]
            for suffix in '.avi', '.mp4', '.mkv':
                prefix = prefix.replace(suffix, '')
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (HOMEDIR, prefix)
            output_dir = '%s/television/unwatched' % OUTPUT_DIR
            remcom_main(input_filename, output_dir, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % input_filename)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('cp %s/Documents/movies/%s %s/'
                           'television/unwatched/\n' % (HOMEDIR, mp4_filename,
                                                        OUTPUT_DIR))
                inpf.write('mv %s/television/unwatched/%s %s/'
                           'Documents/movies/\n' % (OUTPUT_DIR, fname,
                                                    HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s/%s\n'
                           % (HOMEDIR, output_dir, mp4_filename))
            publish_transcode_job_to_queue(mp4_script)
    else:
        for fname in files:
            if fname.endswith('.mp4'):
                continue
            tmp = fname.split('.')[0].split('_')
            show = '_'.join(tmp[:-2])
            season = int(tmp[-2].strip('s'))
            episode = int(tmp[-1].strip('ep'))

            prefix = '%s_s%02d_ep%02d' % (show, season, episode)
            output_dir = '%s/Documents/television/%s/season%d' % (
                OUTPUT_DIR, show, season)
            inavifile = '%s/Documents/movies/%s.avi' % (
                HOMEDIR, prefix)
            mp4_script = '%s/dvdrip/jobs/%s.sh' % (
                HOMEDIR, prefix)
            avifile = '%s/%s.avi' % (
                output_dir, prefix)
            mp4file = '%s/Documents/movies/%s.mp4' % (
                HOMEDIR, prefix)
            mp4file_final = '%s/%s.mp4' % (
                output_dir, prefix)
            remcom_main(inavifile, output_dir, 0, 0)
            if not os.path.exists(mp4_script):
                print('something bad happened %s' % prefix)
                exit(0)
            with open(mp4_script, 'a') as inpf:
                inpf.write('mkdir -p %s\n' % output_dir)
                inpf.write('cp %s %s\n' % (mp4file, mp4file_final))
                inpf.write('mv %s %s/Documents/movies/\n' % (avifile, HOMEDIR))
                inpf.write('rm %s/tmp_avi/%s_0.mpg\n' % (HOMEDIR, prefix))
                inpf.write('%s/bin/make_queue add %s\n' % (HOMEDIR,
                                                           mp4file_final))
            publish_transcode_job_to_queue(mp4_script)
<|endoftext|>


<commit_before>from django.shortcuts import render, get_object_or_404
from rest_framework.generics import RetrieveAPIView, ListAPIView, CreateAPIView
from rest_framework import viewsets
from django.views.generic import DetailView, ListView, TemplateView
from django.views.generic.edit import CreateView
from django.core.exceptions import PermissionDenied
from django.contrib.auth.models import User
from .models import Choropleth, Palette
from .serializers import ChoroplethSerializer, PaletteSerializer
from datasets.serializers import DatasetSerializer
from datasets.models import Dataset

class GalleryView(ListView):
    model = Choropleth
    template_name = "choropleths/gallery.html"
    paginate_by = 10
    queryset = Choropleth.objects.filter(published=1).order_by('-created_at', 'name')

class ChoroplethAPIView(RetrieveAPIView):
    model = Choropleth
    serializer_class = ChoroplethSerializer

class PaletteAPIView(ListAPIView):
    model = Palette
    serializer_class = PaletteSerializer

    def get_queryset(self):
        palettes = Palette.objects.filter(scheme=self.kwargs['pk'])
        return palettes

class ChoroplethList(ListView):
    model = Choropleth
    template_name = 'choropleths/choropleth_list.html'
    paginate_by = 10

    def get_queryset(self):
        user = get_object_or_404(User, pk=self.request.user.id)
        return Choropleth.objects.filter(owner=user).order_by('-modified_at')

class ChoroplethDetail(DetailView):
    model = Choropleth
    template_name = 'choropleths/choropleth_detail.html'

    def get_object(self, **kwargs):
        choropleth = super(ChoroplethDetail, self).get_object(**kwargs)
        if choropleth.owner != self.request.user:
            if choropleth.published == 0:
                raise PermissionDenied()
        return choropleth

class ChoroplethView(DetailView):
    template_name = 'choropleths/choropleth_view.html'
    model = Choropleth

    def get_object(self, **kwargs):
        choropleth = super(ChoroplethView, self).get_object(**kwargs)
        if choropleth.owner != self.request.user:
            if choropleth.published == 0:
                raise PermissionDenied()
        return choropleth

class ChoroplethEdit(DetailView):
    template_name = "choropleths/choropleth_edit.html"
    model = Choropleth

    def get_object(self, **kwargs):
        choropleth = super(ChoroplethEdit, self).get_object(**kwargs)
        if choropleth.owner != self.request.user:
            raise PermissionDenied()
        return choropleth

class ChoroplethCreate(TemplateView):
    template_name = "choropleths/choropleth_create.html"

    def get_context_data(self, **kwargs):
        context = super(ChoroplethCreate, self).get_context_data(**kwargs)
        dataset = Dataset.objects.get(id=kwargs['pk'])
        if dataset.owner == self.request.user:
            context['object'] = dataset
            context['dataset'] = dataset
            return context
        else:
            raise PermissionDenied()

class ChoroplethAPI(viewsets.ModelViewSet):
    model = Choropleth
    serializer_class = ChoroplethSerializer

    def pre_save(self, obj):
        obj.owner = self.request.user
<commit_msg>General refactoring. Added GetPublishedObjectMixin. Cleaned up import statements. Group API views together.<commit_after>from django.shortcuts import render, get_object_or_404
from rest_framework import generics
from rest_framework import viewsets
from django.views import generic
from django.core.exceptions import PermissionDenied
from django.contrib.auth.models import User
from .models import Choropleth, Palette
from .serializers import ChoroplethSerializer, PaletteSerializer
from datasets.serializers import DatasetSerializer
from datasets.models import Dataset

class GetPublishedObjectMixin(object):
    """
    Get Object Mixin

    Retrieves the object if the request user is the owner, or if
    the object is published. Otherwise returns status 403
    """
    def get_object(self, **kwargs):
        gotten_object = super(GetPublishedObjectMixin, self).get_object(**kwargs)
        if gotten_object.owner != self.request.user:
            if gotten_object.published == 0:
                raise PermissionDenied()
        return gotten_object

class GalleryView(generic.ListView):
    model = Choropleth
    template_name = "choropleths/gallery.html"
    paginate_by = 10
    queryset = Choropleth.objects.filter(published=1).order_by('-created_at', 'name')

class ChoroplethList(generic.ListView):
    model = Choropleth
    template_name = 'choropleths/choropleth_list.html'
    paginate_by = 10

    def get_queryset(self):
        user = get_object_or_404(User, pk=self.request.user.id)
        return Choropleth.objects.filter(owner=user).order_by('-modified_at')

class ChoroplethDetail(GetPublishedObjectMixin, generic.DetailView):
    model = Choropleth
    template_name = 'choropleths/choropleth_detail.html'

class ChoroplethView(GetPublishedObjectMixin, generic.DetailView):
    template_name = 'choropleths/choropleth_view.html'
    model = Choropleth

class ChoroplethEdit(GetPublishedObjectMixin, generic.DetailView):
    template_name = "choropleths/choropleth_edit.html"
    model = Choropleth

    def get_object(self, **kwargs):
        """
        Only the owner may show get a status 200
        """
        choropleth = super(ChoroplethEdit, self).get_object(**kwargs)
        if choropleth.owner != self.request.user:
            raise PermissionDenied()
        return choropleth

class ChoroplethCreate(generic.TemplateView):
    template_name = "choropleths/choropleth_create.html"

    def get_context_data(self, **kwargs):
        context = super(ChoroplethCreate, self).get_context_data(**kwargs)
        dataset = Dataset.objects.get(id=kwargs['pk'])
        if dataset.owner == self.request.user:
            context['object'] = dataset
            context['dataset'] = dataset
            return context
        else:
            raise PermissionDenied()

class PaletteAPIView(generics.ListAPIView):
    model = Palette
    serializer_class = PaletteSerializer

    def get_queryset(self):
        palettes = Palette.objects.filter(scheme=self.kwargs['pk'])
        return palettes

class ChoroplethAPI(viewsets.ModelViewSet):
    model = Choropleth
    serializer_class = ChoroplethSerializer

    def pre_save(self, obj):
        obj.owner = self.request.user
<|endoftext|>


<commit_before>__version__ = '1.0'

from gevent import monkey; monkey.patch_all()

import os
import re
import errno
import urllib2
import fcntl
import gevent
import logging
import logging.config
from contextlib import contextmanager
from time import time

from resizer import resize
from httpwhohas import HttpWhoHas

USER_AGENT = 'Katana/%s' % __version__

@contextmanager
def wlock(filename):
    try:
        with open(filename, 'r') as lock:
            while True:
                try:
                    fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
                except IOError as exc:
                    if exc.errno == errno.EAGAIN:
                        gevent.sleep(0.05)
                        continue
                    else:
                        raise
                else:
                    yield
                    break
    except IOError as exc:
        if exc.errno == errno.ENOENT:
            with open(filename, 'wb') as lockw:
                while True:
                    try:
                        fcntl.flock(lockw, fcntl.LOCK_EX | fcntl.LOCK_NB)
                    except IOError as exc:
                        if exc.errno == errno.EAGAIN:
                            gevent.sleep(0.05)
                            continue
                        else:
                            raise
                    else:
                        yield lockw

                        if os.path.exists(filename):
                            if not lockw.closed:
                                lockw.seek(0, 2)
                                if not lockw.tell():
                                    os.unlink(filename)
                            elif not os.path.getsize(filename):
                                os.unlink(filename)
                        break
        else:
            raise

class Timer:
    def __init__(self):
        self.start = time()

    def __str__(self):
        return '%.3f ms' % ((time() - self.start) * 1000)

class Katana(object):

    def __init__(self):
        self.slog = logging.getLogger('katana.server')
        self.clog = logging.getLogger('katana.cleaner')

        self.config = {
            'proxy': None,
            'accel_redirect': False,
            'accel_redirect_path': '/resized',
            'chunk_size': 16 * 1024,
            'thumb_max_width': 1920,
            'thumb_max_height': 1080,
            'thumb_max_quality': 90,
            'thumb_default_quality': 75,
            'origin_fetch_timeout': 1,
            'origin_timeout': 1,
            'origin_mapping': {
                'localhost': {'ips': ['127.0.0.1'], 'headers': {'Host': 'localhost'}}
            },
            'resize_url_re': None,
            'resize_origin': '',
            'resize_cache_path_source': '',
            'resize_cache_path_resized': '',
            'proxy_url_re': None,
            'proxy_origin': '',
            'proxy_cache_path': '',
            'cache_dir': './data',
            'cache_dir_max_usage': 90,
            'clean_older_than': 24,
            'clean_every': 4,
            'clean_dry': False,
            'logging': {'version': 1},
        }
        configfile = os.environ.get('CONFIG_FILE', 'katana.conf')
        if os.path.exists(configfile):
            execfile(configfile, {}, self.config)

        logging.config.dictConfig(self.config['logging'])

        self.hws = HttpWhoHas(proxy=self.config['proxy'], timeout=self.config['origin_timeout'], user_agent=USER_AGENT)
        for name, conf in self.config['origin_mapping'].items():
            self.hws.set_cluster(name, conf['ips'], conf.get('headers'))

    def _get_cache(self, cache):
        if os.path.exists(cache):
            if os.path.getsize(cache):
                return cache
            else:
                os.unlink(cache)
        return None

    def get_file(self, origin, cache):
        try:
            os.makedirs(os.path.dirname(cache))
        except OSError as exc:
            if exc.errno not in (errno.EEXIST, errno.ENOENT):
                raise

        with wlock(cache) as cache_fdw:
            if cache_fdw:
                self.slog.debug('write lock: resolving %s', origin)
                info = self.hws.resolve(origin)
                if info:
                    url = info[1]
                    headers = {'User-Agent': USER_AGENT}
                    if info[2]:
                        headers['Host'] = info[2]
                    try:
                        req = urllib2.Request(url, headers=headers)
                        req = urllib2.urlopen(req, timeout=self.config['origin_fetch_timeout'])
                    except urllib2.HTTPError as exc:
                        self.slog.error('fetching url=%s error: %s', url, exc)
                    except Exception as exc:
                        self.slog.exception('fetching url=%s failed', url)
                    else:
                        while True:
                            chunk = req.read(self.config['chunk_size'])
                            if not chunk:
                                break
                            cache_fdw.write(chunk)
                        self.slog.debug('fetched %s to %s', url, cache)
                        return cache
                else:
                    self.slog.debug('%s not found on origin', cache)

            elif self._get_cache(cache):
                self.slog.debug('read lock: %s found in cache as %s', origin, cache)
                return cache

        return None

    def get_image_resized(self, image_src, cache, width, height, fit, quality, values):
        with wlock(cache) as lockw:
            if lockw:
                if not resize(image_src, cache, width, height, fit, quality):
                    return None

            if self._get_cache(cache):
                return cache

        return None

    def resize(self, values):
        width = values.get('width')
        if width:
            width = int(width)
        else:
            width = 0
        if width > self.config['thumb_max_width']:
            width = self.config['thumb_max_width']

        height = values.get('height')
        if height:
            height = int(height)
        else:
            height = 0
        if height > self.config['thumb_max_height']:
            height = self.config['thumb_max_height']

        quality = values.get('quality')
        if quality:
            quality = int(quality)
        else:
            quality = self.config['thumb_default_quality']
        if quality > self.config['thumb_max_quality']:
            quality = self.config['thumb_max_quality']
        elif quality == 0:
            quality = 1

        fit = True if values.get('fit') else False

        if width == height == 0:
            return None

        values.update({
            'width': width,
            'height': height,
            'fit': '1' if fit else '0',
            'quality': quality,
            })

        origin = self.config['resize_origin'].format(**values)
        cache = '%s%s' % (self.config['cache_dir'], self.config['resize_cache_path_source'].format(**values))
        image_src = self.get_file(origin, cache)
        if image_src:
            os.utime(image_src, None)
            cache = '%s%s' % (self.config['cache_dir'], self.config['resize_cache_path_resized'].format(**values))
            return self.get_image_resized(image_src, cache, width, height, fit, quality, values)
        return None

    def proxy(self, values):
        origin = self.config['proxy_origin'].format(**values)
        cache = '%s%s' % (self.config['cache_dir'], self.config['proxy_cache_path'].format(**values))
        return self.get_file(origin, cache)

    def app(self, environ, start_response):
        timer = Timer()

        if environ['REQUEST_METHOD'] not in ('GET', 'HEAD'):
            start_response('405 Invalid Method', [])
            return []

        image_dst = None
        for action in ('resize', 'proxy'):
            regex = self.config['%s_url_re' % action]
            if regex:
                match = re.match(regex, environ['PATH_INFO'])
                if match:
                    image_dst = getattr(self, action)(match.groupdict())
                    if image_dst:
                        headers = [('Content-Type', 'image/jpeg'), ('X-Response-Time', str(timer)),]
                        if self.config['accel_redirect']:
                            accel_redirect = self.config['accel_redirect_path'] + image_dst[len(self.config['cache_dir']):]
                            headers.append(('X-Accel-Redirect', accel_redirect))
                            start_response('200 OK', headers)
                            return []
                        else:
                            image = open(image_dst, 'rb')
                            try:
                                start_response('200 OK', headers)
                                return environ['wsgi.file_wrapper'](image, self.config['chunk_size'])
                            except KeyError:
                                start_response('200 OK', headers)
                                return  iter(lambda: image.read(self.config['chunk_size']), '')
                    break

        start_response('404 Not Found', [('X-Response-Time', str(timer))])
        return ''

    def clean(self, dry=False):
        try:
            cleaner_lock = open(self.config['cache_dir'] + '/cleaner.lock', 'w')
            fcntl.flock(cleaner_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
            cleaner_lock.write('%s\n' % os.getpid())
        except (IOError, OSError) as exc:
            if exc.errno == errno.EAGAIN:
                self.clog.debug('cleaner already running')
            else:
                self.clog.exception('error while acquiring cleaner lock')
        else:
            clean_older_than = self.config['clean_older_than']
            self.clog.info('cleaner process starting')
            while True:
                st = os.statvfs(self.config['cache_dir'])
                disk_usage = (st.f_blocks - st.f_bfree) / float(st.f_blocks) * 100
                if disk_usage < self.config['cache_dir_max_usage']:
                    self.clog.info('cleaner process ending')
                    break
                now = time()
                for dirpath, dirnames, filenames in os.walk(self.config['cache_dir']):
                    for filename in filenames:
                        filepath = dirpath + '/' + filename
                        atime = os.path.getatime(filepath)
                        age = (now - atime) / 3600.
                        if age > clean_older_than:
                            lock = open(filepath, 'w')
                            try:
                                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
                            except IOError as exc:
                                if exc.errno == errno.EAGAIN:
                                    self.clog.debug('%s already locked', filepath)
                                else:
                                    self.clog.exception('error while cleaning %s', filepath)
                            else:
                                if not dry:
                                    os.unlink(filepath)
                                self.clog.info('cleaned %s : %fh old', filepath, age)
                clean_older_than -= 1
                if clean_older_than < 0:
                    self.clog.error('no more file to clean')
                    break
            cleaner_lock.close()

    def start_cleaner(self):
        while True:
            self.clean(self.config['clean_dry'])
            gevent.sleep(self.config['clean_every'] * 3600)

def create_app():
    return Katana().app

def main():
    import argparse
    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--clean', help='start the cleaner process', action='store_true')
    group.add_argument('--dump', help='print the configuration', action='store_true')
    group.add_argument('--start', help='start the server', action='store_true')
    args = parser.parse_args()

    if args.dump:
        from pprint import pprint
        pprint(Katana().config)

    elif args.clean:
        try:
            Katana().start_cleaner()
        except KeyboardInterrupt:
            pass

    elif args.start:
        from gevent.pywsgi import WSGIServer
        WSGIServer(('', 8088), Katana().app).serve_forever()

if __name__ == '__main__':
    main()
<commit_msg>katana: add support for revalidation of the cache<commit_after>__version__ = '1.0'

from gevent import monkey; monkey.patch_all()

import os
import re
import errno
import urllib2
import fcntl
import gevent
import logging
import logging.config
from contextlib import contextmanager
from time import time

from resizer import resize
from httpwhohas import HttpWhoHas

USER_AGENT = 'Katana/%s' % __version__

@contextmanager
def wlock(filename):
    try:
        with open(filename, 'rb+') as lock:
            while True:
                try:
                    fcntl.flock(lock, fcntl.LOCK_SH | fcntl.LOCK_NB)
                except IOError as exc:
                    if exc.errno == errno.EAGAIN:
                        gevent.sleep(0.05)
                        continue
                    else:
                        raise
                else:
                    yield False, lock
                    break
    except IOError as exc:
        if exc.errno == errno.ENOENT:
            with open(filename, 'wb') as lock:
                while True:
                    try:
                        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
                    except IOError as exc:
                        if exc.errno == errno.EAGAIN:
                            gevent.sleep(0.05)
                            continue
                        else:
                            raise
                    else:
                        yield True, lock

                        if os.path.exists(filename):
                            if not lock.closed:
                                lock.seek(0, 2)
                                if not lock.tell():
                                    os.unlink(filename)
                            elif not os.path.getsize(filename):
                                os.unlink(filename)
                        break
        else:
            raise

class Timer:
    def __init__(self):
        self.start = time()

    def __str__(self):
        return '%.3f ms' % ((time() - self.start) * 1000)

META_MAGIC = 'A'

class Katana(object):

    def __init__(self):
        self.slog = logging.getLogger('katana.server')
        self.clog = logging.getLogger('katana.cleaner')

        self.config = {
            'proxy': None,
            'accel_redirect': False,
            'accel_redirect_path': '/resized',
            'chunk_size': 16 * 1024,
            'thumb_max_width': 1920,
            'thumb_max_height': 1080,
            'thumb_max_quality': 90,
            'thumb_default_quality': 75,
            'origin_fetch_timeout': 3,
            'origin_timeout': 1,
            'origin_mapping': {
                'localhost': {'ips': ['127.0.0.1'], 'headers': {'Host': 'localhost'}}
            },
            'resize_url_re': None,
            'resize_origin': '',
            'resize_cache_path_source': '',
            'resize_cache_path_resized': '',
            'proxy_url_re': None,
            'proxy_origin': '',
            'proxy_cache_path': '',
            'cache_force_expires': False,
            'cache_default_expires': 600,
            'cache_dir': './data',
            'cache_dir_max_usage': 90,
            'clean_older_than': 24,
            'clean_every': 4,
            'clean_dry': False,
            'logging': {'version': 1},
        }
        configfile = os.environ.get('CONFIG_FILE', 'katana.conf')
        if os.path.exists(configfile):
            execfile(configfile, {}, self.config)

        logging.config.dictConfig(self.config['logging'])

        self.hws = HttpWhoHas(proxy=self.config['proxy'], timeout=self.config['origin_timeout'], user_agent=USER_AGENT)
        for name, conf in self.config['origin_mapping'].items():
            self.hws.set_cluster(name, conf['ips'], conf.get('headers'))

    def get_meta(self, cache):
        try:
            with open('%s.meta' % cache, 'r') as cache_meta:
                splitted = cache_meta.read().split('|')
                if splitted[0] == META_MAGIC:
                    magic, expires, last_modified, etag = splitted
                    return {
                        'expires': int(expires),
                        'last_modified': last_modified,
                        'etag': etag,
                    }
                else:
                    self.slog.error('get_meta wrong magic [%s] for %s', splitted[0], cache)
                    os.remove('%s.meta' % cache)
        except (IOError, OSError) as exc:
            self.slog.error('get_meta failed for %s: %s', cache, exc)
        return {}

    def set_meta(self, cache, headers):
        try:
            with open('%s.meta' % cache, 'w') as cache_meta:
                etag = headers.get('etag', '')
                last_modified = headers.get('last-modified', '')
                expires = self.config['cache_default_expires']
                if not self.config['cache_force_expires']:
                    m = re.match('.*max-age=(\d+).*', headers.get('cache-control', ''))
                    if m:
                        expires = int(m.group(1))
                expires = int(time() + expires)
                cache_meta.write('%s|%s|%s|%s' % (META_MAGIC, expires, last_modified, etag))
                return {
                    'expires': expires,
                    'last_modified': last_modified,
                    'etag': etag,
                }
        except (IOError, OSError) as exc:
            self.slog.error('set_meta failed for %s: %s', cache, exc)
        return {}

    def _get_cache(self, cache):
        if os.path.exists(cache):
            if os.path.getsize(cache):
                return cache
            else:
                os.unlink(cache)
        return None

    def get_file(self, origin, cache):
        try:
            os.makedirs(os.path.dirname(cache))
        except OSError as exc:
            if exc.errno not in (errno.EEXIST, errno.ENOENT):
                raise

        with wlock(cache) as (write, cache_fd):
            meta = self.get_meta(cache)
            if 'expires' in meta:
                if time() > meta['expires']:
                    write = True
            if write:
                self.slog.debug('write lock: resolving %s', origin)
                info = self.hws.resolve(origin, etag=meta.get('etag'), last_modified=meta.get('last_modified'))
                if info:
                    url = info['url']
                    if not info['modified']:
                        self.slog.debug('url=%s not modified', url)
                        return cache, self.set_meta(cache, info['headers'])
                    headers = {'User-Agent': USER_AGENT}
                    if info['host']:
                        headers['Host'] = info['host']
                    try:
                        req = urllib2.Request(url, headers=headers)
                        resp = urllib2.urlopen(req, timeout=self.config['origin_fetch_timeout'])
                    except urllib2.HTTPError as exc:
                        self.slog.error('fetching url=%s error: %s', url, exc)
                    except Exception as exc:
                        self.slog.exception('fetching url=%s failed', url)
                    else:
                        meta = self.set_meta(cache, dict(resp.headers))
                        while True:
                            chunk = resp.read(self.config['chunk_size'])
                            if not chunk:
                                break
                            cache_fd.write(chunk)
                        self.slog.debug('fetched %s to %s', url, cache)
                        return cache, meta
                else:
                    self.slog.debug('%s not found on origin', cache)

            elif self._get_cache(cache):
                self.slog.debug('read lock: %s found in cache as %s', origin, cache)
                return cache, meta

        return None, {}

    def get_image_resized(self, image_src, cache, width, height, fit, quality, values):
        with wlock(cache) as (write, cache_fd):
            if write:
                if not resize(image_src, cache, width, height, fit, quality):
                    return None

            if self._get_cache(cache):
                return cache

        return None

    def resize(self, values):
        width = values.get('width')
        if width:
            width = int(width)
        else:
            width = 0
        if width > self.config['thumb_max_width']:
            width = self.config['thumb_max_width']

        height = values.get('height')
        if height:
            height = int(height)
        else:
            height = 0
        if height > self.config['thumb_max_height']:
            height = self.config['thumb_max_height']

        quality = values.get('quality')
        if quality:
            quality = int(quality)
        else:
            quality = self.config['thumb_default_quality']
        if quality > self.config['thumb_max_quality']:
            quality = self.config['thumb_max_quality']
        elif quality == 0:
            quality = 1

        fit = True if values.get('fit') else False

        if width == height == 0:
            return None

        values.update({
            'width': width,
            'height': height,
            'fit': '1' if fit else '0',
            'quality': quality,
            })

        origin = self.config['resize_origin'].format(**values)
        cache = '%s%s' % (self.config['cache_dir'], self.config['resize_cache_path_source'].format(**values))
        image_src, meta = self.get_file(origin, cache)
        if image_src:
            os.utime(image_src, None)
            cache = '%s%s' % (self.config['cache_dir'], self.config['resize_cache_path_resized'].format(**values))
            return self.get_image_resized(image_src, cache, width, height, fit, quality, values)
        return None, meta

    def proxy(self, values):
        origin = self.config['proxy_origin'].format(**values)
        cache = '%s%s' % (self.config['cache_dir'], self.config['proxy_cache_path'].format(**values))
        return self.get_file(origin, cache)

    def app(self, environ, start_response):
        timer = Timer()

        if environ['REQUEST_METHOD'] not in ('GET', 'HEAD'):
            start_response('405 Invalid Method', [])
            return []

        image_dst = None
        for action in ('resize', 'proxy'):
            regex = self.config['%s_url_re' % action]
            if regex:
                match = re.match(regex, environ['PATH_INFO'])
                if match:
                    image_dst, meta = getattr(self, action)(match.groupdict())
                    if image_dst:
                        headers = [('Content-Type', 'image/jpeg'), ('X-Response-Time', str(timer)),]
                        if meta.get('etag'):
                            headers.append(('ETag', meta['etag']))
                        if meta.get('last_modified'):
                            headers.append(('Last-Modified', meta['last_modified']))
                        if self.config['accel_redirect']:
                            accel_redirect = self.config['accel_redirect_path'] + image_dst[len(self.config['cache_dir']):]
                            headers.append(('X-Accel-Redirect', accel_redirect))
                            start_response('200 OK', headers)
                            return []
                        else:
                            image = open(image_dst, 'rb')
                            try:
                                start_response('200 OK', headers)
                                return environ['wsgi.file_wrapper'](image, self.config['chunk_size'])
                            except KeyError:
                                start_response('200 OK', headers)
                                return  iter(lambda: image.read(self.config['chunk_size']), '')
                    break

        start_response('404 Not Found', [('X-Response-Time', str(timer))])
        return ''

    def clean(self, dry=False):
        try:
            cleaner_lock = open(self.config['cache_dir'] + '/cleaner.lock', 'w')
            fcntl.flock(cleaner_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
            cleaner_lock.write('%s\n' % os.getpid())
        except (IOError, OSError) as exc:
            if exc.errno == errno.EAGAIN:
                self.clog.debug('cleaner already running')
            else:
                self.clog.exception('error while acquiring cleaner lock')
        else:
            clean_older_than = self.config['clean_older_than']
            self.clog.info('cleaner process starting')
            while True:
                st = os.statvfs(self.config['cache_dir'])
                disk_usage = (st.f_blocks - st.f_bfree) / float(st.f_blocks) * 100
                if disk_usage < self.config['cache_dir_max_usage']:
                    self.clog.info('cleaner process ending')
                    break
                now = time()
                for dirpath, dirnames, filenames in os.walk(self.config['cache_dir']):
                    for filename in filenames:
                        filepath = dirpath + '/' + filename
                        atime = os.path.getatime(filepath)
                        age = (now - atime) / 3600.
                        if age > clean_older_than:
                            lock = open(filepath, 'w')
                            try:
                                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
                            except IOError as exc:
                                if exc.errno == errno.EAGAIN:
                                    self.clog.debug('%s already locked', filepath)
                                else:
                                    self.clog.exception('error while cleaning %s', filepath)
                            else:
                                if not dry:
                                    os.unlink(filepath)
                                self.clog.info('cleaned %s : %fh old', filepath, age)
                clean_older_than -= 1
                if clean_older_than < 0:
                    self.clog.error('no more file to clean')
                    break
            cleaner_lock.close()

    def start_cleaner(self):
        while True:
            self.clean(self.config['clean_dry'])
            gevent.sleep(self.config['clean_every'] * 3600)

def create_app():
    return Katana().app

def main():
    import argparse
    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--clean', help='start the cleaner process', action='store_true')
    group.add_argument('--dump', help='print the configuration', action='store_true')
    group.add_argument('--start', help='start the server', action='store_true')
    args = parser.parse_args()

    if args.dump:
        from pprint import pprint
        pprint(Katana().config)

    elif args.clean:
        try:
            Katana().start_cleaner()
        except KeyboardInterrupt:
            pass

    elif args.start:
        from gevent.pywsgi import WSGIServer
        WSGIServer(('', 8088), Katana().app).serve_forever()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Create and modify labels on github to have same labels and same color
on all repo
"""
from .github_login import login

REPO_TO_IGNORE = [
    'odoo-community.org',
    'community-data-files',
    'contribute-md-template',
    'website',
    ]

# here is the list of labels we need in each repo
all_labels = {
    '7.0': '000000',
    '8.0': '000000',
    'bug': 'fc2929',
    'duplicate': 'cccccc',
    'enhancement': '84b6eb',
    'help wanted': '159818',
    'invalid': 'e6e6e6',
    'question': 'cc317c',
    'needs fixing': 'eb6420',
    'needs review': 'fbca04',
    'work in progress': '0052cc',
    'wontfix': 'ffffff',
    }

def main():
    gh = login()

    all_repos = gh.iter_user_repos('OCA')

    for repo in all_repos:
        if repo.name in REPO_TO_IGNORE:
            continue
        labels = repo.iter_labels()

        existing_labels = dict((l.name, l.color) for l in labels)

        to_create = []
        to_change_color = []
        for needed_label in all_labels:
            if needed_label not in existing_labels.keys():
                to_create.append(needed_label)
            elif existing_labels[needed_label] != all_labels[needed_label]:
                to_change_color.append(needed_label)

        extra_labels = [l for l in existing_labels if l not in all_labels]

        if to_create:
            print ('Repo %s - Create %s missing labels'
                   % (repo.name, len(to_create)))

            for label_name in to_create:
                success = repo.create_label(label_name, all_labels[label_name])
                if not success:
                    print ("Failed to create a label on '%s'!"
                           " Please check you access right to this repository."
                           % repo.name)

        if to_change_color:
            print ('Repo %s - Update %s labels with wrong color'
                   % (repo.name, len(to_change_color)))

            for label_name in to_change_color:
                success = repo.update_label(label_name, all_labels[label_name])
                if not success:
                    print ("Failed to update a label on '%s'!"
                           " Please check you access right to this repository."
                           % repo.name)

        if extra_labels:
            print ('Repo %s - Found %s extra labels'
                   % (repo.name, len(extra_labels)))
            for label_name in extra_labels:
                print label_name

if __name__ == '__main__':
    main()
<commit_msg>[IMP] Remove versions labels<commit_after># -*- coding: utf-8 -*-
"""
Create and modify labels on github to have same labels and same color
on all repo
"""
from .github_login import login

REPO_TO_IGNORE = [
    'odoo-community.org',
    'community-data-files',
    'contribute-md-template',
    'website',
    ]

# here is the list of labels we need in each repo
all_labels = {
    'bug': 'fc2929',
    'duplicate': 'cccccc',
    'enhancement': '84b6eb',
    'help wanted': '159818',
    'invalid': 'e6e6e6',
    'question': 'cc317c',
    'needs fixing': 'eb6420',
    'needs review': 'fbca04',
    'work in progress': '0052cc',
    'wontfix': 'ffffff',
    }

def main():
    gh = login()

    all_repos = gh.iter_user_repos('OCA')

    for repo in all_repos:
        if repo.name in REPO_TO_IGNORE:
            continue
        labels = repo.iter_labels()

        existing_labels = dict((l.name, l.color) for l in labels)

        to_create = []
        to_change_color = []
        for needed_label in all_labels:
            if needed_label not in existing_labels.keys():
                to_create.append(needed_label)
            elif existing_labels[needed_label] != all_labels[needed_label]:
                to_change_color.append(needed_label)

        extra_labels = [l for l in existing_labels if l not in all_labels]

        if to_create:
            print ('Repo %s - Create %s missing labels'
                   % (repo.name, len(to_create)))

            for label_name in to_create:
                success = repo.create_label(label_name, all_labels[label_name])
                if not success:
                    print ("Failed to create a label on '%s'!"
                           " Please check you access right to this repository."
                           % repo.name)

        if to_change_color:
            print ('Repo %s - Update %s labels with wrong color'
                   % (repo.name, len(to_change_color)))

            for label_name in to_change_color:
                success = repo.update_label(label_name, all_labels[label_name])
                if not success:
                    print ("Failed to update a label on '%s'!"
                           " Please check you access right to this repository."
                           % repo.name)

        if extra_labels:
            print ('Repo %s - Found %s extra labels'
                   % (repo.name, len(extra_labels)))
            for label_name in extra_labels:
                print label_name

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Esto es un script para tratar de romper un passphrase de ecryptfs usando fuerza bruta.

import sys
import pexpect
import datetime
import math

config = {
    "ecryptcomm": "ecryptfs-unwrap-passphrase"
}
inicio = None
final = None
cantcp = 10   # Cantidad de combinaciones probadas por punto.
cantpr = 100  # Cantidad de pruebas por reporte.
cantpp = 1000 # Cantidad de pruebas por paquete.

def probarPalabra(fichpassph, palabra):
    global config
    # print("Probando \"{}\"".format(palabra))
    expdor = pexpect.spawn("{0} {1}" .format(config["ecryptcomm"], fichpassph))
    expdor.expect("Passphrase: ")
    expdor.sendline(palabra)
    resp = expdor.expect(["Error: ", "[a-z0-9]{10,}"])
    if resp == 0:
        # print("Falló la palabra \"{}\"".format(palabra))
        return False
    else:
        # print("La clave \"{}\" fue exitosa.".format(palabra))
        return True

def ataque_diccionario(fichpassph, fuente):
    print("Haciendo el ataque del diccionario")
    # Lee el diccionario.
    roto = False
    with open(fuente, "r") as fich:
        # Prueba cada palabra en el diccionario.
        for linea in fich:
            palabra = linea[:-1].strip()
            roto = probarPalabra(fichpassph, palabra.capitalize())
            if roto:
                print("Entcontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
            roto = probarPalabra(fichpassph, palabra.upper())
            if roto:
                print("Entcontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
            roto = probarPalabra(fichpassph, palabra.lower())
            if roto:
                print("Encontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
        else:
            print("Se probaron todas las palabras disponibles"
                  + " y no se ha podido romper el passphrase")

def lee_partes(fuente):
    partes = []
    with open(fuente, "r") as hfuente:
        # Pon todas las lineas de fuente en partes.
        for linea in hfuente:
            valor = linea[:-1].strip().lower()
            partes.append(valor)
    return partes

def elimina_repeticiones(partes):
    cont = 0
    seg = 0
    while cont < len(partes):
        seg = cont + 1
        while seg < len(partes):
            if partes[cont] == partes[seg]:
                del(partes[seg])
            else:
                seg += 1
        cont += 1

def sumarle_decimal_contador(contador, base, decimal):
    cont = len(contador) - 1
    while (contador[cont] + decimal) >= base:
        decimal += contador[cont]
        contador[cont] = decimal % base
        decimal = decimal // base
        if (decimal > 0) and (cont == 0):
            contador.insert(cont, 0)
        elif (decimal > 0):
            cont -= 1
    contador[cont] += decimal

def completa_ceros(contador, n):
    contador = ([0] * (n - len(contador))) + contador
    return contador

def verificar_menor(conta, contb):
    if len(conta) > len(contb):
        contb = completa_ceros(contb, len(conta))
    elif len(contb) > len(conta):
        conta = completa_ceros(conta, len(contb))
    return conta < contb

def administrar_ataque_combinacion(fichpassph, fuente, conti):
    global cantpp
    roto = False
    partes = lee_partes(fuente)
    elimina_repeticiones(partes)
    base = len(partes)
    contfsel = conti
    while not roto:
        # Selecciona el próximo paquete.
        contisel = contfsel.copy()
        sumarle_decimal_contador(contfsel, base, cantpp)
        # Prueba todas las combinaciones del paquete.
        roto = ataque_combinaciones(fichpassph, partes, contisel, contfsel)

def ataque_combinaciones(fichpassph, partes, conti, contf):
    global inicio
    print("Haciendo el ataque de combinaciones de partes de cadenas")
    print("Probando del {} al {}.".format(conti, contf))
    # print("ataque_combinaciones {} {} {}".format(partes, conti, contf))
    roto = False
    combinacion = ""
    cprob = 0
    base = len(partes)
    print("Probando combinaciones con {} partes.".format(base))
    while verificar_menor(conti, contf):
        # print(conti)
        combinacion = ""
        # Arma la combinación.
        for i in conti:
            combinacion += partes[i]
        # Prueba la combinación.
        roto = probarPalabra(fichpassph, combinacion)
        cprob += 1
        # Imprime el punto o el reporte si toca hacerlo.
        if (cprob % cantcp) == 0:
            print(".", end = "", flush = True)
        if (cprob % cantpr) == 0:
            actual = datetime.datetime.now()
            print("\nCantidad: {} Contador: {} Combinacion: \"{}\" Tiempo: {}".format(
                cprob, conti, combinacion, actual - inicio))
            print("Probando {} combinaciones por minuto".format(
                math.floor(cprob / ((actual - inicio).total_seconds() / 60)) ))
        # Si encontraste la combinación rompe el bucle.
        if roto:
            print("Encontré el passphrase, es la palabra \"{}\"!!".format(combinacion))
            break
        # Aumenta el contador.
        sumarle_decimal_contador(conti, base, 1)
    print("Probé {} combinaciones.".format(cprob))
    return roto

def main():
    global inicio, final
    contadores = [0]
    # Revisa los argumentos.
    if len(sys.argv) < 3:
        print("El uso correcto es \"romper-ecryptfs.py"
              + " /home/pedro/.ecryptfs/wrapped-passphrase\""
              + " /home/pedro/diccionario.txt")
        sys.exit(1)
    for arg in sys.argv[3:]:
        if arg.startswith("--cont="):
            contadores = [int(parte) for parte in arg[len("--cont="):].split("-")]
            print("Iniciando en Contador {}".format(contadores))
    print("Tratando de romper el passpharse {}.".format(sys.argv[1]))
    # Toma el tiempo de inicio.
    inicio = datetime.datetime.now()
    print("Comenzando a las {}".format(inicio))
    administrar_ataque_combinacion(sys.argv[1], sys.argv[2], contadores)
    # Toma el tiempo final
    final = datetime.datetime.now()
    # Haz un reporte final del tiempo
    print("Comenzamos a hacer intentos en: {}".format(inicio))
    print("Terminamos en: {}".format(final))
    print("Tomó: {}".format(final - inicio))

if __name__ == "__main__":
    main()
<commit_msg>romper-ecryptfs.py ahora funciona como cliente servidor, repartiendo el trabajo de probar combinaciones.<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Esto es un script para tratar de romper un passphrase de ecryptfs usando fuerza bruta.

import sys
import pexpect
import datetime
import math
import xmlrpc.server
import xmlrpc.client

config = {
    "ecryptcomm": "ecryptfs-unwrap-passphrase",
    "temppassphrase": "/tmp/wrapped-passphrase-temp"
}

inicio = None
final = None
cantcp = 10   # Cantidad de combinaciones probadas por punto.
cantpr = 100  # Cantidad de pruebas por reporte.
cantpp = 1000 # Cantidad de pruebas por paquete.

pizarron = [] # El pizarrón para llevar el estado de los trabajos.
partes = []   # La lista de las partes.
contfichpassph = None # El contenido del ficher passphrase.
roto = False  # Indica si ya logramos romper el passphrase.
maxtp = datetime.timedelta(seconds = 60 * 10) # El tiempo máximo que se va a esperar un paquete.
conti = [0]   # El contador inicial.

def probarPalabra(fichpassph, palabra):
    global config
    # print("Probando \"{}\"".format(palabra))
    expdor = pexpect.spawn("{0} {1}" .format(config["ecryptcomm"], fichpassph))
    expdor.expect("Passphrase: ")
    expdor.sendline(palabra)
    resp = expdor.expect(["Error: ", "[a-z0-9]{10,}"])
    if resp == 0:
        # print("Falló la palabra \"{}\"".format(palabra))
        return False
    else:
        # print("La clave \"{}\" fue exitosa.".format(palabra))
        return True

def ataque_diccionario(fichpassph, fuente):
    print("Haciendo el ataque del diccionario")
    # Lee el diccionario.
    roto = False
    with open(fuente, "r") as fich:
        # Prueba cada palabra en el diccionario.
        for linea in fich:
            palabra = linea[:-1].strip()
            roto = probarPalabra(fichpassph, palabra.capitalize())
            if roto:
                print("Entcontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
            roto = probarPalabra(fichpassph, palabra.upper())
            if roto:
                print("Entcontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
            roto = probarPalabra(fichpassph, palabra.lower())
            if roto:
                print("Encontré el passphrase la palabra \"{}\"!!".format(palabra))
                break
        else:
            print("Se probaron todas las palabras disponibles"
                  + " y no se ha podido romper el passphrase")

def lee_partes(fuente):
    partes = []
    with open(fuente, "r") as hfuente:
        # Pon todas las lineas de fuente en partes.
        for linea in hfuente:
            valor = linea[:-1].strip().lower()
            partes.append(valor)
    return partes

def elimina_repeticiones(partes):
    cont = 0
    seg = 0
    while cont < len(partes):
        seg = cont + 1
        while seg < len(partes):
            if partes[cont] == partes[seg]:
                del(partes[seg])
            else:
                seg += 1
        cont += 1

def sumarle_decimal_contador(contador, base, decimal):
    cont = len(contador) - 1
    while (contador[cont] + decimal) >= base:
        decimal += contador[cont]
        contador[cont] = decimal % base
        decimal = decimal // base
        if (decimal > 0) and (cont == 0):
            contador.insert(cont, 0)
        elif (decimal > 0):
            cont -= 1
    contador[cont] += decimal

def completa_ceros(contador, n):
    contador = ([0] * (n - len(contador))) + contador
    return contador

def verificar_menor(conta, contb):
    if len(conta) > len(contb):
        contb = completa_ceros(contb, len(conta))
    elif len(contb) > len(conta):
        conta = completa_ceros(conta, len(contb))
    return conta < contb

def lee_fichpassph(fichpassph):
    with open(fichpassph, "rb") as hfich:
        return hfich.read()

def servir_partes():
    global partes
    print("Sirviendo partes ...")
    return partes

def servir_fichpassph():
    global contfichpassph
    print("Sirviendo fichpassph ...")
    return xmlrpc.client.Binary(contfichpassph)

def servir_trabajo():
    global pizarron, cantpp, conti, partes, maxtp, roto
    base = len(partes)
    contisel = None
    contfsel = None
    contfante = None
    tpaquete = None
    print("Sirviendo trabajo ...")
    # Verifica si ya fue roto el passphrase, si lo fue no hay que
    # seguir dando trabajos.
    if roto:
        tpaquete = None
    # Verifica el pizarrón
    elif (pizarron == None) or (len(pizarron) == 0):
        # Si el pizarrón está vacío registra el primer trabajo.
        contisel = conti
        contfsel = conti.copy()
        sumarle_decimal_contador(contfsel, base, cantpp)
        tpaquete = [contisel, contfsel, datetime.datetime.now(), 1]
        pizarron.append(tpaquete)
    else:
        # Sino, si el pizarrón ya existe, ya tiene trabajos,
        # verificalo en busca de trabajos.
        for paquete in pizarron:
            if (paquete[3] == 1) and ((datetime.datetime.now() - paquete[2]) > maxtp):
                # Si encuentras un trabajo que ya se le pasó el tiempo, tomalo.
                paquete[2] = datetime.datetime.now()
                tpaquete = paquete
            elif (contfante != None) and (restar_contador(paquete[0], contfante) > 0):
                # Sino, si encuentras un espacio entre 2 trabajos, tomalo.
                contisel = contfante
                contfsel = contfante.copy()
                tam_esp = restar_contador(paquete[0], contfante)
                if tam_esp >= cantpp:
                    tam_esp = cantpp
                sumarle_decimal_contador(contfsel, base, tam_esp)
                tpaquete = [contisel, contfsel, datetime.datetime.now(), 1]
        else:
            # Sino, agrega uno nuevo al final.
            contisel = pizarron[-1][1]
            contfsel = pizarron[-1][1].copy()
            sumarle_decimal_contador(contfsel, base, cantpp)
            tpaquete = [contisel, contfsel, datetime.datetime.now(), 1]
        pizarron.append(tpaquete)
    print("Sirviendo {}.".format(tpaquete))
    return tpaquete

def servir_avisar_final(paquete):
    global pizarron
    i = 0
    print("Recibiendo aviso de final ...")
    # Recorre los trabajos en el pizarrón
    while i < len(pizarron):
        # Si aparece el trabajo que acaba de terminar ponlo en 0
        if paquete[0] == pizarron[i][0]:
            pizarron[i][3] = 0
        # Si hay un trabajo previo en el pizarron, también terminado
        # concatenalos en uno solo para marcar que ese intervalo ya se
        # reviso.
        if ((i > 0) and (pizarron[i - 1][1] == pizarron[i][0])
            and (pizarron[i][3] == 0) and (pizarron[i - 1][3] == pizarron[i][3])):
            pizarron[i - 1][1] = pizarron[i][1]
            del(pizarron[i])
        else:
            i += 1
    return 0

def servir_avisar_roto(contador):
    global roto, partes
    roto = True
    print("Recibiendo aviso de passphrase roto ...")
    combinacion = armar_combinacion(partes, contador)
    print("Encontré el passphrase, es la palabra \"{}\", la combinacion {}."
          .format(combinacion, contador))

def administrar_servidor(fichpassph, fuente, conti):
    global pizarron, partes, contfichpassph, cantpp, roto
    print("Iniciando servidor")
    partes = lee_partes(fuente)
    elimina_repeticiones(partes)
    contfichpassph = lee_fichpassph(fichpassph)
    # pizarron = leer_pizarron()
    with xmlrpc.server.SimpleXMLRPCServer(("localhost", 8080)) as servidor:
        servidor.register_introspection_functions()
        servidor.register_function(servir_partes)
        servidor.register_function(servir_fichpassph)
        servidor.register_function(servir_trabajo)
        servidor.register_function(servir_avisar_final)
        servidor.register_function(servir_avisar_roto)
        servidor.serve_forever()

def recibir_fichero(servicio, contfichero):
    global config
    with open(config["temppassphrase"], "wb") as hfichero:
        hfichero.write(contfichero.data)

def administrar_cliente(url):
    global config
    roto = False
    cont = []
    with xmlrpc.client.ServerProxy(url) as servicio:
        partes = servicio.servir_partes()
        contfichero = servicio.servir_fichpassph()
        recibir_fichero(servicio, contfichero)
        while not roto:
            trabajo = servicio.servir_trabajo()
            if trabajo == None:
                break
            roto, cont = ataque_combinaciones(
                config["temppassphrase"],
                partes, trabajo[0], trabajo[1])
            if roto:
                servicio.servir_avisar_roto(cont)
            else:
                servicio.servir_avisar_final(trabajo)

def administrar_ataque_combinacion(fichpassph, fuente, conti):
    global cantpp
    roto = False
    cont = []
    partes = lee_partes(fuente)
    elimina_repeticiones(partes)
    base = len(partes)
    contfsel = conti
    while not roto:
        # Selecciona el próximo paquete.
        contisel = contfsel.copy()
        sumarle_decimal_contador(contfsel, base, cantpp)
        # Prueba todas las combinaciones del paquete.
        roto, cont = ataque_combinaciones(fichpassph, partes, contisel, contfsel)

def armar_combinacion(partes, contador):
    combinacion = ""
    for i in contador:
        combinacion += partes[i]
    return combinacion

def ataque_combinaciones(fichpassph, partes, conti, contf):
    global inicio
    print("Haciendo el ataque de combinaciones de partes de cadenas")
    print("Probando del {} al {}.".format(conti, contf))
    # print("ataque_combinaciones {} {} {}".format(partes, conti, contf))
    roto = False
    combinacion = ""
    cprob = 0
    base = len(partes)
    print("Probando combinaciones con {} partes.".format(base))
    while verificar_menor(conti, contf):
        # print(conti)
        combinacion = armar_combinacion(partes, conti)
        # Prueba la combinación.
        roto = probarPalabra(fichpassph, combinacion)
        cprob += 1
        # Imprime el punto o el reporte si toca hacerlo.
        if (cprob % cantcp) == 0:
            print(".", end = "", flush = True)
        if (cprob % cantpr) == 0:
            actual = datetime.datetime.now()
            print("\nCantidad: {} Contador: {} Combinacion: \"{}\" Tiempo: {}".format(
                cprob, conti, combinacion, actual - inicio))
            print("Probando {} combinaciones por minuto".format(
                math.floor(cprob / ((actual - inicio).total_seconds() / 60)) ))
        # Si encontraste la combinación rompe el bucle.
        if roto:
            print("Encontré el passphrase, es la palabra \"{}\"!!".format(combinacion))
            break
        # Aumenta el contador.
        sumarle_decimal_contador(conti, base, 1)
    print("Probé {} combinaciones.".format(cprob))
    return (roto, conti)

def main():
    global inicio, final
    contadores = [0]
    # Revisa los argumentos.
    if ((len(sys.argv) < 3)
        or ((sys.argv[1].lower().strip() == "--servidor")
            and (len(sys.argv) < 4) )):
        print("El uso correcto es: \n    \"romper-ecryptfs.py --servidor "
              + " /home/pedro/.ecryptfs/wrapped-passphrase "
              + "/home/pedro/diccionario.txt\" \n    o\n    \""
              + "romper-ecryptfs.py --cliente http://servidor/\"")
        sys.exit(1)
    # Toma el tiempo de inicio.
    inicio = datetime.datetime.now()
    print("Comenzando a las {}".format(inicio))
    if sys.argv[1].strip().lower() == "--servidor":
        for arg in sys.argv[4:]:
            if arg.startswith("--cont="):
                contadores = [int(parte) for parte in arg[len("--cont="):].split("-")]
                print("Iniciando en Contador {}".format(contadores))
        print("Tratando de romper el passpharse {}.".format(sys.argv[2]))
        administrar_servidor(sys.argv[2], sys.argv[3], contadores)
    elif sys.argv[1].strip().lower() == "--cliente":
        administrar_cliente(sys.argv[2])
    # Toma el tiempo final
    final = datetime.datetime.now()
    # Haz un reporte final del tiempo
    print("Comenzamos a hacer intentos en: {}".format(inicio))
    print("Terminamos en: {}".format(final))
    print("Tomó: {}".format(final - inicio))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/bin/env python3
"""
Use only legal characters from files or current directory

Usage:
    strip_filenames.py [options] [<filename> ...]

Options:
    -l, --lowercase  Only lowercase
    -h, --help  Show this screen and exit.
"""

import sys
import os
from docopt import docopt

# docopt(doc, argv=None, help=True, version=None, options_first=False))

def main():
    opt = docopt(__doc__, sys.argv[1:])
    directory = opt.get("filename", os.listdir())
    legal_characters = ""
    list_N010 = [ str(x) for x in range(10) ]
    list_alpha = [ chr(x+97) for x in range(26) ]
    list_ALPHA = [ chr(x+65) for x in range(26) ]

    legal_characters += "".join(list_N010)
    legal_characters += "".join(list_alpha)
    legal_characters += ".-_~"

    if not opt.get("--lowercase", False):
        legal_characters += "".join(list_N010)

    for a in range(len(directory)):
        newname=""
        for c in directory[a]:
            if c == " ":
                newname += "_"
            elif c not in legal_characters:
                continue
            else:
                newname += c
        print("convert {} to {}".format(directory[a],newname))
        os.rename(directory[a], newname)

if __name__ == "__main__":
    main()
<commit_msg>Fix filename stripping<commit_after>#!/bin/env python3
"""
Use only legal characters from files or current directory

Usage:
    strip_filenames.py [-a]
    strip_filenames.py [options] [<filename> ...]

Options:
    -l, --lowercase  Only lowercase
    -a, --all  Include hidden files
    -h, --help  Show this screen and exit.
"""

import sys
import os
from docopt import docopt

# docopt(doc, argv=None, help=True, version=None, options_first=False))

def main():
    opt = docopt(__doc__, sys.argv[1:])
    directory = opt.get("filename", os.listdir())
    legal_characters = ""
    list_N010 = [ str(x) for x in range(10) ]
    list_alpha = [ chr(x+97) for x in range(26) ]
    list_ALPHA = [ chr(x+65) for x in range(26) ]

    legal_characters += "".join(list_N010)
    legal_characters += "".join(list_alpha)
    legal_characters += ".-_~"

    if not opt.get("--lowercase", False):
        legal_characters += "".join(list_ALPHA)

    for a in range(len(directory)):
        newname = ""
        for c in directory[a]:
            if c == " ":
                newname += "_"
            elif c in legal_characters:
                newname += c
            else:
                continue
        print("convert {} to {}".format(directory[a],newname))
        os.rename(directory[a], newname)

if __name__ == "__main__":
    main()
<|endoftext|>