<commit_before>from django import forms
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from taggit.models import Tag
import django_filters
from elasticsearch_dsl.query import Q

from froide.publicbody.models import PublicBody, Category, Jurisdiction

from .models import FoiRequest
from .widgets import DropDownFilterWidget, DateRangeWidget

def resolution_filter(x):
    return Q('term', resolution=x)

def status_filter(x):
    return Q('term', status=x)

FILTER_ORDER = ('jurisdiction', 'publicbody', 'status', 'category', 'tag')
SUB_FILTERS = {
    'jurisdiction': ('status', 'category', 'tag', 'publicbody')
}

FOIREQUEST_FILTERS = [
    (_("awaiting-classification"), (lambda x:
        Q('term', status='awaiting_classification')),
        'awaiting_classification'),
    (_("successful"), resolution_filter, 'successful'),
    (_("partially-successful"), resolution_filter,
        'partially_successful'),
    (_("refused"), resolution_filter, 'refused'),
    (_("withdrawn"), resolution_filter, 'user_withdrew'),
    (_("withdrawn-costs"), resolution_filter, 'user_withdrew_costs'),
    (_("awaiting-response"), status_filter, 'awaiting_response'),
    (_("overdue"), (lambda x:
        Q('range', due_date={
            'lt': timezone.now()
        }) & Q('term', status='awaiting_response')),
        'overdue'),
    (_("asleep"), status_filter, 'asleep'),
    (_("not-held"), resolution_filter, 'not_held'),
    (_("has-fee"), lambda x: Q('range', costs={'gt': 0}), 'has_fee')
]

FOIREQUEST_FILTERS = [
    (x[0], x[1], x[2],
        FoiRequest.STATUS_RESOLUTION_DICT[x[2]][0],
        FoiRequest.STATUS_RESOLUTION_DICT[x[2]][1])
    for x in FOIREQUEST_FILTERS
]
FOIREQUEST_FILTER_CHOICES = [(x[0], x[3]) for x in FOIREQUEST_FILTERS]
FOIREQUEST_FILTER_DICT = dict([(x[0], x[1:]) for x in FOIREQUEST_FILTERS])
REVERSE_FILTER_DICT = dict([(x[2], x[:2] + x[3:]) for x in FOIREQUEST_FILTERS])
FOIREQUEST_FILTER_RENDER = [(x[0], x[3], x[2]) for x in FOIREQUEST_FILTERS]

FOIREQUEST_LIST_FILTER_CHOICES = [
    x for x in FOIREQUEST_FILTER_CHOICES if x[0] not in {_("awaiting-classification")}
]

def get_active_filters(data):
    for key in FILTER_ORDER:
        if not data.get(key):
            continue
        yield key
        sub_filters = SUB_FILTERS.get(key, ())
        for sub_key in sub_filters:
            if data.get(sub_key):
                yield sub_key
                break
        break

def get_filter_data(filter_kwargs, data):
    query = {}
    for key in get_active_filters(filter_kwargs):
        query[key] = filter_kwargs[key]
    data.update(query)
    return data

class DropDownStatusFilterWidget(DropDownFilterWidget):
    def create_option(self, name, value, label, selected, index,
                      subindex=None, attrs=None):
        option = super(DropDownStatusFilterWidget, self).create_option(
            name, value, label, selected, index, subindex=subindex, attrs=attrs)
        if value:
            status = FOIREQUEST_FILTER_DICT[value][1]
            option['icon'] = 'status-%s' % status
        return option

class BaseFoiRequestFilterSet(django_filters.FilterSet):
    q = django_filters.CharFilter(
        method='auto_query',
        widget=forms.TextInput(
            attrs={
                'placeholder': _('Search requests'),
                'class': 'form-control'
            }
        ),
    )
    FOIREQUEST_FILTER_DICT = FOIREQUEST_FILTER_DICT
    status = django_filters.ChoiceFilter(
        choices=FOIREQUEST_LIST_FILTER_CHOICES,
        label=_('status'),
        empty_label=_('any status'),
        widget=DropDownStatusFilterWidget(
            attrs={
                'label': _('status'),
                'class': 'form-control'
            }
        ),
        method='filter_status',
    )
    jurisdiction = django_filters.ModelChoiceFilter(
        queryset=Jurisdiction.objects.get_visible(),
        to_field_name='slug',
        empty_label=_('all jurisdictions'),
        widget=forms.Select(
            attrs={
                'label': _('jurisdiction'),
                'class': 'form-control'
            }
        ),
        method='filter_jurisdiction'
    )
    category = django_filters.ModelChoiceFilter(
        queryset=Category.objects.get_category_list(),
        to_field_name='slug',
        empty_label=_('all categories'),
        widget=forms.Select(
            attrs={
                'label': _('category'),
                'class': 'form-control'
            }
        ),
        method='filter_category'
    )
    tag = django_filters.ModelChoiceFilter(
        queryset=Tag.objects.all(),
        to_field_name='slug',
        method='filter_tag',
        widget=forms.HiddenInput()
    )
    publicbody = django_filters.ModelChoiceFilter(
        queryset=PublicBody.objects.all(),
        to_field_name='slug',
        method='filter_publicbody',
        widget=forms.HiddenInput()
    )

    first = django_filters.DateFromToRangeFilter(
        method='filter_first',
        widget=DateRangeWidget,
    )
    last = django_filters.DateFromToRangeFilter(
        method='filter_last',
        widget=DateRangeWidget
    )
    sort = django_filters.ChoiceFilter(
        choices=[
            ('-last', _('last message (newest first)')),
            ('last', _('last message (oldest first)')),
            ('-first', _('request date (newest first)')),
            ('first', _('request date (oldest first)')),
        ],
        label=_('sort'),
        empty_label=_('default sort'),
        widget=forms.Select(
            attrs={
                'label': _('sort'),
                'class': 'form-control'
            }
        ),
        method='add_sort',
    )

    class Meta:
        model = FoiRequest
        fields = [
            'q', 'status', 'jurisdiction',
            'category', 'tag', 'publicbody', 'first'
        ]

    def __init__(self, *args, **kwargs):
        self.view = kwargs.pop('view')
        super().__init__(*args, **kwargs)
        self.filters['status'].field.widget.get_url = self.view.make_filter_url

    def filter_queryset(self, queryset):
        """
        Filter the queryset with the underlying form's `cleaned_data`. You must
        call `is_valid()` or `errors` before calling this method.
        This method should be overridden if additional filtering needs to be
        applied to the queryset before it is cached.
        """
        for name, value in self.form.cleaned_data.items():
            queryset = self.filters[name].filter(queryset, value)
            # assert isinstance(queryset, models.QuerySet), \
            #     "Expected '%s.%s' to return a QuerySet, but got a %s instead." \
            #     % (type(self).__name__, name, type(queryset).__name__)
        return queryset

    def auto_query(self, qs, name, value):
        if value:
            return qs.set_query(Q(
                "simple_query_string",
                query=value,
                # analyzer='standard',
                fields=['title^5', 'description^3', 'content'],
                default_operator='and',
                lenient=True
            ))
        return qs

    def filter_status(self, qs, name, value):
        parts = self.FOIREQUEST_FILTER_DICT[value]
        func = parts[0]
        status_name = parts[1]
        return qs.filter(func(status_name))

    def filter_jurisdiction(self, qs, name, value):
        return qs.filter(jurisdiction=value.id)

    def filter_category(self, qs, name, value):
        return qs.filter(categories=value.id)

    def filter_tag(self, qs, name, value):
        return qs.filter(tags=value.id)

    def filter_publicbody(self, qs, name, value):
        return qs.filter(publicbody=value.id)

    def filter_first(self, qs, name, value):
        range_kwargs = {}
        if value.start is not None:
            range_kwargs['gte'] = value.start
        if value.stop is not None:
            range_kwargs['lte'] = value.stop

        return qs.filter(Q('range', first_message=range_kwargs))

    def filter_last(self, qs, name, value):
        range_kwargs = {}
        if value.start is not None:
            range_kwargs['gte'] = value.start
        if value.stop is not None:
            range_kwargs['lte'] = value.stop
        return qs.filter(Q('range', last_message=range_kwargs))

    def add_sort(self, qs, name, value):
        if value:
            return qs.add_sort('%s_message' % value)
        return qs

class FoiRequestFilterSet(BaseFoiRequestFilterSet):
    pass
<commit_msg>Use default manager for public body filtering<commit_after>from django import forms
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from taggit.models import Tag
import django_filters
from elasticsearch_dsl.query import Q

from froide.publicbody.models import PublicBody, Category, Jurisdiction

from .models import FoiRequest
from .widgets import DropDownFilterWidget, DateRangeWidget

def resolution_filter(x):
    return Q('term', resolution=x)

def status_filter(x):
    return Q('term', status=x)

FILTER_ORDER = ('jurisdiction', 'publicbody', 'status', 'category', 'tag')
SUB_FILTERS = {
    'jurisdiction': ('status', 'category', 'tag', 'publicbody')
}

FOIREQUEST_FILTERS = [
    (_("awaiting-classification"), (lambda x:
        Q('term', status='awaiting_classification')),
        'awaiting_classification'),
    (_("successful"), resolution_filter, 'successful'),
    (_("partially-successful"), resolution_filter,
        'partially_successful'),
    (_("refused"), resolution_filter, 'refused'),
    (_("withdrawn"), resolution_filter, 'user_withdrew'),
    (_("withdrawn-costs"), resolution_filter, 'user_withdrew_costs'),
    (_("awaiting-response"), status_filter, 'awaiting_response'),
    (_("overdue"), (lambda x:
        Q('range', due_date={
            'lt': timezone.now()
        }) & Q('term', status='awaiting_response')),
        'overdue'),
    (_("asleep"), status_filter, 'asleep'),
    (_("not-held"), resolution_filter, 'not_held'),
    (_("has-fee"), lambda x: Q('range', costs={'gt': 0}), 'has_fee')
]

FOIREQUEST_FILTERS = [
    (x[0], x[1], x[2],
        FoiRequest.STATUS_RESOLUTION_DICT[x[2]][0],
        FoiRequest.STATUS_RESOLUTION_DICT[x[2]][1])
    for x in FOIREQUEST_FILTERS
]
FOIREQUEST_FILTER_CHOICES = [(x[0], x[3]) for x in FOIREQUEST_FILTERS]
FOIREQUEST_FILTER_DICT = dict([(x[0], x[1:]) for x in FOIREQUEST_FILTERS])
REVERSE_FILTER_DICT = dict([(x[2], x[:2] + x[3:]) for x in FOIREQUEST_FILTERS])
FOIREQUEST_FILTER_RENDER = [(x[0], x[3], x[2]) for x in FOIREQUEST_FILTERS]

FOIREQUEST_LIST_FILTER_CHOICES = [
    x for x in FOIREQUEST_FILTER_CHOICES if x[0] not in {_("awaiting-classification")}
]

def get_active_filters(data):
    for key in FILTER_ORDER:
        if not data.get(key):
            continue
        yield key
        sub_filters = SUB_FILTERS.get(key, ())
        for sub_key in sub_filters:
            if data.get(sub_key):
                yield sub_key
                break
        break

def get_filter_data(filter_kwargs, data):
    query = {}
    for key in get_active_filters(filter_kwargs):
        query[key] = filter_kwargs[key]
    data.update(query)
    return data

class DropDownStatusFilterWidget(DropDownFilterWidget):
    def create_option(self, name, value, label, selected, index,
                      subindex=None, attrs=None):
        option = super(DropDownStatusFilterWidget, self).create_option(
            name, value, label, selected, index, subindex=subindex, attrs=attrs)
        if value:
            status = FOIREQUEST_FILTER_DICT[value][1]
            option['icon'] = 'status-%s' % status
        return option

class BaseFoiRequestFilterSet(django_filters.FilterSet):
    q = django_filters.CharFilter(
        method='auto_query',
        widget=forms.TextInput(
            attrs={
                'placeholder': _('Search requests'),
                'class': 'form-control'
            }
        ),
    )
    FOIREQUEST_FILTER_DICT = FOIREQUEST_FILTER_DICT
    status = django_filters.ChoiceFilter(
        choices=FOIREQUEST_LIST_FILTER_CHOICES,
        label=_('status'),
        empty_label=_('any status'),
        widget=DropDownStatusFilterWidget(
            attrs={
                'label': _('status'),
                'class': 'form-control'
            }
        ),
        method='filter_status',
    )
    jurisdiction = django_filters.ModelChoiceFilter(
        queryset=Jurisdiction.objects.get_visible(),
        to_field_name='slug',
        empty_label=_('all jurisdictions'),
        widget=forms.Select(
            attrs={
                'label': _('jurisdiction'),
                'class': 'form-control'
            }
        ),
        method='filter_jurisdiction'
    )
    category = django_filters.ModelChoiceFilter(
        queryset=Category.objects.get_category_list(),
        to_field_name='slug',
        empty_label=_('all categories'),
        widget=forms.Select(
            attrs={
                'label': _('category'),
                'class': 'form-control'
            }
        ),
        method='filter_category'
    )
    tag = django_filters.ModelChoiceFilter(
        queryset=Tag.objects.all(),
        to_field_name='slug',
        method='filter_tag',
        widget=forms.HiddenInput()
    )
    publicbody = django_filters.ModelChoiceFilter(
        queryset=PublicBody._default_manager.all(),
        to_field_name='slug',
        method='filter_publicbody',
        widget=forms.HiddenInput()
    )

    first = django_filters.DateFromToRangeFilter(
        method='filter_first',
        widget=DateRangeWidget,
    )
    last = django_filters.DateFromToRangeFilter(
        method='filter_last',
        widget=DateRangeWidget
    )
    sort = django_filters.ChoiceFilter(
        choices=[
            ('-last', _('last message (newest first)')),
            ('last', _('last message (oldest first)')),
            ('-first', _('request date (newest first)')),
            ('first', _('request date (oldest first)')),
        ],
        label=_('sort'),
        empty_label=_('default sort'),
        widget=forms.Select(
            attrs={
                'label': _('sort'),
                'class': 'form-control'
            }
        ),
        method='add_sort',
    )

    class Meta:
        model = FoiRequest
        fields = [
            'q', 'status', 'jurisdiction',
            'category', 'tag', 'publicbody', 'first'
        ]

    def __init__(self, *args, **kwargs):
        self.view = kwargs.pop('view')
        super().__init__(*args, **kwargs)
        self.filters['status'].field.widget.get_url = self.view.make_filter_url

    def filter_queryset(self, queryset):
        """
        Filter the queryset with the underlying form's `cleaned_data`. You must
        call `is_valid()` or `errors` before calling this method.
        This method should be overridden if additional filtering needs to be
        applied to the queryset before it is cached.
        """
        for name, value in self.form.cleaned_data.items():
            queryset = self.filters[name].filter(queryset, value)
            # assert isinstance(queryset, models.QuerySet), \
            #     "Expected '%s.%s' to return a QuerySet, but got a %s instead." \
            #     % (type(self).__name__, name, type(queryset).__name__)
        return queryset

    def auto_query(self, qs, name, value):
        if value:
            return qs.set_query(Q(
                "simple_query_string",
                query=value,
                # analyzer='standard',
                fields=['title^5', 'description^3', 'content'],
                default_operator='and',
                lenient=True
            ))
        return qs

    def filter_status(self, qs, name, value):
        parts = self.FOIREQUEST_FILTER_DICT[value]
        func = parts[0]
        status_name = parts[1]
        return qs.filter(func(status_name))

    def filter_jurisdiction(self, qs, name, value):
        return qs.filter(jurisdiction=value.id)

    def filter_category(self, qs, name, value):
        return qs.filter(categories=value.id)

    def filter_tag(self, qs, name, value):
        return qs.filter(tags=value.id)

    def filter_publicbody(self, qs, name, value):
        return qs.filter(publicbody=value.id)

    def filter_first(self, qs, name, value):
        range_kwargs = {}
        if value.start is not None:
            range_kwargs['gte'] = value.start
        if value.stop is not None:
            range_kwargs['lte'] = value.stop

        return qs.filter(Q('range', first_message=range_kwargs))

    def filter_last(self, qs, name, value):
        range_kwargs = {}
        if value.start is not None:
            range_kwargs['gte'] = value.start
        if value.stop is not None:
            range_kwargs['lte'] = value.stop
        return qs.filter(Q('range', last_message=range_kwargs))

    def add_sort(self, qs, name, value):
        if value:
            return qs.add_sort('%s_message' % value)
        return qs

class FoiRequestFilterSet(BaseFoiRequestFilterSet):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.
import boto
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import sys
try:
    import s3_settings
except:
    print ("There should be a s3_settings.py file with the name \
           of the bucket with the photos.")

def allowed_file(filename):
    return '.' in filename and \
              filename.rsplit('.', 1)[1] in s3_settings.ALLOWED_EXTENSIONS

def get_s3_photos(folder):
    """
    Get photos from S3 bucket.

    :returns: A list of photos.
    :rtype: list

    """
    conn = S3Connection(s3_settings.ACCESS_KEY, s3_settings.SECRET)
    mybucket = conn.get_bucket(s3_settings.BUCKET)
    photos = []
    for photo in mybucket.list():
        if allowed_file(photo.name.lower()):
            link = dict(
                url_b="http://%s.s3.amazonaws.com/%s/%s" % (s3_settings.BUCKET, s3_settings.FOLDER, photo.name)
            )
            photos.append(link)

    return photos
<commit_msg>Amend get images<commit_after># -*- coding: utf-8 -*-

# This file is part of PyBOSSA.
#
# PyBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBOSSA.  If not, see <http://www.gnu.org/licenses/>.
import boto
from boto.s3.connection import S3Connection
from boto.s3.key import Key
import sys
try:
    import s3_settings
except:
    print ("There should be a s3_settings.py file with the name \
           of the bucket with the photos.")

def allowed_file(filename):
    return '.' in filename and \
              filename.rsplit('.', 1)[1] in s3_settings.ALLOWED_EXTENSIONS

def get_s3_photos(folder):
    """
    Get photos from S3 bucket.

    :returns: A list of photos.
    :rtype: list

    """
    conn = S3Connection(s3_settings.ACCESS_KEY, s3_settings.SECRET)
    bucket = conn.get_bucket(s3_settings.BUCKET)
    rs = bucket.list(s3_settings.FOLDER)
    photos = []
    for photo in rs:
        if allowed_file(photo.name.lower()):
            link = dict(
                url_b=photo.name
            )
            photos.append(link)

    return photos
<|endoftext|>


<commit_before># vim: set fdm=marker fdc=2 :
# coding=utf-8

# imports {{{1
import vim
import re
from glob import glob
from os import walk
from os.path import join, getmtime, isfile, isdir, exists
from subprocess import Popen, PIPE
from vim_pad.utils import get_save_dir
from vim_pad.pad import PadInfo
from vim_pad.timestamps import natural_timestamp

# globals (caches) {{{1
cached_data = []
cached_timestamps = []
cached_filenames = []

def open_pad(path=None, first_line="", query=''):  # {{{1
    """Creates or opens a note.

    path: a valid path for a note.

    first_line: a string to insert to a new note, if given.
    """
    # we require self.save_dir_set to be set to a valid path
    if get_save_dir() == "":
        vim.command('echom "vim-pad: IMPORTANT: please set g:pad#dir to a valid path in your vimrc."')
        return

    # if no path is provided, we create one using the current time
    if not path:
        path = join(get_save_dir(),
                    PadInfo([first_line]).id + vim.eval("g:pad#default_file_extension"))
    path = path.replace(" ", "\ ")

    if bool(int(vim.eval("g:pad#open_in_split"))):
        if vim.eval('g:pad#position["pads"]') == 'right':
            vim.command("silent! rightbelow"
                    + str(vim.eval("g:pad#window_width")) + "vsplit " + path)
        else:
            vim.command("silent! botright"
                    + str(vim.eval("g:pad#window_height")) + "split " + path)
    else:
        vim.command("silent! edit " + path)

    # we don't keep the buffer when we hide it
    vim.command("set bufhidden=wipe")

    # set the filetype to our default
    if vim.eval('&filetype') in ('', 'conf'):
        vim.command("set filetype=" + vim.eval("g:pad#default_format"))

    # map the local commands
    if bool(int(vim.eval('has("gui_running")'))):
        vim.command("noremap <silent> <buffer> <localleader><delete> :call pad#DeleteThis()<cr>")
    else:
        vim.command("noremap <silent> <buffer> <localleader>dd :call pad#DeleteThis()<cr>")

    vim.command("noremap <silent> <buffer> <localleader>+m :call pad#AddModeline()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>+f :call pad#MoveToFolder()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>-f :call pad#MoveToSaveDir()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>+a :call pad#Archive()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>-a :call pad#Unarchive()<cr>")

    # insert the text in first_line to the buffer, if provided
    if first_line:
        vim.current.buffer.append(first_line, 0)
        vim.command("normal! j")

    # highlight query and jump to it?
    if query != '':
        if vim.eval('g:pad#highlight_query') == '1':
            vim.command("call matchadd('PadQuery', '\c"+query+"')")
        if vim.eval('g:pad#jumpto_query') == '1':
            vim.command("call search('\c"+query+"')")

def listdir_recursive_nohidden(path, archive):  # {{{1
    matches = []
    for root, dirnames, filenames in walk(path, topdown=True):
        for dirname in dirnames:
            if dirname.startswith('.'):
                dirnames.remove(dirname)
            if archive != "!":
                if dirname == "archive":
                    dirnames.remove(dirname)
        matches += [join(root, f) for f in filenames if not f.startswith('.')]
    return matches

def listdir_external(path, archive, query): # {{{1
    search_backend = vim.eval("g:pad#search_backend")
    if search_backend == "grep":
        # we use Perl mode for grep (-P), because it is really fast
        command = ["grep", "-P", "-n", "-r", query, path + "/"]
        if archive != "!":
            command.append("--exclude-dir=archive")
        command.append('--exclude=.*')
        command.append("--exclude-dir=.git")
    elif search_backend == "ack":
        if vim.eval("executable('ack')") == "1":
            ack_path = "ack"
        else:
            ack_path = "/usr/bin/vendor_perl/ack"
        command = [ack_path, query, get_save_dir() + "/", "--noheading"]
        if archive != "!":
            command.append("--ignore-dir=archive")
            command.append('--ignore-file=match:/\./')

    if bool(int(vim.eval("g:pad#search_ignorecase"))):
        command.append("-i")
    command.append("--max-count=1")

    cmd_output = Popen(command, stdout=PIPE, stderr=PIPE).communicate()[0].split("\n")

    return [line.split(":")[0] for line in cmd_output if line != '']

def get_filelist(query=None, archive=None):  # {{{1
    """ __get_filelist(query) -> list_of_notes

    Returns a list of notes. If no query is provided, all the valid filenames
    in self.save_dir are returned in a list, otherwise, return the results of
    grep or ack search for query in self.save_dir.
    """
    local_path = vim.eval("getcwd(). '/'. g:pad#local_dir")
    use_local_dir = vim.eval('g:pad#local_dir') != '' and local_path != get_save_dir()
    if not query or query == "":
        files = listdir_recursive_nohidden(get_save_dir(), archive)

        if use_local_dir:
            files.extend(listdir_recursive_nohidden(local_path, archive))
    else:
        files = listdir_external(get_save_dir(), archive, query)

        if bool(int(vim.eval("g:pad#query_filenames"))):
            matches = filter(lambda i: not isdir(i) and i not in files, glob(join(get_save_dir(), "*"+query+"*")))
            files.extend(matches)

        if bool(int(vim.eval("g:pad#query_dirnames"))):
            matching_dirs = filter(isdir, glob(join(get_save_dir(), "*"+ query+"*")))
            for mdir in matching_dirs:
                files.extend(filter(lambda x: x not in files, listdir_recursive_nohidden(mdir, archive)))

        if use_local_dir:
            files.extend(listdir_external(local_path, archive, query))

    return files

def fill_list(files, queried=False, custom_order=False): # {{{1
    """ Writes the list of notes to the __pad__ buffer.

    files: a list of files to process.

    queried: whether files is the result of a query or not.

    custom_order: whether we should keep the order of the list given (implies queried=True).

    Keeps a cache so we only read the notes when the files have been modified.
    """
    global cached_filenames, cached_timestamps, cached_data

    # we won't want to touch the cache
    if custom_order:
        queried = True

    files = filter(exists, [join(get_save_dir(), f) for f in files])

    timestamps = [getmtime(join(get_save_dir(), f)) for f in files]

    # we will have a new list only on the following cases
    if queried or files != cached_filenames or timestamps != cached_timestamps:
        lines = []
        if not custom_order:
            files = reversed(sorted(files, key=lambda i: getmtime(join(get_save_dir(), i))))
        for pad in files:
            pad_path = join(get_save_dir(), pad)
            if isfile(pad_path):
                pad_path = join(get_save_dir(), pad)
                with open(pad_path) as pad_file:
                    info = PadInfo(pad_file)
                    if info.isEmpty:
                        if bool(int(vim.eval("g:pad#show_dir"))):
                            tail = info.folder + u'\u2e25 '.encode('utf-8') + "[EMPTY]"
                        else:
                            tail = "[EMPTY]"
                    else:
                        if bool(int(vim.eval("g:pad#show_dir"))):
                            tail = info.folder + u'\u2e25 '.encode('utf-8') + u'\u21b2'.encode('utf-8').join((info.summary, info.body))
                        else:
                            tail = u'\u21b2'.encode('utf-8').join((info.summary, info.body))
                    lines.append(pad + " @ " + tail)
            else:
                pass

        # we only update the cache if we are not queried, to preserve the global cache
        if not queried:
            cached_data = lines
            cached_timestamps = timestamps
            cached_filenames = files

    # update natural timestamps
    def add_natural_timestamp(matchobj):
        id_string = matchobj.group("id")
        mtime = str(int(getmtime(join(get_save_dir(), matchobj.group("id")))*1000000))
        return id_string + " @ " + natural_timestamp(mtime).ljust(19) + " │"

    if not queried: # we use the cache
        lines = [re.sub("(?P<id>^.*?) @", add_natural_timestamp, line) for line in cached_data]
    else: # we use the new values in lines
        lines = [re.sub("(?P<id>^.*?) @", add_natural_timestamp, line) for line in lines]

    # we now show the list
    del vim.current.buffer[:] # clear the buffer
    vim.current.buffer.append(list(lines))
    vim.command("normal! dd")

def display(query, archive): # {{{1
    """ Shows a list of notes.

    query: a string representing a regex search. Can be "".

    Builds a list of files for query and then processes it to show the list in the pad format.
    """
    if get_save_dir() == "":
        vim.command('let tmp = confirm("IMPORTANT:\n'\
                'Please set g:pad#dir to a valid path in your vimrc.", "OK", 1, "Error")')
        return
    pad_files = get_filelist(query, archive)
    if len(pad_files) > 0:
        if vim.eval("bufexists('__pad__')") == "1":
            vim.command("bw __pad__")
        if vim.eval('g:pad#position["list"]') == "right":
            vim.command("silent! rightbelow " + str(vim.eval('g:pad#window_width')) + "vnew __pad__")
        else:
            vim.command("silent! botright " + str(vim.eval("g:pad#window_height")) + "new __pad__")
        fill_list(pad_files, query != "")
        if query != "":
            vim.command("let b:pad_query = '" + query + "'")
        vim.command("set filetype=pad")
        vim.command("setlocal nomodifiable")
        vim.command("setlocal statusline=%#PreCondit#\ vim-pad%=%#Comment#" + \
                    "%#Special#q%#Comment#:close\ %#Special#dd%#Comment#:delete\ " + \
                    "%#Special#[-+]a%#Comment#:[un]archive\ %#Special#[-+]f%#Comment#:move\ [from\|to]\ " + \
                    "%#Special#<s-f>%#Comment#:search\ %#Special#<s-s>%#Comment#:sort\ ")
    else:
        print "vim-pad: no pads"

def search_pads(): # {{{1
    """ Aks for a query and lists the matching notes.
    """
    if get_save_dir() == "":
        vim.command('let tmp = confirm("IMPORTANT:\n'\
                'Please set g:pad#dir to a valid path in your vimrc.", "OK", 1, "Error")')
        return
    query = vim.eval('input(">>> ")')
    display(query, "")
    vim.command("redraw!")

def global_incremental_search():  # {{{1
    """ Provides incremental search in normal mode without opening the list.
    """
    query = ""
    should_create_on_enter = False

    vim.command("echohl None")
    vim.command('echo ">> "')
    while True:
        raw_char = vim.eval("getchar()")
        if raw_char in ("13", "27"):
            if raw_char == "13":
                if should_create_on_enter:
                    open_pad(first_line=query)
                    vim.command("echohl None")
                else:
                    display(query, True)
            vim.command("redraw!")
            break
        else:
            try:   # if we can convert to an int, we have a regular key
                int(raw_char)   # we bring up an error on nr2char
                last_char = vim.eval("nr2char(" + raw_char + ")")
                query = query + last_char
            except:  # if we don't, we have some special key
                keycode = unicode(raw_char, errors="ignore")
                if keycode == "kb":  # backspace
                    query = query[:-len(last_char)]
        pad_files = get_filelist(query)
        if pad_files != []:
            info = ""
            vim.command("echohl None")
            should_create_on_enter = False
        else:  # we will create a new pad
            info = "[NEW] "
            vim.command("echohl WarningMsg")
            should_create_on_enter = True
        vim.command("redraw")
        vim.command('echo ">> ' + info + query + '"')

<commit_msg>Behave better when opening over a buffer with unsaved changes<commit_after># vim: set fdm=marker fdc=2 :
# coding=utf-8

# imports {{{1
import vim
import re
from glob import glob
from os import walk
from os.path import join, getmtime, isfile, isdir, exists
from subprocess import Popen, PIPE
from vim_pad.utils import get_save_dir
from vim_pad.pad import PadInfo
from vim_pad.timestamps import natural_timestamp

# globals (caches) {{{1
cached_data = []
cached_timestamps = []
cached_filenames = []

def open_pad(path=None, first_line="", query=''):  # {{{1
    """Creates or opens a note.

    path: a valid path for a note.

    first_line: a string to insert to a new note, if given.
    """
    # we require self.save_dir_set to be set to a valid path
    if get_save_dir() == "":
        vim.command('echom "vim-pad: IMPORTANT: please set g:pad#dir to a valid path in your vimrc."')
        return

    # if no path is provided, we create one using the current time
    if not path:
        path = join(get_save_dir(),
                    PadInfo([first_line]).id + vim.eval("g:pad#default_file_extension"))
    path = path.replace(" ", "\ ")

    def split_for_pad():
        if vim.eval('g:pad#position["pads"]') == 'right':
            vim.command("silent! rightbelow"
                    + str(vim.eval("g:pad#window_width")) + "vsplit " + path)
        else:
            vim.command("silent! botright"
                    + str(vim.eval("g:pad#window_height")) + "split " + path)

    if bool(int(vim.eval("g:pad#open_in_split"))):
        split_for_pad()
    else:
        awa = int(vim.eval("&autowriteall"))
        if bool(int(vim.eval("&modified"))):
            reply = vim.eval('input("vim-pad: the current file has unsaved changes. do you want to save? [Yn] ", "y")')
            if reply == "y":
                vim.command("set autowriteall")
                vim.command("silent! edit " + path)
                if awa == 0:
                    vim.command("set noautowriteall")
                vim.command("redraw")
            else:
                vim.command('echom "vim-pad: will have to open pad in a split"')
                split_for_pad()
        else:
            vim.command("silent! edit " + path)

    # we don't keep the buffer when we hide it
    vim.command("set bufhidden=wipe")

    # set the filetype to our default
    if vim.eval('&filetype') in ('', 'conf'):
        vim.command("set filetype=" + vim.eval("g:pad#default_format"))

    # map the local commands
    if bool(int(vim.eval('has("gui_running")'))):
        vim.command("noremap <silent> <buffer> <localleader><delete> :call pad#DeleteThis()<cr>")
    else:
        vim.command("noremap <silent> <buffer> <localleader>dd :call pad#DeleteThis()<cr>")

    vim.command("noremap <silent> <buffer> <localleader>+m :call pad#AddModeline()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>+f :call pad#MoveToFolder()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>-f :call pad#MoveToSaveDir()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>+a :call pad#Archive()<cr>")
    vim.command("noremap <silent> <buffer> <localleader>-a :call pad#Unarchive()<cr>")

    # insert the text in first_line to the buffer, if provided
    if first_line:
        vim.current.buffer.append(first_line, 0)
        vim.command("normal! j")

    # highlight query and jump to it?
    if query != '':
        if vim.eval('g:pad#highlight_query') == '1':
            vim.command("call matchadd('PadQuery', '\c"+query+"')")
        if vim.eval('g:pad#jumpto_query') == '1':
            vim.command("call search('\c"+query+"')")

def listdir_recursive_nohidden(path, archive):  # {{{1
    matches = []
    for root, dirnames, filenames in walk(path, topdown=True):
        for dirname in dirnames:
            if dirname.startswith('.'):
                dirnames.remove(dirname)
            if archive != "!":
                if dirname == "archive":
                    dirnames.remove(dirname)
        matches += [join(root, f) for f in filenames if not f.startswith('.')]
    return matches

def listdir_external(path, archive, query): # {{{1
    search_backend = vim.eval("g:pad#search_backend")
    if search_backend == "grep":
        # we use Perl mode for grep (-P), because it is really fast
        command = ["grep", "-P", "-n", "-r", query, path + "/"]
        if archive != "!":
            command.append("--exclude-dir=archive")
        command.append('--exclude=.*')
        command.append("--exclude-dir=.git")
    elif search_backend == "ack":
        if vim.eval("executable('ack')") == "1":
            ack_path = "ack"
        else:
            ack_path = "/usr/bin/vendor_perl/ack"
        command = [ack_path, query, get_save_dir() + "/", "--noheading"]
        if archive != "!":
            command.append("--ignore-dir=archive")
            command.append('--ignore-file=match:/\./')

    if bool(int(vim.eval("g:pad#search_ignorecase"))):
        command.append("-i")
    command.append("--max-count=1")

    cmd_output = Popen(command, stdout=PIPE, stderr=PIPE).communicate()[0].split("\n")

    return [line.split(":")[0] for line in cmd_output if line != '']

def get_filelist(query=None, archive=None):  # {{{1
    """ __get_filelist(query) -> list_of_notes

    Returns a list of notes. If no query is provided, all the valid filenames
    in self.save_dir are returned in a list, otherwise, return the results of
    grep or ack search for query in self.save_dir.
    """
    local_path = vim.eval("getcwd(). '/'. g:pad#local_dir")
    use_local_dir = vim.eval('g:pad#local_dir') != '' and local_path != get_save_dir()
    if not query or query == "":
        files = listdir_recursive_nohidden(get_save_dir(), archive)

        if use_local_dir:
            files.extend(listdir_recursive_nohidden(local_path, archive))
    else:
        files = listdir_external(get_save_dir(), archive, query)

        if bool(int(vim.eval("g:pad#query_filenames"))):
            matches = filter(lambda i: not isdir(i) and i not in files, glob(join(get_save_dir(), "*"+query+"*")))
            files.extend(matches)

        if bool(int(vim.eval("g:pad#query_dirnames"))):
            matching_dirs = filter(isdir, glob(join(get_save_dir(), "*"+ query+"*")))
            for mdir in matching_dirs:
                files.extend(filter(lambda x: x not in files, listdir_recursive_nohidden(mdir, archive)))

        if use_local_dir:
            files.extend(listdir_external(local_path, archive, query))

    return files

def fill_list(files, queried=False, custom_order=False): # {{{1
    """ Writes the list of notes to the __pad__ buffer.

    files: a list of files to process.

    queried: whether files is the result of a query or not.

    custom_order: whether we should keep the order of the list given (implies queried=True).

    Keeps a cache so we only read the notes when the files have been modified.
    """
    global cached_filenames, cached_timestamps, cached_data

    # we won't want to touch the cache
    if custom_order:
        queried = True

    files = filter(exists, [join(get_save_dir(), f) for f in files])

    timestamps = [getmtime(join(get_save_dir(), f)) for f in files]

    # we will have a new list only on the following cases
    if queried or files != cached_filenames or timestamps != cached_timestamps:
        lines = []
        if not custom_order:
            files = reversed(sorted(files, key=lambda i: getmtime(join(get_save_dir(), i))))
        for pad in files:
            pad_path = join(get_save_dir(), pad)
            if isfile(pad_path):
                pad_path = join(get_save_dir(), pad)
                with open(pad_path) as pad_file:
                    info = PadInfo(pad_file)
                    if info.isEmpty:
                        if bool(int(vim.eval("g:pad#show_dir"))):
                            tail = info.folder + u'\u2e25 '.encode('utf-8') + "[EMPTY]"
                        else:
                            tail = "[EMPTY]"
                    else:
                        if bool(int(vim.eval("g:pad#show_dir"))):
                            tail = info.folder + u'\u2e25 '.encode('utf-8') + u'\u21b2'.encode('utf-8').join((info.summary, info.body))
                        else:
                            tail = u'\u21b2'.encode('utf-8').join((info.summary, info.body))
                    lines.append(pad + " @ " + tail)
            else:
                pass

        # we only update the cache if we are not queried, to preserve the global cache
        if not queried:
            cached_data = lines
            cached_timestamps = timestamps
            cached_filenames = files

    # update natural timestamps
    def add_natural_timestamp(matchobj):
        id_string = matchobj.group("id")
        mtime = str(int(getmtime(join(get_save_dir(), matchobj.group("id")))*1000000))
        return id_string + " @ " + natural_timestamp(mtime).ljust(19) + " │"

    if not queried: # we use the cache
        lines = [re.sub("(?P<id>^.*?) @", add_natural_timestamp, line) for line in cached_data]
    else: # we use the new values in lines
        lines = [re.sub("(?P<id>^.*?) @", add_natural_timestamp, line) for line in lines]

    # we now show the list
    del vim.current.buffer[:] # clear the buffer
    vim.current.buffer.append(list(lines))
    vim.command("normal! dd")

def display(query, archive): # {{{1
    """ Shows a list of notes.

    query: a string representing a regex search. Can be "".

    Builds a list of files for query and then processes it to show the list in the pad format.
    """
    if get_save_dir() == "":
        vim.command('let tmp = confirm("IMPORTANT:\n'\
                'Please set g:pad#dir to a valid path in your vimrc.", "OK", 1, "Error")')
        return
    pad_files = get_filelist(query, archive)
    if len(pad_files) > 0:
        if vim.eval("bufexists('__pad__')") == "1":
            vim.command("bw __pad__")
        if vim.eval('g:pad#position["list"]') == "right":
            vim.command("silent! rightbelow " + str(vim.eval('g:pad#window_width')) + "vnew __pad__")
        else:
            vim.command("silent! botright " + str(vim.eval("g:pad#window_height")) + "new __pad__")
        fill_list(pad_files, query != "")
        if query != "":
            vim.command("let b:pad_query = '" + query + "'")
        vim.command("set filetype=pad")
        vim.command("setlocal nomodifiable")
        vim.command("setlocal statusline=%#PreCondit#\ vim-pad%=%#Comment#" + \
                    "%#Special#q%#Comment#:close\ %#Special#dd%#Comment#:delete\ " + \
                    "%#Special#[-+]a%#Comment#:[un]archive\ %#Special#[-+]f%#Comment#:move\ [from\|to]\ " + \
                    "%#Special#<s-f>%#Comment#:search\ %#Special#<s-s>%#Comment#:sort\ ")
    else:
        print "vim-pad: no pads"

def search_pads(): # {{{1
    """ Aks for a query and lists the matching notes.
    """
    if get_save_dir() == "":
        vim.command('let tmp = confirm("IMPORTANT:\n'\
                'Please set g:pad#dir to a valid path in your vimrc.", "OK", 1, "Error")')
        return
    query = vim.eval('input(">>> ")')
    display(query, "")
    vim.command("redraw!")

def global_incremental_search():  # {{{1
    """ Provides incremental search in normal mode without opening the list.
    """
    query = ""
    should_create_on_enter = False

    vim.command("echohl None")
    vim.command('echo ">> "')
    while True:
        raw_char = vim.eval("getchar()")
        if raw_char in ("13", "27"):
            if raw_char == "13":
                if should_create_on_enter:
                    open_pad(first_line=query)
                    vim.command("echohl None")
                else:
                    display(query, True)
            vim.command("redraw!")
            break
        else:
            try:   # if we can convert to an int, we have a regular key
                int(raw_char)   # we bring up an error on nr2char
                last_char = vim.eval("nr2char(" + raw_char + ")")
                query = query + last_char
            except:  # if we don't, we have some special key
                keycode = unicode(raw_char, errors="ignore")
                if keycode == "kb":  # backspace
                    query = query[:-len(last_char)]
        pad_files = get_filelist(query)
        if pad_files != []:
            info = ""
            vim.command("echohl None")
            should_create_on_enter = False
        else:  # we will create a new pad
            info = "[NEW] "
            vim.command("echohl WarningMsg")
            should_create_on_enter = True
        vim.command("redraw")
        vim.command('echo ">> ' + info + query + '"')

<|endoftext|>


<commit_before># -*- coding: iso-8859-15 -*-
"""
A wxValidator that matches APDU in hexadecimal such as:
    A4 A0 00 00 02
    A4A0000002

__author__ = "http://www.gemalto.com"

Copyright 2001-2012 gemalto
Author: Jean-Daniel Aussel, mailto:jean-daniel.aussel@gemalto.com

This file is part of pyscard.

pyscard is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

pyscard is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with pyscard; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""
import re
import string
import wx

# a regexp to match ATRs and APDUs
hexbyte = "[0-9a-fA-F]{1,2}"
apduregexp = re.compile("((%s)[ ]*)*" % hexbyte)

class APDUHexValidator(wx.PyValidator):
    '''A wxValidator that matches APDU in hexadecimal such as:
        A4 A0 00 00 02
        A4A0000002'''

    def __init__(self):
        wx.PyValidator.__init__(self)
        self.Bind(wx.EVT_CHAR, self.OnChar)

    def Clone(self):
        return APDUHexValidator()

    def Validate(self, win):
        tc = self.GetWindow()
        value = tc.GetValue()

        if not apduregexp.match(value):
            return False

        return True

    def OnChar(self, event):
        key = event.GetKeyCode()

        if wx.WXK_SPACE == key or chr(key) in string.hexdigits:
            value = event.GetEventObject().GetValue() + chr(key)
            if apduregexp.match(value):
                event.Skip()
            return

        if key < wx.WXK_SPACE or key == wx.WXK_DELETE or key > 255:
            event.Skip()
            return

        if not wx.Validator_IsSilent():
            wx.Bell()

        return
<commit_msg>Documentation: Fix epydoc warning<commit_after># -*- coding: iso-8859-15 -*-
"""
A wxValidator that matches APDU in hexadecimal such as::

    A4 A0 00 00 02
    A4A0000002

__author__ = "http://www.gemalto.com"

Copyright 2001-2012 gemalto
Author: Jean-Daniel Aussel, mailto:jean-daniel.aussel@gemalto.com

This file is part of pyscard.

pyscard is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

pyscard is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with pyscard; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""
import re
import string
import wx

# a regexp to match ATRs and APDUs
hexbyte = "[0-9a-fA-F]{1,2}"
apduregexp = re.compile("((%s)[ ]*)*" % hexbyte)

class APDUHexValidator(wx.PyValidator):
    '''A wxValidator that matches APDU in hexadecimal such as:
        A4 A0 00 00 02
        A4A0000002'''

    def __init__(self):
        wx.PyValidator.__init__(self)
        self.Bind(wx.EVT_CHAR, self.OnChar)

    def Clone(self):
        return APDUHexValidator()

    def Validate(self, win):
        tc = self.GetWindow()
        value = tc.GetValue()

        if not apduregexp.match(value):
            return False

        return True

    def OnChar(self, event):
        key = event.GetKeyCode()

        if wx.WXK_SPACE == key or chr(key) in string.hexdigits:
            value = event.GetEventObject().GetValue() + chr(key)
            if apduregexp.match(value):
                event.Skip()
            return

        if key < wx.WXK_SPACE or key == wx.WXK_DELETE or key > 255:
            event.Skip()
            return

        if not wx.Validator_IsSilent():
            wx.Bell()

        return
<|endoftext|>


<commit_before>from rest_framework import serializers

from .models import Upload

class UploadSerializer(serializers.ModelSerializer):
    class Meta:
        model = Upload
        fields = '__all__'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['guid'].required = True
<commit_msg>Add some documentation for weird init<commit_after>from rest_framework import serializers

from .models import Upload

class UploadSerializer(serializers.ModelSerializer):
    class Meta:
        model = Upload
        fields = '__all__'

    def __init__(self, *args, **kwargs):
        '''
        Add required marker, so OpenAPI schema generator can remove it again
        -.-
        '''
        super().__init__(*args, **kwargs)
        self.fields['guid'].required = True
<|endoftext|>


<commit_before>from __future__ import with_statement
import urllib, os, time, random
from bs4 import BeautifulSoup
from collections import defaultdict

# constant directories and files
DATA_DIR = 'data/'
IMG_DIR = 'data/images/'
DATA = 'data/data'
DATA_LABELS = 'data/labels'

# constant outcome types that HeTexted.com presents us with
INTO_YOU = 'into you'
NOT_INTO_YOU = 'not into you'
VERDICT_STILL_OUT = 'verdict still out'

# mapping our internal representation of the outcomes to the strings that
# designate them in the website
VERDICT_MAP = {"He's into you": INTO_YOU, "He's not into you": NOT_INTO_YOU,
               "Verdict is still out": VERDICT_STILL_OUT}

def mk_file_if_ne (fname):
    if not os.path.exists(fname):
        os.makedirs(fname)

def setup_env ():
    """Creates all dirs if needed"""
    mk_file_if_ne(DATA_DIR)
    mk_file_if_ne(IMG_DIR)

def get_data ():
    """Gets all raw data as text"""
    with open(DATA) as f:
        raw = f.read()
    return raw

def iter_thru_valid_divs (raw):
    """Iterates through parsed HTML and yields: (1) the URL at which a text can
    be found, and (2) the dictionary representing the user votes"""
    soup = BeautifulSoup(raw)
    for div in soup.findAll('div', {'class': 'post-content'}):
        img_src = None
        for match in div.findAll('div', {'class': 'field-text-image'}):
            img_src = match.img['src']
            break

        countmap = defaultdict(lambda: 0)
        for match in div.findAll('div', {'class': 'the-verdict'}):
            for li in match.div.ul.findAll('li'):
                count_type = li.a.text
                count = int(li.text.replace(li.a.text, ""))
                countmap[VERDICT_MAP[count_type]] = count

        if img_src != None:
            yield img_src, countmap

def cv_line_nums (li):
    random.shuffle(li)
    perc = int(0.1*len(li))
    for i in xrange(0, len(li), perc):
        yield li[i:i+perc], li[0:i] + li[i+perc:]

def write_data (lines, ident, datatype):
    with open("%s_%s_%d.txt"% (DATA_LABELS, datatype, ident), 'wb') as w:
        w.write("id\tinto you\tnot into you\tverdict still out\n")
        for line in lines:
            w.write(line)

def setup_data ():
    setup_env()
    raw = get_data()

    labels_file_lines = []
    for i,(url,labels) in enumerate(iter_thru_valid_divs(raw)):
        print 'processing file %d' % i
        filename = "%stxt_msg%d.jpg" % (IMG_DIR, i)

        with open(filename, 'wb') as w:
            try:
                dta = urllib.urlopen(url).read()
            except Exception as err:
                print err
                continue
            w.write(dta)
            #time.sleep(0.25)
            labels_file_lines.append("%d\t%d\t%d\t%d\n" %
                                     (i, labels[INTO_YOU],
                                      labels[NOT_INTO_YOU],
                                      labels[VERDICT_STILL_OUT]))

    for i,(train_chunk,test_chunk) in enumerate(cv_line_nums(labels_file_lines)):
        write_data(train_chunk, i, 'train')
        write_data(test_chunk, i, 'test')

if __name__ == '__main__':
    setup_data()
<commit_msg>Add class comment<commit_after>"""get-data.js results in a potentially very long HTML document rendered in a
browser as the main page of HeTexted.com. This contains `img` tags that link
to the actual texts of the people. This script grabs those texts, as well as the
votes from users on each text regarding whether the text indicates the boy likes
the person who received the text."""

from __future__ import with_statement
import urllib, os, time, random
from bs4 import BeautifulSoup
from collections import defaultdict

# constant directories and files
DATA_DIR = 'data/'
IMG_DIR = 'data/images/'
DATA = 'data/data'
DATA_LABELS = 'data/labels'

# constant outcome types that HeTexted.com presents us with
INTO_YOU = 'into you'
NOT_INTO_YOU = 'not into you'
VERDICT_STILL_OUT = 'verdict still out'

# mapping our internal representation of the outcomes to the strings that
# designate them in the website
VERDICT_MAP = {"He's into you": INTO_YOU, "He's not into you": NOT_INTO_YOU,
               "Verdict is still out": VERDICT_STILL_OUT}

def mk_file_if_ne (fname):
    if not os.path.exists(fname):
        os.makedirs(fname)

def setup_env ():
    """Creates all dirs if needed"""
    mk_file_if_ne(DATA_DIR)
    mk_file_if_ne(IMG_DIR)

def get_data ():
    """Gets all raw data as text"""
    with open(DATA) as f:
        raw = f.read()
    return raw

def iter_thru_valid_divs (raw):
    """Iterates through parsed HTML and yields: (1) the URL at which a text can
    be found, and (2) the dictionary representing the user votes"""
    soup = BeautifulSoup(raw)
    for div in soup.findAll('div', {'class': 'post-content'}):
        img_src = None
        for match in div.findAll('div', {'class': 'field-text-image'}):
            img_src = match.img['src']
            break

        countmap = defaultdict(lambda: 0)
        for match in div.findAll('div', {'class': 'the-verdict'}):
            for li in match.div.ul.findAll('li'):
                count_type = li.a.text
                count = int(li.text.replace(li.a.text, ""))
                countmap[VERDICT_MAP[count_type]] = count

        if img_src != None:
            yield img_src, countmap

def cv_line_nums (li):
    random.shuffle(li)
    perc = int(0.1*len(li))
    for i in xrange(0, len(li), perc):
        yield li[i:i+perc], li[0:i] + li[i+perc:]

def write_data (lines, ident, datatype):
    with open("%s_%s_%d.txt"% (DATA_LABELS, datatype, ident), 'wb') as w:
        w.write("id\tinto you\tnot into you\tverdict still out\n")
        for line in lines:
            w.write(line)

def setup_data ():
    setup_env()
    raw = get_data()

    labels_file_lines = []
    for i,(url,labels) in enumerate(iter_thru_valid_divs(raw)):
        print 'processing file %d' % i
        filename = "%stxt_msg%d.jpg" % (IMG_DIR, i)

        with open(filename, 'wb') as w:
            try:
                dta = urllib.urlopen(url).read()
            except Exception as err:
                print err
                continue
            w.write(dta)
            #time.sleep(0.25)
            labels_file_lines.append("%d\t%d\t%d\t%d\n" %
                                     (i, labels[INTO_YOU],
                                      labels[NOT_INTO_YOU],
                                      labels[VERDICT_STILL_OUT]))

    for i,(train_chunk,test_chunk) in enumerate(cv_line_nums(labels_file_lines)):
        write_data(train_chunk, i, 'train')
        write_data(test_chunk, i, 'test')

if __name__ == '__main__':
    setup_data()
<|endoftext|>


<commit_before>import pandas as pd
import os,fnmatch,re,subprocess,shutil
from collections import defaultdict
from igf_data.illumina.samplesheet import SampleSheet
from igf_data.utils.fileutils import get_temp_dir,remove_dir,copy_local_file

class MergeSingleCellFastq:
  '''
  A class for merging single cell fastq files per lane per sample

  :param fastq_dir: A directory path containing fastq files
  :param samplesheet: A samplesheet file used demultiplexing of bcl files
  :param platform_name: A sequencing platform name
  :param singlecell_tag: A single cell keyword for description field, default '10X'
  :param sampleid_col: A keyword for sample id column of samplesheet, default 'Sample_ID'
  :param samplename_col: A keyword for sample name column of samplesheet, default 'Sample_Name'
  :param orig_sampleid_col: A keyword for original sample id column, default 'Original_Sample_ID'
  :param orig_samplename_col: A keyword for original sample name column, default 'Original_Sample_Name'
  :param description_col: A keyword for description column, default 'Description'
  :param project_col: A keyword for project column, default 'Sample_Project'
  :param pseudo_lane_col: A keyword for pseudo lane column, default 'PseudoLane'
  :param lane_col: A keyword for lane column, default 'Lane'
  :param force_overwrite: A toggle for overwriting output fastqs, default True

  SampleSheet file should contain following columns:
     * Sample_ID: A single cell sample id in the following format, SampleId_{digit}
     * Sample_Name: A single cell sample name in the following format, SampleName_{digit}
     * Original_Sample_ID: An IGF sample id
     * Original_Sample_Name: A sample name provided by user
     * Description: A single cell label, default 10X
  '''
  def __init__(self, fastq_dir,samplesheet,platform_name,singlecell_tag='10X',
               sampleid_col='Sample_ID', samplename_col='Sample_Name',use_ephemeral_space=0,
               orig_sampleid_col='Original_Sample_ID', description_col='Description',
               orig_samplename_col='Original_Sample_Name',project_col='Sample_Project',
               lane_col='Lane', pseudo_lane_col='PseudoLane',force_overwrite=True):
    self.fastq_dir = fastq_dir
    self.samplesheet = samplesheet
    self.platform_name = platform_name
    self.singlecell_tag = singlecell_tag
    self.sampleid_col = sampleid_col
    self.samplename_col = samplename_col
    self.orig_sampleid_col = orig_sampleid_col
    self.description_col = description_col
    self.orig_samplename_col = orig_samplename_col
    self.project_col = project_col
    self.lane_col = lane_col
    self.pseudo_lane_col = pseudo_lane_col
    self.force_overwrite = force_overwrite
    self.use_ephemeral_space = use_ephemeral_space

  def _fetch_lane_and_sample_info_from_samplesheet(self):
    '''
    A internal method for grouping samples per lane based on the samplesheet
    returns a list containing sample and lane information per row
    '''
    try:
      samplesheet_data=SampleSheet(infile=self.samplesheet)                     # read samplesheet file
      if (self.orig_sampleid_col not in samplesheet_data._data_header) or \
         (self.orig_samplename_col not in samplesheet_data._data_header):
        raise ValueError(
      'Samplesheet {0} does not have {1} or {2} column'.\
        format(
          self.samplesheet,
          self.orig_sampleid_col,
          self.orig_samplename_col))                                            # check for required columns in the samplesheet
      samplesheet_data.\
      filter_sample_data(
        condition_key=self.description_col,
        condition_value=self.singlecell_tag,
        method='include')                                                       # filter samplesheet for single cell data
      sample_lane_data = list()
      if self.platform_name=='NEXTSEQ':                                         # hack for nextseq
        samplesheet_data.add_pseudo_lane_for_nextseq()
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.pseudo_lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      elif self.platform_name=='MISEQ':                                         # hack for miseq
        samplesheet_data.add_pseudo_lane_for_miseq()
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.pseudo_lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      elif self.platform_name=='HISEQ4000':                                     # check for hiseq4k
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      else:
        raise ValueError('platform {0} not supported'.format(platform_name))
      return sample_lane_data
    except:
      raise

  @staticmethod
  def _group_singlecell_fastq(sample_data,fastq_dir):
    '''
    A static method for grouping single cell fastq files

    required params:
    sample_data: A list of sample entries from samplesheet
                It should contain following keys for each row:
                lane_id, sample_id, sample_name, project_id
    fastq_dir: A directory path containing fastq files

    returns two dictionary of fastq group, one for single cell samples and
    another for sample information
    '''
    try:
      samples_info = defaultdict(dict)
      sample_files_list = \
        defaultdict(lambda: \
          defaultdict(lambda: \
            defaultdict(lambda: \
              defaultdict(list))))                                              # output data structure
      for sample_record in sample_data:
        sample_lane = sample_record.get('lane_id')
        sample_id = sample_record.get('sample_id')
        sample_name = sample_record.get('sample_name')
        project_id = sample_record.get('project_id')
        samples_info[sample_id]['sample_name'] = sample_name
        samples_info[sample_id]['project_id'] = project_id
        sample_id_regex = re.compile('^{0}_\d$'.format(sample_id))              # regexp for sample id match
        file_name_regex = \
          re.compile(r'^{0}_(\d)_S\d+_L00{1}_([R,I][1,2])_\d+\.fastq(\.gz)?$'.\
                     format(
                       sample_name,
                       sample_lane))                                            # regexp for fastq file match
        for root,_,files in os.walk(fastq_dir):
          for file in files:
            if fnmatch.fnmatch(file, "*.fastq.gz") and \
               not fnmatch.fnmatch(file, "Undetermined_*"):                     # skip undetermined reads
              if re.search(sample_id_regex,os.path.basename(root)) and \
                 re.search(file_name_regex,file):
                sm = re.match(file_name_regex,file)
                if len(sm.groups())>=2:
                  fragment_id=sm.group(1)
                  read_type=sm.group(2)
                  sample_files_list[sample_lane][sample_id][read_type][fragment_id].\
                  append(os.path.join(root,file))                               # add fastqs to samples list
                else:
                  raise ValueError('Failed to determined sample info:{0}, {1}'.\
                                   format(sample_id,file))
      return sample_files_list, samples_info
    except:
      raise

  def merge_fastq_per_lane_per_sample(self):
    '''
    A method for merging single cell fastq files present in input fastq_dir
    per lane per sample basis
    '''
    try:
      sample_data = \
        self._fetch_lane_and_sample_info_from_samplesheet()                     # get sample and lane information from samplesheet
      sample_files, samples_info = \
        self._group_singlecell_fastq(
          sample_data,
          self.fastq_dir)                                                       # get file groups
      all_intermediate_files=list()                                             # empty list for intermediate files
      s_count = 0                                                               # initial count for fastq S value
      for lane_id in sorted(sample_files.keys()):
        if self.platform_name=='NEXTSEQ':
            s_count = 0                                                         # nextseq is weird, reset counter for each lane
        for sample_id in sorted(sample_files[lane_id].keys()):
          s_count += 1                                                          # assign new S value for fastq files
          sample_name = samples_info.get(sample_id)['sample_name']
          project_id = samples_info.get(sample_id)['project_id']                # get sample and project info
          output_path = \
            os.path.join(
              self.fastq_dir,
              project_id,
              sample_id)                                                        # output location is under input fastq_dir
          if not os.path.exists(output_path):
            os.makedirs(output_path,mode=0o770)                                 # create outout directory

          for read_type in sample_files[lane_id][sample_id].keys():             # merge per read type
            output_filename = \
              '{0}_S{1}_L00{2}_{3}_001.fastq.gz'.\
                format(
                  sample_name,
                  s_count,
                  lane_id,
                  read_type)                                                    # assign new output filename
            final_path = os.path.join(output_path,output_filename)              # assign final output path
            if not self.force_overwrite and os.path.exists(final_path):
              raise ValueError('Failed to overwrite existing file {0}'.\
                               format(final_path))

            input_list = list()
            for sc_fragment, file_path in \
              sorted(sample_files[lane_id][sample_id][read_type].items()):
              input_list.extend(file_path)                                      # create list of input fastqs for merge
            if len(input_list) != 4:
              raise ValueError(\
                'expecting 4 files, got {0} for sample {1}, lane {2}, read type {3}'.\
                  format(
                    len(input_list),
                    sample_id,
                    lane_id,
                    read_type))                                                 # checking input files list
            temp_dir = \
              get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)        # get a temp dir
            temp_file = os.path.join(temp_dir,output_filename)                  # assign temp filename
            cmd = ["cat"]+input_list+[">",temp_file]                            # shell command for merging fastq.gz files
            subprocess.check_call(" ".join(cmd),shell=True)                     # exact same command for fastq merge as 10x pipeline
            copy_local_file(temp_file,final_path,force=True)                    # copy file to final location
            remove_dir(temp_dir)                                                # remove temp dir
            for file_path in input_list:
              all_intermediate_files.append(file_path)                          # add fastq to intermediate list
      for file_path in all_intermediate_files:
        os.remove(file_path)                                                    # remove intermediate files once merging is complete
    except:
      raise<commit_msg>fix for R3 reads<commit_after>import pandas as pd
import os,fnmatch,re,subprocess,shutil
from collections import defaultdict
from igf_data.illumina.samplesheet import SampleSheet
from igf_data.utils.fileutils import get_temp_dir,remove_dir,copy_local_file

class MergeSingleCellFastq:
  '''
  A class for merging single cell fastq files per lane per sample

  :param fastq_dir: A directory path containing fastq files
  :param samplesheet: A samplesheet file used demultiplexing of bcl files
  :param platform_name: A sequencing platform name
  :param singlecell_tag: A single cell keyword for description field, default '10X'
  :param sampleid_col: A keyword for sample id column of samplesheet, default 'Sample_ID'
  :param samplename_col: A keyword for sample name column of samplesheet, default 'Sample_Name'
  :param orig_sampleid_col: A keyword for original sample id column, default 'Original_Sample_ID'
  :param orig_samplename_col: A keyword for original sample name column, default 'Original_Sample_Name'
  :param description_col: A keyword for description column, default 'Description'
  :param project_col: A keyword for project column, default 'Sample_Project'
  :param pseudo_lane_col: A keyword for pseudo lane column, default 'PseudoLane'
  :param lane_col: A keyword for lane column, default 'Lane'
  :param force_overwrite: A toggle for overwriting output fastqs, default True

  SampleSheet file should contain following columns:
     * Sample_ID: A single cell sample id in the following format, SampleId_{digit}
     * Sample_Name: A single cell sample name in the following format, SampleName_{digit}
     * Original_Sample_ID: An IGF sample id
     * Original_Sample_Name: A sample name provided by user
     * Description: A single cell label, default 10X
  '''
  def __init__(self, fastq_dir,samplesheet,platform_name,singlecell_tag='10X',
    sampleid_col='Sample_ID', samplename_col='Sample_Name',use_ephemeral_space=0,
    orig_sampleid_col='Original_Sample_ID', description_col='Description',
    orig_samplename_col='Original_Sample_Name',project_col='Sample_Project',
    lane_col='Lane', pseudo_lane_col='PseudoLane',force_overwrite=True):
    """
    Add params
    """
    self.fastq_dir = fastq_dir
    self.samplesheet = samplesheet
    self.platform_name = platform_name
    self.singlecell_tag = singlecell_tag
    self.sampleid_col = sampleid_col
    self.samplename_col = samplename_col
    self.orig_sampleid_col = orig_sampleid_col
    self.description_col = description_col
    self.orig_samplename_col = orig_samplename_col
    self.project_col = project_col
    self.lane_col = lane_col
    self.pseudo_lane_col = pseudo_lane_col
    self.force_overwrite = force_overwrite
    self.use_ephemeral_space = use_ephemeral_space

  def _fetch_lane_and_sample_info_from_samplesheet(self):
    '''
    A internal method for grouping samples per lane based on the samplesheet
    returns a list containing sample and lane information per row
    '''
    try:
      samplesheet_data=SampleSheet(infile=self.samplesheet)                     # read samplesheet file
      if (self.orig_sampleid_col not in samplesheet_data._data_header) or \
         (self.orig_samplename_col not in samplesheet_data._data_header):
        raise ValueError(
      'Samplesheet {0} does not have {1} or {2} column'.\
        format(
          self.samplesheet,
          self.orig_sampleid_col,
          self.orig_samplename_col))                                            # check for required columns in the samplesheet
      samplesheet_data.\
      filter_sample_data(
        condition_key=self.description_col,
        condition_value=self.singlecell_tag,
        method='include')                                                       # filter samplesheet for single cell data
      sample_lane_data = list()
      if self.platform_name=='NEXTSEQ':                                         # hack for nextseq
        samplesheet_data.add_pseudo_lane_for_nextseq()
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.pseudo_lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      elif self.platform_name=='MISEQ':                                         # hack for miseq
        samplesheet_data.add_pseudo_lane_for_miseq()
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.pseudo_lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      elif self.platform_name=='HISEQ4000':                                     # check for hiseq4k
        for group_tag,_ in pd.DataFrame(samplesheet_data._data).\
                              groupby([self.lane_col,
                                       self.orig_sampleid_col,
                                       self.orig_samplename_col,
                                       self.project_col]):
          sample_lane_data.append({
            'lane_id':group_tag[0],
            'sample_id':group_tag[1],
            'sample_name':group_tag[2],
            'project_id':group_tag[3]})
      else:
        raise ValueError('platform {0} not supported'.format(self.platform_name))
      return sample_lane_data
    except:
      raise

  @staticmethod
  def _group_singlecell_fastq(sample_data,fastq_dir):
    '''
    A static method for grouping single cell fastq files

    required params:
    sample_data: A list of sample entries from samplesheet
                It should contain following keys for each row:
                lane_id, sample_id, sample_name, project_id
    fastq_dir: A directory path containing fastq files

    returns two dictionary of fastq group, one for single cell samples and
    another for sample information
    '''
    try:
      samples_info = defaultdict(dict)
      sample_files_list = \
        defaultdict(lambda: \
          defaultdict(lambda: \
            defaultdict(lambda: \
              defaultdict(list))))                                              # output data structure
      for sample_record in sample_data:
        sample_lane = sample_record.get('lane_id')
        sample_id = sample_record.get('sample_id')
        sample_name = sample_record.get('sample_name')
        project_id = sample_record.get('project_id')
        samples_info[sample_id]['sample_name'] = sample_name
        samples_info[sample_id]['project_id'] = project_id
        sample_id_regex = re.compile('^{0}_\d$'.format(sample_id))              # regexp for sample id match
        file_name_regex = \
          re.compile(r'^{0}_(\d)_S\d+_L00{1}_([R,I][1,2,3,4])_\d+\.fastq(\.gz)?$'.\
                     format(
                       sample_name,
                       sample_lane))                                            # regexp for fastq file match
        for root,_,files in os.walk(fastq_dir):
          for file in files:
            if fnmatch.fnmatch(file, "*.fastq.gz") and \
               not fnmatch.fnmatch(file, "Undetermined_*"):                     # skip undetermined reads
              if re.search(sample_id_regex,os.path.basename(root)) and \
                 re.search(file_name_regex,file):
                sm = re.match(file_name_regex,file)
                if len(sm.groups())>=2:
                  fragment_id=sm.group(1)
                  read_type=sm.group(2)
                  sample_files_list[sample_lane][sample_id][read_type][fragment_id].\
                  append(os.path.join(root,file))                               # add fastqs to samples list
                else:
                  raise ValueError('Failed to determined sample info:{0}, {1}'.\
                                   format(sample_id,file))
      return sample_files_list, samples_info
    except:
      raise

  def merge_fastq_per_lane_per_sample(self):
    '''
    A method for merging single cell fastq files present in input fastq_dir
    per lane per sample basis
    '''
    try:
      sample_data = \
        self._fetch_lane_and_sample_info_from_samplesheet()                     # get sample and lane information from samplesheet
      sample_files, samples_info = \
        self._group_singlecell_fastq(
          sample_data,
          self.fastq_dir)                                                       # get file groups
      all_intermediate_files=list()                                             # empty list for intermediate files
      s_count = 0                                                               # initial count for fastq S value
      for lane_id in sorted(sample_files.keys()):
        if self.platform_name=='NEXTSEQ':
            s_count = 0                                                         # nextseq is weird, reset counter for each lane
        for sample_id in sorted(sample_files[lane_id].keys()):
          s_count += 1                                                          # assign new S value for fastq files
          sample_name = samples_info.get(sample_id)['sample_name']
          project_id = samples_info.get(sample_id)['project_id']                # get sample and project info
          output_path = \
            os.path.join(
              self.fastq_dir,
              project_id,
              sample_id)                                                        # output location is under input fastq_dir
          if not os.path.exists(output_path):
            os.makedirs(output_path,mode=0o770)                                 # create outout directory

          for read_type in sample_files[lane_id][sample_id].keys():             # merge per read type
            output_filename = \
              '{0}_S{1}_L00{2}_{3}_001.fastq.gz'.\
                format(
                  sample_name,
                  s_count,
                  lane_id,
                  read_type)                                                    # assign new output filename
            final_path = os.path.join(output_path,output_filename)              # assign final output path
            if not self.force_overwrite and os.path.exists(final_path):
              raise ValueError('Failed to overwrite existing file {0}'.\
                               format(final_path))

            input_list = list()
            for sc_fragment, file_path in \
              sorted(sample_files[lane_id][sample_id][read_type].items()):
              input_list.extend(file_path)                                      # create list of input fastqs for merge
            if len(input_list) != 4:
              raise ValueError(\
                'expecting 4 files, got {0} for sample {1}, lane {2}, read type {3}'.\
                  format(
                    len(input_list),
                    sample_id,
                    lane_id,
                    read_type))                                                 # checking input files list
            temp_dir = \
              get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)        # get a temp dir
            temp_file = os.path.join(temp_dir,output_filename)                  # assign temp filename
            cmd = ["cat"]+input_list+[">",temp_file]                            # shell command for merging fastq.gz files
            subprocess.check_call(" ".join(cmd),shell=True)                     # exact same command for fastq merge as 10x pipeline
            copy_local_file(temp_file,final_path,force=True)                    # copy file to final location
            remove_dir(temp_dir)                                                # remove temp dir
            for file_path in input_list:
              all_intermediate_files.append(file_path)                          # add fastq to intermediate list
      for file_path in all_intermediate_files:
        os.remove(file_path)                                                    # remove intermediate files once merging is complete
    except:
      raise<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Andy Sayler
# 2016
# pytutamen Package
# Access Control Client

### Imports ###

from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import *

import uuid

from . import config
from . import base

### Constants ###

_WAIT_SLEEP = 0.1
_DEFAULT_TIMEOUT = 60

_EP_BOOTSTRAP = "bootstrap"

_KEY_ACCOUNTS = "accounts"

_KEY_CLIENTS = "clients"
_KEY_CLIENTS_CERTS = "{}_certs".format(_KEY_CLIENTS)

_EP_AUTHORIZATIONS = "authorizations"
_KEY_AUTHORIZATIONS = "authorizations"
_KEY_AUTHORIZATIONS_STATUS = "status"
_KEY_AUTHORIZATIONS_TOKEN = "token"
_VAL_AUTHORIZATIONS_STATUS_PENDING = 'pending'
_VAL_AUTHORIZATIONS_STATUS_GRANTED = 'approved'

_EP_VERIFIERS = "verifiers"
_KEY_VERIFIERS = "verifiers"

_EP_PERMISSIONS = "permissions"
_KEY_PERMISSIONS = "permissions"

### Exceptions ###

class ACServerConnectionException(base.ServerConnectionException):
    pass

### Connection Objects ###

class ACServerConnection(base.ServerConnection):

    def __init__(self, ac_server_name=None, conf=None, conf_path=None, **kwargs):

        # Setup Conf
        if not conf:
            conf = config.ClientConfig(conf_path=conf_path)

        # Get Server Name
        if not ac_server_name:
            ac_server_name = conf.defaults_get_ac_server()
            if not ac_server_name:
                raise(ACServerConnectionException("Missing AC Server Name"))

        # Get Server URL
        ac_server_url = conf.ac_server_get_url(ac_server_name)
        if not ac_server_url:
            raise(ACServerConnectionException("Missing AC Server URL"))

        # Call Parent
        super().__init__(server_url=ac_server_url, server_name=ac_server_name,
                         conf=conf, conf_path=conf_path, **kwargs)

### Client Objects ###

class AccessControlClient(object):

    def __init__(self, ac_connection):

        # Check Args
        if not isinstance(ac_connection, ACServerConnection):
            raise(TypeError("'ac_connection' must of an instance of {}".format(ACServerConnection)))

        # Call Parent
        super().__init__()

        # Setup Properties
        self._ac_connection = ac_connection

class BootstrapClient(AccessControlClient):

    def account(self, account_userdata=None, account_uid=None,
                client_userdata=None, client_uid=None, client_csr=None):

        if not client_csr:
            raise ValueError("client_csr required")

        ep = "{}/{}".format(_EP_BOOTSTRAP, _KEY_ACCOUNTS)

        json_out = {'client_csr': client_csr}
        if account_userdata:
            json_out['account_userdata'] = account_userdata
        if client_userdata:
            json_out['client_userdata'] = client_userdata
        if account_uid:
            json_out['account_uid'] = str(account_uid)
        if client_uid:
            json_out['client_uid'] = str(client_uid)

        res = self._ac_connection.http_post(ep, json=json_out)
        res_account_uid = uuid.UUID(res[_KEY_ACCOUNTS][0])
        if account_uid:
            assert res_account_uid == account_uid

        res_client_uid, res_client_cert = res[_KEY_CLIENTS_CERTS].popitem()
        res_client_uid = uuid.UUID(res_client_uid)
        if client_uid:
            assert res_client_uid == client_uid

        return (res_account_uid, res_client_uid, res_client_cert)

class AuthorizationsClient(AccessControlClient):

    def request(self, obj_type, obj_uid, obj_perm, userdata=None):

        if userdata is None:
            userdata = {}

        ep = "{}".format(_EP_AUTHORIZATIONS)

        json_out = {'objperm': obj_perm,
                    'objtype': obj_type,
                    'objuid': str(obj_uid) if obj_uid else "",
                    'userdata': userdata}

        res = self._ac_connection.http_post(ep, json=json_out)
        return uuid.UUID(res[_KEY_AUTHORIZATIONS][0])

    def fetch(self, auth_uid):

        ep = "{}/{}/".format(_KEY_AUTHORIZATIONS, str(auth_uid))

        auth = self._ac_connection.http_get(ep)
        return auth

    def wait_token(self, auth_uid, timeout=_DEFAULT_TIMEOUT):

        auth = self.fetch(auth_uid)
        status = auth[_KEY_AUTHORIZATIONS_STATUS]
        cnt = 0
        while (status == _VAL_AUTHORIZATIONS_STATUS_PENDING):
            if (cnt * _WAIT_SLEEP) > timeout:
                raise Exception("Timed out wating for token")
            time.sleep(_WAIT_SLEEP)
            auth = self.fetch(auth_uid)
            status = auth[_KEY_AUTHORIZATIONS_STATUS]
            cnt += 1
        if (status == _VAL_AUTHORIZATIONS_STATUS_GRANTED):
            return auth[_KEY_AUTHORIZATIONS_TOKEN]
        else:
            return None

class VerifiersClient(AccessControlClient):

    def create(self, uid=None, accounts=None, authenticators=None, userdata=None):

        if uid:
            if not isinstance(uid, uuid.UUID):
                raise TypeError("uid must be uuid.UUID")
        if not accounts:
            accounts = []
        if not authenticators:
            authenticators = []
        if userdata is None:
            userdata = {}

        ep = "{}".format(_EP_VERIFIERS)

        json_out = {}
        if uid:
            json_out['uid'] = str(uid)
        if accounts:
            json_out['accounts'] = accounts
        if authenticators:
            json_out['authenticators'] = accounts
        if userdata:
            json_out['userdata'] = userdata

        res = self._ac_connection.http_post(ep, json=json_out)
        return uuid.UUID(res[_KEY_VERIFIERS][0])

    def fetch(self, uid):

        if not isinstance(uid, uuid.UUID):
            raise TypeError("uid must be uuid.UUID")

        ep = "{}/{}/".format(_EP_VERIFIERS, str(uid))

        verifier = self._ac_connection.http_get(ep)
        return verifier

class PermissionsClient(AccessControlClient):

    def create(self, objtype, objuid=None,
               v_create=None, v_read=None,
               v_modify=None, v_delete=None,
               v_ac=None, v_default=None):

        v_default_str = []
        for v in v_default:
            v_default_str.append(str(v))

        ep = "{}".format(_EP_PERMISSIONS)

        json_out = {'objtype': objtype}
        if objuid:
            json_out['objuid'] = str(objuid)
        if v_create:
            json_out['create'] = v_create
        if v_read:
            json_out['read'] = v_read
        if v_modify:
            json_out['modify'] = v_modify
        if v_delete:
            json_out['delete'] = v_delete
        if v_ac:
            json_out['ac'] = v_ac
        if v_default:
            json_out['default'] = v_default_str

        res = self._ac_connection.http_post(ep, json=json_out)
        res = res[_KEY_PERMISSIONS][0]
        assert(objtype == res['objtype'])
        if objuid:
            assert(objuid == uuid.UUID(res['objuid']))
        return objtype, objuid

    def fetch(self, objtype, objuid):

        ep = "{}/{}/{}/".format(_EP_PERMISSIONS, objtype, str(objuid))

        perms = self._ac_connection.http_get(ep)
        return perms
<commit_msg>Added UID list normilzation<commit_after># -*- coding: utf-8 -*-

# Andy Sayler
# 2016
# pytutamen Package
# Access Control Client

### Imports ###

from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
from builtins import *

import uuid

from . import config
from . import base

### Constants ###

_WAIT_SLEEP = 0.1
_DEFAULT_TIMEOUT = 60

_EP_BOOTSTRAP = "bootstrap"

_KEY_ACCOUNTS = "accounts"

_KEY_CLIENTS = "clients"
_KEY_CLIENTS_CERTS = "{}_certs".format(_KEY_CLIENTS)

_EP_AUTHORIZATIONS = "authorizations"
_KEY_AUTHORIZATIONS = "authorizations"
_KEY_AUTHORIZATIONS_STATUS = "status"
_KEY_AUTHORIZATIONS_TOKEN = "token"
_VAL_AUTHORIZATIONS_STATUS_PENDING = 'pending'
_VAL_AUTHORIZATIONS_STATUS_GRANTED = 'approved'

_EP_VERIFIERS = "verifiers"
_KEY_VERIFIERS = "verifiers"

_EP_PERMISSIONS = "permissions"
_KEY_PERMISSIONS = "permissions"

### Exceptions ###

class ACServerConnectionException(base.ServerConnectionException):
    pass

### Connection Objects ###

class ACServerConnection(base.ServerConnection):

    def __init__(self, ac_server_name=None, conf=None, conf_path=None, **kwargs):

        # Setup Conf
        if not conf:
            conf = config.ClientConfig(conf_path=conf_path)

        # Get Server Name
        if not ac_server_name:
            ac_server_name = conf.defaults_get_ac_server()
            if not ac_server_name:
                raise(ACServerConnectionException("Missing AC Server Name"))

        # Get Server URL
        ac_server_url = conf.ac_server_get_url(ac_server_name)
        if not ac_server_url:
            raise(ACServerConnectionException("Missing AC Server URL"))

        # Call Parent
        super().__init__(server_url=ac_server_url, server_name=ac_server_name,
                         conf=conf, conf_path=conf_path, **kwargs)

### Client Objects ###

class AccessControlClient(object):

    def __init__(self, ac_connection):

        # Check Args
        if not isinstance(ac_connection, ACServerConnection):
            raise(TypeError("'ac_connection' must of an instance of {}".format(ACServerConnection)))

        # Call Parent
        super().__init__()

        # Setup Properties
        self._ac_connection = ac_connection

class BootstrapClient(AccessControlClient):

    def account(self, account_userdata=None, account_uid=None,
                client_userdata=None, client_uid=None, client_csr=None):

        if not client_csr:
            raise ValueError("client_csr required")

        ep = "{}/{}".format(_EP_BOOTSTRAP, _KEY_ACCOUNTS)

        json_out = {'client_csr': client_csr}
        if account_userdata:
            json_out['account_userdata'] = account_userdata
        if client_userdata:
            json_out['client_userdata'] = client_userdata
        if account_uid:
            json_out['account_uid'] = str(account_uid)
        if client_uid:
            json_out['client_uid'] = str(client_uid)

        res = self._ac_connection.http_post(ep, json=json_out)
        res_account_uid = uuid.UUID(res[_KEY_ACCOUNTS][0])
        if account_uid:
            assert res_account_uid == account_uid

        res_client_uid, res_client_cert = res[_KEY_CLIENTS_CERTS].popitem()
        res_client_uid = uuid.UUID(res_client_uid)
        if client_uid:
            assert res_client_uid == client_uid

        return (res_account_uid, res_client_uid, res_client_cert)

class AuthorizationsClient(AccessControlClient):

    def request(self, obj_type, obj_uid, obj_perm, userdata=None):

        if userdata is None:
            userdata = {}

        ep = "{}".format(_EP_AUTHORIZATIONS)

        json_out = {'objperm': obj_perm,
                    'objtype': obj_type,
                    'objuid': str(obj_uid) if obj_uid else "",
                    'userdata': userdata}

        res = self._ac_connection.http_post(ep, json=json_out)
        return uuid.UUID(res[_KEY_AUTHORIZATIONS][0])

    def fetch(self, auth_uid):

        ep = "{}/{}/".format(_KEY_AUTHORIZATIONS, str(auth_uid))

        auth = self._ac_connection.http_get(ep)
        return auth

    def wait_token(self, auth_uid, timeout=_DEFAULT_TIMEOUT):

        auth = self.fetch(auth_uid)
        status = auth[_KEY_AUTHORIZATIONS_STATUS]
        cnt = 0
        while (status == _VAL_AUTHORIZATIONS_STATUS_PENDING):
            if (cnt * _WAIT_SLEEP) > timeout:
                raise Exception("Timed out wating for token")
            time.sleep(_WAIT_SLEEP)
            auth = self.fetch(auth_uid)
            status = auth[_KEY_AUTHORIZATIONS_STATUS]
            cnt += 1
        if (status == _VAL_AUTHORIZATIONS_STATUS_GRANTED):
            return auth[_KEY_AUTHORIZATIONS_TOKEN]
        else:
            return None

class VerifiersClient(AccessControlClient):

    def create(self, uid=None, accounts=None, authenticators=None, userdata=None):

        if uid:
            if not isinstance(uid, uuid.UUID):
                raise TypeError("uid must be uuid.UUID")
        if not accounts:
            accounts = []
        else:
            accounts = [str(a) for a in accounts]
        if not authenticators:
            authenticators = []
        else:
            authenticators = [str(a) for a in authenticators]
        if userdata is None:
            userdata = {}

        ep = "{}".format(_EP_VERIFIERS)

        json_out = {}
        if uid:
            json_out['uid'] = str(uid)
        if accounts:
            json_out['accounts'] = accounts
        if authenticators:
            json_out['authenticators'] = accounts
        if userdata:
            json_out['userdata'] = userdata

        res = self._ac_connection.http_post(ep, json=json_out)
        return uuid.UUID(res[_KEY_VERIFIERS][0])

    def fetch(self, uid):

        if not isinstance(uid, uuid.UUID):
            raise TypeError("uid must be uuid.UUID")

        ep = "{}/{}/".format(_EP_VERIFIERS, str(uid))

        verifier = self._ac_connection.http_get(ep)
        return verifier

class PermissionsClient(AccessControlClient):

    def create(self, objtype, objuid=None,
               v_create=None, v_read=None,
               v_modify=None, v_delete=None,
               v_ac=None, v_default=None):

        if v_create:
            v_create = [str(v) for v in v_create]
        if v_read:
            v_read = [str(v) for v in v_read]
        if v_modify:
            v_modify = [str(v) for v in v_modify]
        if v_delete:
            v_delete = [str(v) for v in v_delete]
        if v_ac:
            v_ac = [str(v) for v in v_ac]
        if v_default:
            v_default = [str(v) for v in v_default]

        ep = "{}".format(_EP_PERMISSIONS)

        json_out = {'objtype': objtype}
        if objuid:
            json_out['objuid'] = str(objuid)
        if v_create:
            json_out['create'] = v_create
        if v_read:
            json_out['read'] = v_read
        if v_modify:
            json_out['modify'] = v_modify
        if v_delete:
            json_out['delete'] = v_delete
        if v_ac:
            json_out['ac'] = v_ac
        if v_default:
            json_out['default'] = v_default

        res = self._ac_connection.http_post(ep, json=json_out)
        res = res[_KEY_PERMISSIONS][0]
        assert(objtype == res['objtype'])
        if objuid:
            assert(objuid == uuid.UUID(res['objuid']))
        return objtype, objuid

    def fetch(self, objtype, objuid):

        ep = "{}/{}/{}/".format(_EP_PERMISSIONS, objtype, str(objuid))

        perms = self._ac_connection.http_get(ep)
        return perms
<|endoftext|>