<commit_before>import datetime
import json
import logging
import operator
import os
import shlex

import pytz
from tastypie import (
    authentication, authorization, bundle, exceptions, fields, resources
)
from twilio.twiml import Response

from django.conf import settings
from django.conf.urls import url
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.http import (
    HttpResponse, HttpResponseBadRequest,
    HttpResponseNotAllowed, HttpResponseNotFound
)

from . import models
from . import forms
from .context_managers import git_checkpoint
from .decorators import requires_taskd_sync, git_managed

logger = logging.getLogger(__name__)

class UserAuthorization(authorization.Authorization):
    def read_list(self, object_list, bundle):
        return object_list.filter(
            pk=bundle.request.user.pk
        )

    def read_detail(self, object_list, bundle):
        return bundle.obj == bundle.request.user

class UserResource(resources.ModelResource):
    def prepend_urls(self):
        return [
            url(
                r"^(?P<resource_name>%s)/status/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('account_status')
            ),
            url(
                r"^(?P<resource_name>%s)/my-certificate/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('my_certificate')
            ),
            url(
                r"^(?P<resource_name>%s)/my-key/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('my_key')
            ),
            url(
                r"^(?P<resource_name>%s)/ca-certificate/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('ca_certificate')
            ),
            url(
                r"^(?P<resource_name>%s)/taskrc/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('taskrc_extras')
            ),
            url(
                r"^(?P<resource_name>%s)/configure-taskd/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('configure_taskd')
            ),
            url(
                r"^(?P<resource_name>%s)/reset-taskd-configuration/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('reset_taskd_configuration')
            )
        ]

    def _send_file(self, out, content_type=None, **kwargs):
        if content_type is None:
            content_type = 'application/octet-stream'
        if out is None or not os.path.isfile(out):
            return HttpResponseNotFound()

        with open(out, 'r') as outfile:
            response = HttpResponse(
                outfile.read(),
                content_type=content_type,
            )
            response['Content-Disposition'] = 'attachment; filename="%s"' % (
                os.path.basename(out)
            )
            return response

    @git_managed("Reset taskd configuration")
    def reset_taskd_configuration(self, request, store=None, **kwargs):
        if request.method != 'POST':
            raise HttpResponseNotAllowed(
                'Only POST requests are allowed'
            )
        store.taskrc.update({
            'taskd.certificate': store.DEFAULT_FILENAMES['certificate'],
            'taskd.key': store.DEFAULT_FILENAMES['key'],
            'taskd.ca': store.server_config['ca.cert'],
            'taskd.server': settings.TASKD_SERVER,
            'taskd.credentials': store.metadata['generated_taskd_credentials']
        })
        return HttpResponse('OK')

    @git_managed("Configuring taskd server")
    def configure_taskd(self, request, store=None, **kwargs):
        if request.method != 'POST':
            raise HttpResponseNotAllowed(
                'Only POST requests are allowed'
            )

        form = forms.TaskdConfigurationForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest(
                json.dumps(form.errors),
                content_type='application/json',
            )

        cert_path = os.path.join(store.local_path, 'custom.private.cert.pem')
        with open(cert_path, 'w') as out:
            out.write(form.cleaned_data['certificate'])

        key_path = os.path.join(store.local_path, 'custom.private.key.pem')
        with open(key_path, 'w') as out:
            out.write(form.cleaned_data['key'])

        ca_path = os.path.join(store.local_path, 'custom.ca.pem')
        with open(ca_path, 'w') as out:
            out.write(form.cleaned_data['ca'])

        # Write files from form to user directory
        store.taskrc.update({
            'taskd.certificate': cert_path,
            'taskd.key': key_path,
            'taskd.ca': ca_path,
            'taskd.server': form.cleaned_data['server'],
            'taskd.credentials': form.cleaned_data['credentials'],
        })

        return HttpResponse('OK')

    def my_certificate(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.certificate_path,
            content_type='application/x-pem-file',
        )

    def my_key(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.key_path,
            content_type='application/x-pem-file',
        )

    def ca_certificate(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.server_config.get('ca.cert'),
            content_type='application/x-pem-file',
        )

    @git_managed("Updating custom taskrc configuration")
    def taskrc_extras(self, request, **kwargs):
        if request.method == 'GET':
            ts = models.TaskStore.get_for_user(request.user)
            return HttpResponse(
                ts.taskrc_extras,
                content_type='text/plain'
            )
        elif request.method == 'PUT':
            ts = models.TaskStore.get_for_user(request.user)
            ts.taskrc_extras = request.body.decode(request.encoding)
            results = ts.apply_extras()
            ts.save()
            return HttpResponse(
                json.dumps(
                    {
                        'success': results[0],
                        'failed': results[1]
                    }
                ),
                content_type='application/json',
            )
        else:
            return HttpResponseNotAllowed(
                'Only GET or PUT requests are allowed'
            )

    def account_status(self, request, **kwargs):
        if request.user.is_authenticated():
            store = models.TaskStore.get_for_user(request.user)
            user_data = {
                'logged_in': True,
                'uid': request.user.pk,
                'username': request.user.username,
                'name': (
                    request.user.first_name
                    if request.user.first_name
                    else request.user.username
                ),
                'email': request.user.email,
                'configured': store.configured,
                'taskd_credentials': store.taskrc.get('taskd.credentials'),
                'taskd_server': store.taskrc.get('taskd.server'),
                'taskd_is_custom': (
                    store.taskrc.get('taskd.server') != settings.TASKD_SERVER
                ),
                'taskrc_extras': store.taskrc_extras,
                'api_key': request.user.api_key.key,
                'sms_url': reverse(
                    'incoming_sms',
                    kwargs={
                        'api_name': 'v1',
                        'resource_name': 'task',
                        'username': request.user.username,
                    }
                )
            }
        else:
            user_data = {
                'logged_in': False,
            }
        return HttpResponse(
            json.dumps(user_data),
            content_type='application/json',
        )

    class Meta:
        queryset = User.objects.all()
        authorization = UserAuthorization()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']
        authentication = authentication.MultiAuthentication(
            authentication.ApiKeyAuthentication(),
            authentication.SessionAuthentication(),
        )

class Task(object):
    DATE_FIELDS = [
        'due', 'entry', 'modified', 'start', 'wait', 'scheduled',
    ]

    def __init__(self, json):
        if not json:
            raise ValueError()
        self.json = json

    def _date_from_taskw(self, value):
        value = datetime.datetime.strptime(
            value,
            '%Y%m%dT%H%M%SZ',
        )
        return value.replace(tzinfo=pytz.UTC)

    def _date_to_taskw(self, value):
        raise NotImplementedError()

    def __getattr__(self, name):
        try:
            value = self.json[name]
            if name in self.DATE_FIELDS:
                value = self._date_from_taskw(value)
            elif name == 'annotations':
                new_value = []
                for annotation in value:
                    annotation['entry'] = self._date_from_taskw(
                        annotation['entry']
                    )
                    new_value.append(annotation)
                value = new_value
            return value
        except KeyError:
            raise AttributeError()

    @property
    def id(self):
        return self.json['id'] if self.json['id'] else None

    @property
    def urgency(self):
        return float(self.json['urgency']) if self.json['urgency'] else None

    def __unicode__(self):
        return self.description

class TaskResource(resources.Resource):
    id = fields.IntegerField(attribute='id', null=True)
    uuid = fields.CharField(attribute='uuid')
    status = fields.CharField(attribute='status')
    urgency = fields.FloatField(attribute='urgency')
    description = fields.CharField(attribute='description')
    priority = fields.CharField(attribute='priority', null=True)
    project = fields.CharField(attribute='project', null=True)
    due = fields.DateTimeField(attribute='due', null=True)
    entry = fields.DateTimeField(attribute='entry', null=True)
    modified = fields.DateTimeField(attribute='modified', null=True)
    start = fields.DateTimeField(attribute='start', null=True)
    wait = fields.DateTimeField(attribute='wait', null=True)
    scheduled = fields.DateTimeField(attribute='scheduled', null=True)
    depends = fields.CharField(attribute='depends', null=True)
    annotations = fields.ListField(attribute='annotations', null=True)
    tags = fields.ListField(attribute='tags', null=True)
    imask = fields.IntegerField(attribute='imask', null=True)

    def prepend_urls(self):
        return [
            url(
                r"^(?P<resource_name>%s)/autoconfigure/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('autoconfigure')
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<username>[\w\d_.-]+)/sms/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('incoming_sms'),
                name="incoming_sms"
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/complete/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('complete')
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/delete/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('delete')
            ),
        ]

    @git_managed("Mark task completed")
    @requires_taskd_sync
    def complete(self, request, uuid, store, **kwargs):
        store.client.task_done(uuid=uuid)
        return HttpResponse(
            status=200
        )

    @git_managed("Delete task")
    def delete(self, request, uuid, **kwargs):
        return HttpResponse(
            status=501
        )

    def incoming_sms(self, request, username, **kwargs):
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            return HttpResponse(status=404)

        r = Response()
        store = models.TaskStore.get_for_user(user)

        with git_checkpoint(store, "Incoming SMS"):
            from_ = request.POST['From']
            body = request.POST['Body']
            task_info = body[4:]

            if not body.lower().startswith('add'):
                r.sms("Bad Request: Unknown command.")
                logger.warning(
                    "Incoming SMS from %s had no recognized command: '%s'" % (
                        from_,
                        body,
                    )
                )
            elif not task_info:
                logger.warning(
                    "Incoming SMS from %s had no content." % (
                        from_,
                        body,
                    )
                )
                r.sms("Bad Request: Empty task.")
            else:
                store.sync()
                task_args = ['add'] + shlex.split(task_info)
                result = store.client._execute(*task_args)
                stdout, stderr = result
                r.sms("Added.")

                logger.info(
                    "Added task from %s; message '%s'; response: '%s'" % (
                        from_,
                        body,
                        stdout,
                    )
                )

                store.sync()
        return HttpResponse(str(r), content_type='application/xml')

    def autoconfigure(self, request, **kwargs):
        store = models.TaskStore.get_for_user(request.user)
        try:
            store.autoconfigure_taskd()
        except Exception as e:
            return HttpResponse(
                json.dumps({
                    'error': str(e)
                }),
                status=500,
                content_type='application/json',
            )
        return HttpResponse(
            json.dumps({
                'status': 'Successfully configured'
            }),
            status=200,
            content_type='application/json',
        )

    def _get_store(self, user):
        return user.task_stores.get()

    def detail_uri_kwargs(self, bundle_or_obj):
        kwargs = {}

        if isinstance(bundle_or_obj, bundle.Bundle):
            kwargs['pk'] = bundle_or_obj.obj.uuid
        else:
            kwargs['pk'] = bundle_or_obj.uuid

        return kwargs

    def apply_sorting(self, obj_list, options=None):
        if options is None:
            options = {}

        parameter_name = 'order_by'

        if hasattr(options, 'getlist'):
            order_bits = options.getlist(parameter_name)
        else:
            order_bits = options.get(parameter_name)

            if not isinstance(order_bits, list, tuple):
                order_bits = [order_bits]

        if not order_bits:
            order_bits = ['-urgency']

        order_by_args = []
        for order_by in order_bits:
            order_by_bits = order_by.split(',')

            field_name = order_by_bits[0]
            reverse = False

            if order_by_bits[0].startswith('-'):
                field_name = order_by_bits[0][1:]
                reverse = True

            order_by_args.append(
                (field_name, reverse)
            )

        order_by_args.reverse()
        for arg, reverse in order_by_args:
            obj_list = sorted(
                obj_list,
                key=operator.attrgetter(arg),
                reverse=reverse,
            )

        return obj_list

    def passes_filters(self, task, filters):
        for key, value in filters.items():
            task_value = getattr(task, key, None)
            if task_value == value:
                return True
        return False

    @requires_taskd_sync
    def obj_get_list(self, bundle, store, **kwargs):
        if hasattr(bundle.request, 'GET'):
            filters = bundle.request.GET.copy()
        filters.update(kwargs)

        key = 'pending'
        if int(bundle.request.GET.get('completed', 0)):
            key = 'completed'

        objects = []
        for task_json in store.client.load_tasks()[key]:
            task = Task(task_json)
            if self.passes_filters(task, filters):
                objects.append(task)

        return objects

    @requires_taskd_sync
    def obj_get(self, bundle, store, **kwargs):
        try:
            return Task(store.client.get_task(uuid=kwargs['pk'])[1])
        except ValueError:
            raise exceptions.NotFound()

    class Meta:
        authentication = authentication.MultiAuthentication(
            authentication.ApiKeyAuthentication(),
            authentication.SessionAuthentication(),
        )
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']
        limit = 1000
        max_limit = 1000
<commit_msg>Fixes two bugs relating to custom certificates:<commit_after>import datetime
import json
import logging
import operator
import os
import shlex

import pytz
from tastypie import (
    authentication, authorization, bundle, exceptions, fields, resources
)
from twilio.twiml import Response

from django.conf import settings
from django.conf.urls import url
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.http import (
    HttpResponse, HttpResponseBadRequest,
    HttpResponseNotAllowed, HttpResponseNotFound
)

from . import models
from . import forms
from .context_managers import git_checkpoint
from .decorators import requires_taskd_sync, git_managed

logger = logging.getLogger(__name__)

class UserAuthorization(authorization.Authorization):
    def read_list(self, object_list, bundle):
        return object_list.filter(
            pk=bundle.request.user.pk
        )

    def read_detail(self, object_list, bundle):
        return bundle.obj == bundle.request.user

class UserResource(resources.ModelResource):
    def prepend_urls(self):
        return [
            url(
                r"^(?P<resource_name>%s)/status/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('account_status')
            ),
            url(
                r"^(?P<resource_name>%s)/my-certificate/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('my_certificate')
            ),
            url(
                r"^(?P<resource_name>%s)/my-key/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('my_key')
            ),
            url(
                r"^(?P<resource_name>%s)/ca-certificate/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('ca_certificate')
            ),
            url(
                r"^(?P<resource_name>%s)/taskrc/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('taskrc_extras')
            ),
            url(
                r"^(?P<resource_name>%s)/configure-taskd/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('configure_taskd')
            ),
            url(
                r"^(?P<resource_name>%s)/reset-taskd-configuration/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('reset_taskd_configuration')
            )
        ]

    def _send_file(self, out, content_type=None, **kwargs):
        if content_type is None:
            content_type = 'application/octet-stream'
        if out is None or not os.path.isfile(out):
            return HttpResponseNotFound()

        with open(out, 'r') as outfile:
            response = HttpResponse(
                outfile.read(),
                content_type=content_type,
            )
            response['Content-Disposition'] = 'attachment; filename="%s"' % (
                os.path.basename(out)
            )
            return response

    @git_managed("Reset taskd configuration")
    def reset_taskd_configuration(self, request, store=None, **kwargs):
        if request.method != 'POST':
            raise HttpResponseNotAllowed(
                'Only POST requests are allowed'
            )
        store.taskrc.update({
            'taskd.certificate': os.path.join(
                store.local_path,
                store.DEFAULT_FILENAMES['certificate']
            ),
            'taskd.key': os.path.join(
                store.local_path,
                store.DEFAULT_FILENAMES['key']
            ),
            'taskd.ca': store.server_config['ca.cert'],
            'taskd.server': settings.TASKD_SERVER,
            'taskd.credentials': store.metadata['generated_taskd_credentials']
        })
        return HttpResponse('OK')

    @git_managed("Configuring taskd server")
    def configure_taskd(self, request, store=None, **kwargs):
        if request.method != 'POST':
            raise HttpResponseNotAllowed(
                'Only POST requests are allowed'
            )

        form = forms.TaskdConfigurationForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest(
                json.dumps(form.errors),
                content_type='application/json',
            )

        cert_path = os.path.join(store.local_path, 'custom.private.cert.pem')
        with open(cert_path, 'w') as out:
            out.write(form.cleaned_data['certificate'])

        key_path = os.path.join(store.local_path, 'custom.private.key.pem')
        with open(key_path, 'w') as out:
            out.write(form.cleaned_data['key'])

        ca_path = os.path.join(store.local_path, 'custom.ca.pem')
        with open(ca_path, 'w') as out:
            out.write(form.cleaned_data['ca'])

        # Write files from form to user directory
        store.taskrc.update({
            'taskd.certificate': cert_path,
            'taskd.key': key_path,
            'taskd.ca': ca_path,
            'taskd.server': form.cleaned_data['server'],
            'taskd.credentials': form.cleaned_data['credentials'],
        })

        return HttpResponse('OK')

    def my_certificate(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.taskrc.get('taskd.certificate'),
            content_type='application/x-pem-file',
        )

    def my_key(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.taskrc.get('taskd.key'),
            content_type='application/x-pem-file',
        )

    def ca_certificate(self, request, **kwargs):
        if request.method != 'GET':
            raise HttpResponseNotAllowed(
                'Only GET requests are allowed'
            )
        ts = models.TaskStore.get_for_user(request.user)
        return self._send_file(
            ts.server_config.get('ca.cert'),
            content_type='application/x-pem-file',
        )

    @git_managed("Updating custom taskrc configuration")
    def taskrc_extras(self, request, **kwargs):
        if request.method == 'GET':
            ts = models.TaskStore.get_for_user(request.user)
            return HttpResponse(
                ts.taskrc_extras,
                content_type='text/plain'
            )
        elif request.method == 'PUT':
            ts = models.TaskStore.get_for_user(request.user)
            ts.taskrc_extras = request.body.decode(request.encoding)
            results = ts.apply_extras()
            ts.save()
            return HttpResponse(
                json.dumps(
                    {
                        'success': results[0],
                        'failed': results[1]
                    }
                ),
                content_type='application/json',
            )
        else:
            return HttpResponseNotAllowed(
                'Only GET or PUT requests are allowed'
            )

    def account_status(self, request, **kwargs):
        if request.user.is_authenticated():
            store = models.TaskStore.get_for_user(request.user)
            user_data = {
                'logged_in': True,
                'uid': request.user.pk,
                'username': request.user.username,
                'name': (
                    request.user.first_name
                    if request.user.first_name
                    else request.user.username
                ),
                'email': request.user.email,
                'configured': store.configured,
                'taskd_credentials': store.taskrc.get('taskd.credentials'),
                'taskd_server': store.taskrc.get('taskd.server'),
                'taskd_is_custom': (
                    store.taskrc.get('taskd.server') != settings.TASKD_SERVER
                ),
                'taskrc_extras': store.taskrc_extras,
                'api_key': request.user.api_key.key,
                'sms_url': reverse(
                    'incoming_sms',
                    kwargs={
                        'api_name': 'v1',
                        'resource_name': 'task',
                        'username': request.user.username,
                    }
                )
            }
        else:
            user_data = {
                'logged_in': False,
            }
        return HttpResponse(
            json.dumps(user_data),
            content_type='application/json',
        )

    class Meta:
        queryset = User.objects.all()
        authorization = UserAuthorization()
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']
        authentication = authentication.MultiAuthentication(
            authentication.ApiKeyAuthentication(),
            authentication.SessionAuthentication(),
        )

class Task(object):
    DATE_FIELDS = [
        'due', 'entry', 'modified', 'start', 'wait', 'scheduled',
    ]

    def __init__(self, json):
        if not json:
            raise ValueError()
        self.json = json

    def _date_from_taskw(self, value):
        value = datetime.datetime.strptime(
            value,
            '%Y%m%dT%H%M%SZ',
        )
        return value.replace(tzinfo=pytz.UTC)

    def _date_to_taskw(self, value):
        raise NotImplementedError()

    def __getattr__(self, name):
        try:
            value = self.json[name]
            if name in self.DATE_FIELDS:
                value = self._date_from_taskw(value)
            elif name == 'annotations':
                new_value = []
                for annotation in value:
                    annotation['entry'] = self._date_from_taskw(
                        annotation['entry']
                    )
                    new_value.append(annotation)
                value = new_value
            return value
        except KeyError:
            raise AttributeError()

    @property
    def id(self):
        return self.json['id'] if self.json['id'] else None

    @property
    def urgency(self):
        return float(self.json['urgency']) if self.json['urgency'] else None

    def __unicode__(self):
        return self.description

class TaskResource(resources.Resource):
    id = fields.IntegerField(attribute='id', null=True)
    uuid = fields.CharField(attribute='uuid')
    status = fields.CharField(attribute='status')
    urgency = fields.FloatField(attribute='urgency')
    description = fields.CharField(attribute='description')
    priority = fields.CharField(attribute='priority', null=True)
    project = fields.CharField(attribute='project', null=True)
    due = fields.DateTimeField(attribute='due', null=True)
    entry = fields.DateTimeField(attribute='entry', null=True)
    modified = fields.DateTimeField(attribute='modified', null=True)
    start = fields.DateTimeField(attribute='start', null=True)
    wait = fields.DateTimeField(attribute='wait', null=True)
    scheduled = fields.DateTimeField(attribute='scheduled', null=True)
    depends = fields.CharField(attribute='depends', null=True)
    annotations = fields.ListField(attribute='annotations', null=True)
    tags = fields.ListField(attribute='tags', null=True)
    imask = fields.IntegerField(attribute='imask', null=True)

    def prepend_urls(self):
        return [
            url(
                r"^(?P<resource_name>%s)/autoconfigure/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('autoconfigure')
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<username>[\w\d_.-]+)/sms/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('incoming_sms'),
                name="incoming_sms"
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/complete/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('complete')
            ),
            url(
                r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/delete/?$" % (
                    self._meta.resource_name
                ),
                self.wrap_view('delete')
            ),
        ]

    @git_managed("Mark task completed")
    @requires_taskd_sync
    def complete(self, request, uuid, store, **kwargs):
        store.client.task_done(uuid=uuid)
        return HttpResponse(
            status=200
        )

    @git_managed("Delete task")
    def delete(self, request, uuid, **kwargs):
        return HttpResponse(
            status=501
        )

    def incoming_sms(self, request, username, **kwargs):
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            return HttpResponse(status=404)

        r = Response()
        store = models.TaskStore.get_for_user(user)

        with git_checkpoint(store, "Incoming SMS"):
            from_ = request.POST['From']
            body = request.POST['Body']
            task_info = body[4:]

            if not body.lower().startswith('add'):
                r.sms("Bad Request: Unknown command.")
                logger.warning(
                    "Incoming SMS from %s had no recognized command: '%s'" % (
                        from_,
                        body,
                    )
                )
            elif not task_info:
                logger.warning(
                    "Incoming SMS from %s had no content." % (
                        from_,
                        body,
                    )
                )
                r.sms("Bad Request: Empty task.")
            else:
                store.sync()
                task_args = ['add'] + shlex.split(task_info)
                result = store.client._execute(*task_args)
                stdout, stderr = result
                r.sms("Added.")

                logger.info(
                    "Added task from %s; message '%s'; response: '%s'" % (
                        from_,
                        body,
                        stdout,
                    )
                )

                store.sync()
        return HttpResponse(str(r), content_type='application/xml')

    def autoconfigure(self, request, **kwargs):
        store = models.TaskStore.get_for_user(request.user)
        try:
            store.autoconfigure_taskd()
        except Exception as e:
            return HttpResponse(
                json.dumps({
                    'error': str(e)
                }),
                status=500,
                content_type='application/json',
            )
        return HttpResponse(
            json.dumps({
                'status': 'Successfully configured'
            }),
            status=200,
            content_type='application/json',
        )

    def _get_store(self, user):
        return user.task_stores.get()

    def detail_uri_kwargs(self, bundle_or_obj):
        kwargs = {}

        if isinstance(bundle_or_obj, bundle.Bundle):
            kwargs['pk'] = bundle_or_obj.obj.uuid
        else:
            kwargs['pk'] = bundle_or_obj.uuid

        return kwargs

    def apply_sorting(self, obj_list, options=None):
        if options is None:
            options = {}

        parameter_name = 'order_by'

        if hasattr(options, 'getlist'):
            order_bits = options.getlist(parameter_name)
        else:
            order_bits = options.get(parameter_name)

            if not isinstance(order_bits, list, tuple):
                order_bits = [order_bits]

        if not order_bits:
            order_bits = ['-urgency']

        order_by_args = []
        for order_by in order_bits:
            order_by_bits = order_by.split(',')

            field_name = order_by_bits[0]
            reverse = False

            if order_by_bits[0].startswith('-'):
                field_name = order_by_bits[0][1:]
                reverse = True

            order_by_args.append(
                (field_name, reverse)
            )

        order_by_args.reverse()
        for arg, reverse in order_by_args:
            obj_list = sorted(
                obj_list,
                key=operator.attrgetter(arg),
                reverse=reverse,
            )

        return obj_list

    def passes_filters(self, task, filters):
        for key, value in filters.items():
            task_value = getattr(task, key, None)
            if task_value == value:
                return True
        return False

    @requires_taskd_sync
    def obj_get_list(self, bundle, store, **kwargs):
        if hasattr(bundle.request, 'GET'):
            filters = bundle.request.GET.copy()
        filters.update(kwargs)

        key = 'pending'
        if int(bundle.request.GET.get('completed', 0)):
            key = 'completed'

        objects = []
        for task_json in store.client.load_tasks()[key]:
            task = Task(task_json)
            if self.passes_filters(task, filters):
                objects.append(task)

        return objects

    @requires_taskd_sync
    def obj_get(self, bundle, store, **kwargs):
        try:
            return Task(store.client.get_task(uuid=kwargs['pk'])[1])
        except ValueError:
            raise exceptions.NotFound()

    class Meta:
        authentication = authentication.MultiAuthentication(
            authentication.ApiKeyAuthentication(),
            authentication.SessionAuthentication(),
        )
        list_allowed_methods = ['get']
        detail_allowed_methods = ['get']
        limit = 1000
        max_limit = 1000
<|endoftext|>


<commit_before>#!/usr/bin/env python

from datetime import timedelta
from dateutil import parser
from random import randint
from time import sleep
from tweepy import Stream, OAuthHandler
from tweepy.streaming import StreamListener
from yconf import BaseConfiguration

import json
import logging
import pymongo
import signal
import sys

__VERSION__ = "0.4"

class SmallhandsListener(StreamListener):
    def __init__(self, db, config, count=0):
        self.db     = db
        self.config = config
        self.count  = count
        self.logger = logging.getLogger(__name__)

    def parse_tweet(self, data):
        try:
            parsed = data
            if isinstance(data, dict):
                parsed = {}
                for key in data:
                    if key == "created_at":
                        data[key] = parser.parse(data[key])
                    parsed[key] = self.parse_tweet(data[key])
            elif isinstance(data, list):
                parsed = []
                for item in data:
                    parsed.append(self.parse_tweet(item))
            return parsed
        except Exception, e:
            self.logger.error("Error parsing tweet: %s" % e)

    def process_tweet(self, data):
        try:
            tweet = self.parse_tweet(json.loads(data))
            if 'created_at' in tweet and 'expire' in self.config.db:
                if 'min_secs' in self.config.db.expire and 'max_secs' in self.config.db.expire:
                    ttl_secs = randint(self.config.db.expire.min_secs, self.config.db.expire.max_secs)
                    tweet['expire_at'] = tweet['created_at'] + timedelta(seconds=ttl_secs)
            return tweet
        except Exception, e:
            self.logger.error("Error processing tweet: %s" % e)

    def on_data(self, data):
        tweet = self.process_tweet(data)
        try:
            self.db['tweets'].insert(tweet)
            if 'user' in tweet:
                if 'expire_at' in tweet:
                    tweet['user']['expire_at'] = tweet['expire_at']
                self.db['users'].update_one({ 'id': tweet['user']['id'] }, { '$set' : tweet['user'] }, upsert=True)
            self.count += 1
            if (self.count % 50) == 0:
                self.logger.info("Wrote 50 tweets (total: %i)" % self.count)
        except pymongo.errors.DuplicateKeyError, e:
            pass
        except Exception, e:
            self.logger.error("Error writing tweet to db: %s" % e)

    def on_error(self, e):
        msg = e
        if e == 401:
            msg = "401 Unauthorized: Authentication credentials were missing or incorrect"
        elif e == 420:
            msg = "420 Rate limiting"
            return sleep(3)
        elif e == 429:
            msg = "429 Too Many Requests"
            return sleep(5)
        self.logger.error("Twitter Streaming API error: '%s'" % msg)

class SmallhandsConfig(BaseConfiguration):
    def makeParser(self):
        parser = super(SmallhandsConfig, self).makeParser()
        parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose/debug output (default: false)", default=False, action="store_true")
        parser.add_argument("-l", "--log-file", dest="log_file", help="Output to log file", type=str, default=None)
        parser.add_argument("-H", "--db-host", dest="db.host", help="MongoDB hostname/ip")
        parser.add_argument("-P", "--db-port", dest="db.port", type=int, help="MongoDB TCP port")
        parser.add_argument("-D", "--db-name", dest="db.name", help="MongoDB Database name (default: smallhands)")
        parser.add_argument("-u", "--db-user", dest="db.user", help="(Optional) MongoDB Username")
        parser.add_argument("-p", "--db-password", dest="db.password", help="(Optional) MongoDB Password")
        parser.add_argument("-d", "--db-authdb", dest="db.authdb", help="MongoDB Authentication Database (default: admin)")
        parser.add_argument("-x", "--db-expire-min", dest="db.expire.min", type=int, help="(Optional) minimum MongoDB TTL expiry in seconds")
        parser.add_argument("-X", "--db-expire-max", dest="db.expire.max", type=int, help="(Optional) maximum MongoDB TTL expiry in seconds")
        parser.add_argument("-k", "--twitter-consumer-key", dest="twitter.consumer.key", help="Twitter Consumer Key (REQUIRED)")
        parser.add_argument("-s", "--twitter-consumer-secret", dest="twitter.consumer.secret", help="Twitter Consumer Secret (REQUIRED)")
        parser.add_argument("-T", "--twitter-access-token", dest="twitter.access.token", help="Twitter Access Token (REQUIRED)")
        parser.add_argument("-S", "--twitter-access-secret", dest="twitter.access.secret", help="Twitter Access Secret (REQUIRED)")
        parser.add_argument("-F", "--twitter-filters", dest="twitter.filters", help="Comma-separated list of filters to Twitter stream (default: @realDonaldTrump)")
        parser.add_argument("--twitter-stream-timeout", dest="twitter.stream.timeout", help="Timeout for Twitter Streaming API, in seconds (default: 30)", default=30, type=int)
        return parser

class Smallhands():
    def __init__(self, log_level=logging.INFO):
        self.log_level = log_level
        self.config    = self.parse_config()
        self.logger    = self.setup_logger()
        self.db_conn   = None
        self.stopped   = False
        self.stream    = None

        # Parse twitter-stream filters
        self.stream_filters = ["@realDonaldTrump"]
        if 'filters' in self.config.twitter:
            self.stream_filters = self.config.twitter.filters.split(",")
        if len(self.stream_filters) < 1:
            self.logger.fatal("No Twitter stream filters!")
            raise e

        self.logger.info("Starting Smallhands version: %s (https://github.com/timvaillancourt/smallhands)" % __VERSION__)
        self.logger.info("\t\t\"I'm going to make database testing great again. Believe me.\"")

    def setup_logger(self):
        try:
            if self.config.verbose:
                if not isinstance(self.config.verbose, str) or self.config.verbose.startswith("true"):
                    self.log_level = logging.DEBUG
            self.logger = logging.getLogger(__name__)
            stream_log  = logging.StreamHandler()
            formatter   = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(module)s:%(lineno)d] %(message)s')
            stream_log.setFormatter(formatter)
            self.logger.addHandler(stream_log)
            if self.config.log_file:
                file_log = logging.FileHandler(self.config.log_file)
                file_log.setFormatter(formatter)
                self.logger.addHandler(file_log)
            self.logger.setLevel(self.log_level)
            return self.logger
        except Exception, e:
            print('Error setting up logger: %s' % e)
            raise e

    def auth_conn(self, conn):
        try:
            if 'user' in self.config.db and 'password' in self.config.db:
                self.logger.debug("Authenticating db connection as user: %s" % self.config.db.user)
                conn[self.config.db.authdb].authenticate(
                    self.config.db.user,
                    self.config.db.password,
                    source=self.config.db.authdb
                )
        except Exception, e:
            self.logger.fatal("Error authenticating to: %s:%i - %s" % (self.config.db.host, self.config.db.port, e))

    def ensure_indices(self, conn, collection, sharded=False):
        try:
            db = conn[self.config.db.name]
            self.logger.debug("Ensuring 'id' index on '%s.%s'" % (self.config.db.name, collection))
            db[collection].create_index([("id", pymongo.ASCENDING)], unique=True)
            if sharded:
                self.logger.debug("Ensuring optional hashed 'id' index on '%s.%s' (for sharding)" % (self.config.db.name, collection))
                db[collection].create_index([('id', pymongo.HASHED)])
            if 'expire' in self.config.db and 'min_secs' in self.config.db.expire and 'max_secs' in self.config.db.expire:
                self.logger.debug("Ensuring optional TTL-expiry 'expire_at' index on '%s.%s'" % (self.config.db.name, collection))
                db[collection].create_index([("expire_at", pymongo.ASCENDING)], expireAfterSeconds=0)
        except Exception, e:
            self.logger.fatal("Error creating collections on: '%s.%s' - %s" % (self.config.db.name, collection, e))
            raise e

    def get_db(self, force=False):
        try:
            if force or not self.db_conn:
                self.logger.debug("Getting mongo connection to '%s:%i'" % (self.config.db.host, self.config.db.port))
                self.db_conn = pymongo.MongoClient(
                    self.config.db.host,
                    self.config.db.port
                )
                self.auth_conn(self.db_conn)

                # Setup indices, setup TTL index conditionally:
                if self.db_conn.is_mongos:
                    self.logger.info("Detected 'mongos' process. Using sharded mode")

                    try:
                        self.logger.info("Ensuring sharding is enabled for db: %s" % self.config.db.name)
                        db = self.db_conn['admin']
                        db.command({ 'enableSharding': self.config.db.name })
                    except pymongo.errors.OperationFailure, e:
                        if e.code != 23:
                            raise e
                    except Exception, e:
                        raise e

                    shard_conn = None
                    try:
                        db = self.db_conn['config']
                        for shard in db.shards.find():
                            shard_conn = pymongo.MongoClient(shard['host'], replicaSet=shard['_id'], readPreference='primary')
                            self.auth_conn(shard_conn)
                            self.logger.info("Ensuring indices on shard: %s" % shard['host'])
                            for collection in ['tweets', 'users']:
                                self.logger.debug("Ensuring shard indices for '%s.%s'" % (self.config.db.name, collection))
                                self.ensure_indices(shard_conn, collection, True)
                    except Exception, e:
                        raise e
                    finally:
                        if shard_conn:
                            shard_conn.close()

                    for collection in ['tweets', 'users']:
                        try:
                            self.logger.info("Ensuring collection is sharded: '%s.%s'" % (self.config.db.name, collection))
                            db = self.db_conn['admin']
                            db.command({ 'shardCollection': '%s.%s' % (self.config.db.name, collection), 'key': { 'id': pymongo.HASHED } })
                        except pymongo.errors.OperationFailure, e:
                            if e.code != 20:
                                raise e
                        except Exception, e:
                            raise e
                else:
                    self.logger.info("Ensuring indices on: '%s:%i'" % (self.config.db.host, self.config.db.port))
                    for collection in ['tweets', 'users']:
                        self.logger.debug("Ensuring indices for '%s.%s'" % (self.config.db.name, collection))
                        self.ensure_indices(self.db_conn, collection)
            return self.db_conn[self.config.db.name]
        except Exception, e:
            self.logger.fatal("Error setting up db: %s" % e)
            raise Exception, 'MongoDB connection error - %s' % e, None

    def get_twitter_auth(self):
        try:
            self.logger.debug("Authenticating Twitter API session")
            auth = OAuthHandler(
                self.config.twitter.consumer.key,
                self.config.twitter.consumer.secret
            )
            auth.set_access_token(
                self.config.twitter.access.key,
                self.config.twitter.access.secret
            )
            return auth
        except Exception, e:
            self.logger.fatal("Error with Twitter auth: %s" % e)
            raise e

    def parse_config(self):
        try:
            config = SmallhandsConfig()
            config.parse(sys.argv[1:])

            # Set defaults:
            if 'db' not in config:
                config.db = {}
            if 'name' not in config.db:
                config.db.name = "smallhands"
            if 'host' not in config.db:
                config.db.host = "localhost"
            if 'port' not in config.db:
                config.db.port = 27017

            # Required fields
            if 'twitter' in config:
                required = [
                    config.twitter.access.key,
                    config.twitter.access.secret,
                    config.twitter.consumer.key,
                    config.twitter.consumer.secret
                ]
                for field in required:
                    try:
                        test = field
                    except Exception, e:
                        raise Exception, 'Required config field "%s" not specified!' % field, None
            else:
                raise Exception, 'Required "twitter" auth key/secret info not set! Please set via config file or command line!', None
            return config
        except Exception, e:
            print("Error parsing config: %s" % e)
            raise e

    def start_stream(self):
        db   = self.get_db()
        auth = self.get_twitter_auth()

        # Start the stream
        try:
            self.logger.info("Listening to Twitter Streaming API for mentions of: %s, writing data to: '%s:%i'" % (self.stream_filters, self.config.db.host, self.config.db.port))
            self.stream = Stream(auth, SmallhandsListener(db, self.config), timeout=self.config.twitter.stream.timeout)
            self.stream.filter(track=self.stream_filters, async=True)
        except Exception, e:
            raise e

    def start(self):
        try:
            self.start_stream()
            while not self.stopped and self.stream:
                if not self.stopped and not self.stream.running:
                    self.logger.error("Restarting streaming due to error!")
                    self.stop_stream()
                    self.start_stream()
                else:
                    sleep(1)
        except Exception, e:
            self.logger.fatal("Error with Twitter Streaming: %s" % e)
            raise e

    def stop_stream(self):
        try:
            self.logger.info("Stopping tweet streaming")
            if self.stream:
                self.stream.disconnect()
        except Exception, e:
            self.logger.fatal("Error stopping streaming: %s" % e)
            raise e

    def stop(self, frame=None, code=None):
        if not self.stopped:
            try:
                self.stopped = True
                self.stop_stream()
                if self.db_conn:
                    self.db_conn.close()
                self.logger.info("Smallhands stopped. Sad!")
                if frame and code:
                    sys.exit(1)
            except Exception, e:
                raise e

if __name__ == "__main__":
    smallhands = None
    try:
        smallhands = Smallhands()
        signal.signal(signal.SIGINT, smallhands.stop)
        signal.signal(signal.SIGTERM, smallhands.stop)
        smallhands.start()
    except Exception, e:
        sys.exit(1)
    finally:
        if smallhands:
            smallhands.stop()
<commit_msg>Update counts/status every 30 seconds, add flag to change<commit_after>#!/usr/bin/env python

from datetime import timedelta
from dateutil import parser
from random import randint
from time import sleep, time
from tweepy import Stream, OAuthHandler
from tweepy.streaming import StreamListener
from yconf import BaseConfiguration

import json
import logging
import pymongo
import signal
import sys

__VERSION__ = "0.4"

class SmallhandsListener(StreamListener):
    def __init__(self, db, config, count=0):
        self.db     = db
        self.config = config
        self.count  = count
        self.logger = logging.getLogger(__name__)

        self.last_report_time  = int(time())
        self.last_report_count = self.count

    def parse_tweet(self, data):
        try:
            parsed = data
            if isinstance(data, dict):
                parsed = {}
                for key in data:
                    if key == "created_at":
                        data[key] = parser.parse(data[key])
                    parsed[key] = self.parse_tweet(data[key])
            elif isinstance(data, list):
                parsed = []
                for item in data:
                    parsed.append(self.parse_tweet(item))
            return parsed
        except Exception, e:
            self.logger.error("Error parsing tweet: %s" % e)

    def process_tweet(self, data):
        try:
            tweet = self.parse_tweet(json.loads(data))
            if 'created_at' in tweet and 'expire' in self.config.db:
                if 'min_secs' in self.config.db.expire and 'max_secs' in self.config.db.expire:
                    ttl_secs = randint(self.config.db.expire.min_secs, self.config.db.expire.max_secs)
                    tweet['expire_at'] = tweet['created_at'] + timedelta(seconds=ttl_secs)
            return tweet
        except Exception, e:
            self.logger.error("Error processing tweet: %s" % e)

    def on_data(self, data):
        tweet = self.process_tweet(data)
        try:
            now = time()
            if int(now) >= int(self.last_report_time) + self.config.report_interval:
                count = self.count - self.last_report_count
                tps = float(count) / float(self.config.report_interval)
                self.logger.info("Processed %i tweets (total: %i) in %i seconds (%f per sec)" % (count, self.count, self.config.report_interval, tps))
                self.last_report_count = self.count
                self.last_report_time  = now
            self.db['tweets'].insert(tweet)
            if 'user' in tweet:
                if 'expire_at' in tweet:
                    tweet['user']['expire_at'] = tweet['expire_at']
                self.db['users'].update_one({ 'id': tweet['user']['id'] }, { '$set' : tweet['user'] }, upsert=True)
            self.count += 1
        except pymongo.errors.DuplicateKeyError, e:
            pass
        except Exception, e:
            self.logger.error("Error writing tweet to db: %s" % e)

    def on_error(self, e):
        msg = e
        if e == 401:
            msg = "401 Unauthorized: Authentication credentials were missing or incorrect"
        elif e == 420:
            msg = "420 Rate limiting"
            return sleep(3)
        elif e == 429:
            msg = "429 Too Many Requests"
            return sleep(5)
        self.logger.error("Twitter Streaming API error: '%s'" % msg)

class SmallhandsConfig(BaseConfiguration):
    def makeParser(self):
        parser = super(SmallhandsConfig, self).makeParser()
        parser.add_argument("-v", "--verbose", dest="verbose", help="Verbose/debug output (default: false)", default=False, action="store_true")
        parser.add_argument("-l", "--log-file", dest="log_file", help="Output to log file", type=str, default=None)
        parser.add_argument("-R", "--report-interval", dest="report_interval", help="Interval in seconds for report updates", type=int, default=30)
        parser.add_argument("-H", "--db-host", dest="db.host", help="MongoDB hostname/ip")
        parser.add_argument("-P", "--db-port", dest="db.port", type=int, help="MongoDB TCP port")
        parser.add_argument("-D", "--db-name", dest="db.name", help="MongoDB Database name (default: smallhands)")
        parser.add_argument("-u", "--db-user", dest="db.user", help="(Optional) MongoDB Username")
        parser.add_argument("-p", "--db-password", dest="db.password", help="(Optional) MongoDB Password")
        parser.add_argument("-d", "--db-authdb", dest="db.authdb", help="MongoDB Authentication Database (default: admin)")
        parser.add_argument("-x", "--db-expire-min", dest="db.expire.min", type=int, help="(Optional) minimum MongoDB TTL expiry in seconds")
        parser.add_argument("-X", "--db-expire-max", dest="db.expire.max", type=int, help="(Optional) maximum MongoDB TTL expiry in seconds")
        parser.add_argument("-k", "--twitter-consumer-key", dest="twitter.consumer.key", help="Twitter Consumer Key (REQUIRED)")
        parser.add_argument("-s", "--twitter-consumer-secret", dest="twitter.consumer.secret", help="Twitter Consumer Secret (REQUIRED)")
        parser.add_argument("-T", "--twitter-access-token", dest="twitter.access.token", help="Twitter Access Token (REQUIRED)")
        parser.add_argument("-S", "--twitter-access-secret", dest="twitter.access.secret", help="Twitter Access Secret (REQUIRED)")
        parser.add_argument("-F", "--twitter-filters", dest="twitter.filters", help="Comma-separated list of filters to Twitter stream (default: @realDonaldTrump)")
        parser.add_argument("--twitter-stream-timeout", dest="twitter.stream.timeout", help="Timeout for Twitter Streaming API, in seconds (default: 30)", default=30, type=int)
        return parser

class Smallhands():
    def __init__(self, log_level=logging.INFO):
        self.log_level = log_level
        self.config    = self.parse_config()
        self.logger    = self.setup_logger()
        self.db_conn   = None
        self.stopped   = False
        self.stream    = None

        # Parse twitter-stream filters
        self.stream_filters = ["@realDonaldTrump"]
        if 'filters' in self.config.twitter:
            self.stream_filters = self.config.twitter.filters.split(",")
        if len(self.stream_filters) < 1:
            self.logger.fatal("No Twitter stream filters!")
            raise e

        self.logger.info("Starting Smallhands version: %s (https://github.com/timvaillancourt/smallhands)" % __VERSION__)
        self.logger.info("\t\t\"I'm going to make database testing great again. Believe me.\"")

    def setup_logger(self):
        try:
            if self.config.verbose:
                if not isinstance(self.config.verbose, str) or self.config.verbose.startswith("true"):
                    self.log_level = logging.DEBUG
            self.logger = logging.getLogger(__name__)
            stream_log  = logging.StreamHandler()
            formatter   = logging.Formatter('[%(asctime)s] [%(levelname)s] [%(threadName)s] [%(module)s:%(lineno)d] %(message)s')
            stream_log.setFormatter(formatter)
            self.logger.addHandler(stream_log)
            if self.config.log_file:
                file_log = logging.FileHandler(self.config.log_file)
                file_log.setFormatter(formatter)
                self.logger.addHandler(file_log)
            self.logger.setLevel(self.log_level)
            return self.logger
        except Exception, e:
            print('Error setting up logger: %s' % e)
            raise e

    def auth_conn(self, conn):
        try:
            if 'user' in self.config.db and 'password' in self.config.db:
                self.logger.debug("Authenticating db connection as user: %s" % self.config.db.user)
                conn[self.config.db.authdb].authenticate(
                    self.config.db.user,
                    self.config.db.password,
                    source=self.config.db.authdb
                )
        except Exception, e:
            self.logger.fatal("Error authenticating to: %s:%i - %s" % (self.config.db.host, self.config.db.port, e))

    def ensure_indices(self, conn, collection, sharded=False):
        try:
            db = conn[self.config.db.name]
            self.logger.debug("Ensuring 'id' index on '%s.%s'" % (self.config.db.name, collection))
            db[collection].create_index([("id", pymongo.ASCENDING)], unique=True)
            if sharded:
                self.logger.debug("Ensuring optional hashed 'id' index on '%s.%s' (for sharding)" % (self.config.db.name, collection))
                db[collection].create_index([('id', pymongo.HASHED)])
            if 'expire' in self.config.db and 'min_secs' in self.config.db.expire and 'max_secs' in self.config.db.expire:
                self.logger.debug("Ensuring optional TTL-expiry 'expire_at' index on '%s.%s'" % (self.config.db.name, collection))
                db[collection].create_index([("expire_at", pymongo.ASCENDING)], expireAfterSeconds=0)
        except Exception, e:
            self.logger.fatal("Error creating collections on: '%s.%s' - %s" % (self.config.db.name, collection, e))
            raise e

    def get_db(self, force=False):
        try:
            if force or not self.db_conn:
                self.logger.debug("Getting mongo connection to '%s:%i'" % (self.config.db.host, self.config.db.port))
                self.db_conn = pymongo.MongoClient(
                    self.config.db.host,
                    self.config.db.port
                )
                self.auth_conn(self.db_conn)

                # Setup indices, setup TTL index conditionally:
                if self.db_conn.is_mongos:
                    self.logger.info("Detected 'mongos' process. Using sharded mode")

                    try:
                        self.logger.info("Ensuring sharding is enabled for db: %s" % self.config.db.name)
                        db = self.db_conn['admin']
                        db.command({ 'enableSharding': self.config.db.name })
                    except pymongo.errors.OperationFailure, e:
                        if e.code != 23:
                            raise e
                    except Exception, e:
                        raise e

                    shard_conn = None
                    try:
                        db = self.db_conn['config']
                        for shard in db.shards.find():
                            shard_conn = pymongo.MongoClient(shard['host'], replicaSet=shard['_id'], readPreference='primary')
                            self.auth_conn(shard_conn)
                            self.logger.info("Ensuring indices on shard: %s" % shard['host'])
                            for collection in ['tweets', 'users']:
                                self.logger.debug("Ensuring shard indices for '%s.%s'" % (self.config.db.name, collection))
                                self.ensure_indices(shard_conn, collection, True)
                    except Exception, e:
                        raise e
                    finally:
                        if shard_conn:
                            shard_conn.close()

                    for collection in ['tweets', 'users']:
                        try:
                            self.logger.info("Ensuring collection is sharded: '%s.%s'" % (self.config.db.name, collection))
                            db = self.db_conn['admin']
                            db.command({ 'shardCollection': '%s.%s' % (self.config.db.name, collection), 'key': { 'id': pymongo.HASHED } })
                        except pymongo.errors.OperationFailure, e:
                            if e.code != 20:
                                raise e
                        except Exception, e:
                            raise e
                else:
                    self.logger.info("Ensuring indices on: '%s:%i'" % (self.config.db.host, self.config.db.port))
                    for collection in ['tweets', 'users']:
                        self.logger.debug("Ensuring indices for '%s.%s'" % (self.config.db.name, collection))
                        self.ensure_indices(self.db_conn, collection)
            return self.db_conn[self.config.db.name]
        except Exception, e:
            self.logger.fatal("Error setting up db: %s" % e)
            raise Exception, 'MongoDB connection error - %s' % e, None

    def get_twitter_auth(self):
        try:
            self.logger.debug("Authenticating Twitter API session")
            auth = OAuthHandler(
                self.config.twitter.consumer.key,
                self.config.twitter.consumer.secret
            )
            auth.set_access_token(
                self.config.twitter.access.key,
                self.config.twitter.access.secret
            )
            return auth
        except Exception, e:
            self.logger.fatal("Error with Twitter auth: %s" % e)
            raise e

    def parse_config(self):
        try:
            config = SmallhandsConfig()
            config.parse(sys.argv[1:])

            # Set defaults:
            if 'db' not in config:
                config.db = {}
            if 'name' not in config.db:
                config.db.name = "smallhands"
            if 'host' not in config.db:
                config.db.host = "localhost"
            if 'port' not in config.db:
                config.db.port = 27017

            # Required fields
            if 'twitter' in config:
                required = [
                    config.twitter.access.key,
                    config.twitter.access.secret,
                    config.twitter.consumer.key,
                    config.twitter.consumer.secret
                ]
                for field in required:
                    try:
                        test = field
                    except Exception, e:
                        raise Exception, 'Required config field "%s" not specified!' % field, None
            else:
                raise Exception, 'Required "twitter" auth key/secret info not set! Please set via config file or command line!', None
            return config
        except Exception, e:
            print("Error parsing config: %s" % e)
            raise e

    def start_stream(self):
        db   = self.get_db()
        auth = self.get_twitter_auth()

        # Start the stream
        try:
            self.logger.info("Listening to Twitter Streaming API for mentions of: %s, writing data to: '%s:%i'" % (self.stream_filters, self.config.db.host, self.config.db.port))
            self.stream = Stream(auth, SmallhandsListener(db, self.config), timeout=self.config.twitter.stream.timeout)
            self.stream.filter(track=self.stream_filters, async=True)
        except Exception, e:
            raise e

    def start(self):
        try:
            self.start_stream()
            while not self.stopped and self.stream:
                if not self.stopped and not self.stream.running:
                    self.logger.error("Restarting streaming due to error!")
                    self.stop_stream()
                    self.start_stream()
                else:
                    sleep(1)
        except Exception, e:
            self.logger.fatal("Error with Twitter Streaming: %s" % e)
            raise e

    def stop_stream(self):
        try:
            self.logger.info("Stopping tweet streaming")
            if self.stream:
                self.stream.disconnect()
        except Exception, e:
            self.logger.fatal("Error stopping streaming: %s" % e)
            raise e

    def stop(self, frame=None, code=None):
        if not self.stopped:
            try:
                self.stopped = True
                self.stop_stream()
                if self.db_conn:
                    self.db_conn.close()
                self.logger.info("Smallhands stopped. Sad!")
                if frame and code:
                    sys.exit(1)
            except Exception, e:
                raise e

if __name__ == "__main__":
    smallhands = None
    try:
        smallhands = Smallhands()
        signal.signal(signal.SIGINT, smallhands.stop)
        signal.signal(signal.SIGTERM, smallhands.stop)
        smallhands.start()
    except Exception, e:
        sys.exit(1)
    finally:
        if smallhands:
            smallhands.stop()
<|endoftext|>


<commit_before># Copyright 2022 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

from django.conf import settings

def support_email(request):
    return {'support_email': getattr(settings, 'SUPPORT_EMAIL', '')}

def google_analytics(request):
    ga_key = getattr(settings, 'GOOGLE_ANALYTICS_KEY', False)
    return {
        'GOOGLE_ANALYTICS_KEY': ga_key,
        'google_analytics': ga_key
    }

def show_alert(request):
    return {'show_alert': settings.SHOW_ALERT}<commit_msg>Reformat with black<commit_after># Copyright 2022 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

from django.conf import settings

def support_email(request):
    return {"support_email": getattr(settings, "SUPPORT_EMAIL", "")}

def google_analytics(request):
    ga_key = getattr(settings, "GOOGLE_ANALYTICS_KEY", False)
    return {"GOOGLE_ANALYTICS_KEY": ga_key, "google_analytics": ga_key}

def show_alert(request):
    return {"show_alert": settings.SHOW_ALERT}
<|endoftext|>


<commit_before>#!/usr/bin/python

###
 # Copyright (c) 2016 Nishant Das Patnaik.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #  http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
###

import os, sys, re, argparse, codecs, subprocess, pwd, glob, shutil, time, zipfile, traceback, plistlib
from termcolor import colored

print """
     ___      .______   .______   .___  ___.   ______   .__   __.
    /   \     |   _  \  |   _  \  |   \/   |  /  __  \  |  \ |  |
   /  ^  \    |  |_)  | |  |_)  | |  \  /  | |  |  |  | |   \|  |
  /  /_\  \   |   ___/  |   ___/  |  |\/|  | |  |  |  | |  . `  |
 /  _____  \  |  |      |  |      |  |  |  | |  `--'  | |  |\   |
/__/     \__\ | _|      | _|      |__|  |__|  \______/  |__| \__|
                        github.com/dpnishant

"""

parser = argparse.ArgumentParser()
parser.add_argument('-ipa', action='store', dest='ipa_path', default='',
                                        help='''(Absolute) Path to IPA''')
parser.add_argument('-mobileprovision', action='store', dest='mobileprovision_path', default='',
                                        help='''(Absolute) Path to embedded.mobileprovision (OPTIONAL)''')
parser.add_argument('-identity', action='store', dest='dev_identity', default='',
                                        help='''Developer Identity Hash (OPTIONAL)''')
parser.add_argument('-uuid', action='store', dest='uuid', default='',
                                        help='''Device UUID (OPTIONAL)''')
parser.add_argument('-v', action='version', version='AppMon IPA Installer v0.1, Copyright 2016 Nishant Das Patnaik')

if len(sys.argv) < 2:
    parser.print_help()
    sys.exit(1)

if not os.path.isdir(os.path.join(os.getcwd(), "apps")):
    os.makedirs(os.path.join(os.getcwd(), "apps"))

def deviceError():
    print "%s" % colored("Error: Is the device connected over USB?", "red", attrs=["bold"])
    sys.exit(255)

def getDeveloperId():
    for id in subprocess.check_output(["node", "node-applesign/bin/ipa-resign.js", "-L"]).split("\n"):
        if "iPhone Developer:" in id:
            return id.split(" ")[0]
        else:
            print "No \"iPhone Developer\" identity found!"
            devID = raw("Enter \"iPhone Developer\" Identity Hash: ")
            return devID

def getMobileProvisionFile():
    PATH = "/Users/%s/Library/Developer/Xcode/DerivedData" % pwd.getpwuid(os.getuid())[0]
    mobileprovision_path = [os.path.join(dp, f) for dp, dn, filenames in os.walk(PATH) for f in filenames if os.path.splitext(f)[1] == '.mobileprovision']
    if len(mobileprovision_path) > 1:
        for path in mobileprovision_path:
            print "%s: %s" % (str(mobileprovision_path.index(path)), path)
        path_index = raw_input('Choose Provision file for IPA re-signing? (e.g. 0, 1...): ')
        mobileprovision_path = mobileprovision_path[int(path_index)]
    else:
        mobileprovision_path = mobileprovision_path[0]
    if not os.path.isfile(mobileprovision_path):
        mobileprovision_path = raw_input('Provide the path to "embedded.mobileprovision" file: ')
    return mobileprovision_path

def getMachOExecutable(app_path):
    try:
        plist_path = os.path.join(app_path, "Info.plist")
        plist = plistlib.readPlist(plist_path)
        executable = plist["CFBundleExecutable"]
        return os.path.join(app_path, executable)
    except Exception as _error:
        try:
            filenames = os.listdir(app_path)
            for filename in filenames:
                path = os.path.join(app_path, filename)
                if not os.path.isdir(path):
                    output = subprocess.check_output(['file', path])
                    if "Mach-O" in output:
                        output = output.split(":")[0]
                        break
            return os.path.join(app_path, output)
        except Exception as __error:
            print traceback.print_exc()

def getDeviceUUID():
    try:
        print "[+] Trying to detect device..."
        uuid = subprocess.check_output(["sudo", "ideviceinfo", "-s"]).split("UniqueDeviceID: ")[1].split("\n")[0]
        device_conn = subprocess.check_output(["sudo", "ios-deploy", "-i", uuid, "--no-wifi", "-c"])
        if "Found %s (" % (uuid) in device_conn:
            print "[+] Found %s connected through USB." % colored(device_conn.split("Found")[1].strip().split(" connected through USB.")[0], "green", attrs=["bold"])
            time.sleep(1)
        else:
            print uuid, device_conn
            deviceError()
    except Exception as e:
            print str(e)
            deviceError()

    return uuid

results = parser.parse_args()
ipa_path = results.ipa_path

if not results.mobileprovision_path:
    mobileprovision_path = getMobileProvisionFile()

if not results.uuid:
    uuid = getDeviceUUID()
else:
    uuid = results.uuid

try:
    if len(results.dev_identity) == 40:
        dev_identity = results.dev_identity
    else:
        dev_identity = getDeveloperId()
except AttributeError:
    dev_identity = getDeveloperId()

work_dir = "/tmp/appmon_ipa/"
unzip_filename = os.path.basename(ipa_path).split(".")[0]
unzip_filepath = os.path.join(work_dir, unzip_filename)
zip_filepath = os.path.join(work_dir, "%s.zip" % unzip_filename)
optool_path = "optool/bin/optool"
injected_zip_filename = "%s-injected.zip" % unzip_filename
injected_zip_filepath = os.path.join(work_dir, injected_zip_filename)
injected_ipa_filename = "%s-injected.ipa" % unzip_filename
injected_ipa_filepath = os.path.join(work_dir, injected_ipa_filename)
iparesign_path = "node-applesign/bin/ipa-resign.js"
resign_name = "%s-injected-resigned" % unzip_filename
resigned_ipa_name = "%s.ipa" % resign_name
extracted_resigned_path = os.path.join(os.getcwd(), "apps", resign_name)
gadget_name = "FridaGadget.dylib"
gadget_path = str(os.path.join(os.getcwd(), gadget_name))

if os.path.isdir(work_dir):
    shutil.rmtree(work_dir)

os.makedirs(work_dir)
subprocess.call(["cp", ipa_path, zip_filepath])
print "[+] Unpacking IPA..."
subprocess.check_output(["unzip", zip_filepath, "-d", unzip_filepath])
payload_path = os.path.join(os.path.abspath(unzip_filepath), "Payload/")

if os.listdir(payload_path)[0].endswith(".app"):
    app_name = os.listdir(payload_path)[0]
    app_path = os.path.join(payload_path, app_name)
    _CodeSignature_path = os.path.join(app_path, "_CodeSignature/")
    executable_filepath = getMachOExecutable(app_path)
    #print executable_filepath
    injected_dylib_path = os.path.join(app_path, "FridaGadget.dylib")

subprocess.check_output(["rm", "-rf", _CodeSignature_path])
subprocess.check_output(["chmod", "755", "FridaGadget.dylib"])
subprocess.check_output(["cp", gadget_path, app_path])
print "[+] Injecting DYLIB..."
subprocess.check_output([optool_path, "install", "-c", "load", "-p", "@executable_path/FridaGadget.dylib", "-t", executable_filepath])
print "[+] Code-signing..."
subprocess.check_output(["codesign", "-fs", 'iPhone Developer', injected_dylib_path])
subprocess.check_output(["codesign", "-fs", 'iPhone Developer', app_path])
subprocess.check_output(["find", unzip_filepath, "-name", '".DS_Store"', "-type", "f", "-delete"])
shutil.make_archive(injected_zip_filepath.strip(".zip"), 'zip', unzip_filepath)
os.rename(injected_zip_filepath, injected_ipa_filepath)
shutil.copy(injected_ipa_filepath, os.getcwd())
subprocess.check_output(["node", iparesign_path, "--without-watchapp", "--identity", dev_identity, "--mobileprovision", mobileprovision_path, injected_ipa_filename])
shutil.copy(os.path.join(os.getcwd(), resigned_ipa_name), os.path.join(os.getcwd(), "%s.zip" % resign_name))
subprocess.check_output(["unzip", os.path.join(os.getcwd(), resigned_ipa_name), "-d", extracted_resigned_path])
subprocess.check_output(["rm", "-rf", work_dir])
subprocess.check_output(["rm", "-rf", os.path.join(os.getcwd(), "%s.zip" % resign_name)])
subprocess.check_output(["rm", "-rf", os.path.join(os.getcwd(), injected_ipa_filename)])
subprocess.check_output(["mv", "./%s" % resigned_ipa_name, "apps/"])
#subprocess.call(["sudo", "ideviceinstaller", "-u", uuid, "-i", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
print "[+] Installing IPA..."
subprocess.check_output(["sudo", "ios-deploy", "-v", "--no-wifi", "-i", uuid, "-b", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
#print(chr(27) + "[2J")
time.sleep(2)
print "%s" % colored("----------------HELP----------------", "green", attrs=["bold"])
print '[+] Wait for "%s", on the debugger console to initialize' % colored("60 seconds", "red", attrs=["bold"])
time.sleep(1)
print '[+] Keep the debugger running to continue using the app. To quit type "%s", in the debugger console.' % colored("quit", attrs=["bold"])
time.sleep(2)
print '[+] The app will remain suspended until you run, (in a different terminal window/tab), \n%s' % colored("frida -U Gadget", "red", attrs=["bold"])
time.sleep(2)
print "[+] Generating launch script..."
message = """#!/bin/sh
# %s app launcher script generated by AppMon
# http://dpnishant.github.com/appmon

sudo ios-deploy -d --no-wifi -i %s --noinstall -b %s/Payload/%s
""" % (unzip_filename, uuid, extracted_resigned_path, app_name)
launcher_path = 'sh ./%s/launch_%s.sh' % (resign_name, unzip_filename)
with codecs.open('%s/launch_%s.sh' % (extracted_resigned_path, unzip_filename), 'w', 'utf-8') as f:
    f.write(message)

time.sleep(2)

print 'NOTE: To launch the installed app, in future, run: \n%s\n' % colored(launcher_path, "blue", attrs=["bold"])
print "%s" % colored("------------------------------------", "green", attrs=["bold"])
time.sleep(2)
print colored("[+] Starting app...")
time.sleep(2)
subprocess.call(["sudo", "ios-deploy", "-v", "--no-wifi", "-i", uuid, "--noinstall", "-b", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
sys.exit(0)
<commit_msg>Added conditional statement in mobileprovision<commit_after>#!/usr/bin/python

###
 # Copyright (c) 2016 Nishant Das Patnaik.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #  http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
###

import os, sys, re, argparse, codecs, subprocess, pwd, glob, shutil, time, zipfile, traceback, plistlib
from termcolor import colored

print """
     ___      .______   .______   .___  ___.   ______   .__   __.
    /   \     |   _  \  |   _  \  |   \/   |  /  __  \  |  \ |  |
   /  ^  \    |  |_)  | |  |_)  | |  \  /  | |  |  |  | |   \|  |
  /  /_\  \   |   ___/  |   ___/  |  |\/|  | |  |  |  | |  . `  |
 /  _____  \  |  |      |  |      |  |  |  | |  `--'  | |  |\   |
/__/     \__\ | _|      | _|      |__|  |__|  \______/  |__| \__|
                        github.com/dpnishant

"""

parser = argparse.ArgumentParser()
parser.add_argument('-ipa', action='store', dest='ipa_path', default='',
                                        help='''(Absolute) Path to IPA''')
parser.add_argument('-mobileprovision', action='store', dest='mobileprovision_path', default='',
                                        help='''(Absolute) Path to embedded.mobileprovision (OPTIONAL)''')
parser.add_argument('-identity', action='store', dest='dev_identity', default='',
                                        help='''Developer Identity Hash (OPTIONAL)''')
parser.add_argument('-uuid', action='store', dest='uuid', default='',
                                        help='''Device UUID (OPTIONAL)''')
parser.add_argument('-v', action='version', version='AppMon IPA Installer v0.1, Copyright 2016 Nishant Das Patnaik')

if len(sys.argv) < 2:
    parser.print_help()
    sys.exit(1)

if not os.path.isdir(os.path.join(os.getcwd(), "apps")):
    os.makedirs(os.path.join(os.getcwd(), "apps"))

def deviceError():
    print "%s" % colored("Error: Is the device connected over USB?", "red", attrs=["bold"])
    sys.exit(255)

def getDeveloperId():
    for id in subprocess.check_output(["node", "node-applesign/bin/ipa-resign.js", "-L"]).split("\n"):
        if "iPhone Developer:" in id:
            return id.split(" ")[0]
        else:
            print "No \"iPhone Developer\" identity found!"
            devID = raw("Enter \"iPhone Developer\" Identity Hash: ")
            return devID

def getMobileProvisionFile():
    PATH = "/Users/%s/Library/Developer/Xcode/DerivedData" % pwd.getpwuid(os.getuid())[0]
    mobileprovision_path = [os.path.join(dp, f) for dp, dn, filenames in os.walk(PATH) for f in filenames if os.path.splitext(f)[1] == '.mobileprovision']
    if len(mobileprovision_path) > 1:
        for path in mobileprovision_path:
            print "%s: %s" % (str(mobileprovision_path.index(path)), path)
        path_index = raw_input('Choose Provision file for IPA re-signing? (e.g. 0, 1...): ')
        mobileprovision_path = mobileprovision_path[int(path_index)]
    else:
        mobileprovision_path = mobileprovision_path[0]
    if not os.path.isfile(mobileprovision_path):
        mobileprovision_path = raw_input('Provide the path to "embedded.mobileprovision" file: ')
    return mobileprovision_path

def getMachOExecutable(app_path):
    try:
        plist_path = os.path.join(app_path, "Info.plist")
        plist = plistlib.readPlist(plist_path)
        executable = plist["CFBundleExecutable"]
        return os.path.join(app_path, executable)
    except Exception as _error:
        try:
            filenames = os.listdir(app_path)
            for filename in filenames:
                path = os.path.join(app_path, filename)
                if not os.path.isdir(path):
                    output = subprocess.check_output(['file', path])
                    if "Mach-O" in output:
                        output = output.split(":")[0]
                        break
            return os.path.join(app_path, output)
        except Exception as __error:
            print traceback.print_exc()

def getDeviceUUID():
    try:
        print "[+] Trying to detect device..."
        uuid = subprocess.check_output(["sudo", "ideviceinfo", "-s"]).split("UniqueDeviceID: ")[1].split("\n")[0]
        device_conn = subprocess.check_output(["sudo", "ios-deploy", "-i", uuid, "--no-wifi", "-c"])
        if "Found %s (" % (uuid) in device_conn:
            print "[+] Found %s connected through USB." % colored(device_conn.split("Found")[1].strip().split(" connected through USB.")[0], "green", attrs=["bold"])
            time.sleep(1)
        else:
            print uuid, device_conn
            deviceError()
    except Exception as e:
            print str(e)
            deviceError()

    return uuid

results = parser.parse_args()
ipa_path = results.ipa_path

if not results.mobileprovision_path:
    mobileprovision_path = getMobileProvisionFile()
else:
    mobileprovision_path = results.mobileprovision_path

if not results.uuid:
    uuid = getDeviceUUID()
else:
    uuid = results.uuid

try:
    if len(results.dev_identity) == 40:
        dev_identity = results.dev_identity
    else:
        dev_identity = getDeveloperId()
except AttributeError:
    dev_identity = getDeveloperId()

work_dir = "/tmp/appmon_ipa/"
unzip_filename = os.path.basename(ipa_path).split(".")[0]
unzip_filepath = os.path.join(work_dir, unzip_filename)
zip_filepath = os.path.join(work_dir, "%s.zip" % unzip_filename)
optool_path = "optool/bin/optool"
injected_zip_filename = "%s-injected.zip" % unzip_filename
injected_zip_filepath = os.path.join(work_dir, injected_zip_filename)
injected_ipa_filename = "%s-injected.ipa" % unzip_filename
injected_ipa_filepath = os.path.join(work_dir, injected_ipa_filename)
iparesign_path = "node-applesign/bin/ipa-resign.js"
resign_name = "%s-injected-resigned" % unzip_filename
resigned_ipa_name = "%s.ipa" % resign_name
extracted_resigned_path = os.path.join(os.getcwd(), "apps", resign_name)
gadget_name = "FridaGadget.dylib"
gadget_path = str(os.path.join(os.getcwd(), gadget_name))

if os.path.isdir(work_dir):
    shutil.rmtree(work_dir)

os.makedirs(work_dir)
subprocess.call(["cp", ipa_path, zip_filepath])
print "[+] Unpacking IPA..."
subprocess.check_output(["unzip", zip_filepath, "-d", unzip_filepath])
payload_path = os.path.join(os.path.abspath(unzip_filepath), "Payload/")

if os.listdir(payload_path)[0].endswith(".app"):
    app_name = os.listdir(payload_path)[0]
    app_path = os.path.join(payload_path, app_name)
    _CodeSignature_path = os.path.join(app_path, "_CodeSignature/")
    executable_filepath = getMachOExecutable(app_path)
    #print executable_filepath
    injected_dylib_path = os.path.join(app_path, "FridaGadget.dylib")

subprocess.check_output(["rm", "-rf", _CodeSignature_path])
subprocess.check_output(["chmod", "755", "FridaGadget.dylib"])
subprocess.check_output(["cp", gadget_path, app_path])
print "[+] Injecting DYLIB..."
subprocess.check_output([optool_path, "install", "-c", "load", "-p", "@executable_path/FridaGadget.dylib", "-t", executable_filepath])
print "[+] Code-signing..."
subprocess.check_output(["codesign", "-fs", 'iPhone Developer', injected_dylib_path])
subprocess.check_output(["codesign", "-fs", 'iPhone Developer', app_path])
subprocess.check_output(["find", unzip_filepath, "-name", '".DS_Store"', "-type", "f", "-delete"])
shutil.make_archive(injected_zip_filepath.strip(".zip"), 'zip', unzip_filepath)
os.rename(injected_zip_filepath, injected_ipa_filepath)
shutil.copy(injected_ipa_filepath, os.getcwd())
subprocess.check_output(["node", iparesign_path, "--without-watchapp", "--identity", dev_identity, "--mobileprovision", mobileprovision_path, injected_ipa_filename])
shutil.copy(os.path.join(os.getcwd(), resigned_ipa_name), os.path.join(os.getcwd(), "%s.zip" % resign_name))
subprocess.check_output(["unzip", os.path.join(os.getcwd(), resigned_ipa_name), "-d", extracted_resigned_path])
subprocess.check_output(["rm", "-rf", work_dir])
subprocess.check_output(["rm", "-rf", os.path.join(os.getcwd(), "%s.zip" % resign_name)])
subprocess.check_output(["rm", "-rf", os.path.join(os.getcwd(), injected_ipa_filename)])
subprocess.check_output(["mv", "./%s" % resigned_ipa_name, "apps/"])
#subprocess.call(["sudo", "ideviceinstaller", "-u", uuid, "-i", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
print "[+] Installing IPA..."
subprocess.check_output(["sudo", "ios-deploy", "-v", "--no-wifi", "-i", uuid, "-b", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
#print(chr(27) + "[2J")
time.sleep(2)
print "%s" % colored("----------------HELP----------------", "green", attrs=["bold"])
print '[+] Wait for "%s", on the debugger console to initialize' % colored("60 seconds", "red", attrs=["bold"])
time.sleep(1)
print '[+] Keep the debugger running to continue using the app. To quit type "%s", in the debugger console.' % colored("quit", attrs=["bold"])
time.sleep(2)
print '[+] The app will remain suspended until you run, (in a different terminal window/tab), \n%s' % colored("frida -U Gadget", "red", attrs=["bold"])
time.sleep(2)
print "[+] Generating launch script..."
message = """#!/bin/sh
# %s app launcher script generated by AppMon
# http://dpnishant.github.com/appmon

sudo ios-deploy -d --no-wifi -i %s --noinstall -b %s/Payload/%s
""" % (unzip_filename, uuid, extracted_resigned_path, app_name)
launcher_path = 'sh ./%s/launch_%s.sh' % (resign_name, unzip_filename)
with codecs.open('%s/launch_%s.sh' % (extracted_resigned_path, unzip_filename), 'w', 'utf-8') as f:
    f.write(message)

time.sleep(2)

print 'NOTE: To launch the installed app, in future, run: \n%s\n' % colored(launcher_path, "blue", attrs=["bold"])
print "%s" % colored("------------------------------------", "green", attrs=["bold"])
time.sleep(2)
print colored("[+] Starting app...")
time.sleep(2)
subprocess.call(["sudo", "ios-deploy", "-v", "--no-wifi", "-i", uuid, "--noinstall", "-b", "%s/Payload/%s" % (extracted_resigned_path, app_name)])
sys.exit(0)
<|endoftext|>


<commit_before>"""
Useful utilities.
"""
import logging
import re

import unidecode
from kyoukai.asphalt import HTTPRequestContext

logger = logging.getLogger("OWAPI")

HOUR_REGEX = re.compile(r"([0-9]*) hours?")
MINUTE_REGEX = re.compile(r"([0-9]*) minutes?")
SECOND_REGEX = re.compile(r"([0-9]*\.?[0-9]*) seconds?")
PERCENT_REGEX = re.compile(r"([0-9]{1,3})\s?\%")

async def with_cache(ctx: HTTPRequestContext, func, *args, expires: int = None, cache_404=False):
    """
    Run a coroutine with cache.

    Stores the result in redis.

    Unless we don't have redis.
    """
    if expires is None:
        expires = 300

    if ctx.app.config["owapi_cache_time"] is not None:
        expires = ctx.app.config["owapi_cache_time"]

    if not ctx.app.config["owapi_use_redis"]:
        # no caching without redis, just call the function
        logger.info("Loading `{}` with disabled cache".format(repr(args)))
        result = await func(ctx, *args)
        return result
    else:
        import aioredis
        assert isinstance(ctx.redis, aioredis.Redis)
        built = func.__name__ + repr(args)
        # Check for the key.
        # Uses a simple func name + repr(args) as the key to use.
        got = await ctx.redis.get(built)
        if got and got != "None":
            if await ctx.redis.ttl(built) == -1:
                logger.info("Caching `{}` for `{}` seconds".format(built, expires))
                await ctx.redis.expire(built, expires)

            logger.info("Cache hit for `{}`".format(built))
            return got.decode()

        logger.info("Cache miss for `{}`".format(built))

        # Call the function.
        result = await func(ctx, *args)
        if result is None and not cache_404:
            # return None, no caching for 404s.
            return None

        # Store the result as cached.
        to_set = result if result else "None"
        logger.info("Storing {} with expiration {}".format(built, expires))
        await ctx.redis.set(built, to_set, expire=expires)
        if to_set == "None":
            return None
        return result

def int_or_string(val: str):
    """
    Loads a value from MO into either an int or string value.

    String is returned if we can't turn it into an int.
    """
    new_s = val.replace(",", "")
    try:
        return float(new_s)
    except ValueError:
        return val

def parse_time(val: str) -> float:
    """
    Parse the time out into minutes.
    """
    unit = val.split(" ")[1]
    if 'minute' in unit:
        # Calculate the hour.
        mins = int(val.split(" ")[0])
        hours = round(mins / 60, 3)
        return hours
    else:
        hours = val.split(" ")[0]
        return float(hours)

def try_extract(value):
    """
    Attempt to extract a meaningful value from the time.
    """
    if value == "--":
        return 0

    get_float = int_or_string(value)
    # If it's changed, return the new int value.
    if get_float != value:
        return get_float

    # Next, try and get a time out of it.
    matched = HOUR_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        return val

    matched = MINUTE_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val /= 60
        return val

    matched = SECOND_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val = (val / 60 / 60)

        return val

    matched = PERCENT_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val = (val / 100)

        return val

    # Check if there's an ':' in it.
    if ':' in value:
        sp = value.split(':')
        # If it's only two, it's mm:ss.
        # Formula is (minutes + (seconds / 60)) / 60
        if len(sp) == 2:
            mins, seconds = map(int, sp)
            mins += seconds / 60
            hours = mins / 60
            return hours

        # If it's three, it's hh:mm:ss.
        # Formula is hours + ((minutes + (seconds / 60)) / 60).
        elif len(sp) == 3:
            hours, mins, seconds = map(int, sp)
            mins += (seconds / 60)
            hours += (mins / 60)
            return hours
    else:
        # Just return the value.
        return value

def sanitize_string(string):
    """
    Convert an arbitrary string into the format used for our json keys
    """
    space_converted = re.sub(r'[-\s]', '_', unidecode.unidecode(string).lower())
    removed_nonalphanumeric = re.sub(r'\W', '', space_converted)
    underscore_normalized = re.sub(r'_{2,}', '_', removed_nonalphanumeric)
    return underscore_normalized.replace("soldier_76", "soldier76") # backwards compatability

def correct_plural_name(name: str, value):
    """
    Convert the argument for Plural Bug
    """
    one = name[name.find('_one_') + 5:name.find('_other_')]
    other = name[name.find('_other_') + 7:name.rfind(one) + len(one) + 1]

    if value == 1:
        return name[:name.find('count_')] + one + name[name.find(other) + len(other):]

    return name[:name.find('count_')] + other + name[name.find(other) + len(other):]
<commit_msg>undo formatting changes<commit_after>"""
Useful utilities.
"""
import logging
import re

import unidecode
from kyoukai.asphalt import HTTPRequestContext

logger = logging.getLogger("OWAPI")

HOUR_REGEX = re.compile(r"([0-9]*) hours?")
MINUTE_REGEX = re.compile(r"([0-9]*) minutes?")
SECOND_REGEX = re.compile(r"([0-9]*\.?[0-9]*) seconds?")
PERCENT_REGEX = re.compile(r"([0-9]{1,3})\s?\%")

async def with_cache(ctx: HTTPRequestContext, func, *args, expires: int = None, cache_404=False):
    """
    Run a coroutine with cache.

    Stores the result in redis.

    Unless we don't have redis.
    """
    if expires is None:
        expires = 300

    if ctx.app.config["owapi_cache_time"] is not None:
        expires = ctx.app.config["owapi_cache_time"]

    if not ctx.app.config["owapi_use_redis"]:
        # no caching without redis, just call the function
        logger.info("Loading `{}` with disabled cache".format(repr(args)))
        result = await func(ctx, *args)
        return result
    else:
        import aioredis
        assert isinstance(ctx.redis, aioredis.Redis)
        built = func.__name__ + repr(args)
        # Check for the key.
        # Uses a simple func name + repr(args) as the key to use.
        got = await ctx.redis.get(built)
        if got and got != "None":
            if await ctx.redis.ttl(built) == -1:
                logger.info("Caching `{}` for `{}` seconds".format(built, expires))
                await ctx.redis.expire(built, expires)

            logger.info("Cache hit for `{}`".format(built))
            return got.decode()

        logger.info("Cache miss for `{}`".format(built))

        # Call the function.
        result = await func(ctx, *args)
        if result is None and not cache_404:
            # return None, no caching for 404s.
            return None

        # Store the result as cached.
        to_set = result if result else "None"
        logger.info("Storing {} with expiration {}".format(built, expires))
        await ctx.redis.set(built, to_set, expire=expires)
        if to_set == "None":
            return None
        return result

def int_or_string(val: str):
    """
    Loads a value from MO into either an int or string value.

    String is returned if we can't turn it into an int.
    """
    new_s = val.replace(",", "")
    try:
        return float(new_s)
    except ValueError:
        return val

def parse_time(val: str) -> float:
    """
    Parse the time out into minutes.
    """
    unit = val.split(" ")[1]
    if 'minute' in unit:
        # Calculate the hour.
        mins = int(val.split(" ")[0])
        hours = round(mins / 60, 3)
        return hours
    else:
        hours = val.split(" ")[0]
        return float(hours)

def try_extract(value):
    """
    Attempt to extract a meaningful value from the time.
    """
    if value == "--":
        return 0

    get_float = int_or_string(value)
    # If it's changed, return the new int value.
    if get_float != value:
        return get_float

    # Next, try and get a time out of it.
    matched = HOUR_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        return val

    matched = MINUTE_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val /= 60
        return val

    matched = SECOND_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val = (val / 60 / 60)

        return val

    matched = PERCENT_REGEX.match(value)
    if matched:
        val = matched.groups()[0]
        val = float(val)
        val = (val / 100)

        return val

    # Check if there's an ':' in it.
    if ':' in value:
        sp = value.split(':')
        # If it's only two, it's mm:ss.
        # Formula is (minutes + (seconds / 60)) / 60
        if len(sp) == 2:
            mins, seconds = map(int, sp)
            mins += seconds / 60
            hours = mins / 60
            return hours

        # If it's three, it's hh:mm:ss.
        # Formula is hours + ((minutes + (seconds / 60)) / 60).
        elif len(sp) == 3:
            hours, mins, seconds = map(int, sp)
            mins += (seconds / 60)
            hours += (mins / 60)
            return hours
    else:
        # Just return the value.
        return value

def sanitize_string(string):
    """
    Convert an arbitrary string into the format used for our json keys
    """
    space_converted = re.sub(r'[-\s]', '_', unidecode.unidecode(string).lower())
    removed_nonalphanumeric = re.sub(r'\W', '', space_converted)
    underscore_normalized = re.sub(r'_{2,}', '_', removed_nonalphanumeric)
    return underscore_normalized.replace("soldier_76", "soldier76") # backwards compatability

def correct_plural_name(name: str, value):
    """
    Convert the argument for Plural Bug
    """
    one = name[name.find('_one_') + 5:name.find('_other_')]
    other = name[name.find('_other_') + 7:name.rfind(one) + len(one) + 1]

    if value == 1:
        return name[:name.find('count_')] + one + name[name.find(other) + len(other):]

    return name[:name.find('count_')] + other + name[name.find(other) + len(other):]
<|endoftext|>