<commit_before>from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.queues import Queue
from tornado.concurrent import Future
import urllib2, urllib
import json
from scrapyd.eggstorage import FilesystemEggStorage
import scrapyd.config
from StringIO import StringIO
import subprocess
import sys
import os
import urlparse
import logging
import tempfile
import shutil
import pkg_resources
import time
from config import AgentConfig
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from stream import MultipartRequestBodyProducer

egg_storage = FilesystemEggStorage(scrapyd.config.Config())

TASK_STATUS_SUCCESS = 'success'
TASK_STATUS_FAIL = 'fail'
from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
register_openers()

class SpiderTask():
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    executor = None
    ret_code = None
    items_file = None

class TaskSlotContainer():
    def __init__(self, max_size=1):
        self.slots = [None] * max_size

    def is_full(self):
        for slot in self.slots:
            if slot is None:
                return False
        return True

    def put_task(self, task):
        if not isinstance(task, SpiderTask):
            raise ValueError('Task in TaskSlotContainer must be SpiderTask type.')
        for i in range(len(self.slots)):
            if self.slots[i] is None:
                self.slots[i] = task
                break

    def remove_task(self, task):
        for i in range(len(self.slots)):
            if self.slots[i] == task:
                self.slots[i] = None
                break;

    def tasks(self):
        for item in self.slots:
            if item is not None:
                yield item

class Executor():
    heartbeat_interval = 10
    checktask_interval = 10
    logger = logging.getLogger('scrapymill.executor')
    task_queue = Queue()

    def __init__(self, config=None):
        self.ioloop = IOLoop.current()

        if config is None:
            config =AgentConfig()
        self.task_slots = TaskSlotContainer(config.getint('slots', 1))
        self.config = config
        self.service_base = 'http://%s:%d' % (config.get('server'), config.getint('server_port'))

    def start(self):
        self.register_node()
        #init heartbeat period callback
        heartbeat_callback = PeriodicCallback(self.send_heartbeat, self.heartbeat_interval*1000)
        heartbeat_callback.start()

        #init checktask period callback
        # the new version use HEARTBEAT response to tell client whether there is new task on server queue
        # so do not start this period method. But if server is an old version without that header info
        # client sill need to poll GET_TASK.
        self.checktask_callback = PeriodicCallback(self.check_task, self.checktask_interval*1000)

        self.ioloop.start()

    def check_header_new_task_on_server(self, headers):
        try:
            new_task_on_server = headers['X-DD-New-Task'] == 'True'
            if new_task_on_server:
                self.ioloop.call_later(0, self.check_task)
        except KeyError:
            # if response contains no 'DD-New-Task' header, it might be the old version server,
            # so use the old GET_TASK pool mode
            if not self.checktask_callback.is_running():
                self.checktask_callback.start()

    def send_heartbeat(self):
        url = urlparse.urljoin(self.service_base, '/nodes/%d/heartbeat' % self.node_id)
        running_tasks = ','.join([task.id for task in self.task_slots.tasks()])
        request = urllib2.Request(url, data='', headers={'X-DD-RunningJobs': running_tasks})
        try:
            res = urllib2.urlopen(request)
            self.check_header_new_task_on_server(res.info())
            response_data = res.read()
        except urllib2.HTTPError as e:
            if e.code == 400:
                logging.warning('Node expired, register now.')
                self.register_node()
        except urllib2.URLError:
            logging.warning('Cannot connect to server.')

    def register_node(self):
        while True:
            try:
                url = urlparse.urljoin(self.service_base, '/nodes')
                request = urllib2.Request(url, data='')
                res = urllib2.urlopen(request)
                self.node_id = json.loads(res.read())['id']
                self.logger.info('node %d registered' % self.node_id)
                return
            except urllib2.URLError:
                logging.warning('Cannot connect to server')
                time.sleep(10)

    def get_next_task(self):
        url = urlparse.urljoin(self.service_base, '/executing/next_task')
        post_data = urllib.urlencode({'node_id': self.node_id})
        request = urllib2.Request(url=url, data=post_data)
        try:
            res = urllib2.urlopen(request)
            response_content = res.read()
            response_data = json.loads(response_content)
            logging.debug(url)
            logging.debug(response_content)
            if response_data['data'] is not None:
                task = SpiderTask()
                task.id = response_data['data']['task']['task_id']
                task.spider_id = response_data['data']['task']['spider_id']
                task.project_name = response_data['data']['task']['project_name']
                task.project_version = response_data['data']['task']['version']
                task.spider_name = response_data['data']['task']['spider_name']
                return task
        except urllib2.URLError:
            logging.warning('Cannot connect to server')

    def test_egg_requirements(self, project):
        logging.debug('enter test_egg')
        version, eggfile = egg_storage.get(project)
        logging.debug(version)
        if eggfile:
            prefix = '%s-%s-' % (project, version)
            fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')

            lf = os.fdopen(fd, 'wb')
            shutil.copyfileobj(eggfile, lf)
            lf.close()
        logging.debug(eggpath)
        try:
            d = pkg_resources.find_distributions(eggpath).next()
        except StopIteration:
            raise ValueError("Unknown or corrupt egg")
        else:
            eggpath = None
        logging.debug(d.requires())
        for require in d.requires():
            subprocess.check_call(['pip', 'install', str(require)])

        try:
            assert 'scrapy.conf' not in sys.modules, "Scrapy settings already loaded"
        finally:
            if eggpath:
                os.remove(eggpath)

    def execute_task(self, task):
        logging.debug(task.project_version)
        logging.debug(egg_storage.list(task.project_name))
        if not task.project_version in egg_storage.list(task.project_name):
            egg_request_url = urlparse.urljoin(self.service_base, '/spiders/%d/egg' % task.spider_id)
            logging.debug(egg_request_url)
            egg_request=urllib2.Request(egg_request_url)
            try:
                res = urllib2.urlopen(egg_request)
            except urllib2.URLError:
                logging.warning("Cannot retrieve job's egg, removing job")
                self.complete_task(task, TASK_STATUS_FAIL, "Cannot retrieve job's egg, removing job")
                return
            egg = StringIO(res.read())

            egg_storage.put(egg, task.project_name, task.project_version)

        try:
            self.test_egg_requirements(task.project_name)
        except ValueError as e:
            self.complete_task(task, TASK_STATUS_FAIL, e.message)
            return

        executor = TaskExecutor(task)
        task.executor = executor
        future, pid = executor.begin_execute()
        self.ioloop.add_future(future, self.task_finished)
        self.post_start_task(task, pid)

    def post_start_task(self, task, pid):
        url = urlparse.urljoin(self.service_base, '/jobs/%s/start' % task.id)
        post_data = urllib.urlencode({'pid':pid})
        request = urllib2.Request(url, post_data)
        urllib2.urlopen(request)

    def complete_task(self, task, status, log):
        url = urlparse.urljoin(self.service_base, '/executing/complete')
        if log is None:
            try:
                with open(task.executor.task_output_file, 'r') as f:
                    log = f.read()
            except Exception as e:
                log = ""
                logging.error(e.message)

        post_data = {
            'task_id': task.id,
            'log': log,
            'status':status,
        }
        if task.items_file and os.path.exists(task.items_file):
            post_data['items'] = open(task.items_file, "rb")
        datagen, headers = multipart_encode(post_data)
        request = HTTPRequest(url, method='POST', headers=headers, body_producer=MultipartRequestBodyProducer(datagen))
        client = AsyncHTTPClient()
        future = client.fetch(request)
        logging.info('task %s finished' % task.id)
        self.task_slots.remove_task(task)

    def task_finished(self, future):
        task = future.result()
        self.complete_task(task, 'success' if task.ret_code == 0 else 'fail', None)

    def check_task(self):
        if not self.task_slots.is_full():
            task = self.get_next_task()
            if task is not None:
                self.task_slots.put_task(task)
                self.execute_task(task)

class TaskExecutor():
    def __init__(self, task):
        self.task = task
        self.task_output_fd = None
        self.task_output_file = None

    def begin_execute(self):
        from w3lib.url import path_to_file_uri
        runner = 'scrapyd.runner'
        pargs = [sys.executable, '-m', runner, 'crawl', self.task.spider_name]

        task_output_dir = os.path.join('logs', self.task.project_name, self.task.spider_name)
        if not os.path.exists(task_output_dir):
            os.makedirs(task_output_dir)
        self.task_output_file = os.path.join(task_output_dir, '%s.log'%self.task.id)
        self.task_output_fd = os.open(self.task_output_file, os.O_CREAT | os.O_WRONLY)
        #pargs = [sys.executable, '-m', runner, 'list']
        env = os.environ.copy()
        logging.debug(env)
        env['SCRAPY_PROJECT'] = str(self.task.project_name)
        env['SCRAPY_JOB'] = self.task.id
        self.task.items_file = os.path.join('items', self.task.project_name, self.task.spider_name, '%s.%s' % (self.task.id, 'jl'))
        env['SCRAPY_FEED_URI'] = path_to_file_uri(self.task.items_file)

        self.p = subprocess.Popen(pargs, env=env, stdout=self.task_output_fd, stderr=self.task_output_fd)
        logging.info('job started %d' % self.p.pid)
        self.future = Future()
        self.check_process_callback = PeriodicCallback(self.check_process, 1000)
        self.check_process_callback.start()
        return self.future, self.p.pid

    def check_process(self):
        execute_result = self.p.poll()
        logging.debug('check process')
        if execute_result is not None:
            logging.info('task complete')
            os.close(self.task_output_fd)
            self.task.ret_code = execute_result
            self.check_process_callback.stop()
            self.future.set_result(self.task)

def run(argv=None):
    config = AgentConfig()
    if config.getboolean('debug'):
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)
    if argv is None:
        argv = sys.argv
    executor = Executor(config)
    executor.start()

if __name__ == '__main__':
    run()
<commit_msg>agent remove tmp egg file after requirements test<commit_after>from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.queues import Queue
from tornado.concurrent import Future
import urllib2, urllib
import json
from scrapyd.eggstorage import FilesystemEggStorage
import scrapyd.config
from StringIO import StringIO
import subprocess
import sys
import os
import urlparse
import logging
import tempfile
import shutil
import pkg_resources
import time
from config import AgentConfig
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from stream import MultipartRequestBodyProducer

egg_storage = FilesystemEggStorage(scrapyd.config.Config())

TASK_STATUS_SUCCESS = 'success'
TASK_STATUS_FAIL = 'fail'
from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
register_openers()

class SpiderTask():
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    executor = None
    ret_code = None
    items_file = None

class TaskSlotContainer():
    def __init__(self, max_size=1):
        self.slots = [None] * max_size

    def is_full(self):
        for slot in self.slots:
            if slot is None:
                return False
        return True

    def put_task(self, task):
        if not isinstance(task, SpiderTask):
            raise ValueError('Task in TaskSlotContainer must be SpiderTask type.')
        for i in range(len(self.slots)):
            if self.slots[i] is None:
                self.slots[i] = task
                break

    def remove_task(self, task):
        for i in range(len(self.slots)):
            if self.slots[i] == task:
                self.slots[i] = None
                break;

    def tasks(self):
        for item in self.slots:
            if item is not None:
                yield item

class Executor():
    heartbeat_interval = 10
    checktask_interval = 10
    logger = logging.getLogger('scrapymill.executor')
    task_queue = Queue()

    def __init__(self, config=None):
        self.ioloop = IOLoop.current()

        if config is None:
            config =AgentConfig()
        self.task_slots = TaskSlotContainer(config.getint('slots', 1))
        self.config = config
        self.service_base = 'http://%s:%d' % (config.get('server'), config.getint('server_port'))

    def start(self):
        self.register_node()
        #init heartbeat period callback
        heartbeat_callback = PeriodicCallback(self.send_heartbeat, self.heartbeat_interval*1000)
        heartbeat_callback.start()

        #init checktask period callback
        # the new version use HEARTBEAT response to tell client whether there is new task on server queue
        # so do not start this period method. But if server is an old version without that header info
        # client sill need to poll GET_TASK.
        self.checktask_callback = PeriodicCallback(self.check_task, self.checktask_interval*1000)

        self.ioloop.start()

    def check_header_new_task_on_server(self, headers):
        try:
            new_task_on_server = headers['X-DD-New-Task'] == 'True'
            if new_task_on_server:
                self.ioloop.call_later(0, self.check_task)
        except KeyError:
            # if response contains no 'DD-New-Task' header, it might be the old version server,
            # so use the old GET_TASK pool mode
            if not self.checktask_callback.is_running():
                self.checktask_callback.start()

    def send_heartbeat(self):
        url = urlparse.urljoin(self.service_base, '/nodes/%d/heartbeat' % self.node_id)
        running_tasks = ','.join([task.id for task in self.task_slots.tasks()])
        request = urllib2.Request(url, data='', headers={'X-DD-RunningJobs': running_tasks})
        try:
            res = urllib2.urlopen(request)
            self.check_header_new_task_on_server(res.info())
            response_data = res.read()
        except urllib2.HTTPError as e:
            if e.code == 400:
                logging.warning('Node expired, register now.')
                self.register_node()
        except urllib2.URLError:
            logging.warning('Cannot connect to server.')

    def register_node(self):
        while True:
            try:
                url = urlparse.urljoin(self.service_base, '/nodes')
                request = urllib2.Request(url, data='')
                res = urllib2.urlopen(request)
                self.node_id = json.loads(res.read())['id']
                self.logger.info('node %d registered' % self.node_id)
                return
            except urllib2.URLError:
                logging.warning('Cannot connect to server')
                time.sleep(10)

    def get_next_task(self):
        url = urlparse.urljoin(self.service_base, '/executing/next_task')
        post_data = urllib.urlencode({'node_id': self.node_id})
        request = urllib2.Request(url=url, data=post_data)
        try:
            res = urllib2.urlopen(request)
            response_content = res.read()
            response_data = json.loads(response_content)
            logging.debug(url)
            logging.debug(response_content)
            if response_data['data'] is not None:
                task = SpiderTask()
                task.id = response_data['data']['task']['task_id']
                task.spider_id = response_data['data']['task']['spider_id']
                task.project_name = response_data['data']['task']['project_name']
                task.project_version = response_data['data']['task']['version']
                task.spider_name = response_data['data']['task']['spider_name']
                return task
        except urllib2.URLError:
            logging.warning('Cannot connect to server')

    def test_egg_requirements(self, project):
        logging.debug('enter test_egg')
        version, eggfile = egg_storage.get(project)
        logging.debug(version)
        if eggfile:
            prefix = '%s-%s-' % (project, version)
            fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')

            lf = os.fdopen(fd, 'wb')
            shutil.copyfileobj(eggfile, lf)
            lf.close()
        logging.debug(eggpath)
        try:
            d = pkg_resources.find_distributions(eggpath).next()
        except StopIteration:
            raise ValueError("Unknown or corrupt egg")
        logging.debug(d.requires())
        for require in d.requires():
            subprocess.check_call(['pip', 'install', str(require)])
        if eggpath:
            logging.debug('removing: ' + eggpath)
            os.remove(eggpath)

    def execute_task(self, task):
        logging.debug(task.project_version)
        logging.debug(egg_storage.list(task.project_name))
        if not task.project_version in egg_storage.list(task.project_name):
            egg_request_url = urlparse.urljoin(self.service_base, '/spiders/%d/egg' % task.spider_id)
            logging.debug(egg_request_url)
            egg_request=urllib2.Request(egg_request_url)
            try:
                res = urllib2.urlopen(egg_request)
            except urllib2.URLError:
                logging.warning("Cannot retrieve job's egg, removing job")
                self.complete_task(task, TASK_STATUS_FAIL, "Cannot retrieve job's egg, removing job")
                return
            egg = StringIO(res.read())

            egg_storage.put(egg, task.project_name, task.project_version)

        try:
            self.test_egg_requirements(task.project_name)
        except ValueError as e:
            self.complete_task(task, TASK_STATUS_FAIL, e.message)
            return

        executor = TaskExecutor(task)
        task.executor = executor
        future, pid = executor.begin_execute()
        self.ioloop.add_future(future, self.task_finished)
        self.post_start_task(task, pid)

    def post_start_task(self, task, pid):
        url = urlparse.urljoin(self.service_base, '/jobs/%s/start' % task.id)
        post_data = urllib.urlencode({'pid':pid})
        request = urllib2.Request(url, post_data)
        urllib2.urlopen(request)

    def complete_task(self, task, status, log):
        url = urlparse.urljoin(self.service_base, '/executing/complete')
        if log is None:
            try:
                with open(task.executor.task_output_file, 'r') as f:
                    log = f.read()
            except Exception as e:
                log = ""
                logging.error(e.message)

        post_data = {
            'task_id': task.id,
            'log': log,
            'status':status,
        }
        if task.items_file and os.path.exists(task.items_file):
            post_data['items'] = open(task.items_file, "rb")
        datagen, headers = multipart_encode(post_data)
        request = HTTPRequest(url, method='POST', headers=headers, body_producer=MultipartRequestBodyProducer(datagen))
        client = AsyncHTTPClient()
        future = client.fetch(request)
        logging.info('task %s finished' % task.id)
        self.task_slots.remove_task(task)

    def task_finished(self, future):
        task = future.result()
        self.complete_task(task, 'success' if task.ret_code == 0 else 'fail', None)

    def check_task(self):
        if not self.task_slots.is_full():
            task = self.get_next_task()
            if task is not None:
                self.task_slots.put_task(task)
                self.execute_task(task)

class TaskExecutor():
    def __init__(self, task):
        self.task = task
        self.task_output_fd = None
        self.task_output_file = None
        self.future = None
        self.p = None
        self.check_process_callback = None

    def begin_execute(self):
        from w3lib.url import path_to_file_uri
        runner = 'scrapyd.runner'
        pargs = [sys.executable, '-m', runner, 'crawl', self.task.spider_name]

        task_output_dir = os.path.join('logs', self.task.project_name, self.task.spider_name)
        if not os.path.exists(task_output_dir):
            os.makedirs(task_output_dir)
        self.task_output_file = os.path.join(task_output_dir, '%s.log'%self.task.id)
        self.task_output_fd = os.open(self.task_output_file, os.O_CREAT | os.O_WRONLY)
        #pargs = [sys.executable, '-m', runner, 'list']
        env = os.environ.copy()
        logging.debug(env)
        env['SCRAPY_PROJECT'] = str(self.task.project_name)
        env['SCRAPY_JOB'] = self.task.id
        self.task.items_file = os.path.join('items', self.task.project_name, self.task.spider_name, '%s.%s' % (self.task.id, 'jl'))
        env['SCRAPY_FEED_URI'] = path_to_file_uri(self.task.items_file)

        self.p = subprocess.Popen(pargs, env=env, stdout=self.task_output_fd, stderr=self.task_output_fd)
        logging.info('job started %d' % self.p.pid)
        self.future = Future()
        self.check_process_callback = PeriodicCallback(self.check_process, 1000)
        self.check_process_callback.start()
        return self.future, self.p.pid

    def check_process(self):
        execute_result = self.p.poll()
        logging.debug('check process')
        if execute_result is not None:
            logging.info('task complete')
            os.close(self.task_output_fd)
            self.task.ret_code = execute_result
            self.check_process_callback.stop()
            self.future.set_result(self.task)

def run(argv=None):
    config = AgentConfig()
    if config.getboolean('debug'):
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)
    if argv is None:
        argv = sys.argv
    executor = Executor(config)
    executor.start()

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>from django.shortcuts import render
from applications.models import Application, PublicationFeedback, Record, PublicationWebsite
from approvals.models import Approval
from django.http import HttpResponse, HttpResponseRedirect
from django.views.generic import TemplateView, ListView, DetailView, CreateView, UpdateView, DeleteView, FormView
from django.db.models import Q
from public import forms as apps_forms
from django.core.urlresolvers import reverse
from django.utils.safestring import SafeText

class PublicApplicationsList(TemplateView):
    #model = appmodel.Application

    template_name = "public/home_page.html"
    def get_context_data(self, **kwargs):

        context = super(PublicApplicationsList, self).get_context_data(**kwargs)
        # app = self.get_object()
        # print self.req
        action =  self.kwargs['action']
        context['action'] = action
        search = None
        query_obj = None

        if action == 'draft':
            query_obj = Q(publish_documents__isnull=False,publish_draft_report__isnull=True) & Q(app_type__in=[3])
        elif action == 'final':
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False,publish_final_report__isnull=True) & Q(app_type__in=[3])
        elif action == 'determination':
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False, publish_final_report__isnull=False,publish_determination_report__isnull=True) & Q(app_type__in=[3])
        else:
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False,publish_final_report__isnull=False) & Q(app_type__in=[3])
            #query_obj = Q(publish_documents__isnull=False) & Q(app_type__in=[3])

        if 'q' in self.request.GET and self.request.GET['q']:
            query_str = self.request.GET['q']
            query_obj = Q(Q(pk__contains=query_str) | Q(title__icontains=query_str) | Q(applicant__email__icontains=query_str) | Q(organisation__name__icontains=query_str) |  Q(assignee__email__icontains=query_str)) & query_obj

        if action is not None:
            context['items'] = Application.objects.filter(query_obj)

        return context

class PublicApplicationFeedback(UpdateView):
    """A view for updating a draft (non-lodged) application.
    """
    model = Application
    form_class = apps_forms.ApplicationPart5
    template_name = 'public/application_form.html'

    def get(self, request, *args, **kwargs):
        # TODO: business logic to check the application may be changed.
        app = self.get_object()
        return super(PublicApplicationFeedback, self).get(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(PublicApplicationFeedback, self).get_context_data(**kwargs)
        app_id = self.kwargs['pk']
        context['page_heading'] = 'Application for new Part 5 - '+app_id
        context['left_sidebar'] = 'yes'
        context['action'] = self.kwargs['action']
        app = self.get_object()

        doclist = app.proposed_development_plans.all()
        context['proposed_development_plans_list'] = []
        for doc in doclist:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=doc.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None

            fileitem = {}
            if pub_web is None:
               fileitem['fileid'] = doc.id
               fileitem['path'] = doc.upload.name
               fileitem['path_short'] = SafeText(doc.upload.name)[19:]
               fileitem['name'] = doc.name
               context['proposed_development_plans_list'].append(fileitem)
            else:
               fileitem['fileid'] = pub_web.published_document.id
               fileitem['path'] = pub_web.published_document.upload.name
               fileitem['path_short'] = SafeText(pub_web.published_document.upload.name)[19:]
               fileitem['name'] = pub_web.published_document.name
               context['proposed_development_plans_list'].append(fileitem)

        if app.river_lease_scan_of_application:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=app.river_lease_scan_of_application.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None
#            print pub_web
            if pub_web is None:
              context['river_lease_scan_of_application'] = app.river_lease_scan_of_application
            else:
#              print "HERE"
#               print pub_web.published_document
              context['river_lease_scan_of_application'] = pub_web.published_document
        return context

    def get_initial(self):
        initial = super(PublicApplicationFeedback, self).get_initial()
        app = self.get_object()

        initial['application_id'] = self.kwargs['pk']
        initial['organisation'] = app.organisation
        if app.river_lease_scan_of_application:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=app.river_lease_scan_of_application.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None
            print pub_web
            #if pub_web is None:
            #   initial['river_lease_scan_of_application'] = app.river_lease_scan_of_application.upload
            #else:
            #   print "HERE"
            #   print pub_web.published_document
            #   initial['river_lease_scan_of_application'] = pub_web.published_document
        return initial

    def post(self, request, *args, **kwargs):
        if request.POST.get('cancel'):
            app = Application.objects.get(id=kwargs['pk'])
            if app.state == app.APP_STATE_CHOICES.new:
                app.delete()
                return HttpResponseRedirect(reverse('application_list'))
            return HttpResponseRedirect(self.get_object().get_absolute_url())
        return super(PublicApplicationFeedback, self).post(request, *args, **kwargs)

    def form_valid(self, form):
        """Override form_valid to set the state to draft is this is a new application.
        """
        forms_data = form.cleaned_data
        self.object = form.save(commit=False)
        app_id = self.kwargs['pk']
        action = self.kwargs['action']
        status=None
        application = Application.objects.get(id=app_id)

        if action == 'draft':
             status='draft'
        elif action == 'final':
             status='final'
        elif action == 'determination':
             status=='determination'

        # ToDO remove dupes of this line below. doesn't need to be called
        # multiple times
        pfcreate = PublicationFeedback.objects.create(application=application,
						      name=forms_data['name'],
                                                      address=forms_data['address'],
                                                      suburb=forms_data['suburb'],
                                                      state=forms_data['state'],
                                                      postcode=forms_data['post_code'],
                                                      phone=forms_data['phone'],
                                                      email=forms_data['email'],
                                                      comments=forms_data['comments'],
                                                      status=status
	)

        if self.request.FILES.get('records'):
            for f in self.request.FILES.getlist('records'):
                doc = Record()
                doc.upload = f
                doc.save()
                pfcreate.records.add(doc)

#    name = CharField(required=False,max_length=255)
#    address = CharField(required=False,max_length=255)
#    suburb = CharField(required=False,max_length=255)
#    post_code = CharField(required=False,max_length=255)
#    phone = CharField(required=False,max_length=255)
#    email = EmailField(required=False,max_length=255)
#    email_confirm = EmailField(required=False,max_length=255)
#    comments = CharField(required=False,max_length=255, widget=Textarea)

#        application = Application.objects.get(id=self.object.id)
 #       self.object.save()
        return HttpResponseRedirect(reverse('public_application_list', args=(action,)))

<commit_msg>Removed Print Line<commit_after>from django.shortcuts import render
from applications.models import Application, PublicationFeedback, Record, PublicationWebsite
from approvals.models import Approval
from django.http import HttpResponse, HttpResponseRedirect
from django.views.generic import TemplateView, ListView, DetailView, CreateView, UpdateView, DeleteView, FormView
from django.db.models import Q
from public import forms as apps_forms
from django.core.urlresolvers import reverse
from django.utils.safestring import SafeText

class PublicApplicationsList(TemplateView):
    #model = appmodel.Application

    template_name = "public/home_page.html"
    def get_context_data(self, **kwargs):

        context = super(PublicApplicationsList, self).get_context_data(**kwargs)
        # app = self.get_object()
        # print self.req
        action =  self.kwargs['action']
        context['action'] = action
        search = None
        query_obj = None

        if action == 'draft':
            query_obj = Q(publish_documents__isnull=False,publish_draft_report__isnull=True) & Q(app_type__in=[3])
        elif action == 'final':
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False,publish_final_report__isnull=True) & Q(app_type__in=[3])
        elif action == 'determination':
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False, publish_final_report__isnull=False,publish_determination_report__isnull=True) & Q(app_type__in=[3])
        else:
            query_obj = Q(publish_documents__isnull=False, publish_draft_report__isnull=False,publish_final_report__isnull=False) & Q(app_type__in=[3])
            #query_obj = Q(publish_documents__isnull=False) & Q(app_type__in=[3])

        if 'q' in self.request.GET and self.request.GET['q']:
            query_str = self.request.GET['q']
            query_obj = Q(Q(pk__contains=query_str) | Q(title__icontains=query_str) | Q(applicant__email__icontains=query_str) | Q(organisation__name__icontains=query_str) |  Q(assignee__email__icontains=query_str)) & query_obj

        if action is not None:
            context['items'] = Application.objects.filter(query_obj)

        return context

class PublicApplicationFeedback(UpdateView):
    """A view for updating a draft (non-lodged) application.
    """
    model = Application
    form_class = apps_forms.ApplicationPart5
    template_name = 'public/application_form.html'

    def get(self, request, *args, **kwargs):
        # TODO: business logic to check the application may be changed.
        app = self.get_object()
        return super(PublicApplicationFeedback, self).get(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(PublicApplicationFeedback, self).get_context_data(**kwargs)
        app_id = self.kwargs['pk']
        context['page_heading'] = 'Application for new Part 5 - '+app_id
        context['left_sidebar'] = 'yes'
        context['action'] = self.kwargs['action']
        app = self.get_object()

        doclist = app.proposed_development_plans.all()
        context['proposed_development_plans_list'] = []
        for doc in doclist:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=doc.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None

            fileitem = {}
            if pub_web is None:
               fileitem['fileid'] = doc.id
               fileitem['path'] = doc.upload.name
               fileitem['path_short'] = SafeText(doc.upload.name)[19:]
               fileitem['name'] = doc.name
               context['proposed_development_plans_list'].append(fileitem)
            else:
               fileitem['fileid'] = pub_web.published_document.id
               fileitem['path'] = pub_web.published_document.upload.name
               fileitem['path_short'] = SafeText(pub_web.published_document.upload.name)[19:]
               fileitem['name'] = pub_web.published_document.name
               context['proposed_development_plans_list'].append(fileitem)

        if app.river_lease_scan_of_application:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=app.river_lease_scan_of_application.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None
#            print pub_web
            if pub_web is None:
              context['river_lease_scan_of_application'] = app.river_lease_scan_of_application
            else:
#              print "HERE"
#               print pub_web.published_document
              context['river_lease_scan_of_application'] = pub_web.published_document
        return context

    def get_initial(self):
        initial = super(PublicApplicationFeedback, self).get_initial()
        app = self.get_object()

        initial['application_id'] = self.kwargs['pk']
        initial['organisation'] = app.organisation
        if app.river_lease_scan_of_application:
            pub_web = None
            try:
               pub_web = PublicationWebsite.objects.get(original_document_id=app.river_lease_scan_of_application.id,application=self.kwargs['pk'])
#               print PublicationWebsite.objects.get(original_document_id=doc.id)
            except:
               pub_web = None
            #if pub_web is None:
            #   initial['river_lease_scan_of_application'] = app.river_lease_scan_of_application.upload
            #else:
            #   print "HERE"
            #   print pub_web.published_document
            #   initial['river_lease_scan_of_application'] = pub_web.published_document
        return initial

    def post(self, request, *args, **kwargs):
        if request.POST.get('cancel'):
            app = Application.objects.get(id=kwargs['pk'])
            if app.state == app.APP_STATE_CHOICES.new:
                app.delete()
                return HttpResponseRedirect(reverse('application_list'))
            return HttpResponseRedirect(self.get_object().get_absolute_url())
        return super(PublicApplicationFeedback, self).post(request, *args, **kwargs)

    def form_valid(self, form):
        """Override form_valid to set the state to draft is this is a new application.
        """
        forms_data = form.cleaned_data
        self.object = form.save(commit=False)
        app_id = self.kwargs['pk']
        action = self.kwargs['action']
        status=None
        application = Application.objects.get(id=app_id)

        if action == 'draft':
             status='draft'
        elif action == 'final':
             status='final'
        elif action == 'determination':
             status=='determination'

        # ToDO remove dupes of this line below. doesn't need to be called
        # multiple times
        pfcreate = PublicationFeedback.objects.create(application=application,
						      name=forms_data['name'],
                                                      address=forms_data['address'],
                                                      suburb=forms_data['suburb'],
                                                      state=forms_data['state'],
                                                      postcode=forms_data['post_code'],
                                                      phone=forms_data['phone'],
                                                      email=forms_data['email'],
                                                      comments=forms_data['comments'],
                                                      status=status
	)

        if self.request.FILES.get('records'):
            for f in self.request.FILES.getlist('records'):
                doc = Record()
                doc.upload = f
                doc.save()
                pfcreate.records.add(doc)

#    name = CharField(required=False,max_length=255)
#    address = CharField(required=False,max_length=255)
#    suburb = CharField(required=False,max_length=255)
#    post_code = CharField(required=False,max_length=255)
#    phone = CharField(required=False,max_length=255)
#    email = EmailField(required=False,max_length=255)
#    email_confirm = EmailField(required=False,max_length=255)
#    comments = CharField(required=False,max_length=255, widget=Textarea)

#        application = Application.objects.get(id=self.object.id)
 #       self.object.save()
        return HttpResponseRedirect(reverse('public_application_list', args=(action,)))

<|endoftext|>


<commit_before># Copyright 2020, 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This program configures and builds Emacs.

It’s an internal implementation detail of the Bazel rules for Emacs Lisp; don’t
use it outside the rules or depend on its behavior."""

import argparse
import os
import pathlib
import shlex
import shutil
import subprocess
import tempfile

def main() -> None:
    """Configures and builds Emacs."""
    parser = argparse.ArgumentParser(allow_abbrev=False)
    parser.add_argument('--source', type=pathlib.Path, required=True)
    parser.add_argument('--install', type=pathlib.Path, required=True)
    parser.add_argument('--cc', type=pathlib.Path, required=True)
    parser.add_argument('--cflags', required=True)
    parser.add_argument('--ldflags', required=True)
    parser.add_argument('--module-header', type=pathlib.Path)
    args = parser.parse_args()
    windows = os.name == 'nt'
    if windows:
        bash = _find_bash(args.cc)
    temp = pathlib.Path(tempfile.mkdtemp(prefix='emacs-build-'))
    build = temp / 'build'
    install = args.install.resolve()
    shutil.copytree(args.source, build)

    def run(*command: str) -> None:
        env = None
        if windows:
            # Building Emacs requires MinGW, see nt/INSTALL.W64.  Therefore we
            # invoke commands through the MinGW shell, see
            # https://www.msys2.org/wiki/Launchers/#the-idea.
            env = dict(os.environ, MSYSTEM='MINGW64', CHERE_INVOKING='1')
            command = [str(bash), '-l', '-c', shlex.join(command)]
        try:
            subprocess.run(command, check=True, cwd=build, env=env,
                           stdin=subprocess.DEVNULL,
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           encoding='utf-8', errors='backslashescape')
        except subprocess.CalledProcessError as ex:
            print('command', ' '.join(map(shlex.quote, command)),
                  'failed, output follows:')
            print(ex.stdout)
            print()
            try:
                config_log = build / 'config.log'
                content = config_log.read_text('utf-8', 'backslashescape')
                print('config.log follows:')
                print(content)
            except FileNotFoundError:
                print('config.log not found')
            print()
            print('temporary build directory is ', temp)
            raise

    run('./configure', '--prefix=' + install.as_posix(),
        '--without-all', '--without-ns', '--with-x-toolkit=no',
        '--without-libgmp',
        # Enable threads explicitly to work around
        # https://debbugs.gnu.org/cgi/bugreport.cgi?bug=30106 in older Emacs
        # versions.  Enable toolkit scrollbars to work around
        # https://debbugs.gnu.org/cgi/bugreport.cgi?bug=37042.
        '--with-modules', '--with-threads', '--with-toolkit-scroll-bars',
        '--disable-build-details',
        'CC=' + args.cc.resolve().as_posix(),
        'CFLAGS=' + args.cflags,
        'LDFLAGS=' + args.ldflags)
    run('make', 'install')
    # Build directory no longer needed, delete it.
    shutil.rmtree(temp, ignore_errors=True)
    # Delete source files that have a corresponding compiled file, as these
    # files don’t work well with Coverage (see
    # e.g. https://debbugs.gnu.org/cgi/bugreport.cgi?bug=40766).
    for compiled in install.glob('share/emacs/*/lisp/**/*.elc'):
        compiled.with_suffix('.el').unlink()
    # Sanity check to verify that the resulting binary works.
    subprocess.run([install / 'bin' / 'emacs', '--quick', '--batch'],
                   check=True, stdin=subprocess.DEVNULL)
    if args.module_header:
        # Copy emacs-module.h to the desired location.
        shutil.copy(install / 'include' / 'emacs-module.h', args.module_header)

def _find_bash(c_compiler: pathlib.Path) -> pathlib.Path:
    if c_compiler.parts[-3:-1] != ('mingw64', 'bin'):
        raise ValueError(f'unsupported C compiler location {c_compiler}')
    msys = c_compiler.parents[2]
    bash = msys / 'usr' / 'bin' / 'bash.exe'
    if not bash.is_file():
        raise FileNotFoundError(f'no Bash program found in {msys}')
    return bash

if __name__ == '__main__':
    main()
<commit_msg>Avoid shlex.join, which is too recent.<commit_after># Copyright 2020, 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This program configures and builds Emacs.

It’s an internal implementation detail of the Bazel rules for Emacs Lisp; don’t
use it outside the rules or depend on its behavior."""

import argparse
import os
import pathlib
import shlex
import shutil
import subprocess
import tempfile

def main() -> None:
    """Configures and builds Emacs."""
    parser = argparse.ArgumentParser(allow_abbrev=False)
    parser.add_argument('--source', type=pathlib.Path, required=True)
    parser.add_argument('--install', type=pathlib.Path, required=True)
    parser.add_argument('--cc', type=pathlib.Path, required=True)
    parser.add_argument('--cflags', required=True)
    parser.add_argument('--ldflags', required=True)
    parser.add_argument('--module-header', type=pathlib.Path)
    args = parser.parse_args()
    windows = os.name == 'nt'
    if windows:
        bash = _find_bash(args.cc)
    temp = pathlib.Path(tempfile.mkdtemp(prefix='emacs-build-'))
    build = temp / 'build'
    install = args.install.resolve()
    shutil.copytree(args.source, build)

    def run(*command: str) -> None:
        env = None
        if windows:
            # Building Emacs requires MinGW, see nt/INSTALL.W64.  Therefore we
            # invoke commands through the MinGW shell, see
            # https://www.msys2.org/wiki/Launchers/#the-idea.
            env = dict(os.environ, MSYSTEM='MINGW64', CHERE_INVOKING='1')
            command = [str(bash), '-l',
                       '-c', ' '.join(map(shlex.quote, command))]
        try:
            subprocess.run(command, check=True, cwd=build, env=env,
                           stdin=subprocess.DEVNULL,
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           encoding='utf-8', errors='backslashescape')
        except subprocess.CalledProcessError as ex:
            print('command', ' '.join(map(shlex.quote, command)),
                  'failed, output follows:')
            print(ex.stdout)
            print()
            try:
                config_log = build / 'config.log'
                content = config_log.read_text('utf-8', 'backslashescape')
                print('config.log follows:')
                print(content)
            except FileNotFoundError:
                print('config.log not found')
            print()
            print('temporary build directory is ', temp)
            raise

    run('./configure', '--prefix=' + install.as_posix(),
        '--without-all', '--without-ns', '--with-x-toolkit=no',
        '--without-libgmp',
        # Enable threads explicitly to work around
        # https://debbugs.gnu.org/cgi/bugreport.cgi?bug=30106 in older Emacs
        # versions.  Enable toolkit scrollbars to work around
        # https://debbugs.gnu.org/cgi/bugreport.cgi?bug=37042.
        '--with-modules', '--with-threads', '--with-toolkit-scroll-bars',
        '--disable-build-details',
        'CC=' + args.cc.resolve().as_posix(),
        'CFLAGS=' + args.cflags,
        'LDFLAGS=' + args.ldflags)
    run('make', 'install')
    # Build directory no longer needed, delete it.
    shutil.rmtree(temp, ignore_errors=True)
    # Delete source files that have a corresponding compiled file, as these
    # files don’t work well with Coverage (see
    # e.g. https://debbugs.gnu.org/cgi/bugreport.cgi?bug=40766).
    for compiled in install.glob('share/emacs/*/lisp/**/*.elc'):
        compiled.with_suffix('.el').unlink()
    # Sanity check to verify that the resulting binary works.
    subprocess.run([install / 'bin' / 'emacs', '--quick', '--batch'],
                   check=True, stdin=subprocess.DEVNULL)
    if args.module_header:
        # Copy emacs-module.h to the desired location.
        shutil.copy(install / 'include' / 'emacs-module.h', args.module_header)

def _find_bash(c_compiler: pathlib.Path) -> pathlib.Path:
    if c_compiler.parts[-3:-1] != ('mingw64', 'bin'):
        raise ValueError(f'unsupported C compiler location {c_compiler}')
    msys = c_compiler.parents[2]
    bash = msys / 'usr' / 'bin' / 'bash.exe'
    if not bash.is_file():
        raise FileNotFoundError(f'no Bash program found in {msys}')
    return bash

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Embedded tweet plugin for Pelican
=================================

This plugin allows you to embed Twitter tweets into your articles.
And also provides a link for Twitter username.

    i.e.

        @username

        will be replaced by a link to Twitter username page.

        @username/status/tweetid

        will be replaced by a `Embedded-tweet`_ API.

.. _Embedded-tweet: https://dev.twitter.com/docs/embedded-tweets

"""

from pelican import signals
import re

def embed_tweet(content):
    if not content._content:
        return

    content._content = re.sub(
        r'(^|[^@\w])@(\w{1,15})\b',
        '\\1@\\2',
        re.sub(
            r'(^|[^@\w])@(\w{1,15})/status/(\d+)\b',
            '\\1
Tweet of \\2/\\3
',
            content._content
        )
    ) + ''

def register():
    signals.content_object_init.connect(embed_tweet)
<commit_msg>Fix embedding problem<commit_after>"""
Embedded tweet plugin for Pelican
=================================

This plugin allows you to embed Twitter tweets into your articles.
And also provides a link for Twitter username.

    i.e.

        @username

        will be replaced by a link to Twitter username page.

        @username/status/tweetid

        will be replaced by a `Embedded-tweet`_ API.

.. _Embedded-tweet: https://dev.twitter.com/docs/embedded-tweets

"""

from pelican import signals
import re

def embed_tweet(instance):
    if not instance._content:
        return

    instance._content = re.sub(
        r'(^|[^@\w])@(\w{1,15})(\b[^/])',
        '\\1@\\2\\3',
        re.sub(
            r'(^|[^@\w])@(\w{1,15})/status/(\d+)\b',
            '\\1
Tweet of \\2/\\3
',
            instance._content
        )
    ) + ''

def register():
    signals.content_object_init.connect(embed_tweet)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import argparse
import Tools
import Align
from  Bio import SeqIO

ap = argparse.ArgumentParser(description="Print gaps in a SAM file.")
ap.add_argument("genome", help="Genome file with a .fai")
ap.add_argument("sam", help="Sam file of alignment.", nargs="+")
ap.add_argument("--onTarget", help="Assume the query encodes the position of the aligned sequence, and make sure at least the chromosomes match.", default=False, action='store_true')
ap.add_argument("--gapFree", help="Print sequences without gaps.", default=None)
ap.add_argument("--minContigLength", help="Only parse alignments from contigs this length or greater", default=0, type=int)
ap.add_argument("--minLength", help="Minimum gap length.", default=50, type=int)
ap.add_argument("--maxLength", help="Maximum gap length.", default=None, type=int)
ap.add_argument("--outFile", help="Print output here, default= stdout", default=None)
ap.add_argument("--context", help="Print surrounding context", default=0, type=int)
ap.add_argument("--condense", help="Pack indels if the matches separating them is less than this value.", default=0, type=int)
ap.add_argument("--tsd", help="Attempt to find Target Site Duplications at most this length", default=20, type=int)
ap.add_argument("--outsam", help="Write the modified condensed sam to a file.", default=None)
ap.add_argument("--minq", help="Minimal mapping quality to consider (10)",default=10,type=int)
ap.add_argument("--qpos", help="Write query position of gaps", default=False,action='store_true')
ap.add_argument("--snv", help="Print SNVs to this file.", default=None)
ap.add_argument("--contigBed", help="Print where contigs map.", default=None)
ap.add_argument("--status", help="Print how far along the alignments are.", default=False, action='store_true')
ap.add_argument("--blacklist", help="Exclude contigs on this list from callsets.", default=None)
ap.add_argument("--removeAdjacentIndels", help="Find instances of SNVs pushed into indels, in the format: NIXMND., and remove these operations.", default=False, action='store_true')
args = ap.parse_args()

genome = file(args.genome, 'r')
handle = open(args.genome, "r")

if (args.outFile is None):
    outFile = sys.stdout
else:
    outFile = open(args.outFile, 'w')

if (args.gapFree is not None):
    gapFree = open(args.gapFree, 'w')

if (args.contigBed is not None):
    contigBed = open(args.contigBed, 'w')

blacklist = {}
if (args.blacklist is not None):
    bl = open(args.blacklist)
    blacklist = { i.strip() : true for i in bl.readlines() }

fai = Tools.ReadFAIFile(args.genome + ".fai")

#genomeDict = SeqIO.to_dict(SeqIO.parse(handle, "fasta"))

if (args.outsam is not None):
    outsam = open(args.outsam, 'w')

snvOut = None
if (args.snv is not None):
    snvOut = open(args.snv, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")
genomeFile = open(args.genome, 'r')

M = 'M'
I = 'I'
D = 'D'
N = 'N'
S = 'S'
H = 'H'
P = 'P'
if (args.sam[0].find(".fofn") >= 0):
    fofnFile = open(args.sam[0])
    samFiles = [line.strip() for line in fofnFile.readlines()]
    args.sam = samFiles
lineNumber = 0
contextLength = 8
#import pdb
import re
coordRe = re.compile(".*(chr.*)\.(\d+)-(\d+).*")
for samFileName in args.sam:
    samFile = open(samFileName)

    for line in samFile:
        lineNumber = lineNumber + 1

        if (line[0] == "@"):
            if (args.outsam is not None):
                outsam.write(line)
            continue
        if (len(line) <= 1):
            continue

        aln = Tools.SAMEntry(line)
        if (aln.title is None):
            continue
	#
	# Use 0-based coordinate system
	#
        aln.tStart -=1
        aln.tEnd -=1
        if (args.onTarget == True):
            coordReMatch = coordRe.match(aln.title)
            if (coordReMatch is not None):
                coordMatchGroups = coordReMatch.groups()
                srcChrom = coordMatchGroups[0]
                srcStart = int(coordMatchGroups[1])
                srcEnd   = int(coordMatchGroups[2])
                if (srcChrom != aln.tName):
                    print "off target chromosome: " + srcChrom + " " + aln.tName
                    continue
                if (((srcStart >= aln.tStart and srcStart < aln.tEnd) or (srcEnd >= aln.tStart and srcEnd < aln.tEnd) or (srcStart < aln.tStart and srcEnd > aln.tEnd )) == False):
                    print "no overlap " + srcChrom + " " + str(srcStart) + " " + str(srcEnd) + " alignment: " + str(aln.tStart) + " "+ str(aln.tEnd)
                    continue

        if (aln.flag & 256 != 0):
            continue
        if (aln.mapqv < args.minq):
            continue

        if (args.contigBed is not None):
            contigBed.write("{}\t{}\t{}\t{}\n".format(aln.tName, aln.tStart, aln.tStart + aln.tlen, aln.title))

        if (args.minContigLength > len(aln.seq)):
            continue
        tPos = aln.tStart
        qPos = 0
        #
        # condense matches.
        #
        packedCigar = []
        i = 0
        i1 = 1
        niter = 0
        maxGap = 0
        maxGapType = 0
        #print str(aln.ops)

        foundGap = False

        if (args.removeAdjacentIndels):
            for i in range(1,len(aln.lengths)-1):
                if (aln.ops[i-1] != 'M' and
                    aln.ops[i+1] != 'M' and
                    aln.ops[i-1] != aln.ops[i+1] and
                    aln.ops[i] == 'M' and
                    aln.lengths[i-1] == aln.lengths[i+1] and
                    aln.lengths[i] < 4):
                    aln.lengths[i-1] = 0
                    aln.lengths[i+1] = 0

            newLengths = []
            newOps = []
            for i in range(0,len(aln.lengths)):
                if (aln.lengths[i] != 0):
                    newLengths.append(aln.lengths[i])
                    newOps.append(aln.ops[i])
            aln.lengths = newLengths
            aln.ops = newOps

        packedOps = []
        packedLengths = []
        i = 0
        if (args.condense > 0):
            while (i < len(aln.lengths)):
                l = aln.lengths[i]
                op = aln.ops[i]
                j = i
                if (op == I or op == D):
                    if (l > maxGap):
                        maxGap = l
                        maxGapType = op

                if (op == I or op == D and i < len(aln.ops) - 2 and aln.ops[i+2][0] == op):
                    matchLen = 0
                    gapLen   = 0
                    while (j+2 < len(aln.ops) and aln.ops[j+2][0] == op and aln.ops[j+1][0] == M and aln.lengths[j+1] < args.condense):

                        matchLen += aln.lengths[j+1]
                        gapLen   += aln.lengths[j+2]
                        j+=2
                    if (j > i):
                        newIndel = (op, l+gapLen)
                        newMatch = (M, matchLen)
                        packedOps.append(op)
                        packedLengths.append(l+gapLen)

                        packedOps.append(M)
                        packedLengths.append(matchLen)

                    else:
                        packedLengths.append(l)
                        packedOps.append(op)

                else:
                    packedLengths.append(l)
                    packedOps.append(op)

                i = j + 1
                niter +=1
                if (niter > len(aln.ops)):
                    print "ERROR! too many interations."
        else:
            packedOps = aln.ops
            packedLengths = aln.lengths

        for i in range(len(packedOps)):
            op = packedOps[i]
            l  = packedLengths[i]

            if (op == N or op == S):
                # Inside match block (if op == M)
                qPos += l
            if (op == M):
                # Inside match block (if op == M)
                if (args.snv is not None):
                    targetSeq = Tools.ExtractSeq((aln.tName, tPos-1,tPos+l-1), genomeFile, fai)
                    querySeq  = aln.seq[qPos:qPos+l]
                    nMis = 0

                    for mp in range(0,len(targetSeq)):
                        if (querySeq[mp].upper() != targetSeq[mp].upper() and targetSeq[mp].upper() != 'N' and querySeq[mp].upper() != 'N'):
                            nMis +=1
                            snvOut.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(aln.tName, tPos+mp, tPos+mp+1, targetSeq[mp], querySeq[mp], aln.title ))
                tPos += l
                qPos += l

            if (op == I):
                if (l >= args.minLength and (args.maxLength is None or l < args.maxLength)):
                    #                    print "gap at " + str(i) + " " + str(op) + " " + str(l) + " " + str(qPos) + " qlen: " + str(len(aln.seq))
                    foundGap = True
                    chrName = aln.tName
                    #gapSeq = aln.seq[max(0,qPos-args.context):min(qPos+l+args.context, len(aln.seq))]
                    gapSeq = aln.seq[qPos:qPos+l]

                    context= aln.seq[qPos+l:min(qPos+l+args.context, len(aln.seq))]
                    if (context == "A"*len(context) or context == "T"*len(context)):
                        homopolymer="T"
                    else:
                        homopolymer="F"
                    tsd = "notsd"
                    if (len(gapSeq) == 0):
                        sys.stderr.write("ERROR, gap seq is of zero length\n")
                    if (args.tsd):
                        # try and find the target site duplications, this may be on either side of the alignemnt
                        tsdSuffix = gapSeq[-args.tsd:]
                        tsdSuffix = tsdSuffix.upper()
                        tsdPrefix = gapSeq[0:args.tsd]
                        tsdPrefix = tsdPrefix.upper()
                        targetPrefix = Tools.ExtractSeq((chrName, tPos-args.tsd,tPos), genomeFile, fai)
#                        targetPrefix = genomeDict[chrName].seq[tPos-args.tsd:tPos]

                        targetPrefix = targetPrefix.upper()
                        #targetSuffix = genomeDict[chrName].seq[tPos:tPos+args.tsd]
                        targetSuffix = Tools.ExtractSeq((chrName, tPos,tPos+args.tsd), genomeFile, fai)
                        targetSuffix = targetSuffix.upper()
                        (sp, ss, sScore) = Align.TSDAlign(tsdSuffix, targetPrefix, 'suffix')
                        (pp, ps, pScore) = Align.TSDAlign(tsdPrefix, targetSuffix, 'prefix')
                        if (sScore > pScore ):
                            tsd = ss
                        elif (pScore > sScore ):
                            tsd = ps

                    nucs = ['A', 'C', 'G', 'T']
                    fracs = [float(gapSeq.count(n))/(len(gapSeq)+1) for n in nucs]
                    doPrint = True
                    for frac in fracs:
                        if (frac > 0.85):
                            doPrint = False
                    if (doPrint):
                        outFile.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "insertion", l, gapSeq, tsd, aln.title, qPos, qPos + l))
                        if (args.context > 0):
                            outFile.write("\t{}".format(homopolymer))
                        if (args.qpos):
                            outFile.write("\t{}\t{}".format(qPos, qPos + len(gapSeq)))
                        outFile.write("\n")

                qPos += l
            if (op == D):
                if (l >= args.minLength and (args.maxLength is None or l < args.maxLength)):
                    foundGap = True
                    chrName = aln.tName
                    if (tPos > fai[chrName][0]):
                        sys.stderr.write("ERROR! tpos is past the genome end." + str(tPos) + " " + str(fai[chrName][0]) + "\n")
                    #delStart = max(tPos - args.context, 0)
                    #delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    delStart = max(tPos - args.context, 0)
                    delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    context= aln.seq[qPos+l:min(qPos+l+args.context, len(aln.seq))]
                    if (context == "A"*len(context) or context == "T"*len(context)):
                        homopolymer="T"
                    else:
                        homopolymer="F"

                    #delSeq = genomeDict[chrName].seq[delStart:delEnd].tostring()
                    delSeq = Tools.ExtractSeq([chrName, delStart, delEnd], genomeFile, fai)

                    outFile.write("{}\t{}\t{}\t{}\t{}\t{}\tno_tsd\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "deletion", l, delSeq, aln.title, qPos, qPos + l))
                    if (args.context > 0):
                        outFile.write("\t{}".format(homopolymer))

                    if (args.qpos):
                        outFile.write("\t{}\t{}".format(qPos, qPos))
                    outFile.write("\n")

                tPos += l
            if (op == H):
                pass

        if (foundGap == False and args.gapFree is not None):
            gapFree.write(aln.tName + "\t" + str(aln.tStart) + "\t" + str(aln.tEnd) + "\t" + aln.title + "\n")

        if (args.outsam is not None):
            packedCigar= ''.join([str(v[0]) + str(v[1]) for v in zip(packedLengths, packedOps)])
            vals = line.split()
            packedLine = '\t'.join(vals[0:5]) + "\t" + packedCigar + '\t'.join(vals[6:]) + "\n"
            outsam.write(packedLine)

if (args.gapFree is not None):
    gapFree.close()

outFile.close()
if (args.outsam is not None):
    outsam.close()
<commit_msg>added support for skipping variants in blacklisted contigs.<commit_after>#!/usr/bin/env python

import sys
import argparse
import Tools
import Align
from  Bio import SeqIO

ap = argparse.ArgumentParser(description="Print gaps in a SAM file.")
ap.add_argument("genome", help="Genome file with a .fai")
ap.add_argument("sam", help="Sam file of alignment.", nargs="+")
ap.add_argument("--onTarget", help="Assume the query encodes the position of the aligned sequence, and make sure at least the chromosomes match.", default=False, action='store_true')
ap.add_argument("--gapFree", help="Print sequences without gaps.", default=None)
ap.add_argument("--minContigLength", help="Only parse alignments from contigs this length or greater", default=0, type=int)
ap.add_argument("--minLength", help="Minimum gap length.", default=50, type=int)
ap.add_argument("--maxLength", help="Maximum gap length.", default=None, type=int)
ap.add_argument("--outFile", help="Print output here, default= stdout", default=None)
ap.add_argument("--context", help="Print surrounding context", default=0, type=int)
ap.add_argument("--condense", help="Pack indels if the matches separating them is less than this value.", default=0, type=int)
ap.add_argument("--tsd", help="Attempt to find Target Site Duplications at most this length", default=20, type=int)
ap.add_argument("--outsam", help="Write the modified condensed sam to a file.", default=None)
ap.add_argument("--minq", help="Minimal mapping quality to consider (10)",default=10,type=int)
ap.add_argument("--qpos", help="Write query position of gaps", default=False,action='store_true')
ap.add_argument("--snv", help="Print SNVs to this file.", default=None)
ap.add_argument("--contigBed", help="Print where contigs map.", default=None)
ap.add_argument("--status", help="Print how far along the alignments are.", default=False, action='store_true')
ap.add_argument("--blacklist", help="Exclude contigs on this list from callsets.", default=None)
ap.add_argument("--removeAdjacentIndels", help="Find instances of SNVs pushed into indels, in the format: NIXMND., and remove these operations.", default=False, action='store_true')
args = ap.parse_args()

genome = file(args.genome, 'r')
handle = open(args.genome, "r")

if (args.outFile is None):
    outFile = sys.stdout
else:
    outFile = open(args.outFile, 'w')

if (args.gapFree is not None):
    gapFree = open(args.gapFree, 'w')

if (args.contigBed is not None):
    contigBed = open(args.contigBed, 'w')

blacklist = {}
if (args.blacklist is not None):
    bl = open(args.blacklist)
    for line in bl:
        v = line.split()
        if (v[0] not in blacklist):
            blacklist[v[0]] = []
        if (len(v) > 1):
            blacklist[v[0]].append(int(v[1])+1)

fai = Tools.ReadFAIFile(args.genome + ".fai")

#genomeDict = SeqIO.to_dict(SeqIO.parse(handle, "fasta"))

if (args.outsam is not None):
    outsam = open(args.outsam, 'w')

snvOut = None
if (args.snv is not None):
    snvOut = open(args.snv, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")
genomeFile = open(args.genome, 'r')

M = 'M'
I = 'I'
D = 'D'
N = 'N'
S = 'S'
H = 'H'
P = 'P'
if (args.sam[0].find(".fofn") >= 0):
    fofnFile = open(args.sam[0])
    samFiles = [line.strip() for line in fofnFile.readlines()]
    args.sam = samFiles
lineNumber = 0
contextLength = 8
#import pdb
import re
coordRe = re.compile(".*(chr.*)\.(\d+)-(\d+).*")
for samFileName in args.sam:
    samFile = open(samFileName)

    for line in samFile:
        lineNumber = lineNumber + 1

        if (line[0] == "@"):
            if (args.outsam is not None):
                outsam.write(line)
            continue
        if (len(line) <= 1):
            continue

        aln = Tools.SAMEntry(line)
        if (aln.title is None):
            continue
	#
	# Use 0-based coordinate system
	#
        aln.tStart -=1
        aln.tEnd -=1
        if (args.onTarget == True):
            coordReMatch = coordRe.match(aln.title)
            if (coordReMatch is not None):
                coordMatchGroups = coordReMatch.groups()
                srcChrom = coordMatchGroups[0]
                srcStart = int(coordMatchGroups[1])
                srcEnd   = int(coordMatchGroups[2])
                if (srcChrom != aln.tName):
                    print "off target chromosome: " + srcChrom + " " + aln.tName
                    continue
                if (((srcStart >= aln.tStart and srcStart < aln.tEnd) or (srcEnd >= aln.tStart and srcEnd < aln.tEnd) or (srcStart < aln.tStart and srcEnd > aln.tEnd )) == False):
                    print "no overlap " + srcChrom + " " + str(srcStart) + " " + str(srcEnd) + " alignment: " + str(aln.tStart) + " "+ str(aln.tEnd)
                    continue

        if (aln.flag & 256 != 0):
            continue
        if (aln.mapqv < args.minq):
            continue

        if (args.contigBed is not None):
            contigBed.write("{}\t{}\t{}\t{}\n".format(aln.tName, aln.tStart, aln.tStart + aln.tlen, aln.title))

        if (args.minContigLength > len(aln.seq)):
            continue

        if (args.blacklist is not None):
            if (aln.title in blacklist):
                if (len(blacklist[aln.title]) == 0):
                    sys.stderr.write("Skipping " + aln.title + " in blacklist.\n")
                    continue
                else:
                    foundPos = False

                    for p in blacklist[aln.title]:
                        if int(aln.tPos) == p:
                            foundPos = True
                            break

                    if foundPos:
                        sys.stderr.write("Skipping " + aln.title + " in blacklist.\n")
                        continue

        tPos = aln.tStart
        qPos = 0
        #
        # condense matches.
        #
        packedCigar = []
        i = 0
        i1 = 1
        niter = 0
        maxGap = 0
        maxGapType = 0
        #print str(aln.ops)

        foundGap = False

        if (args.removeAdjacentIndels):
            for i in range(1,len(aln.lengths)-1):
                if (aln.ops[i-1] != 'M' and
                    aln.ops[i+1] != 'M' and
                    aln.ops[i-1] != aln.ops[i+1] and
                    aln.ops[i] == 'M' and
                    aln.lengths[i-1] == aln.lengths[i+1] and
                    aln.lengths[i] < 4):
                    aln.lengths[i-1] = 0
                    aln.lengths[i+1] = 0

            newLengths = []
            newOps = []
            for i in range(0,len(aln.lengths)):
                if (aln.lengths[i] != 0):
                    newLengths.append(aln.lengths[i])
                    newOps.append(aln.ops[i])
            aln.lengths = newLengths
            aln.ops = newOps

        packedOps = []
        packedLengths = []
        i = 0
        if (args.condense > 0):
            while (i < len(aln.lengths)):
                l = aln.lengths[i]
                op = aln.ops[i]
                j = i
                if (op == I or op == D):
                    if (l > maxGap):
                        maxGap = l
                        maxGapType = op

                if (op == I or op == D and i < len(aln.ops) - 2 and aln.ops[i+2][0] == op):
                    matchLen = 0
                    gapLen   = 0
                    while (j+2 < len(aln.ops) and aln.ops[j+2][0] == op and aln.ops[j+1][0] == M and aln.lengths[j+1] < args.condense):

                        matchLen += aln.lengths[j+1]
                        gapLen   += aln.lengths[j+2]
                        j+=2
                    if (j > i):
                        newIndel = (op, l+gapLen)
                        newMatch = (M, matchLen)
                        packedOps.append(op)
                        packedLengths.append(l+gapLen)

                        packedOps.append(M)
                        packedLengths.append(matchLen)

                    else:
                        packedLengths.append(l)
                        packedOps.append(op)

                else:
                    packedLengths.append(l)
                    packedOps.append(op)

                i = j + 1
                niter +=1
                if (niter > len(aln.ops)):
                    print "ERROR! too many interations."
        else:
            packedOps = aln.ops
            packedLengths = aln.lengths

        for i in range(len(packedOps)):
            op = packedOps[i]
            l  = packedLengths[i]

            if (op == N or op == S):
                # Inside match block (if op == M)
                qPos += l
            if (op == M):
                # Inside match block (if op == M)
                if (args.snv is not None):
                    targetSeq = Tools.ExtractSeq((aln.tName, tPos-1,tPos+l-1), genomeFile, fai)
                    querySeq  = aln.seq[qPos:qPos+l]
                    nMis = 0

                    for mp in range(0,len(targetSeq)):
                        if (querySeq[mp].upper() != targetSeq[mp].upper() and targetSeq[mp].upper() != 'N' and querySeq[mp].upper() != 'N'):
                            nMis +=1
                            snvOut.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(aln.tName, tPos+mp, tPos+mp+1, targetSeq[mp], querySeq[mp], aln.title ))
                tPos += l
                qPos += l

            if (op == I):
                if (l >= args.minLength and (args.maxLength is None or l < args.maxLength)):
                    #                    print "gap at " + str(i) + " " + str(op) + " " + str(l) + " " + str(qPos) + " qlen: " + str(len(aln.seq))
                    foundGap = True
                    chrName = aln.tName
                    #gapSeq = aln.seq[max(0,qPos-args.context):min(qPos+l+args.context, len(aln.seq))]
                    gapSeq = aln.seq[qPos:qPos+l]

                    context= aln.seq[qPos+l:min(qPos+l+args.context, len(aln.seq))]
                    if (context == "A"*len(context) or context == "T"*len(context)):
                        homopolymer="T"
                    else:
                        homopolymer="F"
                    tsd = "notsd"
                    if (len(gapSeq) == 0):
                        sys.stderr.write("ERROR, gap seq is of zero length\n")
                    if (args.tsd):
                        # try and find the target site duplications, this may be on either side of the alignemnt
                        tsdSuffix = gapSeq[-args.tsd:]
                        tsdSuffix = tsdSuffix.upper()
                        tsdPrefix = gapSeq[0:args.tsd]
                        tsdPrefix = tsdPrefix.upper()
                        targetPrefix = Tools.ExtractSeq((chrName, tPos-args.tsd,tPos), genomeFile, fai)
#                        targetPrefix = genomeDict[chrName].seq[tPos-args.tsd:tPos]

                        targetPrefix = targetPrefix.upper()
                        #targetSuffix = genomeDict[chrName].seq[tPos:tPos+args.tsd]
                        targetSuffix = Tools.ExtractSeq((chrName, tPos,tPos+args.tsd), genomeFile, fai)
                        targetSuffix = targetSuffix.upper()
                        (sp, ss, sScore) = Align.TSDAlign(tsdSuffix, targetPrefix, 'suffix')
                        (pp, ps, pScore) = Align.TSDAlign(tsdPrefix, targetSuffix, 'prefix')
                        if (sScore > pScore ):
                            tsd = ss
                        elif (pScore > sScore ):
                            tsd = ps

                    nucs = ['A', 'C', 'G', 'T']
                    fracs = [float(gapSeq.count(n))/(len(gapSeq)+1) for n in nucs]
                    doPrint = True
                    for frac in fracs:
                        if (frac > 0.85):
                            doPrint = False
                    if (doPrint):
                        outFile.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "insertion", l, gapSeq, tsd, aln.title, qPos, qPos + l))
                        if (args.context > 0):
                            outFile.write("\t{}".format(homopolymer))
                        if (args.qpos):
                            outFile.write("\t{}\t{}".format(qPos, qPos + len(gapSeq)))
                        outFile.write("\n")

                qPos += l
            if (op == D):
                if (l >= args.minLength and (args.maxLength is None or l < args.maxLength)):
                    foundGap = True
                    chrName = aln.tName
                    if (tPos > fai[chrName][0]):
                        sys.stderr.write("ERROR! tpos is past the genome end." + str(tPos) + " " + str(fai[chrName][0]) + "\n")
                    #delStart = max(tPos - args.context, 0)
                    #delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    delStart = max(tPos - args.context, 0)
                    delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    context= aln.seq[qPos+l:min(qPos+l+args.context, len(aln.seq))]
                    if (context == "A"*len(context) or context == "T"*len(context)):
                        homopolymer="T"
                    else:
                        homopolymer="F"

                    #delSeq = genomeDict[chrName].seq[delStart:delEnd].tostring()
                    delSeq = Tools.ExtractSeq([chrName, delStart, delEnd], genomeFile, fai)

                    outFile.write("{}\t{}\t{}\t{}\t{}\t{}\tno_tsd\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "deletion", l, delSeq, aln.title, qPos, qPos + l))
                    if (args.context > 0):
                        outFile.write("\t{}".format(homopolymer))

                    if (args.qpos):
                        outFile.write("\t{}\t{}".format(qPos, qPos))
                    outFile.write("\n")

                tPos += l
            if (op == H):
                pass

        if (foundGap == False and args.gapFree is not None):
            gapFree.write(aln.tName + "\t" + str(aln.tStart) + "\t" + str(aln.tEnd) + "\t" + aln.title + "\n")

        if (args.outsam is not None):
            packedCigar= ''.join([str(v[0]) + str(v[1]) for v in zip(packedLengths, packedOps)])
            vals = line.split()
            packedLine = '\t'.join(vals[0:5]) + "\t" + packedCigar + '\t'.join(vals[6:]) + "\n"
            outsam.write(packedLine)

if (args.gapFree is not None):
    gapFree.close()

outFile.close()
if (args.outsam is not None):
    outsam.close()
<|endoftext|>