<commit_before>
"""Crawling classes."""

import asyncio
import builtins
from collections import namedtuple
import datetime as dt
import functools
import logging
from pathlib import Path

from aiohttp import ClientSession, TCPConnector
import gridfs
import magic

from . import config, Db
from .text_utils import doc_to_text, docx_to_json, html_to_lxml, pdf_to_text

_CACHE = Db.get(config.CACHE_DB)
_CACHE = namedtuple('_CACHE', 'file text')(gridfs.GridFS(_CACHE),
                                           _CACHE['text'])

class Crawler:
    """An async request pool and a rudimentary persisent cache."""

    def __init__(self, debug=False, max_reqs=15):
        self._loop = asyncio.get_event_loop()
        self._loop.set_debug(enabled=debug)

        # Limit concurrent requests to `max_reqs` to avoid flooding the
        # server.  `aiohttp.BaseConnector` has also got a `limit` option,
        # but that limits the number of open _sockets_
        self._semaphore = asyncio.Semaphore(max_reqs)

    def __call__(self, task):
        """Set off the crawler."""
        with ClientSession(connector=TCPConnector(use_dns_cache=True),
                           loop=self._loop) \
                as self._session:
            output = self._loop.run_until_complete(task(self)())
        return task.after(output)

    def exec_blocking(self, func):
        """Execute blocking operations independently of the async loop.

        `exec_blocking` wraps `func` inside a coroutine; call it as you
        normally would a coroutine.  Use `exec_blocking` with long-running
        blocking operations.
        """
        return functools.partial(self._loop.run_in_executor, None, func)

    async def gather(self, tasks):
        """Execute the supplied sub-tasks, aggregating their return values."""
        return await asyncio.gather(*tasks, loop=self._loop)

    async def get_text(self, url, *, form_data=None, request_method='get'):
        """Retrieve the decoded content of `url`."""
        exists = _CACHE.text.find_one(dict(url=url,
                                           form_data=form_data,
                                           request_method=request_method))
        if exists:
            return exists['text']

        # Postpone the req until a slot has become available
        async with self._semaphore, \
                self._session.request(request_method, url,
                                      data=form_data) as response:
            text = await response.text()
            _CACHE.text.insert_one(dict(url=url,
                                        form_data=form_data,
                                        request_method=request_method,
                                        text=text))
            return text

    async def get_html(self, url, *, clean=False, **kwargs):
        """Retrieve the lxml'ed text content of `url`."""
        text = await self.get_text(url, **kwargs)
        return await self.exec_blocking(html_to_lxml)(url, text, clean)

    async def get_payload(self, url, *, decode=False):
        """Retrieve the encoded content of `url`."""
        if decode is True:
            return await self._decode_payload(url, await self.get_payload(url))

        exists = _CACHE.file.find_one(dict(url=url))
        if exists:
            return exists.read()

        async with self._semaphore, \
                self._session.request('get', url) as response:
            payload = await response.read()
            _CACHE.file.put(payload, url=url)
            return payload

    async def _decode_payload(self, url, payload):
        DECODE_FUNCS = {b'application/msword': doc_to_text,
                        b'application/pdf': pdf_to_text,
                        b'application/vnd.openxmlformats-officedocument.'
                        b'wordprocessingml.document': docx_to_json}

        try:
            decode_func = DECODE_FUNCS[magic.from_buffer(payload, mime=True)]
        except KeyError:
            raise ValueError('Unable to decode {!r};'
                             ' unknown mime type'.format(url)) from None
        else:
            return decode_func.__name__, \
                await self.exec_blocking(decode_func)(payload)

    @classmethod
    def clear_text_cache(cls):
        """Clear the text cache.

        Manually clear the text cache every so often, in the absence of
        a mechanism to check whether a document is stale.
        """
        _CACHE.text.drop()

    @classmethod
    def dump_cache(cls, cache_path=None):
        """Save the cache on disk."""
        cache_dir = Path(cache_path or 'cache-dump')
        cache_dir.mkdir(exist_ok=True)
        for file in _CACHE.file.find():
            path = Path(cache_dir, file.url.replace('://', '___'))
            path.parent.mkdir(exist_ok=True, parents=True)
            with path.open('wb') as file_handle:
                file_handle.write(file.read())
        for file in _CACHE.text.find():
            path = Path(cache_dir, file['url'].replace('://', '___'))
            path.parent.mkdir(exist_ok=True, parents=True)
            with path.open('w') as file_handle:
                file_handle.write(file['text'])
        with (cache_dir/'VERSION').open('w') as file_handle:
            file_handle.write(dt.datetime.now().isoformat())

class Task:
    """A scraping task primitive."""

    def __init__(self, crawler):
        builtins.logger = logging.getLogger(self.__class__.__name__)
        self.crawler = self.c = crawler

    def after(output):
        """Override to handle the output of the `Task`."""
        return output
<commit_msg>Allow passing query parameters to crawler<commit_after>
"""Crawling classes."""

import asyncio
import builtins
from collections import namedtuple
import datetime as dt
import functools
import logging
from pathlib import Path

from aiohttp import ClientSession, TCPConnector
import gridfs
import magic

from . import config, Db
from .text_utils import doc_to_text, docx_to_json, html_to_lxml, pdf_to_text

_CACHE = Db.get(config.CACHE_DB)
_CACHE = namedtuple('_CACHE', 'file text')(gridfs.GridFS(_CACHE),
                                           _CACHE['text'])

class Crawler:
    """An async request pool and a rudimentary persisent cache."""

    def __init__(self, debug=False, max_reqs=15):
        self._loop = asyncio.get_event_loop()
        self._loop.set_debug(enabled=debug)

        # Limit concurrent requests to `max_reqs` to avoid flooding the
        # server.  `aiohttp.BaseConnector` has also got a `limit` option,
        # but that limits the number of open _sockets_
        self._semaphore = asyncio.Semaphore(max_reqs)

    def __call__(self, task):
        """Set off the crawler."""
        with ClientSession(connector=TCPConnector(use_dns_cache=True),
                           loop=self._loop) \
                as self._session:
            output = self._loop.run_until_complete(task(self)())
        return task.after(output)

    def exec_blocking(self, func):
        """Execute blocking operations independently of the async loop.

        `exec_blocking` wraps `func` inside a coroutine; call it as you
        normally would a coroutine.  Use `exec_blocking` with long-running
        blocking operations.
        """
        return functools.partial(self._loop.run_in_executor, None, func)

    async def gather(self, tasks):
        """Execute the supplied sub-tasks, aggregating their return values."""
        return await asyncio.gather(*tasks, loop=self._loop)

    async def get_text(self, url, *,
                        form_data=None, request_method='get', params=None):
        """Retrieve the decoded content of `url`."""
        exists = _CACHE.text.find_one(dict(url=url,
                                           form_data=form_data,
                                           request_method=request_method))
        if exists:
            return exists['text']

        # Postpone the req until a slot has become available
        async with self._semaphore, \
                self._session.request(request_method, url,
                                      data=form_data, params=params) as response:
            text = await response.text()
            _CACHE.text.insert_one(dict(url=url,
                                        form_data=form_data,
                                        request_method=request_method,
                                        text=text))
            return text

    async def get_html(self, url, *, clean=False, **kwargs):
        """Retrieve the lxml'ed text content of `url`."""
        text = await self.get_text(url, **kwargs)
        return await self.exec_blocking(html_to_lxml)(url, text, clean)

    async def get_payload(self, url, *, decode=False, params=None):
        """Retrieve the encoded content of `url`."""
        if decode is True:
            return await self._decode_payload(url, await self.get_payload(url))

        exists = _CACHE.file.find_one(dict(url=url))
        if exists:
            return exists.read()

        async with self._semaphore, \
                self._session.request('get', url, params=params) as response:
            payload = await response.read()
            _CACHE.file.put(payload, url=url)
            return payload

    async def _decode_payload(self, url, payload):
        DECODE_FUNCS = {b'application/msword': doc_to_text,
                        b'application/pdf': pdf_to_text,
                        b'application/vnd.openxmlformats-officedocument.'
                        b'wordprocessingml.document': docx_to_json}

        try:
            decode_func = DECODE_FUNCS[magic.from_buffer(payload, mime=True)]
        except KeyError:
            raise ValueError('Unable to decode {!r};'
                             ' unknown mime type'.format(url)) from None
        else:
            return decode_func.__name__, \
                await self.exec_blocking(decode_func)(payload)

    @classmethod
    def clear_text_cache(cls):
        """Clear the text cache.

        Manually clear the text cache every so often, in the absence of
        a mechanism to check whether a document is stale.
        """
        _CACHE.text.drop()

    @classmethod
    def dump_cache(cls, cache_path=None):
        """Save the cache on disk."""
        cache_dir = Path(cache_path or 'cache-dump')
        cache_dir.mkdir(exist_ok=True)
        for file in _CACHE.file.find():
            path = Path(cache_dir, file.url.replace('://', '___'))
            path.parent.mkdir(exist_ok=True, parents=True)
            with path.open('wb') as file_handle:
                file_handle.write(file.read())
        for file in _CACHE.text.find():
            path = Path(cache_dir, file['url'].replace('://', '___'))
            path.parent.mkdir(exist_ok=True, parents=True)
            with path.open('w') as file_handle:
                file_handle.write(file['text'])
        with (cache_dir/'VERSION').open('w') as file_handle:
            file_handle.write(dt.datetime.now().isoformat())

class Task:
    """A scraping task primitive."""

    def __init__(self, crawler):
        builtins.logger = logging.getLogger(self.__class__.__name__)
        self.crawler = self.c = crawler

    def after(output):
        """Override to handle the output of the `Task`."""
        return output
<|endoftext|>


<commit_before>from docutils.parsers.rst import directives, Directive, roles
import docutils.core
import os

import pynuts.directives

class Button(Directive):
    """A rest directive who create a button in HTML."""
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False
    option_spec = {'class': directives.class_option}

    def run(self):
        content = ('<input type="button" class="%s" value="%s" onclick="%s"/>'
                  % (
                      ' '.join(self.options.get('class', [])),
                      self.arguments[0], self.arguments[1],
                  ))
        return [docutils.nodes.raw('', content, format='html')]

# The signature of this function is given by docutils
# pylint: disable=R0913,W0613
def editable(name, rawtext, text, lineno, inliner, options=None,
             content=None):
    """."""
    content = '%s' % text
    return [docutils.nodes.raw('', content, format='html')], []

roles.register_canonical_role('editable', editable)

class Script(Directive):
    """A rest directive which creates a script tag in HTML."""
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        document_type, document_id, version, filename = \
            self.arguments[0].split('/')
        cls = self.state.document.settings._pynuts.documents[document_type]
        document = cls(document_id, version)
        path = document.resource_url(os.path.join('javascript', filename))
        content = (''
                   % path)
        return [docutils.nodes.raw('', content, format='html')]

class Button(Directive):
    """A rest directive who create a button in HTML."""
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False
    option_spec = {'class': directives.class_option}

    def run(self):
        content = ('<input type="button" class="%s" value="%s" onclick="%s"/>'
                  % (
                      ' '.join(self.options.get('class', [])),
                      self.arguments[0], self.arguments[1],
                  ))
        return [docutils.nodes.raw('', content, format='html')]

class JQuery(Directive):
    """A rest directive which includes JQuery."""
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        content = ('')
        return [docutils.nodes.raw('', content, format='html')]

class Checkbox(Directive):
    """A rest directive who create a checkbox in HTML."""
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        content = (
            '<input type="checkbox" '
            'onChange="this.setAttribute('
            '\'checked\', this.checked?\'checked\':\'\');"/>')
        return [docutils.nodes.raw('', content, format='html')]

directives.register_directive('checkbox', Checkbox)
directives.register_directive('editable', pynuts.directives.Editable)
directives.register_directive('script', Script)
directives.register_directive('jquery', JQuery)
directives.register_directive('button', Button)
<commit_msg>Update directives according to the new Pynuts API<commit_after>from docutils.parsers.rst import directives, Directive, roles
import docutils.core
import os

import pynuts.directives

class Button(Directive):
    """A rest directive who create a button in HTML."""
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False
    option_spec = {'class': directives.class_option}

    def run(self):
        content = ('<input type="button" class="%s" value="%s" onclick="%s"/>'
                  % (
                      ' '.join(self.options.get('class', [])),
                      self.arguments[0], self.arguments[1],
                  ))
        return [docutils.nodes.raw('', content, format='html')]

# The signature of this function is given by docutils
# pylint: disable=R0913,W0613
def editable(name, rawtext, text, lineno, inliner, options=None,
             content=None):
    """."""
    content = '%s' % text
    return [docutils.nodes.raw('', content, format='html')], []

roles.register_canonical_role('editable', editable)

class Script(Directive):
    """A rest directive which creates a script tag in HTML."""
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        document_type, document_id, version, filename = \
            self.arguments[0].split('/')
        cls = self.state.document.settings._pynuts.documents[document_type]
        document = cls(document_id, version)
        path = document.resource_git_url(os.path.join('javascript', filename))
        content = (''
                   % path)
        return [docutils.nodes.raw('', content, format='html')]

class Button(Directive):
    """A rest directive who create a button in HTML."""
    required_arguments = 2
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False
    option_spec = {'class': directives.class_option}

    def run(self):
        content = ('<input type="button" class="%s" value="%s" onclick="%s"/>'
                  % (
                      ' '.join(self.options.get('class', [])),
                      self.arguments[0], self.arguments[1],
                  ))
        return [docutils.nodes.raw('', content, format='html')]

class JQuery(Directive):
    """A rest directive which includes JQuery."""
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        content = ('')
        return [docutils.nodes.raw('', content, format='html')]

class Checkbox(Directive):
    """A rest directive who create a checkbox in HTML."""
    required_arguments = 0
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        content = (
            '<input type="checkbox" '
            'onChange="this.setAttribute('
            '\'checked\', this.checked?\'checked\':\'\');"/>')
        return [docutils.nodes.raw('', content, format='html')]

directives.register_directive('checkbox', Checkbox)
directives.register_directive('editable', pynuts.directives.Editable)
directives.register_directive('script', Script)
directives.register_directive('jquery', JQuery)
directives.register_directive('button', Button)
<|endoftext|>


<commit_before>"""
This module is used by agent to execute spider task.
"""
# pylint: disable=missing-module-docstring
# pylint: disable=missing-class-docstring
# pylint: disable=missing-function-docstring
import os
import logging
import tempfile
import shutil
from urllib.parse import urlparse
from configparser import ConfigParser
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.gen import coroutine
from tornado import gen
from .config import AgentConfig
from .workspace import RunnerFactory, DictSpiderSettings
from .exceptions import ProcessFailed
from .client import get_client, NoJobAvailable

LOGGER = logging.getLogger(__name__)

TASK_STATUS_SUCCESS = 'success'
TASK_STATUS_FAIL = 'fail'

EXECUTOR_STATUS_OFFLINE = 0
EXECUTOR_STATUS_ONLINE = 1

class SpiderTask:
    """
    Spider task description data fetched from server.
    """
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    spider_parameters = None
    extra_requirements = None
    settings = None
    figure = None

class TaskSlotContainer:
    """
    Slot container to control concurrency running on agent side.
    Agent need allocate a slot before running.
    """
    def __init__(self, max_size=1):
        self._max_size = max_size
        self.slots = [None] * max_size

    def is_full(self):
        for slot in self.slots:
            if slot is None:
                return False
        return True

    def put_task(self, task):
        if not isinstance(task, TaskExecutor):
            raise ValueError('Task in TaskSlotContainer must be '
                             'TaskExecutor type.')
        for i in range(len(self.slots)):
            if self.slots[i] is None:
                self.slots[i] = task
                break

    def remove_task(self, task):
        for i in range(len(self.slots)):
            if self.slots[i] == task:
                self.slots[i] = None
                break

    def tasks(self):
        for item in self.slots:
            if item is not None:
                yield item

    def get_task(self, task_id):
        for item in self.slots:
            if item and item.task.id == task_id:
                return item
        return None

    @property
    def max_size(self):
        return self._max_size

class Executor:
    """
    Main object communicate to server and control tasks running.
    """
    # hearbeat check interval, default: 10 seconds
    heartbeat_interval = 10 * 1000
    # check task interval, default: 10 seconds
    checktask_interval = 10 * 1000
    custom_ssl_cert = False

    def __init__(self, config=None):

        self.ioloop = IOLoop.current()
        self.node_id = None
        self.status = EXECUTOR_STATUS_OFFLINE
        if config is None:
            config = AgentConfig()
        self.config = config
        self.tags = config.get('tags')
        self.checktask_callback = None

        self.task_slots = TaskSlotContainer(config.getint('slots', 1))

        server_base = config.get('server')
        if urlparse(server_base).scheme == '':
            if config.getint('server_https_port'):
                server_https_port = config.getint('server_https_port')
                server_base = 'https://%s:%d' % (server_base,
                                                 server_https_port)
            else:
                server_base = 'http://%s:%d' % (server_base,
                                                config.getint('server_port'))
        self.service_base = server_base
        self.keep_job_files = config.getboolean('debug', False)
        LOGGER.debug('keep_job_files %s', self.keep_job_files)

        node_key = None
        secret_key = None
        node_id = None
        if os.path.exists('conf/node.conf'):
            parser = ConfigParser()
            parser.read('conf/node.conf')
            node_key = parser.get('agent', 'node_key')
            secret_key = parser.get('agent', 'secret_key')
            node_id = int(parser.get('agent', 'node_id'))
        self.client = get_client(config,
                                 app_key=node_key,
                                 app_secret=secret_key,
                                 node_id=node_id)
        self.runner_factory = RunnerFactory(config)

    def start(self):
        self.register_node()
        # init heartbeat period callback
        heartbeat_callback = PeriodicCallback(self.send_heartbeat,
                                              self.heartbeat_interval)
        heartbeat_callback.start()

        self.ioloop.start()

    @coroutine
    def send_heartbeat(self):
        if self.status == EXECUTOR_STATUS_OFFLINE:
            self.register_node()
            return
        running_job_ids = [task_executor.task.id for task_executor in
                                   self.task_slots.tasks()]
        res = self.client.heartbeat(running_job_ids=running_job_ids)
        for job_id in res['kill_job_ids'] or []:
            task_to_kill = self.task_slots.get_task(job_id)
            if task_to_kill:
                LOGGER.info('%s', task_to_kill)
                task_to_kill.kill()
                self.task_slots.remove_task(task_to_kill)

        if res['new_job_available']:
            self.ioloop.call_later(0, self.check_task)

    def register_node(self):
        self.client.login()
        self.status = EXECUTOR_STATUS_ONLINE
        self.node_id = self.client._node_id

    def on_new_task_reach(self, task):
        if task is not None:
            task_executor = self.execute_task(task)
            self.task_slots.put_task(task_executor)

    @coroutine
    def get_next_task(self):
        try:
            task = self.client.get_next_job()
            self.on_new_task_reach(task)
        except NoJobAvailable as ex:
            LOGGER.warn('NoJobAvailable')
        except Exception as ex:
            LOGGER.error(ex)

    def execute_task(self, task):
        package_egg = self.client.get_job_egg(task.id)
        executor = TaskExecutor(task, egg_downloader=package_egg,
                                runner_factory=self.runner_factory,
                                keep_files=self.keep_job_files)
        future = executor.execute()
        self.ioloop.add_future(future, self.task_finished)
        return executor

    def complete_task(self, task_executor, status):
        """
        @type task_executor: TaskExecutor
        """
        self.client.complete_job(task_executor.task.id, status,
                                 items_file=task_executor.items_file,
                                 logs_file=task_executor.output_file)
        self.task_slots.remove_task(task_executor)

    def task_finished(self, future):
        task_executor = future.result()
        status = TASK_STATUS_SUCCESS if task_executor.ret_code == 0 \
            else TASK_STATUS_FAIL
        self.complete_task(task_executor, status)

    def check_task(self):
        if not self.task_slots.is_full():
            self.get_next_task()

class TaskExecutor:
    def __init__(self, task, egg_downloader, runner_factory, keep_files=False):
        """
        @type task: SpiderTask
        @type egg_downloader: ProjectEggDownloader
        @type runner_factory: RunnerFactory
        """
        self.task = task
        self.egg_downloader = egg_downloader
        self._runner_factory = runner_factory
        self._spider_settings = DictSpiderSettings(task.figure or
                                                   task.settings['task'])
        self._f_output = None
        self.output_file = None
        self.items_file = None
        self.ret_code = None
        prefix = 'ddjob-%s-%s-' % (task.project_name, task.id)
        self.workspace_dir = tempfile.mkdtemp(prefix=prefix)
        if not os.path.exists(self.workspace_dir):
            os.makedirs(self.workspace_dir)
        self.output_file = str(os.path.join(self.workspace_dir, 'crawl.log'))
        self._f_output = open(self.output_file, 'w')
        self.on_subprocess_start = None
        self.keep_files = keep_files
        self._runner = None

    @gen.coroutine
    def execute(self):
        try:
            LOGGER.info('start fetch spider egg.')
            runner = self._runner_factory.build(self.egg_downloader)
            self._runner = runner
            crawl_result = yield runner.crawl(self._spider_settings)
            self.items_file = crawl_result.items_file
            self.output_file = crawl_result.crawl_logfile
            result = self.complete(0)
        except ProcessFailed as ex:
            LOGGER.warning('Process Failed when executing task %s: %s',
                           self.task.id, ex)
            error_log = ex.message
            if ex.std_output:
                LOGGER.warning(ex.std_output)
                error_log += ex.std_output
            if ex.err_output:
                LOGGER.warning(ex.err_output)
                error_log += ex.err_output
            result = self.complete_with_error(error_log)
        except Exception as ex:
            LOGGER.error('Error when executing task %s: %s', self.task.id, ex)
            error_log = str(ex)
            result = self.complete_with_error(error_log)
        raise gen.Return(result)

    def result(self):
        return self

    def complete(self, ret_code):
        self._f_output.close()
        self.ret_code = ret_code
        return self.result()

    def complete_with_error(self, error_message):
        LOGGER.debug(error_message)
        self._f_output.write(error_message)
        self._f_output.close()
        self.ret_code = 1
        return self.result()

    def __del__(self):
        if not self.keep_files:
            LOGGER.debug('delete task executor for task %s', self.task.id)
            if self.workspace_dir and os.path.exists(self.workspace_dir):
                shutil.rmtree(self.workspace_dir)
            if self._runner:
                self._runner.clear()
                self._runner = None

    def kill(self):
        if not self._f_output.closed:
            self._f_output.write("Received a kill command, stopping spider.")
        self._runner.kill()
<commit_msg>reformat code.<commit_after>"""
This module is used by agent to execute spider task.
"""
# pylint: disable=missing-module-docstring
# pylint: disable=missing-class-docstring
# pylint: disable=missing-function-docstring
import os
import logging
import tempfile
import shutil
from urllib.parse import urlparse
from configparser import ConfigParser
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado.gen import coroutine
from tornado import gen
from .config import AgentConfig
from .workspace import RunnerFactory, DictSpiderSettings
from .exceptions import ProcessFailed
from .client import get_client, NoJobAvailable

logger = logging.getLogger(__name__)

TASK_STATUS_SUCCESS = 'success'
TASK_STATUS_FAIL = 'fail'

EXECUTOR_STATUS_OFFLINE = 0
EXECUTOR_STATUS_ONLINE = 1

class SpiderTask:
    """
    Spider task description data fetched from server.
    """
    id = None
    spider_id = None
    project_name = None
    spider_name = None
    project_version = None
    spider_parameters = None
    extra_requirements = None
    settings = None
    figure = None

class TaskSlotContainer:
    """
    Slot container to control concurrency running on agent side.
    Agent need allocate a slot before running.
    """
    def __init__(self, max_size=1):
        self._max_size = max_size
        self.slots = [None] * max_size

    def is_full(self):
        for slot in self.slots:
            if slot is None:
                return False
        return True

    def put_task(self, task):
        if not isinstance(task, TaskExecutor):
            raise ValueError('Task in TaskSlotContainer must be '
                             'TaskExecutor type.')
        for i in range(len(self.slots)):
            if self.slots[i] is None:
                self.slots[i] = task
                break

    def remove_task(self, task):
        for i in range(len(self.slots)):
            if self.slots[i] == task:
                self.slots[i] = None
                break

    def tasks(self):
        for item in self.slots:
            if item is not None:
                yield item

    def get_task(self, task_id):
        for item in self.slots:
            if item and item.task.id == task_id:
                return item
        return None

    @property
    def max_size(self):
        return self._max_size

class Executor:
    """
    Main object communicate to server and control tasks running.
    """
    # hearbeat check interval, default: 10 seconds
    heartbeat_interval = 10 * 1000
    # check task interval, default: 10 seconds
    checktask_interval = 10 * 1000
    custom_ssl_cert = False

    def __init__(self, config=None):

        self.ioloop = IOLoop.current()
        self.node_id = None
        self.status = EXECUTOR_STATUS_OFFLINE
        if config is None:
            config = AgentConfig()
        self.config = config
        self.tags = config.get('tags')
        self.checktask_callback = None

        self.task_slots = TaskSlotContainer(config.getint('slots', 1))

        server_base = config.get('server')
        if urlparse(server_base).scheme == '':
            if config.getint('server_https_port'):
                server_https_port = config.getint('server_https_port')
                server_base = 'https://%s:%d' % (server_base,
                                                 server_https_port)
            else:
                server_base = 'http://%s:%d' % (server_base,
                                                config.getint('server_port'))
        self.service_base = server_base
        self.keep_job_files = config.getboolean('debug', False)
        logger.debug('keep_job_files %s', self.keep_job_files)

        node_key = None
        secret_key = None
        node_id = None
        if os.path.exists('conf/node.conf'):
            parser = ConfigParser()
            parser.read('conf/node.conf')
            node_key = parser.get('agent', 'node_key')
            secret_key = parser.get('agent', 'secret_key')
            node_id = int(parser.get('agent', 'node_id'))
        self.client = get_client(config,
                                 app_key=node_key,
                                 app_secret=secret_key,
                                 node_id=node_id)
        self.runner_factory = RunnerFactory(config)

    def start(self):
        self.register_node()
        # init heartbeat period callback
        heartbeat_callback = PeriodicCallback(self.send_heartbeat,
                                              self.heartbeat_interval)
        heartbeat_callback.start()

        self.ioloop.start()

    @coroutine
    def send_heartbeat(self):
        if self.status == EXECUTOR_STATUS_OFFLINE:
            self.register_node()
            return
        running_job_ids = [task_executor.task.id for task_executor in
                                   self.task_slots.tasks()]
        res = self.client.heartbeat(running_job_ids=running_job_ids)
        for job_id in res['kill_job_ids'] or []:
            task_to_kill = self.task_slots.get_task(job_id)
            if task_to_kill:
                logger.info('%s', task_to_kill)
                task_to_kill.kill()
                self.task_slots.remove_task(task_to_kill)

        if res['new_job_available']:
            self.ioloop.call_later(0, self.check_task)

    def register_node(self):
        self.client.login()
        self.status = EXECUTOR_STATUS_ONLINE
        self.node_id = self.client._node_id

    def on_new_task_reach(self, task):
        if task is not None:
            task_executor = self.execute_task(task)
            self.task_slots.put_task(task_executor)

    @coroutine
    def get_next_task(self):
        try:
            task = self.client.get_next_job()
            self.on_new_task_reach(task)
        except NoJobAvailable as ex:
            logger.warn('NoJobAvailable')
        except Exception as ex:
            logger.error(ex)

    def execute_task(self, task):
        package_egg = self.client.get_job_egg(task.id)
        executor = TaskExecutor(task, egg_downloader=package_egg,
                                runner_factory=self.runner_factory,
                                keep_files=self.keep_job_files)
        future = executor.execute()
        self.ioloop.add_future(future, self.task_finished)
        return executor

    def complete_task(self, task_executor, status):
        """
        @type task_executor: TaskExecutor
        """
        self.client.complete_job(task_executor.task.id, status,
                                 items_file=task_executor.items_file,
                                 logs_file=task_executor.output_file)
        self.task_slots.remove_task(task_executor)

    def task_finished(self, future):
        task_executor = future.result()
        status = TASK_STATUS_SUCCESS if task_executor.ret_code == 0 \
            else TASK_STATUS_FAIL
        self.complete_task(task_executor, status)

    def check_task(self):
        if not self.task_slots.is_full():
            self.get_next_task()

class TaskExecutor:
    def __init__(self, task, egg_downloader, runner_factory, keep_files=False):
        """
        @type task: SpiderTask
        @type egg_downloader: ProjectEggDownloader
        @type runner_factory: RunnerFactory
        """
        self.task = task
        self.egg_downloader = egg_downloader
        self._runner_factory = runner_factory
        self._spider_settings = DictSpiderSettings(task.figure or
                                                   task.settings['task'])
        self._f_output = None
        self.output_file = None
        self.items_file = None
        self.ret_code = None
        prefix = 'ddjob-%s-%s-' % (task.project_name, task.id)
        self.workspace_dir = tempfile.mkdtemp(prefix=prefix)
        if not os.path.exists(self.workspace_dir):
            os.makedirs(self.workspace_dir)
        self.output_file = str(os.path.join(self.workspace_dir, 'crawl.log'))
        self._f_output = open(self.output_file, 'w')
        self.on_subprocess_start = None
        self.keep_files = keep_files
        self._runner = None

    @gen.coroutine
    def execute(self):
        try:
            logger.info('start fetch spider egg.')
            runner = self._runner_factory.build(self.egg_downloader)
            self._runner = runner
            crawl_result = yield runner.crawl(self._spider_settings)
            self.items_file = crawl_result.items_file
            self.output_file = crawl_result.crawl_logfile
            result = self.complete(0)
        except ProcessFailed as ex:
            logger.warning('Process Failed when executing task %s: %s',
                           self.task.id, ex)
            error_log = ex.message
            if ex.std_output:
                logger.warning(ex.std_output)
                error_log += ex.std_output
            if ex.err_output:
                logger.warning(ex.err_output)
                error_log += ex.err_output
            result = self.complete_with_error(error_log)
        except Exception as ex:
            logger.error('Error when executing task %s: %s', self.task.id, ex)
            error_log = str(ex)
            result = self.complete_with_error(error_log)
        raise gen.Return(result)

    def result(self):
        return self

    def complete(self, ret_code):
        self._f_output.close()
        self.ret_code = ret_code
        return self.result()

    def complete_with_error(self, error_message):
        logger.debug(error_message)
        self._f_output.write(error_message)
        self._f_output.close()
        self.ret_code = 1
        return self.result()

    def __del__(self):
        if not self.keep_files:
            logger.debug('delete task executor for task %s', self.task.id)
            if self.workspace_dir and os.path.exists(self.workspace_dir):
                shutil.rmtree(self.workspace_dir)
            if self._runner:
                self._runner.clear()
                self._runner = None

    def kill(self):
        if not self._f_output.closed:
            self._f_output.write("Received a kill command, stopping spider.")
        self._runner.kill()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.

from __future__ import absolute_import, division, print_function
from builtins import map
from builtins import zip
from builtins import str

from jinja2 import Environment, PackageLoader

import regex

__all__ = ['serialize']

def _rescale(val, low, high):
    """
    Rescales a list of confidence value between 0 and 1 to an interval [low,
    high].

    Args:
        val (float): List of values in interval (0,1)
        low (float): Lower bound of rescaling interval
        high (float): Upper bound of rescaling interval

    Returns:
        Rescaled value (float).
    """
    return [(high - low) * x + low for x in val]

def max_bbox(boxes):
    """
    Calculates the minimal bounding box containing all boxes contained in an
    iterator.

    Args:
        boxes (iterator): An iterator returning tuples of the format (x0, y0,
                          x1, y1)
    Returns:
        A box covering all bounding boxes in the input argument
    """
    sbox = list(map(sorted, list(zip(*boxes))))
    return (sbox[0][0], sbox[1][0], sbox[2][-1], sbox[3][-1])

def serialize(records, image_name=u'', image_size=(0, 0), writing_mode='horizontal-tb', scripts=None, template='hocr'):
    """
    Serializes a list of ocr_records into an output document.

    Serializes a list of predictions and their corresponding positions by doing
    some hOCR-specific preprocessing and then renders them through one of
    several jinja2 templates.

    Args:
        records (iterable): List of kraken.rpred.ocr_record
        image_name (str): Name of the source image
        image_size (tuple): Dimensions of the source image
        writing_mode (str): Sets the principal layout of lines and the
                            direction in which blocks progress. Valid values
                            are horizontal-tb, vertical-rl, and
                            vertical-lr.
        scripts (list): List of scripts contained in the OCR records
        template (str): Selector for the serialization format. May be
                        'hocr' or 'alto'.
    """
    page = {'lines': [], 'size': image_size, 'name': image_name, 'writing_mode': writing_mode, 'scripts': scripts}
    seg_idx = 0
    for idx, record in enumerate(records):
        line = {'index': idx,
                'bbox': max_bbox(record.cuts),
                'cuts': record.cuts,
                'confidences': record.confidences,
                'recognition': []
                }

        splits = regex.split(r'(\s+)', record.prediction)
        line_offset = 0
        for segment in splits:
            if len(segment) == 0:
                continue
            seg_bbox = max_bbox(record.cuts[line_offset:line_offset + len(segment)])
            line['recognition'].append({'bbox': seg_bbox,
                                        'confidences': record.confidences[line_offset:line_offset + len(segment)],
                                        'text': segment,
                                        'index': seg_idx})
            seg_idx += 1
            line_offset += len(segment)
        page['lines'].append(line)
    env = Environment(loader=PackageLoader('kraken', 'templates'), trim_blocks=True, lstrip_blocks=True)
    env.tests['whitespace'] = str.isspace
    env.filters['rescale'] = _rescale
    tmpl = env.get_template(template)
    return tmpl.render(page=page)
<commit_msg>Serialization multiscript stability fixes<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.

from __future__ import absolute_import, division, print_function
from builtins import map
from builtins import zip
from builtins import str

from jinja2 import Environment, PackageLoader

import regex

__all__ = ['serialize']

def _rescale(val, low, high):
    """
    Rescales a list of confidence value between 0 and 1 to an interval [low,
    high].

    Args:
        val (float): List of values in interval (0,1)
        low (float): Lower bound of rescaling interval
        high (float): Upper bound of rescaling interval

    Returns:
        Rescaled value (float).
    """
    return [(high - low) * x + low for x in val]

def max_bbox(boxes):
    """
    Calculates the minimal bounding box containing all boxes contained in an
    iterator.

    Args:
        boxes (iterator): An iterator returning tuples of the format (x0, y0,
                          x1, y1)
    Returns:
        A box covering all bounding boxes in the input argument
    """
    sbox = list(map(sorted, list(zip(*boxes))))
    return (sbox[0][0], sbox[1][0], sbox[2][-1], sbox[3][-1])

def serialize(records, image_name=u'', image_size=(0, 0), writing_mode='horizontal-tb', scripts=None, template='hocr'):
    """
    Serializes a list of ocr_records into an output document.

    Serializes a list of predictions and their corresponding positions by doing
    some hOCR-specific preprocessing and then renders them through one of
    several jinja2 templates.

    Note: Empty records are ignored for serialization purposes.

    Args:
        records (iterable): List of kraken.rpred.ocr_record
        image_name (str): Name of the source image
        image_size (tuple): Dimensions of the source image
        writing_mode (str): Sets the principal layout of lines and the
                            direction in which blocks progress. Valid values
                            are horizontal-tb, vertical-rl, and
                            vertical-lr.
        scripts (list): List of scripts contained in the OCR records
        template (str): Selector for the serialization format. May be
                        'hocr' or 'alto'.
    """
    page = {'lines': [], 'size': image_size, 'name': image_name, 'writing_mode': writing_mode, 'scripts': scripts}
    seg_idx = 0
    for idx, record in enumerate(records):
        # skip empty records
        if not record.prediction:
            break
        line = {'index': idx,
                'bbox': max_bbox(record.cuts),
                'cuts': record.cuts,
                'confidences': record.confidences,
                'recognition': []
                }

        splits = regex.split(r'(\s+)', record.prediction)
        line_offset = 0
        for segment in splits:
            if len(segment) == 0:
                continue
            seg_bbox = max_bbox(record.cuts[line_offset:line_offset + len(segment)])
            line['recognition'].append({'bbox': seg_bbox,
                                        'confidences': record.confidences[line_offset:line_offset + len(segment)],
                                        'text': segment,
                                        'index': seg_idx})
            seg_idx += 1
            line_offset += len(segment)
        page['lines'].append(line)
    env = Environment(loader=PackageLoader('kraken', 'templates'), trim_blocks=True, lstrip_blocks=True)
    env.tests['whitespace'] = str.isspace
    env.filters['rescale'] = _rescale
    tmpl = env.get_template(template)
    return tmpl.render(page=page)
<|endoftext|>


<commit_before><commit_msg>Revert "<fix>[ansible]: add python2-psutil in aarch64 kylin v10 sp2"<commit_after><|endoftext|>


<commit_before>"""
Defines the ABC for a router.
"""

import abc
import typing

import kyoukai.app
from kyoukai.context import HTTPRequestContext

class ABCRoute(abc.ABC):
    """
    A route is a wrapped coroutine object. It provides utilities that allows Kyoukai to interact with the bare route
    command more easily.

    :param blueprint: The blueprint this route is associated with.
    :param matcher: The match string used to create this route.
        This will vary depending on the router.

    :param methods: A :class:`list` of methods that are allowed in this route.
    :param bound: Internal. Used for views.
    :param run_hooks: If the request should run pre and post hooks, or be exempt.
    """

    def __init__(self, blueprint,
                 matcher: str, methods: list,
                 bound: bool = False,
                 run_hooks: bool = True):
        """
        Create a new Route.
        """
        self._match_str = matcher
        self.allowed_methods = methods
        self._wrapped_coro = None

        self.bp = blueprint

        # Used for bounded routes.
        self._bound = bound

        self._view_class = None

        self.name = "<Undefined route>"

        self._should_run_hooks = run_hooks

    @abc.abstractmethod
    async def invoke(self, ctx: HTTPRequestContext, args: typing.Iterable = None, exception: Exception = None):
        """
        Invokes the route, running it.

        :param ctx: The current HTTP context for the route.
        :param args: The expanded arguments from the route, placed into the
        :param exception: Called on an error handler. Is None otherwise.

        :return: The :class:`Response` created.
        """

    def __call__(self, coro):
        """
        Sets the coroutine.
        """
        return self.create(coro)

    def create(self, coro):
        """
        Creates a route from the coroutine.
        """
        self._wrapped_coro = coro
        self.name = coro.__name__
        return self

    # Used for bound routes.
    @property
    def self(self):
        if self._bound:
            return self._view_class
        else:
            return

    @self.setter
    def self(self, vc):
        if self._bound:
            self._view_class = vc
        else:
            raise TypeError("Attempted to update view class on unbounded route")

    def method_allowed(self, method: str):
        """
        Checks if a method is allowed in a route.

        :param method: The HTTP method.
        :return: True if it is allowed for this route, False if not.
        """
        return method.lower() in [_.lower() for _ in self.allowed_methods]

class ABCRouter(abc.ABC):
    """
    A router defines how routes are routed from an incoming client connection.

    Before 1.10, routes were purely routed based on a regular expression. In 1.10 and above, the Router is
    responsible for routing, which (by default) is a regular expression but can be extended to use something like the
    Werkzeug router.
    """

    def __init__(self, app: 'kyoukai.app.Kyoukai'):
        self.app = app

    @staticmethod
    def calculate_405_errorhandler(routes: typing.Iterable):
        """
        Calculates what Blueprint the 405 error handler should run on.

        :param routes: A list of route objects to run on.
        :return: The top-most blueprint to use.
        """
        common_blueprints = None
        for route in routes:
            full_match = set(route.bp.tree_path)
            # If common_blueprints is defined, we can do an intersection on it.
            # Otherwise, we just set it to the current set of Blueprints.
            if common_blueprints is not None:
                common_blueprints = common_blueprints.intersection(full_match)
            else:
                common_blueprints = full_match

        # Turn that set back into a list, and sort it by depth.
        cmn_bp = list(common_blueprints)
        cmn_bp = sorted(cmn_bp, key=lambda bp: bp.depth)
        # Get the bottom most blueprint, which is the best ancestor for both of these in the tree.
        blueprint_to_use = cmn_bp[-1]

        return blueprint_to_use

    @abc.abstractmethod
    def match(self, route: str, method: str) -> typing.Tuple[ABCRoute, typing.Iterable]:
        """
        Matches a route.

        :param route: The route path to match.
        :param method: The method of the route.

        :returns: A two-item tuple:
            The route that was matched,
            An iterable of items to be passed to invoke. This can be any iterable - the route invoke method must
            handle it appropriately.

        :raises: :class:`HTTPException` with 404 if the route was not matched.
        :raises: :class:`HTTPException` with 405 if the route was matched but no methods matched.
        """
<commit_msg>Add should_convert property to base routes.<commit_after>"""
Defines the ABC for a router.
"""

import abc
import typing

import kyoukai.app
from kyoukai.context import HTTPRequestContext

class ABCRoute(abc.ABC):
    """
    A route is a wrapped coroutine object. It provides utilities that allows Kyoukai to interact with the bare route
    command more easily.

    :param blueprint: The blueprint this route is associated with.
    :param matcher: The match string used to create this route.
        This will vary depending on the router.

    :param methods: A :class:`list` of methods that are allowed in this route.
    :param bound: Internal. Used for views.
    :param run_hooks: If the request should run pre and post hooks, or be exempt.
    """

    def __init__(self, blueprint,
                 matcher: str, methods: list,
                 bound: bool = False,
                 run_hooks: bool = True):
        """
        Create a new Route.
        """
        self._match_str = matcher
        self.allowed_methods = methods
        self._wrapped_coro = None

        self.bp = blueprint

        # Used for bounded routes.
        self._bound = bound

        self._view_class = None

        self.name = "<Undefined route>"

        self._should_run_hooks = run_hooks

        self.should_convert = True

    @abc.abstractmethod
    async def invoke(self, ctx: HTTPRequestContext, args: typing.Iterable = None, exception: Exception = None):
        """
        Invokes the route, running it.

        :param ctx: The current HTTP context for the route.
        :param args: The expanded arguments from the route, placed into the
        :param exception: Called on an error handler. Is None otherwise.

        :return: The :class:`Response` created.
        """

    def __call__(self, coro):
        """
        Sets the coroutine.
        """
        return self.create(coro)

    def create(self, coro):
        """
        Creates a route from the coroutine.
        """
        self._wrapped_coro = coro
        self.name = coro.__name__
        return self

    # Used for bound routes.
    @property
    def self(self):
        if self._bound:
            return self._view_class
        else:
            return

    @self.setter
    def self(self, vc):
        if self._bound:
            self._view_class = vc
        else:
            raise TypeError("Attempted to update view class on unbounded route")

    def method_allowed(self, method: str):
        """
        Checks if a method is allowed in a route.

        :param method: The HTTP method.
        :return: True if it is allowed for this route, False if not.
        """
        return method.lower() in [_.lower() for _ in self.allowed_methods]

class ABCRouter(abc.ABC):
    """
    A router defines how routes are routed from an incoming client connection.

    Before 1.10, routes were purely routed based on a regular expression. In 1.10 and above, the Router is
    responsible for routing, which (by default) is a regular expression but can be extended to use something like the
    Werkzeug router.
    """

    def __init__(self, app: 'kyoukai.app.Kyoukai'):
        self.app = app

    @staticmethod
    def calculate_405_errorhandler(routes: typing.Iterable):
        """
        Calculates what Blueprint the 405 error handler should run on.

        :param routes: A list of route objects to run on.
        :return: The top-most blueprint to use.
        """
        common_blueprints = None
        for route in routes:
            full_match = set(route.bp.tree_path)
            # If common_blueprints is defined, we can do an intersection on it.
            # Otherwise, we just set it to the current set of Blueprints.
            if common_blueprints is not None:
                common_blueprints = common_blueprints.intersection(full_match)
            else:
                common_blueprints = full_match

        # Turn that set back into a list, and sort it by depth.
        cmn_bp = list(common_blueprints)
        cmn_bp = sorted(cmn_bp, key=lambda bp: bp.depth)
        # Get the bottom most blueprint, which is the best ancestor for both of these in the tree.
        blueprint_to_use = cmn_bp[-1]

        return blueprint_to_use

    @abc.abstractmethod
    def match(self, route: str, method: str) -> typing.Tuple[ABCRoute, typing.Iterable]:
        """
        Matches a route.

        :param route: The route path to match.
        :param method: The method of the route.

        :returns: A two-item tuple:
            The route that was matched,
            An iterable of items to be passed to invoke. This can be any iterable - the route invoke method must
            handle it appropriately.

        :raises: :class:`HTTPException` with 404 if the route was not matched.
        :raises: :class:`HTTPException` with 405 if the route was matched but no methods matched.
        """
<|endoftext|>


<commit_before>#!/usr/bin/python
# Simple script to parse the devpost dump and place results in a json

import os
import json
from multiprocessing import Pool
import multiprocessing
from bs4 import BeautifulSoup

OUTPUT_FNAME="devpostdump.json"
DUMP_DIR = "output/"

projects = [os.path.join(DUMP_DIR, f) for f in os.listdir(DUMP_DIR)]
# projects = projects[:100]

projects_json = []

def process_project(inp):
    i, project = inp
    print "%d %s" % (i, project)

    proj_html = BeautifulSoup(open(project, 'r').read(), 'html.parser')

    proj_data = {}
    proj_data['name'] = proj_html.find(id='app-title').string
    proj_data['id'] = project[len(DUMP_DIR):]

    # Number of likes and comments
    num_likes =    proj_html.find('span', { 'class' : 'ss-heart' }).next_sibling.next_sibling
    proj_data['num_likes'] = int(num_likes.string) if num_likes is not None else 0
    num_comments = proj_html.find('span', { 'class' : 'ss-quote' }).next_sibling.next_sibling
    proj_data['num_comments'] = int(num_comments.string) if num_comments is not None else 0

    # Length of the description
    proj_data['description_length'] = len(proj_html.find(id="app-details").get_text())

    # Number of contributors
    proj_data['num_contributors'] = len(proj_html.find_all('li', { 'class' : 'software-team-member' }))

    # Tags
    proj_data['tags'] = sorted([tag.string for tag in proj_html.find_all('span', { 'class' : 'cp-tag' })])

    # Hackathon details
    hackathon_deets = proj_html.find('div', { 'class' : 'software-list-content' })
    if hackathon_deets:
        proj_data['hackathon_name'] = hackathon_deets.find('a').string
        proj_data['num_prizes'] = len(hackathon_deets.find_all('span', { 'class' : 'winner' }))
    else:
        proj_data['hackathon_name'] = 0
        proj_data['num_prizes'] = 0

    return proj_data

if __name__ == '__main__':
    num_cores = multiprocessing.cpu_count()
    p = Pool(num_cores)
    j = p.map(process_project, enumerate(projects[:1000]))
    print "Creating json file"
    with open(OUTPUT_FNAME, "w+") as f:
        f.write(json.dumps(j))

<commit_msg>Prune non hackathon hacks and get timestamp<commit_after>#!/usr/bin/python
# Simple script to parse the devpost dump and place results in a json

import os
import json
from multiprocessing import Pool
import multiprocessing
from bs4 import BeautifulSoup

OUTPUT_FNAME="devpostdump.json"
DUMP_DIR = "output/"

projects = [os.path.join(DUMP_DIR, f) for f in os.listdir(DUMP_DIR)]
# projects = projects[:100]

projects_json = []

def process_project(inp):
    i, project = inp
    print "%d %s" % (i, project)

    proj_html = BeautifulSoup(open(project, 'r').read(), 'html.parser')

    proj_data = {}
    proj_data['name'] = proj_html.find(id='app-title').string
    proj_data['id'] = project[len(DUMP_DIR):]

    # Number of likes and comments
    num_likes =    proj_html.find('span', { 'class' : 'ss-heart' }).next_sibling.next_sibling
    proj_data['num_likes'] = int(num_likes.string) if num_likes is not None else 0
    num_comments = proj_html.find('span', { 'class' : 'ss-quote' }).next_sibling.next_sibling
    proj_data['num_comments'] = int(num_comments.string) if num_comments is not None else 0

    # Length of the description
    proj_data['description_length'] = len(proj_html.find(id="app-details").get_text())

    # Number of contributors
    proj_data['num_contributors'] = len(proj_html.find_all('li', { 'class' : 'software-team-member' }))

    # Tags
    proj_data['tags'] = sorted([tag.string for tag in proj_html.find_all('span', { 'class' : 'cp-tag' })])

    # Hackathon details
    hackathon_deets = proj_html.find('div', { 'class' : 'software-list-content' })
    if hackathon_deets:
        proj_data['hackathon_name'] = hackathon_deets.find('a').string
        proj_data['num_prizes'] = len(hackathon_deets.find_all('span', { 'class' : 'winner' }))
    else:
        # We are coing to skip non hackathon hacks for now
        return None

    proj_data['timestamp'] = proj_html.find('time').get('datetime')

    return proj_data

if __name__ == '__main__':
    num_cores = multiprocessing.cpu_count()
    p = Pool(num_cores)
    j = p.map(process_project, enumerate(projects[:100]))
    j = filter(lambda x: x != None, j)
    print "Creating json file"
    with open(OUTPUT_FNAME, "w+") as f:
        f.write(json.dumps(j))

<|endoftext|>


<commit_before>from __future__ import division

from libtbx.phil import parse

master_phil = parse("""
nrefl = 0
  .type = int
nproc = 0
  .type = int
shoebox_size {
  x = 10
    .type = int
  y = 10
    .type = int
  z = 10
    .type = int
}
spot_size {
  x = 1.0
    .type = float
  y = 1.0
    .type = float
  z = 1.0
    .type = float
}
counts = 0
  .type = int
background = 0
  .type = int
pixel_mask = *none static precise
  .type = choice
background_method = *xds mosflm
  .type = choice
integration_methpd = *xds mosflm
  .type = choice
output {
  over = None
    .type = path
  under = None
    .type = path
  all = None
    .type = path
}
""")

def simple_gaussian_spots(params):
  from dials.model.data import ReflectionList
  from dials.algorithms import shoebox
  import random
  import math

  # generate mask and peak values

  from dials.algorithms.shoebox import MaskCode
  mask_peak = MaskCode.Valid|MaskCode.Foreground
  mask_back = MaskCode.Valid|MaskCode.Background

  rlist = ReflectionList(params.nrefl)

  from dials.util.command_line import ProgressBar
  p = ProgressBar(title = 'Generating reflections')

  for j in range(params.nrefl):
    p.update(j * 100.0 / params.nrefl)
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()
    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, params.shoebox_size.x, 0, params.shoebox_size.y,
                             0, params.shoebox_size.z)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  p.finished('Generating %d reflections' % params.nrefl)
  shoebox.allocate(rlist)

  p = ProgressBar(title = 'Generating shoeboxes')

  for _, refl in enumerate(rlist):

    p.update(_ * 100.0 / params.nrefl)
    mask = refl.shoebox_mask

    if params.pixel_mask == 'precise':
      # flag everything as background: peak will me assigned later
      for j in range(len(mask)):
        mask[j] = mask_back
    elif params.pixel_mask == 'none' or params.pixel_mask == None:
      # flag we have no idea what anything is
      mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
      for j in range(len(mask)):
        mask[j] = mask_none
    elif params.pixel_mask == 'static':
      from math import sqrt
      x0 = params.shoebox_size.x / 2
      y0 = params.shoebox_size.y / 2
      z0 = params.shoebox_size.z / 2
      sx = params.spot_size.x
      sy = params.spot_size.y
      sz = params.spot_size.z
      for k in range(mask.all()[0]):
        for j in range(mask.all()[1]):
          for i in range(mask.all()[2]):
            d = ((i - x0) / sx)**2 + ((j - y0) / sy)**2 + ((k - z0) / sz)**2
            if d <= 1.0:
              mask[k,j,i] = MaskCode.Valid | MaskCode.Foreground
            else:
              mask[k,j,i] = MaskCode.Valid | MaskCode.Background# | MaskCode.Foreground

    sbox = refl.shoebox

    # reflection itself, including setting the peak region if we're doing that

    counts_true = 0
    for j in range(params.counts):
      x = int(random.gauss(params.shoebox_size.x / 2, params.spot_size.x))
      y = int(random.gauss(params.shoebox_size.y / 2, params.spot_size.y))
      z = int(random.gauss(params.shoebox_size.z / 2, params.spot_size.z))
      if x < 0 or x >= params.shoebox_size.x:
        continue
      if y < 0 or y >= params.shoebox_size.y:
        continue
      if z < 0 or z >= params.shoebox_size.z:
        continue
      sbox[z, y, x] += 1
      counts_true += 1
      if params.pixel_mask == 'precise':
        mask[z, y, x] = mask_peak

    refl.intensity = counts_true

    # background:flat; FIXME can replace this with Poissonian added random
    # number, which will give the same answer...

    for j in range(params.background * len(sbox)):
      x = random.randint(0, params.shoebox_size.x - 1)
      y = random.randint(0, params.shoebox_size.y - 1)
      z = random.randint(0, params.shoebox_size.z - 1)
      sbox[z, y, x] += 1

  p.finished('Generating %d shoeboxes' % params.nrefl)

  return rlist

def background_xds(rlist):
  from dials.algorithms.background import XdsSubtractor
  background = XdsSubtractor()
  background(None, None, rlist)
  return

def background_inclined(rlist):
  from dials.algorithms.background import InclinedSubtractor
  background = InclinedSubtractor()
  background(None, None, rlist)
  return

def integrate_3d_summation(rlist):
  from dials.algorithms.integration import Summation3d
  integration = Summation3d()
  integration(None, None, rlist)
  return

def main(params):
  rlist = simple_gaussian_spots(params)
  correct_intensities = [r.intensity for r in rlist]
  for r in rlist:
    r.intensity = 0

  if params.background_method == 'xds':
    background_xds(rlist)
  elif params.background_method == 'mosflm':
    assert(params.pixel_mask)
    assert(params.pixel_mask != 'none')
    background_inclined(rlist)

  integrate_3d_summation(rlist)

  integrated_intensities = [r.intensity for r in rlist]

  # now scan through the reflection list and find those where the integration
  # gave an apparently duff answer i.e. outside of 3 sigma from correct value

  from dials.model.data import ReflectionList
  import math

  overestimates = ReflectionList()
  underestimates = ReflectionList()

  for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)):
    sigma = math.sqrt(c)
    if math.fabs(c - i) < 3 * sigma:
      continue
    if i > params.counts:
      overestimates.append(rlist[j])
    else:
      underestimates.append(rlist[j])

  print '%d overestimates, %d underestimates' % (len(overestimates),
                                                 len(underestimates))

  # now pickle these, perhaps

  import cPickle as pickle

  if params.output.under:
    pickle.dump(underestimates, open(params.output.under, 'w'))
  if params.output.over:
    pickle.dump(overestimates, open(params.output.over, 'w'))
  if params.output.all:
    pickle.dump(rlist, open(params.output.all, 'w'))

if __name__ == '__main__':
  import sys

  # FIXME add Phil parameters for
  # nrefl - done
  # nproc - done
  # size of box - done x y z
  # width of spot - done x y z
  # rotation of spot density - not done
  # background plane method with B(x, y) = c + ax + by; don't know how to make
  #                                                   ; random dist. like this
  # counts
  # background
  # background method
  # integration method
  # set proper background mask or no
  # set static mask

  working_phil = master_phil
  for arg in sys.argv[1:]:
    working_phil = working_phil.fetch(parse(arg))
  main(working_phil.extract())
<commit_msg>Allow funny size, orientation of test spot: cctbx.python reflgen.py nrefl=10 counts=10000 background=0 output.all=all_refl.pickle pixel_mask=all background_method=xds shoebox_size.x=20 shoebox_size.y=20 shoebox_size.z=20 spot_size.x=1 spot_size.y=3 spot_size.z=1 spot_offset.x=-0.5 spot_offset.y=-0.5 spot_offset.z=-0.5 rotation.axis.x=0 rotation.axis.y=0 rotation.axis.z=1 rotation.angle=45<commit_after>from __future__ import division

from libtbx.phil import parse

master_phil = parse("""
nrefl = 0
  .type = int
nproc = 0
  .type = int
shoebox_size {
  x = 10
    .type = int
  y = 10
    .type = int
  z = 10
    .type = int
}
spot_size {
  x = 1.0
    .type = float
  y = 1.0
    .type = float
  z = 1.0
    .type = float
}
spot_offset {
  x = 0.0
    .type = float
  y = 0.0
    .type = float
  z = 0.0
    .type = float
}
counts = 0
  .type = int
background = 0
  .type = int
pixel_mask = *all static precise
  .type = choice
background_method = *xds mosflm
  .type = choice
integration_methpd = *xds mosflm
  .type = choice
output {
  over = None
    .type = path
  under = None
    .type = path
  all = None
    .type = path
}
rotation {
  axis {
    x = 0.0
      .type = float
    y = 0.0
      .type = float
    z = 0.0
      .type = float
  }

  angle = 0.0
    .type = float
}
""")

def simple_gaussian_spots(params):
  from dials.model.data import ReflectionList
  from dials.algorithms import shoebox
  import random
  import math

  from scitbx import matrix
  r = params.rotation
  axis = matrix.col((r.axis.x, r.axis.y, r.axis.z))
  if axis.length() > 0:
    rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True)
  else:
    rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1))

  # generate mask and peak values

  from dials.algorithms.shoebox import MaskCode
  mask_peak = MaskCode.Valid|MaskCode.Foreground
  mask_back = MaskCode.Valid|MaskCode.Background

  rlist = ReflectionList(params.nrefl)

  from dials.util.command_line import ProgressBar
  p = ProgressBar(title = 'Generating reflections')

  for j in range(params.nrefl):
    p.update(j * 100.0 / params.nrefl)
    rlist[j].miller_index = (random.randint(0, 20),
                             random.randint(0, 20),
                             random.randint(0, 20))
    rlist[j].rotation_angle = 2 * math.pi * random.random()
    rlist[j].beam_vector = (0, 0, 0)
    rlist[j].image_coord_px = (0, 0)
    rlist[j].image_coord_mm = (0, 0)
    rlist[j].frame_number = 0
    rlist[j].panel_number = 0
    rlist[j].bounding_box = (0, params.shoebox_size.x, 0, params.shoebox_size.y,
                             0, params.shoebox_size.z)
    rlist[j].centroid_position = (0, 0, 0)
    rlist[j].centroid_variance = (0, 0, 0)
    rlist[j].centroid_sq_width = (0, 0, 0)
    rlist[j].intensity = 0
    rlist[j].intensity_variance = 0
    rlist[j].corrected_intensity = 0
    rlist[j].corrected_intensity_variance = 0

  p.finished('Generating %d reflections' % params.nrefl)
  shoebox.allocate(rlist)

  p = ProgressBar(title = 'Generating shoeboxes')

  for _, refl in enumerate(rlist):

    p.update(_ * 100.0 / params.nrefl)
    mask = refl.shoebox_mask

    if params.pixel_mask == 'precise':
      # flag everything as background: peak will me assigned later
      for j in range(len(mask)):
        mask[j] = mask_back
    elif params.pixel_mask == 'all':
      # flag we have no idea what anything is
      mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background
      for j in range(len(mask)):
        mask[j] = mask_none
    elif params.pixel_mask == 'static':
      from math import sqrt
      x0 = params.shoebox_size.x / 2
      y0 = params.shoebox_size.y / 2
      z0 = params.shoebox_size.z / 2
      sx = params.spot_size.x
      sy = params.spot_size.y
      sz = params.spot_size.z
      for k in range(mask.all()[0]):
        for j in range(mask.all()[1]):
          for i in range(mask.all()[2]):
            d = ((i - x0) / sx)**2 + ((j - y0) / sy)**2 + ((k - z0) / sz)**2
            if d <= 1.0:
              mask[k,j,i] = MaskCode.Valid | MaskCode.Foreground
            else:
              mask[k,j,i] = MaskCode.Valid | MaskCode.Background# | MaskCode.Foreground

    sbox = refl.shoebox

    # reflection itself, including setting the peak region if we're doing that

    counts_true = 0
    for j in range(params.counts):
      _x = random.gauss(0, params.spot_size.x)
      _y = random.gauss(0, params.spot_size.y)
      _z = random.gauss(0, params.spot_size.z)

      Rxyz = rotation * matrix.col((_x, _y, _z)).elems

      x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2)
      y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2)
      z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2)

      if x < 0 or x >= params.shoebox_size.x:
        continue
      if y < 0 or y >= params.shoebox_size.y:
        continue
      if z < 0 or z >= params.shoebox_size.z:
        continue
      sbox[z, y, x] += 1
      counts_true += 1
      if params.pixel_mask == 'precise':
        mask[z, y, x] = mask_peak

    refl.intensity = counts_true

    # background:flat; FIXME can replace this with Poissonian added random
    # number, which will give the same answer...

    for j in range(params.background * len(sbox)):
      x = random.randint(0, params.shoebox_size.x - 1)
      y = random.randint(0, params.shoebox_size.y - 1)
      z = random.randint(0, params.shoebox_size.z - 1)
      sbox[z, y, x] += 1

  p.finished('Generating %d shoeboxes' % params.nrefl)

  return rlist

def background_xds(rlist):
  from dials.algorithms.background import XdsSubtractor
  background = XdsSubtractor()
  background(None, None, rlist)
  return

def background_inclined(rlist):
  from dials.algorithms.background import InclinedSubtractor
  background = InclinedSubtractor()
  background(None, None, rlist)
  return

def integrate_3d_summation(rlist):
  from dials.algorithms.integration import Summation3d
  integration = Summation3d()
  integration(None, None, rlist)
  return

def main(params):
  rlist = simple_gaussian_spots(params)
  correct_intensities = [r.intensity for r in rlist]
  for r in rlist:
    r.intensity = 0

  if params.background_method == 'xds':
    background_xds(rlist)
  elif params.background_method == 'mosflm':
    assert(params.pixel_mask != 'all')
    background_inclined(rlist)

  integrate_3d_summation(rlist)

  integrated_intensities = [r.intensity for r in rlist]

  # now scan through the reflection list and find those where the integration
  # gave an apparently duff answer i.e. outside of 3 sigma from correct value

  from dials.model.data import ReflectionList
  import math

  overestimates = ReflectionList()
  underestimates = ReflectionList()

  for j, (c, i) in enumerate(zip(correct_intensities, integrated_intensities)):
    sigma = math.sqrt(c)
    if math.fabs(c - i) < 3 * sigma:
      continue
    if i > params.counts:
      overestimates.append(rlist[j])
    else:
      underestimates.append(rlist[j])

  print '%d overestimates, %d underestimates' % (len(overestimates),
                                                 len(underestimates))

  # now pickle these, perhaps

  import cPickle as pickle

  if params.output.under:
    pickle.dump(underestimates, open(params.output.under, 'w'))
  if params.output.over:
    pickle.dump(overestimates, open(params.output.over, 'w'))
  if params.output.all:
    pickle.dump(rlist, open(params.output.all, 'w'))

if __name__ == '__main__':
  import sys

  # FIXME add Phil parameters for
  # nrefl - done
  # nproc - done
  # size of box - done x y z
  # width of spot - done x y z
  # rotation of spot density - not done
  # background plane method with B(x, y) = c + ax + by; don't know how to make
  #                                                   ; random dist. like this
  # counts
  # background
  # background method
  # integration method
  # set proper background mask or no
  # set static mask

  working_phil = master_phil
  for arg in sys.argv[1:]:
    working_phil = working_phil.fetch(parse(arg))
  main(working_phil.extract())
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import platform
import signal
import shutil
import time
import curses
# from PIL import Image

if platform.system() == "Darwin":
    # use mock classes
    from mockcamera import PiCamera
else:
    # assume we're on a raspberry pi
    from picamera import PiCamera

# create access to camera
camera = PiCamera()

# create overlays
# img0 = Image.open('recording.png')
# pad0 = Image.new('RGB', (
#     ((img0.size[0] + 31) // 32) * 32,
#     ((img0.size[1] + 15) // 16) * 16))
# pad0.paste(img0, (0, 0))
# o0 = camera.add_overlay(pad0.tostring(), size=img0.size)
# o0.layer = 3
# o0.alpha = 0

# img1 = Image.open('not-recording.png')
# pad1 = Image.new('RGB', (
#     ((img1.size[0] + 31) // 32) * 32,
#     ((img1.size[1] + 15) // 16) * 16))
# pad1.paste(img1, (0, 0))
# o1 = camera.add_overlay(pad1.tostring(), size=img1.size)
# o1.layer = 4
# o1.alpha = 0

# create list of properties to display
properties = [
    "analog_gain",
    "annotate_text",
    "annotate_text_size",
    # "awb_gains",
    "awb_mode",
    "brightness",
    "color_effects",
    "contrast",
    "crop",
    "digital_gain",
    "drc_strength",
    "exposure_compensation",
    "exposure_mode",
    "exposure_speed",
    "flash_mode",
    "framerate",
    "framerate_delta",
    "hflip",
    "image_denoise",
    "image_effect",
    "iso",
    # the follwing is a read-only boolean property and it is not available on
    # the Pi 3
    # "led",
    "meter_mode",
    "resolution",
    "rotation",
    "saturation",
    "sensor_mode",
    "sharpness",
    "shutter_speed",
    "vflip",
    "video_denoise",
    "video_stabilization",
    "zoom"
]

# get maximum property name length for padding during display
max_len = 0

for property in properties:
    property_len = len(property)
    if max_len < property_len:
        max_len = property_len

def update_screen(stdscr):
    global max_len, properties
    row = 0
    col = 0
    max_col = 0

    # clear screen
    stdscr.clear()

    # display properties
    for key in properties:
        # just in case we have an invalid property, wrap this iteration in
        # a try/except statement
        try:
            # display key/value on screen
            # NOTE: we probably need to handle the case where the current
            # string cannot fit in the current row
            value = getattr(camera, key)
            string = key.rjust(max_len) + ": " + str(value)
            stdscr.addstr(row, col, string)

            # update max_col
            current_end_col = col + len(string)

            if current_end_col > max_col:
                max_col = current_end_col

            # update current row
            row = row + 1

            if row >= curses.LINES:
                row = 0
                col = max_col + 2

                # we're off of the display
                if col >= curses.COLS:
                    break
        except:
            print("An exception occurred", file=sys.stderr)

    # move the cursor to a sensible location and update the screen
    stdscr.move(curses.LINES - 1, 0)
    stdscr.refresh()

def sigwinch_handler(n, frame):
    # based on example 5 at
    # http://www.programcreek.com/python/example/9546/curses.KEY_RESIZE
    size = shutil.get_terminal_size()
    curses.resizeterm(size.lines, size.columns)
    curses.ungetch(curses.KEY_RESIZE)

# display all current property values
def main(stdscr):
    preview = False
    recording = False

    while True:
        update_screen(stdscr)

        ch = stdscr.getch()

        # TODO: process keys here
        if ch == ord("q"):
            break
        elif ch == ord("-"):
            if camera.brightness > 0:
                camera.brightness -= 1
        elif ch == ord("="):
            if camera.brightness < 100:
                camera.brightness += 1
        elif ch == ord("_"):
            if camera.contrast > -100:
                camera.contrast -= 1
        elif ch == ord("+"):
            if camera.contrast < 100:
                camera.contrast += 1
        elif ch == ord("p"):
            preview = not preview
            if preview:
                camera.start_preview()
            else:
                camera.stop_preview()
        elif ch == ord("r"):
            recording = not recording
            if recording:
                timestamp = int(time.time())
                filename = "g2x-{}.h264".format(timestamp)
                camera.start_recording(filename)
                # o0.alpha = 255
                # o1.alpha = 0
                camera.annotate_text = "Recording"
            else:
                camera.stop_recording()
                # o0.alpha = 0
                # o1.alpha = 255
                camera.annotate_text = ""
        else:
            print("Unhandled key: " + ch, file=sys.stderr)

try:
    signal.signal(signal.SIGWINCH, sigwinch_handler)
    curses.wrapper(main)
finally:
    camera.close()
<commit_msg>Cast character value to string<commit_after>#!/usr/bin/env python3

import sys
import platform
import signal
import shutil
import time
import curses
# from PIL import Image

if platform.system() == "Darwin":
    # use mock classes
    from mockcamera import PiCamera
else:
    # assume we're on a raspberry pi
    from picamera import PiCamera

# create access to camera
camera = PiCamera()

# create overlays
# img0 = Image.open('recording.png')
# pad0 = Image.new('RGB', (
#     ((img0.size[0] + 31) // 32) * 32,
#     ((img0.size[1] + 15) // 16) * 16))
# pad0.paste(img0, (0, 0), img0)
# o0 = camera.add_overlay(pad0.tostring(), size=img0.size)
# o0.layer = 3
# o0.alpha = 0

# img1 = Image.open('not-recording.png')
# pad1 = Image.new('RGB', (
#     ((img1.size[0] + 31) // 32) * 32,
#     ((img1.size[1] + 15) // 16) * 16))
# pad1.paste(img1, (0, 0), img1)
# o1 = camera.add_overlay(pad1.tostring(), size=img1.size)
# o1.layer = 4
# o1.alpha = 0

# create list of properties to display
properties = [
    "analog_gain",
    "annotate_text",
    "annotate_text_size",
    # "awb_gains",
    "awb_mode",
    "brightness",
    "color_effects",
    "contrast",
    "crop",
    "digital_gain",
    "drc_strength",
    "exposure_compensation",
    "exposure_mode",
    "exposure_speed",
    "flash_mode",
    "framerate",
    "framerate_delta",
    "hflip",
    "image_denoise",
    "image_effect",
    "iso",
    # the follwing is a read-only boolean property and it is not available on
    # the Pi 3
    # "led",
    "meter_mode",
    "resolution",
    "rotation",
    "saturation",
    "sensor_mode",
    "sharpness",
    "shutter_speed",
    "vflip",
    "video_denoise",
    "video_stabilization",
    "zoom"
]

# get maximum property name length for padding during display
max_len = 0

for property in properties:
    property_len = len(property)
    if max_len < property_len:
        max_len = property_len

def update_screen(stdscr):
    global max_len, properties
    row = 0
    col = 0
    max_col = 0

    # clear screen
    stdscr.clear()

    # display properties
    for key in properties:
        # just in case we have an invalid property, wrap this iteration in
        # a try/except statement
        try:
            # display key/value on screen
            # NOTE: we probably need to handle the case where the current
            # string cannot fit in the current row
            value = getattr(camera, key)
            string = key.rjust(max_len) + ": " + str(value)
            stdscr.addstr(row, col, string)

            # update max_col
            current_end_col = col + len(string)

            if current_end_col > max_col:
                max_col = current_end_col

            # update current row
            row = row + 1

            if row >= curses.LINES:
                row = 0
                col = max_col + 2

                # we're off of the display
                if col >= curses.COLS:
                    break
        except:
            print("An exception occurred", file=sys.stderr)

    # move the cursor to a sensible location and update the screen
    stdscr.move(curses.LINES - 1, 0)
    stdscr.refresh()

def sigwinch_handler(n, frame):
    # based on example 5 at
    # http://www.programcreek.com/python/example/9546/curses.KEY_RESIZE
    size = shutil.get_terminal_size()
    curses.resizeterm(size.lines, size.columns)
    curses.ungetch(curses.KEY_RESIZE)

# display all current property values
def main(stdscr):
    preview = False
    recording = False

    while True:
        update_screen(stdscr)

        ch = stdscr.getch()

        # TODO: process keys here
        if ch == ord("q"):
            break
        elif ch == ord("-"):
            if camera.brightness > 0:
                camera.brightness -= 1
        elif ch == ord("="):
            if camera.brightness < 100:
                camera.brightness += 1
        elif ch == ord("_"):
            if camera.contrast > -100:
                camera.contrast -= 1
        elif ch == ord("+"):
            if camera.contrast < 100:
                camera.contrast += 1
        elif ch == ord("p"):
            preview = not preview
            if preview:
                camera.start_preview()
            else:
                camera.stop_preview()
        elif ch == ord("r"):
            recording = not recording
            if recording:
                timestamp = int(time.time())
                filename = "g2x-{}.h264".format(timestamp)
                camera.start_recording(filename)
                # o0.alpha = 255
                # o1.alpha = 0
                camera.annotate_text = "Recording"
            else:
                camera.stop_recording()
                # o0.alpha = 0
                # o1.alpha = 255
                camera.annotate_text = ""
        else:
            print("Unhandled key: " + str(ch), file=sys.stderr)

try:
    signal.signal(signal.SIGWINCH, sigwinch_handler)
    curses.wrapper(main)
finally:
    camera.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

DESC = """EPP script to copy 'concentration' and 'concentration unit' for each
sample sample in the 'Qubit Result File' to the 'Concentration' and 'Conc. Units'
fields of the output analytes of the process.

Warnings are generated to the user and stored in regular log file wich allso
contains regular execution information in the folowing cases:

1) missing row names (samples) in file
2) duplicated row names (samples)
3) missing value (concentrations)
4) values found but for some reason are not successfully copied:

Can be executed in the background or triggered by a user pressing a "blue button".

Written by Maya Brandi
"""

import os
import sys
import logging
import numpy as np

from argparse import ArgumentParser
from requests import HTTPError
from genologics.lims import Lims
from genologics.config import BASEURI,USERNAME,PASSWORD
from genologics.entities import Process
from genologics.epp import EppLogger
from genologics.epp import ReadResultFiles
from genologics.epp import set_field

def main(lims, pid, epp_logger):
    process = Process(lims,id = pid)
    sample_names = map(lambda a: a.name, process.analytes()[0])
    target_files = process.result_files()
    file_handler = ReadResultFiles(process)
    files = file_handler.shared_files['Qubit Result File']
    qubit_result_file = file_handler.format_file(files,
                                                 name = 'Qubit Result File',
                                                 first_header = 'Sample',
                                                 find_keys = sample_names)
    missing_samples = 0
    low_conc = 0
    bad_formated = 0
    abstract = []
    udfs = dict(process.udf.items())
    if udfs.has_key("Minimum required concentration (ng/ul)"):
        min_conc = udfs["Minimum required concentration (ng/ul)"]
    else:
        min_conc = None
        abstract.append("Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!")
    for target_file in target_files:
        print '*****'
        print target_file
        print target_file.udf.container
        print target_file.files
        print target_filelocation
        print target_file.root
        print dir(target_file)
        sample = target_file.samples[0].name
        print target_file.samples
        if qubit_result_file.has_key(sample):
            sample_mesurements = qubit_result_file[sample]
            if "Sample Concentration" in sample_mesurements.keys():
                conc, unit = sample_mesurements["Sample Concentration"]
                if conc == 'Out Of Range':
                    target_file.qc_flag = "FAILED"
                elif conc.replace('.','').isdigit():
                    conc = float(conc)
                    if unit == 'ng/mL':
                        conc = np.true_divide(conc, 1000)
                    if min_conc:
                        if conc < min_conc:
                            target_file.qc_flag = "FAILED"
                            low_conc +=1
                        else:
                            target_file.qc_flag = "PASSED"
                    target_file.udf['Concentration'] = conc
                    target_file.udf['Conc. Units'] = 'ng/ul'
                else:
                    bad_formated += 1
                set_field(target_file)
        else:
            missing_samples += 1

    if low_conc:
        abstract.append('{0}/{1} samples have low concentration.'.format(low_conc, len(target_files)))
    if missing_samples:
        abstract.append('{0}/{1} samples are missing in Qubit Result File.'.format(missing_samples, len(target_files)))
    if bad_formated:
        abstract.append('There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'.format(bad_formated))

    print >> sys.stderr, ' '.join(abstract)

if __name__ == "__main__":
    parser = ArgumentParser(description=DESC)
    parser.add_argument('--pid', default = '24-38458', dest = 'pid',
                        help='Lims id for current Process')
    parser.add_argument('--log', dest = 'log',
                        help=('File name for standard log file, '
                              'for runtime information and problems.'))

    args = parser.parse_args()

    lims = Lims(BASEURI,USERNAME,PASSWORD)
    lims.check_version()

    with EppLogger(log_file=args.log, lims=lims, prepend=True) as epp_logger:
        main(lims, args.pid, epp_logger)

<commit_msg>debugging missing pool names<commit_after>#!/usr/bin/env python

DESC = """EPP script to copy 'concentration' and 'concentration unit' for each
sample sample in the 'Qubit Result File' to the 'Concentration' and 'Conc. Units'
fields of the output analytes of the process.

Warnings are generated to the user and stored in regular log file wich allso
contains regular execution information in the folowing cases:

1) missing row names (samples) in file
2) duplicated row names (samples)
3) missing value (concentrations)
4) values found but for some reason are not successfully copied:

Can be executed in the background or triggered by a user pressing a "blue button".

Written by Maya Brandi
"""

import os
import sys
import logging
import numpy as np

from argparse import ArgumentParser
from requests import HTTPError
from genologics.lims import Lims
from genologics.config import BASEURI,USERNAME,PASSWORD
from genologics.entities import Process
from genologics.epp import EppLogger
from genologics.epp import ReadResultFiles
from genologics.epp import set_field

def main(lims, pid, epp_logger):
    process = Process(lims,id = pid)
    sample_names = map(lambda a: a.name, process.analytes()[0])
    target_files = process.result_files()
    file_handler = ReadResultFiles(process)
    files = file_handler.shared_files['Qubit Result File']
    qubit_result_file = file_handler.format_file(files,
                                                 name = 'Qubit Result File',
                                                 first_header = 'Sample',
                                                 find_keys = sample_names)
    missing_samples = 0
    low_conc = 0
    bad_formated = 0
    abstract = []
    udfs = dict(process.udf.items())
    if udfs.has_key("Minimum required concentration (ng/ul)"):
        min_conc = udfs["Minimum required concentration (ng/ul)"]
    else:
        min_conc = None
        abstract.append("Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!")
    for target_file in target_files:
        print '*****'
        print target_file
        print target_file.container
        print target_file.files
        print target_file.location
        print target_file.root
        print dir(target_file)
        sample = target_file.samples[0].name
        print target_file.samples
        if qubit_result_file.has_key(sample):
            sample_mesurements = qubit_result_file[sample]
            if "Sample Concentration" in sample_mesurements.keys():
                conc, unit = sample_mesurements["Sample Concentration"]
                if conc == 'Out Of Range':
                    target_file.qc_flag = "FAILED"
                elif conc.replace('.','').isdigit():
                    conc = float(conc)
                    if unit == 'ng/mL':
                        conc = np.true_divide(conc, 1000)
                    if min_conc:
                        if conc < min_conc:
                            target_file.qc_flag = "FAILED"
                            low_conc +=1
                        else:
                            target_file.qc_flag = "PASSED"
                    target_file.udf['Concentration'] = conc
                    target_file.udf['Conc. Units'] = 'ng/ul'
                else:
                    bad_formated += 1
                set_field(target_file)
        else:
            missing_samples += 1

    if low_conc:
        abstract.append('{0}/{1} samples have low concentration.'.format(low_conc, len(target_files)))
    if missing_samples:
        abstract.append('{0}/{1} samples are missing in Qubit Result File.'.format(missing_samples, len(target_files)))
    if bad_formated:
        abstract.append('There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'.format(bad_formated))

    print >> sys.stderr, ' '.join(abstract)

if __name__ == "__main__":
    parser = ArgumentParser(description=DESC)
    parser.add_argument('--pid', default = '24-38458', dest = 'pid',
                        help='Lims id for current Process')
    parser.add_argument('--log', dest = 'log',
                        help=('File name for standard log file, '
                              'for runtime information and problems.'))

    args = parser.parse_args()

    lims = Lims(BASEURI,USERNAME,PASSWORD)
    lims.check_version()

    with EppLogger(log_file=args.log, lims=lims, prepend=True) as epp_logger:
        main(lims, args.pid, epp_logger)

<|endoftext|>