<commit_before># This avoids a plugin dependency issue with pytest-forked/xdist:
# https://github.com/pytest-dev/pytest/issues/935
import pytest

#: Maximum time an individual test case (function) can run
MAX_CASE_RUN_SECS = 30

@pytest.fixture
def animation_fc(fc):
    fc.sr_animation_run = animation_run
    return fc

def animation_run(fc, sim_name, compute_model, reports, **kwargs):
    from pykern import pkconfig
    from pykern import pkunit
    from pykern.pkcollections import PKDict
    from pykern.pkdebug import pkdp, pkdlog
    from sirepo import srunit
    import re
    import time

    data = fc.sr_sim_data(sim_name)
    run = fc.sr_run_sim(data, compute_model, **kwargs)
    for r, a in reports.items():
        if 'runSimulation' in a:
            f = fc.sr_run_sim(data, r)
            for k, v in a.items():
                m = re.search('^expect_(.+)', k)
                if m:
                    pkunit.pkre(
                        v(i) if callable(v) else v,
                        str(f.get(m.group(1))),
                    )
            continue
        if 'frame_index' in a:
            c = [a.get('frame_index')]
        else:
            c = range(run.get(a.get('frame_count_key', 'frameCount')))
            assert c, \
                'frame_count_key={} or frameCount={} is zero'.format(
                    a.get('frame_count_key'), a.get('frameCount'),
                )
        pkdlog('frameReport={} count={}', r, c)
        import sirepo.sim_data

        s = sirepo.sim_data.get_class(fc.sr_sim_type)
        for i in c:
            pkdlog('frameIndex={} frameCount={}', i, run.get('frameCount'))
            f = fc.sr_get_json(
                'simulationFrame',
                PKDict(frame_id=s.frame_id(data, run, r, i)),
            )
            for k, v in a.items():
                m = re.search('^expect_(.+)', k)
                if m:
                    pkunit.pkre(
                        v(i) if callable(v) else v,
                        str(f.get(m.group(1))),
                    )

@pytest.fixture
def auth_fc(auth_fc_module):
    # set the sentinel
    auth_fc_module.cookie_jar.clear()
    auth_fc_module.sr_get_root()
    auth_fc_module.sr_email_confirm = email_confirm
    return auth_fc_module

@pytest.fixture
def auth_fc_module(request):
    from sirepo import srunit
    from pykern.pkcollections import PKDict

    cfg = PKDict(
        SIREPO_AUTH_EMAIL_FROM_EMAIL='x',
        SIREPO_AUTH_EMAIL_FROM_NAME='x',
        SIREPO_AUTH_EMAIL_SMTP_PASSWORD='x',
        SIREPO_AUTH_EMAIL_SMTP_SERVER='dev',
        SIREPO_AUTH_EMAIL_SMTP_USER='x',
        SIREPO_AUTH_GITHUB_CALLBACK_URI='/uri',
        SIREPO_AUTH_GITHUB_KEY='key',
        SIREPO_AUTH_GITHUB_SECRET='secret',
        SIREPO_AUTH_GUEST_EXPIRY_DAYS='1',
        SIREPO_AUTH_METHODS='email:guest',
    )
    if 'email3_test' in str(request.fspath):
        cfg.SIREPO_AUTH_METHODS += ':github'
    else:
        cfg.SIREPO_AUTH_DEPRECATED_METHODS = 'github'
    return srunit.flask_client(cfg=cfg)

def email_confirm(fc, resp, display_name=None):
    import re
    from pykern.pkcollections import PKDict

    fc.sr_get(resp.url)
    m = re.search(r'/(\w+)$', resp.url)
    assert m
    r = PKDict(token=m.group(1))
    if display_name:
        r.displayName = display_name
    fc.sr_post(
        resp.url,
        r,
        raw_response=True,
    )

@pytest.fixture(scope='function')
def fc(request, fc_module):
    """Flask client based logged in to specific code of test

    Defaults to myapp.
    """
    import sirepo.srunit

    c = _sim_type(request)
    if fc_module.sr_uid:
        if fc_module.sr_sim_type != c:
            fc_module.sr_get_root(sim_type=c)
    else:
        fc_module.sr_login_as_guest(sim_type=c)
    return fc_module

@pytest.fixture(scope='module')
def fc_module(request):
    import sirepo.srunit
    p, fc = _job_supervisor_start(request)
    if p:
        yield fc
        p.terminate()
        p.wait()
    else:
        yield sirepo.srunit.flask_client()

@pytest.fixture
def import_req(request):
    import flask

    flask.g = {}

    def w(path):
        import sirepo.http_request
        req = sirepo.http_request.parse_params(
            filename=path.basename,
            folder='/import_test',
            template=True,
            type=_sim_type(request),
        )
        # Supports read() for elegant and zgoubi
        req.file_stream = path
        return req

    return w

def pytest_collection_modifyitems(session, config, items):
    """Restrict which tests are running"""
    import importlib
    import sirepo.feature_config
    from pykern.pkcollections import PKDict
    import os

    s = PKDict(
        elegant='sdds',
        srw='srwl_bl',
        synergia='synergia',
        warp='warp',
        zgoubi='zgoubi',
    )
    all_codes = set(sirepo.feature_config.ALL_CODES)
    codes = set()
    import_fail = PKDict()
    res = set()
    skip_list = os.environ.get('SIREPO_PYTEST_SKIP', '').split(':')
    for i in items:
        if i.fspath.purebasename in skip_list:
            i.add_marker(pytest.mark.skip(reason="SIREPO_PYTEST_SKIP"))
            continue
        c = [x for x in all_codes if x in i.name]
        if not c:
            continue
        c = c[0]
        if c in import_fail:
            i.add_marker(import_fail[c])
            continue
        if c not in all_codes:
            i.add_marker(
                pytest.mark.skip(
                    reason='skipping code={} not in codes={}'.format(c, all_codes),
                ),
            )
            continue
        try:
            m = s.get(c)
            if m:
                importlib.import_module(m)
        except Exception:
            import_fail[c] = pytest.mark.skip(reason='unable to import={}'.format(m))
            i.add_marker(import_fail[c])
            continue
        codes.add(c)
    if not codes:
        return
    codes.add('myapp')
    import sirepo.srunit
    sirepo.srunit.CONFTEST_ALL_CODES = ':'.join(codes)

@pytest.hookimpl(tryfirst=True)
def pytest_runtest_protocol(item, *args, **kwargs):
    import signal
    from pykern import pkunit

    def _timeout(*args, **kwargs):
        pkunit.pkfail('MAX_CASE_RUN_SECS={} exceeded', MAX_CASE_RUN_SECS)

    # Seems to be the only way to get the module under test
    m = item._request.module
    is_new = m != pkunit.module_under_test

    if is_new:
        signal.signal(signal.SIGALRM, _timeout)
    pkunit.module_under_test = m
    signal.alarm(MAX_CASE_RUN_SECS)
    if is_new:
        from pykern import pkio
        pkio.unchecked_remove(pkunit.work_dir())

@pytest.hookimpl(tryfirst=True)
def pytest_configure(config):
    config._parser.parse_setoption(
        # run each test file in a separate process
        # the native trace works better, e.g. for emacs
        ['--tb=native'],
        config.option,
        namespace=config.option,
    )

def _job_supervisor_check(env):
    import sirepo.job
    import socket
    import subprocess

    try:
        o = subprocess.check_output(
            ['pyenv', 'exec', 'sirepo', 'job_supervisor', '--help'],
            env=env,
            stderr=subprocess.STDOUT,
        )
    except subprocess.CalledProcessError as e:
        from pykern.pkdebug import pkdlog

        pkdlog('job_supervisor --help exit={} output={}', e.returncode, e.output)
        raise
    assert 'usage: sirepo job_supervisor' in str(o)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.bind((sirepo.job.DEFAULT_IP, int(sirepo.job.DEFAULT_PORT)))
    except Exception:
        raise AssertionError(
            'job_supervisor still running on port={}'.format(sirepo.job.DEFAULT_PORT),
        )
    finally:
        s.close()

def _job_supervisor_setup():
    """setup the supervisor"""
    import os
    from pykern.pkcollections import PKDict

    env = PKDict()
    for k, v in os.environ.items():
        if ('PYENV' in k or 'PYTHON' in k):
            continue
        if k in ('PATH', 'LD_LIBRARY_PATH'):
            v2 = []
            for x in v.split(':'):
                if x and 'py2' not in x:
                    v2.append(x)
            v = ':'.join(v2)
        env[k] = v
    cfg = PKDict(
        PYKERN_PKDEBUG_WANT_PID_TIME='1',
        SIREPO_FEATURE_CONFIG_JOB='1',
    )
    env.pkupdate(
        PYENV_VERSION='py3',
        PYTHONUNBUFFERED='1',
        **cfg
    )

    import sirepo.srunit

    fc = sirepo.srunit.flask_client(cfg=cfg)
    import sirepo.srdb
    env['SIREPO_SRDB_ROOT'] = str(sirepo.srdb.root())
    _job_supervisor_check(env)
    return (env, fc)

def _job_supervisor_start(request):
    import os
    if not os.environ.get('SIREPO_FEATURE_CONFIG_JOB'):
        return None, None

    from pykern import pkunit
    from pykern.pkcollections import PKDict
    import sirepo.job
    import subprocess
    import time

    env, fc = _job_supervisor_setup()
    p = subprocess.Popen(
        ['pyenv', 'exec', 'sirepo', 'job_supervisor'],
        env=env,
    )
    for i in range(30):
        r = fc.sr_post('jobSupervisorPing', PKDict(simulationType=fc.SR_SIM_TYPE_DEFAULT))
        if r.state == 'ok':
            break
        time.sleep(.1)
    else:
        pkunit.pkfail('could not connect to {}', sirepo.job.SERVER_PING_ABS_URI)
    return p, fc

def _sim_type(request):
    import sirepo.feature_config

    for c in sirepo.feature_config.ALL_CODES:
        if c in request.function.func_name or c in str(request.fspath):
            return c
    return 'myapp'
<commit_msg>check for  SIREPO_FEATURE_CONFIG_JOB explicitly<commit_after># This avoids a plugin dependency issue with pytest-forked/xdist:
# https://github.com/pytest-dev/pytest/issues/935
import pytest

#: Maximum time an individual test case (function) can run
MAX_CASE_RUN_SECS = 30

@pytest.fixture
def animation_fc(fc):
    fc.sr_animation_run = animation_run
    return fc

def animation_run(fc, sim_name, compute_model, reports, **kwargs):
    from pykern import pkconfig
    from pykern import pkunit
    from pykern.pkcollections import PKDict
    from pykern.pkdebug import pkdp, pkdlog
    from sirepo import srunit
    import re
    import time

    data = fc.sr_sim_data(sim_name)
    run = fc.sr_run_sim(data, compute_model, **kwargs)
    for r, a in reports.items():
        if 'runSimulation' in a:
            f = fc.sr_run_sim(data, r)
            for k, v in a.items():
                m = re.search('^expect_(.+)', k)
                if m:
                    pkunit.pkre(
                        v(i) if callable(v) else v,
                        str(f.get(m.group(1))),
                    )
            continue
        if 'frame_index' in a:
            c = [a.get('frame_index')]
        else:
            c = range(run.get(a.get('frame_count_key', 'frameCount')))
            assert c, \
                'frame_count_key={} or frameCount={} is zero'.format(
                    a.get('frame_count_key'), a.get('frameCount'),
                )
        pkdlog('frameReport={} count={}', r, c)
        import sirepo.sim_data

        s = sirepo.sim_data.get_class(fc.sr_sim_type)
        for i in c:
            pkdlog('frameIndex={} frameCount={}', i, run.get('frameCount'))
            f = fc.sr_get_json(
                'simulationFrame',
                PKDict(frame_id=s.frame_id(data, run, r, i)),
            )
            for k, v in a.items():
                m = re.search('^expect_(.+)', k)
                if m:
                    pkunit.pkre(
                        v(i) if callable(v) else v,
                        str(f.get(m.group(1))),
                    )

@pytest.fixture
def auth_fc(auth_fc_module):
    # set the sentinel
    auth_fc_module.cookie_jar.clear()
    auth_fc_module.sr_get_root()
    auth_fc_module.sr_email_confirm = email_confirm
    return auth_fc_module

@pytest.fixture
def auth_fc_module(request):
    from sirepo import srunit
    from pykern.pkcollections import PKDict

    cfg = PKDict(
        SIREPO_AUTH_EMAIL_FROM_EMAIL='x',
        SIREPO_AUTH_EMAIL_FROM_NAME='x',
        SIREPO_AUTH_EMAIL_SMTP_PASSWORD='x',
        SIREPO_AUTH_EMAIL_SMTP_SERVER='dev',
        SIREPO_AUTH_EMAIL_SMTP_USER='x',
        SIREPO_AUTH_GITHUB_CALLBACK_URI='/uri',
        SIREPO_AUTH_GITHUB_KEY='key',
        SIREPO_AUTH_GITHUB_SECRET='secret',
        SIREPO_AUTH_GUEST_EXPIRY_DAYS='1',
        SIREPO_AUTH_METHODS='email:guest',
    )
    if 'email3_test' in str(request.fspath):
        cfg.SIREPO_AUTH_METHODS += ':github'
    else:
        cfg.SIREPO_AUTH_DEPRECATED_METHODS = 'github'
    return srunit.flask_client(cfg=cfg)

def email_confirm(fc, resp, display_name=None):
    import re
    from pykern.pkcollections import PKDict

    fc.sr_get(resp.url)
    m = re.search(r'/(\w+)$', resp.url)
    assert m
    r = PKDict(token=m.group(1))
    if display_name:
        r.displayName = display_name
    fc.sr_post(
        resp.url,
        r,
        raw_response=True,
    )

@pytest.fixture(scope='function')
def fc(request, fc_module):
    """Flask client based logged in to specific code of test

    Defaults to myapp.
    """
    import sirepo.srunit

    c = _sim_type(request)
    if fc_module.sr_uid:
        if fc_module.sr_sim_type != c:
            fc_module.sr_get_root(sim_type=c)
    else:
        fc_module.sr_login_as_guest(sim_type=c)
    return fc_module

@pytest.fixture(scope='module')
def fc_module(request):
    import sirepo.srunit
    p, fc = _job_supervisor_start(request)
    if p:
        yield fc
        p.terminate()
        p.wait()
    else:
        yield sirepo.srunit.flask_client()

@pytest.fixture
def import_req(request):
    import flask

    flask.g = {}

    def w(path):
        import sirepo.http_request
        req = sirepo.http_request.parse_params(
            filename=path.basename,
            folder='/import_test',
            template=True,
            type=_sim_type(request),
        )
        # Supports read() for elegant and zgoubi
        req.file_stream = path
        return req

    return w

def pytest_collection_modifyitems(session, config, items):
    """Restrict which tests are running"""
    import importlib
    import sirepo.feature_config
    from pykern.pkcollections import PKDict
    import os

    s = PKDict(
        elegant='sdds',
        srw='srwl_bl',
        synergia='synergia',
        warp='warp',
        zgoubi='zgoubi',
    )
    all_codes = set(sirepo.feature_config.ALL_CODES)
    codes = set()
    import_fail = PKDict()
    res = set()
    skip_list = os.environ.get('SIREPO_PYTEST_SKIP', '').split(':')
    for i in items:
        if i.fspath.purebasename in skip_list:
            i.add_marker(pytest.mark.skip(reason="SIREPO_PYTEST_SKIP"))
            continue
        c = [x for x in all_codes if x in i.name]
        if not c:
            continue
        c = c[0]
        if c in import_fail:
            i.add_marker(import_fail[c])
            continue
        if c not in all_codes:
            i.add_marker(
                pytest.mark.skip(
                    reason='skipping code={} not in codes={}'.format(c, all_codes),
                ),
            )
            continue
        try:
            m = s.get(c)
            if m:
                importlib.import_module(m)
        except Exception:
            import_fail[c] = pytest.mark.skip(reason='unable to import={}'.format(m))
            i.add_marker(import_fail[c])
            continue
        codes.add(c)
    if not codes:
        return
    codes.add('myapp')
    import sirepo.srunit
    sirepo.srunit.CONFTEST_ALL_CODES = ':'.join(codes)

@pytest.hookimpl(tryfirst=True)
def pytest_runtest_protocol(item, *args, **kwargs):
    import signal
    from pykern import pkunit

    def _timeout(*args, **kwargs):
        pkunit.pkfail('MAX_CASE_RUN_SECS={} exceeded', MAX_CASE_RUN_SECS)

    # Seems to be the only way to get the module under test
    m = item._request.module
    is_new = m != pkunit.module_under_test

    if is_new:
        signal.signal(signal.SIGALRM, _timeout)
    pkunit.module_under_test = m
    signal.alarm(MAX_CASE_RUN_SECS)
    if is_new:
        from pykern import pkio
        pkio.unchecked_remove(pkunit.work_dir())

@pytest.hookimpl(tryfirst=True)
def pytest_configure(config):
    config._parser.parse_setoption(
        # run each test file in a separate process
        # the native trace works better, e.g. for emacs
        ['--tb=native'],
        config.option,
        namespace=config.option,
    )

def _job_supervisor_check(env):
    import sirepo.job
    import socket
    import subprocess

    try:
        o = subprocess.check_output(
            ['pyenv', 'exec', 'sirepo', 'job_supervisor', '--help'],
            env=env,
            stderr=subprocess.STDOUT,
        )
    except subprocess.CalledProcessError as e:
        from pykern.pkdebug import pkdlog

        pkdlog('job_supervisor --help exit={} output={}', e.returncode, e.output)
        raise
    assert 'usage: sirepo job_supervisor' in str(o)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.bind((sirepo.job.DEFAULT_IP, int(sirepo.job.DEFAULT_PORT)))
    except Exception:
        raise AssertionError(
            'job_supervisor still running on port={}'.format(sirepo.job.DEFAULT_PORT),
        )
    finally:
        s.close()

def _job_supervisor_setup():
    """setup the supervisor"""
    import os
    from pykern.pkcollections import PKDict

    env = PKDict()
    for k, v in os.environ.items():
        if ('PYENV' in k or 'PYTHON' in k):
            continue
        if k in ('PATH', 'LD_LIBRARY_PATH'):
            v2 = []
            for x in v.split(':'):
                if x and 'py2' not in x:
                    v2.append(x)
            v = ':'.join(v2)
        env[k] = v
    cfg = PKDict(
        PYKERN_PKDEBUG_WANT_PID_TIME='1',
        SIREPO_FEATURE_CONFIG_JOB='1',
    )
    env.pkupdate(
        PYENV_VERSION='py3',
        PYTHONUNBUFFERED='1',
        **cfg
    )

    import sirepo.srunit

    fc = sirepo.srunit.flask_client(cfg=cfg)
    import sirepo.srdb
    env['SIREPO_SRDB_ROOT'] = str(sirepo.srdb.root())
    _job_supervisor_check(env)
    return (env, fc)

def _job_supervisor_start(request):
    import os
    if not os.environ.get('SIREPO_FEATURE_CONFIG_JOB') == '1':
        return None, None

    from pykern import pkunit
    from pykern.pkcollections import PKDict
    import sirepo.job
    import subprocess
    import time

    env, fc = _job_supervisor_setup()
    p = subprocess.Popen(
        ['pyenv', 'exec', 'sirepo', 'job_supervisor'],
        env=env,
    )
    for i in range(30):
        r = fc.sr_post('jobSupervisorPing', PKDict(simulationType=fc.SR_SIM_TYPE_DEFAULT))
        if r.state == 'ok':
            break
        time.sleep(.1)
    else:
        pkunit.pkfail('could not connect to {}', sirepo.job.SERVER_PING_ABS_URI)
    return p, fc

def _sim_type(request):
    import sirepo.feature_config

    for c in sirepo.feature_config.ALL_CODES:
        if c in request.function.func_name or c in str(request.fspath):
            return c
    return 'myapp'
<|endoftext|>


<commit_before>"""

Original EmailParser Code by Ian Lewis:
http://www.ianlewis.org/en/parsing-email-attachments-python
Licensed under MIT

"""
from datetime import datetime
import time
from StringIO import StringIO
from email.Header import decode_header
from email.Parser import Parser
from email.utils import parseaddr, parsedate_tz, getaddresses, decode_rfc2231
import imaplib

def get_unread_mails(host, port, user, password, ssl=True):
    klass = imaplib.IMAP4
    if ssl:
        klass = imaplib.IMAP4_SSL
    mail = klass(host, port)
    mail.login(user, password)
    try:
        status, count = mail.select('Inbox')
        typ, data = mail.search(None, 'UNSEEN')
        for num in data[0].split():
            status, data = mail.fetch(num, '(RFC822)')
            yield data[0][1]
    finally:
        mail.close()
        mail.logout()

def make_address(email, name=None):
    if name:
        return "%s <%s>" % (name, email)
    return email

class UnsupportedMailFormat(Exception):
    pass

class EmailParser(object):

    def parse_attachment(self, message_part):
        content_disposition = message_part.get("Content-Disposition", None)
        if content_disposition:
            dispositions = content_disposition.strip().split(";")
            if content_disposition and dispositions[0].lower() == "attachment":
                file_data = message_part.get_payload(decode=True)
                if file_data is None:
                    file_data = ""
                attachment = StringIO(file_data)
                attachment.content_type = message_part.get_content_type()
                attachment.size = len(file_data)
                attachment.name = None
                attachment.create_date = None
                attachment.mod_date = None
                attachment.read_date = None

                for param in dispositions[1:]:
                    name,value = param.split("=", 1)
                    name = name.lower().strip()
                    value = value.strip()
                    if value.startswith('"') and value.endswith('"'):
                        value = value[1:-1]
                    if name == "filename":
                        attachment.name = value
                    elif name == "create-date":
                        attachment.create_date = value  #TODO: datetime
                    elif name == "modification-date":
                        attachment.mod_date = value #TODO: datetime
                    elif name == "read-date":
                        attachment.read_date = value #TODO: datetime
                return attachment
        return None

    def parse_date(self, date):
        date = parsedate_tz(date)
        if date is None:
            return None
        return (datetime.fromtimestamp(time.mktime(date[:9])), date[9])

    def parse(self, content):
        p = Parser()
        msgobj = p.parsestr(content)
        if msgobj['Subject'] is not None:
            decodefrag = decode_header(msgobj['Subject'])
            subj_fragments = []
            for s , enc in decodefrag:
                if enc:
                    try:
                        s = unicode(s , enc)
                    except UnicodeDecodeError:
                        # desperate move here
                        try:
                            s = s.decode("latin1")
                        except:
                            pass
                subj_fragments.append(s)
            subject = ''.join(subj_fragments)
            subject = subject.replace('\n\t', " ")
        else:
            subject = None
        attachments = []
        body = None
        html = None
        for part in msgobj.walk():
            attachment = self.parse_attachment(part)
            if attachment:
                attachments.append(attachment)
            elif part.get_content_type() == "text/plain":
                if body is None:
                    body = ""
                charset = part.get_content_charset() or 'ascii'
                body += unicode(
                    part.get_payload(decode=True),
                    charset,
                    'replace').encode('utf8','replace')
            elif part.get_content_type() == "text/html":
                if html is None:
                    html = ""
                charset = part.get_content_charset() or 'ascii'
                html += unicode(
                    part.get_payload(decode=True),
                    charset,
                    'replace').encode('utf8','replace')
        tos = getaddresses(msgobj.get_all('To', []))
        tos.extend(getaddresses(msgobj.get_all('X-Original-To', [])))
        ccs = getaddresses(msgobj.get_all('Cc', []))
        resent_tos = getaddresses(msgobj.get_all('resent-to', []))
        resent_ccs = getaddresses(msgobj.get_all('resent-cc', []))
        date = self.parse_date(msgobj.get("Date"))
        return {
            'msgobj': msgobj,
            'date': date,
            'subject': subject,
            'body': body,
            'html': html,
            'from': parseaddr(msgobj.get('From')),
            'to': tos,
            'cc': ccs,
            'resent_to': resent_tos,
            'resent_cc': resent_ccs,
            'attachments': attachments
        }

 # uses the email flatten
 #        out_file = StringIO.StringIO()
 #        message_gen = Generator(out_file, mangle_from_=False, maxheaderlen=60)
 #        message_gen.flatten(message)
 #        message_text = out_file.getvalue()

 #        fixes mime encoding issues (for display within html)
 #        clean_text = quopri.decodestring(message_text)

<commit_msg>Refactor parsing email body and content-dispositions<commit_after>"""

Original EmailParser Code by Ian Lewis:
http://www.ianlewis.org/en/parsing-email-attachments-python
Licensed under MIT

"""
from datetime import datetime
import time
from StringIO import StringIO
from email.Header import decode_header
from email.Parser import Parser
from email.utils import parseaddr, parsedate_tz, getaddresses
import imaplib

def get_unread_mails(host, port, user, password, ssl=True):
    klass = imaplib.IMAP4
    if ssl:
        klass = imaplib.IMAP4_SSL
    mail = klass(host, port)
    mail.login(user, password)
    try:
        status, count = mail.select('Inbox')
        typ, data = mail.search(None, 'UNSEEN')
        for num in data[0].split():
            status, data = mail.fetch(num, '(RFC822)')
            yield data[0][1]
    finally:
        mail.close()
        mail.logout()

def make_address(email, name=None):
    if name:
        return "%s <%s>" % (name, email)
    return email

class UnsupportedMailFormat(Exception):
    pass

class EmailParser(object):

    def parse_dispositions(self, dispo):
        dispos = dispo.strip().split(";")
        dispo_name = dispos[0].lower()
        dispo_dict = {}
        for param in dispos[1:]:
            name, value = param.split("=", 1)
            name = name.lower().strip()
            value = value.strip()
            if value.startswith('"') and value.endswith('"'):
                value = value[1:-1]
            dispo_dict[name] = value
        return dispo_name, dispo_dict

    def parse_attachment(self, message_part):
        content_disposition = message_part.get("Content-Disposition", None)
        if content_disposition:
            dispo_type, dispo_dict = self.parse_dispositions(content_disposition)
            if dispo_type == "attachment" or (dispo_type == 'inline' and
                    'filename' in dispo_dict):
                file_data = message_part.get_payload(decode=True)
                if file_data is None:
                    file_data = ""
                attachment = StringIO(file_data)
                attachment.content_type = message_part.get_content_type()
                attachment.size = len(file_data)
                attachment.name = None
                attachment.create_date = None
                attachment.mod_date = None
                attachment.read_date = None

                if "filename" in dispo_dict:
                    attachment.name = dispo_dict['filename']
                elif "create-date" in dispo_dict:
                    attachment.create_date = dispo_dict['create-date']  # TODO: datetime
                elif "modification-date" in dispo_dict:
                    attachment.mod_date = dispo_dict['modification-date']  # TODO: datetime
                elif "read-date" in dispo_dict:
                    attachment.read_date = dispo_dict['read-date']  # TODO: datetime
                return attachment
        return None

    def parse_date(self, date):
        date = parsedate_tz(date)
        if date is None:
            return None
        return (datetime.fromtimestamp(time.mktime(date[:9])), date[9])

    def parse_body(self, parts, attachments, body, html):
        for part in parts:
            attachment = self.parse_attachment(part)
            if attachment:
                attachments.append(attachment)
            elif part.get_content_type() == "text/plain":
                charset = part.get_content_charset() or 'ascii'
                body.append(unicode(
                    part.get_payload(decode=True),
                    charset, 'replace').encode('utf8', 'replace'))
            elif part.get_content_type() == "text/html":
                charset = part.get_content_charset() or 'ascii'
                html.append(unicode(
                    part.get_payload(decode=True),
                    charset,
                    'replace').encode('utf8', 'replace'))

    def parse(self, content):
        p = Parser()
        msgobj = p.parsestr(content)
        if msgobj['Subject'] is not None:
            decodefrag = decode_header(msgobj['Subject'])
            subj_fragments = []
            for s, enc in decodefrag:
                if enc:
                    try:
                        s = unicode(s, enc)
                    except UnicodeDecodeError:
                        # desperate move here
                        try:
                            s = s.decode("latin1")
                        except:
                            pass
                subj_fragments.append(s)
            subject = ''.join(subj_fragments)
            subject = subject.replace('\n\t', " ")
        else:
            subject = None
        attachments = []
        body = []
        html = []
        self.parse_body(msgobj.walk(), attachments, body, html)
        body = '\n'.join(body)
        html = '\n'.join(html)
        tos = getaddresses(msgobj.get_all('To', []))
        tos.extend(getaddresses(msgobj.get_all('X-Original-To', [])))
        ccs = getaddresses(msgobj.get_all('Cc', []))
        resent_tos = getaddresses(msgobj.get_all('resent-to', []))
        resent_ccs = getaddresses(msgobj.get_all('resent-cc', []))
        date = self.parse_date(msgobj.get("Date"))
        return {
            'msgobj': msgobj,
            'date': date,
            'subject': subject,
            'body': body,
            'html': html,
            'from': parseaddr(msgobj.get('From')),
            'to': tos,
            'cc': ccs,
            'resent_to': resent_tos,
            'resent_cc': resent_ccs,
            'attachments': attachments
        }

 # uses the email flatten
 #        out_file = StringIO.StringIO()
 #        message_gen = Generator(out_file, mangle_from_=False, maxheaderlen=60)
 #        message_gen.flatten(message)
 #        message_text = out_file.getvalue()

 #        fixes mime encoding issues (for display within html)
 #        clean_text = quopri.decodestring(message_text)

if __name__ == '__main__':
    p = EmailParser()
    email = p.parse(file('../foirequest/tests/test_mail_03.txt').read())
    for i, at in enumerate(email['attachments']):
        file(getattr(at, 'name', 'test'), 'w').write(at.read())
<|endoftext|>


<commit_before>"""pytest configuration.
"""

from __future__ import print_function
import sys
import os.path
import datetime
import re
from random import getrandbits
import zlib
import subprocess
import shutil
import tempfile
import logging
from distutils.version import StrictVersion as Version
import pytest
import icat
import icat.config
try:
    from suds.sax.date import UtcTimezone
except ImportError:
    UtcTimezone = None

# Note that pytest captures stderr, so we won't see any logging by
# default.  But since Suds uses logging, it's better to still have
# a well defined basic logging configuration in place.
logging.basicConfig(level=logging.INFO)

testdir = os.path.dirname(__file__)

# ============================= helper ===============================

if sys.version_info < (3, 0):
    def buf(seq):
        return buffer(bytearray(seq))
else:
    def buf(seq):
        return bytearray(seq)

class DummyDatafile(object):
    """A dummy file with random content to be used for test upload.
    """
    def __init__(self, directory, name, size, date=None):
        if date is not None:
            date = (date, date)
        self.name = name
        self.fname = os.path.join(directory, name)
        chunksize = 8192
        crc32 = 0
        with open(self.fname, 'wb') as f:
            while size > 0:
                if chunksize > size:
                    chunksize = size
                chunk = buf(getrandbits(8) for _ in range(chunksize))
                size -= chunksize
                crc32 = zlib.crc32(chunk, crc32)
                f.write(chunk)
        if date:
            os.utime(self.fname, date)
        self.crc32 = "%x" % (crc32 & 0xffffffff)
        self.stat = os.stat(self.fname)
        self.size = self.stat.st_size
        if UtcTimezone:
            mtime = int(self.stat.st_mtime)
            self.mtime = datetime.datetime.fromtimestamp(mtime, UtcTimezone())
        else:
            self.mtime = None

def getConfig(confSection="root", **confArgs):
    """Get the configuration, skip on ConfigError.
    """
    confFile = os.path.join(testdir, "data", "icat.cfg")
    if not os.path.isfile(confFile):
        pytest.skip("no test ICAT server configured")
    try:
        confArgs['args'] = ["-c", confFile, "-s", confSection]
        client, conf = icat.config.Config(**confArgs).getconfig()
        conf.cmdargs = ["-c", conf.configFile[0], "-s", conf.configSection]
        return (client, conf)
    except icat.ConfigError as err:
        pytest.skip(str(err))

class tmpSessionId:
    """Temporarily switch to another sessionId in an ICAT client.
    """
    def __init__(self, client, sessionId):
        self.client = client
        self.saveSessionId = client.sessionId
        self.sessionId = sessionId
    def __enter__(self):
        self.client.sessionId = self.sessionId
        return self.client
    def __exit__(self, type, value, tb):
        self.client.sessionId = self.saveSessionId

class tmpClient:
    """A temporary client using an own configuration,
    such as login as another user.
    """
    def __init__(self, **confArgs):
        (self.client, self.conf) = getConfig(**confArgs)
    def __enter__(self):
        self.client.login(self.conf.auth, self.conf.credentials)
        return self.client
    def __exit__(self, type, value, tb):
        self.client.logout()

def gettestdata(fname):
    fname = os.path.join(testdir, "data", fname)
    assert os.path.isfile(fname)
    return fname

def get_icat_version():
    client, _ = getConfig(needlogin=False)
    return client.apiversion

# ICAT server version we talk to.  Ignore any errors from
# get_icat_version(), if something fails (e.g. no server is configured
# at all), set a dummy zero version number.
try:
    icat_version = get_icat_version()
except:
    icat_version = Version("0.0")

def require_icat_version(minversion, reason):
    if icat_version < minversion:
        pytest.skip("need ICAT server version %s or newer: %s"
                    % (minversion, reason))

def callscript(scriptname, args, stdin=None, stdout=None, stderr=None):
    script = os.path.join(testdir, "scripts", scriptname)
    cmd = [sys.executable, script] + args
    print("\n>", *cmd)
    subprocess.check_call(cmd, stdin=stdin, stdout=stdout, stderr=stderr)

yaml_filter = (re.compile(r"^# (Date|Service|ICAT-API|Generator): .*$"),
               r"# \1: ###")
xml_filter = (re.compile(r"^\s*<(date|service|apiversion|generator)>.*</\1>$"),
              r"  <\1>###</\1>")

def filter_file(infile, outfile, pattern, repl):
    """Filter a text file.

    This may be needed to compare some test output file with
    predefined results, because some information in the file might not
    depend on the actual test but rather dynamically change with each
    call.  Such as the header of a dump file that contains date and
    ICAT version.
    """
    with open(infile, 'rt') as inf, open(outfile, 'wt') as outf:
        while True:
            l = inf.readline()
            if not l:
                break
            l = re.sub(pattern, repl, l)
            outf.write(l)

# ============================ fixtures ==============================

# Deliberately not using the 'tmpdir' fixture provided by pytest,
# because it seem to use a predictable directory name in /tmp wich is
# insecure.

class TmpDir(object):
    """Provide a temporary directory.
    """
    def __init__(self):
        self.dir = tempfile.mkdtemp(prefix="python-icat-test-")
    def __del__(self):
        self.cleanup()
    def cleanup(self):
        if self.dir:
            shutil.rmtree(self.dir)
        self.dir = None

@pytest.fixture(scope="session")
def tmpdirsec(request):
    tmpdir = TmpDir()
    request.addfinalizer(tmpdir.cleanup)
    return tmpdir

@pytest.fixture(scope="session")
def standardCmdArgs():
    _, conf = getConfig()
    return conf.cmdargs

testcontent = gettestdata("icatdump.yaml")

@pytest.fixture(scope="session")
def setupicat(standardCmdArgs):
    require_icat_version("4.4.0", "need InvestigationGroup")
    callscript("wipeicat.py", standardCmdArgs)
    args = standardCmdArgs + ["-f", "YAML", "-i", testcontent]
    callscript("icatingest.py", args)

# ============================= hooks ================================

def pytest_report_header(config):
    """Add information on the icat package used in the tests.
    """
    modpath = os.path.dirname(os.path.abspath(icat.__file__))
    if icat_version > "0.0":
        server = icat_version
    else:
        server = "-"
    return [ "python-icat: %s (%s)" % (icat.__version__, icat.__revision__),
             "             %s" % (modpath),
             "icat.server: %s" % server]

<commit_msg>Silence logging from Suds in the tests.<commit_after>"""pytest configuration.
"""

from __future__ import print_function
import sys
import os.path
import datetime
import re
from random import getrandbits
import zlib
import subprocess
import shutil
import tempfile
import logging
from distutils.version import StrictVersion as Version
import pytest
import icat
import icat.config
try:
    from suds.sax.date import UtcTimezone
except ImportError:
    UtcTimezone = None

# Note that pytest captures stderr, so we won't see any logging by
# default.  But since Suds uses logging, it's better to still have
# a well defined basic logging configuration in place.
logging.basicConfig(level=logging.INFO)
# Newer pytest versions show the logs at level DEBUG in case of an
# error.  The problem is that suds is rather chatty, so it will
# clutter the output to an extent that we wont be able to see
# anything.  Silence it.
logging.getLogger('suds.client').setLevel(logging.CRITICAL)
logging.getLogger('suds').setLevel(logging.ERROR)

testdir = os.path.dirname(__file__)

# ============================= helper ===============================

if sys.version_info < (3, 0):
    def buf(seq):
        return buffer(bytearray(seq))
else:
    def buf(seq):
        return bytearray(seq)

class DummyDatafile(object):
    """A dummy file with random content to be used for test upload.
    """
    def __init__(self, directory, name, size, date=None):
        if date is not None:
            date = (date, date)
        self.name = name
        self.fname = os.path.join(directory, name)
        chunksize = 8192
        crc32 = 0
        with open(self.fname, 'wb') as f:
            while size > 0:
                if chunksize > size:
                    chunksize = size
                chunk = buf(getrandbits(8) for _ in range(chunksize))
                size -= chunksize
                crc32 = zlib.crc32(chunk, crc32)
                f.write(chunk)
        if date:
            os.utime(self.fname, date)
        self.crc32 = "%x" % (crc32 & 0xffffffff)
        self.stat = os.stat(self.fname)
        self.size = self.stat.st_size
        if UtcTimezone:
            mtime = int(self.stat.st_mtime)
            self.mtime = datetime.datetime.fromtimestamp(mtime, UtcTimezone())
        else:
            self.mtime = None

def getConfig(confSection="root", **confArgs):
    """Get the configuration, skip on ConfigError.
    """
    confFile = os.path.join(testdir, "data", "icat.cfg")
    if not os.path.isfile(confFile):
        pytest.skip("no test ICAT server configured")
    try:
        confArgs['args'] = ["-c", confFile, "-s", confSection]
        client, conf = icat.config.Config(**confArgs).getconfig()
        conf.cmdargs = ["-c", conf.configFile[0], "-s", conf.configSection]
        return (client, conf)
    except icat.ConfigError as err:
        pytest.skip(str(err))

class tmpSessionId:
    """Temporarily switch to another sessionId in an ICAT client.
    """
    def __init__(self, client, sessionId):
        self.client = client
        self.saveSessionId = client.sessionId
        self.sessionId = sessionId
    def __enter__(self):
        self.client.sessionId = self.sessionId
        return self.client
    def __exit__(self, type, value, tb):
        self.client.sessionId = self.saveSessionId

class tmpClient:
    """A temporary client using an own configuration,
    such as login as another user.
    """
    def __init__(self, **confArgs):
        (self.client, self.conf) = getConfig(**confArgs)
    def __enter__(self):
        self.client.login(self.conf.auth, self.conf.credentials)
        return self.client
    def __exit__(self, type, value, tb):
        self.client.logout()

def gettestdata(fname):
    fname = os.path.join(testdir, "data", fname)
    assert os.path.isfile(fname)
    return fname

def get_icat_version():
    client, _ = getConfig(needlogin=False)
    return client.apiversion

# ICAT server version we talk to.  Ignore any errors from
# get_icat_version(), if something fails (e.g. no server is configured
# at all), set a dummy zero version number.
try:
    icat_version = get_icat_version()
except:
    icat_version = Version("0.0")

def require_icat_version(minversion, reason):
    if icat_version < minversion:
        pytest.skip("need ICAT server version %s or newer: %s"
                    % (minversion, reason))

def callscript(scriptname, args, stdin=None, stdout=None, stderr=None):
    script = os.path.join(testdir, "scripts", scriptname)
    cmd = [sys.executable, script] + args
    print("\n>", *cmd)
    subprocess.check_call(cmd, stdin=stdin, stdout=stdout, stderr=stderr)

yaml_filter = (re.compile(r"^# (Date|Service|ICAT-API|Generator): .*$"),
               r"# \1: ###")
xml_filter = (re.compile(r"^\s*<(date|service|apiversion|generator)>.*</\1>$"),
              r"  <\1>###</\1>")

def filter_file(infile, outfile, pattern, repl):
    """Filter a text file.

    This may be needed to compare some test output file with
    predefined results, because some information in the file might not
    depend on the actual test but rather dynamically change with each
    call.  Such as the header of a dump file that contains date and
    ICAT version.
    """
    with open(infile, 'rt') as inf, open(outfile, 'wt') as outf:
        while True:
            l = inf.readline()
            if not l:
                break
            l = re.sub(pattern, repl, l)
            outf.write(l)

# ============================ fixtures ==============================

# Deliberately not using the 'tmpdir' fixture provided by pytest,
# because it seem to use a predictable directory name in /tmp wich is
# insecure.

class TmpDir(object):
    """Provide a temporary directory.
    """
    def __init__(self):
        self.dir = tempfile.mkdtemp(prefix="python-icat-test-")
    def __del__(self):
        self.cleanup()
    def cleanup(self):
        if self.dir:
            shutil.rmtree(self.dir)
        self.dir = None

@pytest.fixture(scope="session")
def tmpdirsec(request):
    tmpdir = TmpDir()
    request.addfinalizer(tmpdir.cleanup)
    return tmpdir

@pytest.fixture(scope="session")
def standardCmdArgs():
    _, conf = getConfig()
    return conf.cmdargs

testcontent = gettestdata("icatdump.yaml")

@pytest.fixture(scope="session")
def setupicat(standardCmdArgs):
    require_icat_version("4.4.0", "need InvestigationGroup")
    callscript("wipeicat.py", standardCmdArgs)
    args = standardCmdArgs + ["-f", "YAML", "-i", testcontent]
    callscript("icatingest.py", args)

# ============================= hooks ================================

def pytest_report_header(config):
    """Add information on the icat package used in the tests.
    """
    modpath = os.path.dirname(os.path.abspath(icat.__file__))
    if icat_version > "0.0":
        server = icat_version
    else:
        server = "-"
    return [ "python-icat: %s (%s)" % (icat.__version__, icat.__revision__),
             "             %s" % (modpath),
             "icat.server: %s" % server]

<|endoftext|>


<commit_before>"""
evepaste.parsers.view_contents
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse results from 'View Contents' of a ship.

"""
from collections import defaultdict
import re

from evepaste.utils import regex_match_lines, f_int

VIEWCONT_LIST_RE = re.compile(r"""^([\S ]*)\t
                                  ([\S ]*)\t
                                  (Cargo\ Hold|
                                   (Drone|Fuel)\ Bay|
                                   (Low|Medium|High|Rig)\ Slot|
                                   Subsystem|
                                   )\t
                                  ([\d,\.]+)$""", re.X)
STATION_CONTAINER_RE = re.compile(r"""^([\S ]*)\t
                                       ([\S ]*)\t
                                       ([\d,\.]+)$""", re.X)

def parse_view_contents(lines):
    """ Parse view contents

    :param string paste_string: String pasted from view contents window
    """
    matches, bad_lines = regex_match_lines(VIEWCONT_LIST_RE, lines)
    matches2, bad_lines2 = regex_match_lines(STATION_CONTAINER_RE, bad_lines)

    items = defaultdict(int)
    print matches[0:1]
    for name, group, location, _, _, quantity in matches:
        items[(name, group, location)] += f_int(quantity)

    results = [{'name': name,
                'group': group,
                'location': location,
                'quantity': quantity}
               for (name, group, location), quantity in sorted(items.items())]

    items2 = defaultdict(int)
    for name, group, quantity in matches2:
        items2[(name, group)] += f_int(quantity)
    results2 = [{'name': name,
                 'group': group,
                 'quantity': quantity}
                for (name, group), quantity in sorted(items2.items())]

    return results + results2, bad_lines2
<commit_msg>Removes debugging<commit_after>"""
evepaste.parsers.view_contents
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse results from 'View Contents' of a ship.

"""
from collections import defaultdict
import re

from evepaste.utils import regex_match_lines, f_int

VIEWCONT_LIST_RE = re.compile(r"""^([\S ]*)\t
                                  ([\S ]*)\t
                                  (Cargo\ Hold|
                                   (Drone|Fuel)\ Bay|
                                   (Low|Medium|High|Rig)\ Slot|
                                   Subsystem|
                                   )\t
                                  ([\d,\.]+)$""", re.X)
STATION_CONTAINER_RE = re.compile(r"""^([\S ]*)\t
                                       ([\S ]*)\t
                                       ([\d,\.]+)$""", re.X)

def parse_view_contents(lines):
    """ Parse view contents

    :param string paste_string: String pasted from view contents window
    """
    matches, bad_lines = regex_match_lines(VIEWCONT_LIST_RE, lines)
    matches2, bad_lines2 = regex_match_lines(STATION_CONTAINER_RE, bad_lines)

    items = defaultdict(int)
    for name, group, location, _, _, quantity in matches:
        items[(name, group, location)] += f_int(quantity)

    results = [{'name': name,
                'group': group,
                'location': location,
                'quantity': quantity}
               for (name, group, location), quantity in sorted(items.items())]

    items2 = defaultdict(int)
    for name, group, quantity in matches2:
        items2[(name, group)] += f_int(quantity)
    results2 = [{'name': name,
                 'group': group,
                 'quantity': quantity}
                for (name, group), quantity in sorted(items2.items())]

    return results + results2, bad_lines2
<|endoftext|>


<commit_before>import random

from django.conf import settings
from django.contrib.gis.geos import Point
from django.db.models import Prefetch
from django.shortcuts import get_object_or_404

from rest_framework import mixins, permissions, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.settings import api_settings
from rest_framework.throttling import UserRateThrottle

from froide.foirequest.api_views import throttle_action
from froide.foirequest.models import FoiRequest

from .filters import (
    CategoryFilter,
    CustomSearchFilter,
    FeaturedFilter,
    RandomOrderFilter,
    StatusFilter,
)
from .geocode import run_geocode
from .models import (
    Answer,
    Campaign,
    CampaignSubscription,
    InformationObject,
    Question,
    Questionaire,
    Report,
)
from .providers.base import BaseProvider
from .serializers import CampaignProviderRequestSerializer, InformationObjectSerializer

def get_lat_lng(request):
    try:
        lat = float(request.GET.get("lat"))
    except (ValueError, TypeError):
        raise ValueError
    try:
        lng = float(request.GET.get("lng"))
    except (ValueError, TypeError):
        raise ValueError
    return lat, lng

class AddLocationPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        campaign_id = request.data.get("campaign")
        if campaign_id is None:
            return False
        campaign = Campaign.objects.get(id=campaign_id)
        return campaign.get_provider().CREATE_ALLOWED

class AddLocationThrottle(UserRateThrottle):
    scope = "campaign-createlocation"
    THROTTLE_RATES = {
        scope: "3/day",
    }

class InformationObjectViewSet(
    mixins.CreateModelMixin,
    mixins.RetrieveModelMixin,
    mixins.ListModelMixin,
    viewsets.GenericViewSet,
):
    RANDOM_COUNT = 3
    SEARCH_COUNT = 10
    serializer_class = InformationObjectSerializer
    pagination_class = api_settings.DEFAULT_PAGINATION_CLASS
    filter_backends = [
        CustomSearchFilter,
        StatusFilter,
        CategoryFilter,
        FeaturedFilter,
        RandomOrderFilter,
    ]
    search_fields = ["translations__title", "translations__subtitle"]

    def get_serializer_context(self):
        context = super().get_serializer_context()
        lang = self.request.GET.get("language", settings.LANGUAGE_CODE)
        context.update({"language": lang})
        return context

    def get_permissions(self):
        if self.action == "create":
            permission_classes = [AddLocationPermission]
        else:
            permission_classes = [permissions.AllowAny]
        return [permission() for permission in permission_classes]

    @throttle_action((AddLocationThrottle,))
    def create(self, request, *args, **kwargs):
        return super().create(request, *args, **kwargs)

    def perform_create(self, serializer):
        obj = serializer.save()
        obj.ident = "custom_{}".format(str(obj.id))
        point = self.get_geo(obj)
        obj.geo = point
        obj.save()

    def retrieve(self, request, *args, **kwargs):
        campaign = self.get_campaign()
        language = self.request.GET.get("language", settings.LANGUAGE_CODE)
        campaign.set_current_language(language)
        provider = campaign.get_provider()
        ident = kwargs.pop("pk")
        obj = provider.get_by_ident(ident)
        data = provider.get_provider_item_data(obj)
        data["publicbody"] = provider.get_publicbody(ident)
        data["publicbodies"] = provider.get_publicbodies(ident)
        data["makeRequestURL"] = provider.get_request_url(ident, language=language)
        data["userRequestCount"] = provider.get_user_request_count(request.user)
        serializer = CampaignProviderRequestSerializer(
            data, context={"request": request}
        )
        return Response(serializer.data)

    def get_campaign(self):
        campaign_id = self.request.GET.get("campaign")
        if self.request.user.is_staff:
            qs = Campaign.objects.all()
        else:
            qs = Campaign.objects.get_public()
        return get_object_or_404(qs, id=campaign_id)

    def get_queryset(self):
        campaign = self.get_campaign()
        iobjs = InformationObject.objects.filter(campaign=campaign)
        iobjs = iobjs.prefetch_related(
            Prefetch(
                "foirequests", queryset=FoiRequest.objects.order_by("-first_message")
            )
        )
        iobjs = iobjs.prefetch_related("campaign")
        iobjs = iobjs.prefetch_related("categories")
        return iobjs

    def get_geo(self, obj):
        if obj.address and not obj.geo:
            geo = run_geocode(obj.address)
            if geo:
                lat_lng = geo[0]
                return Point(lat_lng[1], lat_lng[0])

    @action(detail=False, methods=["post"])
    def report(self, request):
        questionaire_id = int(request.data.get("questionaire"))
        iobj_id = int(request.data.get("informationObject"))
        answers = request.data.get("answers")
        report_id = request.data.get("report")

        questionaire = Questionaire.objects.get(id=questionaire_id)
        information_object = InformationObject.objects.get(id=iobj_id)

        if report_id:
            report = Report.objects.get(id=report_id)
            report.answer_set.all().delete()
        else:
            report = Report.objects.create(
                questionaire=questionaire, informationsobject=information_object
            )

        for answer in answers:
            question_id = int(answer["questionId"])
            question = Question.objects.get(id=question_id)
            Answer.objects.create(
                text=answer["answer"], report=report, question=question
            )
        return Response({"report": report.id})

    @action(detail=False, methods=["post"])
    def subscribe(self, request):
        email = request.data.get("email")
        if request.user.is_authenticated:
            email = request.user.email

        campaign_id = request.data.get("campaign")
        subscribe = request.data.get("subscribe")

        if email and campaign_id:
            try:
                campaign = Campaign.objects.get(id=campaign_id)
                if subscribe:
                    obj, created = CampaignSubscription.objects.get_or_create(
                        campaign=campaign, email=email
                    )
                    return Response({"email": obj.email, "campaign": obj.campaign.id})
                else:
                    try:
                        obj = CampaignSubscription.objects.get(
                            campaign=campaign, email=email
                        ).delete()
                    except CampaignSubscription.DoesNotExist:
                        pass
            except Campaign.DoesNotExist:
                return Response({"error": "Campaign does not exist"})
        return Response({})

    @action(detail=False, methods=["get"])
    def random(self, request):
        campaign = self.get_campaign()

        provider = campaign.get_provider()

        filters = {"requested": False}

        data = provider.search(**filters)
        if data:
            random_data = random.choices(data, k=self.RANDOM_COUNT)
            return Response(random_data)
        return Response(data)

    @action(detail=False, methods=["get"])
    def search(self, request):
        campaign = self.get_campaign()

        provider = campaign.get_provider()

        filters = {"q": request.GET.get("q", "")}

        try:
            if "featured" in request.GET:
                filters["featured"] = int(request.GET["featured"])
        except ValueError:
            pass

        try:
            if "requested" in request.GET:
                filters["requested"] = int(request.GET["requested"])
        except ValueError:
            pass

        # TODO: geocode
        # location / coordinates
        # if location is not None:
        #     location_search = True
        #     point, formatted_address = geocode(location, address=False)

        try:
            lat, lng = get_lat_lng(request)
            filters.update(
                {
                    "coordinates": Point(lng, lat),
                }
            )
        except ValueError:
            pass

        try:
            filters["zoom"] = int(request.GET.get("zoom"))
        except (ValueError, TypeError):
            pass
        try:
            filters["radius"] = int(request.GET.get("radius"))
        except (ValueError, TypeError):
            pass

        data = provider.search(**filters)

        if not type(provider) == BaseProvider:
            iobjs = BaseProvider(campaign).search(**filters)
            data = data + iobjs

        return Response(data)
<commit_msg>Validate user-given language for API with settings<commit_after>import random

from django.conf import settings
from django.contrib.gis.geos import Point
from django.db.models import Prefetch
from django.shortcuts import get_object_or_404

from rest_framework import mixins, permissions, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.settings import api_settings
from rest_framework.throttling import UserRateThrottle

from froide.foirequest.api_views import throttle_action
from froide.foirequest.models import FoiRequest

from .filters import (
    CategoryFilter,
    CustomSearchFilter,
    FeaturedFilter,
    RandomOrderFilter,
    StatusFilter,
)
from .geocode import run_geocode
from .models import (
    Answer,
    Campaign,
    CampaignSubscription,
    InformationObject,
    Question,
    Questionaire,
    Report,
)
from .providers.base import BaseProvider
from .serializers import CampaignProviderRequestSerializer, InformationObjectSerializer

def get_lat_lng(request):
    try:
        lat = float(request.GET.get("lat"))
    except (ValueError, TypeError):
        raise ValueError
    try:
        lng = float(request.GET.get("lng"))
    except (ValueError, TypeError):
        raise ValueError
    return lat, lng

def get_language(request):
    lang = request.GET.get("language", settings.LANGUAGE_CODE)
    if lang not in dict(settings.LANGUAGES):
        return settings.LANGUAGE_CODE
    return lang

class AddLocationPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        campaign_id = request.data.get("campaign")
        if campaign_id is None:
            return False
        campaign = Campaign.objects.get(id=campaign_id)
        return campaign.get_provider().CREATE_ALLOWED

class AddLocationThrottle(UserRateThrottle):
    scope = "campaign-createlocation"
    THROTTLE_RATES = {
        scope: "3/day",
    }

class InformationObjectViewSet(
    mixins.CreateModelMixin,
    mixins.RetrieveModelMixin,
    mixins.ListModelMixin,
    viewsets.GenericViewSet,
):
    RANDOM_COUNT = 3
    SEARCH_COUNT = 10
    serializer_class = InformationObjectSerializer
    pagination_class = api_settings.DEFAULT_PAGINATION_CLASS
    filter_backends = [
        CustomSearchFilter,
        StatusFilter,
        CategoryFilter,
        FeaturedFilter,
        RandomOrderFilter,
    ]
    search_fields = ["translations__title", "translations__subtitle"]

    def get_serializer_context(self):
        context = super().get_serializer_context()
        context.update({"language": get_language(self.request)})
        return context

    def get_permissions(self):
        if self.action == "create":
            permission_classes = [AddLocationPermission]
        else:
            permission_classes = [permissions.AllowAny]
        return [permission() for permission in permission_classes]

    @throttle_action((AddLocationThrottle,))
    def create(self, request, *args, **kwargs):
        return super().create(request, *args, **kwargs)

    def perform_create(self, serializer):
        obj = serializer.save()
        obj.ident = "custom_{}".format(str(obj.id))
        point = self.get_geo(obj)
        obj.geo = point
        obj.save()

    def retrieve(self, request, *args, **kwargs):
        campaign = self.get_campaign()
        language = get_language(self.request)
        campaign.set_current_language(language)
        provider = campaign.get_provider()
        ident = kwargs.pop("pk")
        obj = provider.get_by_ident(ident)
        data = provider.get_provider_item_data(obj)
        data["publicbody"] = provider.get_publicbody(ident)
        data["publicbodies"] = provider.get_publicbodies(ident)
        data["makeRequestURL"] = provider.get_request_url(ident, language=language)
        data["userRequestCount"] = provider.get_user_request_count(request.user)
        serializer = CampaignProviderRequestSerializer(
            data, context={"request": request}
        )
        return Response(serializer.data)

    def get_campaign(self):
        campaign_id = self.request.GET.get("campaign")
        if self.request.user.is_staff:
            qs = Campaign.objects.all()
        else:
            qs = Campaign.objects.get_public()
        return get_object_or_404(qs, id=campaign_id)

    def get_queryset(self):
        campaign = self.get_campaign()
        iobjs = InformationObject.objects.filter(campaign=campaign)
        iobjs = iobjs.prefetch_related(
            Prefetch(
                "foirequests", queryset=FoiRequest.objects.order_by("-first_message")
            )
        )
        iobjs = iobjs.prefetch_related("campaign")
        iobjs = iobjs.prefetch_related("categories")
        return iobjs

    def get_geo(self, obj):
        if obj.address and not obj.geo:
            geo = run_geocode(obj.address)
            if geo:
                lat_lng = geo[0]
                return Point(lat_lng[1], lat_lng[0])

    @action(detail=False, methods=["post"])
    def report(self, request):
        questionaire_id = int(request.data.get("questionaire"))
        iobj_id = int(request.data.get("informationObject"))
        answers = request.data.get("answers")
        report_id = request.data.get("report")

        questionaire = Questionaire.objects.get(id=questionaire_id)
        information_object = InformationObject.objects.get(id=iobj_id)

        if report_id:
            report = Report.objects.get(id=report_id)
            report.answer_set.all().delete()
        else:
            report = Report.objects.create(
                questionaire=questionaire, informationsobject=information_object
            )

        for answer in answers:
            question_id = int(answer["questionId"])
            question = Question.objects.get(id=question_id)
            Answer.objects.create(
                text=answer["answer"], report=report, question=question
            )
        return Response({"report": report.id})

    @action(detail=False, methods=["post"])
    def subscribe(self, request):
        email = request.data.get("email")
        if request.user.is_authenticated:
            email = request.user.email

        campaign_id = request.data.get("campaign")
        subscribe = request.data.get("subscribe")

        if email and campaign_id:
            try:
                campaign = Campaign.objects.get(id=campaign_id)
                if subscribe:
                    obj, created = CampaignSubscription.objects.get_or_create(
                        campaign=campaign, email=email
                    )
                    return Response({"email": obj.email, "campaign": obj.campaign.id})
                else:
                    try:
                        obj = CampaignSubscription.objects.get(
                            campaign=campaign, email=email
                        ).delete()
                    except CampaignSubscription.DoesNotExist:
                        pass
            except Campaign.DoesNotExist:
                return Response({"error": "Campaign does not exist"})
        return Response({})

    @action(detail=False, methods=["get"])
    def random(self, request):
        campaign = self.get_campaign()

        provider = campaign.get_provider()

        filters = {"requested": False}

        data = provider.search(**filters)
        if data:
            random_data = random.choices(data, k=self.RANDOM_COUNT)
            return Response(random_data)
        return Response(data)

    @action(detail=False, methods=["get"])
    def search(self, request):
        campaign = self.get_campaign()

        provider = campaign.get_provider()

        filters = {"q": request.GET.get("q", "")}

        try:
            if "featured" in request.GET:
                filters["featured"] = int(request.GET["featured"])
        except ValueError:
            pass

        try:
            if "requested" in request.GET:
                filters["requested"] = int(request.GET["requested"])
        except ValueError:
            pass

        # TODO: geocode
        # location / coordinates
        # if location is not None:
        #     location_search = True
        #     point, formatted_address = geocode(location, address=False)

        try:
            lat, lng = get_lat_lng(request)
            filters.update(
                {
                    "coordinates": Point(lng, lat),
                }
            )
        except ValueError:
            pass

        try:
            filters["zoom"] = int(request.GET.get("zoom"))
        except (ValueError, TypeError):
            pass
        try:
            filters["radius"] = int(request.GET.get("radius"))
        except (ValueError, TypeError):
            pass

        data = provider.search(**filters)

        if not type(provider) == BaseProvider:
            iobjs = BaseProvider(campaign).search(**filters)
            data = data + iobjs

        return Response(data)
<|endoftext|>


<commit_before>#
# Copyright (C) 2013 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""configobj backend for anyconfig.
"""
from __future__ import absolute_import

import configobj
import anyconfig.backend.base

class Parser(anyconfig.backend.base.Parser):
    """
    Parser for Ini-like config files which configobj supports.

    - Backend: configobj (https://pypi.python.org/pypi/configobj/)
    - Limitations: None obvious
    - Special options:

      - All options passed to configobj.ConfigObj.__init__ should work.
    """
    _type = "configobj"
    _priority = 10
    _load_opts = ["cls", "configspec", "encoding", "interpolation",
                  "raise_errors", "list_values", "create_empty", "file_error",
                  "stringify", "indent_type", "default_encoding", "unrepr",
                  "_inspec", ]
    _dump_opts = ["cls", "encoding", "list_values", "indent_type",
                  "default_encoding", "unrepr", "write_empty_values", ]

    @classmethod
    def load_impl(cls, config_fp, **kwargs):
        """
        :param config_fp:  Config file object
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: dict object holding config parameters
        """
        return configobj.ConfigObj(config_fp, **kwargs)

    @classmethod
    def dumps_impl(cls, data, **kwargs):
        """
        :param data: Data to dump :: dict
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: string represents the configuration
        """
        conf = configobj.ConfigObj(**kwargs)
        conf.update(data)
        conf.filename = None

        return '\n'.join(conf.write())

    @classmethod
    def dump_impl(cls, data, config_path, **kwargs):
        """
        :param data: Data to dump :: dict
        :param config_path: Dump destination file path
        :param kwargs: backend-specific optional keyword parameters :: dict
        """
        conf = configobj.ConfigObj(**kwargs)
        conf.update(data)

        conf.write(open(config_path, 'w'))

# vim:sw=4:ts=4:et:
<commit_msg>fix a bug that configobj process not str but bytes internally when output to files<commit_after>#
# Copyright (C) 2013 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""configobj backend for anyconfig.
"""
from __future__ import absolute_import

import configobj
import anyconfig.backend.base

class Parser(anyconfig.backend.base.Parser):
    """
    Parser for Ini-like config files which configobj supports.

    - Backend: configobj (https://pypi.python.org/pypi/configobj/)
    - Limitations: None obvious
    - Special options:

      - All options passed to configobj.ConfigObj.__init__ should work.
    """
    _type = "configobj"
    _priority = 10
    _load_opts = ["cls", "configspec", "encoding", "interpolation",
                  "raise_errors", "list_values", "create_empty", "file_error",
                  "stringify", "indent_type", "default_encoding", "unrepr",
                  "_inspec", ]
    _dump_opts = ["cls", "encoding", "list_values", "indent_type",
                  "default_encoding", "unrepr", "write_empty_values", ]

    @classmethod
    def load_impl(cls, config_fp, **kwargs):
        """
        :param config_fp:  Config file object
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: dict object holding config parameters
        """
        return configobj.ConfigObj(config_fp, **kwargs)

    @classmethod
    def dumps_impl(cls, data, **kwargs):
        """
        :param data: Data to dump :: dict
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: string represents the configuration
        """
        conf = configobj.ConfigObj(**kwargs)
        conf.update(data)
        conf.filename = None

        return '\n'.join(conf.write())

    @classmethod
    def dump_impl(cls, data, config_path, **kwargs):
        """
        :param data: Data to dump :: dict
        :param config_path: Dump destination file path
        :param kwargs: backend-specific optional keyword parameters :: dict
        """
        conf = configobj.ConfigObj(**kwargs)
        conf.update(data)

        conf.write(open(config_path, 'wb'))

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from datetime import datetime as dtdatetime
from dateutil.parser import parse
from functools import wraps
from flask.ext.cache import Cache

import pickle
from .helpers import *

from . import app
from .ratelimit import redis

# a long time, when the cars will fly...
INFINITE = 365 * 24 * 3600
if redis:
    redis_prefix = app.config['CACHE']['CACHE_KEY_PREFIX'].encode('utf-8') + b'KEY-ITEM/'

cache = Cache(app, config=app.config['CACHE'])

#
# CACHER BY ARGS FILTERS
#
# ======================
def get_key_list():
    """List all cached keys"""
    if redis:
        try:
            key_list = redis.keys(redis_prefix + b'*')
            keys = {}
            for key in key_list:
                val = redis.get(key)
                decoded = pickle.loads(val)
                keys[key[len(redis_prefix):]] = decoded
        except Exception as e:
            app.logger.debug('Error decoding key {key} [{e}]'.format(key=key, e=e))
    else:
        keys = cache.get('KEY-LIST')

    if not keys:
        return {}
    return keys

def add_key_list(key, timeout, time):
    """Save cache key to a listable set"""
    val = (timeout, time)
    if redis:
        # A more efficient way to touch a single key instead of retrieving the full set
        app.logger.debug('Adding key to REDIS {key} {val}'.format(key=key, val=val))
        v = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
        return redis.set(redis_prefix + key, v)

    keys = get_key_list()
    keys[key] = val
    return cache.set('KEY-LIST', keys, timeout=INFINITE)

def renew_key_list(key):
    if redis:
        # A more efficient way to touch a single key instead of retrieving the full set
        try:
            (timeout, time) = pickle.loads(redis.get(redis_prefix + key).decode('utf-8'))
            return add_key_list(key, timeout, dtdatetime.now())
        except:
            pass

    keys = get_key_list()
    if key in keys:
        (timeout, time) = keys[key]

    if timeout and time:
        return add_key_list(key, timeout, dtdatetime.now())

    return False

def cacher(f):
    """Caches methods for model classes"""
    @wraps(f)
    def decorated(*args, **kwargs):
        if not app.config['CACHING']:
            return f(*args, **kwargs)

        key = pickle.dumps((f.__name__, args, kwargs), pickle.HIGHEST_PROTOCOL)
        # TODO: lower the cache time depending on the to_date parameter
        # if present use that date as maxdate (else now)
        # if maxdate is > now() - 2 months (configurable)
        # timeout variable (the closer to now, the lesser the value)
        timeout = app.config['CACHE_MIN_TIMEOUT']
        now = dtdatetime.now()
        if 'to_date' in kwargs and kwargs['to_date'] != None:
            datemax = parse(kwargs['to_date'])
        else:
            datemax = now
        delta = now - datemax
        if delta.days > 60:
            timeout = INFINITE  # 1 year time caching
        elif delta.total_seconds() > app.config['CACHE_MIN_TIMEOUT']:
            timeout = int(delta.total_seconds())

        app.logger.debug('CACHER FOR FUNCTION: {0} TIMEOUT: {1}s KEY: {2}'.format(f.__name__, timeout, key))

        cached = cache.get(str(key))
        if cached is not None:
            if 'BYPASS_CACHING' in app.config and app.config['BYPASS_CACHING'] and app.debug:
                app.logger.debug('--Bypassing caching result---')
            else:
                app.logger.debug('--Cached result--')
                return cached

        app.logger.debug('--Real result (not cached)--')

        add_key_list(key, timeout, now)
        result = f(*args, **kwargs)
        cache.set(str(key), result, timeout=timeout)
        return result

    return decorated

def get_key_parts(key):
    """Decodes a cache key"""
    try:
        parts = pickle.loads(key)
        func = parts[0]
        args = ()
        kwargs = {}
        for k in parts:
            if isinstance(k, dict):
                kwargs = k
            elif not isinstance(k, str):
                args = k
        # print (func, kwargs, args)
        clas = None
        if args and hasattr(args[0], func):
            clas = args[0]
            args = args[1:]
            func = getattr(clas, func)
        return (clas, func, args, kwargs)
    except Exception as e:
        app.logger.debug("ERROR DECODING Key '{0}' Exception {1}".format(key, str(e)))

    return False

def get_key_functions(keys, force=False):
    """Returns a list with functions and parameters (ready to execute) associated with the keys in cache"""
    funcs = []
    for key, (timeout, time) in keys.items():
        delta = dtdatetime.now() - time
        # print (key, timeout, time.isoformat(), delta.total_seconds(), timeout / delta.total_seconds(), delta.total_seconds() / timeout)
        # if delta.total_seconds() / timeout < 0.75 and (timeout - delta.total_seconds()) > 60:
        if not force and (timeout - delta.total_seconds()) > 60:
            app.logger.debug("BYPASSING Key '{0}' with timeout {1} still have {2} seconds to expire".format(key, timeout, timeout - delta.total_seconds()))
            continue
        app.logger.debug("PROCESSING Key '{0}' with timeout {1}  about to expire in {2} seconds".format(key, timeout, timeout - delta.total_seconds()))

        key_parts = get_key_parts(key)
        if key_parts is False:
            app.logger.debug("INVALID KEY '{0}' BYPASSING...".format(key))
            continue

        (clas, func, args, kwargs) = key_parts

        if clas:
            funcs.append((key, clas, func, args, kwargs))
        else:

            if func in locals():
                funcs.append((locals()[func], args, dict(kwargs)))
            elif func in globals():
                funcs.append((globals()[func], args, dict(kwargs)))
            else:
                funcs.append((key, None, func, args, dict(kwargs)))

        # # Retrieving content from class
        # if clas:
        #     parts = clas.split('.')
        #     mod = importlib.import_module('.'.join(parts[:-1]))

        #     if inspect.ismodule(mod):
        #         mod = getattr(mod, parts[-1])
        #         instance = getattr(mod, func)
        #         funcs.append((instance, args, dict(kwargs)))

    return funcs
<commit_msg>pep8 comparison style<commit_after># -*- coding: utf-8 -*-
from datetime import datetime as dtdatetime
from dateutil.parser import parse
from functools import wraps
from flask.ext.cache import Cache

import pickle
from .helpers import *

from . import app
from .ratelimit import redis

# a long time, when the cars will fly...
INFINITE = 365 * 24 * 3600
if redis:
    redis_prefix = app.config['CACHE']['CACHE_KEY_PREFIX'].encode('utf-8') + b'KEY-ITEM/'

cache = Cache(app, config=app.config['CACHE'])

#
# CACHER BY ARGS FILTERS
#
# ======================
def get_key_list():
    """List all cached keys"""
    if redis:
        try:
            key_list = redis.keys(redis_prefix + b'*')
            keys = {}
            for key in key_list:
                val = redis.get(key)
                decoded = pickle.loads(val)
                keys[key[len(redis_prefix):]] = decoded
        except Exception as e:
            app.logger.debug('Error decoding key {key} [{e}]'.format(key=key, e=e))
    else:
        keys = cache.get('KEY-LIST')

    if not keys:
        return {}
    return keys

def add_key_list(key, timeout, time):
    """Save cache key to a listable set"""
    val = (timeout, time)
    if redis:
        # A more efficient way to touch a single key instead of retrieving the full set
        app.logger.debug('Adding key to REDIS {key} {val}'.format(key=key, val=val))
        v = pickle.dumps(val, pickle.HIGHEST_PROTOCOL)
        return redis.set(redis_prefix + key, v)

    keys = get_key_list()
    keys[key] = val
    return cache.set('KEY-LIST', keys, timeout=INFINITE)

def renew_key_list(key):
    if redis:
        # A more efficient way to touch a single key instead of retrieving the full set
        try:
            (timeout, time) = pickle.loads(redis.get(redis_prefix + key).decode('utf-8'))
            return add_key_list(key, timeout, dtdatetime.now())
        except:
            pass

    keys = get_key_list()
    if key in keys:
        (timeout, time) = keys[key]

    if timeout and time:
        return add_key_list(key, timeout, dtdatetime.now())

    return False

def cacher(f):
    """Caches methods for model classes"""
    @wraps(f)
    def decorated(*args, **kwargs):
        if not app.config['CACHING']:
            return f(*args, **kwargs)

        key = pickle.dumps((f.__name__, args, kwargs), pickle.HIGHEST_PROTOCOL)
        # TODO: lower the cache time depending on the to_date parameter
        # if present use that date as maxdate (else now)
        # if maxdate is > now() - 2 months (configurable)
        # timeout variable (the closer to now, the lesser the value)
        timeout = app.config['CACHE_MIN_TIMEOUT']
        now = dtdatetime.now()
        if 'to_date' in kwargs and kwargs['to_date'] is not None:
            datemax = parse(kwargs['to_date'])
        else:
            datemax = now
        delta = now - datemax
        if delta.days > 60:
            timeout = INFINITE  # 1 year time caching
        elif delta.total_seconds() > app.config['CACHE_MIN_TIMEOUT']:
            timeout = int(delta.total_seconds())

        app.logger.debug('CACHER FOR FUNCTION: {0} TIMEOUT: {1}s KEY: {2}'.format(f.__name__, timeout, key))

        cached = cache.get(str(key))
        if cached is not None:
            if 'BYPASS_CACHING' in app.config and app.config['BYPASS_CACHING'] and app.debug:
                app.logger.debug('--Bypassing caching result---')
            else:
                app.logger.debug('--Cached result--')
                return cached

        app.logger.debug('--Real result (not cached)--')

        add_key_list(key, timeout, now)
        result = f(*args, **kwargs)
        cache.set(str(key), result, timeout=timeout)
        return result

    return decorated

def get_key_parts(key):
    """Decodes a cache key"""
    try:
        parts = pickle.loads(key)
        func = parts[0]
        args = ()
        kwargs = {}
        for k in parts:
            if isinstance(k, dict):
                kwargs = k
            elif not isinstance(k, str):
                args = k
        # print (func, kwargs, args)
        clas = None
        if args and hasattr(args[0], func):
            clas = args[0]
            args = args[1:]
            func = getattr(clas, func)
        return (clas, func, args, kwargs)
    except Exception as e:
        app.logger.debug("ERROR DECODING Key '{0}' Exception {1}".format(key, str(e)))

    return False

def get_key_functions(keys, force=False):
    """Returns a list with functions and parameters (ready to execute) associated with the keys in cache"""
    funcs = []
    for key, (timeout, time) in keys.items():
        delta = dtdatetime.now() - time
        # print (key, timeout, time.isoformat(), delta.total_seconds(), timeout / delta.total_seconds(), delta.total_seconds() / timeout)
        # if delta.total_seconds() / timeout < 0.75 and (timeout - delta.total_seconds()) > 60:
        if not force and (timeout - delta.total_seconds()) > 60:
            app.logger.debug("BYPASSING Key '{0}' with timeout {1} still have {2} seconds to expire".format(key, timeout, timeout - delta.total_seconds()))
            continue
        app.logger.debug("PROCESSING Key '{0}' with timeout {1}  about to expire in {2} seconds".format(key, timeout, timeout - delta.total_seconds()))

        key_parts = get_key_parts(key)
        if key_parts is False:
            app.logger.debug("INVALID KEY '{0}' BYPASSING...".format(key))
            continue

        (clas, func, args, kwargs) = key_parts

        if clas:
            funcs.append((key, clas, func, args, kwargs))
        else:

            if func in locals():
                funcs.append((locals()[func], args, dict(kwargs)))
            elif func in globals():
                funcs.append((globals()[func], args, dict(kwargs)))
            else:
                funcs.append((key, None, func, args, dict(kwargs)))

        # # Retrieving content from class
        # if clas:
        #     parts = clas.split('.')
        #     mod = importlib.import_module('.'.join(parts[:-1]))

        #     if inspect.ismodule(mod):
        #         mod = getattr(mod, parts[-1])
        #         instance = getattr(mod, func)
        #         funcs.append((instance, args, dict(kwargs)))

    return funcs
<|endoftext|>


<commit_before>from copy import copy
import traceback
import uuid
try:
    from cStringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

from twisted.python import log

from .utils import get_spider_list, JsonResource, UtilsCache, native_stringify_dict

class WsResource(JsonResource):

    def __init__(self, root):
        JsonResource.__init__(self)
        self.root = root

    def render(self, txrequest):
        try:
            return JsonResource.render(self, txrequest).encode('utf-8')
        except Exception as e:
            if self.root.debug:
                return traceback.format_exc().encode('utf-8')
            log.err()
            r = {"node_name": self.root.nodename, "status": "error", "message": str(e)}
            return self.render_object(r, txrequest).encode('utf-8')

class DaemonStatus(WsResource):

    def render_GET(self, txrequest):
        pending = sum(q.count() for q in self.root.poller.queues.values())
        running = len(self.root.launcher.processes)
        finished = len(self.root.launcher.finished)

        return {"node_name": self.root.nodename, "status":"ok", "pending": pending, "running": running, "finished": finished}

class Schedule(WsResource):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        settings = args.pop('setting', [])
        settings = dict(x.split('=', 1) for x in settings)
        args = dict((k, v[0]) for k, v in args.items())
        project = args.pop('project')
        spider = args.pop('spider')
        version = args.get('_version', '')
        spiders = get_spider_list(project, version=version)
        if not spider in spiders:
            return {"status": "error", "message": "spider '%s' not found" % spider}
        args['settings'] = settings
        jobid = args.pop('jobid', uuid.uuid1().hex)
        args['_job'] = jobid
        self.root.scheduler.schedule(project, spider, **args)
        return {"node_name": self.root.nodename, "status": "ok", "jobid": jobid}

class Cancel(WsResource):

    def render_POST(self, txrequest):
        args = dict((k, v[0])
                    for k, v in native_stringify_dict(copy(txrequest.args),
                                    keys_only=False).items())
        project = args['project']
        jobid = args['job']
        signal = args.get('signal', 'TERM')
        prevstate = None
        queue = self.root.poller.queues[project]
        c = queue.remove(lambda x: x["_job"] == jobid)
        if c:
            prevstate = "pending"
        spiders = self.root.launcher.processes.values()
        for s in spiders:
            if s.job == jobid:
                s.transport.signalProcess(signal)
                prevstate = "running"
        return {"node_name": self.root.nodename, "status": "ok", "prevstate": prevstate}

class AddVersion(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args[b'project'][0].decode('utf-8')
        version = txrequest.args[b'version'][0].decode('utf-8')
        eggf = BytesIO(txrequest.args[b'egg'][0])
        self.root.eggstorage.put(eggf, project, version)
        spiders = get_spider_list(project, version=version)
        self.root.update_projects()
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok", "project": project, "version": version, \
            "spiders": len(spiders)}

class ListProjects(WsResource):

    def render_GET(self, txrequest):
        projects = list(self.root.scheduler.list_projects())
        return {"node_name": self.root.nodename, "status": "ok", "projects": projects}

class ListVersions(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        versions = self.root.eggstorage.list(project)
        return {"node_name": self.root.nodename, "status": "ok", "versions": versions}

class ListSpiders(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        version = args.get('_version', [''])[0]
        spiders = get_spider_list(project, runner=self.root.runner, version=version)
        return {"node_name": self.root.nodename, "status": "ok", "spiders": spiders}

class ListJobs(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        spiders = self.root.launcher.processes.values()
        queue = self.root.poller.queues[project]
        pending = [
            {"spider": x["name"], "id": x["_job"]}
            for x in queue.list()
        ]
        running = [
            {
                "spider": s.spider,
                "id": s.job, "pid": s.pid,
                "start_time": s.start_time.isoformat(' '),
            } for s in spiders if s.project == project
        ]
        finished = [
            {
                "spider": s.spider, "id": s.job,
                "start_time": s.start_time.isoformat(' '),
                "end_time": s.end_time.isoformat(' '),
            } for s in self.root.launcher.finished if s.project == project
        ]
        return {"node_name": self.root.nodename, "status": "ok",
                "pending": pending, "running": running, "finished": finished}

class DeleteProject(WsResource):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        self._delete_version(project)
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok"}

    def _delete_version(self, project, version=None):
        self.root.eggstorage.delete(project, version)
        self.root.update_projects()

class DeleteVersion(DeleteProject):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        version = args['version'][0]
        self._delete_version(project, version)
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok"}
<commit_msg>datetime.isoformat(' ') is equivalent to str()<commit_after>from copy import copy
import traceback
import uuid
try:
    from cStringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

from twisted.python import log

from .utils import get_spider_list, JsonResource, UtilsCache, native_stringify_dict

class WsResource(JsonResource):

    def __init__(self, root):
        JsonResource.__init__(self)
        self.root = root

    def render(self, txrequest):
        try:
            return JsonResource.render(self, txrequest).encode('utf-8')
        except Exception as e:
            if self.root.debug:
                return traceback.format_exc().encode('utf-8')
            log.err()
            r = {"node_name": self.root.nodename, "status": "error", "message": str(e)}
            return self.render_object(r, txrequest).encode('utf-8')

class DaemonStatus(WsResource):

    def render_GET(self, txrequest):
        pending = sum(q.count() for q in self.root.poller.queues.values())
        running = len(self.root.launcher.processes)
        finished = len(self.root.launcher.finished)

        return {"node_name": self.root.nodename, "status":"ok", "pending": pending, "running": running, "finished": finished}

class Schedule(WsResource):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        settings = args.pop('setting', [])
        settings = dict(x.split('=', 1) for x in settings)
        args = dict((k, v[0]) for k, v in args.items())
        project = args.pop('project')
        spider = args.pop('spider')
        version = args.get('_version', '')
        spiders = get_spider_list(project, version=version)
        if not spider in spiders:
            return {"status": "error", "message": "spider '%s' not found" % spider}
        args['settings'] = settings
        jobid = args.pop('jobid', uuid.uuid1().hex)
        args['_job'] = jobid
        self.root.scheduler.schedule(project, spider, **args)
        return {"node_name": self.root.nodename, "status": "ok", "jobid": jobid}

class Cancel(WsResource):

    def render_POST(self, txrequest):
        args = dict((k, v[0])
                    for k, v in native_stringify_dict(copy(txrequest.args),
                                    keys_only=False).items())
        project = args['project']
        jobid = args['job']
        signal = args.get('signal', 'TERM')
        prevstate = None
        queue = self.root.poller.queues[project]
        c = queue.remove(lambda x: x["_job"] == jobid)
        if c:
            prevstate = "pending"
        spiders = self.root.launcher.processes.values()
        for s in spiders:
            if s.job == jobid:
                s.transport.signalProcess(signal)
                prevstate = "running"
        return {"node_name": self.root.nodename, "status": "ok", "prevstate": prevstate}

class AddVersion(WsResource):

    def render_POST(self, txrequest):
        project = txrequest.args[b'project'][0].decode('utf-8')
        version = txrequest.args[b'version'][0].decode('utf-8')
        eggf = BytesIO(txrequest.args[b'egg'][0])
        self.root.eggstorage.put(eggf, project, version)
        spiders = get_spider_list(project, version=version)
        self.root.update_projects()
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok", "project": project, "version": version, \
            "spiders": len(spiders)}

class ListProjects(WsResource):

    def render_GET(self, txrequest):
        projects = list(self.root.scheduler.list_projects())
        return {"node_name": self.root.nodename, "status": "ok", "projects": projects}

class ListVersions(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        versions = self.root.eggstorage.list(project)
        return {"node_name": self.root.nodename, "status": "ok", "versions": versions}

class ListSpiders(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        version = args.get('_version', [''])[0]
        spiders = get_spider_list(project, runner=self.root.runner, version=version)
        return {"node_name": self.root.nodename, "status": "ok", "spiders": spiders}

class ListJobs(WsResource):

    def render_GET(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        spiders = self.root.launcher.processes.values()
        queue = self.root.poller.queues[project]
        pending = [
            {"spider": x["name"], "id": x["_job"]}
            for x in queue.list()
        ]
        running = [
            {
                "spider": s.spider,
                "id": s.job, "pid": s.pid,
                "start_time": str(s.start_time),
            } for s in spiders if s.project == project
        ]
        finished = [
            {
                "spider": s.spider, "id": s.job,
                "start_time": str(s.start_time),
                "end_time": str(s.end_time)
            } for s in self.root.launcher.finished if s.project == project
        ]
        return {"node_name": self.root.nodename, "status": "ok",
                "pending": pending, "running": running, "finished": finished}

class DeleteProject(WsResource):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        self._delete_version(project)
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok"}

    def _delete_version(self, project, version=None):
        self.root.eggstorage.delete(project, version)
        self.root.update_projects()

class DeleteVersion(DeleteProject):

    def render_POST(self, txrequest):
        args = native_stringify_dict(copy(txrequest.args), keys_only=False)
        project = args['project'][0]
        version = args['version'][0]
        self._delete_version(project, version)
        UtilsCache.invalid_cache(project)
        return {"node_name": self.root.nodename, "status": "ok"}
<|endoftext|>