<commit_before>from base64 import b64decode, standard_b64decode, urlsafe_b64decode
from io import StringIO
from django.http.response import HttpResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt
import openid2rp
from Care4Care.settings import STATICFILES_DIRS

def oidtest(request):
    ax = (("http://axschema.org/eid/card-validity/end",
           "http://axschema.org/person/gender",
           "http://axschema.org/contact/postalAddress/home",
           "http://axschema.org/namePerson/first",
           "http://axschema.org/eid/photo",
           "http://axschema.org/eid/card-validity/begin",
           "http://axschema.org/contact/city/home",
           "http://axschema.org/contact/postalCode/home",
           "http://axschema.org/birthDate",
           "http://openid.net/schema/birthDate/birthYear",
           "http://openid.net/schema/birthDate/birthMonth",
           "http://openid.net/schema/birthDate/birthday",
           "http://axschema.org/eid/pob",
           "http://axschema.org/eid/card-number",
           "http://axschema.org/eid/nationality",
           "http://axschema.org/namePerson/last",
           "http://axschema.org/namePerson",
           "http://axschema.org/eid/rrn",
           # "http://axschema.org/eid/cert/auth",
           "http://axschema.org/eid/age"), ())
    uri = "https://www.e-contract.be/eid-idp/endpoints/openid/ident"
    kind, claimedId = openid2rp.normalize_uri(uri)
    res = openid2rp.discover(claimedId)
    if res is not None:
        services, op_endpoint, op_local = res
        session = openid2rp.associate(services, op_endpoint)
        redirect_url = openid2rp.request_authentication(
            services,
            op_endpoint,
            session['assoc_handle'],
            "http://127.0.0.1:8000/tests/openid2",
            claimedId, op_local,
            sreg=((), ()),
            ax=ax
        )

        response = HttpResponse()
        response['Location'] = redirect_url
        response.status_code=303
        return response

        # return render_to_response('OIDTest.html',
        #                           {
        #                               'services' : services,
        #                               'op_endpoint' : op_endpoint,
        #                               'op_local' : op_local,
        #                               'kind' : kind,
        #                               'claimedID' : claimedId,
        #                               'redirect_url' : redirect_url
        #                               }
        # )

@csrf_exempt
def oidtest2(request):
    # signed, claimedID = openid2rp.verify(request.POST, None, None, True)
    # printy(get_ax(request.POST))
    ax = get_ax(request.POST)
    args = ax.copy()
    args.update({'ax' : ax})
    strIO = StringIO()
    print(ax, file=strIO)
    args.update({'printy' : strIO.getvalue()})
    return render(request, "OIDTest.html", args)

def get_ax(response):
    ax = 'ax' + "."
    oax = 'openid.' + ax
    res = {}
    for k, v in response.items():
        if k.startswith(oax+"type."):
            k = k.rsplit('.',1)[1]
            value_name = oax+"value."+k
            if ax+"value."+k not in response['openid.signed']:
                continue
            res[v] = response[value_name]
    return _get_readable_ax(res)

def _get_readable_ax(ax):
    res = {}
    AX = {"http://axschema.org/eid/card-validity/end" : 'card_validity_end',
           "http://axschema.org/person/gender" : 'gender',
           "http://axschema.org/contact/postalAddress/home" : 'address',
           "http://axschema.org/namePerson/first" : 'firstname',
           "http://axschema.org/eid/photo" : 'photo',
           "http://axschema.org/eid/card-validity/begin" : 'card_validity_start',
           "http://axschema.org/contact/city/home" : 'city',
           "http://axschema.org/contact/postalCode/home" : 'postal_code',
           "http://axschema.org/birthDate" : 'birth_date',
           "http://openid.net/schema/birthDate/birthYear" : 'birth_year',
           "http://openid.net/schema/birthDate/birthMonth" : 'birth_month',
           "http://openid.net/schema/birthDate/birthday" : 'birth_day',
           "http://axschema.org/eid/pob" : 'birth_place',
           "http://axschema.org/eid/card-number" : 'card_number',
           "http://axschema.org/eid/nationality" : 'nationality',
           "http://axschema.org/namePerson/last" : 'lastname',
           "http://axschema.org/namePerson" : 'fullname',
           "http://axschema.org/eid/rrn" : 'register_number',
           "http://axschema.org/eid/age" : 'age'
    }
    for key, value in ax.items():
        if key.endswith('photo'):
            filename = STATICFILES_DIRS[0] + '/images/photo.jpg'
            #value = value.replace('-', '+').replace('_', '/')
            if len(value) % 3 != 0: value += '=';
            if len(value) % 3 != 0: value += '=';
            with open(filename, 'wb') as fd:
                fd.write(urlsafe_b64decode(value))
                fd.close()
            value = filename
        res[AX[key]] = value
    return res<commit_msg>never cache<commit_after>from base64 import b64decode, standard_b64decode, urlsafe_b64decode
from io import StringIO
from django.http.response import HttpResponse
from django.shortcuts import render
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
import openid2rp
from Care4Care.settings import STATICFILES_DIRS

@never_cache
def oidtest(request):
    ax = (("http://axschema.org/eid/card-validity/end",
           "http://axschema.org/person/gender",
           "http://axschema.org/contact/postalAddress/home",
           "http://axschema.org/namePerson/first",
           "http://axschema.org/eid/photo",
           "http://axschema.org/eid/card-validity/begin",
           "http://axschema.org/contact/city/home",
           "http://axschema.org/contact/postalCode/home",
           "http://axschema.org/birthDate",
           "http://openid.net/schema/birthDate/birthYear",
           "http://openid.net/schema/birthDate/birthMonth",
           "http://openid.net/schema/birthDate/birthday",
           "http://axschema.org/eid/pob",
           "http://axschema.org/eid/card-number",
           "http://axschema.org/eid/nationality",
           "http://axschema.org/namePerson/last",
           "http://axschema.org/namePerson",
           "http://axschema.org/eid/rrn",
           # "http://axschema.org/eid/cert/auth",
           "http://axschema.org/eid/age"), ())
    uri = "https://www.e-contract.be/eid-idp/endpoints/openid/ident"
    kind, claimedId = openid2rp.normalize_uri(uri)
    res = openid2rp.discover(claimedId)
    if res is not None:
        services, op_endpoint, op_local = res
        session = openid2rp.associate(services, op_endpoint)
        redirect_url = openid2rp.request_authentication(
            services,
            op_endpoint,
            session['assoc_handle'],
            "http://127.0.0.1:8000/tests/openid2",
            claimedId, op_local,
            sreg=((), ()),
            ax=ax
        )

        response = HttpResponse()
        response['Location'] = redirect_url
        response.status_code=303
        return response

        # return render_to_response('OIDTest.html',
        #                           {
        #                               'services' : services,
        #                               'op_endpoint' : op_endpoint,
        #                               'op_local' : op_local,
        #                               'kind' : kind,
        #                               'claimedID' : claimedId,
        #                               'redirect_url' : redirect_url
        #                               }
        # )

@csrf_exempt
@never_cache
def oidtest2(request):
    # signed, claimedID = openid2rp.verify(request.POST, None, None, True)
    # printy(get_ax(request.POST))
    ax = get_ax(request.POST)
    args = ax.copy()
    args.update({'ax' : ax})
    strIO = StringIO()
    print(ax, file=strIO)
    args.update({'printy' : strIO.getvalue()})
    return render(request, "OIDTest.html", args)

def get_ax(response):
    ax = 'ax' + "."
    oax = 'openid.' + ax
    res = {}
    for k, v in response.items():
        if k.startswith(oax+"type."):
            k = k.rsplit('.',1)[1]
            value_name = oax+"value."+k
            if ax+"value."+k not in response['openid.signed']:
                continue
            res[v] = response[value_name]
    return _get_readable_ax(res)

def _get_readable_ax(ax):
    res = {}
    AX = {"http://axschema.org/eid/card-validity/end" : 'card_validity_end',
           "http://axschema.org/person/gender" : 'gender',
           "http://axschema.org/contact/postalAddress/home" : 'address',
           "http://axschema.org/namePerson/first" : 'firstname',
           "http://axschema.org/eid/photo" : 'photo',
           "http://axschema.org/eid/card-validity/begin" : 'card_validity_start',
           "http://axschema.org/contact/city/home" : 'city',
           "http://axschema.org/contact/postalCode/home" : 'postal_code',
           "http://axschema.org/birthDate" : 'birth_date',
           "http://openid.net/schema/birthDate/birthYear" : 'birth_year',
           "http://openid.net/schema/birthDate/birthMonth" : 'birth_month',
           "http://openid.net/schema/birthDate/birthday" : 'birth_day',
           "http://axschema.org/eid/pob" : 'birth_place',
           "http://axschema.org/eid/card-number" : 'card_number',
           "http://axschema.org/eid/nationality" : 'nationality',
           "http://axschema.org/namePerson/last" : 'lastname',
           "http://axschema.org/namePerson" : 'fullname',
           "http://axschema.org/eid/rrn" : 'register_number',
           "http://axschema.org/eid/age" : 'age'
    }
    for key, value in ax.items():
        if key.endswith('photo'):
            filename = STATICFILES_DIRS[0] + '/images/photo.jpg'
            #value = value.replace('-', '+').replace('_', '/')
            if len(value) % 3 != 0: value += '=';
            if len(value) % 3 != 0: value += '=';
            with open(filename, 'wb') as fd:
                fd.write(urlsafe_b64decode(value))
                fd.close()
            value = filename
        res[AX[key]] = value
    return res<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import datetime
import os
import unittest

from airflow import AirflowException, settings
from airflow.bin import cli
from airflow.jobs import BackfillJob, SchedulerJob
from airflow.models import DAG, DagBag, DagRun, Pool, TaskInstance as TI
from airflow.operators import DummyOperator
from airflow.utils.state import State
from airflow.utils.timeout import timeout
from airflow.utils.db import provide_session

DEV_NULL = '/dev/null'
DEFAULT_DATE = datetime.datetime(2016, 1, 1)

class BackfillJobTest(unittest.TestCase):

    def setUp(self):
        self.parser = cli.CLIFactory.get_parser()
        self.dagbag = DagBag()

    def test_backfill_examples(self):
        """
        Test backfilling example dags
        """
        dags = [
            dag for dag in self.dagbag.dags.values()
            if dag.dag_id in ('example_bash_operator',)]
        for dag in dags:
            dag.clear(
                start_date=DEFAULT_DATE,
                end_date=DEFAULT_DATE)
        for dag in dags:
            job = BackfillJob(
                dag=dag,
                start_date=DEFAULT_DATE,
                end_date=DEFAULT_DATE)
            job.run()

    def test_trap_executor_error(self):
        """
        Test that errors setting up tasks (before tasks run) are caught.

        Executors run tasks with the `airflow run` command. If a task runs,
        its state (success, failure, or other) is stored in the database and
        `airflow run` exits without error. However, if an error is raised
        before the task runs, then the task won't be able to update its status.
        This can put the executor into an infinite loop of trying to run the
        task.

        To counteract that, the executor traps errors coming from
        `airflow run` (essentially looking for returncode != 0).
        This unit test creates such an error by trying
        to run a subdag whose dag_id has changed and therefore can't be
        found. If the trap is working properly, the error will be caught
        and the Backfill will report failures. If the trap is not working,
        the job will run infinitely (the unit test uses a timeout to protect
        against that case).

        Test for https://github.com/airbnb/airflow/pull/1220
        """
        dag = self.dagbag.get_dag('test_raise_executor_error')
        dag.clear()
        job = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)
        # run with timeout because this creates an infinite loop if not
        # caught
        def run_with_timeout():
            with timeout(seconds=30):
                job.run()
        self.assertRaises(AirflowException, run_with_timeout)

    def test_backfill_pooled_tasks(self):
        """
        Test that queued tasks are executed by BackfillJob

        Test for https://github.com/airbnb/airflow/pull/1225
        """
        session = settings.Session()
        pool = Pool(pool='test_backfill_pooled_task_pool', slots=1)
        session.add(pool)
        session.commit()

        dag = self.dagbag.get_dag('test_backfill_pooled_task_dag')
        dag.clear()

        job = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)

        # run with timeout because this creates an infinite loop if not
        # caught
        with timeout(seconds=30):
            job.run()

        ti = TI(
            task=dag.get_task('test_backfill_pooled_task'),
            execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        self.assertEqual(ti.state, State.SUCCESS)

    def test_backfill_depends_on_past(self):
        """
        Test that backfill resects ignore_depends_on_past
        """
        dag = self.dagbag.get_dag('test_depends_on_past')
        dag.clear()
        run_date = DEFAULT_DATE + datetime.timedelta(days=5)

        # backfill should deadlock
        self.assertRaisesRegexp(
            AirflowException,
            'BackfillJob is deadlocked',
            BackfillJob(dag=dag, start_date=run_date, end_date=run_date).run)

        BackfillJob(
            dag=dag,
            start_date=run_date,
            end_date=run_date,
            ignore_first_depends_on_past=True).run()

        # ti should have succeeded
        ti = TI(dag.tasks[0], run_date)
        ti.refresh_from_db()
        self.assertEquals(ti.state, State.SUCCESS)

    def test_cli_backfill_depends_on_past(self):
        """
        Test that CLI respects -I argument
        """
        dag_id = 'test_dagrun_states_deadlock'
        run_date = DEFAULT_DATE + datetime.timedelta(days=1)
        args = [
            'backfill',
            dag_id,
            '-l',
            '-s',
            run_date.isoformat(),
        ]
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()

        self.assertRaisesRegexp(
            AirflowException,
            'BackfillJob is deadlocked',
            cli.backfill,
            self.parser.parse_args(args))

        cli.backfill(self.parser.parse_args(args + ['-I']))
        ti = TI(dag.get_task('test_depends_on_past'), run_date)
        ti.refresh_from_db()
        # task ran
        self.assertEqual(ti.state, State.SUCCESS)

class SchedulerJobTest(unittest.TestCase):

    def setUp(self):
        self.dagbag = DagBag()

    @provide_session
    def evaluate_dagrun(
            self,
            dag_id,
            first_task_state,
            second_task_state,
            dagrun_state,
            run_kwargs=None,
            advance_execution_date=False,
            session=None):
        """
        Helper for testing DagRun states with simple two-task DAGS
        """
        if run_kwargs is None:
            run_kwargs = {}

        scheduler = SchedulerJob()
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()
        dr = scheduler.schedule_dag(dag)
        if advance_execution_date:
            # run a second time to schedule a dagrun after the start_date
            dr = scheduler.schedule_dag(dag)
        ex_date = dr.execution_date

        try:
            dag.run(start_date=ex_date, end_date=ex_date, **run_kwargs)
        except AirflowException:
            pass

        # test tasks
        task_1, task_2 = dag.tasks
        ti = TI(task_1, ex_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, first_task_state)
        ti = TI(task_2, ex_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, second_task_state)

        # load dagrun
        dr = session.query(DagRun).filter(
            DagRun.dag_id == dag.dag_id,
            DagRun.execution_date == ex_date
        ).first()

        # dagrun is running
        self.assertEqual(dr.state, State.RUNNING)

        dag.get_active_runs()

        # dagrun failed
        self.assertEqual(dr.state, dagrun_state)

    def test_dagrun_fail(self):
        """
        DagRuns with one failed and one incomplete root task -> FAILED
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_fail',
            first_task_state=State.FAILED,
            second_task_state=State.UPSTREAM_FAILED,
            dagrun_state=State.FAILED)

    def test_dagrun_success(self):
        """
        DagRuns with one failed and one successful root task -> SUCCESS
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_success',
            first_task_state=State.FAILED,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS)

    def test_dagrun_root_fail(self):
        """
        DagRuns with one successful and one failed root task -> FAILED
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_root_fail',
            first_task_state=State.SUCCESS,
            second_task_state=State.FAILED,
            dagrun_state=State.FAILED)

    def test_dagrun_deadlock(self):
        """
        Deadlocked DagRun is marked a failure

        Test that a deadlocked dagrun is marked as a failure by having
        depends_on_past and an execution_date after the start_date
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=None,
            second_task_state=None,
            dagrun_state=State.FAILED,
            advance_execution_date=True)

    def test_scheduler_pooled_tasks(self):
        """
        Test that the scheduler handles queued tasks correctly
        See issue #1299
        """
        session = settings.Session()
        if not (
                session.query(Pool)
                .filter(Pool.pool == 'test_queued_pool')
                .first()):
            pool = Pool(pool='test_queued_pool', slots=5)
            session.merge(pool)
            session.commit()
        session.close()

        dag_id = 'test_scheduled_queued_tasks'
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()

        scheduler = SchedulerJob(dag_id, num_runs=10)
        scheduler.run()

        task_1 = dag.tasks[0]
        ti = TI(task_1, dag.start_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, State.FAILED)

        dag.clear()

    def test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date(self):
        """
        DagRun is marked a success if ignore_first_depends_on_past=True

        Test that an otherwise-deadlocked dagrun is marked as a success
        if ignore_first_depends_on_past=True and the dagrun execution_date
        is after the start_date.
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=State.SUCCESS,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS,
            advance_execution_date=True,
            run_kwargs=dict(ignore_first_depends_on_past=True))

    def test_dagrun_deadlock_ignore_depends_on_past(self):
        """
        Test that ignore_first_depends_on_past doesn't affect results
        (this is the same test as
        test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date except
        that start_date == execution_date so depends_on_past is irrelevant).
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=State.SUCCESS,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS,
            run_kwargs=dict(ignore_first_depends_on_past=True))

    def test_scheduler_start_date(self):
        """
        Test that the scheduler respects start_dates, even when DAGS have run
        """

        dag_id = 'test_start_date_scheduling'
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()
        self.assertTrue(dag.start_date > DEFAULT_DATE)

        scheduler = SchedulerJob(dag_id, num_runs=2)
        scheduler.run()

        # zero tasks ran
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)

        # previously, running this backfill would kick off the Scheduler
        # because it would take the most recent run and start from there
        # That behavior still exists, but now it will only do so if after the
        # start date
        backfill = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)
        backfill.run()

        # one task ran
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)

        scheduler = SchedulerJob(dag_id, num_runs=2)
        scheduler.run()

        # still one task
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
<commit_msg>Include all example dags in backfill unit test<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import datetime
import logging
import os
import unittest

from airflow import AirflowException, settings
from airflow.bin import cli
from airflow.jobs import BackfillJob, SchedulerJob
from airflow.models import DAG, DagBag, DagRun, Pool, TaskInstance as TI
from airflow.operators import DummyOperator
from airflow.utils.state import State
from airflow.utils.timeout import timeout
from airflow.utils.db import provide_session

DEV_NULL = '/dev/null'
DEFAULT_DATE = datetime.datetime(2016, 1, 1)

class BackfillJobTest(unittest.TestCase):

    def setUp(self):
        self.parser = cli.CLIFactory.get_parser()
        self.dagbag = DagBag(include_examples=True)

    def test_backfill_examples(self):
        """
        Test backfilling example dags
        """

        # some DAGs really are just examples... but try to make them work!
        skip_dags = [
            'example_http_operator',
        ]

        logger = logging.getLogger('BackfillJobTest.test_backfill_examples')
        dags = [
            dag for dag in self.dagbag.dags.values()
            if 'example_dags' in dag.full_filepath
            and dag.dag_id not in skip_dags
            ]

        for dag in dags:
            dag.clear(
                start_date=DEFAULT_DATE,
                end_date=DEFAULT_DATE)

        for i, dag in enumerate(sorted(dags, key=lambda d: d.dag_id)):
            logger.info('*** Running example DAG #{}: {}'.format(i, dag.dag_id))
            job = BackfillJob(
                dag=dag,
                start_date=DEFAULT_DATE,
                end_date=DEFAULT_DATE,
                ignore_first_depends_on_past=True)
            job.run()

    def test_trap_executor_error(self):
        """
        Test that errors setting up tasks (before tasks run) are caught.

        Executors run tasks with the `airflow run` command. If a task runs,
        its state (success, failure, or other) is stored in the database and
        `airflow run` exits without error. However, if an error is raised
        before the task runs, then the task won't be able to update its status.
        This can put the executor into an infinite loop of trying to run the
        task.

        To counteract that, the executor traps errors coming from
        `airflow run` (essentially looking for returncode != 0).
        This unit test creates such an error by trying
        to run a subdag whose dag_id has changed and therefore can't be
        found. If the trap is working properly, the error will be caught
        and the Backfill will report failures. If the trap is not working,
        the job will run infinitely (the unit test uses a timeout to protect
        against that case).

        Test for https://github.com/airbnb/airflow/pull/1220
        """
        dag = self.dagbag.get_dag('test_raise_executor_error')
        dag.clear()
        job = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)
        # run with timeout because this creates an infinite loop if not
        # caught
        def run_with_timeout():
            with timeout(seconds=30):
                job.run()
        self.assertRaises(AirflowException, run_with_timeout)

    def test_backfill_pooled_tasks(self):
        """
        Test that queued tasks are executed by BackfillJob

        Test for https://github.com/airbnb/airflow/pull/1225
        """
        session = settings.Session()
        pool = Pool(pool='test_backfill_pooled_task_pool', slots=1)
        session.add(pool)
        session.commit()

        dag = self.dagbag.get_dag('test_backfill_pooled_task_dag')
        dag.clear()

        job = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)

        # run with timeout because this creates an infinite loop if not
        # caught
        with timeout(seconds=30):
            job.run()

        ti = TI(
            task=dag.get_task('test_backfill_pooled_task'),
            execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        self.assertEqual(ti.state, State.SUCCESS)

    def test_backfill_depends_on_past(self):
        """
        Test that backfill resects ignore_depends_on_past
        """
        dag = self.dagbag.get_dag('test_depends_on_past')
        dag.clear()
        run_date = DEFAULT_DATE + datetime.timedelta(days=5)

        # backfill should deadlock
        self.assertRaisesRegexp(
            AirflowException,
            'BackfillJob is deadlocked',
            BackfillJob(dag=dag, start_date=run_date, end_date=run_date).run)

        BackfillJob(
            dag=dag,
            start_date=run_date,
            end_date=run_date,
            ignore_first_depends_on_past=True).run()

        # ti should have succeeded
        ti = TI(dag.tasks[0], run_date)
        ti.refresh_from_db()
        self.assertEquals(ti.state, State.SUCCESS)

    def test_cli_backfill_depends_on_past(self):
        """
        Test that CLI respects -I argument
        """
        dag_id = 'test_dagrun_states_deadlock'
        run_date = DEFAULT_DATE + datetime.timedelta(days=1)
        args = [
            'backfill',
            dag_id,
            '-l',
            '-s',
            run_date.isoformat(),
        ]
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()

        self.assertRaisesRegexp(
            AirflowException,
            'BackfillJob is deadlocked',
            cli.backfill,
            self.parser.parse_args(args))

        cli.backfill(self.parser.parse_args(args + ['-I']))
        ti = TI(dag.get_task('test_depends_on_past'), run_date)
        ti.refresh_from_db()
        # task ran
        self.assertEqual(ti.state, State.SUCCESS)

class SchedulerJobTest(unittest.TestCase):

    def setUp(self):
        self.dagbag = DagBag()

    @provide_session
    def evaluate_dagrun(
            self,
            dag_id,
            first_task_state,
            second_task_state,
            dagrun_state,
            run_kwargs=None,
            advance_execution_date=False,
            session=None):
        """
        Helper for testing DagRun states with simple two-task DAGS
        """
        if run_kwargs is None:
            run_kwargs = {}

        scheduler = SchedulerJob()
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()
        dr = scheduler.schedule_dag(dag)
        if advance_execution_date:
            # run a second time to schedule a dagrun after the start_date
            dr = scheduler.schedule_dag(dag)
        ex_date = dr.execution_date

        try:
            dag.run(start_date=ex_date, end_date=ex_date, **run_kwargs)
        except AirflowException:
            pass

        # test tasks
        task_1, task_2 = dag.tasks
        ti = TI(task_1, ex_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, first_task_state)
        ti = TI(task_2, ex_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, second_task_state)

        # load dagrun
        dr = session.query(DagRun).filter(
            DagRun.dag_id == dag.dag_id,
            DagRun.execution_date == ex_date
        ).first()

        # dagrun is running
        self.assertEqual(dr.state, State.RUNNING)

        dag.get_active_runs()

        # dagrun failed
        self.assertEqual(dr.state, dagrun_state)

    def test_dagrun_fail(self):
        """
        DagRuns with one failed and one incomplete root task -> FAILED
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_fail',
            first_task_state=State.FAILED,
            second_task_state=State.UPSTREAM_FAILED,
            dagrun_state=State.FAILED)

    def test_dagrun_success(self):
        """
        DagRuns with one failed and one successful root task -> SUCCESS
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_success',
            first_task_state=State.FAILED,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS)

    def test_dagrun_root_fail(self):
        """
        DagRuns with one successful and one failed root task -> FAILED
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_root_fail',
            first_task_state=State.SUCCESS,
            second_task_state=State.FAILED,
            dagrun_state=State.FAILED)

    def test_dagrun_deadlock(self):
        """
        Deadlocked DagRun is marked a failure

        Test that a deadlocked dagrun is marked as a failure by having
        depends_on_past and an execution_date after the start_date
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=None,
            second_task_state=None,
            dagrun_state=State.FAILED,
            advance_execution_date=True)

    def test_scheduler_pooled_tasks(self):
        """
        Test that the scheduler handles queued tasks correctly
        See issue #1299
        """
        session = settings.Session()
        if not (
                session.query(Pool)
                .filter(Pool.pool == 'test_queued_pool')
                .first()):
            pool = Pool(pool='test_queued_pool', slots=5)
            session.merge(pool)
            session.commit()
        session.close()

        dag_id = 'test_scheduled_queued_tasks'
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()

        scheduler = SchedulerJob(dag_id, num_runs=10)
        scheduler.run()

        task_1 = dag.tasks[0]
        ti = TI(task_1, dag.start_date)
        ti.refresh_from_db()
        self.assertEqual(ti.state, State.FAILED)

        dag.clear()

    def test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date(self):
        """
        DagRun is marked a success if ignore_first_depends_on_past=True

        Test that an otherwise-deadlocked dagrun is marked as a success
        if ignore_first_depends_on_past=True and the dagrun execution_date
        is after the start_date.
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=State.SUCCESS,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS,
            advance_execution_date=True,
            run_kwargs=dict(ignore_first_depends_on_past=True))

    def test_dagrun_deadlock_ignore_depends_on_past(self):
        """
        Test that ignore_first_depends_on_past doesn't affect results
        (this is the same test as
        test_dagrun_deadlock_ignore_depends_on_past_advance_ex_date except
        that start_date == execution_date so depends_on_past is irrelevant).
        """
        self.evaluate_dagrun(
            dag_id='test_dagrun_states_deadlock',
            first_task_state=State.SUCCESS,
            second_task_state=State.SUCCESS,
            dagrun_state=State.SUCCESS,
            run_kwargs=dict(ignore_first_depends_on_past=True))

    def test_scheduler_start_date(self):
        """
        Test that the scheduler respects start_dates, even when DAGS have run
        """

        dag_id = 'test_start_date_scheduling'
        dag = self.dagbag.get_dag(dag_id)
        dag.clear()
        self.assertTrue(dag.start_date > DEFAULT_DATE)

        scheduler = SchedulerJob(dag_id, num_runs=2)
        scheduler.run()

        # zero tasks ran
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 0)

        # previously, running this backfill would kick off the Scheduler
        # because it would take the most recent run and start from there
        # That behavior still exists, but now it will only do so if after the
        # start date
        backfill = BackfillJob(
            dag=dag,
            start_date=DEFAULT_DATE,
            end_date=DEFAULT_DATE)
        backfill.run()

        # one task ran
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)

        scheduler = SchedulerJob(dag_id, num_runs=2)
        scheduler.run()

        # still one task
        session = settings.Session()
        self.assertEqual(
            len(session.query(TI).filter(TI.dag_id == dag_id).all()), 1)
<|endoftext|>


<commit_before># coding=utf8

import os
import sys
import logging
logging.basicConfig(level=logging.INFO)

import toml

sys.path.insert(0, '..')
from skylark import Database, database, DBAPI_MAPPINGS, DatabaseType,\
    Model

from models import User, Post

dbapi_name = os.environ.get('DBAPI', 'MySQLdb')
dbapi = __import__(dbapi_name)
configs = toml.loads(open('conf.toml').read())[dbapi_name]

db_type_mappings = {
    'pymysql': 'mysql',
    'MySQLdb': 'mysql',
    'sqlite3': 'sqlite',
}

db_type = db_type_mappings[dbapi_name]

user_sql = open('%s.user.sql' % db_type).read()
post_sql = open('%s.post.sql' % db_type).read()

database.set_dbapi(dbapi)
logging.info('Using DBAPI %s' % dbapi_name)
database.config(**configs)
database.set_autocommit(True)

class Test(object):

    def setUp(self):
        database.execute(user_sql)
        database.execute(post_sql)

    def tearDown(self):
        database.execute("drop table t_post")
        database.execute("drop table t_user")

class TestDatabase_:

    def setUp(self):
        self.database = DatabaseType()

    def test_alias(self):
        assert database is Database

    def __shouldnt_import(self, name):
        try:
            __import__(name)
            raise Exception
        except ImportError:
            pass

    def test_init(self):
        assert self.database.conn is None
        assert self.database.configs == {}
        assert self.database.dbapi.module.__name__ in DBAPI_MAPPINGS

        # test load orders
        name = self.database.dbapi.module.__name__

        if name == 'MySQLdb':
            assert __import__('MySQLdb')
        elif name == 'pymysql':
            assert __import__('pymysql')
            self.__shouldnt_import('MySQLdb')
        elif name == 'sqlite3':
            assert __import__('sqlite3')
            self.__shouldnt_import('MySQLdb')
            self.__shouldnt_import('pymysql')

    def test_set_dbapi(self):
        self.database.set_dbapi(dbapi)
        assert self.database.dbapi.module.__name__ == dbapi_name

class TestDatabase(Test):

    def setUp(self):
        self.database = DatabaseType()
        self.database.set_dbapi(dbapi)
        super(TestDatabase, self).setUp()

    def test_config(self):
        assert self.database.configs == {}
        self.database.config(**configs)
        assert self.database.configs
        assert not self.database.conn
        self.database.connect()
        assert self.database.conn
        self.database.config(**configs)
        assert not self.database.dbapi.conn_is_open(self.database.conn)

    def test_connect(self):
        assert self.database.conn is None
        self.database.config(**configs)
        conn1 = self.database.connect()
        conn2 = self.database.connect()
        assert conn1 is not conn2
        assert self.database.dbapi.conn_is_open(conn1)
        assert self.database.dbapi.conn_is_open(conn2)

    def test_get_conn(self):
        assert self.database.conn is None
        self.database.config(**configs)
        conn = self.database.get_conn()
        assert conn and self.database.dbapi.conn_is_open(conn)
        assert conn and self.database.dbapi.conn_is_alive(conn)
        conn1 = self.database.get_conn()
        assert conn1 is conn

    def test___del__(self):
        self.database.config(**configs)
        self.database.connect()
        conn = self.database.conn
        dbapi = self.database.dbapi
        del self.database
        assert not dbapi.conn_is_open(conn)

    def test_execute(self):
        self.database.config(**configs)
        cursor = self.database.execute(
            "insert into t_user (name, email) values ('jack', 'i@gmail.com')")
        assert cursor.lastrowid == 1

    def test_execute_sql(self):
        pass

    def change(self):
        self.database.configs(**configs)
        self.database.connect()
        old_conn = self.database.conn
        self.database.execute('create database skylarktests2')
        self.database.change('skylarktests2')
        assert self.database.configs != configs
        if db_type == 'mysql':
            self.database.conn is old_conn
        else:
            self.database.conn is not old_conn
            assert not self.database.dbapi.conn_is_open(old_conn)

    def test_transaction(self):
        db = self.database
        db.config(**configs)
        db.execute("insert into t_user (name, email) values ('j', 'j@i.com')")
        db.set_autocommit(False)
        t = db.transaction()
        try:
            db.execute("insert into t_user set x;")  # syntax error
        except Exception:
            t.rollback()
        else:
            raise Exception
        cursor = db.execute('select count(*) from t_user;')
        db.conn.commit()  # !important
        assert cursor.fetchone()[0] == 1

        with db.transaction() as t:
            db.execute(
                "insert into t_user (name, email) values ('a', 'a@b.com')")
            db.execute(
                "insert into t_user (name, email) values ('a', 'a@b.com')")
        cursor = db.execute('select count(*) from t_user')
        db.commit()
        assert cursor.fetchone()[0] == 3

class TestField_:

    def test_name(self):
        assert User.name.name == 'name'
        assert User.email.name == 'email'
        assert Post.name.name == 'name'
        assert Post.user_id.name == 'user_id'

    def test_fullname(self):
        assert User.name.fullname == 't_user.name'
        assert User.email.fullname == 't_user.email'
        assert Post.name.fullname == 't_post.name'
        assert Post.user_id.fullname == 't_post.user_id'

    def test_model(self):
        assert User.id.model is User
        assert User.name.model is User
        assert User.email.model is User
        assert Post.name.model is Post
        assert Post.user_id.model is Post

    def test_alias(self):
        fd = User.id.alias('user_id')
        assert fd.name == 'user_id'
        assert fd.inst is User.id

class TestPrimaryKey_:

    def test_is_primarykey(self):
        assert User.id.is_primarykey is True
        assert User.name.is_primarykey is False
        assert User.email.is_primarykey is False
        assert Post.post_id.is_primarykey is True
        assert Post.user_id.is_primarykey is False
        assert Post.name.is_primarykey is False

class TestForeignKey_:

    def test_is_foreignkey(self):
        assert User.id.is_foreignkey is False
        assert User.name.is_foreignkey is False
        assert User.email.is_foreignkey is False
        assert Post.post_id.is_foreignkey is False
        assert Post.name.is_foreignkey is False
        assert Post.user_id.is_foreignkey is True

class TestAlias_:

    def setUp(self):
        self._alias = User.name.alias('username')

    def test_name(self):
        assert self._alias.name == 'username'

    def test_inst(self):
        assert self._alias.inst is User.name

class TestModel_:

    def test_table_name(self):
        class MyModel(Model):
            pass
        assert MyModel.table_name == 'my_model'

        class Member(Model):
            pass
        assert Member.table_name == 'member'

        class Cat(Model):
            table_name = 'cute_cat'
        assert Cat.table_name == 'cute_cat'

    def test_table_prefix(self):
        class Users(Model):
            table_prefix = 't_'
        assert Users.table_name == 't_users'

        class CuteDog(Model):
            table_prefix = 'dd_'
        assert CuteDog.table_name == 'dd_cute_dog'

        class Dog(Model):
            table_name = 'custom_table_name'
            table_prefix = 'd_'
        assert Dog.table_name == 'd_custom_table_name'

    def test_table_prefix_is_inheritable(self):
        class X(Model):
            table_prefix = 't_'

        class A(X):
            pass

        class B(X):
            pass

        assert A.table_prefix is X.table_prefix
        assert B.table_prefix is X.table_prefix
        assert A.table_name == 't_a'
        assert B.table_name == 't_b'

    def test_primarykey(self):
        assert User.primarykey is User.id
        assert Post.primarykey is Post.post_id

    def test_fields(self):
        field_names = set(User.fields.keys())
        _field_names = set(('id', 'name', 'email'))
        assert field_names == _field_names
        field_names = set(Post.fields.keys())
        _field_names = set(('post_id', 'name', 'user_id'))
        assert field_names == _field_names

class TestModel(Test):

    def test_insert(self):
        query = User.insert(name='jack', email='jack@gmail.com')
        assert query.execute() == 1
        assert User.count() == 1
        user = User.getone()
        assert user.name == 'jack' and user.email == 'jack@gmail.com'

    def test_update(self):
        user = User.create(name='jack', email='jack@gmail.com')
        assert user.id == 1
        assert user.name == 'jack'
        assert user.email == 'jack@gmail.com'
        assert User.count() == 1

        query = User.at(1).update(email='jack@g.com')
        rows_affected = query.execute()
        assert rows_affected == 1
        user = User.getone()
        assert user.id == 1 and user.email == 'jack@g.com'

    def test_inst_in_model(self):
        user = User.create(name='jack', email='jack@gmail.com')

        # inst with `_in_db=True` won't call db to run a query
        database.conn.close()  # close conn to test if any sql was executed
        database.conn = None
        assert user in User
        assert database.conn is None

        # inst without `_in_db=True` call a query
        assert User(name='amy') not in User
        assert database.conn is not None

    def test_select(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        User.create(name='tom', email='tom@gmail.com')
        ### select all fields
        query = User.select()
        result = query.execute()
        assert result.count == 3
        for user in result.all():
            assert '%s@gmail.com' % user.name == user.email
        ### select part fields
        query = User.select(User.name)
        result = query.execute()
        assert result.count == 3
        jack = result.one()
        assert jack.name == 'jack'
        amy = result.one()
        assert amy.name == 'amy'
        tom = result.one()
        assert tom.name == 'tom'
        assert result.one() is None
        ### select with where
        query = User.where(name='jack').select()
        result = query.execute()
        assert result.count == 1
        assert result.one().name == 'jack'

    def test_delete(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        query = User.at(3).delete()
        rows_affected = query.execute()
        assert rows_affected == 0
        query = User.at(1).delete()
        rows_affected = query.execute()
        assert rows_affected == 1
        assert User.count() == 1

    def test_create(self):
        user = User.create(name='jack', email='jack@gmail.com')
        assert user.data == {
            'name': 'jack', 'email': 'jack@gmail.com', 'id': 1
        }
        assert user in User
        assert user._in_db
        assert User.count() == 1

        _user = User.getone()
        assert _user.data == user.data
        assert _user._in_db

    def test_where(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        User.create(name='tom', email='tom@gmail.com')

        assert ['jack'] == [
            user.name for user in User.where(name='jack').select()]

        assert [1] == [
            user.id for user in User.where(id=1).select()
        ]

class TestSelectResult(Test):

    def test_count(self):
        pass

    def test_one(self):
        pass

    def test_tuples(self):
        pass

    def test_selected_inst_in_db(self):
        pass

class TestOperators(Test):
    pass
<commit_msg>attemp to slove the __del__ testcase<commit_after># coding=utf8

import os
import sys
import logging
logging.basicConfig(level=logging.INFO)

import toml

sys.path.insert(0, '..')
from skylark import Database, database, DBAPI_MAPPINGS, DatabaseType,\
    Model

from models import User, Post

dbapi_name = os.environ.get('DBAPI', 'MySQLdb')
dbapi = __import__(dbapi_name)
configs = toml.loads(open('conf.toml').read())[dbapi_name]

db_type_mappings = {
    'pymysql': 'mysql',
    'MySQLdb': 'mysql',
    'sqlite3': 'sqlite',
}

db_type = db_type_mappings[dbapi_name]

user_sql = open('%s.user.sql' % db_type).read()
post_sql = open('%s.post.sql' % db_type).read()

database.set_dbapi(dbapi)
logging.info('Using DBAPI %s' % dbapi_name)
database.config(**configs)
database.set_autocommit(True)

class Test(object):

    def setUp(self):
        database.execute(user_sql)
        database.execute(post_sql)

    def tearDown(self):
        database.execute("drop table t_post")
        database.execute("drop table t_user")

class TestDatabase_:

    def setUp(self):
        self.database = DatabaseType()

    def test_alias(self):
        assert database is Database

    def __shouldnt_import(self, name):
        try:
            __import__(name)
            raise Exception
        except ImportError:
            pass

    def test_init(self):
        assert self.database.conn is None
        assert self.database.configs == {}
        assert self.database.dbapi.module.__name__ in DBAPI_MAPPINGS

        # test load orders
        name = self.database.dbapi.module.__name__

        if name == 'MySQLdb':
            assert __import__('MySQLdb')
        elif name == 'pymysql':
            assert __import__('pymysql')
            self.__shouldnt_import('MySQLdb')
        elif name == 'sqlite3':
            assert __import__('sqlite3')
            self.__shouldnt_import('MySQLdb')
            self.__shouldnt_import('pymysql')

    def test_set_dbapi(self):
        self.database.set_dbapi(dbapi)
        assert self.database.dbapi.module.__name__ == dbapi_name

class TestDatabase(Test):

    def setUp(self):
        self.database = DatabaseType()
        self.database.set_dbapi(dbapi)
        super(TestDatabase, self).setUp()

    def test_config(self):
        assert self.database.configs == {}
        self.database.config(**configs)
        assert self.database.configs
        assert not self.database.conn
        self.database.connect()
        assert self.database.conn
        self.database.config(**configs)
        assert not self.database.dbapi.conn_is_open(self.database.conn)

    def test_connect(self):
        assert self.database.conn is None
        self.database.config(**configs)
        conn1 = self.database.connect()
        conn2 = self.database.connect()
        assert conn1 is not conn2
        assert self.database.dbapi.conn_is_open(conn1)
        assert self.database.dbapi.conn_is_open(conn2)

    def test_get_conn(self):
        assert self.database.conn is None
        self.database.config(**configs)
        conn = self.database.get_conn()
        assert conn and self.database.dbapi.conn_is_open(conn)
        assert conn and self.database.dbapi.conn_is_alive(conn)
        conn1 = self.database.get_conn()
        assert conn1 is conn

    def test___del__(self):
        self.database.config(**configs)
        self.database.connect()
        conn = self.database.conn
        dbapi = self.database.dbapi
        del self.database
        conn = None
        assert not dbapi.conn_is_open(conn)

    def test_execute(self):
        self.database.config(**configs)
        cursor = self.database.execute(
            "insert into t_user (name, email) values ('jack', 'i@gmail.com')")
        assert cursor.lastrowid == 1

    def test_execute_sql(self):
        pass

    def change(self):
        self.database.configs(**configs)
        self.database.connect()
        old_conn = self.database.conn
        self.database.execute('create database skylarktests2')
        self.database.change('skylarktests2')
        assert self.database.configs != configs
        if db_type == 'mysql':
            self.database.conn is old_conn
        else:
            self.database.conn is not old_conn
            assert not self.database.dbapi.conn_is_open(old_conn)

    def test_transaction(self):
        db = self.database
        db.config(**configs)
        db.execute("insert into t_user (name, email) values ('j', 'j@i.com')")
        db.set_autocommit(False)
        t = db.transaction()
        try:
            db.execute("insert into t_user set x;")  # syntax error
        except Exception:
            t.rollback()
        else:
            raise Exception
        cursor = db.execute('select count(*) from t_user;')
        db.conn.commit()  # !important
        assert cursor.fetchone()[0] == 1

        with db.transaction() as t:
            db.execute(
                "insert into t_user (name, email) values ('a', 'a@b.com')")
            db.execute(
                "insert into t_user (name, email) values ('a', 'a@b.com')")
        cursor = db.execute('select count(*) from t_user')
        db.commit()
        assert cursor.fetchone()[0] == 3

class TestField_:

    def test_name(self):
        assert User.name.name == 'name'
        assert User.email.name == 'email'
        assert Post.name.name == 'name'
        assert Post.user_id.name == 'user_id'

    def test_fullname(self):
        assert User.name.fullname == 't_user.name'
        assert User.email.fullname == 't_user.email'
        assert Post.name.fullname == 't_post.name'
        assert Post.user_id.fullname == 't_post.user_id'

    def test_model(self):
        assert User.id.model is User
        assert User.name.model is User
        assert User.email.model is User
        assert Post.name.model is Post
        assert Post.user_id.model is Post

    def test_alias(self):
        fd = User.id.alias('user_id')
        assert fd.name == 'user_id'
        assert fd.inst is User.id

class TestPrimaryKey_:

    def test_is_primarykey(self):
        assert User.id.is_primarykey is True
        assert User.name.is_primarykey is False
        assert User.email.is_primarykey is False
        assert Post.post_id.is_primarykey is True
        assert Post.user_id.is_primarykey is False
        assert Post.name.is_primarykey is False

class TestForeignKey_:

    def test_is_foreignkey(self):
        assert User.id.is_foreignkey is False
        assert User.name.is_foreignkey is False
        assert User.email.is_foreignkey is False
        assert Post.post_id.is_foreignkey is False
        assert Post.name.is_foreignkey is False
        assert Post.user_id.is_foreignkey is True

class TestAlias_:

    def setUp(self):
        self._alias = User.name.alias('username')

    def test_name(self):
        assert self._alias.name == 'username'

    def test_inst(self):
        assert self._alias.inst is User.name

class TestModel_:

    def test_table_name(self):
        class MyModel(Model):
            pass
        assert MyModel.table_name == 'my_model'

        class Member(Model):
            pass
        assert Member.table_name == 'member'

        class Cat(Model):
            table_name = 'cute_cat'
        assert Cat.table_name == 'cute_cat'

    def test_table_prefix(self):
        class Users(Model):
            table_prefix = 't_'
        assert Users.table_name == 't_users'

        class CuteDog(Model):
            table_prefix = 'dd_'
        assert CuteDog.table_name == 'dd_cute_dog'

        class Dog(Model):
            table_name = 'custom_table_name'
            table_prefix = 'd_'
        assert Dog.table_name == 'd_custom_table_name'

    def test_table_prefix_is_inheritable(self):
        class X(Model):
            table_prefix = 't_'

        class A(X):
            pass

        class B(X):
            pass

        assert A.table_prefix is X.table_prefix
        assert B.table_prefix is X.table_prefix
        assert A.table_name == 't_a'
        assert B.table_name == 't_b'

    def test_primarykey(self):
        assert User.primarykey is User.id
        assert Post.primarykey is Post.post_id

    def test_fields(self):
        field_names = set(User.fields.keys())
        _field_names = set(('id', 'name', 'email'))
        assert field_names == _field_names
        field_names = set(Post.fields.keys())
        _field_names = set(('post_id', 'name', 'user_id'))
        assert field_names == _field_names

class TestModel(Test):

    def test_insert(self):
        query = User.insert(name='jack', email='jack@gmail.com')
        assert query.execute() == 1
        assert User.count() == 1
        user = User.getone()
        assert user.name == 'jack' and user.email == 'jack@gmail.com'

    def test_update(self):
        user = User.create(name='jack', email='jack@gmail.com')
        assert user.id == 1
        assert user.name == 'jack'
        assert user.email == 'jack@gmail.com'
        assert User.count() == 1

        query = User.at(1).update(email='jack@g.com')
        rows_affected = query.execute()
        assert rows_affected == 1
        user = User.getone()
        assert user.id == 1 and user.email == 'jack@g.com'

    def test_inst_in_model(self):
        user = User.create(name='jack', email='jack@gmail.com')

        # inst with `_in_db=True` won't call db to run a query
        database.conn.close()  # close conn to test if any sql was executed
        database.conn = None
        assert user in User
        assert database.conn is None

        # inst without `_in_db=True` call a query
        assert User(name='amy') not in User
        assert database.conn is not None

    def test_select(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        User.create(name='tom', email='tom@gmail.com')
        ### select all fields
        query = User.select()
        result = query.execute()
        assert result.count == 3
        for user in result.all():
            assert '%s@gmail.com' % user.name == user.email
        ### select part fields
        query = User.select(User.name)
        result = query.execute()
        assert result.count == 3
        jack = result.one()
        assert jack.name == 'jack'
        amy = result.one()
        assert amy.name == 'amy'
        tom = result.one()
        assert tom.name == 'tom'
        assert result.one() is None
        ### select with where
        query = User.where(name='jack').select()
        result = query.execute()
        assert result.count == 1
        assert result.one().name == 'jack'

    def test_delete(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        query = User.at(3).delete()
        rows_affected = query.execute()
        assert rows_affected == 0
        query = User.at(1).delete()
        rows_affected = query.execute()
        assert rows_affected == 1
        assert User.count() == 1

    def test_create(self):
        user = User.create(name='jack', email='jack@gmail.com')
        assert user.data == {
            'name': 'jack', 'email': 'jack@gmail.com', 'id': 1
        }
        assert user in User
        assert user._in_db
        assert User.count() == 1

        _user = User.getone()
        assert _user.data == user.data
        assert _user._in_db

    def test_where(self):
        User.create(name='jack', email='jack@gmail.com')
        User.create(name='amy', email='amy@gmail.com')
        User.create(name='tom', email='tom@gmail.com')

        assert ['jack'] == [
            user.name for user in User.where(name='jack').select()]

        assert [1] == [
            user.id for user in User.where(id=1).select()
        ]

class TestSelectResult(Test):

    def test_count(self):
        pass

    def test_one(self):
        pass

    def test_tuples(self):
        pass

    def test_selected_inst_in_db(self):
        pass

class TestOperators(Test):
    pass
<|endoftext|>


<commit_before>#! python3
# -*- coding: utf-8 -*-
"""Checks messages in database and sends PM if expiration time passed.

This module checks messages saved in a MySQL-compatible database and sends a reminder
via PM if the expiration time has passed. If so, the message is removed from the
database.
"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

import clashcallerbot_database as db

# Logger
logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

def main():
    while True:
        time.sleep(30)

        # Get list of messages older than current datetime
        now = datetime.datetime.now(datetime.timezone.utc)
        messages = db.get_messages(now)

        if not messages:
            continue

        # Send reminder PM
        for message in messages:
            tid, link, msg, _exp, usr = message
            send_reminder(link, msg, usr)

            # Delete message from database
            db.delete_message(tid)

def send_reminder(link: str, msg: str, usr: str)-> bool:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    Returns:
        True if successful, False otherwise.
    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')
        return False
    return True

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        parent: Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
        return parent
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<commit_msg>Update clashcallerbot_reply.py added logger messages<commit_after>#! python3
# -*- coding: utf-8 -*-
"""Checks messages in database and sends PM if expiration time passed.

This module checks messages saved in a MySQL-compatible database and sends a reminder
via PM if the expiration time has passed. If so, the message is removed from the
database.
"""

import praw
import praw.exceptions

import logging.config
import datetime
import time

import clashcallerbot_database as db

# Logger
logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
logging.raiseExceptions = True  # Production mode if False (no console sys.stderr output)
logger = logging.getLogger('reply')

# Generate reddit instance
reddit = praw.Reddit('clashcallerreply')  # Section name in praw.ini
subreddit = reddit.subreddit('ClashCallerBot')  # Limit scope for testing purposes

def main():
    while True:
        time.sleep(30)

        # Get list of messages older than current datetime
        now = datetime.datetime.now(datetime.timezone.utc)
        messages = db.get_messages(now)

        if not messages:
            logger.error('No old messages.')
            continue

        # Send reminder PM
        for message in messages:
            tid, link, msg, _exp, usr = message
            logger.debug(f'Found message: {message}')
            if send_reminder(link, msg, usr):
                logger.info(f'Reminder sent to {usr}.')

            # Delete message from database
            if db.delete_message(tid):
                logger.info(f'Message deleted.')

def send_reminder(link: str, msg: str, usr: str)-> bool:
    """Sends reminder PM to username.

    Function sends given permalink and message to given username.

    Args:
         link:  Permalink to comment.
         msg:   Message in comment that was saved.
         usr:   User to send reminder to.

    Returns:
        True if successful, False otherwise.
    """
    subject = 'ClashCallerBot Private Message Here!'
    permalink = 'https://np.reddit.com' + link  # Permalinks are missing prefix
    parent = get_parent(link)
    message = f"""**The message:** {msg}
              **The original comment:** {permalink}
              **The parent comment or submission:** {parent}

              Thank you for entrusting us with your warring needs,
              - ClashCallerBot

              [^(More info)](https://www.reddit.com/r/ClashCallerBot/comments/4e9vo7/clashcallerbot_info/)
              """
    try:
        reddit.redditor(usr).message(subject, message.replace('              ', ''))

    except praw.exceptions.PRAWException as err:
        logger.exception(f'send_reminder: {err}')
        return False
    return True

def get_parent(link: str) -> str:
    """Fetch parent comment or submission.

    Function gets parent comment of given permalink or submission if top level comment.

    Args:
         link:  Permalink to get parent of.

    Returns:
        parent: Parent comment link or default string.
    """
    permalink = 'https://www.reddit.com' + link  # Permalink is missing prefix
    parent = 'Parent comment not found.'  # Default string
    try:
        comment = reddit.comment(url=permalink)  # Fetch comment by URL
        parent = 'https://np.reddit.com' + comment.submission.permalink
    except praw.exceptions as err:
        logger.exception(f'get_parent: {err}')
        return parent
    return parent

# If run directly, instead of imported as a module, run main():
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

"""
Tests all commands that start with 'cfy nodes'
"""

from uuid import uuid4

from mock import MagicMock
from cloudify_rest_client.nodes import Node
from cloudify_rest_client.node_instances import NodeInstance

from cloudify_cli.tests import cli_runner
from cloudify_cli.tests.commands.test_cli_command import CliCommandTest

class NodesTest(CliCommandTest):

    def setUp(self):
        super(NodesTest, self).setUp()
        self._create_cosmo_wd_settings()

    def test_nodes_get(self):
        self.client.nodes.get = MagicMock(return_value=node_get_mock())
        self.client.node_instances.list = \
            MagicMock(return_value=[node_instance_get_mock()])
        cli_runner.run_cli('cfy nodes get --node-id mongod -d nodecellar')

        with self.assertRaises(SystemExit) as sys_exit:
            cli_runner.run_cli('cfy nodes get --node-id mongod')
        self.assertNotEquals(sys_exit.exception.code, 0)

        with self.assertRaises(SystemExit) as sys_exit:
            cli_runner.run_cli('cfy nodes get -d nodecellar')
        self.assertNotEquals(sys_exit.exception.code, 0)

    def test_nodes_list(self):
        self.client.nodes.list = MagicMock(return_value=[node_get_mock(),
                                                         node_get_mock()])
        cli_runner.run_cli('cfy nodes list')
        cli_runner.run_cli('cfy nodes list -d nodecellar')

def node_get_mock():
    return Node({
        'id': uuid4(),
        'deployment_id': 'deployment-id',
        'type': 'Compute',
        'number_of_instances': '1',
        'planned_number_of_instances': '2',
        'properties': {
            'port': '8080'
        }
    })

def node_instance_get_mock():
    return NodeInstance({
        'id': uuid4(),
        'deployment_id': 'deployment_id',
        'host_id': 'host_id',
        'node_id': 'node_id',
        'state': 'started',
        'runtime_properties': {
            'floating_ip': '127.0.0.1'
        }
    })
<commit_msg>CFY-3076 fixed nodes command test<commit_after>########
# Copyright (c) 2015 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

"""
Tests all commands that start with 'cfy nodes'
"""

from uuid import uuid4

from mock import MagicMock
from cloudify_rest_client.nodes import Node
from cloudify_rest_client.node_instances import NodeInstance

from cloudify_cli.tests import cli_runner
from cloudify_cli.tests.commands.test_cli_command import CliCommandTest

class NodesTest(CliCommandTest):

    def setUp(self):
        super(NodesTest, self).setUp()
        self._create_cosmo_wd_settings()

    def test_nodes_get(self):
        self.client.nodes.get = MagicMock(return_value=node_get_mock())
        self.client.node_instances.list = \
            MagicMock(return_value=[node_instance_get_mock()])
        cli_runner.run_cli('cfy nodes get --node-id mongod -d nodecellar')

        with self.assertRaises(SystemExit) as sys_exit:
            cli_runner.run_cli('cfy nodes get --node-id mongod')
        self.assertNotEquals(sys_exit.exception.code, 0)

        with self.assertRaises(SystemExit) as sys_exit:
            cli_runner.run_cli('cfy nodes get -d nodecellar')
        self.assertNotEquals(sys_exit.exception.code, 0)

    def test_nodes_list(self):
        self.client.nodes.list = MagicMock(return_value=[node_get_mock(),
                                                         node_get_mock()])
        cli_runner.run_cli('cfy nodes list')
        cli_runner.run_cli('cfy nodes list -d nodecellar')

def node_get_mock():
    return Node({
        'id': uuid4(),
        'deployment_id': 'deployment-id',
        'blueprint_id': 'blueprint_id',
        'host_id': 'host_id',
        'type': 'Compute',
        'number_of_instances': '1',
        'planned_number_of_instances': '2',
        'properties': {
            'port': '8080'
        }
    })

def node_instance_get_mock():
    return NodeInstance({
        'id': uuid4(),
        'deployment_id': 'deployment_id',
        'host_id': 'host_id',
        'node_id': 'node_id',
        'state': 'started',
        'runtime_properties': {
            'floating_ip': '127.0.0.1'
        }
    })
<|endoftext|>


<commit_before>"""
Ludolph: Monitoring Jabber bot
Copyright (C) 2012-2015 Erigones, s. r. o.
This file is part of Ludolph.

See the file LICENSE for copying permission.
"""
<commit_msg>Updated copyright year<commit_after>"""
Ludolph: Monitoring Jabber bot
Copyright (C) 2012-2016 Erigones, s. r. o.
This file is part of Ludolph.

See the file LICENSE for copying permission.
"""
<|endoftext|>


<commit_before>'''Pretty-printing of intermediate representation code.
'''

import math

from transf.lib import *
from transf import parse

from lang import box
from lang.box import op
from lang.box import kw
from lang.box import lit
from lang.box import sym
from lang.box import commas
from lang.box import Path

#######################################################################
# Int Literals

def _entropy(seq, states):
	state_freqs = dict.fromkeys(states, 0)
	for state in seq:
		state_freqs[state] += 1
	entropy = 0
	nstates = len(seq)
	for freq in state_freqs.itervalues():
		prob = float(freq)/nstates
		if prob:
			entropy -= prob*math.log(prob)
	return entropy

@util.Adaptor
def intrepr(term):
	'''Represent integers, choosing the most suitable (lowest entropy)
	representation.
	'''

	val = term.value

	d = "%d" % abs(val)
	x = "%x" % abs(val)
	sd = _entropy(d, "0123456789")
	sx = _entropy(x, "0123456789abcdef")
	if sx < sd:
		rep = hex(val)
	else:
		rep = str(val)

	return term.factory.makeStr(rep)

intlit = intrepr * box.const

#######################################################################
# Types

parse.Transfs('''
sign = {
	Signed -> H([ <<kw> "signed">, " " ])
|	Unsigned -> H([ <<kw> "unsigned"> , " " ])
|	NoSign -> ""
}

size = {
	8 -> <<kw> "char">
|	16 -> H([ <<kw> "short">, " ", <<kw> "int"> ])
|	32 -> <<kw> "int">
|	64 -> H([ <<kw> "long">, " ", <<kw> "int"> ])
|	n -> H([ "int", <<strings.tostr> n> ])
}

type = rec type : {
	Void
		-> <<kw> "void">
|	Bool
		-> <<kw> "bool">
|	Int(size, sign)
		-> H([ <<sign> sign>, <<size> size> ])
|	Float(32)
		-> <<kw> "float">
|	Float(64)
		-> <<kw> "double">
|	Char(size)
		-> <<kw> "char">
|	Pointer(size, type)
		-> H([ <<type> type>, " ", <<op> "*"> ])
|	Array(type)
		-> H([ <<type> type>, "[", "]" ])
|	Blob(size)
		-> H([ "blob", <<strings.tostr> size> ])
|	_ -> "???"
}
''')

#######################################################################
# Operator precendence.
#
# See http://www.difranco.net/cop2220/op-prec.htm

unOpPrec = parse.Rule('''
	Not -> 1
|	Neg -> 1
''')

binOpPrec = parse.Rule('''
	And(Bool) -> 10
|	Or(Bool) -> 11
|	And(_) -> 7
|	Or(_) -> 9
|	Xor(_) -> 8
|	LShift -> 4
|	RShift -> 4
|	Plus -> 4 # force parenthesis inside shifts (was 3)
|	Minus -> 4 # force parenthesis inside shifts (was 3)
|	Mult -> 2
|	Div -> 2
|	Mod -> 2
|	Eq -> 6
|	NotEq -> 6
|	Lt -> 5
|	LtEq -> 5
|	Gt -> 5
|	GtEq -> 5
''')

exprPrec = parse.Rule('''
	Lit(_, _) -> 0
|	Sym(_) -> 0
|	Cast(_, _) -> 1
|	Addr(_) -> 1
|	Ref(_) -> 1
|	Unary(op, _) -> <<unOpPrec>op>
|	Binary(op, _, _) -> <<binOpPrec>op>
|	Cond(_, _, _) -> 13
|	Call(_, _) -> 0
''')

stmtPrec = parse.Transf('''
	!99
''')

#######################################################################
# Expressions

unOp = parse.Rule('''
	Not(Bool) -> "!"
|	Not(_) -> "~"
|	Neg -> "-"
''')

binOp = parse.Rule('''
	And(Bool) -> "&&"
|	Or(Bool) -> "||"
|	And(_) -> "&"
|	Or(_) -> "|"
|	Xor(_) -> "^"
|	LShift -> "<<"
|	RShift -> ">>"
|	Plus -> "+"
|	Minus -> "-"
|	Mult -> "*"
|	Div -> "/"
|	Mod -> "%"
|	Eq -> "=="
|	NotEq -> "!="
|	Lt -> "<"
|	LtEq -> "<="
|	Gt -> ">"
|	GtEq -> ">="
''')

exprKern = util.Proxy()

parse.Transfs('''
SubExpr(Cmp) =
	let
		pprec = !prec, # parent precedence
		prec = exprPrec
	in
		if Cmp(!prec, !pprec) then
			!H([ "(", <exprKern>, ")" ])
		else
			exprKern
		end
	end

''')
subExpr = SubExpr(arith.Gt)
subExprEq = SubExpr(arith.Geq)

exprKern.subject = Path(parse.Rule('''
	Lit(Int(_,_), value)
		-> <<intlit> value>
|	Lit(type, value)
		-> <<lit> value>
|	Sym(name)
		-> <<sym> name>
|	Cast(type, expr)
		-> H([ "(", <<type>type>, ")", " ", <<subExpr>expr> ])
|	Unary(op, expr)
		-> H([ <<unOp>op>, <<subExpr>expr> ])
|	Binary(op, lexpr, rexpr)
		-> H([ <<subExpr>lexpr>, " ", <<binOp>op>, " ", <<subExprEq>rexpr> ])
|	Cond(cond, texpr, fexpr)
		-> H([ <<subExpr>cond>, " ", <<op>"?">, " ", <<subExpr>texpr>, " ", <<op>":">, " ", <<subExpr>fexpr> ])
|	Call(addr, args)
		-> H([ <<subExpr>addr>, "(", <<Map(subExpr);commas> args>, ")" ])
|	Addr(addr)
		-> H([ <<op>"&">, <<subExpr>addr> ])
|	Ref(expr)
		-> H([ <<op>"*">, <<subExpr>expr> ])
'''))

expr = parse.Transf('''
	let
		prec = exprPrec
	in
		exprKern
	end
''')

#######################################################################
# Statements

arg = parse.Rule('''
	Arg(type, name)
		-> H([ <<type>type>, " ", name ])
''')

stmt = util.Proxy()

stmts = parse.Transf('''
	!V( <Map(stmt)> )
''')

stmtKern = parse.Rule('''
	Assign(Void, NoExpr, src)
		-> H([ <<expr>src> ])
|	Assign(_, dst, src)
		-> H([ <<expr>dst>, " ", <<op>"=">, " ", <<expr>src> ])
|	If(cond, _, _)
		-> H([ <<kw>"if">, "(", <<expr>cond>, ")" ])
|	While(cond, _)
		-> H([ <<kw>"while">, "(", <<expr>cond>, ")" ])
|	DoWhile(cond, _)
		-> H([ <<kw>"while">, "(", <<expr>cond>, ")" ])
|	Var(type, name, NoExpr)
		-> H([ <<type>type>, " ", name ])
|	Var(type, name, val)
		-> H([ <<type>type>, " ", name, "=", <<expr>val> ])
|	Function(type, name, args, stmts)
		-> H([ <<type>type>, " ", name, "(", <<Map(arg);commas> args>, ")" ])
|	Label(name)
		-> H([ name, ":" ])
|	GoTo(label)
		-> H([ <<kw>"goto">, " ", <<expr>label> ])
|	Ret(_, NoExpr)
		-> H([ <<kw>"return"> ])
|	Ret(_, value)
		-> H([ <<kw>"return">, " ", <<expr>value> ])
|	NoStmt
		-> ""
|	Asm(opcode, operands)
		-> H([ <<kw>"asm">, "(", <<commas>[<<lit> opcode>, *<<Map(expr)>operands>]>, ")" ])
''')

parse.Transfs('''
ppLabel = {
	Label
		-> D( <stmtKern> )
}

ppBlock = {
	Block( stmts )
		-> V([
			D("{"),
				<<stmts>stmts>,
			D("}")
		])
}

ppIf = {
	If(_, true, NoStmt)
		-> V([
			<stmtKern>,
				I( <<stmt>true> )
		])
|	If(_, true, false)
		-> V([
			<stmtKern>,
				I( <<stmt>true> ),
			H([ <<kw>"else"> ]),
				I( <<stmt>false> )
		])
}

ppWhile = {
	While(_, body)
		-> V([
			<stmtKern>,
				I( <<stmt>body> )
		])
}

ppDoWhile = {
	DoWhile(_, body)
		-> V([
			H([ <<kw>"do"> ]),
				I( <<stmt>body> ),
			!H([ <stmtKern>, ";" ])
		])
}

ppFunction = {
	Function(_, _, _, stmts)
		-> D(V([
			<stmtKern>,
			"{",
				I(V([ <<stmts>stmts> ])),
			"}"
		]))
}

ppDefault =
	!H([ <stmtKern>, ";" ])

''')

stmt.subject = Path(parse.Transf('''
	switch project.name
		case "Label": ppLabel
		case "Block": ppBlock
		case "If": ppIf
		case "While": ppWhile
		case "DoWhile": ppDoWhile
		case "Function": ppFunction
		else: ppDefault
	end
'''))

module = Path(parse.Rule('''
	Module(stmts)
		-> V([
			I( <<stmts>stmts> )
		])
'''))

#######################################################################
# Test

if __name__ == '__main__':
	import aterm.factory
	import box
	import check

	print commas('[1,2,3]')

	factory = aterm.factory.factory

	exprTestCases = [
		('Binary(Plus(Int(32,Signed)),Lit(Int(32,Unsigned),1),Sym("x"))', '1 + x\n'),
		('Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])}', ''),
		('Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}', '')
	]

	for inputStr, output in exprTestCases:
		input = factory.parse(inputStr)

		print check.expr(input)
		result = expr(inputStr)
		print result
		print box.stringify(result)
		print output
		print

	stmtTestCases = [
		('Label("label")', 'label:\n'),
		('Asm("ret",[])', 'asm("ret");\n'),
		('Asm("mov",[Sym("ax"), Lit(Int(32,Signed),1234)])', 'asm("mov", ax, 1234);\n'),
		('Function(Void,"main",[],[])', 'void main()\n{\n}\n'),
		('Assign(Void,Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])},Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}',''),
		('Assign(Blob(32{Path([0,0,1,0])}){Path([0,1,0])},Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])},Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}',''),
		('If(Binary(Eq(Int(32,Signed)),Binary(Or(Int(32,NoSign)),Binary(Xor(Int(32,NoSign)),Sym("NF"),Sym("OF")),Sym("ZF")),Lit(Int(32,Signed),1)),GoTo(Sym(".L4")),NoStmt)', ''),
	]

	for inputStr, output in stmtTestCases:
		input = factory.parse(inputStr)

		print input
		print check.stmt(input)
		result = stmt(input)
		print result
		print box.stringify(result)
		print output
		print

<commit_msg>Rephrase integer types.<commit_after>'''Pretty-printing of intermediate representation code.
'''

import math

from transf.lib import *
from transf import parse

from lang import box
from lang.box import op
from lang.box import kw
from lang.box import lit
from lang.box import sym
from lang.box import commas
from lang.box import Path

#######################################################################
# Int Literals

def _entropy(seq, states):
	state_freqs = dict.fromkeys(states, 0)
	for state in seq:
		state_freqs[state] += 1
	entropy = 0
	nstates = len(seq)
	for freq in state_freqs.itervalues():
		prob = float(freq)/nstates
		if prob:
			entropy -= prob*math.log(prob)
	return entropy

@util.Adaptor
def intrepr(term):
	'''Represent integers, choosing the most suitable (lowest entropy)
	representation.
	'''

	val = term.value

	d = "%d" % abs(val)
	x = "%x" % abs(val)
	sd = _entropy(d, "0123456789")
	sx = _entropy(x, "0123456789abcdef")
	if sx < sd:
		rep = hex(val)
	else:
		rep = str(val)

	return term.factory.makeStr(rep)

intlit = intrepr * box.const

#######################################################################
# Types

parse.Transfs('''
sign = {
	Signed -> H([ <<kw> "signed">, " " ])
|	Unsigned -> H([ <<kw> "unsigned"> , " " ])
|	NoSign -> ""
}

size =
	switch id
	case 8:
		!<<kw> "char">
	case 16:
		!H([ <<kw> "short">, " ", <<kw> "int"> ])
	case 32:
		!<<kw> "int">
	case 64:
		!H([ <<kw> "long">, " ", <<kw> "int"> ])
	else:
		!H([ "int", <<strings.tostr> n> ])
	end

type = rec type : {
	Void
		-> <<kw> "void">
|	Bool
		-> <<kw> "bool">
|	Int(size, sign)
		-> H([ <<sign> sign>, <<size> size> ])
|	Float(32)
		-> <<kw> "float">
|	Float(64)
		-> <<kw> "double">
|	Char(size)
		-> <<kw> "char">
|	Pointer(size, type)
		-> H([ <<type> type>, " ", <<op> "*"> ])
|	Array(type)
		-> H([ <<type> type>, "[", "]" ])
|	Blob(size)
		-> H([ "blob", <<strings.tostr> size> ])
|	_ -> "???"
}
''')

#######################################################################
# Operator precendence.
#
# See http://www.difranco.net/cop2220/op-prec.htm

unOpPrec = parse.Rule('''
	Not -> 1
|	Neg -> 1
''')

binOpPrec = parse.Rule('''
	And(Bool) -> 10
|	Or(Bool) -> 11
|	And(_) -> 7
|	Or(_) -> 9
|	Xor(_) -> 8
|	LShift -> 4
|	RShift -> 4
|	Plus -> 4 # force parenthesis inside shifts (was 3)
|	Minus -> 4 # force parenthesis inside shifts (was 3)
|	Mult -> 2
|	Div -> 2
|	Mod -> 2
|	Eq -> 6
|	NotEq -> 6
|	Lt -> 5
|	LtEq -> 5
|	Gt -> 5
|	GtEq -> 5
''')

exprPrec = parse.Rule('''
	Lit(_, _) -> 0
|	Sym(_) -> 0
|	Cast(_, _) -> 1
|	Addr(_) -> 1
|	Ref(_) -> 1
|	Unary(op, _) -> <<unOpPrec>op>
|	Binary(op, _, _) -> <<binOpPrec>op>
|	Cond(_, _, _) -> 13
|	Call(_, _) -> 0
''')

stmtPrec = parse.Transf('''
	!99
''')

#######################################################################
# Expressions

unOp = parse.Rule('''
	Not(Bool) -> "!"
|	Not(_) -> "~"
|	Neg -> "-"
''')

binOp = parse.Rule('''
	And(Bool) -> "&&"
|	Or(Bool) -> "||"
|	And(_) -> "&"
|	Or(_) -> "|"
|	Xor(_) -> "^"
|	LShift -> "<<"
|	RShift -> ">>"
|	Plus -> "+"
|	Minus -> "-"
|	Mult -> "*"
|	Div -> "/"
|	Mod -> "%"
|	Eq -> "=="
|	NotEq -> "!="
|	Lt -> "<"
|	LtEq -> "<="
|	Gt -> ">"
|	GtEq -> ">="
''')

exprKern = util.Proxy()

parse.Transfs('''
SubExpr(Cmp) =
	let
		pprec = !prec, # parent precedence
		prec = exprPrec
	in
		if Cmp(!prec, !pprec) then
			!H([ "(", <exprKern>, ")" ])
		else
			exprKern
		end
	end

''')
subExpr = SubExpr(arith.Gt)
subExprEq = SubExpr(arith.Geq)

exprKern.subject = Path(parse.Rule('''
	Lit(Int(_,_), value)
		-> <<intlit> value>
|	Lit(type, value)
		-> <<lit> value>
|	Sym(name)
		-> <<sym> name>
|	Cast(type, expr)
		-> H([ "(", <<type>type>, ")", " ", <<subExpr>expr> ])
|	Unary(op, expr)
		-> H([ <<unOp>op>, <<subExpr>expr> ])
|	Binary(op, lexpr, rexpr)
		-> H([ <<subExpr>lexpr>, " ", <<binOp>op>, " ", <<subExprEq>rexpr> ])
|	Cond(cond, texpr, fexpr)
		-> H([ <<subExpr>cond>, " ", <<op>"?">, " ", <<subExpr>texpr>, " ", <<op>":">, " ", <<subExpr>fexpr> ])
|	Call(addr, args)
		-> H([ <<subExpr>addr>, "(", <<Map(subExpr);commas> args>, ")" ])
|	Addr(addr)
		-> H([ <<op>"&">, <<subExpr>addr> ])
|	Ref(expr)
		-> H([ <<op>"*">, <<subExpr>expr> ])
'''))

expr = parse.Transf('''
	let
		prec = exprPrec
	in
		exprKern
	end
''')

#######################################################################
# Statements

arg = parse.Rule('''
	Arg(type, name)
		-> H([ <<type>type>, " ", name ])
''')

stmt = util.Proxy()

stmts = parse.Transf('''
	!V( <Map(stmt)> )
''')

stmtKern = parse.Rule('''
	Assign(Void, NoExpr, src)
		-> H([ <<expr>src> ])
|	Assign(_, dst, src)
		-> H([ <<expr>dst>, " ", <<op>"=">, " ", <<expr>src> ])
|	If(cond, _, _)
		-> H([ <<kw>"if">, "(", <<expr>cond>, ")" ])
|	While(cond, _)
		-> H([ <<kw>"while">, "(", <<expr>cond>, ")" ])
|	DoWhile(cond, _)
		-> H([ <<kw>"while">, "(", <<expr>cond>, ")" ])
|	Var(type, name, NoExpr)
		-> H([ <<type>type>, " ", name ])
|	Var(type, name, val)
		-> H([ <<type>type>, " ", name, "=", <<expr>val> ])
|	Function(type, name, args, stmts)
		-> H([ <<type>type>, " ", name, "(", <<Map(arg);commas> args>, ")" ])
|	Label(name)
		-> H([ name, ":" ])
|	GoTo(label)
		-> H([ <<kw>"goto">, " ", <<expr>label> ])
|	Ret(_, NoExpr)
		-> H([ <<kw>"return"> ])
|	Ret(_, value)
		-> H([ <<kw>"return">, " ", <<expr>value> ])
|	NoStmt
		-> ""
|	Asm(opcode, operands)
		-> H([ <<kw>"asm">, "(", <<commas>[<<lit> opcode>, *<<Map(expr)>operands>]>, ")" ])
''')

parse.Transfs('''
ppLabel = {
	Label
		-> D( <stmtKern> )
}

ppBlock = {
	Block( stmts )
		-> V([
			D("{"),
				<<stmts>stmts>,
			D("}")
		])
}

ppIf = {
	If(_, true, NoStmt)
		-> V([
			<stmtKern>,
				I( <<stmt>true> )
		])
|	If(_, true, false)
		-> V([
			<stmtKern>,
				I( <<stmt>true> ),
			H([ <<kw>"else"> ]),
				I( <<stmt>false> )
		])
}

ppWhile = {
	While(_, body)
		-> V([
			<stmtKern>,
				I( <<stmt>body> )
		])
}

ppDoWhile = {
	DoWhile(_, body)
		-> V([
			H([ <<kw>"do"> ]),
				I( <<stmt>body> ),
			!H([ <stmtKern>, ";" ])
		])
}

ppFunction = {
	Function(_, _, _, stmts)
		-> D(V([
			<stmtKern>,
			"{",
				I(V([ <<stmts>stmts> ])),
			"}"
		]))
}

ppDefault =
	!H([ <stmtKern>, ";" ])

''')

stmt.subject = Path(parse.Transf('''
	switch project.name
		case "Label": ppLabel
		case "Block": ppBlock
		case "If": ppIf
		case "While": ppWhile
		case "DoWhile": ppDoWhile
		case "Function": ppFunction
		else: ppDefault
	end
'''))

module = Path(parse.Rule('''
	Module(stmts)
		-> V([
			I( <<stmts>stmts> )
		])
'''))

#######################################################################
# Test

if __name__ == '__main__':
	import aterm.factory
	import box
	import check

	print commas('[1,2,3]')

	factory = aterm.factory.factory

	exprTestCases = [
		('Binary(Plus(Int(32,Signed)),Lit(Int(32,Unsigned),1),Sym("x"))', '1 + x\n'),
		('Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])}', ''),
		('Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}', '')
	]

	for inputStr, output in exprTestCases:
		input = factory.parse(inputStr)

		print check.expr(input)
		result = expr(inputStr)
		print result
		print box.stringify(result)
		print output
		print

	stmtTestCases = [
		('Label("label")', 'label:\n'),
		('Asm("ret",[])', 'asm("ret");\n'),
		('Asm("mov",[Sym("ax"), Lit(Int(32,Signed),1234)])', 'asm("mov", ax, 1234);\n'),
		('Function(Void,"main",[],[])', 'void main()\n{\n}\n'),
		('Assign(Void,Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])},Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}',''),
		('Assign(Blob(32{Path([0,0,1,0])}){Path([0,1,0])},Sym("eax"{Path([0,1,1,0])}){Path([1,1,0])},Lit(Int(32{Path([0,0,2,1,0])},Signed{Path([1,0,2,1,0])}){Path([0,2,1,0])},1234{Path([1,2,1,0])}){Path([2,1,0])}){Path([1,0]),Id,2}',''),
		('If(Binary(Eq(Int(32,Signed)),Binary(Or(Int(32,NoSign)),Binary(Xor(Int(32,NoSign)),Sym("NF"),Sym("OF")),Sym("ZF")),Lit(Int(32,Signed),1)),GoTo(Sym(".L4")),NoStmt)', ''),
	]

	for inputStr, output in stmtTestCases:
		input = factory.parse(inputStr)

		print input
		print check.stmt(input)
		result = stmt(input)
		print result
		print box.stringify(result)
		print output
		print

<|endoftext|>


<commit_before>import os
import fileinput

def modify(path, build):
    config = os.path.join(path, 'config/forge.cfg')
    replacements = {
        'removeErroringEntities=false': 'removeErroringEntities=true',
        'removeErroringTileEntities=false': 'removeErroringTileEntities=true'
    }

    if not os.path.isfile(config):
        return True

    for line in fileinput.FileInput(config, inplace=True):
        for find, replace in replacements.iteritems():
            line = line.replace(find, replace)

        print line,<commit_msg>Add forge libraries to patch helper<commit_after>import os
import fileinput
import requests
import zipfile
import distutils.dir_util
import tempfile

patchfiles = 'gdn/static/cache/forgepatch01'

def patchForge(path):

    if not os.path.exists(patchfiles):
        print 'Downloading forgepatch'
        r = requests.get('http://s3.amazonaws.com/SpaceZips/forgepatch.zip', stream=True)

        handle, temp_file_path = tempfile.mkstemp()

        with open(temp_file_path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()

        with zipfile.ZipFile(temp_file_path) as z:
            z.extractall(patchfiles)

        os.remove(temp_file_path)

    distutils.dir_util.copy_tree(patchfiles, path)

def modify(path, build):
    patchForge(path)

    config = os.path.join(path, 'config/forge.cfg')
    replacements = {
        'removeErroringEntities=false': 'removeErroringEntities=true',
        'removeErroringTileEntities=false': 'removeErroringTileEntities=true'
    }

    if not os.path.isfile(config):
        return True

    for line in fileinput.FileInput(config, inplace=True):
        for find, replace in replacements.iteritems():
            line = line.replace(find, replace)

        print line,<|endoftext|>


<commit_before>from sys import stderr
from API_operaciones.mysql_connection import mysql2 as mysql
from API_operaciones.mysql_connection import app
import MySQLdb

def print_err(*args, **kwargs):
    print(*args, file=stderr, **kwargs)

class bd_descripcion:
  """ Esta clase tiene metodos para revisar
  que los datos requeridos si existan """
  tablas = []
  descripcionTablas = {}
  idPorTabla = {}

  """ Constructor"""
  def __init__(self):
    db_cursor = mysql.cursor()
    # Obtenemos todos los nombres de las tablas
    querry = "SHOW TABLES"
    db_cursor.execute(querry)
    tablasDB = db_cursor.fetchall()
    # Por cada tabla
    for tabla in tablasDB:
      self.insertarTabla(tabla[0])
      # Obtenemos la llave primaria
      querry = "SELECT k.column_name \
                FROM information_schema.table_constraints t \
                JOIN information_schema.key_column_usage k \
                USING(constraint_name,table_schema,table_name) \
                WHERE t.constraint_type='PRIMARY KEY' \
                AND t.table_schema=%s \
                AND t.table_name=%s"
      db_cursor.execute(querry, (app.config['MYSQL_DB'], tabla[0],))
      llave = db_cursor.fetchone()
      self.insertarIdPrincipalTabla(tabla[0], llave[0])
      # Obtenemos los nombres de las columnas
      querry = "SELECT `COLUMN_NAME`\
                FROM `INFORMATION_SCHEMA`.`COLUMNS`\
                WHERE `TABLE_SCHEMA`='%s'\
                AND `TABLE_NAME`='%s'"
      db_cursor.execute(querry, (app.config['MYSQL_DB'], tabla[0]))
      columnas = db_cursor.fetchall()
      for columna in columnas:
        self.insertarDescripcion(tabla[0], columna[0])

  """ Permite insertar el nombre de una tabla"""
  def insertarTabla(self, nuevasTablas):
    if isinstance(nuevasTablas,list):
      self.tablas.extend(nuevasTablas)
    elif isinstance(nuevasTablas, str):
      self.tablas.append(nuevasTablas)
    else:
      print_err("ERROR parametro nuevasTablas invalido")

  """ Permite insertar los campos de una tabla"""
  def insertarDescripcion(self, nombreBD, elemento):
    if nombreBD in self.tablas:
      if isinstance(elemento, str):
        if nombreBD not in self.descripcionTablas:
          self.descripcionTablas[nombreBD] = []
        self.descripcionTablas[nombreBD].append(elemento)
      else:
        print_err("ERROR parametro elemento es invalido")
    else:
      print_err("ERROR La tabla no existe")

  """ Permite insertar la llave primaria de una talba"""
  def insertarIdPrincipalTabla(self, nombreTabla, nombreID):
    if nombreTabla in self.tablas:
      if isinstance(nombreID, str):
        self.idPorTabla[nombreTabla] = nombreID
      else:
        print_err("ERROR parametro nombrID invalido")
    else:
      print_err("ERROR La tabla no existe")

  """ Permite obtener todos los campos de la tabla"""
  def obtenerCamposTabla(self, tablaNombre):
    if tablaNombre in self.tablas:
      return self.descripcionTablas[tablaNombre]
    else:
      print_err("ERROR La tabla no existe")
      return None

  """ Permite revisar si una tabla existe"""
  def tablaExiste(self, tablaNombre):
    if isinstance(tablaNombre, str):
      if tablaNombre in self.tablas:
        return True
      else:
        return False
    else:
      print_err("ERROR, tabla nombre no es un string")
      return None

  """ Permite revisar si una tabla cuenta con el campoNombre"""
  def campoExisteEnTabla(self, tablaNombre, campoNombre):
    if tablaNombre in self.tablas:
      if campoNombre in self.descripcionTablas[tablaNombre]:
        return True
      else:
        return False
    else:
      return False

  """ Permite obtener el ID de una tabla"""
  def obtenerTablaId(self, tablaNombre):
    if tablaNombre in self.tablas:
      if tablaNombre in self.idPorTabla:
        return self.idPorTabla[tablaNombre]
      else:
        print_err("ERROR La tabla no existe en idPorTabla")
        return None
    else:
      print_err("ERROR La tabla no existe")
      return None

pimcBD = bd_descripcion()
<commit_msg>python bug<commit_after>from sys import stderr
from API_operaciones.mysql_connection import mysql2 as mysql
from API_operaciones.mysql_connection import app
import MySQLdb

def print_err(*args, **kwargs):
    print(*args, file=stderr, **kwargs)

class bd_descripcion:
  """ Esta clase tiene metodos para revisar
  que los datos requeridos si existan """
  tablas = []
  descripcionTablas = {}
  idPorTabla = {}

  """ Constructor"""
  def __init__(self):
    db_cursor = mysql.cursor()
    # Obtenemos todos los nombres de las tablas
    querry = "SHOW TABLES"
    db_cursor.execute(querry)
    tablasDB = db_cursor.fetchall()
    # Por cada tabla
    for tabla in tablasDB:
      self.insertarTabla(tabla[0])
      # Obtenemos la llave primaria
      querry = "SELECT k.column_name \
                FROM information_schema.table_constraints t \
                JOIN information_schema.key_column_usage k \
                USING(constraint_name,table_schema,table_name) \
                WHERE t.constraint_type='PRIMARY KEY' \
                AND t.table_schema=%s \
                AND t.table_name=%s"
      db_cursor.execute(querry, (app.config['MYSQL_DB'], tabla[0],))
      llave = db_cursor.fetchone()
      self.insertarIdPrincipalTabla(tabla[0], llave[0])
      # Obtenemos los nombres de las columnas
      querry = "SELECT `COLUMN_NAME`\
                FROM `INFORMATION_SCHEMA`.`COLUMNS`\
                WHERE `TABLE_SCHEMA`=%s\
                AND `TABLE_NAME`=%s"
      db_cursor.execute(querry, (app.config['MYSQL_DB'], tabla[0]))
      columnas = db_cursor.fetchall()
      for columna in columnas:
        self.insertarDescripcion(tabla[0], columna[0])

  """ Permite insertar el nombre de una tabla"""
  def insertarTabla(self, nuevasTablas):
    if isinstance(nuevasTablas,list):
      self.tablas.extend(nuevasTablas)
    elif isinstance(nuevasTablas, str):
      self.tablas.append(nuevasTablas)
    else:
      print_err("ERROR parametro nuevasTablas invalido")

  """ Permite insertar los campos de una tabla"""
  def insertarDescripcion(self, nombreBD, elemento):
    if nombreBD in self.tablas:
      if isinstance(elemento, str):
        if nombreBD not in self.descripcionTablas:
          self.descripcionTablas[nombreBD] = []
        self.descripcionTablas[nombreBD].append(elemento)
      else:
        print_err("ERROR parametro elemento es invalido")
    else:
      print_err("ERROR La tabla no existe")

  """ Permite insertar la llave primaria de una talba"""
  def insertarIdPrincipalTabla(self, nombreTabla, nombreID):
    if nombreTabla in self.tablas:
      if isinstance(nombreID, str):
        self.idPorTabla[nombreTabla] = nombreID
      else:
        print_err("ERROR parametro nombrID invalido")
    else:
      print_err("ERROR La tabla no existe")

  """ Permite obtener todos los campos de la tabla"""
  def obtenerCamposTabla(self, tablaNombre):
    if tablaNombre in self.tablas:
      return self.descripcionTablas[tablaNombre]
    else:
      print_err("ERROR La tabla no existe")
      return None

  """ Permite revisar si una tabla existe"""
  def tablaExiste(self, tablaNombre):
    if isinstance(tablaNombre, str):
      if tablaNombre in self.tablas:
        return True
      else:
        return False
    else:
      print_err("ERROR, tabla nombre no es un string")
      return None

  """ Permite revisar si una tabla cuenta con el campoNombre"""
  def campoExisteEnTabla(self, tablaNombre, campoNombre):
    if tablaNombre in self.tablas:
      if campoNombre in self.descripcionTablas[tablaNombre]:
        return True
      else:
        return False
    else:
      return False

  """ Permite obtener el ID de una tabla"""
  def obtenerTablaId(self, tablaNombre):
    if tablaNombre in self.tablas:
      if tablaNombre in self.idPorTabla:
        return self.idPorTabla[tablaNombre]
      else:
        print_err("ERROR La tabla no existe en idPorTabla")
        return None
    else:
      print_err("ERROR La tabla no existe")
      return None

pimcBD = bd_descripcion()
<|endoftext|>