<commit_before>"""RipMyProfessor URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin
import debug_toolbar

urlpatterns = [
    url(r'^index/|^courses/', include('rip.urls')),
    url(r'^admin/', admin.site.urls),
    #url(r'^__debug__/', include(debug_toolbar.urls)),
]
<commit_msg>V2.60 reducing requirement.txt<commit_after>"""RipMyProfessor URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin
#import debug_toolbar

urlpatterns = [
    url(r'^index/|^courses/', include('rip.urls')),
    url(r'^admin/', admin.site.urls),
    #url(r'^__debug__/', include(debug_toolbar.urls)),
]
<|endoftext|>


<commit_before>from box import uniq_f11
from box import box_append, box_remove, box_overwrite
from box import subbox_append, subbox_remove, subbox_overwrite
from box import box_sort, subbox_sort

def test_uniq_f11():
    seq = [1, 2, 3, 2]
    assert uniq_f11(seq) == [1, 2, 3]

def test_append_remove():
    data = {}
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, 1001, 'il', ['60000', 1])
    box_append(data, 1001, 'il', ['60001', 1])
    assert data == {'1001': {'na': ['foo'], 'il': ['60000', '1', '60001', '1']}}

    data = {}
    box_append(data, '1001', 'na', 'foo')
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, '1001', 'na', 'bar', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    del data['1001']['na']
    box_append(data, '1001', 'na', ['foo', 'bar'], dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}

    data = {}
    box_append(data, 1001, 'na', 'foo')
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': []}}
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': []}}
    box_remove(data, 1001, 'na', 'bar')
    assert data == {'1001': {'na': []}}
    box_append(data, 1001, 'na', ['foo', 'bar'], dedup=True)
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': ['bar']}}
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['bar', 'foo']}}
    box_remove(data, 1001, 'na', 'bar')
    assert data == {'1001': {'na': ['foo']}}

    data = {}
    box_append(data, 1001, 'na', 'foo')
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, 1001, 'na', 'bar', dedup=True)
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    box_append(data, 1001, 'na', 'bar', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    box_remove(data, 1001, 'na', 'foo')
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['bar', 'foo']}}

def test_overwrite():
    data = {}
    box_overwrite(data, 1, 2, ['3'])
    assert data == {'1': {'2': ['3']}}

    data = {'1001': {'na': ['Oleg the Loudmouth']}}
    box_overwrite(data, 1001, 'na', ['Phydeaux, RIP'])
    assert data == {'1001': {'na': ['Phydeaux, RIP']}}

def test_append2_remove2():
    '''
    These tests are a copy of test_append_remove with an extra level
    '''
    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': ['foo']}}}

    data = {}
    subbox_append(data, '1001', 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo']}}}
    subbox_append(data, '1001', 'LI', 'hl', 'bar', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    del data['1001']['LI']['hl']
    subbox_append(data, '1001', 'LI', 'hl', ['foo', 'bar'], dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}

    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'bar')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_append(data, 1001, 'LI', 'hl', ['foo', 'bar'], dedup=True)
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': ['bar']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['bar', 'foo']}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'bar')
    assert data == {'1001': {'LI': {'hl': ['foo']}}}

    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'bar', dedup=True)
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'bar', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['bar', 'foo']}}}

def test_overwrite2():
    data = {}
    subbox_overwrite(data, 1, 2, 3, ['3'])
    assert data == {'1': {'2': {'3': ['3']}}}

    data = {'1001': {'LI': {'wh': ['10000']}}}
    subbox_overwrite(data, 1001, 'LI', 'wh', ['10001'])
    assert data == {'1001': {'LI': {'wh': ['10001']}}}

def test_sorts():
    data = {'1001': {'ah': ['3', '2', '1', '11'],
                     'PL': {'un': ['1003', '1002', '10001', '1001']}}}

    box_sort(data, '1001', 'ah')
    assert data['1001']['ah'] == ['1', '2', '3', '11']

    subbox_sort(data, '1001', 'PL', 'un')
    assert data['1001']['PL']['un'] == ['1001', '1002', '1003', '10001']
<commit_msg>flycheck<commit_after>from box import uniq_f11
from box import box_append, box_remove, box_overwrite
from box import subbox_append, subbox_remove, subbox_overwrite
from box import box_sort, subbox_sort

def test_uniq_f11():
    seq = [1, 2, 3, 2]
    assert uniq_f11(seq) == [1, 2, 3]

def test_append_remove():
    data = {}
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, 1001, 'il', ['60000', 1])
    box_append(data, 1001, 'il', ['60001', 1])
    assert data == {'1001': {'na': ['foo'], 'il': ['60000', '1', '60001', '1']}}

    data = {}
    box_append(data, '1001', 'na', 'foo')
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, '1001', 'na', 'bar', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    del data['1001']['na']
    box_append(data, '1001', 'na', ['foo', 'bar'], dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}

    data = {}
    box_append(data, 1001, 'na', 'foo')
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': []}}
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': []}}
    box_remove(data, 1001, 'na', 'bar')
    assert data == {'1001': {'na': []}}
    box_append(data, 1001, 'na', ['foo', 'bar'], dedup=True)
    box_remove(data, 1001, 'na', 'foo')
    assert data == {'1001': {'na': ['bar']}}
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['bar', 'foo']}}
    box_remove(data, 1001, 'na', 'bar')
    assert data == {'1001': {'na': ['foo']}}

    data = {}
    box_append(data, 1001, 'na', 'foo')
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo']}}
    box_append(data, 1001, 'na', 'bar', dedup=True)
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    box_append(data, 1001, 'na', 'bar', dedup=True)
    assert data == {'1001': {'na': ['foo', 'bar']}}
    box_remove(data, 1001, 'na', 'foo')
    box_append(data, 1001, 'na', 'foo', dedup=True)
    assert data == {'1001': {'na': ['bar', 'foo']}}

def test_overwrite():
    data = {}
    box_overwrite(data, 1, 2, ['3'])
    assert data == {'1': {'2': ['3']}}

    data = {'1001': {'na': ['Oleg the Loudmouth']}}
    box_overwrite(data, 1001, 'na', ['Phydeaux, RIP'])
    assert data == {'1001': {'na': ['Phydeaux, RIP']}}

def test_append2_remove2():
    '''
    These tests are a copy of test_append_remove with an extra level
    '''
    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': ['foo']}}}

    data = {}
    subbox_append(data, '1001', 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo']}}}
    subbox_append(data, '1001', 'LI', 'hl', 'bar', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    del data['1001']['LI']['hl']
    subbox_append(data, '1001', 'LI', 'hl', ['foo', 'bar'], dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}

    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'bar')
    assert data == {'1001': {'LI': {'hl': []}}}
    subbox_append(data, 1001, 'LI', 'hl', ['foo', 'bar'], dedup=True)
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    assert data == {'1001': {'LI': {'hl': ['bar']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['bar', 'foo']}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'bar')
    assert data == {'1001': {'LI': {'hl': ['foo']}}}

    data = {}
    subbox_append(data, 1001, 'LI', 'hl', 'foo')
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'bar', dedup=True)
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    subbox_append(data, 1001, 'LI', 'hl', 'bar', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['foo', 'bar']}}}
    subbox_remove(data, 1001, 'LI', 'hl', 'foo')
    subbox_append(data, 1001, 'LI', 'hl', 'foo', dedup=True)
    assert data == {'1001': {'LI': {'hl': ['bar', 'foo']}}}

def test_overwrite2():
    data = {}
    subbox_overwrite(data, 1, 2, 3, ['3'])
    assert data == {'1': {'2': {'3': ['3']}}}

    data = {'1001': {'LI': {'wh': ['10000']}}}
    subbox_overwrite(data, 1001, 'LI', 'wh', ['10001'])
    assert data == {'1001': {'LI': {'wh': ['10001']}}}

def test_sorts():
    data = {'1001': {'ah': ['3', '2', '1', '11'],
                     'PL': {'un': ['1003', '1002', '10001', '1001']}}}

    box_sort(data, '1001', 'ah')
    assert data['1001']['ah'] == ['1', '2', '3', '11']

    subbox_sort(data, '1001', 'PL', 'un')
    assert data['1001']['PL']['un'] == ['1001', '1002', '1003', '10001']
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from bst import Node
from random import randint
import pytest

@pytest.fixture()
def rand_setup():
    root = Node(randint(1, 100))
    for idx in range(20):
        val = randint(1, 100)
        try:
            root.insert(val)
        except AttributeError:
            continue

    return root

@pytest.fixture()
def fixed_setup():
    root = Node(25)
    root.insert(15)
    root.insert(30)
    root.insert(9)
    root.insert(17)
    root.insert(21)
    root.insert(39)
    root.insert(12)
    root.insert(24)
    root.insert(40)
    root.insert(14)

    return root

def test_insert(rand_setup):
    pre_size = rand_setup.size()
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.insert(rand) is None
    rand_setup.insert(rand)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_contains(rand_setup):
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.contains(rand) is True

def test_size(rand_setup):
    pre_size = rand_setup.size()
    rand = randint(1, 100)
    rand_setup.insert(rand)
    rand_setup.insert(rand)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_depth(fixed_setup):
    assert fixed_setup.left.depth() == 5
    assert fixed_setup.right.depth() == 4
    fixed_setup.insert(13)
    assert fixed_setup.left.depth() == 6

def test_balance(rand_setup):
    left = rand_setup.left.depth()
    right = rand_setup.right.depth()
    if left > right:
        assert rand_setup.balance() < 0
    elif right > left:
        assert rand_setup.balance() > 0
    else:
        assert rand_setup.balance() == 0
<commit_msg>Fix test errors<commit_after>from __future__ import unicode_literals
from bst import Node
from random import randint
import pytest

@pytest.fixture()
def rand_setup():
    root = Node(randint(1, 100))
    for idx in range(20):
        val = randint(1, 100)
        try:
            root.insert(val)
        except AttributeError:
            continue

    return root

@pytest.fixture()
def fixed_setup():
    root = Node(25)
    root.insert(15)
    root.insert(30)
    root.insert(9)
    root.insert(17)
    root.insert(21)
    root.insert(39)
    root.insert(12)
    root.insert(24)
    root.insert(40)
    root.insert(14)

    return root

def test_insert(rand_setup):
    pre_size = rand_setup.size()
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.insert(rand) is None
    rand_setup.insert(rand)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_contains(rand_setup):
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.contains(rand) is True

def test_size(rand_setup):
    pre_size = rand_setup.size()
    new = 200
    rand_setup.insert(new)
    rand_setup.insert(new)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_depth(fixed_setup):
    assert fixed_setup.left.depth() == 4
    assert fixed_setup.right.depth() == 3
    fixed_setup.insert(13)
    assert fixed_setup.left.depth() == 5

def test_balance(rand_setup):
    left = rand_setup.left.depth()
    right = rand_setup.right.depth()
    if left > right:
        assert rand_setup.balance() == 1
    elif right > left:
        assert rand_setup.balance() == -1
    else:
        assert rand_setup.balance() == 0
<|endoftext|>


<commit_before>#!/usr/bin/env python

# pylint: disable=protected-access

import os
import unittest

from mock import patch

os.environ["DEN_ACCESS_TOKEN"] = "TEST"
from den import record

class RecordTestCase(unittest.TestCase):

    def test_missing_env_variable_raises_key_error(self):
        with patch.dict("os.environ", {}):
            del os.environ["DEN_ACCESS_TOKEN"]
            with self.assertRaisesRegexp(KeyError, r"Please set the environment variable 'DEN_ACCESS_TOKEN'."):
                reload(record)

    def test_get_api_url(self):
        expected = "https://developer-api.nest.com?auth=%s" % record.NEST_API_ACCESS_TOKEN
        actual = record._get_api_url()
        self.assertEqual(expected, actual)

        expected = "https://developer-api.nest.com/structures?auth=%s" % record.NEST_API_ACCESS_TOKEN
        for path in ["structures", "/structures", "/structures/", "structures/"]:
            self.assertEqual(expected, record._get_api_url(path))

    def test_is_event(self):
        self.assertTrue(record._is_event("event:"))
        self.assertFalse(record._is_event("event"))
        self.assertFalse(record._is_event("data:"))

    def test_is_data(self):
        self.assertTrue(record._is_data("data:"))
        self.assertFalse(record._is_data("data"))
        self.assertFalse(record._is_data("event:"))

    def test_process_event_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_event(line))

    def test_process_event_returns_none_for_keep_alive_line(self):
        self.assertIsNone(record._process_event("event: keep-alive"))
        self.assertIsNone(record._process_event("event:keep-alive"))

    def test_process_event_returns_event_for_valid_line(self):
        expected = "Test event"
        actual = record._process_event("event: " + expected)
        self.assertEqual(expected, actual)
        actual = record._process_event("event:" + expected)
        self.assertEqual(expected, actual)

    def test_process_data_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_data(line))

if __name__ == "__main__":
    unittest.main(verbosity=2)
<commit_msg>record - Add test.<commit_after>#!/usr/bin/env python

# pylint: disable=protected-access

import json
import os
import unittest

from mock import patch

os.environ["DEN_ACCESS_TOKEN"] = "TEST"
from den import record

class RecordTestCase(unittest.TestCase):

    def test_missing_env_variable_raises_key_error(self):
        with patch.dict("os.environ", {}):
            del os.environ["DEN_ACCESS_TOKEN"]
            with self.assertRaisesRegexp(KeyError, r"Please set the environment variable 'DEN_ACCESS_TOKEN'."):
                reload(record)

    def test_get_api_url(self):
        expected = "https://developer-api.nest.com?auth=%s" % record.NEST_API_ACCESS_TOKEN
        actual = record._get_api_url()
        self.assertEqual(expected, actual)

        expected = "https://developer-api.nest.com/structures?auth=%s" % record.NEST_API_ACCESS_TOKEN
        for path in ["structures", "/structures", "/structures/", "structures/"]:
            self.assertEqual(expected, record._get_api_url(path))

    def test_is_event(self):
        self.assertTrue(record._is_event("event:"))
        self.assertFalse(record._is_event("event"))
        self.assertFalse(record._is_event("data:"))

    def test_is_data(self):
        self.assertTrue(record._is_data("data:"))
        self.assertFalse(record._is_data("data"))
        self.assertFalse(record._is_data("event:"))

    def test_process_event_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_event(line))

    def test_process_event_returns_none_for_keep_alive_line(self):
        self.assertIsNone(record._process_event("event: keep-alive"))
        self.assertIsNone(record._process_event("event:keep-alive"))

    def test_process_event_returns_event_for_valid_line(self):
        expected = "Test event"
        actual = record._process_event("event: " + expected)
        self.assertEqual(expected, actual)
        actual = record._process_event("event:" + expected)
        self.assertEqual(expected, actual)

    def test_process_data_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_data(line))

    def test_precess_data_returns_a_list_for_valid_line(self):
        expected = [{"key": "val"}]
        actual = record._process_data("data: " + json.dumps(expected))
        self.assertEqual(expected, actual)
        actual = record._process_data("data:" + json.dumps(expected))
        self.assertEqual(expected, actual)

if __name__ == "__main__":
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>#!/sw64/bin/python2.7

import time
import os
import subprocess
import sys
import shutil

sys.path.append('..')
import great3

# Set which branches to test...
experiments = [
    'control',
    #'real_gal',
    #'real_psf',
    #'multiepoch',
    #'full',
]
obs_type = [
    'ground',
    'space',
]
shear_type = [
    'constant',
    'variable',
]

root = 'test_run'
subfield_min = 199    # Start with the last regular field if we want to check out the deep fields.
subfield_max = 201    # Total of 3 sub-fields
data_dir = 'great3_fit_data'    # This should be set up as a sim-link to your Dropbox folder.
ps_dir = '../inputs/ps/tables'
seed = 12345                    # Whatever.  (But not zero.)

# Clean up possible residue from previous runs.
shutil.rmtree(root, ignore_errors=True)

# Build catalogs, etc.
t1 = time.time()
great3.constants.nrows = 20
great3.constants.ncols = 20
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, ps_dir=ps_dir,
           seed=seed, steps=['metaparameters', 'catalogs', 'config']
)
t2 = time.time()
print
print 'Time for great3.run up to config = ',t2-t1
print

# build images using galsim_yaml
dirs = []
os.chdir(root)
for exp in experiments:
    e = exp[0]
    if e == 'r': e = exp[5]
    for obs in obs_type:
        o = obs[0]
        for shear in shear_type:
            s = shear[0]
            f = e + o + s + '.yaml'
            dirs.append( os.path.join(root,exp,obs,shear) )
            t1 = time.time()
            p = subprocess.Popen(['galsim_yaml',f,'-v1'])
            p.communicate() # wait until done
            t2 = time.time()
            print
            print 'Time for galsim_yaml',f,'= ',t2-t1
            print
os.chdir('..')

# Build images using great3.run
t1 = time.time()
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, ps_dir=ps_dir,
           seed=seed, steps=['images']
)
t2 = time.time()
print
print 'Time for great3.run images = ',t2-t1
print

# Check that images are the same.
print 'Checking diffs: (No output means success)'
sys.stdout.flush()
for dir in dirs:
    for i in range(subfield_min, subfield_max+1):
        f1 = os.path.join(dir,'image-%03d-0.fits'%i)
        f2 = os.path.join(dir,'yaml_image-%03d-0.fits'%i)
        p = subprocess.Popen(['diff',f1,f2],stderr=subprocess.STDOUT)
        p.communicate()

# Now package up the data that should be public
t1 = time.time()
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, seed=seed, steps=['packages']
)
t2 = time.time()
print
print 'Time for great3.run packages = ',t2-t1
print
<commit_msg>Add time for all galsim runs (#42)<commit_after>#!/sw64/bin/python2.7

import time
import os
import subprocess
import sys
import shutil

sys.path.append('..')
import great3

# Set which branches to test...
experiments = [
    'control',
    #'real_gal',
    #'real_psf',
    #'multiepoch',
    #'full',
]
obs_type = [
    'ground',
    'space',
]
shear_type = [
    'constant',
    'variable',
]

root = 'test_run'
subfield_min = 199    # Start with the last regular field if we want to check out the deep fields.
subfield_max = 201    # Total of 3 sub-fields
data_dir = 'great3_fit_data'    # This should be set up as a sim-link to your Dropbox folder.
ps_dir = '../inputs/ps/tables'
seed = 12345                    # Whatever.  (But not zero.)

# Clean up possible residue from previous runs.
shutil.rmtree(root, ignore_errors=True)

# Build catalogs, etc.
t1 = time.time()
great3.constants.nrows = 20
great3.constants.ncols = 20
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, ps_dir=ps_dir,
           seed=seed, steps=['metaparameters', 'catalogs', 'config']
)
t2 = time.time()
print
print 'Time for great3.run up to config = ',t2-t1
print

# build images using galsim_yaml
t1 = time.time()
dirs = []
os.chdir(root)
for exp in experiments:
    e = exp[0]
    if e == 'r': e = exp[5]
    for obs in obs_type:
        o = obs[0]
        for shear in shear_type:
            s = shear[0]
            f = e + o + s + '.yaml'
            dirs.append( os.path.join(root,exp,obs,shear) )
            t3 = time.time()
            p = subprocess.Popen(['galsim_yaml',f,'-v1'])
            p.communicate() # wait until done
            t4 = time.time()
            print
            print 'Time for galsim_yaml',f,'= ',t4-t3
            print
os.chdir('..')
t2 = time.time()
print
print 'Total time for galsim_yaml = ',t2-t1
print

# Build images using great3.run
t1 = time.time()
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, ps_dir=ps_dir,
           seed=seed, steps=['images']
)
t2 = time.time()
print
print 'Time for great3.run images = ',t2-t1
print

# Check that images are the same.
print 'Checking diffs: (No output means success)'
sys.stdout.flush()
for dir in dirs:
    for i in range(subfield_min, subfield_max+1):
        f1 = os.path.join(dir,'image-%03d-0.fits'%i)
        f2 = os.path.join(dir,'yaml_image-%03d-0.fits'%i)
        p = subprocess.Popen(['diff',f1,f2],stderr=subprocess.STDOUT)
        p.communicate()

# Now package up the data that should be public
t1 = time.time()
great3.run(root, subfield_min=subfield_min, subfield_max=subfield_max,
           experiments=experiments, obs_type=obs_type, shear_type=shear_type,
           gal_dir=data_dir, seed=seed, steps=['packages']
)
t2 = time.time()
print
print 'Time for great3.run packages = ',t2-t1
print
<|endoftext|>


<commit_before># Django settings for Rynda project.

import os

from django.core.exceptions import ImproperlyConfigured

def get_env_var(name):
    try:
        return os.env[name]
    except KeyError:
        raise ImproperlyConfigured("Set the %s env variable" % name)

here = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x)

SITE_ROOT = here('..', '..')
DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Moscow'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'ru'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'static'),
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'l(39jm())q%720m2@p%wz)bep@lrfb*7k+66%+epnwes!pcbh2'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    # default template context processors
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    # required by django-admin-tools
    'django.core.context_processors.request',
)
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'Rynda.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'Rynda.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'templates')
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    #
    'admin_tools',
    'admin_tools.theming',
    'admin_tools.menu',
    'admin_tools.dashboard',
    #
    'django.contrib.admin',
    'django.contrib.gis',
    'django_filters',
    'rest_framework',
    'south',
    'templated_emails',
    'message',
    'users',
    'geozones',
    'core',
    'feed',
    'api',
)

AUTHENTICATION_BACKENDS = (
    'core.backends.IonAuth',
    'core.backends.EmailAuthBackend',
    'django.contrib.auth.backends.ModelBackend',
)

ADMIN_TOOLS_THEMING_CSS = 'admin_tools/css/theming.css'
AUTH_PROFILE_MODULE = 'users.users'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

LOGIN_REDIRECT_URL = '/'
<commit_msg>Fixed get_env_var function<commit_after># Django settings for Rynda project.

import os

from django.core.exceptions import ImproperlyConfigured

def get_env_var(name):
    try:
        return os.environ[name]
    except KeyError:
        raise ImproperlyConfigured("Set the %s env variable" % name)

here = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x)

SITE_ROOT = here('..', '..')
DEBUG = False
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': '',                      # Or path to database file if using sqlite3.
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Europe/Moscow'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'ru'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'static'),
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'l(39jm())q%720m2@p%wz)bep@lrfb*7k+66%+epnwes!pcbh2'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    # default template context processors
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    # required by django-admin-tools
    'django.core.context_processors.request',
)
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'Rynda.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'Rynda.wsgi.application'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(SITE_ROOT, 'templates')
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    #
    'admin_tools',
    'admin_tools.theming',
    'admin_tools.menu',
    'admin_tools.dashboard',
    #
    'django.contrib.admin',
    'django.contrib.gis',
    'django_filters',
    'rest_framework',
    'south',
    'templated_emails',
    'message',
    'users',
    'geozones',
    'core',
    'feed',
    'api',
)

AUTHENTICATION_BACKENDS = (
    'core.backends.IonAuth',
    'core.backends.EmailAuthBackend',
    'django.contrib.auth.backends.ModelBackend',
)

ADMIN_TOOLS_THEMING_CSS = 'admin_tools/css/theming.css'
AUTH_PROFILE_MODULE = 'users.users'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

LOGIN_REDIRECT_URL = '/'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import logging
import json
import posixpath

from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _

from desktop.lib.i18n import force_unicode
from desktop.lib.rest.http_client import HttpClient, RestException
from desktop.lib.rest.resource import Resource

from notebook.connectors.base import Api, QueryError

LOG = logging.getLogger(__name__)
_JSON_CONTENT_TYPE = 'application/json'
_API_VERSION = 'v1'
SESSIONS = {}
SESSION_KEY = '%(username)s-%(connector_name)s'

n = 0

def query_error_handler(func):
  def decorator(*args, **kwargs):
    try:
      return func(*args, **kwargs)
    except RestException as e:
      try:
        message = force_unicode(json.loads(e.message)['errors'])
      except:
        message = e.message
      message = force_unicode(message)
      raise QueryError(message)
    except Exception as e:
      message = force_unicode(str(e))
      raise QueryError(message)
  return decorator

class FlinkSqlApi(Api):

  def __init__(self, user, interpreter=None):
    Api.__init__(self, user, interpreter=interpreter)

    self.options = interpreter['options']
    api_url = self.options['url']

    self.db = FlinkSqlClient(user=user, api_url=api_url)

  @query_error_handler
  def create_session(self, lang=None, properties=None):
    session = self._get_session()

    response = {
      'type': lang,
      'id': session['session_id']
    }

    return response

  def _get_session(self):
    session_key = SESSION_KEY % {
      'username': self.user.username,
      'connector_name': self.interpreter['name']
    }

    if session_key not in SESSIONS:
      SESSIONS[session_key] = self.db.create_session()

    try:
      self.db.session_heartbeat(session_id=SESSIONS[session_key]['session_id'])
    except Exception as e:
      if 'Session: %(id)s does not exist' % SESSIONS[session_key] in str(e):
        LOG.warn('Session: %(id)s does not exist, opening a new one' % SESSIONS[session_key])
        SESSIONS[session_key] = self.db.create_session()
      else:
        raise e

    SESSIONS[session_key]['id'] = SESSIONS[session_key]['session_id']

    return SESSIONS[session_key]

  @query_error_handler
  def execute(self, notebook, snippet):
    global n
    n = 0
    session = self._get_session()
    session_id = session['id']
    job_id = None

    statement = snippet['statement'].strip().rstrip(';')

    resp = self.db.execute_statement(session_id=session_id, statement=statement)

    if resp['statement_types'][0] == 'SELECT':
      job_id = resp['results'][0]['data'][0][0]
      data, description = [], []
      # TODO: change_flags
    else:
      data, description = resp['results'][0]['data'], resp['results'][0]['columns']

    has_result_set = data is not None

    return {
      'sync': job_id is None,
      'has_result_set': has_result_set,
      'guid': job_id,
      'result': {
        'has_more': job_id is not None,
        'data': data if job_id is None else [],
        'meta': [{
            'name': col['name'],
            'type': col['type'],
            'comment': ''
          }
          for col in description
        ] if has_result_set else [],
        'type': 'table'
      }
    }

  @query_error_handler
  def check_status(self, notebook, snippet):
    global n
    response = {}
    session = self._get_session()

    status = 'expired'

    if snippet.get('result'):
      statement_id = snippet['result']['handle']['guid']
      if session:
        if not statement_id:  # Sync result
          status = 'available'
        else:
          try:
            resp = self.db.fetch_status(session['id'], statement_id)
            if resp.get('status') == 'RUNNING':
              status = 'streaming'
              response['result'] = self.fetch_result(notebook, snippet, n, False)
            elif resp.get('status') == 'FINISHED':
              status = 'available'
            elif resp.get('status') == 'FAILED':
              status = 'failed'
            elif resp.get('status') == 'CANCELED':
              status = 'expired'
          except Exception as e:
            if '%s does not exist in current session' % statement_id in str(e):
              LOG.warn('Job: %s does not exist' % statement_id)
            else:
              raise e

    response['status'] = status

    return response

  @query_error_handler
  def fetch_result(self, notebook, snippet, rows, start_over):
    global n
    session = self._get_session()
    statement_id = snippet['result']['handle']['guid']
    token = n #rows

    resp = self.db.fetch_results(session['id'], job_id=statement_id, token=token)

    next_result = resp.get('next_result_uri')
    if next_result:
      n = int(next_result.rsplit('/', 1)[-1])

    return {
        'has_more': bool(next_result),
        'data': resp['results'][0]['data'],  # No escaping...
        'meta': [{
            'name': column['name'],
            'type': column['type'],
            'comment': ''
          }
          for column in resp['results'][0]['columns']
        ],
        'type': 'table'
    }

  @query_error_handler
  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
    response = {}

    if database is None:
      response['databases'] = self.show_databases()
    elif table is None:
      response['tables_meta'] = self.show_tables(database)
    elif column is None:
      columns = self.get_columns(database, table)
      response['columns'] = [col['name'] for col in columns]
      response['extended_columns'] = [{
          'comment': col.get('comment'),
          'name': col.get('name'),
          'type': col['type']
        }
        for col in columns
      ]

    return response

  def show_databases(self):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='SHOW DATABASES')

    return [db[0] for db in resp['results'][0]['data']]

  def show_tables(self, database):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='USE %(database)s' % {'database': database})
    resp = self.db.execute_statement(session_id=session_id, statement='SHOW TABLES')

    return [table[0] for table in resp['results'][0]['data']]

  def get_columns(self, database, table):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='USE %(database)s' % {'database': database})
    resp = self.db.execute_statement(session_id=session_id, statement='DESCRIBE %(table)s' % {'table': table})

    columns = resp['results'][0]['columns']

    return [{
        'name': col['name'],
        'type': col['type'],  # Types to unify
        'comment': '',
      }
      for col in columns
    ]

  def cancel(self, notebook, snippet):
    session = self._get_session()
    statement_id = snippet['result']['handle']['guid']

    try:
      self.db.close_statement(session_id=session['id'], job_id=statement_id)
    except Exception as e:
      if 'does not exist in current session:' in str(e):
        return {'status': -1}  # skipped
      else:
        raise e

    return {'status': 0}

  def close_session(self, session):
    session = self._get_session()
    self.db.close_session(session['id'])

class FlinkSqlClient():
  '''
  Implements https://github.com/ververica/flink-sql-gateway
  Could be a pip module or sqlalchemy dialect in the future.
  '''

  def __init__(self, user, api_url):
    self.user = user
    self._url = posixpath.join(api_url + '/' + _API_VERSION + '/')
    self._client = HttpClient(self._url, logger=LOG)
    self._root = Resource(self._client)

  def __str__(self):
    return "FlinkClient at %s" % (self._url,)

  def info(self):
    return self._root.get('info')

  def create_session(self, **properties):
    data = {
        "session_name": "test",  # optional
        "planner": "blink",  # required, "old"/"blink"
        "execution_type": "streaming",  # required, "batch"/"streaming"
        "properties": {  # optional
            "key": "value"
        }
    }
    data.update(properties)

    return self._root.post('sessions', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)

  def session_heartbeat(self, session_id):
    return self._root.post('sessions/%(session_id)s/heartbeat' % {'session_id': session_id})

  def execute_statement(self, session_id, statement):
    data = {
        "statement": statement,  # required
        "execution_timeout": ""  # execution time limit in milliseconds, optional, but required for stream SELECT ?
    }

    return self._root.post(
        'sessions/%(session_id)s/statements' % {
        'session_id': session_id
      },
      data=json.dumps(data),
      contenttype=_JSON_CONTENT_TYPE
    )

  def fetch_status(self, session_id, job_id):
    return self._root.get(
      'sessions/%(session_id)s/jobs/%(job_id)s/status' % {
        'session_id': session_id,
        'job_id': job_id
      }
    )

  def fetch_results(self, session_id, job_id, token=0):
    return self._root.get(
      'sessions/%(session_id)s/jobs/%(job_id)s/result/%(token)s' % {
        'session_id': session_id,
        'job_id': job_id,
        'token': token
      }
    )

  def close_statement(self, session_id, job_id):
    return self._root.delete(
      'sessions/%(session_id)s/jobs/%(job_id)s' % {
        'session_id': session_id,
        'job_id': job_id,
      }
    )

  def close_session(self, session_id):
    return self._root.delete(
      'sessions/%(session_id)s' % {
        'session_id': session_id,
      }
    )
<commit_msg>[flink] Avoid closing session on page refresh or editor close for now<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import logging
import json
import posixpath

from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _

from desktop.lib.i18n import force_unicode
from desktop.lib.rest.http_client import HttpClient, RestException
from desktop.lib.rest.resource import Resource

from notebook.connectors.base import Api, QueryError

LOG = logging.getLogger(__name__)
_JSON_CONTENT_TYPE = 'application/json'
_API_VERSION = 'v1'
SESSIONS = {}
SESSION_KEY = '%(username)s-%(connector_name)s'

n = 0

def query_error_handler(func):
  def decorator(*args, **kwargs):
    try:
      return func(*args, **kwargs)
    except RestException as e:
      try:
        message = force_unicode(json.loads(e.message)['errors'])
      except:
        message = e.message
      message = force_unicode(message)
      raise QueryError(message)
    except Exception as e:
      message = force_unicode(str(e))
      raise QueryError(message)
  return decorator

class FlinkSqlApi(Api):

  def __init__(self, user, interpreter=None):
    Api.__init__(self, user, interpreter=interpreter)

    self.options = interpreter['options']
    api_url = self.options['url']

    self.db = FlinkSqlClient(user=user, api_url=api_url)

  @query_error_handler
  def create_session(self, lang=None, properties=None):
    session = self._get_session()

    response = {
      'type': lang,
      'id': session['session_id']
    }

    return response

  def _get_session(self):
    session_key = SESSION_KEY % {
      'username': self.user.username,
      'connector_name': self.interpreter['name']
    }

    if session_key not in SESSIONS:
      SESSIONS[session_key] = self.db.create_session()

    try:
      self.db.session_heartbeat(session_id=SESSIONS[session_key]['session_id'])
    except Exception as e:
      if 'Session: %(id)s does not exist' % SESSIONS[session_key] in str(e):
        LOG.warn('Session: %(id)s does not exist, opening a new one' % SESSIONS[session_key])
        SESSIONS[session_key] = self.db.create_session()
      else:
        raise e

    SESSIONS[session_key]['id'] = SESSIONS[session_key]['session_id']

    return SESSIONS[session_key]

  @query_error_handler
  def execute(self, notebook, snippet):
    global n
    n = 0
    session = self._get_session()
    session_id = session['id']
    job_id = None

    statement = snippet['statement'].strip().rstrip(';')

    resp = self.db.execute_statement(session_id=session_id, statement=statement)

    if resp['statement_types'][0] == 'SELECT':
      job_id = resp['results'][0]['data'][0][0]
      data, description = [], []
      # TODO: change_flags
    else:
      data, description = resp['results'][0]['data'], resp['results'][0]['columns']

    has_result_set = data is not None

    return {
      'sync': job_id is None,
      'has_result_set': has_result_set,
      'guid': job_id,
      'result': {
        'has_more': job_id is not None,
        'data': data if job_id is None else [],
        'meta': [{
            'name': col['name'],
            'type': col['type'],
            'comment': ''
          }
          for col in description
        ] if has_result_set else [],
        'type': 'table'
      }
    }

  @query_error_handler
  def check_status(self, notebook, snippet):
    global n
    response = {}
    session = self._get_session()

    status = 'expired'

    if snippet.get('result'):
      statement_id = snippet['result']['handle']['guid']
      if session:
        if not statement_id:  # Sync result
          status = 'available'
        else:
          try:
            resp = self.db.fetch_status(session['id'], statement_id)
            if resp.get('status') == 'RUNNING':
              status = 'streaming'
              response['result'] = self.fetch_result(notebook, snippet, n, False)
            elif resp.get('status') == 'FINISHED':
              status = 'available'
            elif resp.get('status') == 'FAILED':
              status = 'failed'
            elif resp.get('status') == 'CANCELED':
              status = 'expired'
          except Exception as e:
            if '%s does not exist in current session' % statement_id in str(e):
              LOG.warn('Job: %s does not exist' % statement_id)
            else:
              raise e

    response['status'] = status

    return response

  @query_error_handler
  def fetch_result(self, notebook, snippet, rows, start_over):
    global n
    session = self._get_session()
    statement_id = snippet['result']['handle']['guid']
    token = n #rows

    resp = self.db.fetch_results(session['id'], job_id=statement_id, token=token)

    next_result = resp.get('next_result_uri')
    if next_result:
      n = int(next_result.rsplit('/', 1)[-1])

    return {
        'has_more': bool(next_result),
        'data': resp['results'][0]['data'],  # No escaping...
        'meta': [{
            'name': column['name'],
            'type': column['type'],
            'comment': ''
          }
          for column in resp['results'][0]['columns']
        ],
        'type': 'table'
    }

  @query_error_handler
  def autocomplete(self, snippet, database=None, table=None, column=None, nested=None, operation=None):
    response = {}

    if database is None:
      response['databases'] = self.show_databases()
    elif table is None:
      response['tables_meta'] = self.show_tables(database)
    elif column is None:
      columns = self.get_columns(database, table)
      response['columns'] = [col['name'] for col in columns]
      response['extended_columns'] = [{
          'comment': col.get('comment'),
          'name': col.get('name'),
          'type': col['type']
        }
        for col in columns
      ]

    return response

  def show_databases(self):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='SHOW DATABASES')

    return [db[0] for db in resp['results'][0]['data']]

  def show_tables(self, database):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='USE %(database)s' % {'database': database})
    resp = self.db.execute_statement(session_id=session_id, statement='SHOW TABLES')

    return [table[0] for table in resp['results'][0]['data']]

  def get_columns(self, database, table):
    session = self._get_session()
    session_id = session['id']

    resp = self.db.execute_statement(session_id=session_id, statement='USE %(database)s' % {'database': database})
    resp = self.db.execute_statement(session_id=session_id, statement='DESCRIBE %(table)s' % {'table': table})

    columns = resp['results'][0]['columns']

    return [{
        'name': col['name'],
        'type': col['type'],  # Types to unify
        'comment': '',
      }
      for col in columns
    ]

  def cancel(self, notebook, snippet):
    session = self._get_session()
    statement_id = snippet['result']['handle']['guid']

    try:
      self.db.close_statement(session_id=session['id'], job_id=statement_id)
    except Exception as e:
      if 'does not exist in current session:' in str(e):
        return {'status': -1}  # skipped
      else:
        raise e

    return {'status': 0}

  def close_session(self, session):
    # Avoid closing session on page refresh or editor close for now
    pass
    # session = self._get_session()
    # self.db.close_session(session['id'])

class FlinkSqlClient():
  '''
  Implements https://github.com/ververica/flink-sql-gateway
  Could be a pip module or sqlalchemy dialect in the future.
  '''

  def __init__(self, user, api_url):
    self.user = user
    self._url = posixpath.join(api_url + '/' + _API_VERSION + '/')
    self._client = HttpClient(self._url, logger=LOG)
    self._root = Resource(self._client)

  def __str__(self):
    return "FlinkClient at %s" % (self._url,)

  def info(self):
    return self._root.get('info')

  def create_session(self, **properties):
    data = {
        "session_name": "test",  # optional
        "planner": "blink",  # required, "old"/"blink"
        "execution_type": "streaming",  # required, "batch"/"streaming"
        "properties": {  # optional
            "key": "value"
        }
    }
    data.update(properties)

    return self._root.post('sessions', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)

  def session_heartbeat(self, session_id):
    return self._root.post('sessions/%(session_id)s/heartbeat' % {'session_id': session_id})

  def execute_statement(self, session_id, statement):
    data = {
        "statement": statement,  # required
        "execution_timeout": ""  # execution time limit in milliseconds, optional, but required for stream SELECT ?
    }

    return self._root.post(
        'sessions/%(session_id)s/statements' % {
        'session_id': session_id
      },
      data=json.dumps(data),
      contenttype=_JSON_CONTENT_TYPE
    )

  def fetch_status(self, session_id, job_id):
    return self._root.get(
      'sessions/%(session_id)s/jobs/%(job_id)s/status' % {
        'session_id': session_id,
        'job_id': job_id
      }
    )

  def fetch_results(self, session_id, job_id, token=0):
    return self._root.get(
      'sessions/%(session_id)s/jobs/%(job_id)s/result/%(token)s' % {
        'session_id': session_id,
        'job_id': job_id,
        'token': token
      }
    )

  def close_statement(self, session_id, job_id):
    return self._root.delete(
      'sessions/%(session_id)s/jobs/%(job_id)s' % {
        'session_id': session_id,
        'job_id': job_id,
      }
    )

  def close_session(self, session_id):
    return self._root.delete(
      'sessions/%(session_id)s' % {
        'session_id': session_id,
      }
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
The main program that is the entry point for the SeedboxManager application.
Provides the ability to configure and start up processing.
"""
from __future__ import absolute_import
import logging
import os
import sys

import lockfile
from lockfile import pidlockfile
from oslo.config import cfg

from seedbox import db
from seedbox import logext as logmgr
from seedbox import options
from seedbox import process

LOG = logging.getLogger(__name__)

def main():
    """
    Entry point for seedmgr
    """

    # processes all command-line inputs that control how we execute
    # logging, run mode, etc.; and we get a handle back to access
    # the info
    options.initialize(sys.argv[1:])

    # configure our logging
    logmgr.configure()
    cfg.CONF.log_opt_values(LOG, logging.DEBUG)

    # need to create a lock to make sure multiple instances do not start at
    # the same time because we are running as a cron.
    filelock = os.path.join(cfg.CONF.config_dir, 'seedmgr.lock')
    lock = pidlockfile.PIDLockFile(filelock, timeout=10)
    try:
        with lock:

            # setup database and initialize connection
            db.dbapi(cfg.CONF)

            # time to start processing
            process.start()

    except lockfile.LockTimeout as lockerr:
        # if we have managed timeout, it means there is another instance
        # already running so we will simply bow out and let the existing
        # one still run.
        LOG.info('Already running; if not delete lock file: %s', lockerr)
<commit_msg>clean up to further simplify implementation of pid lockfile<commit_after>#!/usr/bin/env python
"""
The main program that is the entry point for the SeedboxManager application.
Provides the ability to configure and start up processing.
"""
from __future__ import absolute_import
import logging
import os
import sys

import lockfile
from lockfile import pidlockfile
from oslo.config import cfg

from seedbox import db
from seedbox import logext as logmgr
from seedbox import options
from seedbox import process

LOG = logging.getLogger(__name__)

def main():
    """
    Entry point for seedmgr
    """

    # processes all command-line inputs that control how we execute
    # logging, run mode, etc.; and we get a handle back to access
    # the info
    options.initialize(sys.argv[1:])

    # configure our logging
    logmgr.configure()
    cfg.CONF.log_opt_values(LOG, logging.DEBUG)

    try:
        # need to create a lock to make sure multiple instances do not start
        # at the same time because we are running as a cron.
        with pidlockfile.PIDLockFile(os.path.join(cfg.CONF.config_dir,
                                     'seedmgr.lock'), timeout=10):

            # setup database and initialize connection
            db.dbapi(cfg.CONF)

            # time to start processing
            process.start()

    except lockfile.LockTimeout as lockerr:
        # if we have managed timeout, it means there is another instance
        # already running so we will simply bow out and let the existing
        # one still run.
        LOG.info('Already running; if not delete lock file: %s', lockerr)
<|endoftext|>


<commit_before># Python Version: 3.x
# -*- coding: utf-8 -*-
import onlinejudge.service
import onlinejudge.problem
from onlinejudge.problem import LabeledString, TestCase
import onlinejudge.dispatch
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import io
import posixpath
import json
import requests
import urllib.parse
import zipfile
import collections
import itertools
from typing import *

@utils.singleton
class AOJService(onlinejudge.service.Service):

    def get_url(self):
        return 'http://judge.u-aizu.ac.jp/onlinejudge/'

    def get_name(self):
        return 'aoj'

    @classmethod
    def from_url(cls, s: str) -> Optional['AOJService']:
        # example: http://judge.u-aizu.ac.jp/onlinejudge/
        result = urllib.parse.urlparse(s)
        if result.scheme in ('', 'http', 'https') \
                and result.netloc == 'judge.u-aizu.ac.jp':
            return cls()
        return None

class AOJProblem(onlinejudge.problem.Problem):
    def __init__(self, problem_id):
        self.problem_id = problem_id

    def download(self, session: Optional[requests.Session] = None, is_system: bool = False) -> List[TestCase]:
        if is_system:
            return self.download_system(session=session)
        else:
            return self.download_samples(session=session)

    def download_samples(self, session: Optional[requests.Session] = None) -> List[TestCase]:
        session = session or utils.new_default_session()
        # get samples via the official API
        # reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2Fsamples%2F%7BproblemId%7D_GET
        url = 'https://judgedat.u-aizu.ac.jp/testcases/samples/{}'.format(self.problem_id)
        resp = utils.request('GET', url, session=session)
        samples: List[TestCase] = []
        for sample in json.loads(resp.content):
            samples += [ TestCase(
                LabeledString(str(sample['serial']), sample['in']),
                LabeledString(str(sample['serial']), sample['out']),
                ) ]
        return samples

    def download_system(self, session: Optional[requests.Session] = None) -> List[TestCase]:
        session = session or utils.new_default_session()
        get_url = lambda case, type: 'https://judgedat.u-aizu.ac.jp/testcases/{}/{}/{}'.format(self.problem_id, case, type)
        testcases: List[TestCase] = []
        for case in itertools.count(1):
            # input
            resp = utils.request('GET', get_url(case, 'in'), session=session)
            in_txt = resp.text
            if in_txt.strip() == '/* This is a single file for multiple testcases. serial should be 1. */':
                break
            if in_txt.strip() == '/* Test case #{} for problem {} is not available. */'.format(case, self.problem_id):
                break
            # output
            resp = utils.request('GET', get_url(case, 'out'), session=session)
            out_txt = resp.text
            assert out_txt.strip() != '/* This is a single file for multiple testcases. serial should be 1. */'
            testcases += [ TestCase(
                LabeledString('in%d.txt' % case, in_txt),
                LabeledString('out%d.txt' % case, out_txt),
                ) ]
        return testcases

    def get_url(self) -> str:
        return 'http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id={}'.format(self.problem_id)

    @classmethod
    def from_url(cls, s: str) -> Optional['AOJProblem']:
        # example: http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id=1169
        # example: http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id=DSL_1_A&lang=jp
        result = urllib.parse.urlparse(s)
        querystring = urllib.parse.parse_qs(result.query)
        if result.scheme in ('', 'http', 'https') \
                and result.netloc == 'judge.u-aizu.ac.jp' \
                and utils.normpath(result.path) == '/onlinejudge/description.jsp' \
                and querystring.get('id') \
                and len(querystring['id']) == 1:
            n, = querystring['id']
            return cls(n)
        return None

    def get_service(self) -> AOJService:
        return AOJService()

onlinejudge.dispatch.services += [ AOJService ]
onlinejudge.dispatch.problems += [ AOJProblem ]
<commit_msg>#194: use the official API of AOJ for system cases<commit_after># Python Version: 3.x
# -*- coding: utf-8 -*-
import onlinejudge.service
import onlinejudge.problem
from onlinejudge.problem import LabeledString, TestCase
import onlinejudge.dispatch
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import io
import posixpath
import json
import requests
import urllib.parse
import zipfile
import collections
import itertools
from typing import *

@utils.singleton
class AOJService(onlinejudge.service.Service):

    def get_url(self):
        return 'http://judge.u-aizu.ac.jp/onlinejudge/'

    def get_name(self):
        return 'aoj'

    @classmethod
    def from_url(cls, s: str) -> Optional['AOJService']:
        # example: http://judge.u-aizu.ac.jp/onlinejudge/
        result = urllib.parse.urlparse(s)
        if result.scheme in ('', 'http', 'https') \
                and result.netloc == 'judge.u-aizu.ac.jp':
            return cls()
        return None

class AOJProblem(onlinejudge.problem.Problem):
    def __init__(self, problem_id):
        self.problem_id = problem_id

    def download(self, session: Optional[requests.Session] = None, is_system: bool = False) -> List[TestCase]:
        if is_system:
            return self.download_system(session=session)
        else:
            return self.download_samples(session=session)

    def download_samples(self, session: Optional[requests.Session] = None) -> List[TestCase]:
        session = session or utils.new_default_session()
        # get samples via the official API
        # reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2Fsamples%2F%7BproblemId%7D_GET
        url = 'https://judgedat.u-aizu.ac.jp/testcases/samples/{}'.format(self.problem_id)
        resp = utils.request('GET', url, session=session)
        samples: List[TestCase] = []
        for sample in json.loads(resp.content):
            samples += [ TestCase(
                LabeledString(str(sample['serial']), sample['in']),
                LabeledString(str(sample['serial']), sample['out']),
                ) ]
        return samples

    def download_system(self, session: Optional[requests.Session] = None) -> List[TestCase]:
        session = session or utils.new_default_session()

        # get header
        # reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2F%7BproblemId%7D%2Fheader_GET
        url = 'https://judgedat.u-aizu.ac.jp/testcases/{}/header'.format(self.problem_id)
        resp = utils.request('GET', url, session=session)
        header = json.loads(resp.content)

        # get testcases via the official API
        testcases: List[TestCase] = []
        for header in header['headers']:
            # reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2F%7BproblemId%7D%2F%7Bserial%7D_GET
            url = 'https://judgedat.u-aizu.ac.jp/testcases/{}/{}'.format(self.problem_id, header['serial'])
            resp = utils.request('GET', url, session=session)
            testcase = json.loads(resp.content)
            testcases += [ TestCase(
                LabeledString(header['name'],  testcase['in']),
                LabeledString(header['name'], testcase['out']),
                ) ]
        return testcases

    def get_url(self) -> str:
        return 'http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id={}'.format(self.problem_id)

    @classmethod
    def from_url(cls, s: str) -> Optional['AOJProblem']:
        # example: http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id=1169
        # example: http://judge.u-aizu.ac.jp/onlinejudge/description.jsp?id=DSL_1_A&lang=jp
        result = urllib.parse.urlparse(s)
        querystring = urllib.parse.parse_qs(result.query)
        if result.scheme in ('', 'http', 'https') \
                and result.netloc == 'judge.u-aizu.ac.jp' \
                and utils.normpath(result.path) == '/onlinejudge/description.jsp' \
                and querystring.get('id') \
                and len(querystring['id']) == 1:
            n, = querystring['id']
            return cls(n)
        return None

    def get_service(self) -> AOJService:
        return AOJService()

onlinejudge.dispatch.services += [ AOJService ]
onlinejudge.dispatch.problems += [ AOJProblem ]
<|endoftext|>


<commit_before>VERSION = 27  # Please update this whenever .client or .services changes
# Please update MIN_VERSION_OVERRIDE and MIN_CLIENT_VERSION_OVERRIDE as needed
<commit_msg>Bump version 28<commit_after>VERSION = 28  # Please update this whenever .client or .services changes
# Please update MIN_VERSION_OVERRIDE and MIN_CLIENT_VERSION_OVERRIDE as needed
<|endoftext|>


<commit_before>import datetime
import json
import threading
import time
import traceback

import etcd
from pytz import utc

from tendrl.commons.event import Event
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.message import ExceptionMessage
from tendrl.commons.objects import AtomExecutionFailedError
from tendrl.commons.utils import alert_utils
from tendrl.commons.utils import etcd_utils
from tendrl.commons.utils import log_utils as logger
from tendrl.commons.utils import time_utils

DEFAULT_JOB_TTL = 172800

class JobConsumerThread(threading.Thread):

    def __init__(self):
        super(JobConsumerThread, self).__init__()
        self.daemon = True
        self.name = "Job consumer thread"
        self._complete = threading.Event()

    def run(self):
        logger.log(
            "debug",
            NS.publisher_id,
            {"message": "%s running" % self.__class__.__name__}
        )
        while not self._complete.is_set():
            _job_sync_interval = 5
            NS.node_context = NS.node_context.load()
            NS.tendrl_context = NS.tendrl_context.load()
            if "tendrl/monitor" not in NS.node_context.tags:
                if NS.tendrl_context.integration_id is None or \
                        NS.node_context.fqdn is None:
                    time.sleep(_job_sync_interval)
                    continue
            if "tendrl/monitor" in NS.node_context.tags:
                _job_sync_interval = 3

            time.sleep(_job_sync_interval)
            try:
                jobs = etcd_utils.read("/queue")
            except etcd.EtcdKeyNotFound:
                continue

            for job in jobs.leaves:
                # Check job not already locked by some agent
                jid = job.key.split('/')[-1]
                job_lock_key = "/queue/%s/locked_by" % jid
                try:
                    _locked_by = etcd_utils.read(job_lock_key).value
                    if _locked_by:
                        continue
                except etcd.EtcdKeyNotFound:
                    pass

                _job_thread = threading.Thread(
                    target=process_job, args=(jid,)
                )
                _job_thread.daemon = True
                _job_thread.start()
                _job_thread.join()

    def stop(self):
        self._complete.set()

def process_job(jid):
    job_obj = NS.tendrl.objects.Job(job_id=jid).load()
    NS.node_context = NS.node_context.load()
    # Check job not already "finished", or "processing"
    try:
        if job_obj.status in ["finished", "processing"]:
            return
    except etcd.EtcdKeyNotFound:
        pass

    try:
        _timeout = None
        _timeout = job_obj.timeout
        if _timeout:
            _timeout = _timeout.lower()
    except etcd.EtcdKeyNotFound:
        pass

    # tendrl-node-agent tagged as tendrl/monitor will ensure
    # >10 min old "new" jobs are timed out and marked as
    # "failed" (the parent job of these jobs will also be
    # marked as "failed")
    if "tendrl/monitor" in NS.node_context.tags and \
        _timeout == "yes":
        _valid_until = None
        try:
            _valid_until = job_obj.valid_until
        except etcd.EtcdKeyNotFound:
            pass

        if _valid_until:
            _now_epoch = (time_utils.now() -
                          datetime.datetime(1970, 1,
                                            1).replace(
                              tzinfo=utc)).total_seconds()
            if int(_now_epoch) >= int(_valid_until):
                # Job has "new" status since 10 minutes,
                # mark status as "failed" and Job.error =
                # "Timed out"
                try:
                    job_obj = job_obj.load()
                    job_obj.status = "failed"
                    job_obj.save()
                except etcd.EtcdCompareFailed:
                    pass
                else:
                    job = NS.tendrl.objects.Job(job_id=jid).load()
                    _msg = str("Timed-out (>10min as 'new')")
                    job.errors = _msg
                    job.save()
                    if job.payload.get('parent') is None:
                        alert_utils.alert_job_status(
                            "failed",
                            "Job timed out (job_id: %s)" % jid,
                            integration_id=NS.tendrl_context.integration_id or
                            job.payload['parameters'].get(
                                'TendrlContext.integration_id'
                            ),
                            cluster_name=NS.tendrl_context.cluster_name or
                            job.payload['parameters'].get(
                                'TendrlContext.cluster_name'
                            )
                        )
                    return
        else:
            _now_plus_10 = time_utils.now() + datetime.timedelta(minutes=10)
            _epoch_start = datetime.datetime(1970, 1, 1).replace(tzinfo=utc)

            # noinspection PyTypeChecker
            _now_plus_10_epoch = (_now_plus_10 -
                                  _epoch_start).total_seconds()
            job_obj = job_obj.load()
            job_obj.valid_util = int(_now_plus_10_epoch)
            job_obj.save()

    job = NS.tendrl.objects.Job(job_id=jid).load()
    if job.payload["type"] == NS.type and \
            job.status == "new":
        # Job routing
        # Flows created by tendrl-api use 'tags' from flow
        # definition to target jobs
        _tag_match = False
        if job.payload.get("tags", []):
            for flow_tag in job.payload['tags']:
                if flow_tag in NS.node_context.tags:
                    _tag_match = True

        if not _tag_match:
            _job_tags = ", ".join(job.payload.get("tags", []))
            _msg = "Node (%s)(type: %s)(tags: %s) will not " \
                   "process job-%s (tags: %s)" % \
                   (NS.node_context.node_id, NS.type,
                    NS.node_context.tags, jid,
                    _job_tags)
            logger.log(
                "debug",
                NS.publisher_id,
                {"message": _msg}
            )
            return

        try:
            try:
                job_status_key = "/queue/%s/status" % job.job_id
                etcd_utils.write(job_status_key,
                                 "processing",
                                 prevValue="new")
            except etcd.EtcdKeyNotFound:
                # if status watchable attribute not present
                # then it will be created when job save happens
                pass
            lock_info = dict(node_id=NS.node_context.node_id,
                             fqdn=NS.node_context.fqdn,
                             tags=NS.node_context.tags,
                             type=NS.type)
            job = job.load()
            job.locked_by = json.dumps(lock_info)
            job.status = "processing"
            job.save(ttl=DEFAULT_JOB_TTL)
        except etcd.EtcdCompareFailed:
            # job is already being processed by some tendrl
            # agent
            return

        the_flow = None
        try:
            current_ns, flow_name, obj_name = \
                _extract_fqdn(job.payload['run'])

            if obj_name:
                runnable_flow = current_ns.ns.get_obj_flow(
                    obj_name, flow_name)
            else:
                runnable_flow = current_ns.ns.get_flow(flow_name)

            the_flow = runnable_flow(parameters=job.payload[
                'parameters'], job_id=job.job_id)
            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Starting Job %s" %
                            job.job_id},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id']
            )

            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Running %s" %
                            job.payload['run'].split('.')[-1]},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id']
            )
            the_flow.run()
            try:
                job = job.load()
                job.status = "finished"
                job.save()
            except etcd.EtcdCompareFailed:
                # This should not happen!
                _msg = "Cannot mark job as 'finished', " \
                       "current job status invalid"
                raise FlowExecutionFailedError(_msg)

            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Job (%s) for %s finished. "
                            % (
                                job.job_id,
                                job.payload['run'].split('.')[-1])},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id'],
            )
            if job.payload.get('parent') is None:
                alert_utils.alert_job_status(
                    "finished",
                    "%s (job ID: %s) completed successfully " % (
                        job.payload['run'].split('.')[-1],
                        job.job_id),
                    integration_id=NS.tendrl_context.integration_id or
                    job.payload['parameters'].get(
                        'TendrlContext.integration_id'
                    ),
                    cluster_name=NS.tendrl_context.cluster_name or
                    job.payload['parameters'].get(
                        'TendrlContext.cluster_name'
                    )
                )
        except (FlowExecutionFailedError,
                AtomExecutionFailedError,
                Exception) as e:
            _trace = str(traceback.format_exc(e))
            _msg = "Failure in Job %s Flow %s with error:" % \
                   (job.job_id, job.payload['run'])
            Event(
                ExceptionMessage(
                    priority="error",
                    publisher=NS.publisher_id,
                    payload={"message": _msg + _trace,
                             "exception": e
                             }
                )
            )
            if the_flow:
                logger.log(
                    "error",
                    NS.publisher_id,
                    {"message": _msg + "\n" + _trace},
                    job_id=job.job_id,
                    flow_id=the_flow.parameters['flow_id']
                )
            else:
                logger.log(
                    "error",
                    NS.publisher_id,
                    {"message": _msg + "\n" + _trace}
                )

            try:
                job = job.load()
                job.status = "failed"
                job.save()
            except etcd.EtcdCompareFailed:
                # This should not happen!
                _msg = "Cannot mark job as 'failed', current" \
                       "job status invalid"
                raise FlowExecutionFailedError(_msg)
            else:
                job = job.load()
                job.errors = _trace
                if job.payload.get('parent') is None:
                    alert_utils.alert_job_status(
                        "failed",
                        "Job failed (job_id: %s)" % job.job_id,
                        integration_id=NS.tendrl_context.integration_id or
                        job.payload['parameters'].get(
                            'TendrlContext.integration_id'
                        ),
                        cluster_name=NS.tendrl_context.cluster_name or
                        job.payload['parameters'].get(
                            'TendrlContext.cluster_name'
                        )
                    )
                job.save()

def _extract_fqdn(flow_fqdn):
    ns, flow_name = flow_fqdn.split(".flows.")
    obj_name = None

    # check if the flow is bound to any object
    try:
        ns, obj_name = ns.split(".objects.")
    except ValueError:
        pass

    ns_str = ns.split(".")[-1]
    if "integrations" in ns:
        return getattr(NS.integrations, ns_str), flow_name, obj_name
    else:
        return getattr(NS, ns_str), flow_name, obj_name
<commit_msg>Update __init__.py<commit_after>import datetime
import json
import threading
import time
import traceback

import etcd
from pytz import utc

from tendrl.commons.event import Event
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.message import ExceptionMessage
from tendrl.commons.objects import AtomExecutionFailedError
from tendrl.commons.utils import alert_utils
from tendrl.commons.utils import etcd_utils
from tendrl.commons.utils import log_utils as logger
from tendrl.commons.utils import time_utils

DEFAULT_JOB_TTL = 172800

class JobConsumerThread(threading.Thread):

    def __init__(self):
        super(JobConsumerThread, self).__init__()
        self.daemon = True
        self.name = "Job consumer thread"
        self._complete = threading.Event()

    def run(self):
        logger.log(
            "debug",
            NS.publisher_id,
            {"message": "%s running" % self.__class__.__name__}
        )
        while not self._complete.is_set():
            _job_sync_interval = 5
            NS.node_context = NS.node_context.load()
            NS.tendrl_context = NS.tendrl_context.load()
            if "tendrl/monitor" not in NS.node_context.tags:
                if NS.tendrl_context.integration_id is None or \
                        NS.node_context.fqdn is None:
                    time.sleep(_job_sync_interval)
                    continue
            if "tendrl/monitor" in NS.node_context.tags:
                _job_sync_interval = 3

            time.sleep(_job_sync_interval)
            try:
                jobs = etcd_utils.read("/queue")
            except etcd.EtcdKeyNotFound:
                continue

            for job in jobs.leaves:
                # Check job not already locked by some agent
                jid = job.key.split('/')[-1]
                job_lock_key = "/queue/%s/locked_by" % jid
                try:
                    _locked_by = etcd_utils.read(job_lock_key).value
                    if _locked_by:
                        continue
                except etcd.EtcdKeyNotFound:
                    pass

                _job_thread = threading.Thread(
                    target=process_job, args=(jid,)
                )
                _job_thread.daemon = True
                _job_thread.start()
                _job_thread.join()

    def stop(self):
        self._complete.set()

def process_job(jid):
    job = NS.tendrl.objects.Job(job_id=jid).load()
    if job.status in [None, ""]:
        job.status = "new"
        job.save()

    NS.node_context = NS.node_context.load()
    # Check job not already "finished", or "processing"
    try:
        if job.status in ["finished", "processing"]:
            return
    except etcd.EtcdKeyNotFound:
        pass

    try:
        _timeout = None
        _timeout = job.timeout
        if _timeout:
            _timeout = _timeout.lower()
    except etcd.EtcdKeyNotFound:
        pass

    # tendrl-node-agent tagged as tendrl/monitor will ensure
    # >10 min old "new" jobs are timed out and marked as
    # "failed" (the parent job of these jobs will also be
    # marked as "failed")
    if "tendrl/monitor" in NS.node_context.tags and \
        _timeout == "yes":
        _valid_until = job.valid_until

        if _valid_until:
            _now_epoch = (time_utils.now() -
                          datetime.datetime(1970, 1,
                                            1).replace(
                              tzinfo=utc)).total_seconds()
            if int(_now_epoch) >= int(_valid_until):
                # Job has "new" status since 10 minutes,
                # mark status as "failed" and Job.error =
                # "Timed out"
                try:
                    job = job.load()
                    job.status = "failed"
                    job.save()
                except etcd.EtcdCompareFailed:
                    pass
                else:
                    job = NS.tendrl.objects.Job(job_id=jid).load()
                    _msg = str("Timed-out (>10min as 'new')")
                    job.errors = _msg
                    job.save()
                    if job.payload.get('parent') is None:
                        alert_utils.alert_job_status(
                            "failed",
                            "Job timed out (job_id: %s)" % jid,
                            integration_id=NS.tendrl_context.integration_id or
                            job.payload['parameters'].get(
                                'TendrlContext.integration_id'
                            ),
                            cluster_name=NS.tendrl_context.cluster_name or
                            job.payload['parameters'].get(
                                'TendrlContext.cluster_name'
                            )
                        )
                    return
        else:
            _now_plus_10 = time_utils.now() + datetime.timedelta(minutes=10)
            _epoch_start = datetime.datetime(1970, 1, 1).replace(tzinfo=utc)

            _now_plus_10_epoch = (_now_plus_10 -
                                  _epoch_start).total_seconds()
            job = job.load()
            job.valid_util = int(_now_plus_10_epoch)
            job.save()

    job = NS.tendrl.objects.Job(job_id=jid).load()
    if job.payload["type"] == NS.type and \
            job.status == "new":
        # Job routing
        # Flows created by tendrl-api use 'tags' from flow
        # definition to target jobs
        _tag_match = False
        if job.payload.get("tags", []):
            for flow_tag in job.payload['tags']:
                if flow_tag in NS.node_context.tags:
                    _tag_match = True

        if not _tag_match:
            _job_tags = ", ".join(job.payload.get("tags", []))
            _msg = "Node (%s)(type: %s)(tags: %s) will not " \
                   "process job-%s (tags: %s)" % \
                   (NS.node_context.node_id, NS.type,
                    NS.node_context.tags, jid,
                    _job_tags)
            logger.log(
                "debug",
                NS.publisher_id,
                {"message": _msg}
            )
            return

        try:
            try:
                job_status_key = "/queue/%s/status" % job.job_id
                etcd_utils.write(job_status_key,
                                 "processing",
                                 prevValue="new")
            except etcd.EtcdKeyNotFound:
                # if status watchable attribute not present
                # then it will be created when job save happens
                pass
            lock_info = dict(node_id=NS.node_context.node_id,
                             fqdn=NS.node_context.fqdn,
                             tags=NS.node_context.tags,
                             type=NS.type)
            job = job.load()
            job.locked_by = lock_info
            job.status = "processing"
            job.save(ttl=DEFAULT_JOB_TTL)
        except etcd.EtcdCompareFailed:
            # job is already being processed by some tendrl
            # agent
            return

        the_flow = None
        try:
            current_ns, flow_name, obj_name = \
                _extract_fqdn(job.payload['run'])

            if obj_name:
                runnable_flow = current_ns.ns.get_obj_flow(
                    obj_name, flow_name)
            else:
                runnable_flow = current_ns.ns.get_flow(flow_name)

            the_flow = runnable_flow(parameters=job.payload[
                'parameters'], job_id=job.job_id)
            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Starting Job %s" %
                            job.job_id},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id']
            )

            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Running %s" %
                            job.payload['run'].split('.')[-1]},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id']
            )
            job = job.load()
            lock_info = dict(node_id=NS.node_context.node_id,
                             fqdn=NS.node_context.fqdn,
                             tags=NS.node_context.tags,
                             type=NS.type)
            if job.locked_by == lock_info:
                the_flow.run()

            try:
                job = job.load()
                job.status = "finished"
                job.save()
            except etcd.EtcdCompareFailed:
                # This should not happen!
                _msg = "Cannot mark job as 'finished', " \
                       "current job status invalid"
                raise FlowExecutionFailedError(_msg)

            logger.log(
                "info",
                NS.publisher_id,
                {"message": "Job (%s) for %s finished. "
                            % (
                                job.job_id,
                                job.payload['run'].split('.')[-1])},
                job_id=job.job_id,
                flow_id=the_flow.parameters['flow_id'],
            )
            if job.payload.get('parent') is None:
                alert_utils.alert_job_status(
                    "finished",
                    "%s (job ID: %s) completed successfully " % (
                        job.payload['run'].split('.')[-1],
                        job.job_id),
                    integration_id=NS.tendrl_context.integration_id or
                    job.payload['parameters'].get(
                        'TendrlContext.integration_id'
                    ),
                    cluster_name=NS.tendrl_context.cluster_name or
                    job.payload['parameters'].get(
                        'TendrlContext.cluster_name'
                    )
                )
        except (FlowExecutionFailedError,
                AtomExecutionFailedError,
                Exception) as e:
            _trace = str(traceback.format_exc(e))
            _msg = "Failure in Job %s Flow %s with error:" % \
                   (job.job_id, job.payload['run'])
            Event(
                ExceptionMessage(
                    priority="error",
                    publisher=NS.publisher_id,
                    payload={"message": _msg + _trace,
                             "exception": e
                             }
                )
            )
            if the_flow:
                logger.log(
                    "error",
                    NS.publisher_id,
                    {"message": _msg + "\n" + _trace},
                    job_id=job.job_id,
                    flow_id=the_flow.parameters['flow_id']
                )
            else:
                logger.log(
                    "error",
                    NS.publisher_id,
                    {"message": _msg + "\n" + _trace}
                )

            try:
                job = job.load()
                job.status = "failed"
                job.save()
            except etcd.EtcdCompareFailed:
                # This should not happen!
                _msg = "Cannot mark job as 'failed', current" \
                       "job status invalid"
                raise FlowExecutionFailedError(_msg)
            else:
                job = job.load()
                job.errors = _trace
                if job.payload.get('parent') is None:
                    alert_utils.alert_job_status(
                        "failed",
                        "Job failed (job_id: %s)" % job.job_id,
                        integration_id=NS.tendrl_context.integration_id or
                        job.payload['parameters'].get(
                            'TendrlContext.integration_id'
                        ),
                        cluster_name=NS.tendrl_context.cluster_name or
                        job.payload['parameters'].get(
                            'TendrlContext.cluster_name'
                        )
                    )
                job.save()

def _extract_fqdn(flow_fqdn):
    ns, flow_name = flow_fqdn.split(".flows.")
    obj_name = None

    # check if the flow is bound to any object
    try:
        ns, obj_name = ns.split(".objects.")
    except ValueError:
        pass

    ns_str = ns.split(".")[-1]
    if "integrations" in ns:
        return getattr(NS.integrations, ns_str), flow_name, obj_name
    else:
        return getattr(NS, ns_str), flow_name, obj_name
<|endoftext|>