<commit_before>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import itertools
import json
import re

from oslo.config import cfg
import yaml

from heat.common import exception
from heat.common.i18n import _

cfg.CONF.import_opt('max_template_size', 'heat.common.config')

if hasattr(yaml, 'CSafeLoader'):
    yaml_loader = yaml.CSafeLoader
else:
    yaml_loader = yaml.SafeLoader

if hasattr(yaml, 'CSafeDumper'):
    yaml_dumper = yaml.CSafeDumper
else:
    yaml_dumper = yaml.SafeDumper

def _construct_yaml_str(self, node):
    # Override the default string handling function
    # to always return unicode objects
    return self.construct_scalar(node)
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
# datetime.data which causes problems in API layer when being processed by
# openstack.common.jsonutils. Therefore, make unicode string out of timestamps
# until jsonutils can handle dates.
yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                            _construct_yaml_str)

def simple_parse(tmpl_str):
    try:
        tpl = json.loads(tmpl_str)
    except ValueError:
        try:
            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
        except yaml.YAMLError as yea:
            msg = _('Error parsing template: %s') % yea
            raise ValueError(msg)
        else:
            if tpl is None:
                tpl = {}

    if not isinstance(tpl, dict):
        raise ValueError(_('The template is not a JSON object '
                           'or YAML mapping.'))

    return tpl

def parse(tmpl_str):
    """Takes a string and returns a dict containing the parsed structure.

    This includes determination of whether the string is using the
    JSON or YAML format.
    """
    if len(tmpl_str) > cfg.CONF.max_template_size:
        msg = (_('Template exceeds maximum allowed size (%s bytes)') %
               cfg.CONF.max_template_size)
        raise exception.RequestLimitExceeded(message=msg)
    tpl = simple_parse(tmpl_str)
    # Looking for supported version keys in the loaded template
    if not ('HeatTemplateFormatVersion' in tpl
            or 'heat_template_version' in tpl
            or 'AWSTemplateFormatVersion' in tpl):
        raise ValueError(_("Template format version not found."))
    return tpl

def convert_json_to_yaml(json_str):
    """Convert AWS JSON template format to Heat YAML format.

    :param json_str: a string containing the AWS JSON template format.
    :returns: the equivalent string containing the Heat YAML format.
    """

    # Replace AWS format version with Heat format version
    json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
                      '', json_str)

    # insert a sortable order into the key to preserve file ordering
    key_order = itertools.count()

    def order_key(matchobj):
        key = '%s"__%05d__order__%s" :' % (
            matchobj.group(1),
            next(key_order),
            matchobj.group(2))
        return key
    key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M)
    json_str = key_re.sub(order_key, json_str)

    # parse the string as json to a python structure
    tpl = yaml.load(json_str, Loader=yaml_loader)

    # dump python structure to yaml
    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
    yml = yaml.dump(tpl, Dumper=yaml_dumper)

    # remove ordering from key names
    yml = re.sub('__\d*__order__', '', yml)
    return yml
<commit_msg>Fix translating error in unittests<commit_after>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import itertools
import json
import re
import six

from oslo.config import cfg
import yaml

from heat.common import exception
from heat.common.i18n import _

cfg.CONF.import_opt('max_template_size', 'heat.common.config')

if hasattr(yaml, 'CSafeLoader'):
    yaml_loader = yaml.CSafeLoader
else:
    yaml_loader = yaml.SafeLoader

if hasattr(yaml, 'CSafeDumper'):
    yaml_dumper = yaml.CSafeDumper
else:
    yaml_dumper = yaml.SafeDumper

def _construct_yaml_str(self, node):
    # Override the default string handling function
    # to always return unicode objects
    return self.construct_scalar(node)
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
# datetime.data which causes problems in API layer when being processed by
# openstack.common.jsonutils. Therefore, make unicode string out of timestamps
# until jsonutils can handle dates.
yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                            _construct_yaml_str)

def simple_parse(tmpl_str):
    try:
        tpl = json.loads(tmpl_str)
    except ValueError:
        try:
            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
        except yaml.YAMLError as yea:
            yea = six.text_type(yea)
            msg = _('Error parsing template: %s') % yea
            raise ValueError(msg)
        else:
            if tpl is None:
                tpl = {}

    if not isinstance(tpl, dict):
        raise ValueError(_('The template is not a JSON object '
                           'or YAML mapping.'))

    return tpl

def parse(tmpl_str):
    """Takes a string and returns a dict containing the parsed structure.

    This includes determination of whether the string is using the
    JSON or YAML format.
    """
    if len(tmpl_str) > cfg.CONF.max_template_size:
        msg = (_('Template exceeds maximum allowed size (%s bytes)') %
               cfg.CONF.max_template_size)
        raise exception.RequestLimitExceeded(message=msg)
    tpl = simple_parse(tmpl_str)
    # Looking for supported version keys in the loaded template
    if not ('HeatTemplateFormatVersion' in tpl
            or 'heat_template_version' in tpl
            or 'AWSTemplateFormatVersion' in tpl):
        raise ValueError(_("Template format version not found."))
    return tpl

def convert_json_to_yaml(json_str):
    """Convert AWS JSON template format to Heat YAML format.

    :param json_str: a string containing the AWS JSON template format.
    :returns: the equivalent string containing the Heat YAML format.
    """

    # Replace AWS format version with Heat format version
    json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
                      '', json_str)

    # insert a sortable order into the key to preserve file ordering
    key_order = itertools.count()

    def order_key(matchobj):
        key = '%s"__%05d__order__%s" :' % (
            matchobj.group(1),
            next(key_order),
            matchobj.group(2))
        return key
    key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M)
    json_str = key_re.sub(order_key, json_str)

    # parse the string as json to a python structure
    tpl = yaml.load(json_str, Loader=yaml_loader)

    # dump python structure to yaml
    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
    yml = yaml.dump(tpl, Dumper=yaml_dumper)

    # remove ordering from key names
    yml = re.sub('__\d*__order__', '', yml)
    return yml
<|endoftext|>


<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright 2017 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""Testing for `hera_mc.geo_location and geo_handling`."""

from __future__ import absolute_import, division, print_function

import os.path
import subprocess
from argparse import Namespace

import six
import pytest
import numpy as np

from .. import (cm_sysutils, cm_partconnect, cm_hookup, cm_utils, utils,
                cm_sysdef, cm_dossier, cm_active)

@pytest.fixture(scope='function')
def sys_handle(mcsession):
    return cm_sysutils.Handling(mcsession)

def test_ever_fully_connected(sys_handle):
    now_list = sys_handle.get_connected_stations(
        at_date='now', hookup_type='parts_hera')
    assert len(now_list) == 12

def test_publish_summary(sys_handle):
    msg = sys_handle.publish_summary()
    assert msg == 'Not on "main"'

def test_random_update(sys_handle):
    si = cm_sysutils.SystemInfo()
    si.update_arrays(None)

def test_other_hookup(sys_handle, mcsession, capsys):
    at_date = cm_utils.get_astropytime('2017-07-03')
    hookup = cm_hookup.Hookup(session=mcsession)
    assert hookup.cached_hookup_dict is None

    hu = hookup.get_hookup(['A700'], 'all', at_date='now', exact_match=True,
                           hookup_type='parts_hera')
    out = hookup.show_hookup(hu, cols_to_show=['station'], state='all',
                             revs=True, ports=True)
    assert 'HH700:A <ground' in out
    hu = hookup.get_hookup('A700,A701', 'all', at_date='now', exact_match=True,
                           hookup_type='parts_hera')
    assert 'A700:H' in hu.keys()
    hookup.write_hookup_cache_to_file(log_msg='For testing.')
    hu = hookup.get_hookup('HH', 'all', at_date='now', exact_match=True, use_cache=True)
    assert len(hu) == 0
    hu = hookup.get_hookup('cache', pol='all', hookup_type='parts_hera')
    out = hookup.show_hookup(hu, state='all', output_format='csv')
    assert '1230375618' in out
    out = hookup.show_hookup(hu, output_format='html')
    assert 'NBP700' in out
    out = hookup.show_hookup({}, cols_to_show=['station'], state='all',
                             revs=True, ports=True)
    assert out is None
    hufc = hookup.get_hookup_from_db(['HH'], 'e', at_date='now')
    assert len(hufc.keys()), 13
    hufc = hookup.get_hookup_from_db(['N700'], 'e', at_date='now')
    assert len(hufc.keys()) == 3
    assert hookup.hookup_type, 'parts_hera'
    gptf = hookup.get_part_types_found([])
    assert len(gptf) == 0

def test_hookup_notes(mcsession, capsys):
    hookup = cm_hookup.Hookup(session=mcsession)
    hu = hookup.get_hookup(['HH'])
    notes = hookup.get_notes(hu)
    assert len(notes) == 13
    print(hookup.show_notes(hu))
    captured = capsys.readouterr()
    assert '---HH700:A---' in captured.out.strip()

def test_hookup_dossier(sys_handle, capsys):
    sysdef = cm_sysdef.Sysdef()
    hude = cm_dossier.HookupEntry(entry_key='testing:key', sysdef=sysdef)
    hude.get_hookup_type_and_column_headers('x', 'rusty_scissors')
    captured = capsys.readouterr()
    assert 'Parts did not conform to any hookup_type' in captured.out.strip()
    print(hude)
    captured = capsys.readouterr()
    assert '<testing:key:' in captured.out.strip()
    hude.get_part_from_type('rusty_scissors', include_revs=True,
                            include_ports=True)
    hude.columns = {'x': 'y', 'hu': 'b', 'm': 'l'}
    hude.hookup['hu'] = []
    hude.hookup['hu'].append(Namespace(
        downstream_part='shoe', down_part_rev='testing',
        downstream_input_port='nail', upstream_output_port='screw'))
    xret = hude.get_part_from_type('b', include_revs=True,
                                   include_ports=True)
    assert xret['hu'] == 'shoe:testing<screw'

    # test errors
    hude = cm_dossier.HookupEntry(entry_key='testing:key', sysdef=sysdef)
    hude_dict = hude._to_dict()

    pytest.raises(ValueError, cm_dossier.HookupEntry,
                  entry_key='testing:key', input_dict=hude_dict)
    pytest.raises(ValueError, cm_dossier.HookupEntry, sysdef=sysdef,
                  input_dict=hude_dict)
    pytest.raises(ValueError, cm_dossier.HookupEntry,
                  entry_key='testing:key')
    pytest.raises(ValueError, cm_dossier.HookupEntry, sysdef=sysdef)

def test_sysdef(sys_handle, mcsession):
    sysdef = cm_sysdef.Sysdef()
    active = cm_active.ActiveData(session=mcsession)
    active.load_parts(at_date=None)
    active.load_connections(at_date=None)
    part = Namespace(hpn='N700', hpn_rev='A', hptype='node')
    part.connections = Namespace(input_ports=['loc0', '@mars'],
                                 output_ports=[])
    hl = sysdef.handle_redirect_part_types(part, active)
    assert len(hl), 4
    part.hpn = 'doorknob'
    part.part_type = 'node'
    sysdef.setup(part, pol='all')
    assert 'E<loc0' in sysdef.ppkeys
    sysdef.setup(part, pol='e')
    assert 'E<loc0' in sysdef.ppkeys
    part.connections.input_ports = ['@loc1', 'top', 'bottom']
    sysdef.setup(part, pol='e')
    assert 'E<loc0' in sysdef.ppkeys
    rg = Namespace(direction='up', part='apart', rev='A', port='e1', pol='e')
    op = [Namespace(upstream_part='apart', upstream_output_port='eb',
                    downstream_part='bpart', downstream_input_port='e1')]
    op.append(Namespace(upstream_part='aprt', upstream_output_port='eb',
                        downstream_part='bprt', downstream_input_port='ea'))
    rg.port = 'e4'
    A = Namespace(hpn='apart', part_type='front-end')
    B = [Namespace(hpn='apart', part_type='cable-rfof'),
         Namespace(hpn='bpart', part_type='cable-rfof')]
    rg.port = 'rug'
    op[0].downstream_input_port = 'rug'
    pytest.raises(ValueError, sysdef.find_hookup_type, 'dull_knife',
                  'parts_not')
    part = Namespace(part_type='nope')
    pytest.raises(ValueError, sysdef.setup, part, pol='nope',
                  hookup_type='parts_hera')
    curr = active.set_times(cm_utils.get_astropytime('2017-07-03'))
    assert int(curr) == 1183075218
    active.load_apriori('now')
    assert active.apriori['HH700:A'].status == 'not_connected'

def test_hookup_cache_file_info(sys_handle, mcsession):
    hookup = cm_hookup.Hookup(session=mcsession)
    cfi = hookup.hookup_cache_file_info()
    assert 'json does not exist' in cfi

def test_correlator_info(sys_handle):
    corr_dict = sys_handle.get_cminfo_correlator(hookup_type='parts_hera')
    ant_names = corr_dict['antenna_names']
    assert len(ant_names) == 12

    corr_inputs = corr_dict['correlator_inputs']

    stn_types = corr_dict['station_types']

    index = np.where(np.array(ant_names) == 'HH703')[0]
    assert len(index) == 1
    index = index[0]

    assert stn_types[index] == 'herahexw'
    assert corr_inputs[index] == ('e10>SNPA000700', 'n8>SNPA000700')

    assert ([int(name.split('HH')[1]) for name in ant_names]
            == corr_dict['antenna_numbers'])

    assert set(corr_dict['antenna_numbers']) == set(range(701, 713))

    assert corr_dict['cm_version'] is not None

    # cm_version should be the same as the git hash of m&c for the test data
    mc_dir = os.path.dirname(os.path.realpath(__file__))
    mc_git_hash = subprocess.check_output(['git', '-C', mc_dir, 'rev-parse',
                                           'HEAD'],
                                          stderr=subprocess.STDOUT).strip()

    # In Python 3, we sometimes get Unicode, sometimes bytes
    if isinstance(mc_git_hash, six.binary_type):
        mc_git_hash = utils.bytes_to_str(mc_git_hash)

    assert corr_dict['cm_version'] == mc_git_hash

    expected_keys = ['antenna_numbers', 'antenna_names', 'station_types',
                     'correlator_inputs', 'antenna_utm_datum_vals', 'epoch',
                     'antenna_utm_tiles', 'antenna_utm_eastings',
                     'antenna_utm_northings', 'antenna_positions',
                     'cm_version', 'cofa_lat', 'cofa_lon', 'cofa_alt']
    assert set(corr_dict.keys()) == set(expected_keys)

    cofa = sys_handle.cofa()[0]
    assert cofa.lat == corr_dict['cofa_lat']
    assert cofa.lon == corr_dict['cofa_lon']
    assert cofa.elevation == corr_dict['cofa_alt']

def test_get_pam_from_hookup(sys_handle, mcsession):
    hookup = cm_hookup.Hookup(mcsession)
    hud = hookup.get_hookup(['HH701'], at_date='2019-07-03', exact_match=True,
                            hookup_type='parts_hera')
    pams = hud[list(hud.keys())[0]].get_part_from_type(
        'post-amp', include_ports=True, include_revs=True)
    assert len(pams) == 2
    # the actual pam number (the thing written on the case)
    assert pams['E<ground'] == 'e>PAM701:A<e'

def test_get_pam_info(sys_handle, mcsession):
    sys_h = cm_sysutils.Handling(mcsession)
    pams = sys_h.get_part_at_station_from_type(
        'HH701', '2019-07-03', 'post-amp', include_ports=False,
        hookup_type='parts_hera')
    assert len(pams) == 1
    # the actual pam number (the thing written on the case)
    assert pams['HH701:A']['E<ground'] == 'PAM701'

def test_apriori_antenna(sys_handle, mcsession, capsys):
    cm_partconnect.update_apriori_antenna('HH701', 'needs_checking',
                                          '1214382618', session=mcsession)
    d = sys_handle.get_apriori_antenna_status_for_rtp('needs_checking')
    assert d == 'HH701'
    g = sys_handle.get_apriori_antenna_status_set()
    assert g['needs_checking'][0] == 'HH701'
    pytest.raises(ValueError, cm_partconnect.update_apriori_antenna,
                  'HH3', 'not_one', '1214482618', session=mcsession)
    g = sys_handle.get_apriori_status_for_antenna('HH701')
    assert g == 'needs_checking'
    cm_partconnect.update_apriori_antenna(
        'HH700', 'needs_checking', '1214482818', '1214483000',
        session=mcsession)
    pytest.raises(ValueError, cm_partconnect.update_apriori_antenna, 'HH700',
                  'needs_checking', '1214482838', session=mcsession)
    print(cm_partconnect.AprioriAntenna())
    captured = capsys.readouterr()
    assert '<None:' in captured.out.strip()
<commit_msg>added test to catch missing line<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright 2017 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""Testing for `hera_mc.geo_location and geo_handling`."""

from __future__ import absolute_import, division, print_function

import os.path
import subprocess
from argparse import Namespace

import six
import pytest
import numpy as np

from .. import (cm_sysutils, cm_partconnect, cm_hookup, cm_utils, utils,
                cm_sysdef, cm_dossier, cm_active)

@pytest.fixture(scope='function')
def sys_handle(mcsession):
    return cm_sysutils.Handling(mcsession)

def test_ever_fully_connected(sys_handle):
    now_list = sys_handle.get_connected_stations(
        at_date='now', hookup_type='parts_hera')
    assert len(now_list) == 12

def test_publish_summary(sys_handle):
    msg = sys_handle.publish_summary()
    assert msg == 'Not on "main"'

def test_random_update(sys_handle):
    si = cm_sysutils.SystemInfo()
    si.update_arrays(None)

def test_other_hookup(sys_handle, mcsession, capsys):
    at_date = cm_utils.get_astropytime('2017-07-03')
    hookup = cm_hookup.Hookup(session=mcsession)
    assert hookup.cached_hookup_dict is None

    hu = hookup.get_hookup(['A700'], 'all', at_date='now', exact_match=True,
                           hookup_type='parts_hera')
    out = hookup.show_hookup(hu, cols_to_show=['station'], state='all',
                             revs=True, ports=True)
    assert 'HH700:A <ground' in out
    hu = hookup.get_hookup('A700,A701', 'all', at_date='now', exact_match=True,
                           hookup_type='parts_hera')
    assert 'A700:H' in hu.keys()
    hookup.write_hookup_cache_to_file(log_msg='For testing.')
    hu = hookup.get_hookup('HH', 'all', at_date='now', exact_match=True, use_cache=True)
    assert len(hu) == 0
    hu = hookup.get_hookup('cache', pol='all', hookup_type='parts_hera')
    out = hookup.show_hookup(hu, state='all', output_format='csv')
    assert '1230375618' in out
    out = hookup.show_hookup(hu, output_format='html')
    assert 'NBP700' in out
    out = hookup.show_hookup({}, cols_to_show=['station'], state='all',
                             revs=True, ports=True)
    assert out is None
    hufc = hookup.get_hookup_from_db(['HH'], 'e', at_date='now')
    assert len(hufc.keys()), 13
    hufc = hookup.get_hookup_from_db(['N700'], 'e', at_date='now')
    assert len(hufc.keys()) == 3
    assert hookup.hookup_type, 'parts_hera'
    gptf = hookup.get_part_types_found([])
    assert len(gptf) == 0

def test_hookup_notes(mcsession, capsys):
    hookup = cm_hookup.Hookup(session=mcsession)
    hu = hookup.get_hookup(['HH'])
    notes = hookup.get_notes(hu)
    assert len(notes) == 13
    print(hookup.show_notes(hu))
    captured = capsys.readouterr()
    assert '---HH700:A---' in captured.out.strip()

def test_hookup_dossier(sys_handle, capsys):
    sysdef = cm_sysdef.Sysdef()
    hude = cm_dossier.HookupEntry(entry_key='testing:key', sysdef=sysdef)
    hude.get_hookup_type_and_column_headers('x', 'rusty_scissors')
    captured = capsys.readouterr()
    assert 'Parts did not conform to any hookup_type' in captured.out.strip()
    print(hude)
    captured = capsys.readouterr()
    assert '<testing:key:' in captured.out.strip()
    hude.get_part_from_type('rusty_scissors', include_revs=True,
                            include_ports=True)
    hude.columns = {'x': 'y', 'hu': 'b', 'm': 'l'}
    hude.hookup['hu'] = []
    hude.hookup['hu'].append(Namespace(
        downstream_part='shoe', down_part_rev='testing',
        downstream_input_port='nail', upstream_output_port='screw'))
    xret = hude.get_part_from_type('b', include_revs=True,
                                   include_ports=True)
    assert xret['hu'] == 'shoe:testing<screw'

    # test errors
    hude = cm_dossier.HookupEntry(entry_key='testing:key', sysdef=sysdef)
    hude_dict = hude._to_dict()

    pytest.raises(ValueError, cm_dossier.HookupEntry,
                  entry_key='testing:key', input_dict=hude_dict)
    pytest.raises(ValueError, cm_dossier.HookupEntry, sysdef=sysdef,
                  input_dict=hude_dict)
    pytest.raises(ValueError, cm_dossier.HookupEntry,
                  entry_key='testing:key')
    pytest.raises(ValueError, cm_dossier.HookupEntry, sysdef=sysdef)

def test_sysdef(sys_handle, mcsession):
    sysdef = cm_sysdef.Sysdef()
    active = cm_active.ActiveData(session=mcsession)
    active.load_parts(at_date=None)
    active.load_connections(at_date=None)
    part = Namespace(hpn='N700', hpn_rev='A', hptype='node')
    part.connections = Namespace(input_ports=['loc0', '@mars'],
                                 output_ports=[])
    hl = sysdef.handle_redirect_part_types(part, active)
    assert len(hl), 4
    part.hpn = 'doorknob'
    part.part_type = 'node'
    sysdef.setup(part, pol='all')
    assert 'E<loc0' in sysdef.ppkeys
    sysdef.setup(part, pol='e')
    assert 'E<loc0' in sysdef.ppkeys
    part.connections.input_ports = ['@loc1', 'top', 'bottom']
    sysdef.setup(part, pol='e')
    assert 'E<loc0' in sysdef.ppkeys
    rg = Namespace(direction='up', part='apart', rev='A', port='e1', pol='e')
    op = [Namespace(upstream_part='apart', upstream_output_port='eb',
                    downstream_part='bpart', downstream_input_port='e1')]
    op.append(Namespace(upstream_part='aprt', upstream_output_port='eb',
                        downstream_part='bprt', downstream_input_port='ea'))
    rg.port = 'e4'
    A = Namespace(hpn='apart', part_type='front-end')
    B = [Namespace(hpn='apart', part_type='cable-rfof'),
         Namespace(hpn='bpart', part_type='cable-rfof')]
    rg.port = 'rug'
    op[0].downstream_input_port = 'rug'
    pytest.raises(ValueError, sysdef.find_hookup_type, 'dull_knife',
                  'parts_not')
    part = Namespace(part_type='nope')
    pytest.raises(ValueError, sysdef.setup, part, pol='nope',
                  hookup_type='parts_hera')
    curr = active.set_times(cm_utils.get_astropytime('2017-07-03'))
    assert int(curr) == 1183075218
    active.load_apriori('now')
    assert active.apriori['HH700:A'].status == 'not_connected'
    ap = cm_partconnect.AprioriAntenna()
    ap.antenna = 'HH700'
    ap.start_gpstime = int(cm_utils.get_astropytime('2019-01-01').gps)
    ap.status = 'duplicate'
    mcsession.add(ap)
    mcsession.commit()
    pytest.raises(ValueError, active.load_apriori, 'now')

def test_hookup_cache_file_info(sys_handle, mcsession):
    hookup = cm_hookup.Hookup(session=mcsession)
    cfi = hookup.hookup_cache_file_info()
    assert 'json does not exist' in cfi

def test_correlator_info(sys_handle):
    corr_dict = sys_handle.get_cminfo_correlator(hookup_type='parts_hera')
    ant_names = corr_dict['antenna_names']
    assert len(ant_names) == 12

    corr_inputs = corr_dict['correlator_inputs']

    stn_types = corr_dict['station_types']

    index = np.where(np.array(ant_names) == 'HH703')[0]
    assert len(index) == 1
    index = index[0]

    assert stn_types[index] == 'herahexw'
    assert corr_inputs[index] == ('e10>SNPA000700', 'n8>SNPA000700')

    assert ([int(name.split('HH')[1]) for name in ant_names]
            == corr_dict['antenna_numbers'])

    assert set(corr_dict['antenna_numbers']) == set(range(701, 713))

    assert corr_dict['cm_version'] is not None

    # cm_version should be the same as the git hash of m&c for the test data
    mc_dir = os.path.dirname(os.path.realpath(__file__))
    mc_git_hash = subprocess.check_output(['git', '-C', mc_dir, 'rev-parse',
                                           'HEAD'],
                                          stderr=subprocess.STDOUT).strip()

    # In Python 3, we sometimes get Unicode, sometimes bytes
    if isinstance(mc_git_hash, six.binary_type):
        mc_git_hash = utils.bytes_to_str(mc_git_hash)

    assert corr_dict['cm_version'] == mc_git_hash

    expected_keys = ['antenna_numbers', 'antenna_names', 'station_types',
                     'correlator_inputs', 'antenna_utm_datum_vals', 'epoch',
                     'antenna_utm_tiles', 'antenna_utm_eastings',
                     'antenna_utm_northings', 'antenna_positions',
                     'cm_version', 'cofa_lat', 'cofa_lon', 'cofa_alt']
    assert set(corr_dict.keys()) == set(expected_keys)

    cofa = sys_handle.cofa()[0]
    assert cofa.lat == corr_dict['cofa_lat']
    assert cofa.lon == corr_dict['cofa_lon']
    assert cofa.elevation == corr_dict['cofa_alt']

def test_get_pam_from_hookup(sys_handle, mcsession):
    hookup = cm_hookup.Hookup(mcsession)
    hud = hookup.get_hookup(['HH701'], at_date='2019-07-03', exact_match=True,
                            hookup_type='parts_hera')
    pams = hud[list(hud.keys())[0]].get_part_from_type(
        'post-amp', include_ports=True, include_revs=True)
    assert len(pams) == 2
    # the actual pam number (the thing written on the case)
    assert pams['E<ground'] == 'e>PAM701:A<e'

def test_get_pam_info(sys_handle, mcsession):
    sys_h = cm_sysutils.Handling(mcsession)
    pams = sys_h.get_part_at_station_from_type(
        'HH701', '2019-07-03', 'post-amp', include_ports=False,
        hookup_type='parts_hera')
    assert len(pams) == 1
    # the actual pam number (the thing written on the case)
    assert pams['HH701:A']['E<ground'] == 'PAM701'

def test_apriori_antenna(sys_handle, mcsession, capsys):
    cm_partconnect.update_apriori_antenna('HH701', 'needs_checking',
                                          '1214382618', session=mcsession)
    d = sys_handle.get_apriori_antenna_status_for_rtp('needs_checking')
    assert d == 'HH701'
    g = sys_handle.get_apriori_antenna_status_set()
    assert g['needs_checking'][0] == 'HH701'
    pytest.raises(ValueError, cm_partconnect.update_apriori_antenna,
                  'HH3', 'not_one', '1214482618', session=mcsession)
    g = sys_handle.get_apriori_status_for_antenna('HH701')
    assert g == 'needs_checking'
    cm_partconnect.update_apriori_antenna(
        'HH700', 'needs_checking', '1214482818', '1214483000',
        session=mcsession)
    pytest.raises(ValueError, cm_partconnect.update_apriori_antenna, 'HH700',
                  'needs_checking', '1214482838', session=mcsession)
    print(cm_partconnect.AprioriAntenna())
    captured = capsys.readouterr()
    assert '<None:' in captured.out.strip()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Cassandra using version of BaseServer."""

import os
import simplejson
import zlib
import pprint
import urllib
import traceback
import logging
import time
from uuid import uuid4

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet import reactor
from telephus.protocol import ManagedCassandraClientFactory
from telephus.client import CassandraClient
from .base import BaseServer
from ..pagegetter import PageGetter

from txredisapi import RedisShardingConnection
from mixins.jobgetter import JobGetterMixin

PP = pprint.PrettyPrinter(indent=4)

logger = logging.getLogger(__name__)

class CassandraServer(BaseServer, JobGetterMixin):

    redis_client = None

    def __init__(self, config):
        super(CassandraServer, self).__init__(config)
        self.cassandra_cf_content = config["cassandra_cf_content"]
        self.cassandra_cf_temp_content = config["cassandra_cf_temp_content"]
        # Cassandra Clients & Factorys
        factory = ManagedCassandraClientFactory(config["cassandra_keyspace"])
        self.cassandra_client = CassandraClient(factory)
        reactor.connectTCP(
            config["cassandra_server"],
            config.get("cassandra_port", 9160),
            factory)
        # Negative Cache
        self.disable_negative_cache = config.get("disable_negative_cache", False)
        # Redis
        self.redis_hosts = config["redis_hosts"]
        # deltas
        self.delta_log_enabled = config.get('delta_log_enabled', False)
        self.delta_log_path = config.get('delta_log_path', '/tmp/deltas/')
        self.delta_enabled = config.get('delta_enabled', False)
        # create the log path if required & enabled
        if self.delta_enabled and self.delta_log_enabled and not os.path.exists(self.delta_log_path):
            os.makedirs(self.delta_log_path)
        self.cassandra_cf_delta = config.get('cassandra_cf_delta', None)
        self.cassandra_cf_delta_user = config.get('cassandra_cf_delta_user', None)
        # sanity check config;  if cfs aren't set, turn deltas off
        if not all([self.cassandra_cf_delta, self.cassandra_cf_delta_user]):
            logger.warn('Disabling cassandra deltas; both cf_delta and'
                ' cf_delta_user must be set in the config.')
            self.delta_enabled = False
        self.setupJobGetter(config)

    def start(self):
        start_deferred = super(CassandraServer, self).start()
        start_deferred.addCallback(self._cassandraStart)
        return start_deferred

    @inlineCallbacks
    def _cassandraStart(self, started=False):
        logger.debug("Starting Cassandra components.")
        try:
            self.redis_client = yield RedisShardingConnection(self.redis_hosts)
        except Exception, e:
            logger.error("Could not connect to Redis: %s" % e)
            self.shutdown()
            raise Exception("Could not connect to Redis.")
        self.pg = PageGetter(
            self.cassandra_client,
            redis_client=self.redis_client,
            disable_negative_cache=self.disable_negative_cache,
            rq=self.rq)
        returnValue(True)

    def shutdown(self):
        # Shutdown things here.
        return super(CassandraServer, self).shutdown()

    def logDelta(self, uuid, old, new, delta):
        """Log a delta in a way we can examine later or incorporate into our
        unit testing corpus."""
        if not self.delta_log_enabled: return
        basepath = os.path.join(self.delta_log_path, uuid)
        with open(basepath + '.old.js', 'w') as f:
            f.write(simplejson.dumps(old, indent=2))
        with open(basepath + '.new.js', 'w') as f:
            f.write(simplejson.dumps(new, indent=2))
        with open(basepath + '.res.js', 'w') as f:
            f.write(simplejson.dumps(delta, indent=2))

    @inlineCallbacks
    def executeJob(self, job):
        def iterate_deltas(deltas):
            if isinstance(deltas, dict):
                for key,value in delta.iteritems():
                    yield key, value
            else:
                for data in deltas:
                    ts = time.time()
                    yield '%0.2f:%s' % (ts, uuid4().hex), data

        user_id = job.user_account["user_id"]
        new_data = yield super(CassandraServer, self).executeJob(job)
        if new_data is None:
            return
        ts = time.time()
        if self.delta_enabled:
            delta_func = self.functions[job.function_name]["delta"]
            if delta_func is not None:
                service = job.subservice.split('/')[0]
                old_data = yield self.getData(user_id, job.uuid)
                # TODO: make sure we check that old_data exists
                delta = delta_func(new_data, old_data)
                self.logDelta(job.uuid, old_data, new_data, delta)
                logger.debug("Got delta: %s ..." % str(delta)[:1000])
                for delta_id, data in iterate_deltas(delta):
                    category = self.functions[job.function_name]['category']
                    service = job.subservice.split('/')[0]
                    user_column = '%s:%s:%s' % (delta_id, category, service, job.subservice)
                    logger.info("Inserting delta id %s, user column: %s"  % (str(delta_id), user_column))
                    mapping = {
                        'data': zlib.compress(simplejson.dumps(data)),
                        'old_data': zlib.compress(simplejson.dumps(old_data)),
                        'user_id': str(user_id),
                        'category': category,
                        'service': service,
                        'subservice': job.subservice,
                        'uuid': job.uuid,
                    }
                    yield self.cassandra_client.batch_insert(
                        key=str(delta_id),
                        column_family=self.cassandra_cf_delta,
                        mapping=mapping)
                    yield self.cassandra_client.insert(
                        key=str(user_id),
                        column_family=self.cassandra_cf_delta_user,
                        column=user_column,
                        value=''
                    )

        yield self.cassandra_client.insert(
            str(user_id),
            self.cassandra_cf_content,
            zlib.compress(simplejson.dumps(new_data)),
            column=job.uuid)
        returnValue(new_data)

    @inlineCallbacks
    def getData(self, user_id, uuid):
        data = yield self.cassandra_client.get(
            key=str(user_id),
            column_family=self.cassandra_cf_content,
            column=uuid)
        returnValue(simplejson.loads(zlib.decompress(data.column.value)))

    @inlineCallbacks
    def deleteReservation(self, uuid):
        """Delete a reservation by uuid."""
        # FIXME: this function is unnecessarily coupled to the job object;
        # only a uuid is needed to delete a reservation
        logger.info('Deleting UUID from spider_service table: %s' % uuid)
        yield self.mysql.runQuery('DELETE FROM spider_service WHERE uuid=%s', uuid)
        url = 'http://%s:%s/function/schedulerserver/remoteremovefromheap?%s' % (
            self.scheduler_server,
            self.scheduler_server_port,
            urllib.urlencode({'uuid': uuid}))
        logger.info('Sending UUID to scheduler to be dequeued: %s' % url)
        try:
            yield self.rq.getPage(url=url)
        except Exception, e:
            tb = traceback.format_exc()
            logger.error("failed to deque job %s on scheduler (url was: %s):\n%s" % (
                uuid, url, tb))
            # TODO: punt here?
        logger.info('Deleting UUID from Cassandra: %s' % uuid)
        yield self.cassandra_client.remove(
            uuid,
            self.cassandra_cf_content)
        returnValue({'success':True})

<commit_msg>missing a %s<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Cassandra using version of BaseServer."""

import os
import simplejson
import zlib
import pprint
import urllib
import traceback
import logging
import time
from uuid import uuid4

from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet import reactor
from telephus.protocol import ManagedCassandraClientFactory
from telephus.client import CassandraClient
from .base import BaseServer
from ..pagegetter import PageGetter

from txredisapi import RedisShardingConnection
from mixins.jobgetter import JobGetterMixin

PP = pprint.PrettyPrinter(indent=4)

logger = logging.getLogger(__name__)

class CassandraServer(BaseServer, JobGetterMixin):

    redis_client = None

    def __init__(self, config):
        super(CassandraServer, self).__init__(config)
        self.cassandra_cf_content = config["cassandra_cf_content"]
        self.cassandra_cf_temp_content = config["cassandra_cf_temp_content"]
        # Cassandra Clients & Factorys
        factory = ManagedCassandraClientFactory(config["cassandra_keyspace"])
        self.cassandra_client = CassandraClient(factory)
        reactor.connectTCP(
            config["cassandra_server"],
            config.get("cassandra_port", 9160),
            factory)
        # Negative Cache
        self.disable_negative_cache = config.get("disable_negative_cache", False)
        # Redis
        self.redis_hosts = config["redis_hosts"]
        # deltas
        self.delta_log_enabled = config.get('delta_log_enabled', False)
        self.delta_log_path = config.get('delta_log_path', '/tmp/deltas/')
        self.delta_enabled = config.get('delta_enabled', False)
        # create the log path if required & enabled
        if self.delta_enabled and self.delta_log_enabled and not os.path.exists(self.delta_log_path):
            os.makedirs(self.delta_log_path)
        self.cassandra_cf_delta = config.get('cassandra_cf_delta', None)
        self.cassandra_cf_delta_user = config.get('cassandra_cf_delta_user', None)
        # sanity check config;  if cfs aren't set, turn deltas off
        if not all([self.cassandra_cf_delta, self.cassandra_cf_delta_user]):
            logger.warn('Disabling cassandra deltas; both cf_delta and'
                ' cf_delta_user must be set in the config.')
            self.delta_enabled = False
        self.setupJobGetter(config)

    def start(self):
        start_deferred = super(CassandraServer, self).start()
        start_deferred.addCallback(self._cassandraStart)
        return start_deferred

    @inlineCallbacks
    def _cassandraStart(self, started=False):
        logger.debug("Starting Cassandra components.")
        try:
            self.redis_client = yield RedisShardingConnection(self.redis_hosts)
        except Exception, e:
            logger.error("Could not connect to Redis: %s" % e)
            self.shutdown()
            raise Exception("Could not connect to Redis.")
        self.pg = PageGetter(
            self.cassandra_client,
            redis_client=self.redis_client,
            disable_negative_cache=self.disable_negative_cache,
            rq=self.rq)
        returnValue(True)

    def shutdown(self):
        # Shutdown things here.
        return super(CassandraServer, self).shutdown()

    def logDelta(self, uuid, old, new, delta):
        """Log a delta in a way we can examine later or incorporate into our
        unit testing corpus."""
        if not self.delta_log_enabled: return
        basepath = os.path.join(self.delta_log_path, uuid)
        with open(basepath + '.old.js', 'w') as f:
            f.write(simplejson.dumps(old, indent=2))
        with open(basepath + '.new.js', 'w') as f:
            f.write(simplejson.dumps(new, indent=2))
        with open(basepath + '.res.js', 'w') as f:
            f.write(simplejson.dumps(delta, indent=2))

    @inlineCallbacks
    def executeJob(self, job):
        def iterate_deltas(deltas):
            if isinstance(deltas, dict):
                for key,value in delta.iteritems():
                    yield key, value
            else:
                for data in deltas:
                    ts = time.time()
                    delta_uuid = '%0.2f:%s' % (ts, uuid4().hex)
                    yield delta_uuid, data

        user_id = job.user_account["user_id"]
        new_data = yield super(CassandraServer, self).executeJob(job)
        if new_data is None:
            return
        ts = time.time()
        if self.delta_enabled:
            delta_func = self.functions[job.function_name]["delta"]
            if delta_func is not None:
                service = job.subservice.split('/')[0]
                old_data = yield self.getData(user_id, job.uuid)
                # TODO: make sure we check that old_data exists
                delta = delta_func(new_data, old_data)
                self.logDelta(job.uuid, old_data, new_data, delta)
                logger.debug("Got delta: %s ..." % str(delta)[:1000])
                for delta_id, data in iterate_deltas(delta):
                    category = self.functions[job.function_name]['category']
                    service = job.subservice.split('/')[0]
                    user_column = '%s:%s:%s:%s' % (delta_id, category, service, job.subservice)
                    logger.info("Inserting delta id %s, user column: %s"  % (str(delta_id), user_column))
                    mapping = {
                        'data': zlib.compress(simplejson.dumps(data)),
                        'old_data': zlib.compress(simplejson.dumps(old_data)),
                        'user_id': str(user_id),
                        'category': category,
                        'service': service,
                        'subservice': job.subservice,
                        'uuid': job.uuid,
                    }
                    yield self.cassandra_client.batch_insert(
                        key=str(delta_id),
                        column_family=self.cassandra_cf_delta,
                        mapping=mapping)
                    yield self.cassandra_client.insert(
                        key=str(user_id),
                        column_family=self.cassandra_cf_delta_user,
                        column=user_column,
                        value=''
                    )

        yield self.cassandra_client.insert(
            str(user_id),
            self.cassandra_cf_content,
            zlib.compress(simplejson.dumps(new_data)),
            column=job.uuid)
        returnValue(new_data)

    @inlineCallbacks
    def getData(self, user_id, uuid):
        data = yield self.cassandra_client.get(
            key=str(user_id),
            column_family=self.cassandra_cf_content,
            column=uuid)
        returnValue(simplejson.loads(zlib.decompress(data.column.value)))

    @inlineCallbacks
    def deleteReservation(self, uuid):
        """Delete a reservation by uuid."""
        # FIXME: this function is unnecessarily coupled to the job object;
        # only a uuid is needed to delete a reservation
        logger.info('Deleting UUID from spider_service table: %s' % uuid)
        yield self.mysql.runQuery('DELETE FROM spider_service WHERE uuid=%s', uuid)
        url = 'http://%s:%s/function/schedulerserver/remoteremovefromheap?%s' % (
            self.scheduler_server,
            self.scheduler_server_port,
            urllib.urlencode({'uuid': uuid}))
        logger.info('Sending UUID to scheduler to be dequeued: %s' % url)
        try:
            yield self.rq.getPage(url=url)
        except Exception, e:
            tb = traceback.format_exc()
            logger.error("failed to deque job %s on scheduler (url was: %s):\n%s" % (
                uuid, url, tb))
            # TODO: punt here?
        logger.info('Deleting UUID from Cassandra: %s' % uuid)
        yield self.cassandra_client.remove(
            uuid,
            self.cassandra_cf_content)
        returnValue({'success':True})

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
VERSION = '1.0'
<commit_msg>start version 1.1-devel<commit_after># -*- coding: utf-8 -*-
VERSION = '1.1-devel'
<|endoftext|>


<commit_before>from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rfile import create_popen_file

from hippy.objects.nullobject import w_Null
from hippy.builtin import wrap, Optional, ExitFunctionWithError

@wrap(['interp', str])
def escapeshellarg(interp, arg):
    s = StringBuilder(len(arg) + 2)
    s.append("'")
    for c in arg:
        if c == "'":
            s.append("'")
            s.append("\\")
            s.append("'")
            s.append("'")
        else:
            s.append(c)
    s.append("'")
    return interp.space.wrap(s.build())

@wrap(['interp', str, Optional('reference'), Optional('reference')],
      error=False, name='exec')
def exec_(interp, cmd, r_output=None, r_return_var=None):
    space = interp.space
    if not cmd:
        raise ExitFunctionWithError('Cannot execute a blank command')
    if r_output is not None:
        if not space.is_array(r_output.deref_temp()):
            r_output.store(space.new_array_from_list([]))
        w_output = r_output.deref_unique()
    else:
        w_output = None
    try:
        pfile = create_popen_file(cmd, 'r')
    except OSError:
        raise ExitFunctionWithError('Unable to fork [%s]' % cmd, w_Null)
    last_line = ''
    while True:
        line = pfile.readline()
        if not line:
            break
        last_line = line.rstrip()
        if w_output:
            w_output.appenditem_inplace(space, space.newstr(last_line))
    exitcode = pfile.close()
    if r_return_var is not None:
        r_return_var.store(space.wrap(exitcode))
    return space.newstr(last_line)

@wrap(['interp', str, Optional('reference')], error=False)
def passthru(interp, cmd, r_return_var=None):
    space = interp.space
    if not cmd:
        raise ExitFunctionWithError('Cannot execute a blank command')
    try:
        pfile = create_popen_file(cmd, 'r')
    except OSError:
        raise ExitFunctionWithError('Unable to fork [%s]' % cmd, w_Null)
    last_line = ''
    while True:
        line = pfile.read()
        if not line:
            break
        interp.writestr(line, buffer=False)
    exitcode = pfile.close()
    if r_return_var is not None:
        r_return_var.store(space.wrap(exitcode))
    return space.newstr(last_line)

@wrap(['interp', str])
def shell_exec(interp, cmd):
    try:
        r_pfile = create_popen_file(cmd, 'r')
    except OSError:
        interp.warn("Unable to execute '%s'" % cmd)
        return w_Null
    res = r_pfile.read(-1)
    if res:
        return interp.space.wrap(res)
    else:
        return w_Null
<commit_msg>Implement system()<commit_after>from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rfile import create_popen_file

from hippy.objects.nullobject import w_Null
from hippy.builtin import wrap, Optional, ExitFunctionWithError

@wrap(['interp', str])
def escapeshellarg(interp, arg):
    s = StringBuilder(len(arg) + 2)
    s.append("'")
    for c in arg:
        if c == "'":
            s.append("'")
            s.append("\\")
            s.append("'")
            s.append("'")
        else:
            s.append(c)
    s.append("'")
    return interp.space.wrap(s.build())

@wrap(['interp', str, Optional('reference'), Optional('reference')],
      error=False, name='exec')
def exec_(interp, cmd, r_output=None, r_return_var=None):
    space = interp.space
    if not cmd:
        raise ExitFunctionWithError('Cannot execute a blank command')
    if r_output is not None:
        if not space.is_array(r_output.deref_temp()):
            r_output.store(space.new_array_from_list([]))
        w_output = r_output.deref_unique()
    else:
        w_output = None
    try:
        pfile = create_popen_file(cmd, 'r')
    except OSError:
        raise ExitFunctionWithError('Unable to fork [%s]' % cmd, w_Null)
    last_line = ''
    while True:
        line = pfile.readline()
        if not line:
            break
        last_line = line.rstrip()
        if w_output:
            w_output.appenditem_inplace(space, space.newstr(last_line))
    exitcode = pfile.close()
    if r_return_var is not None:
        r_return_var.store(space.wrap(exitcode))
    return space.newstr(last_line)

@wrap(['interp', str, Optional('reference')], error=False)
def passthru(interp, cmd, r_return_var=None):
    space = interp.space
    if not cmd:
        raise ExitFunctionWithError('Cannot execute a blank command')
    try:
        pfile = create_popen_file(cmd, 'r')
    except OSError:
        raise ExitFunctionWithError('Unable to fork [%s]' % cmd, w_Null)
    last_line = ''
    while True:
        line = pfile.read()
        if not line:
            break
        interp.writestr(line, buffer=False)
    exitcode = pfile.close()
    if r_return_var is not None:
        r_return_var.store(space.wrap(exitcode))
    return space.newstr(last_line)

@wrap(['interp', str, Optional('reference')], error=False)
def system(interp, cmd, r_return_var=None):
    space = interp.space
    if not cmd:
        raise ExitFunctionWithError('Cannot execute a blank command')
    try:
        pfile = create_popen_file(cmd, 'r')
    except OSError:
        raise ExitFunctionWithError('Unable to fork [%s]' % cmd, w_Null)
    last_line = ''
    while True:
        line = pfile.readline()
        if not line:
            break
        interp.writestr(line, buffer=True)
        last_line = line
    exitcode = pfile.close()
    if r_return_var is not None:
        r_return_var.store(space.wrap(exitcode))
    return space.newstr(last_line.rstrip())

@wrap(['interp', str])
def shell_exec(interp, cmd):
    try:
        r_pfile = create_popen_file(cmd, 'r')
    except OSError:
        interp.warn("Unable to execute '%s'" % cmd)
        return w_Null
    res = r_pfile.read(-1)
    if res:
        return interp.space.wrap(res)
    else:
        return w_Null
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import operator

from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError
from django.core.cache import cache
from django.utils import timezone
from django.conf import settings
from django.template.defaultfilters import slugify
from django.contrib.sites.models import Site

from polymorphic import PolymorphicModel
from polymorphic.showfields import ShowFieldContent

from opps.core.cache import _cache_key
from opps.core.models import Publishable, Slugged, Channeling, Imaged
from opps.core.tags.models import Tagged
from opps.db.models.fields import JSONField
from opps.boxes.models import BaseBox

from .managers import ContainerManager
from .signals import shorturl_generate, delete_container
from .tasks import check_mirror_channel, check_mirror_site

class Container(PolymorphicModel, ShowFieldContent, Publishable, Slugged,
                Channeling, Imaged, Tagged):
    title = models.CharField(_(u"Title"), max_length=140, db_index=True)
    hat = models.CharField(
        _(u"Hat"),
        max_length=140,
        null=True, blank=True,
    )
    short_url = models.URLField(
        _(u"Short URL"),
        null=True, blank=True,
    )
    child_class = models.CharField(
        _(u'Child class'),
        max_length=30,
        null=True, blank=True,
        db_index=True
    )
    child_module = models.CharField(
        _(u'Child module'),
        max_length=120,
        null=True, blank=True,
        db_index=True
    )
    child_app_label = models.CharField(
        _(u'Child app label'),
        max_length=30,
        null=True, blank=True,
        db_index=True
    )
    show_on_root_channel = models.BooleanField(
        _(u"Show on root channel?"),
        default=True
    )
    source = models.CharField(
        _('Source'),
        null=True, blank=True,
        max_length=255)
    json = JSONField(_(u"Customized"),
                     null=True, blank=True)

    objects = ContainerManager()

    def __unicode__(self):
        return u"{0}".format(self.get_absolute_url())

    def __repr__(self):
        val = self.__unicode__()
        if isinstance(val, str):
            return val
        elif not isinstance(val, unicode):
            val = unicode(val)
        return val.encode('utf8')

    class Meta:
        ordering = ['-date_available']
        verbose_name = _(u'Container')
        verbose_name_plural = _(u'Containers')
        unique_together = ("site", "channel", "slug")

    def save(self, *args, **kwargs):
        if not self.channel_name:
            self.channel_name = self.channel.name
        if not self.channel_long_slug:
            self.channel_long_slug = self.channel.long_slug
        self.child_class = self.__class__.__name__
        self.child_module = self.__class__.__module__
        self.child_app_label = self._meta.app_label
        if self.slug == u"":
            self.slug = slugify(self.title)

        models.signals.post_save.connect(shorturl_generate,
                                         sender=self.__class__)
        super(Container, self).save(*args, **kwargs)
        if settings.OPPS_MIRROR_CHANNEL and (
            self.mirror_channel or self.mirror_site)\
                and self.child_class != u"Mirror":
            check_mirror_channel.delay(
                container=self, Mirror=Mirror)
            check_mirror_site.delay(
                container=self, Mirror=Mirror)

    def get_absolute_url(self):
        if self.channel.homepage:
            return u"/{0}.html".format(self.slug)
        return u"/{0}/{1}.html".format(self.channel_long_slug, self.slug)

    @classmethod
    def get_children_models(cls):
        children = models.get_models()
        return [model for model in children
                if (model is not None and
                    issubclass(model, cls) and
                    model is not cls)]

    def get_thumb(self):
        return self.main_image

    @property
    def search_category(self):
        """for use in search result"""
        return _(self.child_class)

    def get_http_absolute_url(self):
        return u"http://{0}{1}".format(self.site_domain,
                                       self.get_absolute_url())
    get_http_absolute_url.short_description = _(u'Get HTTP Absolute URL')

    def recommendation(self, child_class=False, query_slice=[None, 10]):

        if not child_class:
            child_class = self.child_class

        now = timezone.now()
        start = now - timezone.timedelta(
            days=settings.OPPS_RECOMMENDATION_RANGE_DAYS
        )

        cachekey = _cache_key(
            u'{0}-recommendation'.format(self.__class__.__name__),
            self.__class__, self.site_domain,
            u"{0}-{1}-{2}".format(child_class, self.channel_long_slug,
                                  self.slug))
        getcache = cache.get(cachekey)
        if getcache:
            return getcache

        containers = Container.objects.filter(
            site_domain=self.site_domain,
            child_class=child_class,
            channel_long_slug=self.channel_long_slug,
            date_available__range=(start, now),
            published=True
        ).exclude(pk=self.pk)

        tag_list = []
        if self.tags:
            tag_list = [t for t in self.tags.split(',')[:3]]
            containers = containers.filter(
                reduce(operator.or_, (Q(tags__contains=tag) for tag in
                                      tag_list)),
            )
        containers = containers.distinct().order_by(
            '-date_available')[slice(*query_slice)]

        _list = [a for a in containers]

        cache.set(cachekey, _list, 3600)
        return _list

    def inbox(self, containerbox=None):
        obj = ContainerBoxContainers.objects
        if containerbox:
            return obj.get(container=self.id,
                           containerbox__slug=containerbox)
        return obj.filter(container=self.id)

    def custom_fields(self):
        import json
        if not self.json:
            return
        return json.loads(self.json)

class ContainerImage(models.Model):
    container = models.ForeignKey(
        'containers.Container',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container'),
    )
    order = models.PositiveIntegerField(_(u'Order'), default=0)
    image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Image'),
    )

    caption = models.CharField(
        _(u"Caption"),
        max_length=255,
        blank=True,
        null=True
    )

    class Meta:
        verbose_name = _(u'Container image')
        verbose_name_plural = _(u'Container images')
        ordering = ('order',)

    def __unicode__(self):
        if self.image:
            return u"{0}".format(self.image.title)
        return u'Id:{0} - Order:{1}'.format(self.id, self.order)

class ContainerBox(BaseBox):

    title = models.CharField(
        _(u"Title"),
        null=True,
        blank=True,
        max_length=140,
    )
    title_url = models.CharField(
        _(u"Title Link"),
        null=True,
        blank=True,
        max_length=250,
    )
    main_image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Main Image'),
    )
    main_image_caption = models.CharField(
        _(u"Main Image Caption"),
        max_length=4000,
        blank=True,
        null=True,
        help_text=_(u'Maximum characters 4000'),
    )
    containers = models.ManyToManyField(
        'containers.Container',
        null=True, blank=True,
        verbose_name=_(u'Container'),
        related_name='containerbox_containers',
        through='containers.ContainerBoxContainers'
    )
    queryset = models.ForeignKey(
        'boxes.QuerySet',
        null=True, blank=True,
        related_name='containerbox_querysets',
        verbose_name=_(u'Query Set')
    )

    content_group = models.CharField(
        _(u"Content Group"),
        max_length=255,
        default='default',
        help_text=_(u"Boxes in the same group do not allow repetitions")
    )

    class Meta:
        verbose_name = _(u'Container box')
        verbose_name_plural = _(u'Containers boxes')

    def __init__(self, *args, **kwargs):
        """
        to avoid re-execution of methods
        its results are cached in a local storage
        per instance cache
        """
        super(ContainerBox, self).__init__(*args, **kwargs)
        if not hasattr(self, 'local_cache'):
            self.local_cache = {}

    @property
    def has_content(self):
        # TODO: should check start/end available_dates
        if self.containerboxcontainers_set.exists():
            return True
        qs = self.get_queryset(use_local_cache=False)
        if qs and qs.exists():
            return True

    def ordered_containers(self, field='order', exclude_ids=None):
        cached = self.local_cache.get('ordered_containers', None)
        if cached:
            return cached

        exclude_ids = exclude_ids or []

        now = timezone.now()
        fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        if not fallback:
            qs = self.containers.filter(
                models.Q(containerboxcontainers__date_end__gte=now) |
                models.Q(containerboxcontainers__date_end__isnull=True),
                published=True,
                date_available__lte=now,
                containerboxcontainers__date_available__lte=now
            ).exclude(
                id__in=exclude_ids
            ).order_by('containerboxcontainers__order').distinct()
        else:

            site_master = cache.get("site_master")

            if not site_master:
                site_master = Site.objects.order_by('id')[0]
                cache.set('site_master', site_master, 3600)

            boxes = [self]

            if fallback and site_master.pk != self.site_id:
                try:
                    master_box = self.__class__.objects.get(
                        site=site_master, slug=self.slug
                    )
                    boxes.insert(0, master_box)
                except self.__class__.DoesNotExist:
                    pass

            qs = ContainerBoxContainers.objects.filter(
                models.Q(date_end__gte=now) |
                models.Q(date_end__isnull=True),
                models.Q(container__published=True,
                         container__date_available__lte=now) |
                models.Q(container__isnull=True),
                containerbox__in=boxes,
                date_available__lte=now,
            ).exclude(
                container__id__in=exclude_ids
            ).order_by('-containerbox__site__id', 'order').distinct()

        self.local_cache['ordered_containers'] = qs
        return qs

    def ordered_box_containers(self, exclude_ids=None):

        fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        if fallback:
            return self.ordered_containers(exclude_ids=exclude_ids)

        cached = self.local_cache.get('ordered_box_containers')
        if cached:
            return cached

        exclude_ids = exclude_ids or []

        now = timezone.now()
        qs = self.containerboxcontainers_set.filter(
            models.Q(date_end__gte=now) |
            models.Q(date_end__isnull=True),
            date_available__lte=now
        ).exclude(
            container__id__in=exclude_ids
        ).order_by('order').distinct()

        self.local_cache['ordered_box_containers'] = qs
        return qs

    def get_containers_queryset(self):
        if self.queryset:
            return self.queryset.get_queryset()
        else:
            return self.ordered_containers()

    def get_queryset(self, exclude_ids=None, use_local_cache=True):
        cached = self.local_cache.get('get_queryset')
        if use_local_cache and cached:
            return cached

        queryset = self.queryset and self.queryset.get_queryset(
            content_group=self.content_group,
            exclude_ids=exclude_ids,
            use_local_cache=use_local_cache
        )
        if use_local_cache:
            self.local_cache['get_queryset'] = queryset
        return queryset

class ContainerBoxContainers(models.Model):
    URL_TARGET_CHOICES = (
        ('_blank', _(u'Load in a new window')),
        ('_self', _(u'Load in the same frame as it was clicked')),
        ('_parent', _(u'Load in the parent frameset')),
        ('_top', _(u'Load in the full body of the window'))
    )

    containerbox = models.ForeignKey(
        'containers.ContainerBox',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container Box'),
    )
    container = models.ForeignKey(
        'containers.Container',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container'),
    )

    order = models.PositiveIntegerField(_(u'Order'), default=0)
    aggregate = models.BooleanField(_(u'Aggregate container'), default=False)
    highlight = models.BooleanField(_(u'Highlight container'), default=False)
    date_available = models.DateTimeField(_(u"Date available"),
                                          default=timezone.now, null=True)
    date_end = models.DateTimeField(_(u"End date"), null=True, blank=True)

    title = models.CharField(_(u"Title"), max_length=140,
                             null=True, blank=True)

    hat = models.CharField(
        _(u"Hat"),
        max_length=140,
        null=True, blank=True,
    )

    main_image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Main Image'),
    )
    main_image_caption = models.CharField(
        _(u"Main Image Caption"),
        max_length=4000,
        blank=True,
        null=True,
        help_text=_(u'Maximum characters 4000'),
    )

    url = models.CharField(_(u"URL"), max_length=255, null=True, blank=True)
    url_target = models.CharField(_(u"URL Target"), max_length=255,
                                  choices=URL_TARGET_CHOICES, default="_self",
                                  null=True, blank=True)

    class Meta:
        ordering = ('order',)
        verbose_name = _(u'Article box articles')
        verbose_name_plural = _(u'Article boxes articles')
        ordering = ('order', 'aggregate',)

    def __unicode__(self):
        if self.container:
            return u"{0}-{1}".format(self.containerbox.slug,
                                     self.container.slug)
        else:
            return u"{0}".format(self.containerbox.slug)

    def clean(self):
        if not self.container and not self.url:
            raise ValidationError(_(u'Please select a Container or insert a '
                                    u'URL!'))

        if self.container and not self.container.published:
            raise ValidationError(_(u'Article not published!'))

class ContainerRelated(models.Model):
    container = models.ForeignKey(
        'containers.Container',
        verbose_name=_(u'Post'),
        null=True,
        blank=True,
        related_name='containerrelated_container',
        on_delete=models.SET_NULL
    )
    related = models.ForeignKey(
        'containers.Container',
        verbose_name=_(u'Related Post'),
        null=True,
        blank=True,
        related_name='containerrelated_related',
        on_delete=models.SET_NULL
    )
    order = models.PositiveIntegerField(_(u'Order'), default=0)

    class Meta:
        verbose_name = _('Related content')
        verbose_name_plural = _('Related contents')
        ordering = ('order',)

    def __unicode__(self):
        return u"{0}->{1}".format(self.related.slug, self.container.slug)

class Mirror(Container):
    container = models.ForeignKey('containers.Container',
                                  related_name='containers_mirror',
                                  verbose_name=_(u'Container'))

    class Meta:
        verbose_name = _(u'Mirror')
        verbose_name_plural = _(u'Mirrors')

models.signals.post_delete.connect(delete_container, sender=Container)
<commit_msg>Fix related_name in ContainerRelated<commit_after># -*- coding: utf-8 -*-
import operator

from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError
from django.core.cache import cache
from django.utils import timezone
from django.conf import settings
from django.template.defaultfilters import slugify
from django.contrib.sites.models import Site

from polymorphic import PolymorphicModel
from polymorphic.showfields import ShowFieldContent

from opps.core.cache import _cache_key
from opps.core.models import Publishable, Slugged, Channeling, Imaged
from opps.core.tags.models import Tagged
from opps.db.models.fields import JSONField
from opps.boxes.models import BaseBox

from .managers import ContainerManager
from .signals import shorturl_generate, delete_container
from .tasks import check_mirror_channel, check_mirror_site

class Container(PolymorphicModel, ShowFieldContent, Publishable, Slugged,
                Channeling, Imaged, Tagged):
    title = models.CharField(_(u"Title"), max_length=140, db_index=True)
    hat = models.CharField(
        _(u"Hat"),
        max_length=140,
        null=True, blank=True,
    )
    short_url = models.URLField(
        _(u"Short URL"),
        null=True, blank=True,
    )
    child_class = models.CharField(
        _(u'Child class'),
        max_length=30,
        null=True, blank=True,
        db_index=True
    )
    child_module = models.CharField(
        _(u'Child module'),
        max_length=120,
        null=True, blank=True,
        db_index=True
    )
    child_app_label = models.CharField(
        _(u'Child app label'),
        max_length=30,
        null=True, blank=True,
        db_index=True
    )
    show_on_root_channel = models.BooleanField(
        _(u"Show on root channel?"),
        default=True
    )
    source = models.CharField(
        _('Source'),
        null=True, blank=True,
        max_length=255)
    json = JSONField(_(u"Customized"),
                     null=True, blank=True)

    objects = ContainerManager()

    def __unicode__(self):
        return u"{0}".format(self.get_absolute_url())

    def __repr__(self):
        val = self.__unicode__()
        if isinstance(val, str):
            return val
        elif not isinstance(val, unicode):
            val = unicode(val)
        return val.encode('utf8')

    class Meta:
        ordering = ['-date_available']
        verbose_name = _(u'Container')
        verbose_name_plural = _(u'Containers')
        unique_together = ("site", "channel", "slug")

    def save(self, *args, **kwargs):
        if not self.channel_name:
            self.channel_name = self.channel.name
        if not self.channel_long_slug:
            self.channel_long_slug = self.channel.long_slug
        self.child_class = self.__class__.__name__
        self.child_module = self.__class__.__module__
        self.child_app_label = self._meta.app_label
        if self.slug == u"":
            self.slug = slugify(self.title)

        models.signals.post_save.connect(shorturl_generate,
                                         sender=self.__class__)
        super(Container, self).save(*args, **kwargs)
        if settings.OPPS_MIRROR_CHANNEL and (
            self.mirror_channel or self.mirror_site)\
                and self.child_class != u"Mirror":
            check_mirror_channel.delay(
                container=self, Mirror=Mirror)
            check_mirror_site.delay(
                container=self, Mirror=Mirror)

    def get_absolute_url(self):
        if self.channel.homepage:
            return u"/{0}.html".format(self.slug)
        return u"/{0}/{1}.html".format(self.channel_long_slug, self.slug)

    @classmethod
    def get_children_models(cls):
        children = models.get_models()
        return [model for model in children
                if (model is not None and
                    issubclass(model, cls) and
                    model is not cls)]

    def get_thumb(self):
        return self.main_image

    @property
    def search_category(self):
        """for use in search result"""
        return _(self.child_class)

    def get_http_absolute_url(self):
        return u"http://{0}{1}".format(self.site_domain,
                                       self.get_absolute_url())
    get_http_absolute_url.short_description = _(u'Get HTTP Absolute URL')

    def recommendation(self, child_class=False, query_slice=[None, 10]):

        if not child_class:
            child_class = self.child_class

        now = timezone.now()
        start = now - timezone.timedelta(
            days=settings.OPPS_RECOMMENDATION_RANGE_DAYS
        )

        cachekey = _cache_key(
            u'{0}-recommendation'.format(self.__class__.__name__),
            self.__class__, self.site_domain,
            u"{0}-{1}-{2}".format(child_class, self.channel_long_slug,
                                  self.slug))
        getcache = cache.get(cachekey)
        if getcache:
            return getcache

        containers = Container.objects.filter(
            site_domain=self.site_domain,
            child_class=child_class,
            channel_long_slug=self.channel_long_slug,
            date_available__range=(start, now),
            published=True
        ).exclude(pk=self.pk)

        tag_list = []
        if self.tags:
            tag_list = [t for t in self.tags.split(',')[:3]]
            containers = containers.filter(
                reduce(operator.or_, (Q(tags__contains=tag) for tag in
                                      tag_list)),
            )
        containers = containers.distinct().order_by(
            '-date_available')[slice(*query_slice)]

        _list = [a for a in containers]

        cache.set(cachekey, _list, 3600)
        return _list

    def inbox(self, containerbox=None):
        obj = ContainerBoxContainers.objects
        if containerbox:
            return obj.get(container=self.id,
                           containerbox__slug=containerbox)
        return obj.filter(container=self.id)

    def custom_fields(self):
        import json
        if not self.json:
            return
        return json.loads(self.json)

class ContainerImage(models.Model):
    container = models.ForeignKey(
        'containers.Container',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container'),
    )
    order = models.PositiveIntegerField(_(u'Order'), default=0)
    image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Image'),
    )

    caption = models.CharField(
        _(u"Caption"),
        max_length=255,
        blank=True,
        null=True
    )

    class Meta:
        verbose_name = _(u'Container image')
        verbose_name_plural = _(u'Container images')
        ordering = ('order',)

    def __unicode__(self):
        if self.image:
            return u"{0}".format(self.image.title)
        return u'Id:{0} - Order:{1}'.format(self.id, self.order)

class ContainerBox(BaseBox):

    title = models.CharField(
        _(u"Title"),
        null=True,
        blank=True,
        max_length=140,
    )
    title_url = models.CharField(
        _(u"Title Link"),
        null=True,
        blank=True,
        max_length=250,
    )
    main_image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Main Image'),
    )
    main_image_caption = models.CharField(
        _(u"Main Image Caption"),
        max_length=4000,
        blank=True,
        null=True,
        help_text=_(u'Maximum characters 4000'),
    )
    containers = models.ManyToManyField(
        'containers.Container',
        null=True, blank=True,
        verbose_name=_(u'Container'),
        related_name='containerbox_containers',
        through='containers.ContainerBoxContainers'
    )
    queryset = models.ForeignKey(
        'boxes.QuerySet',
        null=True, blank=True,
        related_name='containerbox_querysets',
        verbose_name=_(u'Query Set')
    )

    content_group = models.CharField(
        _(u"Content Group"),
        max_length=255,
        default='default',
        help_text=_(u"Boxes in the same group do not allow repetitions")
    )

    class Meta:
        verbose_name = _(u'Container box')
        verbose_name_plural = _(u'Containers boxes')

    def __init__(self, *args, **kwargs):
        """
        to avoid re-execution of methods
        its results are cached in a local storage
        per instance cache
        """
        super(ContainerBox, self).__init__(*args, **kwargs)
        if not hasattr(self, 'local_cache'):
            self.local_cache = {}

    @property
    def has_content(self):
        # TODO: should check start/end available_dates
        if self.containerboxcontainers_set.exists():
            return True
        qs = self.get_queryset(use_local_cache=False)
        if qs and qs.exists():
            return True

    def ordered_containers(self, field='order', exclude_ids=None):
        cached = self.local_cache.get('ordered_containers', None)
        if cached:
            return cached

        exclude_ids = exclude_ids or []

        now = timezone.now()
        fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        if not fallback:
            qs = self.containers.filter(
                models.Q(containerboxcontainers__date_end__gte=now) |
                models.Q(containerboxcontainers__date_end__isnull=True),
                published=True,
                date_available__lte=now,
                containerboxcontainers__date_available__lte=now
            ).exclude(
                id__in=exclude_ids
            ).order_by('containerboxcontainers__order').distinct()
        else:

            site_master = cache.get("site_master")

            if not site_master:
                site_master = Site.objects.order_by('id')[0]
                cache.set('site_master', site_master, 3600)

            boxes = [self]

            if fallback and site_master.pk != self.site_id:
                try:
                    master_box = self.__class__.objects.get(
                        site=site_master, slug=self.slug
                    )
                    boxes.insert(0, master_box)
                except self.__class__.DoesNotExist:
                    pass

            qs = ContainerBoxContainers.objects.filter(
                models.Q(date_end__gte=now) |
                models.Q(date_end__isnull=True),
                models.Q(container__published=True,
                         container__date_available__lte=now) |
                models.Q(container__isnull=True),
                containerbox__in=boxes,
                date_available__lte=now,
            ).exclude(
                container__id__in=exclude_ids
            ).order_by('-containerbox__site__id', 'order').distinct()

        self.local_cache['ordered_containers'] = qs
        return qs

    def ordered_box_containers(self, exclude_ids=None):

        fallback = getattr(settings, 'OPPS_MULTISITE_FALLBACK', False)
        if fallback:
            return self.ordered_containers(exclude_ids=exclude_ids)

        cached = self.local_cache.get('ordered_box_containers')
        if cached:
            return cached

        exclude_ids = exclude_ids or []

        now = timezone.now()
        qs = self.containerboxcontainers_set.filter(
            models.Q(date_end__gte=now) |
            models.Q(date_end__isnull=True),
            date_available__lte=now
        ).exclude(
            container__id__in=exclude_ids
        ).order_by('order').distinct()

        self.local_cache['ordered_box_containers'] = qs
        return qs

    def get_containers_queryset(self):
        if self.queryset:
            return self.queryset.get_queryset()
        else:
            return self.ordered_containers()

    def get_queryset(self, exclude_ids=None, use_local_cache=True):
        cached = self.local_cache.get('get_queryset')
        if use_local_cache and cached:
            return cached

        queryset = self.queryset and self.queryset.get_queryset(
            content_group=self.content_group,
            exclude_ids=exclude_ids,
            use_local_cache=use_local_cache
        )
        if use_local_cache:
            self.local_cache['get_queryset'] = queryset
        return queryset

class ContainerBoxContainers(models.Model):
    URL_TARGET_CHOICES = (
        ('_blank', _(u'Load in a new window')),
        ('_self', _(u'Load in the same frame as it was clicked')),
        ('_parent', _(u'Load in the parent frameset')),
        ('_top', _(u'Load in the full body of the window'))
    )

    containerbox = models.ForeignKey(
        'containers.ContainerBox',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container Box'),
    )
    container = models.ForeignKey(
        'containers.Container',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Container'),
    )

    order = models.PositiveIntegerField(_(u'Order'), default=0)
    aggregate = models.BooleanField(_(u'Aggregate container'), default=False)
    highlight = models.BooleanField(_(u'Highlight container'), default=False)
    date_available = models.DateTimeField(_(u"Date available"),
                                          default=timezone.now, null=True)
    date_end = models.DateTimeField(_(u"End date"), null=True, blank=True)

    title = models.CharField(_(u"Title"), max_length=140,
                             null=True, blank=True)

    hat = models.CharField(
        _(u"Hat"),
        max_length=140,
        null=True, blank=True,
    )

    main_image = models.ForeignKey(
        'images.Image',
        null=True, blank=True,
        on_delete=models.SET_NULL,
        verbose_name=_(u'Main Image'),
    )
    main_image_caption = models.CharField(
        _(u"Main Image Caption"),
        max_length=4000,
        blank=True,
        null=True,
        help_text=_(u'Maximum characters 4000'),
    )

    url = models.CharField(_(u"URL"), max_length=255, null=True, blank=True)
    url_target = models.CharField(_(u"URL Target"), max_length=255,
                                  choices=URL_TARGET_CHOICES, default="_self",
                                  null=True, blank=True)

    class Meta:
        ordering = ('order',)
        verbose_name = _(u'Article box articles')
        verbose_name_plural = _(u'Article boxes articles')
        ordering = ('order', 'aggregate',)

    def __unicode__(self):
        if self.container:
            return u"{0}-{1}".format(self.containerbox.slug,
                                     self.container.slug)
        else:
            return u"{0}".format(self.containerbox.slug)

    def clean(self):
        if not self.container and not self.url:
            raise ValidationError(_(u'Please select a Container or insert a '
                                    u'URL!'))

        if self.container and not self.container.published:
            raise ValidationError(_(u'Article not published!'))

class ContainerRelated(models.Model):
    container = models.ForeignKey(
        'containers.Container',
        verbose_name=_(u'Container'),
        null=True,
        blank=True,
        related_name='containerrelated_container',
        on_delete=models.SET_NULL
    )
    related = models.ForeignKey(
        'containers.Container',
        verbose_name=_(u'Related Container'),
        null=True,
        blank=True,
        related_name="%(app_label)s_%(class)s_container",
        on_delete=models.SET_NULL
    )
    order = models.PositiveIntegerField(_(u'Order'), default=0)

    class Meta:
        verbose_name = _('Related content')
        verbose_name_plural = _('Related contents')
        ordering = ('order',)

    def __unicode__(self):
        return u"{0}->{1}".format(self.related.slug, self.container.slug)

class Mirror(Container):
    container = models.ForeignKey('containers.Container',
                                  related_name='containers_mirror',
                                  verbose_name=_(u'Container'))

    class Meta:
        verbose_name = _(u'Mirror')
        verbose_name_plural = _(u'Mirrors')

models.signals.post_delete.connect(delete_container, sender=Container)
<|endoftext|>


<commit_before>#!/usr/bin/python

import socket
import time
import sys
import random

count = 5
try:
  count = int(sys.argv[1])
except:
  pass

timem = 1000000
goodenough = 30000 # delay of ans in us
servers = []

socket.setdefaulttimeout(2)

def testserver(srv,timem):
  ctime = timem
  s = socket.socket()
  try:
    stime = time.time()
    s.connect((server,8074))
    etime = time.time()
    s.close()
    return int((etime-stime)*timem)
  except socket.error:
    return timem*10

if __name__ == '__main__':

  # generate all the IPs
  for o4 in xrange(2,100):
    servers.append("91.214.237."+str(o4))

  # test servers
  while len(servers) > 0:

    # pick a random server
    server = random.choice(servers)
    servers.remove(server)

    times = ""
    ok = 1

    # test it if it is ok 3 times
    for i in xrange(0,3):
      delay = testserver(server,timem)
      if delay > goodenough:
        ok = 0
        break
      else:
        times += str(delay)+"us "

    if ok == 1:
      print str(server)+" "+str(times)
      break
    else:
      continue
<commit_msg>home_stuff/gg_server_tester.py - improved check if the server is alive and fast<commit_after>#!/usr/bin/python

import socket
import time
import sys
import random

count = 5
try:
  count = int(sys.argv[1])
except:
  pass

timem = 1000000
goodenough = 60000 # delay of ans including reading 2 first bytes in us
servers = []

socket.setdefaulttimeout(2)

def testserver(srv,timem):
  s = socket.socket()
  try:
    stime = time.time()
    s.connect((server,8074))
    data = s.recv(2)
    if str(data).encode('hex') == "0100":
      etime = time.time()
    else:
      etime = stime + timem*10
    s.close()
    return int((etime-stime)*timem)
  except socket.error:
    return timem*10

if __name__ == '__main__':

  # generate all the IPs
  for o4 in xrange(2,100):
    servers.append("91.214.237."+str(o4))

  # test servers
  while len(servers) > 0:

    # pick a random server
    server = random.choice(servers)
    servers.remove(server)

    times = ""
    ok = 1

    # test it if it is ok 3 times
    for i in xrange(0,3):
      delay = testserver(server,timem)
      if delay > goodenough:
        ok = 0
        break
      else:
        times += str(delay)+"us "

    if ok == 1:
      print str(server)+" "+str(times)
      break
    else:
      continue
<|endoftext|>


<commit_before>import argparse
from io import BytesIO
from binascii import unhexlify
from time import sleep

from sqlalchemy.exc import IntegrityError

from pycoin.block import Block

from .models import DBSession, merge_nulldatas_from_block_obj
from .compatibility import bitcoind

if __name__ == "__main__":
    session = DBSession

    parser = argparse.ArgumentParser()
    parser.add_argument('--block-height', help='Start from this block height', type=int, required=True)
    args = parser.parse_args()
    block_height = args.block_height

    def get_block(block_hash, hex_summary=True):
        return bitcoind.getblock(block_hash, hex_summary)

    def block_as_bytesio(block_hash):
        return BytesIO(unhexlify(get_block(block_hash, False)))

    while True:
        try:
            block_hash = bitcoind.getblockhash(block_height)
            block = Block.parse(block_as_bytesio(block_hash))
            merge_nulldatas_from_block_obj(block, block_hash, block_height, verbose=True)
            block_height += 1
        except KeyboardInterrupt:
            print("Exiting.")
            break
        except IntegrityError as e:
            session.rollback()
            block_height += 1  # probably a duplicate entry
            print(e, 'skipping')
        except Exception as e:
            session.rollback()
            print(e, type(e))
            sleep(1)
<commit_msg>Add logging debug<commit_after>import argparse
from io import BytesIO
from binascii import unhexlify
from time import sleep
import logging

from sqlalchemy.exc import IntegrityError

from pycoin.block import Block

from .models import DBSession, merge_nulldatas_from_block_obj
from .compatibility import bitcoind

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    session = DBSession

    parser = argparse.ArgumentParser()
    parser.add_argument('--block-height', help='Start from this block height', type=int, required=True)
    args = parser.parse_args()
    block_height = args.block_height

    def get_block(block_hash, hex_summary=True):
        return bitcoind.getblock(block_hash, hex_summary)

    def block_as_bytesio(block_hash):
        return BytesIO(unhexlify(get_block(block_hash, False)))

    while True:
        try:
            block_hash = bitcoind.getblockhash(block_height)
            block = Block.parse(block_as_bytesio(block_hash))
            merge_nulldatas_from_block_obj(block, block_hash, block_height, verbose=True)
            block_height += 1
        except KeyboardInterrupt:
            print("Exiting.")
            break
        except IntegrityError as e:
            session.rollback()
            block_height += 1  # probably a duplicate entry
            print(e, 'skipping')
        except Exception as e:
            session.rollback()
            print(e, type(e))
            sleep(1)
<|endoftext|>