<commit_before>import cPickle as pickle
import os
import json
import logging
import shutil
import time
from collections import defaultdict
from datetime import datetime, timedelta
from operator import itemgetter

import hb
import settings_local as settings

log = logging.getLogger('glow')

# The default version number we look for.
FX = settings.FIREFOX_VERSION
JSON_DIR = os.path.join(settings.BASE_DIR, 'json')
PICKLE = settings.path('glow.pickle')
BACKUP = PICKLE + '.bak'

hbase = hb.Client(settings.HBASE_HOST, settings.HBASE_PORT,
                  settings.HBASE_TABLES['new'])

# Maps {country: continent}.
continents = json.load(open(settings.path('continents.json')))

# Maps {country code: name}.
countries = json.load(open(settings.path('countries.json')))

# Maps {country code: {region code: name}}.
regions = json.load(open(settings.path('regions.json')))
geo = (continents, countries, regions)

# We're not supposed to show downloads for these countries (607127#c10):
# Cuba, Iran, Syria, N. Korea, Myanmar, Sudan. Go figure.
REDACTED = ('CU', 'IR', 'SY', 'KP', 'MM', 'SD')

##
## 1. The part that talks to Hbase and collects data.
##

# These contain global download totals that get updated every time we process a
# new chunk of data.
G = {
    'total': 0,
    'counts': [],
    'arc': {},
    'version': 6,
}

# This should be a lambda but pickle can't pickle a lambda.
def defaultdict_int():
    return defaultdict(int)

# The global locale count aggregator.
# {continent: {country: {region: {city: total}}}}
G['arc'] = dict((k, {}) for k in continents.values())
for country, continent in continents.items():
    G['arc'][continent][country] = defaultdict(defaultdict_int)
for country, regions in regions.items():
    continent = continents[country]
    for region in regions:
        G['arc'][continent][country][region] = defaultdict(int)

def row_name(dt):
    """Convert a datetime into the Hbase timestamp format."""
    # TODO: mobile.
    return 'firefox::%s:%s' % (FX, dt.strftime('%Y-%m-%dT%H:%M:00.000'))

def time_sequence(dt, num=100):
    for i in xrange(num):
        yield dt + timedelta(minutes=i)

def row_sum(row):
    return sum(row.columns.itervalues()) if row else 0

def get_counts(dt, num=1):
    """Get `num` minutes of download counts starting at `dt`."""
    if num == 1:
        rows = hbase.row(row_name(dt), ['product'])
    else:
        rows = hbase.scanner(row_name(dt), ['product']).list(num)
    return [(t.utctimetuple()[:5], row_sum(row))
            for t, row in zip(time_sequence(dt, num), rows)]

def extend_counts(counts):
    for t, count in counts:
        G['total'] += count
        G['counts'].append((t, G['total']))
    G['counts'] = G['counts'][-60:]

def process_locations(rows):
    """
    Break up the hbase rows into a list of
    [(continent, country, region, city, lat, lon, num_downloads)].

    The cumulative count in `arc` is updated inline.
    """
    # Get local names for fast lookups in the loop.
    arc = G['arc']
    continents, countries, regions = geo
    rv = []
    total = 0
    for row in rows:
        new = []
        # We localize country names on the client.
        for key, val in row.columns.iteritems():
            total += val
            country, region, city, lat, lon = key.split(':')[-5:]
            if country in REDACTED:
                continue
            try:
                # Sometimes maxmind gives us regions named '  ' or '00'. Those
                # are invalid. The frontend expects invalid regions named ''.
                if region.strip() in ('', '00'):
                    region = ''
                    log.debug('Renaming region: %s.' % key)
                continent = continents[country]
                arc[continent][country][region][city] += val
                new.append((continent, country, region, city,
                            lat, lon, val))
            except (KeyError, ValueError):
                log.error('skipping key: %s' % key, exc_info=True)
                pass
        rv.append((total, new))
    return rv

def _get_locations(dt, num=1):
    """Get `num` minutes of download locations starting at `dt`."""
    if num == 1:
        rows = hbase.row(row_name(dt), ['location:'])
    else:
        rows = hbase.scanner(row_name(dt), ['location:']).list(num)
    locs = process_locations(rows)
    return [(t.utctimetuple()[:5], r)
            for t, r in zip(time_sequence(dt, num), locs)]

def get_map(dt, num=1):
    """Get a list of [`dt`, num_rows, [(lat, long, num_downloads)]]."""
    # Get (time, num_rows, [(lat, long, hits)]) for each datetime.
    times = [(t, (num, [r[-3:] for r in rows]))
             for t, (num, rows) in _get_locations(dt, num)]
    hits = [row for t in times for row in t[1][1]] * 20
    return (times[0][0], len(hits), hits)

def get_arc():
    """
    Aggregate the location data into an easy json structure:

        (None, total,
         [continent, total, [country, total, [region, total, [city, total]]]])

    The expected format of `data` is:

        {continent: {country: {region: {city: total}}}}

    Each outer total is the sum of its childrens' inner totals.
    """
    def unpack(dict_):
        """Unpack a (key, (v1, v2)) structure into (key, v1, v2)."""
        return revsort((a.strip(), b, c) for a, (b, c) in dict_.iteritems())

    revsort = lambda xs: sorted(xs, key=itemgetter(1), reverse=True)
    continents, world_sum = {}, 0
    for continent, country_dict in G['arc'].iteritems():
        countries, continent_sum = {}, 0
        for country, region_dict in country_dict.iteritems():
            regions, country_sum = {}, 0
            for region, cities in region_dict.iteritems():
                total = sum(cities.itervalues())
                if total:
                    cs = [(k.strip(), v) for k, v in cities.iteritems()]
                    regions[region] = [total, revsort(cs)]
                    country_sum += total
            if country_sum:
                countries[country] = [country_sum, unpack(regions)]
                continent_sum += country_sum
        if continent_sum:
            continents[continent] = [continent_sum, unpack(countries)]
            world_sum += continent_sum
    return (None, world_sum, unpack(continents))

##
## 2. The main loop.
##

def makedirs(d):
    if not os.path.exists(d):
        log.info('Making dir %s.' % d)
        os.makedirs(d)

def write_files(dt, count_data=None, map_data=None, arc_data=None,
                interval=60):
    """Write all the data dicts we were given to their files."""
    log.info('Writing data for %s.' % dt)
    xs = {'count': count_data, 'map': map_data, 'arc': arc_data}
    for name, data in xs.items():
        if not data:
            continue
        fmt = '%Y/%m/%d/%H/%M/{name}.json'.format(name=name)
        path = os.path.join(JSON_DIR, dt.strftime(fmt))
        next = (dt + timedelta(seconds=interval)).strftime(fmt)
        makedirs(os.path.dirname(path))
        d = {'next': next, 'interval': interval, 'data': data}
        json.dump(d, open(path, 'w'), separators=(',', ':'))

def collect(dt):
    """Grab Hbase data, write json files, save internal state."""
    log.info('Fetching data for %s.' % dt)
    extend_counts(get_counts(dt))
    write_files(dt, G['counts'], get_map(dt), get_arc())
    dump_state(dt)

def now():
    # Live one minute in the past so Hbase has time to collect a full minute of
    # data before we start talking to it.
    return datetime.utcnow() - timedelta(minutes=1)

def do_the_stuff_to_the_thing():
    dt = now()
    next = dt + timedelta(minutes=1)
    # Wait until :30 to give Hbase some processing time.
    if dt.second < 30:
        log.info('Waiting until :30 past.')
        time.sleep(30 - dt.second)

    collect(dt)

    # Sleep until the next minute comes around.
    wait = next.replace(second=30) - now().replace(microsecond=0)
    # The delta will be around -1 days, 86400 seconds if we're into the next
    # minute already.
    if wait.seconds <= 60:
        log.info('Sleeping for %s seconds.' % wait.seconds)
        time.sleep(wait.seconds)
    else:
        log.info('Skipping sleep.')

def main():
    load_state()
    log.info('Looping, infinitely.')
    while 1:
        try:
            do_the_stuff_to_the_thing()
        except hb.exceptions:
            log.error('Recycling Hbase connection.', exc_info=True)
            hbase.recycle()

#
# 3. Saving and loading application state.
#

def dump_state(dt):
    """Dump all the global aggregators so we can pick at the same spot."""
    log.info('Saving state for %s.' % dt)
    if os.path.exists(PICKLE):
        shutil.copyfile(PICKLE, BACKUP)
    d = {'G': G, 'last_update': dt}
    pickle.dump(d, open(PICKLE, 'w'))

def load_state():
    """Figure out where we left off, catch up on old data if needed."""
    if not (os.path.exists(PICKLE) or os.path.exists(BACKUP)):
        return
    log.info('Found a pickle, picking it up.')
    try:
        d = pickle.load(open(PICKLE))
    except Exception:
        log.error('Trouble opening pickle.', exc_info=True)
        if os.path.exists(BACKUP):
            log.info('Loading backup pickle.')
            d = pickle.load(open(BACKUP))

    if d['G'].get('version') == G['version']:
        for k, v in d['G'].items():
            G[k] = v
    else:
        log.info('Skipping out of date pickle (want v%s).' % G['version'])

    dt = now()
    delta = dt - d['last_update'].replace(second=0)
    if delta.seconds > 60:
        log.info('Missing %s minutes. Catching up.' % (delta.seconds / 60))
        for i in xrange(1, delta.seconds / 60):
            collect(d['last_update'] + timedelta(minutes=i))

    # Collect once more if the clock rolled over during catchup.
    if now().minute != dt.minute:
        log.info('Rollover!')
        load_state()

    # Wait until the next minute if the last update was at 1:15:00 and the
    # current time is less than 1:16:00 so we don't count twice.
    if now().minute == d['last_update'].minute:
        log.info('Waiting for the minute to roll over.')
        time.sleep(60 - now().second)

#
# 4. Cleanup.
#

def cleanup():
    # Delete all the data from two days ago. This expects to run in cron daily
    # so there won't be any data older than two days.
    d = (now() - timedelta(days=2)).strftime('%Y/%m/%d')
    path = os.path.join(JSON_DIR, d)
    if os.path.exists(path):
        log.info('Dropping %s.' % path)
        shutil.rmtree(path)
        # Try to delete the month and year directories. rmdir only works if the
        # directory is empty, so this will clean up empty directories.
        try:
            os.rmdir(os.path.dirname(path))
            os.rmdir(os.path.dirname(os.path.dirname(path)))
        except OSError:
            pass
<commit_msg>ignore (0,0) in the map<commit_after>import cPickle as pickle
import os
import json
import logging
import shutil
import time
from collections import defaultdict
from datetime import datetime, timedelta
from operator import itemgetter

import hb
import settings_local as settings

log = logging.getLogger('glow')

# The default version number we look for.
FX = settings.FIREFOX_VERSION
JSON_DIR = os.path.join(settings.BASE_DIR, 'json')
PICKLE = settings.path('glow.pickle')
BACKUP = PICKLE + '.bak'

hbase = hb.Client(settings.HBASE_HOST, settings.HBASE_PORT,
                  settings.HBASE_TABLES['new'])

# Maps {country: continent}.
continents = json.load(open(settings.path('continents.json')))

# Maps {country code: name}.
countries = json.load(open(settings.path('countries.json')))

# Maps {country code: {region code: name}}.
regions = json.load(open(settings.path('regions.json')))
geo = (continents, countries, regions)

# We're not supposed to show downloads for these countries (607127#c10):
# Cuba, Iran, Syria, N. Korea, Myanmar, Sudan. Go figure.
REDACTED = ('CU', 'IR', 'SY', 'KP', 'MM', 'SD')

##
## 1. The part that talks to Hbase and collects data.
##

# These contain global download totals that get updated every time we process a
# new chunk of data.
G = {
    'total': 0,
    'counts': [],
    'arc': {},
    'version': 6,
}

# This should be a lambda but pickle can't pickle a lambda.
def defaultdict_int():
    return defaultdict(int)

# The global locale count aggregator.
# {continent: {country: {region: {city: total}}}}
G['arc'] = dict((k, {}) for k in continents.values())
for country, continent in continents.items():
    G['arc'][continent][country] = defaultdict(defaultdict_int)
for country, regions in regions.items():
    continent = continents[country]
    for region in regions:
        G['arc'][continent][country][region] = defaultdict(int)

def row_name(dt):
    """Convert a datetime into the Hbase timestamp format."""
    # TODO: mobile.
    return 'firefox::%s:%s' % (FX, dt.strftime('%Y-%m-%dT%H:%M:00.000'))

def time_sequence(dt, num=100):
    for i in xrange(num):
        yield dt + timedelta(minutes=i)

def row_sum(row):
    return sum(row.columns.itervalues()) if row else 0

def get_counts(dt, num=1):
    """Get `num` minutes of download counts starting at `dt`."""
    if num == 1:
        rows = hbase.row(row_name(dt), ['product'])
    else:
        rows = hbase.scanner(row_name(dt), ['product']).list(num)
    return [(t.utctimetuple()[:5], row_sum(row))
            for t, row in zip(time_sequence(dt, num), rows)]

def extend_counts(counts):
    for t, count in counts:
        G['total'] += count
        G['counts'].append((t, G['total']))
    G['counts'] = G['counts'][-60:]

def process_locations(rows):
    """
    Break up the hbase rows into a list of
    [(continent, country, region, city, lat, lon, num_downloads)].

    The cumulative count in `arc` is updated inline.
    """
    # Get local names for fast lookups in the loop.
    arc = G['arc']
    continents, countries, regions = geo
    rv = []
    total = 0
    for row in rows:
        new = []
        # We localize country names on the client.
        for key, val in row.columns.iteritems():
            total += val
            country, region, city, lat, lon = key.split(':')[-5:]
            if country in REDACTED:
                continue
            try:
                # Sometimes maxmind gives us regions named '  ' or '00'. Those
                # are invalid. The frontend expects invalid regions named ''.
                if region.strip() in ('', '00'):
                    region = ''
                    log.debug('Renaming region: %s.' % key)
                # (0, 0) means the download is from a satellite/proxy.
                if float(lat) == float(lon) == 0:
                    continue
                continent = continents[country]
                arc[continent][country][region][city] += val
                new.append((continent, country, region, city,
                            lat, lon, val))
            except (KeyError, ValueError):
                log.error('skipping key: %s' % key, exc_info=True)
                pass
        rv.append((total, new))
    return rv

def _get_locations(dt, num=1):
    """Get `num` minutes of download locations starting at `dt`."""
    if num == 1:
        rows = hbase.row(row_name(dt), ['location:'])
    else:
        rows = hbase.scanner(row_name(dt), ['location:']).list(num)
    locs = process_locations(rows)
    return [(t.utctimetuple()[:5], r)
            for t, r in zip(time_sequence(dt, num), locs)]

def get_map(dt, num=1):
    """Get a list of [`dt`, num_rows, [(lat, long, num_downloads)]]."""
    # Get (time, num_rows, [(lat, long, hits)]) for each datetime.
    times = [(t, (num, [r[-3:] for r in rows]))
             for t, (num, rows) in _get_locations(dt, num)]
    hits = [row for t in times for row in t[1][1]] * 20
    return (times[0][0], len(hits), hits)

def get_arc():
    """
    Aggregate the location data into an easy json structure:

        (None, total,
         [continent, total, [country, total, [region, total, [city, total]]]])

    The expected format of `data` is:

        {continent: {country: {region: {city: total}}}}

    Each outer total is the sum of its childrens' inner totals.
    """
    def unpack(dict_):
        """Unpack a (key, (v1, v2)) structure into (key, v1, v2)."""
        return revsort((a.strip(), b, c) for a, (b, c) in dict_.iteritems())

    revsort = lambda xs: sorted(xs, key=itemgetter(1), reverse=True)
    continents, world_sum = {}, 0
    for continent, country_dict in G['arc'].iteritems():
        countries, continent_sum = {}, 0
        for country, region_dict in country_dict.iteritems():
            regions, country_sum = {}, 0
            for region, cities in region_dict.iteritems():
                total = sum(cities.itervalues())
                if total:
                    cs = [(k.strip(), v) for k, v in cities.iteritems()]
                    regions[region] = [total, revsort(cs)]
                    country_sum += total
            if country_sum:
                countries[country] = [country_sum, unpack(regions)]
                continent_sum += country_sum
        if continent_sum:
            continents[continent] = [continent_sum, unpack(countries)]
            world_sum += continent_sum
    return (None, world_sum, unpack(continents))

##
## 2. The main loop.
##

def makedirs(d):
    if not os.path.exists(d):
        log.info('Making dir %s.' % d)
        os.makedirs(d)

def write_files(dt, count_data=None, map_data=None, arc_data=None,
                interval=60):
    """Write all the data dicts we were given to their files."""
    log.info('Writing data for %s.' % dt)
    xs = {'count': count_data, 'map': map_data, 'arc': arc_data}
    for name, data in xs.items():
        if not data:
            continue
        fmt = '%Y/%m/%d/%H/%M/{name}.json'.format(name=name)
        path = os.path.join(JSON_DIR, dt.strftime(fmt))
        next = (dt + timedelta(seconds=interval)).strftime(fmt)
        makedirs(os.path.dirname(path))
        d = {'next': next, 'interval': interval, 'data': data}
        json.dump(d, open(path, 'w'), separators=(',', ':'))

def collect(dt):
    """Grab Hbase data, write json files, save internal state."""
    log.info('Fetching data for %s.' % dt)
    extend_counts(get_counts(dt))
    write_files(dt, G['counts'], get_map(dt), get_arc())
    dump_state(dt)

def now():
    # Live one minute in the past so Hbase has time to collect a full minute of
    # data before we start talking to it.
    return datetime.utcnow() - timedelta(minutes=1)

def do_the_stuff_to_the_thing():
    dt = now()
    next = dt + timedelta(minutes=1)
    # Wait until :30 to give Hbase some processing time.
    if dt.second < 30:
        log.info('Waiting until :30 past.')
        time.sleep(30 - dt.second)

    collect(dt)

    # Sleep until the next minute comes around.
    wait = next.replace(second=30) - now().replace(microsecond=0)
    # The delta will be around -1 days, 86400 seconds if we're into the next
    # minute already.
    if wait.seconds <= 60:
        log.info('Sleeping for %s seconds.' % wait.seconds)
        time.sleep(wait.seconds)
    else:
        log.info('Skipping sleep.')

def main():
    load_state()
    log.info('Looping, infinitely.')
    while 1:
        try:
            do_the_stuff_to_the_thing()
        except hb.exceptions:
            log.error('Recycling Hbase connection.', exc_info=True)
            hbase.recycle()

#
# 3. Saving and loading application state.
#

def dump_state(dt):
    """Dump all the global aggregators so we can pick at the same spot."""
    log.info('Saving state for %s.' % dt)
    if os.path.exists(PICKLE):
        shutil.copyfile(PICKLE, BACKUP)
    d = {'G': G, 'last_update': dt}
    pickle.dump(d, open(PICKLE, 'w'))

def load_state():
    """Figure out where we left off, catch up on old data if needed."""
    if not (os.path.exists(PICKLE) or os.path.exists(BACKUP)):
        return
    log.info('Found a pickle, picking it up.')
    try:
        d = pickle.load(open(PICKLE))
    except Exception:
        log.error('Trouble opening pickle.', exc_info=True)
        if os.path.exists(BACKUP):
            log.info('Loading backup pickle.')
            d = pickle.load(open(BACKUP))

    if d['G'].get('version') == G['version']:
        for k, v in d['G'].items():
            G[k] = v
    else:
        log.info('Skipping out of date pickle (want v%s).' % G['version'])

    dt = now()
    delta = dt - d['last_update'].replace(second=0)
    if delta.seconds > 60:
        log.info('Missing %s minutes. Catching up.' % (delta.seconds / 60))
        for i in xrange(1, delta.seconds / 60):
            collect(d['last_update'] + timedelta(minutes=i))

    # Collect once more if the clock rolled over during catchup.
    if now().minute != dt.minute:
        log.info('Rollover!')
        load_state()

    # Wait until the next minute if the last update was at 1:15:00 and the
    # current time is less than 1:16:00 so we don't count twice.
    if now().minute == d['last_update'].minute:
        log.info('Waiting for the minute to roll over.')
        time.sleep(60 - now().second)

#
# 4. Cleanup.
#

def cleanup():
    # Delete all the data from two days ago. This expects to run in cron daily
    # so there won't be any data older than two days.
    d = (now() - timedelta(days=2)).strftime('%Y/%m/%d')
    path = os.path.join(JSON_DIR, d)
    if os.path.exists(path):
        log.info('Dropping %s.' % path)
        shutil.rmtree(path)
        # Try to delete the month and year directories. rmdir only works if the
        # directory is empty, so this will clean up empty directories.
        try:
            os.rmdir(os.path.dirname(path))
            os.rmdir(os.path.dirname(os.path.dirname(path)))
        except OSError:
            pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

def test_generate_tests(testdir):
    """Make sure that pytest accepts our fixture."""

    # create a temporary pytest test module
    testdir.makepyfile("""
        def test_factorial(wish):
            assert wish(0) == 1
            assert wish(1) == 1
            assert wish(21) == 51090942171709440000
    """)

    # run pytest with the following cmd args
    result = testdir.runpytest(
        '--wish-modules=math',
        '-v',
    )

    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        '*::test_factorial*math:factorial*XPASS',
    ])

    # make sure that that we get a '0' exit code for the testsuite
    assert result.ret == 0

def test_help_message(testdir):
    result = testdir.runpytest(
        '--help',
    )
    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        'wish:',
        '*--wish-modules*',
        '*--wish-fail*',
    ])
<commit_msg>test support for skipping tests when no wish_modules option is provided<commit_after># -*- coding: utf-8 -*-

TEST_FACTORIAL = """
def test_factorial(wish):
    assert wish(0) == 1
    assert wish(1) == 1
    assert wish(21) == 51090942171709440000
"""

def test_generate_tests(testdir):
    """Make sure that pytest accepts our fixture."""

    # create a temporary pytest test module
    testdir.makepyfile(TEST_FACTORIAL)

    # run pytest with the following cmd args
    result = testdir.runpytest(
        '--wish-modules=math',
        '-v',
    )

    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        '*::test_factorial*math:factorial*XPASS',
    ])

    # make sure that that we get a '0' exit code for the testsuite
    assert result.ret == 0

def test_skip_tests(testdir):
    """Make sure that pytest accepts our fixture."""

    # create a temporary pytest test module
    testdir.makepyfile(TEST_FACTORIAL)

    # run pytest with the following cmd args
    result = testdir.runpytest(
        '-v',
    )

    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        '*::test_factorial*wish*SKIPPED',
    ])

    # make sure that that we get a '0' exit code for the testsuite
    assert result.ret == 0

def test_help_message(testdir):
    result = testdir.runpytest(
        '--help',
    )
    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        'wish:',
        '*--wish-modules*',
        '*--wish-fail*',
    ])
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
import sys
import shutil
import tempfile
import logging
import getpass
import subprocess
import time

from unittest import TestCase
from nose.tools import nottest
from mock import patch
import git

from dusty import constants
from dusty.config import write_default_config, save_config_value, get_config, save_config
from dusty.compiler.spec_assembler import get_specs_repo
from dusty.commands.repos import override_repo
from dusty.cli import main as client_entrypoint
from dusty.schemas.base_schema_class import get_specs_from_path, DustySpecs
from dusty.systems.docker import exec_in_container, get_docker_client, get_container_for_app_or_service, get_docker_env
from dusty.systems.nfs import client as nfs_client
from dusty.path import parent_dir
from dusty.subprocess import call_demoted
from .fixtures import basic_specs_fixture
from dusty.log import client_logger, DustyClientTestingSocketHandler
from dusty.memoize import reset_memoize_cache

class TestCaptureHandler(logging.Handler):
    def __init__(self, lst):
        super(TestCaptureHandler, self).__init__()
        self.lst = lst

    def emit(self, record):
        self.lst.append(self.format(record))

class DustyTestCase(TestCase):
    def setUp(self):
        self.temp_config_path = tempfile.mkstemp()[1]
        self.temp_specs_path = tempfile.mkdtemp()
        self.temp_repos_path = tempfile.mkdtemp()

        constants.CONFIG_PATH = self.temp_config_path
        write_default_config()
        save_config_value(constants.CONFIG_SPECS_REPO_KEY, 'github.com/org/dusty-specs')
        override_repo(get_specs_repo().remote_path, self.temp_specs_path)
        basic_specs_fixture()

        self.client_output = []
        self.capture_handler = TestCaptureHandler(self.client_output)
        logging.getLogger(constants.SOCKET_LOGGER_NAME).addHandler(self.capture_handler)
        reset_memoize_cache()

    def tearDown(self):
        os.remove(self.temp_config_path)
        shutil.rmtree(self.temp_specs_path)
        shutil.rmtree(self.temp_repos_path)
        logging.getLogger(constants.SOCKET_LOGGER_NAME).removeHandler(self.capture_handler)
        reset_memoize_cache()

    @nottest
    @patch('dusty.schemas.base_schema_class.get_specs_from_path')
    def make_test_specs(self, specs_dict, fake_specs_from_path):
        fake_specs_from_path.return_value = specs_dict
        return DustySpecs('')

    @property
    def last_client_output(self):
        return self.client_output[-1] if self.client_output else None

class CommandError(Exception):
    pass

class SysExit(Exception):
    pass

class DustyIntegrationTestCase(TestCase):
    """This test case intentionally avoids mocking whenever possible
    in order to get as close as possible to the actual state that
    would be experienced on a system running Dusty. Therefore,
    integration tests are possibly destructive if run on a user's
    machine. To help protect the user from running them accidentally,
    integration tests will refuse to run unless the environment
    variable DUSTY_ALLOW_INTEGRATION_TESTS is set.

    Note that this also assumes it is running against an actual
    Dusty daemon process on the local host."""
    def setUp(self):
        if not os.getenv('DUSTY_ALLOW_INTEGRATION_TESTS'):
            raise RuntimeError('You must set the env var DUSTY_ALLOW_INTEGRATION_TESTS to run integration tests. '
                               'This may affect your local config, do not run integration tests on your actual '
                               "machine unless you know what you're doing!")
        self.handler = DustyClientTestingSocketHandler()
        client_logger.addHandler(self.handler)
        self.previous_config = get_config()
        self._clear_stdout()
        self.overridden_specs_path = tempfile.mkdtemp()
        write_default_config()
        save_config_value(constants.CONFIG_SETUP_KEY, True)
        save_config_value(constants.CONFIG_SPECS_REPO_KEY, 'github.com/gamechanger/dusty-example-specs')

        self.tests_user = os.getenv('DUSTY_INTEGRATION_TESTS_USER', self.current_user)
        save_config_value(constants.CONFIG_MAC_USERNAME_KEY, self.tests_user)

        override_repo(get_specs_repo().remote_path, self.overridden_specs_path)
        self.fake_local_repo_location = '/tmp/fake-repo'
        self._set_up_fake_local_repo('/tmp/fake-repo')
        self._clear_stdout()
        self.exec_docker_processes = []
        reset_memoize_cache()

    def tearDown(self):
        for exec_docker_process in self.exec_docker_processes:
            try:
                os.kill(exec_docker_process.pid)
            except:
                pass
        shutil.rmtree(self.overridden_specs_path)
        # if os.path.exists(constants.REPOS_DIR):
        #     shutil.rmtree(constants.REPOS_DIR)
        if os.path.exists(constants.COMPOSE_DIR):
            shutil.rmtree(constants.COMPOSE_DIR)
        shutil.rmtree('/tmp/fake-repo')
        save_config(self.previous_config)
        self.handler.log_to_client_output = ''
        client_logger.removeHandler(self.handler)
        nfs_client.unmount_all_repos()
        reset_memoize_cache()

    def _clear_stdout(self):
        self.stdout_start = len(sys.stdout.getvalue())

    @property
    def current_user(self):
        return getpass.getuser()

    @property
    def CommandError(self):
        """Pure convenience to avoid having to import this explicitly"""
        return CommandError

    @property
    def stdout(self):
        """Returns any stdout that has been generated *since* the last time
        clear_stdout was called."""
        return sys.stdout.getvalue()[self.stdout_start:].strip()

    def exec_docker_patch(self, *args):
        docker_executable = subprocess.check_output('which docker', shell=True).rstrip('\n')
        args = [docker_executable] + [a for a in args]
        self.exec_docker_processes.append(subprocess.Popen(args=args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=get_docker_env()))

    @patch('sys.exit')
    def run_command(self, args, fake_exit, raise_on_error=True):
        """Run a command through the Dusty client entrypoint, e.g. simulating
        the Dusty CLI as close as possible without having to call a subprocess.
        This command raises if the command fails, otherwise it returns the
        stdout generated by the command."""
        with patch('dusty.commands.utils.exec_docker', wraps=self.exec_docker_patch) as fake_exec_docker:
            fake_exit.side_effect = SysExit('sys.exit called')
            self.fake_exec_docker = fake_exec_docker
            sys.argv = ['dusty'] + args.split(' ')
            try:
                client_entrypoint()
            except SysExit:
                pass
            for call in fake_exit.mock_calls:
                name, args, kwargs = call
                if len(args) == 1 and args[0] > 0 and raise_on_error:
                    self._clear_stdout()
                    raise CommandError('Command {} returned with exit code: {}'.format(' '.join(sys.argv), args[0]))
            result = self.stdout
            self._clear_stdout()
            return result

    def _set_up_fake_local_repo(self, path='/tmp/fake-repo'):
        repo = git.Repo.init(path)
        with open(os.path.join(path, 'README.md'), 'w') as f:
            f.write('# {}'.format(path.split('/')[-1]))
        repo.index.add([os.path.join(path, 'README.md')])
        repo.index.commit('Initial commit')

    def _in_same_line(self, string, *values):
        for line in string.splitlines():
            if all(value in line for value in values):
                return True
        return False

    def _container_exists_in_set(self, service_name, include_exited):
        client = get_docker_client()
        container = get_container_for_app_or_service(client, service_name, include_exited=include_exited)
        return bool(container)

    def _image_exists(self, image_to_find):
        client = get_docker_client()
        all_images = client.images(all=True)
        return any(image_to_find in image['RepoTags'] for image in all_images)

    def _file_exists_in_container(self, service_name, file_path):
        call_demoted(['boot2docker', 'ssh', 'sync']) # flush any outstanding writes
        file_exists_str = self.exec_in_container(service_name,
            'sh -c \'[ -f {} ] && echo "yes" || echo "no"\''.format(file_path)).rstrip()
        return file_exists_str == 'yes'

    def exec_in_container(self, service_name, command):
        client = get_docker_client()
        container = get_container_for_app_or_service(client, service_name, raise_if_not_found=True)
        return exec_in_container(client, container, *command.split(' '))

    def remove_path_in_container(self, service_name, file_path):
        self.exec_in_container(service_name, 'rm -rf {}'.format(file_path))

    def write_file_in_container(self, service_name, file_path, contents):
        self.exec_in_container(service_name, 'mkdir -p {}'.format(parent_dir(file_path)))
        self.exec_in_container(service_name, 'sh -c "echo -n {} > {}"'.format(contents, file_path))

    def container_id(self, service_name):
        client = get_docker_client()
        return get_container_for_app_or_service(client, service_name, include_exited=True, raise_if_not_found=True)['Id']

    def inspect_container(self, service_name):
        container_id = self.container_id(service_name)
        client = get_docker_client()
        return client.inspect_container(container_id)

    def remove_container(self, service_name):
        container_id = self.container_id(service_name)
        client = get_docker_client()
        client.remove_container(container_id, force=True)

    @staticmethod
    def retriable_assertion(backoff, max_retries):
        def retriable_fn_wrapper(fn):
            def retriable_fn(*args, **kwargs):
                for i in range(0, max_retries):
                    try:
                        return fn(*args, **kwargs)
                    except AssertionError:
                        print 'Retriable assertion failure {}'.format(i + 1)
                        if i == max_retries - 1:
                            raise
                    time.sleep(backoff)
            return retriable_fn
        return retriable_fn_wrapper

    def assertInSameLine(self, string, *values):
        self.assertTrue(self._in_same_line(string, *values))

    def assertNotInSameLine(self, string, *values):
        self.assertFalse(self._in_same_line(string, *values))

    def assertFileContents(self, file_path, contents):
        with open(file_path, 'r') as f:
            self.assertEqual(f.read(), contents)

    def assertFileContentsInContainer(self, service_name, file_path, contents):
        self.assertEqual(self.exec_in_container(service_name, 'cat {}'.format(file_path)), contents)

    def assertFileNotInContainer(self, service_name, file_path):
        self.assertFalse(self._file_exists_in_container(service_name, file_path))

    def assertFileInContainer(self, service_name, file_path):
        self.assertTrue(self._file_exists_in_container(service_name, file_path))

    def assertContainerRunning(self, service_name):
        self.assertTrue(self._container_exists_in_set(service_name, False))

    def assertContainerIsNotRunning(self, service_name):
        self.assertFalse(self._container_exists_in_set(service_name, False))

    def assertContainerExists(self, service_name):
        self.assertTrue(self._container_exists_in_set(service_name, True))

    def assertContainerDoesNotExist(self, service_name):
        self.assertFalse(self._container_exists_in_set(service_name, True))

    def assertImageExists(self, image):
        self.assertTrue(self._image_exists(image))

    def assertImageDoesNotExist(self, image):
        self.assertFalse(self._image_exists(image))

    def assertExecDocker(self, *args):
        self.fake_exec_docker.assert_called_with(*args)

    def assertConfigValue(self, key, value):
        config = get_config()
        self.assertEqual(config[key], value)
<commit_msg>cleanup exports state in tests<commit_after>from __future__ import absolute_import

import os
import sys
import shutil
import tempfile
import logging
import getpass
import subprocess
import time

from unittest import TestCase
from nose.tools import nottest
from mock import patch
import git

from dusty import constants
from dusty.config import write_default_config, save_config_value, get_config, save_config
from dusty.compiler.spec_assembler import get_specs_repo
from dusty.commands.repos import override_repo
from dusty.cli import main as client_entrypoint
from dusty.schemas.base_schema_class import get_specs_from_path, DustySpecs
from dusty.systems.docker import exec_in_container, get_docker_client, get_container_for_app_or_service, get_docker_env
from dusty.systems.nfs import client as nfs_client
from dusty.systems.nfs import server as nfs_server
from dusty.path import parent_dir
from dusty.subprocess import call_demoted
from .fixtures import basic_specs_fixture
from dusty.log import client_logger, DustyClientTestingSocketHandler
from dusty.memoize import reset_memoize_cache

class TestCaptureHandler(logging.Handler):
    def __init__(self, lst):
        super(TestCaptureHandler, self).__init__()
        self.lst = lst

    def emit(self, record):
        self.lst.append(self.format(record))

class DustyTestCase(TestCase):
    def setUp(self):
        self.temp_config_path = tempfile.mkstemp()[1]
        self.temp_specs_path = tempfile.mkdtemp()
        self.temp_repos_path = tempfile.mkdtemp()

        constants.CONFIG_PATH = self.temp_config_path
        write_default_config()
        save_config_value(constants.CONFIG_SPECS_REPO_KEY, 'github.com/org/dusty-specs')
        override_repo(get_specs_repo().remote_path, self.temp_specs_path)
        basic_specs_fixture()

        self.client_output = []
        self.capture_handler = TestCaptureHandler(self.client_output)
        logging.getLogger(constants.SOCKET_LOGGER_NAME).addHandler(self.capture_handler)
        reset_memoize_cache()

    def tearDown(self):
        os.remove(self.temp_config_path)
        shutil.rmtree(self.temp_specs_path)
        shutil.rmtree(self.temp_repos_path)
        logging.getLogger(constants.SOCKET_LOGGER_NAME).removeHandler(self.capture_handler)
        reset_memoize_cache()

    @nottest
    @patch('dusty.schemas.base_schema_class.get_specs_from_path')
    def make_test_specs(self, specs_dict, fake_specs_from_path):
        fake_specs_from_path.return_value = specs_dict
        return DustySpecs('')

    @property
    def last_client_output(self):
        return self.client_output[-1] if self.client_output else None

class CommandError(Exception):
    pass

class SysExit(Exception):
    pass

class DustyIntegrationTestCase(TestCase):
    """This test case intentionally avoids mocking whenever possible
    in order to get as close as possible to the actual state that
    would be experienced on a system running Dusty. Therefore,
    integration tests are possibly destructive if run on a user's
    machine. To help protect the user from running them accidentally,
    integration tests will refuse to run unless the environment
    variable DUSTY_ALLOW_INTEGRATION_TESTS is set.

    Note that this also assumes it is running against an actual
    Dusty daemon process on the local host."""
    def setUp(self):
        if not os.getenv('DUSTY_ALLOW_INTEGRATION_TESTS'):
            raise RuntimeError('You must set the env var DUSTY_ALLOW_INTEGRATION_TESTS to run integration tests. '
                               'This may affect your local config, do not run integration tests on your actual '
                               "machine unless you know what you're doing!")
        self.handler = DustyClientTestingSocketHandler()
        client_logger.addHandler(self.handler)
        self.previous_config = get_config()
        self._clear_stdout()
        self.overridden_specs_path = tempfile.mkdtemp()
        write_default_config()
        save_config_value(constants.CONFIG_SETUP_KEY, True)
        save_config_value(constants.CONFIG_SPECS_REPO_KEY, 'github.com/gamechanger/dusty-example-specs')

        self.tests_user = os.getenv('DUSTY_INTEGRATION_TESTS_USER', self.current_user)
        save_config_value(constants.CONFIG_MAC_USERNAME_KEY, self.tests_user)

        override_repo(get_specs_repo().remote_path, self.overridden_specs_path)
        self.fake_local_repo_location = '/tmp/fake-repo'
        self._set_up_fake_local_repo('/tmp/fake-repo')
        self._clear_stdout()
        self.exec_docker_processes = []
        reset_memoize_cache()

    def tearDown(self):
        for exec_docker_process in self.exec_docker_processes:
            try:
                os.kill(exec_docker_process.pid)
            except:
                pass
        shutil.rmtree(self.overridden_specs_path)
        # if os.path.exists(constants.REPOS_DIR):
        #     shutil.rmtree(constants.REPOS_DIR)
        if os.path.exists(constants.COMPOSE_DIR):
            shutil.rmtree(constants.COMPOSE_DIR)
        shutil.rmtree('/tmp/fake-repo')
        save_config(self.previous_config)
        self.handler.log_to_client_output = ''
        client_logger.removeHandler(self.handler)
        nfs_client.unmount_all_repos()
        reset_memoize_cache()
        nfs_server._write_exports_config(set())

    def _clear_stdout(self):
        self.stdout_start = len(sys.stdout.getvalue())

    @property
    def current_user(self):
        return getpass.getuser()

    @property
    def CommandError(self):
        """Pure convenience to avoid having to import this explicitly"""
        return CommandError

    @property
    def stdout(self):
        """Returns any stdout that has been generated *since* the last time
        clear_stdout was called."""
        return sys.stdout.getvalue()[self.stdout_start:].strip()

    def exec_docker_patch(self, *args):
        docker_executable = subprocess.check_output('which docker', shell=True).rstrip('\n')
        args = [docker_executable] + [a for a in args]
        self.exec_docker_processes.append(subprocess.Popen(args=args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=get_docker_env()))

    @patch('sys.exit')
    def run_command(self, args, fake_exit, raise_on_error=True):
        """Run a command through the Dusty client entrypoint, e.g. simulating
        the Dusty CLI as close as possible without having to call a subprocess.
        This command raises if the command fails, otherwise it returns the
        stdout generated by the command."""
        with patch('dusty.commands.utils.exec_docker', wraps=self.exec_docker_patch) as fake_exec_docker:
            fake_exit.side_effect = SysExit('sys.exit called')
            self.fake_exec_docker = fake_exec_docker
            sys.argv = ['dusty'] + args.split(' ')
            try:
                client_entrypoint()
            except SysExit:
                pass
            for call in fake_exit.mock_calls:
                name, args, kwargs = call
                if len(args) == 1 and args[0] > 0 and raise_on_error:
                    self._clear_stdout()
                    raise CommandError('Command {} returned with exit code: {}'.format(' '.join(sys.argv), args[0]))
            result = self.stdout
            self._clear_stdout()
            return result

    def _set_up_fake_local_repo(self, path='/tmp/fake-repo'):
        repo = git.Repo.init(path)
        with open(os.path.join(path, 'README.md'), 'w') as f:
            f.write('# {}'.format(path.split('/')[-1]))
        repo.index.add([os.path.join(path, 'README.md')])
        repo.index.commit('Initial commit')

    def _in_same_line(self, string, *values):
        for line in string.splitlines():
            if all(value in line for value in values):
                return True
        return False

    def _container_exists_in_set(self, service_name, include_exited):
        client = get_docker_client()
        container = get_container_for_app_or_service(client, service_name, include_exited=include_exited)
        return bool(container)

    def _image_exists(self, image_to_find):
        client = get_docker_client()
        all_images = client.images(all=True)
        return any(image_to_find in image['RepoTags'] for image in all_images)

    def _file_exists_in_container(self, service_name, file_path):
        call_demoted(['boot2docker', 'ssh', 'sync']) # flush any outstanding writes
        file_exists_str = self.exec_in_container(service_name,
            'sh -c \'[ -f {} ] && echo "yes" || echo "no"\''.format(file_path)).rstrip()
        return file_exists_str == 'yes'

    def exec_in_container(self, service_name, command):
        client = get_docker_client()
        container = get_container_for_app_or_service(client, service_name, raise_if_not_found=True)
        return exec_in_container(client, container, *command.split(' '))

    def remove_path_in_container(self, service_name, file_path):
        self.exec_in_container(service_name, 'rm -rf {}'.format(file_path))

    def write_file_in_container(self, service_name, file_path, contents):
        self.exec_in_container(service_name, 'mkdir -p {}'.format(parent_dir(file_path)))
        self.exec_in_container(service_name, 'sh -c "echo -n {} > {}"'.format(contents, file_path))

    def container_id(self, service_name):
        client = get_docker_client()
        return get_container_for_app_or_service(client, service_name, include_exited=True, raise_if_not_found=True)['Id']

    def inspect_container(self, service_name):
        container_id = self.container_id(service_name)
        client = get_docker_client()
        return client.inspect_container(container_id)

    def remove_container(self, service_name):
        container_id = self.container_id(service_name)
        client = get_docker_client()
        client.remove_container(container_id, force=True)

    @staticmethod
    def retriable_assertion(backoff, max_retries):
        def retriable_fn_wrapper(fn):
            def retriable_fn(*args, **kwargs):
                for i in range(0, max_retries):
                    try:
                        return fn(*args, **kwargs)
                    except AssertionError:
                        print 'Retriable assertion failure {}'.format(i + 1)
                        if i == max_retries - 1:
                            raise
                    time.sleep(backoff)
            return retriable_fn
        return retriable_fn_wrapper

    def assertInSameLine(self, string, *values):
        self.assertTrue(self._in_same_line(string, *values))

    def assertNotInSameLine(self, string, *values):
        self.assertFalse(self._in_same_line(string, *values))

    def assertFileContents(self, file_path, contents):
        with open(file_path, 'r') as f:
            self.assertEqual(f.read(), contents)

    def assertFileContentsInContainer(self, service_name, file_path, contents):
        self.assertEqual(self.exec_in_container(service_name, 'cat {}'.format(file_path)), contents)

    def assertFileNotInContainer(self, service_name, file_path):
        self.assertFalse(self._file_exists_in_container(service_name, file_path))

    def assertFileInContainer(self, service_name, file_path):
        self.assertTrue(self._file_exists_in_container(service_name, file_path))

    def assertContainerRunning(self, service_name):
        self.assertTrue(self._container_exists_in_set(service_name, False))

    def assertContainerIsNotRunning(self, service_name):
        self.assertFalse(self._container_exists_in_set(service_name, False))

    def assertContainerExists(self, service_name):
        self.assertTrue(self._container_exists_in_set(service_name, True))

    def assertContainerDoesNotExist(self, service_name):
        self.assertFalse(self._container_exists_in_set(service_name, True))

    def assertImageExists(self, image):
        self.assertTrue(self._image_exists(image))

    def assertImageDoesNotExist(self, image):
        self.assertFalse(self._image_exists(image))

    def assertExecDocker(self, *args):
        self.fake_exec_docker.assert_called_with(*args)

    def assertConfigValue(self, key, value):
        config = get_config()
        self.assertEqual(config[key], value)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import hashlib
import sqlite3

from settings import *

if not os.path.exists("blog.db"):
    conn = sqlite3.connect("blog.db")
    c = conn.cursor()
    print("Did not find any database file.")
    c.execute('''CREATE TABLE admin_config
             (username text NOT NULL PRIMARY KEY, password text NOT NULL, token text);''')
    print("Creat Admin Config......")
    c.execute('''CREATE TABLE articles
             (id integer NOT NULL PRIMARY KEY autoincrement, title text NOT NULL, content text NOT NULL, tag text NOT NULL, datetime text NOT NULL);''')
    c.execute('''INSERT INTO admin_config VALUES (\"%s\", \"%s\", "token");''' % (login_username, hashlib.md5("admin").hexdigest()))
    c.execute("CREATE UNIQUE INDEX articles_id ON articles(id);")
    print("Creat Article Database......")
    c.execute('''CREATE TABLE pages
             (id integer NOT NULL PRIMARY KEY, title text NOT NULL, content text NOT NULL);''')
    print("Creat Page Database......")
    c.execute('''CREATE TABLE tags
             (id integer NOT NULL PRIMARY KEY autoincrement, name text NOT NULL, article_id integer NOT NULL);''')
    print("Creat Tag Database......")
    c.execute('''''')
    print("Creat Index......")
    conn.commit()
    conn.close()
    print("Successful to creat database file.")
else:
    print("Database File already exists.")
    print("Failed to create database file.")<commit_msg>Python 2.x bug fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import hashlib
import sqlite3

from settings import *

if not os.path.exists("blog.db"):
    conn = sqlite3.connect("blog.db")
    c = conn.cursor()
    print("Did not find any database file.")
    c.execute('''CREATE TABLE admin_config
             (username text NOT NULL PRIMARY KEY, password text NOT NULL, token text);''')
    print("Creat Admin Config......")
    c.execute('''CREATE TABLE articles
             (id integer NOT NULL PRIMARY KEY autoincrement, title text NOT NULL, content text NOT NULL, tag text NOT NULL, datetime text NOT NULL);''')
    c.execute('''INSERT INTO admin_config VALUES (\"%s\", \"%s\", "token");''' % (login_username, hashlib.md5("admin".encode('utf-8')).hexdigest()))
    c.execute("CREATE UNIQUE INDEX articles_id ON articles(id);")
    print("Creat Article Database......")
    c.execute('''CREATE TABLE pages
             (id integer NOT NULL PRIMARY KEY, title text NOT NULL, content text NOT NULL);''')
    print("Creat Page Database......")
    c.execute('''CREATE TABLE tags
             (id integer NOT NULL PRIMARY KEY autoincrement, name text NOT NULL, article_id integer NOT NULL);''')
    print("Creat Tag Database......")
    c.execute('''''')
    print("Creat Index......")
    conn.commit()
    conn.close()
    print("Successful to creat database file.")
else:
    print("Database File already exists.")
    print("Failed to create database file.")<|endoftext|>


<commit_before>#!/usr/bin/python
#coding:utf-8
"""
  GMM classification
  one GMM per class, tuned by maximum likelihood
  至于如何来调每个 GMM中components的个数 这个就如同 k-means如何选择合适的k的大小一样
"""
import numpy as np

X = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5],[4,5,6,7]])

class GMM(object):

	"""docstring for GMM algorithm
       add this for test
    """
	def __init__(self, K_or_centroids, X):
		super(GMM, self).__init__()
		#self.K_or_centroids = K_or_centroids
		self.N, self.D = np.shape(X)
		self.Lprev = -np.inf
		self.L = 0.0
		self.threshold = 1e-7
		self.alpha = 1e-3
		if np.isscalar(K_or_centroids):
                        self.K = K_or_centroids
                        #randomly pick centroids
                        rndp = np.random.permutation(self.N) # 选 top-K 个 points 做 centroids
#			rndp = np.array([2,4,0,1,3])
                        self.centroids = X[rndp[0:self.K]] # 前 K 行
                else:
                        self.K = np.shape(K_or_centroids)[0]
                        self.centroids = K_or_centroids

		self.pMiu = self.centroids
                self.pPi = np.zeros((1,self.K))
                self.pSigma = np.zeros((self.K, self.D, self.D))

	def gmm(self,X, K_or_centroids):# Here, X is for training
		"""
		X: N by D matrix
		K_or_centroids:
		Px: N x K matrix indicating the probability of each component generating each point
		MODEL: a structure containing the parameters for a GMM:
			MODEL.Miu: K x D matrix
			MODEL.Sigma: D x D x K matrix
			MODEL.Pi: 1 x K vector
		"""

		# initial values
		[self.pMiu, self.pPi, self.pSigma] = self.init_params(X)
		it = 0
		while True:
			print "it: ", it
			it+=1
			Px = self.calc_prob(X) # 计算后验概率 N by K

			# new value for pGamma (N by K)
			# pGamma[i,k] = \frac{Px[i,k]}{sum_{j=1}^K(Px(i,j))}
			# i = 0,1,...,N-1  k = 0,1,...,K-1
			# pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i的多少比例(比如 0.3个point i)分到 component k 中去
			pGamma = np.multiply(Px , np.kron(np.ones((self.N,1)),self.pPi)) # instead of repmat in matlab
			# pGamma N by K
			pGamma = pGamma / np.kron(np.ones((1,self.K)),np.transpose(np.mat(np.sum(pGamma,1))))

			# new value for parameters of each component
			Nk = np.sum(pGamma,0)
			print "Nk[0]: ", np.array(Nk)[0]
			Nk = np.array(Nk)[0]
			self.pMiu = np.diag(1./Nk)* np.transpose(pGamma) * X

			print "self.pMiu: ", self.pMiu

			self.pPi = Nk / self.N

			print "self.pPi: ", self.pPi

			for kk in range(self.K):
				Xshift = X - np.kron(np.ones((self.N,1)), self.pMiu[kk,:])
				print "Xshift: ", Xshift, np.shape(Xshift)
				candidateSigma=(np.mat(np.transpose(Xshift)) * np.mat(np.diag(np.array(pGamma[:,kk])[:,0])) * np.mat(Xshift)) / Nk[kk]
				print "np.diag(np.array(pGamma[:,kk])[:,0]: ", np.diag(np.array(pGamma[:,kk])[:,0])
				print "Nk[kk]: ", Nk[kk], np.shape(Nk[kk])

				print "candidateSigma: ", candidateSigma
				self.pSigma[kk,:,:] = self.regular(candidateSigma)
				print "regular candidateSigma: ", self.regular(candidateSigma)

			print "self.pSigma: ", self.pSigma

			# check for convergence
			self.L = np.sum(np.log(Px*np.transpose(np.mat(self.pPi))))
			print "self.L, self.Lprev: ", self.L, self.Lprev

			if (self.L - self.Lprev < self.threshold) or it >= 200:
				print "self.L, self.Lprev: ", self.L, self.Lprev
				break
			self.Lprev = self.L

	def init_params(self,X): # X is trainX
		pMiu = self.centroids
		pPi = np.zeros((1,self.K))
		pSigma = np.zeros((self.K, self.D, self.D))
		# hard assignment x to each centroids
		distmat = np.kron(np.ones((1,self.K)), np.sum(np.multiply(X,X),1)) + np.kron(np.ones((self.N,1)),np.transpose(np.sum(np.multiply(pMiu,pMiu),1))) - 2*X*np.transpose(pMiu)
		labels = np.array(np.transpose(np.argmin(distmat,1)))[0]# transpose(N by 1) -> 1 by N

		print '---init---'
		for k in range(self.K):
	#		print "labels: ",labels,np.shape(labels)
			Xk = X[labels==k]
	#		print "Xk: ", np.shape(Xk)
			print Xk
	#		print "np.shape(Xk)[0]: ", np.shape(Xk)[0]
	#		print "k: ",k
	#		print "pPi: ", np.shape(pPi)
			pPi[0,k] = np.shape(Xk)[0] / (self.N+0.0)
			#print "pSigma: ", pSigma, np.shape(pSigma)
	#		print "Xk: ",Xk
	#		print "cov: ",np.cov(Xk,rowvar=0)
			covm = np.cov(Xk,rowvar=0)

			pSigma[k,:,:] = self.regular(np.cov(Xk,rowvar=0))
			print "k = ",k, "pSigma[k,:,:]: ", pSigma[k,:,:]
		return (pMiu, pPi, pSigma)

	def regular(self,sigma):
		if len(np.shape(sigma))==0:
                	sigma = np.array([sigma])
		else :
			sigma = np.array(sigma)
		sigma[np.diag(np.diag(sigma)<self.alpha)] = 0.0

		return  sigma + self.alpha * np.identity(self.D)

	def calc_prob(self,X):
		# 返回Px 1 by N
		# N(x_i|\mu_k,\Sigma_k) = \frac{1}{(2*pi)^{\frac{d}{2}} |\Sigma|^\frac{1}{2}}
				# exp\{ -\frac{1}{2}(x-\mu)^T \Sigma^{-1}(x-\mu)\}
		Px = np.zeros((self.N,self.K))

		for k in range(self.K):
			# 计算 N(X|\mu_k,\Sigma_k)
			Xshift = X - np.kron(np.ones((self.N,1)),self.pMiu[k]) # Xshift N by D
	#		print "Xshift: ", Xshift
	#			print "pSigma[k,:,:]: ", self.pSigma[k,:,:]
			inv_pSigma = np.linalg.inv(self.pSigma[k,:,:]) # D by D
			#tmp = np.sum(np.multiply((Xshift * inv_pSigma),Xshift) , 1) # sum((X*D).*X,2)
	#		print "Xshift: ", np.shape(Xshift), "inv_pSigma: ",np.shape(inv_pSigma), "diagonal: ",(Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
			tmp = (Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
	#		print "tmp: ",tmp, np.shape(tmp)
			tmp = tmp[0] # 1 by N
	#		print "tmp[0]: ", tmp
	#		print "inv_pSigma: ", inv_pSigma
	#		print "np.linalg.det(inv_pSigma): ", np.linalg.det(inv_pSigma)
			coef = (2*np.pi)**(-self.D/2)*np.sqrt(np.linalg.det(inv_pSigma))
			#print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
			Px[:,k] = coef*np.exp(-0.5*np.array(tmp))
	#		print "np.array(tmp): ",np.array(tmp)
	#		print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
		print "calc_prob Px: ", Px
		return Px
		# Px[i,k] = N(x_i|\mu_k, \Sigma_k)
		# pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i的多少比例(比如 0.3个point i)分到 component k 中去

def calc_prob(gmmModel,testX):
        # 返回Px 1 by N
        # N(x_i|\mu_k,\Sigma_k) = \frac{1}{(2*pi)^{\frac{d}{2}} |\Sigma|^\frac{1}{2}}
                        # exp\{ -\frac{1}{2}(x-\mu)^T \Sigma^{-1}(x-\mu)\}
        N = np.shape(testX)[0]
        Px = np.zeros((N,gmmModel.K))

        for k in range(gmmModel.K):
                # 计算 N(X|\mu_k,\Sigma_k)
                Xshift = testX - np.kron(np.ones((N,1)),gmmModel.pMiu[k]) # Xshift N by D
#               print "Xshift: ", Xshift
#                       print "pSigma[k,:,:]: ", self.pSigma[k,:,:]
                inv_pSigma = np.linalg.inv(gmmModel.pSigma[k,:,:]) # D by D
                #tmp = np.sum(np.multiply((Xshift * inv_pSigma),Xshift) , 1) # sum((X*D).*X,2)
#               print "Xshift: ", np.shape(Xshift), "inv_pSigma: ",np.shape(inv_pSigma), "diagonal: ",(Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
                tmp = (Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
#               print "tmp: ",tmp, np.shape(tmp)
                tmp = tmp[0] # 1 by N
#               print "tmp[0]: ", tmp
#               print "inv_pSigma: ", inv_pSigma
#               print "np.linalg.det(inv_pSigma): ", np.linalg.det(inv_pSigma)
                coef = (2*np.pi)**(-gmmModel.D/2)*np.sqrt(np.linalg.det(inv_pSigma))
                #print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
                Px[:,k] = coef*np.exp(-0.5*np.array(tmp))
#               print "np.array(tmp): ",np.array(tmp)
#               print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
        print "calc_prob Px: ", Px
        return Px
        # Px[i,k] = N(x_i|\mu_k, \Sigma_k)
        # pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i>的多少比例(比如 0.3个point i)分到 component k 中去

def predictIndx(trainedGMMinstance,newX):
	"""
	Px: N by K 取第i行最大的值的 index 作为 point i 的 component
	"""
	Px = calc_prob(trainedGMMinstance,newX)
	print "np.argmax(Px,1): ", np.argmax(Px,1)
	return np.argmax(Px,1)

def predictValue(trainedGMMinstance, newX):
	Px = calc_prob(trainedGMMinstance,newX)
	print "np.max(Px,1): ", np.max(Px,1)
        return np.max(Px,1)

def evaluate(re,ans):# re, ans are two list
    correct = 0
    for i in range(len(re)):
            if re[i] + 1 == ans[i]:
                    correct += 1
    acc = (0.0+correct) / len(re)
    return acc

if __name__=="__main__":
	# prepare data
    X = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5],[4,5,6,7]])
    #N,D = np.shape(X)
    gmm = GMM(3,X)
    gmm.gmm(X,3)
    testX = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5]])
    predictIndx(gmm,testX)
    predictValue(gmm, testX)
<commit_msg>remove test<commit_after>#!/usr/bin/python
#coding:utf-8
"""
  GMM classification
  one GMM per class, tuned by maximum likelihood
  至于如何来调每个 GMM中components的个数 这个就如同 k-means如何选择合适的k的大小一样
"""
import numpy as np

X = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5],[4,5,6,7]])

class GMM(object):

	"""docstring for GMM algorithm"""
	def __init__(self, K_or_centroids, X):
		super(GMM, self).__init__()
		#self.K_or_centroids = K_or_centroids
		self.N, self.D = np.shape(X)
		self.Lprev = -np.inf
		self.L = 0.0
		self.threshold = 1e-7
		self.alpha = 1e-3
		if np.isscalar(K_or_centroids):
                        self.K = K_or_centroids
                        #randomly pick centroids
                        rndp = np.random.permutation(self.N) # 选 top-K 个 points 做 centroids
#			rndp = np.array([2,4,0,1,3])
                        self.centroids = X[rndp[0:self.K]] # 前 K 行
                else:
                        self.K = np.shape(K_or_centroids)[0]
                        self.centroids = K_or_centroids

		self.pMiu = self.centroids
                self.pPi = np.zeros((1,self.K))
                self.pSigma = np.zeros((self.K, self.D, self.D))

	def gmm(self,X, K_or_centroids):# Here, X is for training
		"""
		X: N by D matrix
		K_or_centroids:
		Px: N x K matrix indicating the probability of each component generating each point
		MODEL: a structure containing the parameters for a GMM:
			MODEL.Miu: K x D matrix
			MODEL.Sigma: D x D x K matrix
			MODEL.Pi: 1 x K vector
		"""

		# initial values
		[self.pMiu, self.pPi, self.pSigma] = self.init_params(X)
		it = 0
		while True:
			print "it: ", it
			it+=1
			Px = self.calc_prob(X) # 计算后验概率 N by K

			# new value for pGamma (N by K)
			# pGamma[i,k] = \frac{Px[i,k]}{sum_{j=1}^K(Px(i,j))}
			# i = 0,1,...,N-1  k = 0,1,...,K-1
			# pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i的多少比例(比如 0.3个point i)分到 component k 中去
			pGamma = np.multiply(Px , np.kron(np.ones((self.N,1)),self.pPi)) # instead of repmat in matlab
			# pGamma N by K
			pGamma = pGamma / np.kron(np.ones((1,self.K)),np.transpose(np.mat(np.sum(pGamma,1))))

			# new value for parameters of each component
			Nk = np.sum(pGamma,0)
			print "Nk[0]: ", np.array(Nk)[0]
			Nk = np.array(Nk)[0]
			self.pMiu = np.diag(1./Nk)* np.transpose(pGamma) * X

			print "self.pMiu: ", self.pMiu

			self.pPi = Nk / self.N

			print "self.pPi: ", self.pPi

			for kk in range(self.K):
				Xshift = X - np.kron(np.ones((self.N,1)), self.pMiu[kk,:])
				print "Xshift: ", Xshift, np.shape(Xshift)
				candidateSigma=(np.mat(np.transpose(Xshift)) * np.mat(np.diag(np.array(pGamma[:,kk])[:,0])) * np.mat(Xshift)) / Nk[kk]
				print "np.diag(np.array(pGamma[:,kk])[:,0]: ", np.diag(np.array(pGamma[:,kk])[:,0])
				print "Nk[kk]: ", Nk[kk], np.shape(Nk[kk])

				print "candidateSigma: ", candidateSigma
				self.pSigma[kk,:,:] = self.regular(candidateSigma)
				print "regular candidateSigma: ", self.regular(candidateSigma)

			print "self.pSigma: ", self.pSigma

			# check for convergence
			self.L = np.sum(np.log(Px*np.transpose(np.mat(self.pPi))))
			print "self.L, self.Lprev: ", self.L, self.Lprev

			if (self.L - self.Lprev < self.threshold) or it >= 200:
				print "self.L, self.Lprev: ", self.L, self.Lprev
				break
			self.Lprev = self.L

	def init_params(self,X): # X is trainX
		pMiu = self.centroids
		pPi = np.zeros((1,self.K))
		pSigma = np.zeros((self.K, self.D, self.D))
		# hard assignment x to each centroids
		distmat = np.kron(np.ones((1,self.K)), np.sum(np.multiply(X,X),1)) + np.kron(np.ones((self.N,1)),np.transpose(np.sum(np.multiply(pMiu,pMiu),1))) - 2*X*np.transpose(pMiu)
		labels = np.array(np.transpose(np.argmin(distmat,1)))[0]# transpose(N by 1) -> 1 by N

		print '---init---'
		for k in range(self.K):
	#		print "labels: ",labels,np.shape(labels)
			Xk = X[labels==k]
	#		print "Xk: ", np.shape(Xk)
			print Xk
	#		print "np.shape(Xk)[0]: ", np.shape(Xk)[0]
	#		print "k: ",k
	#		print "pPi: ", np.shape(pPi)
			pPi[0,k] = np.shape(Xk)[0] / (self.N+0.0)
			#print "pSigma: ", pSigma, np.shape(pSigma)
	#		print "Xk: ",Xk
	#		print "cov: ",np.cov(Xk,rowvar=0)
			covm = np.cov(Xk,rowvar=0)

			pSigma[k,:,:] = self.regular(np.cov(Xk,rowvar=0))
			print "k = ",k, "pSigma[k,:,:]: ", pSigma[k,:,:]
		return (pMiu, pPi, pSigma)

	def regular(self,sigma):
		if len(np.shape(sigma))==0:
                	sigma = np.array([sigma])
		else :
			sigma = np.array(sigma)
		sigma[np.diag(np.diag(sigma)<self.alpha)] = 0.0

		return  sigma + self.alpha * np.identity(self.D)

	def calc_prob(self,X):
		# 返回Px 1 by N
		# N(x_i|\mu_k,\Sigma_k) = \frac{1}{(2*pi)^{\frac{d}{2}} |\Sigma|^\frac{1}{2}}
				# exp\{ -\frac{1}{2}(x-\mu)^T \Sigma^{-1}(x-\mu)\}
		Px = np.zeros((self.N,self.K))

		for k in range(self.K):
			# 计算 N(X|\mu_k,\Sigma_k)
			Xshift = X - np.kron(np.ones((self.N,1)),self.pMiu[k]) # Xshift N by D
	#		print "Xshift: ", Xshift
	#			print "pSigma[k,:,:]: ", self.pSigma[k,:,:]
			inv_pSigma = np.linalg.inv(self.pSigma[k,:,:]) # D by D
			#tmp = np.sum(np.multiply((Xshift * inv_pSigma),Xshift) , 1) # sum((X*D).*X,2)
	#		print "Xshift: ", np.shape(Xshift), "inv_pSigma: ",np.shape(inv_pSigma), "diagonal: ",(Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
			tmp = (Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
	#		print "tmp: ",tmp, np.shape(tmp)
			tmp = tmp[0] # 1 by N
	#		print "tmp[0]: ", tmp
	#		print "inv_pSigma: ", inv_pSigma
	#		print "np.linalg.det(inv_pSigma): ", np.linalg.det(inv_pSigma)
			coef = (2*np.pi)**(-self.D/2)*np.sqrt(np.linalg.det(inv_pSigma))
			#print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
			Px[:,k] = coef*np.exp(-0.5*np.array(tmp))
	#		print "np.array(tmp): ",np.array(tmp)
	#		print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
		print "calc_prob Px: ", Px
		return Px
		# Px[i,k] = N(x_i|\mu_k, \Sigma_k)
		# pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i的多少比例(比如 0.3个point i)分到 component k 中去

def calc_prob(gmmModel,testX):
        # 返回Px 1 by N
        # N(x_i|\mu_k,\Sigma_k) = \frac{1}{(2*pi)^{\frac{d}{2}} |\Sigma|^\frac{1}{2}}
                        # exp\{ -\frac{1}{2}(x-\mu)^T \Sigma^{-1}(x-\mu)\}
        N = np.shape(testX)[0]
        Px = np.zeros((N,gmmModel.K))

        for k in range(gmmModel.K):
                # 计算 N(X|\mu_k,\Sigma_k)
                Xshift = testX - np.kron(np.ones((N,1)),gmmModel.pMiu[k]) # Xshift N by D
#               print "Xshift: ", Xshift
#                       print "pSigma[k,:,:]: ", self.pSigma[k,:,:]
                inv_pSigma = np.linalg.inv(gmmModel.pSigma[k,:,:]) # D by D
                #tmp = np.sum(np.multiply((Xshift * inv_pSigma),Xshift) , 1) # sum((X*D).*X,2)
#               print "Xshift: ", np.shape(Xshift), "inv_pSigma: ",np.shape(inv_pSigma), "diagonal: ",(Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
                tmp = (Xshift * inv_pSigma * np.transpose(Xshift)).diagonal()
#               print "tmp: ",tmp, np.shape(tmp)
                tmp = tmp[0] # 1 by N
#               print "tmp[0]: ", tmp
#               print "inv_pSigma: ", inv_pSigma
#               print "np.linalg.det(inv_pSigma): ", np.linalg.det(inv_pSigma)
                coef = (2*np.pi)**(-gmmModel.D/2)*np.sqrt(np.linalg.det(inv_pSigma))
                #print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
                Px[:,k] = coef*np.exp(-0.5*np.array(tmp))
#               print "np.array(tmp): ",np.array(tmp)
#               print "Px[:,k]: ",Px[:,k], np.shape(Px[:,k]),"tmp: ", np.shape(tmp)
        print "calc_prob Px: ", Px
        return Px
        # Px[i,k] = N(x_i|\mu_k, \Sigma_k)
        # pGamma[i,k] 对Px[i,k] 进行归一化 存的是 point i 由 component k 生成的概率 或者说把 point i>的多少比例(比如 0.3个point i)分到 component k 中去

def predictIndx(trainedGMMinstance,newX):
	"""
	Px: N by K 取第i行最大的值的 index 作为 point i 的 component
	"""
	Px = calc_prob(trainedGMMinstance,newX)
	print "np.argmax(Px,1): ", np.argmax(Px,1)
	return np.argmax(Px,1)

def predictValue(trainedGMMinstance, newX):
	Px = calc_prob(trainedGMMinstance,newX)
	print "np.max(Px,1): ", np.max(Px,1)
        return np.max(Px,1)

def evaluate(re,ans):# re, ans are two list
    correct = 0
    for i in range(len(re)):
            if re[i] + 1 == ans[i]:
                    correct += 1
    acc = (0.0+correct) / len(re)
    return acc

if __name__=="__main__":
	# prepare data
    X = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5],[4,5,6,7]])
    #N,D = np.shape(X)
    gmm = GMM(3,X)
    gmm.gmm(X,3)
    testX = np.mat([[1,2,3,4],[4,3,2,1],[5,6,7,8],[8,7,6,5]])
    predictIndx(gmm,testX)
    predictValue(gmm, testX)
<|endoftext|>


<commit_before># Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
"""Elements' labels and properties for objects."""
# pylint: disable=too-few-public-methods
# pylint: disable=invalid-name

from lib.constants import objects, roles

class Lhn(object):
  """Elements' labels and properties for LHN menu."""
  class __metaclass__(type):
    def __init__(cls, *args):
      for object_ in objects.ALL_PLURAL:
        setattr(cls, object_, object_.lower())
      cls.DIRECTIVES_MEMBERS = (
          cls.REGULATIONS,
          cls.POLICIES,
          cls.STANDARDS,
          cls.CONTRACTS,
          cls.CLAUSES,
          cls.SECTIONS)
      cls.CONTROLS_OR_OBJECTIVES_MEMBERS = (
          cls.CONTROLS,
          cls.OBJECTIVES)
      cls.PEOPLE_OR_GROUPS_MEMBERS = (
          cls.PEOPLE,
          cls.ORG_GROUPS,
          cls.VENDORS,
          cls.ACCESS_GROUPS)
      cls.ASSETS_OR_BUSINESS_MEMBERS = (
          cls.SYSTEMS,
          cls.PROCESSES,
          cls.DATA_ASSETS,
          cls.PRODUCTS,
          cls.PROJECTS,
          cls.FACILITIES,
          cls.MARKETS)
      cls.RISKS_OR_THREATS_MEMBERS = (
          cls.RISKS,
          cls.THREATS)
  CONTROLS_OR_OBJECTIVES = "controls_or_objectives"
  PEOPLE_OR_GROUPS = "people_or_groups"
  ASSETS_OR_BUSINESS = "assets_or_business"
  RISKS_OR_THREATS = "risks_or_threats"

class WidgetBar(object):
  """Elements' labels and properties for Generic widget bar."""
  INFO = "Info"

  class __metaclass__(type):
    def __init__(cls, *args):
      for object_ in objects.ALL_PLURAL:
        setattr(cls, object_, object_.lower())

class AdminWidgetRoles(object):
  """Elements' labels (role scopes) and properties for Roles widget
 at Admin Dashboard.
 """
  _ADMIN_SCOPE = roles.ADMIN.upper()
  _SYS_SCOPE = roles.SYSTEM.upper()
  _PRG_SCOPE = roles.PRIVATE_PROGRAM.upper()
  _WF_SCOPE = roles.WORKFLOW.upper()
  # role scopes
  ROLE_SCOPE_ADMINISTRATOR = (roles.ADMINISTRATOR, _ADMIN_SCOPE)
  ROLE_SCOPE_CREATOR = (roles.CREATOR, _SYS_SCOPE)
  ROLE_SCOPE_EDITOR = (roles.EDITOR, _SYS_SCOPE)
  ROLE_SCOPE_READER = (roles.READER, _SYS_SCOPE)
  ROLE_SCOPE_PROGRAM_EDITOR = (roles.PROGRAM_EDITOR, _PRG_SCOPE)
  ROLE_SCOPE_PROGRAM_MANAGER = (roles.PROGRAM_MANAGER, _PRG_SCOPE)
  ROLE_SCOPE_PROGRAM_READER = (roles.PROGRAM_READER, _PRG_SCOPE)
  ROLE_SCOPE_WORKFLOW_MEMBER = (roles.WORKFLOW_MEMBER, _WF_SCOPE)
  ROLE_SCOPE_WORKFLOW_MANAGER = (roles.WORKFLOW_MANAGER, _WF_SCOPE)
  ROLE_SCOPES_LIST = [ROLE_SCOPE_ADMINISTRATOR,
                      ROLE_SCOPE_CREATOR,
                      ROLE_SCOPE_EDITOR,
                      ROLE_SCOPE_PROGRAM_EDITOR,
                      ROLE_SCOPE_PROGRAM_MANAGER,
                      ROLE_SCOPE_PROGRAM_READER,
                      ROLE_SCOPE_READER,
                      ROLE_SCOPE_WORKFLOW_MEMBER,
                      ROLE_SCOPE_WORKFLOW_MANAGER]
  ROLE_SCOPES_DICT = dict(ROLE_SCOPES_LIST)

class AdminWidgetEvents(object):
  """Elements' labels and properties (regular expression) for Event widget
 at Admin Dashboard.
 """
  WIDGET_HEADER = "Events"
  TREE_VIEW_ROW_REGEXP = r"^.+\s(by.+)\son\s" + \
      r"(\d{2}/\d{2}/\d{4}\s\d{2}:\d{2}:\d{2}\s[A,P]M)"

class AdminWidgetCustomAttrs(object):
  """Elements' labels (custom attributes scopes) for Custom Attributes
 widget at Admin Dashboard.
 """
  # possible types of custom attributes
  WIDGET_HEADER = "Custom Attributes"
  TEXT = "Text"
  RICH_TEXT = "Rich Text"
  DATE = "Date"
  CHECKBOX = "Checkbox"
  DROPDOWN = "Dropdown"
  PERSON = "Map:Person"

  ALL_ATTRS_TYPES = (TEXT, RICH_TEXT, DATE, CHECKBOX, DROPDOWN, PERSON)

class Common(object):
  """Common elements' labels and properties for objects."""
  TITLE = "Title"
  DESCRIPTION = "Description"
  CODE = "Code"
  TRUE = "true"
  FALSE = "false"

class Base(object):
  """Base elements' labels and properties for objects."""
  WIDGET_INFO_HEADER_FORMAT = "{} Info"
  TYPE = "Type"
  STATE = "State"
  REFERENCE_URL = "Reference URL"
  EFFECTIVE_DATE = "Effective Date"
  STOP_DATE = "Stop Date"
  URL = "URL"

class CommonModalCreate(object):
  """Common elements' labels and properties for Modal to Create object.
 """
  HIDE_ALL_OPT_FIELDS = "Hide all optional fields"
  SHOW_ALL_OPT_FIELDS = "Show all optional fields"
  SAVE_AND_CLOSE = "Save & Close"

class CommonModalSetVisibleFields(object):
  """Common elements' labels and properties for Modal widow that Select visible
 fields for Tree View.
 """
  MODAL_HEADER_FORMAT = "Set visible fields for {}"
  TITLE = Common.TITLE
  CODE = Common.CODE
  STATE = Base.STATE
  LAST_UPDATED = "Last Updated"
  SET_FIELDS = "Set Fields"

class TransformationSetVisibleFields(CommonModalSetVisibleFields):
  """To transformation elements' labels and properties for Modal to Set
 visible fields for object as Tree View headers.
 """
  OWNER = "Owner"
  VERIFIED = "Verified"
  STATUS = "Status"
  AUDIT_LEAD = "Audit Lead"
  MANAGER = "Manager"
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT

class CommonProgram(Common):
  """Common elements' labels and properties for Programs objects."""
  # pylint: disable=too-many-instance-attributes
  PROGRAM = objects.get_normal_form(objects.get_singular(objects.PROGRAMS))
  TITLE = Common.TITLE
  MANAGER = "Manager"
  PROGRAM_URL = "Program URL"
  REFERENCE_URL = Base.REFERENCE_URL
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  NOTES = "Notes"
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  STATE = Base.STATE

class CommonAudit(Common):
  """Common elements' labels and properties for Audits objects."""
  # pylint: disable=too-many-instance-attributes
  AUDIT = objects.get_normal_form(objects.get_singular(objects.AUDITS))
  TITLE = Common.TITLE
  STATUS = "Status"
  PLANNED_START_DATE = "Planned Start Date"
  PLANNED_END_DATE = "Planned End Date"
  PLANNED_REPORT_PERIOD = "Planned Report Period"
  AUDIT_LEAD = "Audit Lead"
  AUDIT_FIRM = " Audit Firm"
  AUDITORS = "Auditors"
  ADD_AUDITOR = "+ Add Auditor"
  AUDIT_FOLDER = "Audit Folder"
  ASSIGN_FOLDER = "Assign folder"

class CommonControl(object):
  """Common elements' labels and properties for Controls objects."""
  CONTROL = objects.get_normal_form(objects.get_singular(objects.CONTROLS))

class CommonAssessment(object):
  """Common elements' labels and properties for Assessments objects."""
  ASMT = objects.get_normal_form(objects.get_singular(objects.ASSESSMENTS))

class CommonAssessmentTemplate(object):
  """Common elements' labels and properties for Assessment Templates objects.
 """
  ASMT_TMPL = objects.get_normal_form(
      objects.get_singular(objects.ASSESSMENT_TEMPLATES))

class CommonIssue(object):
  """Common elements' labels and properties for Issues objects."""
  ISSUE = objects.get_normal_form(objects.get_singular(objects.ISSUES))

class ObjectStates(object):
  """States for objects."""
  DRAFT = "Draft"
  DEPRECATED = "Deprecated"
  ACTIVE = "Active"

class BaseStates(object):
  """Common states for Audit and Assessment objects."""
  IN_PROGRESS = "In progress"
  COMPLETED = "Completed"

class AuditStates(BaseStates):
  """States for Audits objects."""
  PLANNED = "Planned"
  MANAGER_REVIEW = "Manager Review"
  READY_FOR_EXT_REVIEW = "Ready for External Review"

class AssessmentStates(BaseStates):
  """States for Assessments objects."""
  NOT_STARTED = "Not Started"
  READY_FOR_REVIEW = "Ready for Review"
  VERIFIED = "Verified"

class IssueStates(ObjectStates):
  """States for Issues objects."""

class ProgramInfoWidget(CommonProgram):
  """Elements' labels and properties for Programs Info widgets."""
  WIDGET_HEADER = Base.WIDGET_INFO_HEADER_FORMAT.format(CommonProgram.PROGRAM)

class AuditInfoWidget(CommonAudit):
  """Elements' labels and properties for Audits Info widgets."""
  WIDGET_HEADER = Base.WIDGET_INFO_HEADER_FORMAT.format(CommonAudit.AUDIT)
  TITLE_UPPER = CommonAudit.TITLE.upper()
  AUDIT_LEAD_UPPER = CommonAudit.AUDIT_LEAD.upper()
  CODE_UPPER = CommonAudit.CODE.upper()

class AssessmentTemplateModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Assessment Templates.
 """
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonAssessmentTemplate.ASMT_TMPL)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE,
      CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.LAST_UPDATED)

class AssessmentModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Assessments.
 """
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonAssessment.ASMT)
  VERIFIED = TransformationSetVisibleFields.VERIFIED
  CONCLUSION_DESIGN = "Conclusion: Design"
  CONCLUSION_OPERATION = "Conclusion: Operation"
  FINISHED_DATE = "Finished Date"
  VERIFIED_DATE = "Verified Date"
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  TYPE = Base.TYPE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.STATE, VERIFIED,
      CommonModalSetVisibleFields.LAST_UPDATED)

class ControlModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Controls.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonControl.CONTROL)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  KIND_NATURE = "Kind/Nature"
  FRAUD_RELATED = "Fraud Related"
  SIGNIFICANCE = "Significance"
  TYPE_MEANS = "Type/Means"
  FREQUENCY = "Frequency"
  ASSERTIONS = "Assertions"
  CATEGORIES = "Categories"
  PRINCIPAL_ASSIGNEE = roles.PRINCIPAL_ASSIGNEE
  SECONDARY_ASSIGNEE = roles.SECONDARY_ASSIGNEE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, OWNER,
      CommonModalSetVisibleFields.CODE, CommonModalSetVisibleFields.STATE,
      PRIMARY_CONTACT)

class IssueModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Issues.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonIssue.ISSUE)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  REVIEW_STATE = "Review State"
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, OWNER,
      CommonModalSetVisibleFields.CODE, CommonModalSetVisibleFields.STATE,
      CommonModalSetVisibleFields.LAST_UPDATED)

class ProgramModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Programs.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonProgram.PROGRAM)
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  REVIEW_STATE = "Review State"
  MANAGER = CommonProgram.MANAGER
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.STATE, PRIMARY_CONTACT, MANAGER)
<commit_msg>Update of selenium tests<commit_after># Copyright (C) 2017 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
"""Elements' labels and properties for objects."""
# pylint: disable=too-few-public-methods
# pylint: disable=invalid-name

from lib.constants import objects, roles

class Lhn(object):
  """Elements' labels and properties for LHN menu."""
  class __metaclass__(type):
    def __init__(cls, *args):
      for object_ in objects.ALL_PLURAL:
        setattr(cls, object_, object_.lower())
      cls.DIRECTIVES_MEMBERS = (
          cls.REGULATIONS,
          cls.POLICIES,
          cls.STANDARDS,
          cls.CONTRACTS,
          cls.CLAUSES,
          cls.SECTIONS)
      cls.CONTROLS_OR_OBJECTIVES_MEMBERS = (
          cls.CONTROLS,
          cls.OBJECTIVES)
      cls.PEOPLE_OR_GROUPS_MEMBERS = (
          cls.PEOPLE,
          cls.ORG_GROUPS,
          cls.VENDORS,
          cls.ACCESS_GROUPS)
      cls.ASSETS_OR_BUSINESS_MEMBERS = (
          cls.SYSTEMS,
          cls.PROCESSES,
          cls.DATA_ASSETS,
          cls.PRODUCTS,
          cls.PROJECTS,
          cls.FACILITIES,
          cls.MARKETS)
      cls.RISKS_OR_THREATS_MEMBERS = (
          cls.RISKS,
          cls.THREATS)
  CONTROLS_OR_OBJECTIVES = "controls_or_objectives"
  PEOPLE_OR_GROUPS = "people_or_groups"
  ASSETS_OR_BUSINESS = "assets_or_business"
  RISKS_OR_THREATS = "risks_or_threats"

class WidgetBar(object):
  """Elements' labels and properties for Generic widget bar."""
  INFO = "Info"

  class __metaclass__(type):
    def __init__(cls, *args):
      for object_ in objects.ALL_PLURAL:
        setattr(cls, object_, object_.lower())

class AdminWidgetRoles(object):
  """Elements' labels (role scopes) and properties for Roles widget
 at Admin Dashboard.
 """
  _ADMIN_SCOPE = roles.ADMIN.upper()
  _SYS_SCOPE = roles.SYSTEM.upper()
  _PRG_SCOPE = roles.PRIVATE_PROGRAM.upper()
  _WF_SCOPE = roles.WORKFLOW.upper()
  # role scopes
  ROLE_SCOPE_ADMINISTRATOR = (roles.ADMINISTRATOR, _ADMIN_SCOPE)
  ROLE_SCOPE_CREATOR = (roles.CREATOR, _SYS_SCOPE)
  ROLE_SCOPE_EDITOR = (roles.EDITOR, _SYS_SCOPE)
  ROLE_SCOPE_READER = (roles.READER, _SYS_SCOPE)
  ROLE_SCOPE_PROGRAM_EDITOR = (roles.PROGRAM_EDITOR, _PRG_SCOPE)
  ROLE_SCOPE_PROGRAM_MANAGER = (roles.PROGRAM_MANAGER, _PRG_SCOPE)
  ROLE_SCOPE_PROGRAM_READER = (roles.PROGRAM_READER, _PRG_SCOPE)
  ROLE_SCOPE_WORKFLOW_MEMBER = (roles.WORKFLOW_MEMBER, _WF_SCOPE)
  ROLE_SCOPE_WORKFLOW_MANAGER = (roles.WORKFLOW_MANAGER, _WF_SCOPE)
  ROLE_SCOPES_LIST = [ROLE_SCOPE_ADMINISTRATOR,
                      ROLE_SCOPE_CREATOR,
                      ROLE_SCOPE_EDITOR,
                      ROLE_SCOPE_PROGRAM_EDITOR,
                      ROLE_SCOPE_PROGRAM_MANAGER,
                      ROLE_SCOPE_PROGRAM_READER,
                      ROLE_SCOPE_READER,
                      ROLE_SCOPE_WORKFLOW_MEMBER,
                      ROLE_SCOPE_WORKFLOW_MANAGER]
  ROLE_SCOPES_DICT = dict(ROLE_SCOPES_LIST)

class AdminWidgetEvents(object):
  """Elements' labels and properties (regular expression) for Event widget
 at Admin Dashboard.
 """
  WIDGET_HEADER = "Events"
  TREE_VIEW_ROW_REGEXP = r"^.+\s(by.+)\son\s" + \
      r"(\d{2}/\d{2}/\d{4}\s\d{2}:\d{2}:\d{2}\s[A,P]M)"

class AdminWidgetCustomAttrs(object):
  """Elements' labels (custom attributes scopes) for Custom Attributes
 widget at Admin Dashboard.
 """
  # possible types of custom attributes
  WIDGET_HEADER = "Custom Attributes"
  TEXT = "Text"
  RICH_TEXT = "Rich Text"
  DATE = "Date"
  CHECKBOX = "Checkbox"
  DROPDOWN = "Dropdown"
  PERSON = "Map:Person"

  ALL_ATTRS_TYPES = (TEXT, RICH_TEXT, DATE, CHECKBOX, DROPDOWN, PERSON)

class Common(object):
  """Common elements' labels and properties for objects."""
  TITLE = "Title"
  DESCRIPTION = "Description"
  CODE = "Code"
  TRUE = "true"
  FALSE = "false"

class Base(object):
  """Base elements' labels and properties for objects."""
  WIDGET_INFO_HEADER_FORMAT = "{} Info"
  TYPE = "Type"
  STATE = "State"
  REFERENCE_URL = "Reference URL"
  EFFECTIVE_DATE = "Effective Date"
  STOP_DATE = "Stop Date"
  URL = "URL"

class CommonModalCreate(object):
  """Common elements' labels and properties for Modal to Create object.
 """
  HIDE_ALL_OPT_FIELDS = "Hide all optional fields"
  SHOW_ALL_OPT_FIELDS = "Show all optional fields"
  SAVE_AND_CLOSE = "Save & Close"

class CommonModalSetVisibleFields(object):
  """Common elements' labels and properties for Modal widow that Select visible
 fields for Tree View.
 """
  MODAL_HEADER_FORMAT = "Set visible fields for {}"
  TITLE = Common.TITLE
  CODE = Common.CODE
  STATE = Base.STATE
  LAST_UPDATED = "Last Updated"
  SET_FIELDS = "Set Fields"

class TransformationSetVisibleFields(CommonModalSetVisibleFields):
  """To transformation elements' labels and properties for Modal to Set
 visible fields for object as Tree View headers.
 """
  OWNER = "Owner"
  VERIFIED = "Verified"
  STATUS = "Status"
  AUDIT_LEAD = "Internal Audit Lead"
  MANAGER = "Manager"
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT

class CommonProgram(Common):
  """Common elements' labels and properties for Programs objects."""
  # pylint: disable=too-many-instance-attributes
  PROGRAM = objects.get_normal_form(objects.get_singular(objects.PROGRAMS))
  TITLE = Common.TITLE
  MANAGER = "Manager"
  PROGRAM_URL = "Program URL"
  REFERENCE_URL = Base.REFERENCE_URL
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  NOTES = "Notes"
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  STATE = Base.STATE

class CommonAudit(Common):
  """Common elements' labels and properties for Audits objects."""
  # pylint: disable=too-many-instance-attributes
  AUDIT = objects.get_normal_form(objects.get_singular(objects.AUDITS))
  TITLE = Common.TITLE
  STATUS = "Status"
  PLANNED_START_DATE = "Planned Start Date"
  PLANNED_END_DATE = "Planned End Date"
  PLANNED_REPORT_PERIOD = "Report Period"
  AUDIT_LEAD = "Internal Audit Lead"
  AUDIT_FIRM = " Audit Firm"
  AUDITORS = "Auditors"
  ADD_AUDITOR = "+ Add Auditor"
  AUDIT_FOLDER = "Audit Folder"
  ASSIGN_FOLDER = "Assign folder"

class CommonControl(object):
  """Common elements' labels and properties for Controls objects."""
  CONTROL = objects.get_normal_form(objects.get_singular(objects.CONTROLS))

class CommonAssessment(object):
  """Common elements' labels and properties for Assessments objects."""
  ASMT = objects.get_normal_form(objects.get_singular(objects.ASSESSMENTS))

class CommonAssessmentTemplate(object):
  """Common elements' labels and properties for Assessment Templates objects.
 """
  ASMT_TMPL = objects.get_normal_form(
      objects.get_singular(objects.ASSESSMENT_TEMPLATES))

class CommonIssue(object):
  """Common elements' labels and properties for Issues objects."""
  ISSUE = objects.get_normal_form(objects.get_singular(objects.ISSUES))

class ObjectStates(object):
  """States for objects."""
  DRAFT = "Draft"
  DEPRECATED = "Deprecated"
  ACTIVE = "Active"

class BaseStates(object):
  """Common states for Audit and Assessment objects."""
  IN_PROGRESS = "In progress"
  COMPLETED = "Completed"

class AuditStates(BaseStates):
  """States for Audits objects."""
  PLANNED = "Planned"
  MANAGER_REVIEW = "Manager Review"
  READY_FOR_EXT_REVIEW = "Ready for External Review"

class AssessmentStates(BaseStates):
  """States for Assessments objects."""
  NOT_STARTED = "Not Started"
  READY_FOR_REVIEW = "Ready for Review"
  VERIFIED = "Verified"

class IssueStates(ObjectStates):
  """States for Issues objects."""

class ProgramInfoWidget(CommonProgram):
  """Elements' labels and properties for Programs Info widgets."""
  WIDGET_HEADER = Base.WIDGET_INFO_HEADER_FORMAT.format(CommonProgram.PROGRAM)

class AuditInfoWidget(CommonAudit):
  """Elements' labels and properties for Audits Info widgets."""
  WIDGET_HEADER = Base.WIDGET_INFO_HEADER_FORMAT.format(CommonAudit.AUDIT)
  TITLE_UPPER = CommonAudit.TITLE.upper()
  AUDIT_LEAD_UPPER = CommonAudit.AUDIT_LEAD.upper()
  CODE_UPPER = CommonAudit.CODE.upper()

class AssessmentTemplateModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Assessment Templates.
 """
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonAssessmentTemplate.ASMT_TMPL)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE,
      CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.LAST_UPDATED)

class AssessmentModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Assessments.
 """
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonAssessment.ASMT)
  VERIFIED = TransformationSetVisibleFields.VERIFIED
  CONCLUSION_DESIGN = "Conclusion: Design"
  CONCLUSION_OPERATION = "Conclusion: Operation"
  FINISHED_DATE = "Finished Date"
  VERIFIED_DATE = "Verified Date"
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  TYPE = Base.TYPE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.STATE, VERIFIED,
      CommonModalSetVisibleFields.LAST_UPDATED)

class ControlModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Controls.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonControl.CONTROL)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  KIND_NATURE = "Kind/Nature"
  FRAUD_RELATED = "Fraud Related"
  SIGNIFICANCE = "Significance"
  TYPE_MEANS = "Type/Means"
  FREQUENCY = "Frequency"
  ASSERTIONS = "Assertions"
  CATEGORIES = "Categories"
  PRINCIPAL_ASSIGNEE = roles.PRINCIPAL_ASSIGNEE
  SECONDARY_ASSIGNEE = roles.SECONDARY_ASSIGNEE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, OWNER,
      CommonModalSetVisibleFields.CODE, CommonModalSetVisibleFields.STATE,
      PRIMARY_CONTACT)

class IssueModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Issues.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonIssue.ISSUE)
  OWNER = TransformationSetVisibleFields.OWNER
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  REVIEW_STATE = "Review State"
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, OWNER,
      CommonModalSetVisibleFields.CODE, CommonModalSetVisibleFields.STATE,
      CommonModalSetVisibleFields.LAST_UPDATED)

class ProgramModalSetVisibleFields(CommonModalSetVisibleFields):
  """Common elements' labels and properties for Modal to Set visible
 fields for Programs.
 """
  # pylint: disable=too-many-instance-attributes
  MODAL_HEADER = CommonModalSetVisibleFields.MODAL_HEADER_FORMAT.format(
      CommonProgram.PROGRAM)
  PRIMARY_CONTACT = roles.PRIMARY_CONTACT
  SECONDARY_CONTACT = roles.SECONDARY_CONTACT
  REVIEW_STATE = "Review State"
  MANAGER = CommonProgram.MANAGER
  URL = Base.URL
  REFERENCE_URL = Base.REFERENCE_URL
  EFFECTIVE_DATE = Base.EFFECTIVE_DATE
  STOP_DATE = Base.STOP_DATE
  DEFAULT_SET_FIELDS = (
      CommonModalSetVisibleFields.TITLE, CommonModalSetVisibleFields.CODE,
      CommonModalSetVisibleFields.STATE, PRIMARY_CONTACT, MANAGER)
<|endoftext|>