<commit_before>import argparse
import sys
import os

import teuthology.openstack

def main(argv=sys.argv[1:]):
    sys.exit(teuthology.openstack.main(parse_args(argv), argv))

def get_key_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--key-name',
        help='OpenStack keypair name',
    )
    parser.add_argument(
        '--key-filename',
        help='path to the ssh private key',
    )
    return parser

def get_suite_parser():
    parser = argparse.ArgumentParser()
    # copy/pasted from scripts/suite.py
    parser.add_argument(
        'config_yaml',
        nargs='*',
        help='Optional extra job yaml to include',
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true', default=None,
        help='be more verbose',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true', default=None,
        help='Do a dry run; do not schedule anything',
    )
    parser.add_argument(
        '-s', '--suite',
        help='The suite to schedule',
    )
    parser.add_argument(
        '-c', '--ceph',
        help='The ceph branch to run against',
        default=os.getenv('TEUTH_CEPH_BRANCH', 'master'),
    )
    parser.add_argument(
        '-k', '--kernel',
        help=('The kernel branch to run against; if not '
              'supplied, the installed kernel is unchanged'),
    )
    parser.add_argument(
        '-f', '--flavor',
        help=("The kernel flavor to run against: ('basic',"
              "'gcov', 'notcmalloc')"),
        default='basic',
    )
    parser.add_argument(
        '-d', '--distro',
        help='Distribution to run against',
    )
    parser.add_argument(
        '--suite-branch',
        help='Use this suite branch instead of the ceph branch',
        default=os.getenv('TEUTH_SUITE_BRANCH', 'master'),
    )
    parser.add_argument(
        '-e', '--email',
        help='When tests finish or time out, send an email here',
    )
    parser.add_argument(
        '-N', '--num',
        help='Number of times to run/queue the job',
        type=int,
        default=1,
    )
    parser.add_argument(
        '-l', '--limit',
        metavar='JOBS',
        help='Queue at most this many jobs',
        type=int,
    )
    parser.add_argument(
        '--subset',
        help=('Instead of scheduling the entire suite, break the '
              'set of jobs into <outof> pieces (each of which will '
              'contain each facet at least once) and schedule '
              'piece <index>.  Scheduling 0/<outof>, 1/<outof>, '
              '2/<outof> ... <outof>-1/<outof> will schedule all '
              'jobs in the suite (many more than once).')
    )
    parser.add_argument(
        '-p', '--priority',
        help='Job priority (lower is sooner)',
        type=int,
        default=1000,
    )
    parser.add_argument(
        '--timeout',
        help=('How long, in seconds, to wait for jobs to finish '
              'before sending email. This does not kill jobs.'),
        type=int,
        default=43200,
    )
    parser.add_argument(
        '--filter',
        help=('Only run jobs whose description contains at least one '
              'of the keywords in the comma separated keyword '
              'string specified. ')
    )
    parser.add_argument(
        '--filter-out',
        help=('Do not run jobs whose description contains any of '
              'the keywords in the comma separated keyword '
              'string specified. ')
    )
    parser.add_argument(
        '--throttle',
        help=('When scheduling, wait SLEEP seconds between jobs. '
              'Useful to avoid bursts that may be too hard on '
              'the underlying infrastructure or exceed OpenStack API '
              'limits (server creation per minute for instance).'),
        type=int,
        default=15,
    )
    parser.add_argument(
        '--suite-relpath',
        help=('Look for tasks and suite definitions in this'
              'subdirectory of the suite repo.'),
    )
    parser.add_argument(
        '-r', '--rerun',
        help=('Attempt to reschedule a run, selecting only those'
              'jobs whose status are mentioned by'
              '--rerun-status.'
              'Note that this is implemented by scheduling an'
              'entirely new suite and including only jobs whose'
              'descriptions match the selected ones. It does so'
              'using the same logic as --filter.'
              'Of all the flags that were passed when scheduling'
              'the original run, the resulting one will only'
              'inherit the suite value. Any others must be'
              'passed as normal while scheduling with this'
              'feature.'),
    )
    parser.add_argument(
        '-R', '--rerun-statuses',
        help=("A comma-separated list of statuses to be used"
              "with --rerun. Supported statuses are: 'dead',"
              "'fail', 'pass', 'queued', 'running', 'waiting'"),
        default='fail,dead',
    )
    parser.add_argument(
        '-D', '--distroversion', '--distro-version',
        help='Distro version to run against',
    )
    parser.add_argument(
        '-n', '--newest',
        help=('Search for the newest revision built on all'
              'required distro/versions, starting from'
              'either --ceph or --sha1, backtracking'
              'up to <newest> commits'),
        type=int,
        default=0,
    )
    parser.add_argument(
        '-S', '--sha1',
        help=('The ceph sha1 to run against (overrides -c)'
              'If both -S and -c are supplied, -S wins, and'
              'there is no validation that sha1 is contained'
              'in branch')
    )
    parser.add_argument(
        '--ceph-repo',
        help=("Query this repository for Ceph branch and SHA1"),
        default=os.getenv('TEUTH_CEPH_REPO', 'https://github.com/ceph/ceph'),
    )
    parser.add_argument(
        '--suite-repo',
        help=("Use tasks and suite definition in this repository"),
        default=os.getenv('TEUTH_SUITE_REPO', 'https://github.com/ceph/ceph'),
    )
    return parser

def get_openstack_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--wait',
        action='store_true', default=None,
        help='block until the suite is finished',
    )
    parser.add_argument(
        '--name',
        help='OpenStack primary instance name',
        default='teuthology',
    )
    parser.add_argument(
        '--simultaneous-jobs',
        help='maximum number of jobs running in parallel',
        type=int,
        default=1,
    )
    parser.add_argument(
        '--teardown',
        action='store_true', default=None,
        help='destroy the cluster, if it exists',
    )
    parser.add_argument(
        '--teuthology-git-url',
        help="git clone url for teuthology",
        default=os.getenv('TEUTH_REPO', 'https://github.com/ceph/teuthology'),
    )
    parser.add_argument(
        '--teuthology-branch',
        help="use this teuthology branch instead of master",
        default=os.getenv('TEUTH_BRANCH', 'master'),
    )
    parser.add_argument(
        '--upload',
        action='store_true', default=False,
        help='upload archives to an rsync server',
    )
    parser.add_argument(
        '--archive-upload',
        help='rsync destination to upload archives',
        default='ubuntu@teuthology-logs.public.ceph.com:./',
    )
    parser.add_argument(
        '--archive-upload-url',
        help='Public facing URL where archives are uploaded',
        default='http://teuthology-logs.public.ceph.com',
    )
    parser.add_argument(
        '--test-repo',
        action='append',
        help=('Package repository, or repositories, to be added on test nodes. '
              'Repository to be specified as a NAME:URL pair. Multiple '
              'repositories can be provided with multiple usage. '
              'For example --test-repo foo:http://example.com/repo/foo '
              '--test-repo bar:http://example.com/repo/bar specifies two '
              'test package repositories named "foo" and "bar", respectively.'),
        default=None,
    )
    return parser

def get_parser():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[
            get_suite_parser(),
            get_key_parser(),
            get_openstack_parser(),
        ],
        conflict_handler='resolve',
        add_help=False,
        description="""
Run a suite of ceph integration tests. A suite is a directory containing
facets. A facet is a directory containing config snippets. Running a suite
means running teuthology for every configuration combination generated by
taking one config snippet from each facet. Any config files passed on the
command line will be used for every combination, and will override anything in
the suite. By specifying a subdirectory in the suite argument, it is possible
to limit the run to a specific facet. For instance -s upgrade/dumpling-x only
runs the dumpling-x facet of the upgrade suite.

Display the http and ssh access to follow the progress of the suite
and analyze results.

  firefox http://183.84.234.3:8081/
  ssh -i teuthology-admin.pem ubuntu@183.84.234.3

""")
    return parser

def parse_args(argv):
    return get_parser().parse_args(argv)
<commit_msg>teuthology-openstack: update help for --test-repo argument<commit_after>import argparse
import sys
import os

import teuthology.openstack

def main(argv=sys.argv[1:]):
    sys.exit(teuthology.openstack.main(parse_args(argv), argv))

def get_key_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--key-name',
        help='OpenStack keypair name',
    )
    parser.add_argument(
        '--key-filename',
        help='path to the ssh private key',
    )
    return parser

def get_suite_parser():
    parser = argparse.ArgumentParser()
    # copy/pasted from scripts/suite.py
    parser.add_argument(
        'config_yaml',
        nargs='*',
        help='Optional extra job yaml to include',
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true', default=None,
        help='be more verbose',
    )
    parser.add_argument(
        '--dry-run',
        action='store_true', default=None,
        help='Do a dry run; do not schedule anything',
    )
    parser.add_argument(
        '-s', '--suite',
        help='The suite to schedule',
    )
    parser.add_argument(
        '-c', '--ceph',
        help='The ceph branch to run against',
        default=os.getenv('TEUTH_CEPH_BRANCH', 'master'),
    )
    parser.add_argument(
        '-k', '--kernel',
        help=('The kernel branch to run against; if not '
              'supplied, the installed kernel is unchanged'),
    )
    parser.add_argument(
        '-f', '--flavor',
        help=("The kernel flavor to run against: ('basic',"
              "'gcov', 'notcmalloc')"),
        default='basic',
    )
    parser.add_argument(
        '-d', '--distro',
        help='Distribution to run against',
    )
    parser.add_argument(
        '--suite-branch',
        help='Use this suite branch instead of the ceph branch',
        default=os.getenv('TEUTH_SUITE_BRANCH', 'master'),
    )
    parser.add_argument(
        '-e', '--email',
        help='When tests finish or time out, send an email here',
    )
    parser.add_argument(
        '-N', '--num',
        help='Number of times to run/queue the job',
        type=int,
        default=1,
    )
    parser.add_argument(
        '-l', '--limit',
        metavar='JOBS',
        help='Queue at most this many jobs',
        type=int,
    )
    parser.add_argument(
        '--subset',
        help=('Instead of scheduling the entire suite, break the '
              'set of jobs into <outof> pieces (each of which will '
              'contain each facet at least once) and schedule '
              'piece <index>.  Scheduling 0/<outof>, 1/<outof>, '
              '2/<outof> ... <outof>-1/<outof> will schedule all '
              'jobs in the suite (many more than once).')
    )
    parser.add_argument(
        '-p', '--priority',
        help='Job priority (lower is sooner)',
        type=int,
        default=1000,
    )
    parser.add_argument(
        '--timeout',
        help=('How long, in seconds, to wait for jobs to finish '
              'before sending email. This does not kill jobs.'),
        type=int,
        default=43200,
    )
    parser.add_argument(
        '--filter',
        help=('Only run jobs whose description contains at least one '
              'of the keywords in the comma separated keyword '
              'string specified. ')
    )
    parser.add_argument(
        '--filter-out',
        help=('Do not run jobs whose description contains any of '
              'the keywords in the comma separated keyword '
              'string specified. ')
    )
    parser.add_argument(
        '--throttle',
        help=('When scheduling, wait SLEEP seconds between jobs. '
              'Useful to avoid bursts that may be too hard on '
              'the underlying infrastructure or exceed OpenStack API '
              'limits (server creation per minute for instance).'),
        type=int,
        default=15,
    )
    parser.add_argument(
        '--suite-relpath',
        help=('Look for tasks and suite definitions in this'
              'subdirectory of the suite repo.'),
    )
    parser.add_argument(
        '-r', '--rerun',
        help=('Attempt to reschedule a run, selecting only those'
              'jobs whose status are mentioned by'
              '--rerun-status.'
              'Note that this is implemented by scheduling an'
              'entirely new suite and including only jobs whose'
              'descriptions match the selected ones. It does so'
              'using the same logic as --filter.'
              'Of all the flags that were passed when scheduling'
              'the original run, the resulting one will only'
              'inherit the suite value. Any others must be'
              'passed as normal while scheduling with this'
              'feature.'),
    )
    parser.add_argument(
        '-R', '--rerun-statuses',
        help=("A comma-separated list of statuses to be used"
              "with --rerun. Supported statuses are: 'dead',"
              "'fail', 'pass', 'queued', 'running', 'waiting'"),
        default='fail,dead',
    )
    parser.add_argument(
        '-D', '--distroversion', '--distro-version',
        help='Distro version to run against',
    )
    parser.add_argument(
        '-n', '--newest',
        help=('Search for the newest revision built on all'
              'required distro/versions, starting from'
              'either --ceph or --sha1, backtracking'
              'up to <newest> commits'),
        type=int,
        default=0,
    )
    parser.add_argument(
        '-S', '--sha1',
        help=('The ceph sha1 to run against (overrides -c)'
              'If both -S and -c are supplied, -S wins, and'
              'there is no validation that sha1 is contained'
              'in branch')
    )
    parser.add_argument(
        '--ceph-repo',
        help=("Query this repository for Ceph branch and SHA1"),
        default=os.getenv('TEUTH_CEPH_REPO', 'https://github.com/ceph/ceph'),
    )
    parser.add_argument(
        '--suite-repo',
        help=("Use tasks and suite definition in this repository"),
        default=os.getenv('TEUTH_SUITE_REPO', 'https://github.com/ceph/ceph'),
    )
    return parser

def get_openstack_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--wait',
        action='store_true', default=None,
        help='block until the suite is finished',
    )
    parser.add_argument(
        '--name',
        help='OpenStack primary instance name',
        default='teuthology',
    )
    parser.add_argument(
        '--simultaneous-jobs',
        help='maximum number of jobs running in parallel',
        type=int,
        default=1,
    )
    parser.add_argument(
        '--teardown',
        action='store_true', default=None,
        help='destroy the cluster, if it exists',
    )
    parser.add_argument(
        '--teuthology-git-url',
        help="git clone url for teuthology",
        default=os.getenv('TEUTH_REPO', 'https://github.com/ceph/teuthology'),
    )
    parser.add_argument(
        '--teuthology-branch',
        help="use this teuthology branch instead of master",
        default=os.getenv('TEUTH_BRANCH', 'master'),
    )
    parser.add_argument(
        '--upload',
        action='store_true', default=False,
        help='upload archives to an rsync server',
    )
    parser.add_argument(
        '--archive-upload',
        help='rsync destination to upload archives',
        default='ubuntu@teuthology-logs.public.ceph.com:./',
    )
    parser.add_argument(
        '--archive-upload-url',
        help='Public facing URL where archives are uploaded',
        default='http://teuthology-logs.public.ceph.com',
    )
    parser.add_argument(
        '--test-repo',
        action='append',
        help=('Package repository to be added on test nodes, which are specified '
              'as NAME:URL, NAME!PRIORITY:URL or @FILENAME, for details see below.'),
        default=None,
    )
    return parser

def get_parser():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        parents=[
            get_suite_parser(),
            get_key_parser(),
            get_openstack_parser(),
        ],
        conflict_handler='resolve',
        add_help=False,
        epilog="""test repos:

Test repository can be specified using --test-repo optional argument
with value in the following formats:  NAME:URL, NAME!PRIORITY:URL
or @FILENAME. See examples:

1) Essential usage requires to provide repo name and url:

    --test-repo foo:http://example.com/repo/foo

2) Repo can be prioritized by adding a number after '!' symbol
   in the name:

    --test-repo 'bar!10:http://example.com/repo/bar'

3) Repo data can be taken from a file by simply adding '@' symbol
   at the beginning argument value, for example from yaml:

    --test-repo @path/to/foo.yaml

  where `foo.yaml` contains one or more records like:

  - name: foo
    priority: 1
    url: http://example.com/repo/foo

4) Or from json file:

    --test-repo @path/to/foo.json

   where `foo.json` content is:

   [{"name":"foo","priority":1,"url":"http://example.com/repo/foo"}]

Several repos can be provided with multiple usage of --test-repo and/or
you can provide several repos within one yaml or json file.
The repositories are added in the order they appear in the command line or
in the file. Example:

    ---
    # The foo0 repo will be included first, after all that have any priority,
    # in particular after foo1 because it has lowest priority
    - name: foo0
      url: http://example.com/repo/foo0
    # The foo1 will go after foo2 because it has lower priority then foo2
    - name: foo1
      url: http://example.com/repo/foo1
      priority: 2
    # The foo2 will go first because it has highest priority
    - name: foo2
      url: http://example.com/repo/foo2
      priority: 1
    # The foo3 will go after foo0 because it appears after it in this file
    - name: foo3
      url: http://example.com/repo/foo3

Equivalent json file content below:

    [
      {
        "name": "foo0",
        "url": "http://example.com/repo/foo0"
      },
      {
        "name": "foo1",
        "url": "http://example.com/repo/foo1",
        "priority": 2
      },
      {
        "name": "foo2",
        "url": "http://example.com/repo/foo2",
        "priority": 1
      },
      {
        "name": "foo3",
        "url": "http://example.com/repo/foo3"
      }
    ]

At the moment supported only files with extensions: .yaml, .yml, .json, .jsn.

teuthology-openstack %s
""" % teuthology.__version__,
        description="""
Run a suite of ceph integration tests. A suite is a directory containing
facets. A facet is a directory containing config snippets. Running a suite
means running teuthology for every configuration combination generated by
taking one config snippet from each facet. Any config files passed on the
command line will be used for every combination, and will override anything in
the suite. By specifying a subdirectory in the suite argument, it is possible
to limit the run to a specific facet. For instance -s upgrade/dumpling-x only
runs the dumpling-x facet of the upgrade suite.

Display the http and ssh access to follow the progress of the suite
and analyze results.

  firefox http://183.84.234.3:8081/
  ssh -i teuthology-admin.pem ubuntu@183.84.234.3

""")
    return parser

def parse_args(argv):
    return get_parser().parse_args(argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
# import getpass
import logging
import os
import shutil
import socket
import subprocess
import threading
import time

from django.conf import settings

# from nose.tools import assert_equal, assert_true

from desktop.lib.paths import get_run_root
from hadoop import pseudo_hdfs4

from sqoop.client import SqoopClient
from sqoop.conf import SERVER_URL

service_lock = threading.Lock()

LOG = logging.getLogger(__name__)

class SqoopServerProvider(object):
  """
  Setup a Sqoop server.
  """
  TEST_PORT = '18080'
  TEST_SHUTDOWN_PORT = '18081'
  HOME = get_run_root('ext/sqoop/sqoop')

  requires_hadoop = True
  is_running = False

  @classmethod
  def setup_class(cls):
    cls.cluster = pseudo_hdfs4.shared_cluster()
    cls.client, callback = cls.get_shared_server()
    cls.shutdown = [callback]

  @classmethod
  def initialize(cls, tmpdir):
    hadoop_conf_dir = os.path.join(tmpdir, 'conf')
    base_dir = os.path.join(tmpdir, 'sqoop')
    log_dir = os.path.join(base_dir, 'logs')
    conf_dir = os.path.join(base_dir, 'conf')
    old_conf_dir = os.path.join(SqoopServerProvider.HOME, 'server/conf')

    if not os.path.exists(hadoop_conf_dir):
      os.mkdir(hadoop_conf_dir)
    if not os.path.exists(base_dir):
      os.mkdir(base_dir)
    if not os.path.exists(log_dir):
      os.mkdir(log_dir)
    if not os.path.exists(conf_dir):
      os.mkdir(conf_dir)

    for _file in ('sqoop.properties', 'sqoop_bootstrap.properties'):
      with open(os.path.join(old_conf_dir, _file), 'r') as _original:
        with open(os.path.join(conf_dir, _file), 'w') as _new:
          for _line in _original:
            line = _line.replace('${test.log.dir}', log_dir)
            line = line.replace('${test.hadoop.conf.dir}', hadoop_conf_dir)
            line = line.replace('${test.base.dir}', base_dir)
            _new.write(line)
    # This sets JAVA_OPTS with a sqoop conf... we need to use our own.
    os.chmod(os.path.join(SqoopServerProvider.HOME, 'server/bin/setenv.sh'), 0)

  @classmethod
  def start(cls, cluster):
    """
    Start oozie process.
    """
    SqoopServerProvider.initialize(cluster._tmpdir)

    env = os.environ
    env['CATALINA_HOME'] = os.path.join(SqoopServerProvider.HOME, 'server')
    env['CATALINA_PID'] = os.path.join(cluster._tmpdir, 'sqoop/sqoop.pid')
    env['CATALINA_OPTS'] = """
      -Dtest.log.dir=%(log_dir)s
      -Dtest.host.local=%(host)s
      -Dsqoop.http.port=%(http_port)s
      -Dsqoop.admin.port=%(admin_port)s
    """ % {
      'log_dir': os.path.join(cluster._tmpdir, 'sqoop/logs'),
      'host': socket.getfqdn(),
      'http_port': SqoopServerProvider.TEST_PORT,
      'admin_port': SqoopServerProvider.TEST_SHUTDOWN_PORT
    }
    env['SQOOP_HTTP_PORT'] = SqoopServerProvider.TEST_PORT
    env['SQOOP_ADMIN_PORT'] = SqoopServerProvider.TEST_SHUTDOWN_PORT
    env['JAVA_OPTS'] = '-Dsqoop.config.dir=%s' % os.path.join(cluster._tmpdir, 'sqoop/conf')
    args = [os.path.join(SqoopServerProvider.HOME, 'bin/sqoop.sh'), 'server', 'start']

    LOG.info("Executing %s, env %s, cwd %s" % (repr(args), repr(env), cluster._tmpdir))
    process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
    return process

  @classmethod
  def get_shared_server(cls, username='sqoop', language=settings.LANGUAGE_CODE):
    callback = lambda: None

    service_lock.acquire()

    if not SqoopServerProvider.is_running:
      LOG.info('\nStarting a Mini Sqoop. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')

      finish = (
        SERVER_URL.set_for_testing("http://%s:%s/sqoop" % (socket.getfqdn(), SqoopServerProvider.TEST_PORT)),
      )

      # Setup
      cluster = pseudo_hdfs4.shared_cluster()

      p = cls.start(cluster)

      def kill():
        with open(os.path.join(cluster._tmpdir, 'sqoop/sqoop.pid'), 'r') as pidfile:
          pid = pidfile.read()
          LOG.info("Killing Sqoop server (pid %s)." % pid)
          os.kill(int(pid), 9)
          p.wait()
      atexit.register(kill)

      start = time.time()
      started = False
      sleep = 0.01

      client = SqoopClient(SERVER_URL.get(), username, language)

      while not started and time.time() - start < 60.0:
        status = None
        try:
          LOG.info('Check Sqoop status...')
          version = client.get_version()
          if version:
            started = True
            break
          time.sleep(sleep)
          sleep *= 2
        except Exception, e:
          LOG.info('Sqoop server not started yet: %s' % e)
          time.sleep(sleep)
          sleep *= 2
          pass
      if not started:
        service_lock.release()
        raise Exception("Sqoop server took too long to come up.")

      def shutdown():
        for f in finish:
          f()
        cluster.stop()
      callback = shutdown

      SqoopServerProvider.is_running = True

    else:
      client = SqoopClient(SERVER_URL.get(), username, language)

    service_lock.release()

    return client, callback
<commit_msg>[sqoop] Fix tests<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
# import getpass
import logging
import os
import shutil
import socket
import subprocess
import threading
import time

from django.conf import settings

# from nose.tools import assert_equal, assert_true

from desktop.lib.paths import get_run_root
from hadoop import pseudo_hdfs4

from sqoop.client import SqoopClient
from sqoop.conf import SERVER_URL

service_lock = threading.Lock()

LOG = logging.getLogger(__name__)

class SqoopServerProvider(object):
  """
  Setup a Sqoop server.
  """
  TEST_PORT = '19080'
  TEST_SHUTDOWN_PORT = '19081'
  HOME = get_run_root('ext/sqoop/sqoop')

  requires_hadoop = True
  is_running = False

  @classmethod
  def setup_class(cls):
    cls.cluster = pseudo_hdfs4.shared_cluster()
    cls.client, callback = cls.get_shared_server()
    cls.shutdown = [callback]

  @classmethod
  def initialize(cls, tmpdir):
    hadoop_conf_dir = os.path.join(tmpdir, 'conf')
    base_dir = os.path.join(tmpdir, 'sqoop')
    log_dir = os.path.join(base_dir, 'logs')
    conf_dir = os.path.join(base_dir, 'conf')
    old_conf_dir = os.path.join(SqoopServerProvider.HOME, 'server/conf')

    if not os.path.exists(hadoop_conf_dir):
      os.mkdir(hadoop_conf_dir)
    if not os.path.exists(base_dir):
      os.mkdir(base_dir)
    if not os.path.exists(log_dir):
      os.mkdir(log_dir)
    if not os.path.exists(conf_dir):
      os.mkdir(conf_dir)

    for _file in ('sqoop.properties', 'sqoop_bootstrap.properties'):
      with open(os.path.join(old_conf_dir, _file), 'r') as _original:
        with open(os.path.join(conf_dir, _file), 'w') as _new:
          for _line in _original:
            line = _line.replace('${test.log.dir}', log_dir)
            line = line.replace('${test.hadoop.conf.dir}', hadoop_conf_dir)
            line = line.replace('${test.base.dir}', base_dir)
            _new.write(line)
    # This sets JAVA_OPTS with a sqoop conf... we need to use our own.
    os.chmod(os.path.join(SqoopServerProvider.HOME, 'server/bin/setenv.sh'), 0)

  @classmethod
  def start(cls, cluster):
    """
    Start oozie process.
    """
    SqoopServerProvider.initialize(cluster._tmpdir)

    env = os.environ
    env['CATALINA_HOME'] = os.path.join(SqoopServerProvider.HOME, 'server')
    env['CATALINA_PID'] = os.path.join(cluster._tmpdir, 'sqoop/sqoop.pid')
    env['CATALINA_OPTS'] = """
      -Dtest.log.dir=%(log_dir)s
      -Dtest.host.local=%(host)s
      -Dsqoop.http.port=%(http_port)s
      -Dsqoop.admin.port=%(admin_port)s
    """ % {
      'log_dir': os.path.join(cluster._tmpdir, 'sqoop/logs'),
      'host': socket.getfqdn(),
      'http_port': SqoopServerProvider.TEST_PORT,
      'admin_port': SqoopServerProvider.TEST_SHUTDOWN_PORT
    }
    env['SQOOP_HTTP_PORT'] = SqoopServerProvider.TEST_PORT
    env['SQOOP_ADMIN_PORT'] = SqoopServerProvider.TEST_SHUTDOWN_PORT
    env['JAVA_OPTS'] = '-Dsqoop.config.dir=%s' % os.path.join(cluster._tmpdir, 'sqoop/conf')
    args = [os.path.join(SqoopServerProvider.HOME, 'bin/sqoop.sh'), 'server', 'start']

    LOG.info("Executing %s, env %s, cwd %s" % (repr(args), repr(env), cluster._tmpdir))
    process = subprocess.Popen(args=args, env=env, cwd=cluster._tmpdir, stdin=subprocess.PIPE)
    return process

  @classmethod
  def get_shared_server(cls, username='sqoop', language=settings.LANGUAGE_CODE):
    callback = lambda: None

    service_lock.acquire()

    if not SqoopServerProvider.is_running:
      LOG.info('\nStarting a Mini Sqoop. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')

      finish = (
        SERVER_URL.set_for_testing("http://%s:%s/sqoop" % (socket.getfqdn(), SqoopServerProvider.TEST_PORT)),
      )

      # Setup
      cluster = pseudo_hdfs4.shared_cluster()

      p = cls.start(cluster)

      def kill():
        with open(os.path.join(cluster._tmpdir, 'sqoop/sqoop.pid'), 'r') as pidfile:
          pid = pidfile.read()
          LOG.info("Killing Sqoop server (pid %s)." % pid)
          os.kill(int(pid), 9)
          p.wait()
      atexit.register(kill)

      start = time.time()
      started = False
      sleep = 0.01

      client = SqoopClient(SERVER_URL.get(), username, language)

      while not started and time.time() - start < 60.0:
        status = None
        try:
          LOG.info('Check Sqoop status...')
          version = client.get_version()
          if version:
            started = True
            break
          time.sleep(sleep)
          sleep *= 2
        except Exception, e:
          LOG.info('Sqoop server not started yet: %s' % e)
          time.sleep(sleep)
          sleep *= 2
          pass
      if not started:
        service_lock.release()
        raise Exception("Sqoop server took too long to come up.")

      def shutdown():
        for f in finish:
          f()
        cluster.stop()
      callback = shutdown

      SqoopServerProvider.is_running = True

    else:
      client = SqoopClient(SERVER_URL.get(), username, language)

    service_lock.release()

    return client, callback
<|endoftext|>


<commit_before>"""Elasticsearch utilities not general enough to lift into pyelasticsearch"""

from flask import current_app
from werkzeug.exceptions import NotFound

from dxr.config import FORMAT

UNINDEXED_STRING = {
    'type': 'string',
    'index': 'no',
}

UNINDEXED_INT = {
    'type': 'integer',
    'index': 'no',
}

TREE = 'tree'  # 'tree' doctype

def frozen_configs():
    """Return a list of dicts, each describing a tree of the current format
    version."""
    return filtered_query(current_app.dxr_config.es_catalog_index,
                          TREE,
                          filter={'format': FORMAT},
                          sort=['name'],
                          size=10000)

def frozen_config(tree_name):
    """Return the bits of config that are "frozen" in place upon indexing.

    Return the ES "tree" doc for the given tree at the current format
    version.

    """
    frozen = current_app.es.get(current_app.dxr_config.es_catalog_index,
                                TREE,
                                '%s/%s' % (FORMAT, tree_name))
    try:
        return frozen['_source']
    except KeyError:
        # If nothing is found, we still get a hash, but it has no _source key.
        raise NotFound('No such tree as %s' % tree_name)

def es_alias_or_not_found(tree):
    """Return the elasticsearch alias for a tree, or raise NotFound."""
    return frozen_config(tree)['es_alias']

def filtered_query(index, doc_type, filter, sort=None, size=1, include=None, exclude=None):
    """Do a simple, filtered term query, returning an iterable of _sources.

    This is just a mindless upfactoring. It probably shouldn't be blown up
    into a full-fledged API.

    ``include`` and ``exclude`` are mutually exclusive for now.

    """
    query = {
            'query': {
                'filtered': {
                    'query': {
                        'match_all': {}
                    },
                    'filter': {
                        'term': filter
                    }
                }
            }
        }
    if sort:
        query['sort'] = sort
    if include is not None:
        query['_source'] = {'include': include}
    elif exclude is not None:
        query['_source'] = {'exclude': exclude}
    return _sources(current_app.es.search(
        query,
        index=index,
        doc_type=doc_type,
        size=size))

def _sources(search_results):
    """Return just the _source attributes of some ES search results."""
    return [r['_source'] for r in search_results['hits']['hits']]
<commit_msg>Have a 404 on /favicon.ico, not an exception.<commit_after>"""Elasticsearch utilities not general enough to lift into pyelasticsearch"""

from flask import current_app
from pyelasticsearch import ElasticHttpNotFoundError
from werkzeug.exceptions import NotFound

from dxr.config import FORMAT

UNINDEXED_STRING = {
    'type': 'string',
    'index': 'no',
}

UNINDEXED_INT = {
    'type': 'integer',
    'index': 'no',
}

TREE = 'tree'  # 'tree' doctype

def frozen_configs():
    """Return a list of dicts, each describing a tree of the current format
    version."""
    return filtered_query(current_app.dxr_config.es_catalog_index,
                          TREE,
                          filter={'format': FORMAT},
                          sort=['name'],
                          size=10000)

def frozen_config(tree_name):
    """Return the bits of config that are "frozen" in place upon indexing.

    Return the ES "tree" doc for the given tree at the current format
    version. Raise NotFound if the tree

    """
    try:
        frozen = current_app.es.get(current_app.dxr_config.es_catalog_index,
                                    TREE,
                                    '%s/%s' % (FORMAT, tree_name))
        return frozen['_source']
    except (ElasticHttpNotFoundError, KeyError):
        # If nothing is found, we still get a hash, but it has no _source key.
        raise NotFound('No such tree as %s' % tree_name)

def es_alias_or_not_found(tree):
    """Return the elasticsearch alias for a tree, or raise NotFound."""
    return frozen_config(tree)['es_alias']

def filtered_query(index, doc_type, filter, sort=None, size=1, include=None, exclude=None):
    """Do a simple, filtered term query, returning an iterable of _sources.

    This is just a mindless upfactoring. It probably shouldn't be blown up
    into a full-fledged API.

    ``include`` and ``exclude`` are mutually exclusive for now.

    """
    query = {
            'query': {
                'filtered': {
                    'query': {
                        'match_all': {}
                    },
                    'filter': {
                        'term': filter
                    }
                }
            }
        }
    if sort:
        query['sort'] = sort
    if include is not None:
        query['_source'] = {'include': include}
    elif exclude is not None:
        query['_source'] = {'exclude': exclude}
    return _sources(current_app.es.search(
        query,
        index=index,
        doc_type=doc_type,
        size=size))

def _sources(search_results):
    """Return just the _source attributes of some ES search results."""
    return [r['_source'] for r in search_results['hits']['hits']]
<|endoftext|>


<commit_before>from django.contrib.auth import hashers
from django.contrib.auth.models import User
from django.core import validators
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from molo.core.models import (
    TranslatablePageMixin, PreventDeleteMixin, Main, index_pages_after_copy)
from molo.core.molo_wagtail_models import MoloPage
from molo.core.utils import generate_slug
from phonenumber_field.modelfields import PhoneNumberField
from wagtail.wagtailcore.models import Site
from wagtail.contrib.settings.models import BaseSetting, register_setting
from wagtail.wagtailadmin.edit_handlers import (
    FieldPanel, MultiFieldPanel, PageChooserPanel)

@register_setting
class UserProfilesSettings(BaseSetting):
    show_mobile_number_field = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add mobile number field to registration"),
    )
    mobile_number_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Mobile number required"),
    )
    country_code = models.CharField(
        max_length=4,
        null=True, blank=True,
        verbose_name=_(
            "The country code that should be added to a user's number for "
            "this site"),
        help_text=_("For example: +27 for South Africa, +44 for England")
    )
    prevent_phone_number_in_username = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Prevent phone number in username / display name"),
    )

    show_email_field = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add email field to registration")
    )
    email_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Email required")
    )

    prevent_email_in_username = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Prevent email in username / display name"),
    )

    show_security_question_fields = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add security question fields to registration")
    )
    security_questions_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Security questions required")
    )
    num_security_questions = models.PositiveSmallIntegerField(
        default=1,
        verbose_name=_("Number of security questions asked for "
                       "password recovery")
    )
    password_recovery_retries = models.PositiveSmallIntegerField(
        default=5,
        verbose_name=_("Max number of password recovery retries before "
                       "lockout")
    )
    terms_and_conditions = models.ForeignKey(
        'wagtailcore.Page',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text=_('Choose a footer page')
    )
    activate_display_name = models.BooleanField(
        default=True,
        editable=True,
        verbose_name=_("Activate Display Name"),
    )
    capture_display_name_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Display Name is activated, "
                    "and Capture On Registration is not activated, "
                    "The display name field will be captured on done page."),
    )
    display_name_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Display Name Required"),
    )
    activate_gender = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Gender"),
    )
    capture_gender_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Gender is activated, "
                    "and Capture On Registration is not activated, "
                    "The Gender field will be captured on done page."),
    )
    gender_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Gender Required"),
    )
    activate_dob = models.BooleanField(
        default=True,
        editable=True,
        verbose_name=_("Activate Date Of Birth"),
    )
    capture_dob_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Date Of Birth is activated, "
                    "and Capture On Registration is not activated, "
                    "The Date Of Birth field will be captured on done page."),
    )
    dob_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Date Of Birth Required"),
    )
    activate_location = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Location"),
    )
    capture_location_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Location is activated, "
                    "and Capture On Registration is not activated, "
                    "The Location field will be captured on done page."),
    )
    location_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Location Required"),
    )
    activate_education_level = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Education Level"),
    )
    capture_education_level_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Education Level is activated, "
                    "and Capture On Registration is not activated, "
                    "The Education Level field will be captured "
                    "on done page."),
    )
    activate_education_level_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Education Level Required"),
    )

    panels = [
        MultiFieldPanel(
            [
                FieldPanel('show_mobile_number_field'),
                FieldPanel('mobile_number_required'),
                FieldPanel('country_code'),
                FieldPanel('prevent_phone_number_in_username'),
            ],
            heading="Mobile Number Settings", ),
        MultiFieldPanel(
            [
                FieldPanel('show_email_field'),
                FieldPanel('email_required'),
                FieldPanel('prevent_email_in_username'),
            ],
            heading="Email Settings", ),
        MultiFieldPanel(
            [
                FieldPanel("show_security_question_fields"),
                FieldPanel("security_questions_required"),
                FieldPanel("num_security_questions"),
                FieldPanel("password_recovery_retries"),
            ],
            heading="Security Question Settings", ),
        MultiFieldPanel(
            [
                PageChooserPanel('terms_and_conditions'),
            ],
            heading="Terms and Conditions on registration", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_display_name'),
                FieldPanel('capture_display_name_on_reg'),
                FieldPanel('display_name_required'),
            ],
            heading="Display Name", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_gender'),
                FieldPanel('capture_gender_on_reg'),
                FieldPanel('gender_required'),
            ],
            heading="Gender", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_dob'),
                FieldPanel('capture_dob_on_reg'),
                FieldPanel('dob_required'),
            ],
            heading="Date Of Birth", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_location'),
                FieldPanel('capture_location_on_reg'),
                FieldPanel('location_required'),
            ],
            heading="Location", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_education_level'),
                FieldPanel('capture_education_level_on_reg'),
                FieldPanel('activate_education_level_required'),
            ],
            heading="Education Level", )
    ]
    # TODO: mobile_number_required field shouldn't be shown
    # if show_mobile_number_field is False

@python_2_unicode_compatible
class SecurityQuestion(TranslatablePageMixin, MoloPage):
    parent_page_types = ['SecurityQuestionIndexPage']
    subpage_types = []

    class Meta:
        verbose_name = _("Security Question")

    def __str__(self):
        return self.title

SecurityQuestion.content_panels = [
    FieldPanel("title", classname="full title")
]
SecurityQuestion.promote_panels = []
SecurityQuestion.settings_panels = []

class SecurityQuestionIndexPage(MoloPage, PreventDeleteMixin):
    parent_page_types = ['core.Main']
    subpage_types = ["SecurityQuestion"]

    def copy(self, *args, **kwargs):
        site = kwargs['to'].get_site()
        main = site.root_page
        SecurityQuestionIndexPage.objects.child_of(main).delete()
        super(SecurityQuestionIndexPage, self).copy(*args, **kwargs)

SecurityQuestionIndexPage.content_panels = [
    FieldPanel('title', classname='full title'),
]

@receiver(index_pages_after_copy, sender=Main)
def create_security_question_index_page(sender, instance, **kwargs):
    if not instance.get_children().filter(
            title='Security Questions').exists():
        security_index = SecurityQuestionIndexPage(
            title='Security Questions', slug=('security-questions-%s' % (
                generate_slug(instance.title), )))
        instance.add_child(instance=security_index)
        security_index.save_revision().publish()

class UserProfile(models.Model):
    user = models.OneToOneField(User, related_name="profile", primary_key=True)
    date_of_birth = models.DateField(null=True)
    alias = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    gender = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    location = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    education_level = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    avatar = models.ImageField(
        'avatar',
        max_length=100,
        upload_to='users/profile',
        blank=True,
        null=True)

    mobile_number = PhoneNumberField(blank=True, null=True, unique=False)
    security_question_answers = models.ManyToManyField(
        SecurityQuestion,
        through="SecurityAnswer"
    )
    site = models.ForeignKey(Site, blank=True, null=True)
    admin_sites = models.ManyToManyField(
        Site, related_name='admin_sites'
    )
    migrated_username = models.CharField(
        _('migrated_username'),
        max_length=40,
        validators=[
            validators.RegexValidator(
                r'^[\w.@+-]+$',
                _('Enter a valid username. This value may contain only '
                  'letters, numbers ' 'and @/./+/-/_ characters.')
            ),
        ],
        null=True, blank=True
    )
    fcm_registration_token = models.CharField(max_length=256, null=True)

    panels = [
        FieldPanel('admin_sites',),
    ]

@receiver(post_save, sender=User)
def user_profile_handler(sender, instance, created, **kwargs):
    if created:
        profile = UserProfile(user=instance)
        profile.site = Site.objects.get(is_default_site=True)
        profile.save()

class SecurityAnswer(models.Model):
    user = models.ForeignKey(UserProfile)
    question = models.ForeignKey(SecurityQuestion)
    answer = models.CharField(max_length=150, null=False, blank=False)

    def set_answer(self, raw_answer):
        self.answer = hashers.make_password(raw_answer.strip().lower())

    def check_answer(self, raw_answer):
        def setter(raw_answer):
            self.set_answer(raw_answer)
            self.save(update_fields=["answer"])

        return hashers.check_password(
            raw_answer.strip().lower(),
            self.answer,
            setter
        )

    def save(self, is_import=False, *args, **kwargs):
        # checks if this save is coming from an import so we don't hash a hash
        if not is_import and not self.id:
            self.set_answer(self.answer)
        super(SecurityAnswer, self).save(*args, **kwargs)
<commit_msg>add uuid field<commit_after>import uuid
from django.contrib.auth import hashers
from django.contrib.auth.models import User
from django.core import validators
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _

from molo.core.models import (
    TranslatablePageMixin, PreventDeleteMixin, Main, index_pages_after_copy)
from molo.core.molo_wagtail_models import MoloPage
from molo.core.utils import generate_slug
from phonenumber_field.modelfields import PhoneNumberField
from wagtail.wagtailcore.models import Site
from wagtail.contrib.settings.models import BaseSetting, register_setting
from wagtail.wagtailadmin.edit_handlers import (
    FieldPanel, MultiFieldPanel, PageChooserPanel)

@register_setting
class UserProfilesSettings(BaseSetting):
    show_mobile_number_field = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add mobile number field to registration"),
    )
    mobile_number_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Mobile number required"),
    )
    country_code = models.CharField(
        max_length=4,
        null=True, blank=True,
        verbose_name=_(
            "The country code that should be added to a user's number for "
            "this site"),
        help_text=_("For example: +27 for South Africa, +44 for England")
    )
    prevent_phone_number_in_username = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Prevent phone number in username / display name"),
    )

    show_email_field = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add email field to registration")
    )
    email_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Email required")
    )

    prevent_email_in_username = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Prevent email in username / display name"),
    )

    show_security_question_fields = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Add security question fields to registration")
    )
    security_questions_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Security questions required")
    )
    num_security_questions = models.PositiveSmallIntegerField(
        default=1,
        verbose_name=_("Number of security questions asked for "
                       "password recovery")
    )
    password_recovery_retries = models.PositiveSmallIntegerField(
        default=5,
        verbose_name=_("Max number of password recovery retries before "
                       "lockout")
    )
    terms_and_conditions = models.ForeignKey(
        'wagtailcore.Page',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text=_('Choose a footer page')
    )
    activate_display_name = models.BooleanField(
        default=True,
        editable=True,
        verbose_name=_("Activate Display Name"),
    )
    capture_display_name_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Display Name is activated, "
                    "and Capture On Registration is not activated, "
                    "The display name field will be captured on done page."),
    )
    display_name_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Display Name Required"),
    )
    activate_gender = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Gender"),
    )
    capture_gender_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Gender is activated, "
                    "and Capture On Registration is not activated, "
                    "The Gender field will be captured on done page."),
    )
    gender_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Gender Required"),
    )
    activate_dob = models.BooleanField(
        default=True,
        editable=True,
        verbose_name=_("Activate Date Of Birth"),
    )
    capture_dob_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Date Of Birth is activated, "
                    "and Capture On Registration is not activated, "
                    "The Date Of Birth field will be captured on done page."),
    )
    dob_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Date Of Birth Required"),
    )
    activate_location = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Location"),
    )
    capture_location_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Location is activated, "
                    "and Capture On Registration is not activated, "
                    "The Location field will be captured on done page."),
    )
    location_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Location Required"),
    )
    activate_education_level = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Activate Education Level"),
    )
    capture_education_level_on_reg = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Capture On Registration"),
        help_text=_("If Education Level is activated, "
                    "and Capture On Registration is not activated, "
                    "The Education Level field will be captured "
                    "on done page."),
    )
    activate_education_level_required = models.BooleanField(
        default=False,
        editable=True,
        verbose_name=_("Education Level Required"),
    )

    panels = [
        MultiFieldPanel(
            [
                FieldPanel('show_mobile_number_field'),
                FieldPanel('mobile_number_required'),
                FieldPanel('country_code'),
                FieldPanel('prevent_phone_number_in_username'),
            ],
            heading="Mobile Number Settings", ),
        MultiFieldPanel(
            [
                FieldPanel('show_email_field'),
                FieldPanel('email_required'),
                FieldPanel('prevent_email_in_username'),
            ],
            heading="Email Settings", ),
        MultiFieldPanel(
            [
                FieldPanel("show_security_question_fields"),
                FieldPanel("security_questions_required"),
                FieldPanel("num_security_questions"),
                FieldPanel("password_recovery_retries"),
            ],
            heading="Security Question Settings", ),
        MultiFieldPanel(
            [
                PageChooserPanel('terms_and_conditions'),
            ],
            heading="Terms and Conditions on registration", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_display_name'),
                FieldPanel('capture_display_name_on_reg'),
                FieldPanel('display_name_required'),
            ],
            heading="Display Name", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_gender'),
                FieldPanel('capture_gender_on_reg'),
                FieldPanel('gender_required'),
            ],
            heading="Gender", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_dob'),
                FieldPanel('capture_dob_on_reg'),
                FieldPanel('dob_required'),
            ],
            heading="Date Of Birth", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_location'),
                FieldPanel('capture_location_on_reg'),
                FieldPanel('location_required'),
            ],
            heading="Location", ),
        MultiFieldPanel(
            [
                FieldPanel('activate_education_level'),
                FieldPanel('capture_education_level_on_reg'),
                FieldPanel('activate_education_level_required'),
            ],
            heading="Education Level", )
    ]
    # TODO: mobile_number_required field shouldn't be shown
    # if show_mobile_number_field is False

@python_2_unicode_compatible
class SecurityQuestion(TranslatablePageMixin, MoloPage):
    parent_page_types = ['SecurityQuestionIndexPage']
    subpage_types = []

    class Meta:
        verbose_name = _("Security Question")

    def __str__(self):
        return self.title

SecurityQuestion.content_panels = [
    FieldPanel("title", classname="full title")
]
SecurityQuestion.promote_panels = []
SecurityQuestion.settings_panels = []

class SecurityQuestionIndexPage(MoloPage, PreventDeleteMixin):
    parent_page_types = ['core.Main']
    subpage_types = ["SecurityQuestion"]

    def copy(self, *args, **kwargs):
        site = kwargs['to'].get_site()
        main = site.root_page
        SecurityQuestionIndexPage.objects.child_of(main).delete()
        super(SecurityQuestionIndexPage, self).copy(*args, **kwargs)

SecurityQuestionIndexPage.content_panels = [
    FieldPanel('title', classname='full title'),
]

@receiver(index_pages_after_copy, sender=Main)
def create_security_question_index_page(sender, instance, **kwargs):
    if not instance.get_children().filter(
            title='Security Questions').exists():
        security_index = SecurityQuestionIndexPage(
            title='Security Questions', slug=('security-questions-%s' % (
                generate_slug(instance.title), )))
        instance.add_child(instance=security_index)
        security_index.save_revision().publish()

class UserProfile(models.Model):
    user = models.OneToOneField(User, related_name="profile", primary_key=True)
    uuid = models.UUIDField(default=uuid.uuid4, unique=True)
    date_of_birth = models.DateField(null=True)
    alias = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    gender = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    location = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    education_level = models.CharField(
        max_length=128,
        blank=True,
        null=True)
    avatar = models.ImageField(
        'avatar',
        max_length=100,
        upload_to='users/profile',
        blank=True,
        null=True)

    mobile_number = PhoneNumberField(blank=True, null=True, unique=False)
    security_question_answers = models.ManyToManyField(
        SecurityQuestion,
        through="SecurityAnswer"
    )
    site = models.ForeignKey(Site, blank=True, null=True)
    admin_sites = models.ManyToManyField(
        Site, related_name='admin_sites'
    )
    migrated_username = models.CharField(
        _('migrated_username'),
        max_length=40,
        validators=[
            validators.RegexValidator(
                r'^[\w.@+-]+$',
                _('Enter a valid username. This value may contain only '
                  'letters, numbers ' 'and @/./+/-/_ characters.')
            ),
        ],
        null=True, blank=True
    )
    fcm_registration_token = models.CharField(max_length=256, null=True)

    panels = [
        FieldPanel('admin_sites',),
    ]

@receiver(post_save, sender=User)
def user_profile_handler(sender, instance, created, **kwargs):
    if created:
        profile = UserProfile(user=instance)
        profile.site = Site.objects.get(is_default_site=True)
        profile.save()

class SecurityAnswer(models.Model):
    user = models.ForeignKey(UserProfile)
    question = models.ForeignKey(SecurityQuestion)
    answer = models.CharField(max_length=150, null=False, blank=False)

    def set_answer(self, raw_answer):
        self.answer = hashers.make_password(raw_answer.strip().lower())

    def check_answer(self, raw_answer):
        def setter(raw_answer):
            self.set_answer(raw_answer)
            self.save(update_fields=["answer"])

        return hashers.check_password(
            raw_answer.strip().lower(),
            self.answer,
            setter
        )

    def save(self, is_import=False, *args, **kwargs):
        # checks if this save is coming from an import so we don't hash a hash
        if not is_import and not self.id:
            self.set_answer(self.answer)
        super(SecurityAnswer, self).save(*args, **kwargs)
<|endoftext|>


<commit_before>from bitcoin_tools.analysis.plots import get_cdf
from bitcoin_tools.analysis.status.data_processing import get_samples, get_filtered_samples
from bitcoin_tools.analysis.status.plots import plots_from_samples
from bitcoin_tools.analysis.status.utils import aggregate_dust_np
from bitcoin_tools import CFG
from ujson import load
from getopt import getopt
from sys import argv

def compare_dust(dust_files, legend, version, sufix=''):
    """
    Compares dust of two given dust files.

    :param dust_files: List of dust file names
    :type dust_files: list of str
    :param legend: Legend for the charts
    :type legend: list of str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return: None
    :rtype: None
    """

    outs = ["cmp_dust_utxos"+sufix, "cmp_dust_value"+sufix, "cmp_dust_data_len"+sufix]
    totals = ['total_utxos', 'total_value', 'total_data_len']

    utxos = []
    value = []
    length = []

    for f in dust_files:
        data = load(open(CFG.data_path + f))
        utxos.append(data['npest_utxos'])
        value.append(data['npest_value'])
        length.append(data['npest_data_len'])

    xs_utxos, ys_utxos = [sorted(u.keys(), key=int) for u in utxos], [sorted(u.values(), key=int) for u in utxos]
    xs_value, ys_value = [sorted(v.keys(), key=int) for v in value], [sorted(v.values(), key=int) for v in value]
    xs_length, ys_length = [sorted(l.keys(), key=int) for l in length], [sorted(l.values(), key=int) for l in length]

    x_samples = [xs_utxos, xs_value, xs_length]
    y_samples = [ys_utxos, ys_value, ys_length]

    for xs, ys, out, total in zip(x_samples, y_samples, outs, totals):
        plots_from_samples(xs=xs, ys=ys, save_fig=out, legend=legend, xlabel='Fee rate(sat/byte)',
                           ylabel="Number of UTXOs", version=str(version))

        # Get values in percentage
        ys_perc = []
        for y_samples in ys:
            y_perc = [y / float(data[total]) for y in y_samples]
            ys_perc.append(y_perc)

        plots_from_samples(xs=xs, ys=ys_perc, save_fig='perc_' + out, legend=legend,
                           xlabel='Fee rate(sat/byte)', ylabel="Number of UTXOs", version=str(version))

def compare_attribute(fin_names, x_attribute, out_name, version, xlabel='', legend=''):
    """
    Performs a comparative analysis between different files and a fixed attribute. Useful to compare the evolution
    of a parameter throughout different snapshots.

    :param fin_names: List of file names to load data from.
    :type fin_names: list str
    :param x_attribute: Attribute to be compared.
    :type x_attribute: str
    :param out_name: Name of the generated chart.
    :type out_name: str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :param xlabel: Label of the x axis of the resulting chart.
    :type xlabel: str
    :param legend: Legend to be included in the chart.
    :type legend: str
    :return: None
    :rtyp: None
    """

    samples = [get_samples(x_attribute, fin) for fin in fin_names]

    xs = []
    ys = []
    for _ in range(len(samples)):
        x, y = get_cdf(samples.pop(0).values(), normalize=True)
        xs.append(x)
        ys.append(y)

    plots_from_samples(xs=xs, ys=ys, xlabel=xlabel, save_fig=out_name, version=str(version), legend=legend,
                       log_axis='x', ylabel="Number of UTXOs", legend_loc=2)

def comparative_data_analysis(tx_fin_name, utxo_fin_name, version):
    """
    Performs a comparative data analysis between a transaction dump data file and an utxo dump one.

    :param tx_fin_name: Input file path which contains the chainstate transaction dump.
    :type: str
    :param utxo_fin_name: Input file path which contains the chainstate utxo dump.
    :type: str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return: None
    :rtype: None
    """

    # Generate plots with both transaction and utxo data (f_parsed_txs and f_parsed_utxos)
    tx_attributes = ['total_value', 'height']
    utxo_attributes = ['amount', 'tx_height']

    xlabels = ['Amount (Satoshi)', 'Height']
    out_names = ['tx_utxo_amount', 'tx_utxo_height']
    legends = [['Tx.', 'UTXO'], ['Tx.', 'UTXO']]
    legend_locations = [1, 2]

    tx_samples = get_samples(tx_attributes, tx_fin_name)
    utxo_samples = get_samples(utxo_attributes, utxo_fin_name)

    for tx_attr, utxo_attr, label, out, legend, leg_loc in zip(tx_attributes, utxo_attributes, xlabels, out_names,
                                                               legends, legend_locations):
        xs_txs, ys_txs = get_cdf(tx_samples[tx_attr], normalize=True)
        xs_utxos, ys_utxos = get_cdf(utxo_samples[utxo_attr], normalize=True)

        plots_from_samples(xs=[xs_txs, xs_utxos], ys=[ys_txs, ys_utxos], xlabel=label, save_fig=out,
                           version=str(version), legend=legend, legend_loc=leg_loc, ylabel="Number of registers")

def run_experiment(version, f_dust, f_parsed_utxos, f_parsed_txs):
    """
    Runs the whole experiment. You may comment the parts of it you are not interested in to save time.

     :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return:
    """

    print "Running comparative data analysis."
    # Comparative dust analysis between different snapshots
    fin_names = [str(version) + '/height-' + str(i) + 'K/' + f_parsed_utxos + '.json' for i in range(100, 550, 50)]
    dust_files = [str(version) + '/height-' + str(i) + 'K/' + f_dust + '.json' for i in range(100, 550, 50)]
    legend = [str(i) + 'K' for i in range(100, 550, 50)]

    print "Comparing dust from different snapshots."
    # Get dust files from different dates to compare (Change / Add the ones you'll need)
    compare_dust(dust_files=dust_files, legend=legend, version=version)

    # Dust comparision counting only P2PKH outputs
    dust_files = [str(version) + '/height-' + str(i) + 'K/' + f_dust + '_p2pkh_only.json' for i in range(100, 550, 50)]

    # for utxo_fin, dust_fout in zip(fin_names, dust_files):
    #     aggregate_dust_np(utxo_fin, dust_fout, fltr=lambda x: x['out_type'] == 0)

    compare_dust(dust_files=dust_files, legend=legend, version=version, sufix='_p2pkh')

    # Comparative analysis between different snapshots
    # UTXO amount comparison
    print "Comparing UTXO amount from different snapshots."
    compare_attribute(fin_names=fin_names, x_attribute='amount', xlabel='Amount (Satoshi)', legend=legend,
                      out_name='cmp_utxo_amount', version=version)

    # UTXO size comparison
    print "Comparing UTXO size from different snapshots."
    compare_attribute(fin_names=fin_names, x_attribute='register_len', xlabel='Size (bytes)', legend=legend,
                      out_name='cmp_utxo_size', version=version)

    # Comparative data analysis (transactions and UTXOs)
    comparative_data_analysis(f_parsed_txs, f_parsed_utxos, version)

if __name__ == '__main__':

    f_dust = 'dust_wp2sh'
    f_parsed_utxos = 'parsed_utxos_wp2sh'
    f_parsed_txs = 'parsed_txs'

    # Set version and chainstate dir name
    version = 0.15

    run_experiment(version, f_dust, f_parsed_utxos, f_parsed_txs)<commit_msg>PEP 8<commit_after>from bitcoin_tools.analysis.plots import get_cdf
from bitcoin_tools.analysis.status.data_processing import get_samples
from bitcoin_tools.analysis.status.plots import plots_from_samples
from bitcoin_tools import CFG
from ujson import load

def compare_dust(dust_files, legend, version, sufix=''):
    """
    Compares dust of two given dust files.

    :param dust_files: List of dust file names
    :type dust_files: list of str
    :param legend: Legend for the charts
    :type legend: list of str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return: None
    :rtype: None
    """

    outs = ["cmp_dust_utxos"+sufix, "cmp_dust_value"+sufix, "cmp_dust_data_len"+sufix]
    totals = ['total_utxos', 'total_value', 'total_data_len']

    utxos = []
    value = []
    length = []

    for f in dust_files:
        data = load(open(CFG.data_path + f))
        utxos.append(data['npest_utxos'])
        value.append(data['npest_value'])
        length.append(data['npest_data_len'])

    xs_utxos, ys_utxos = [sorted(u.keys(), key=int) for u in utxos], [sorted(u.values(), key=int) for u in utxos]
    xs_value, ys_value = [sorted(v.keys(), key=int) for v in value], [sorted(v.values(), key=int) for v in value]
    xs_length, ys_length = [sorted(l.keys(), key=int) for l in length], [sorted(l.values(), key=int) for l in length]

    x_samples = [xs_utxos, xs_value, xs_length]
    y_samples = [ys_utxos, ys_value, ys_length]

    for xs, ys, out, total in zip(x_samples, y_samples, outs, totals):
        plots_from_samples(xs=xs, ys=ys, save_fig=out, legend=legend, xlabel='Fee rate(sat/byte)',
                           ylabel="Number of UTXOs", version=str(version))

        # Get values in percentage
        ys_perc = []
        for y_samples in ys:
            y_perc = [y / float(data[total]) for y in y_samples]
            ys_perc.append(y_perc)

        plots_from_samples(xs=xs, ys=ys_perc, save_fig='perc_' + out, legend=legend,
                           xlabel='Fee rate(sat/byte)', ylabel="Number of UTXOs", version=str(version))

def compare_attribute(fin_names, x_attribute, out_name, version, xlabel='', legend=''):
    """
    Performs a comparative analysis between different files and a fixed attribute. Useful to compare the evolution
    of a parameter throughout different snapshots.

    :param fin_names: List of file names to load data from.
    :type fin_names: list str
    :param x_attribute: Attribute to be compared.
    :type x_attribute: str
    :param out_name: Name of the generated chart.
    :type out_name: str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :param xlabel: Label of the x axis of the resulting chart.
    :type xlabel: str
    :param legend: Legend to be included in the chart.
    :type legend: str
    :return: None
    :rtyp: None
    """

    samples = [get_samples(x_attribute, fin) for fin in fin_names]

    xs = []
    ys = []
    for _ in range(len(samples)):
        x, y = get_cdf(samples.pop(0).values(), normalize=True)
        xs.append(x)
        ys.append(y)

    plots_from_samples(xs=xs, ys=ys, xlabel=xlabel, save_fig=out_name, version=str(version), legend=legend,
                       log_axis='x', ylabel="Number of UTXOs", legend_loc=2)

def comparative_data_analysis(tx_fin_name, utxo_fin_name, version):
    """
    Performs a comparative data analysis between a transaction dump data file and an utxo dump one.

    :param tx_fin_name: Input file path which contains the chainstate transaction dump.
    :type: str
    :param utxo_fin_name: Input file path which contains the chainstate utxo dump.
    :type: str
    :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return: None
    :rtype: None
    """

    # Generate plots with both transaction and utxo data (f_parsed_txs and f_parsed_utxos)
    tx_attributes = ['total_value', 'height']
    utxo_attributes = ['amount', 'tx_height']

    xlabels = ['Amount (Satoshi)', 'Height']
    out_names = ['tx_utxo_amount', 'tx_utxo_height']
    legends = [['Tx.', 'UTXO'], ['Tx.', 'UTXO']]
    legend_locations = [1, 2]

    tx_samples = get_samples(tx_attributes, tx_fin_name)
    utxo_samples = get_samples(utxo_attributes, utxo_fin_name)

    for tx_attr, utxo_attr, label, out, legend, leg_loc in zip(tx_attributes, utxo_attributes, xlabels, out_names,
                                                               legends, legend_locations):
        xs_txs, ys_txs = get_cdf(tx_samples[tx_attr], normalize=True)
        xs_utxos, ys_utxos = get_cdf(utxo_samples[utxo_attr], normalize=True)

        plots_from_samples(xs=[xs_txs, xs_utxos], ys=[ys_txs, ys_utxos], xlabel=label, save_fig=out,
                           version=str(version), legend=legend, legend_loc=leg_loc, ylabel="Number of registers")

def run_experiment(version, f_dust, f_parsed_utxos, f_parsed_txs):
    """
    Runs the whole experiment. You may comment the parts of it you are not interested in to save time.

     :param version: Bitcoin core version, used to decide the folder in which to store the data.
    :type version: float
    :return:
    """

    print "Running comparative data analysis."
    # Comparative dust analysis between different snapshots
    fin_names = [str(version) + '/height-' + str(i) + 'K/' + f_parsed_utxos + '.json' for i in range(100, 550, 50)]
    dust_files = [str(version) + '/height-' + str(i) + 'K/' + f_dust + '.json' for i in range(100, 550, 50)]
    legend = [str(i) + 'K' for i in range(100, 550, 50)]

    print "Comparing dust from different snapshots."
    # Get dust files from different dates to compare (Change / Add the ones you'll need)
    compare_dust(dust_files=dust_files, legend=legend, version=version)

    # Dust comparision counting only P2PKH outputs
    dust_files = [str(version) + '/height-' + str(i) + 'K/' + f_dust + '_p2pkh_only.json' for i in range(100, 550, 50)]

    # for utxo_fin, dust_fout in zip(fin_names, dust_files):
    #     aggregate_dust_np(utxo_fin, dust_fout, fltr=lambda x: x['out_type'] == 0)

    compare_dust(dust_files=dust_files, legend=legend, version=version, sufix='_p2pkh')

    # Comparative analysis between different snapshots
    # UTXO amount comparison
    print "Comparing UTXO amount from different snapshots."
    compare_attribute(fin_names=fin_names, x_attribute='amount', xlabel='Amount (Satoshi)', legend=legend,
                      out_name='cmp_utxo_amount', version=version)

    # UTXO size comparison
    print "Comparing UTXO size from different snapshots."
    compare_attribute(fin_names=fin_names, x_attribute='register_len', xlabel='Size (bytes)', legend=legend,
                      out_name='cmp_utxo_size', version=version)

    # Comparative data analysis (transactions and UTXOs)
    comparative_data_analysis(f_parsed_txs, f_parsed_utxos, version)

if __name__ == '__main__':

    f_dust = 'dust_wp2sh'
    f_parsed_utxos = 'parsed_utxos_wp2sh'
    f_parsed_txs = 'parsed_txs'

    # Set version and chainstate dir name
    version = 0.15

    run_experiment(version, f_dust, f_parsed_utxos, f_parsed_txs)<|endoftext|>


<commit_before>#! /usr/bin/env python

import os
import sys
import json
import shutil

# Process the command-line arguments
if (len(sys.argv) != 2) and (len(sys.argv) != 3):
    print "Usage: %s <data_file> <output_folder>" % sys.argv[0]
    print
    sys.exit(-1)

data_file = sys.argv[1]

if len(sys.argv) == 3:
    dest = sys.argv[2]
else:
    dest = './html'

dest = os.path.abspath(dest)

# Try to import the user's custom settings
CHARACTERS_TO_IGNORE = []
CHARACTERS_COLORS = {}
try:
    module = __import__('settings')

    if hasattr(module, 'CHARACTERS_TO_IGNORE'):
        CHARACTERS_TO_IGNORE = module.CHARACTERS_TO_IGNORE

    if hasattr(module, 'CHARACTERS_COLORS'):
        CHARACTERS_COLORS = module.CHARACTERS_COLORS
except:
    pass

# Open the data file and convert its content to JSON
input_file = open(data_file, 'r')
lines = filter(lambda x: len(x) > 0, map(lambda x: x.strip(), input_file.readlines()))
input_file.close()

processed_lines = []
for line in lines:
    if line == 'TitanPlayedTimes = {':
        line = '{'
    elif line.startswith('}'):
        if processed_lines[-1].endswith(','):
            processed_lines[-1] = processed_lines[-1][:-1]
    elif line.find('=') > 0:
        (name, value) = tuple(map(lambda x: x.strip(), line.split('=')))
        if value.startswith("'") and value.endswith("',"):
            value = '"' + value[1:-2] + '",'
        if name.startswith('["') and name.endswith('"]'):
            line = name[1:-1] + ': ' + value
        elif name.startswith('[') and name.endswith(']'):
            line = '"' + name[1:-1] + '": ' + value
    processed_lines.append(line)

entries_stringified = ''.join(processed_lines)
entries = json.loads(entries_stringified)

# Retrieves a sorted list of all the timestamps in the data file, without missing days
first_day = None
last_day = None
for name, details in entries.items():
    if first_day is not None:
        last_day = max(last_day, details['last'])
        first_day = min(first_day, min(map(lambda x: int(x), filter(lambda x: (x != u'last') and (x != u'class'), details.keys()))))
    else:
        last_day = details['last']
        first_day = min(map(lambda x: int(x), filter(lambda x: (x != u'last') and (x != u'class'), details.keys())))

timestamps = range(first_day, last_day + 1, 3600 * 24)

# Retrieves a sorted list of all the characters in the data file
names = entries.keys()
names.sort()

# Generate the HTML page
if os.path.exists(dest):
    shutil.rmtree(dest)

shutil.copytree(os.path.join(os.path.dirname(sys.argv[0]), 'templates'), dest)

input_file = open(os.path.join(dest, 'index.html'), 'r')
content = input_file.read()
input_file.close()

content = content.replace('$ENTRIES$', entries_stringified). \
                  replace('$TIMETAMPS$', str(timestamps)).   \
                  replace('$CHARACTER_NAMES$', str(map(lambda x: str(x), names))). \
                  replace('$CHARACTERS_TO_IGNORE$', str(CHARACTERS_TO_IGNORE)). \
                  replace('$CHARACTERS_COLORS$', str(CHARACTERS_COLORS))

output_file = open(os.path.join(dest, 'index.html'), 'w')
output_file.write(content)
output_file.close()
<commit_msg>[Visualization] plot-html.py now supports the new fields (money, level and levels_history)<commit_after>#! /usr/bin/env python

import os
import sys
import json
import shutil

# Process the command-line arguments
if (len(sys.argv) != 2) and (len(sys.argv) != 3):
    print "Usage: %s <data_file> <output_folder>" % sys.argv[0]
    print
    sys.exit(-1)

data_file = sys.argv[1]

if len(sys.argv) == 3:
    dest = sys.argv[2]
else:
    dest = './html'

dest = os.path.abspath(dest)

# Try to import the user's custom settings
CHARACTERS_TO_IGNORE = []
CHARACTERS_COLORS = {}
try:
    module = __import__('settings')

    if hasattr(module, 'CHARACTERS_TO_IGNORE'):
        CHARACTERS_TO_IGNORE = module.CHARACTERS_TO_IGNORE

    if hasattr(module, 'CHARACTERS_COLORS'):
        CHARACTERS_COLORS = module.CHARACTERS_COLORS
except:
    pass

# Open the data file and convert its content to JSON
input_file = open(data_file, 'r')
lines = filter(lambda x: len(x) > 0, map(lambda x: x.strip(), input_file.readlines()))
input_file.close()

processed_lines = []
in_array = False
for line in lines:
    if line == 'TitanPlayedTimes = {':
        line = '{'
    elif line.startswith('}'):
        if processed_lines[-1].endswith(','):
            processed_lines[-1] = processed_lines[-1][:-1]
        if in_array:
            line = ']' + line[1:]
            in_array = False
    elif line.find('=') > 0:
        (name, value) = tuple(map(lambda x: x.strip(), line.split('=')))
        if value.startswith("'") and value.endswith("',"):
            value = '"' + value[1:-2] + '",'
        if name.startswith('["') and name.endswith('"]'):
            line = name[1:-1] + ': ' + value
        elif name.startswith('[') and name.endswith(']'):
            line = '"' + name[1:-1] + '": ' + value
    elif line.find(', -- ['):
        if processed_lines[-1].endswith(': {'):
            processed_lines[-1] = processed_lines[-1][:-1] + '['
        in_array = True
        line = line[:line.find(', -- [') + 1]
    processed_lines.append(line)

entries_stringified = ''.join(processed_lines)
entries = json.loads(entries_stringified)

# Retrieves a sorted list of all the timestamps in the data file, without missing days
first_day = None
last_day = None
keys_to_ignore = [u'last', u'class', u'money', u'level', u'levels_history']
for name, details in entries.items():
    if first_day is not None:
        last_day = max(last_day, details['last'])
        first_day = min(first_day, min(map(lambda x: int(x), filter(lambda x: x not in keys_to_ignore, details.keys()))))
    else:
        last_day = details['last']
        first_day = min(map(lambda x: int(x), filter(lambda x: x not in keys_to_ignore, details.keys())))

timestamps = range(first_day, last_day + 1, 3600 * 24)

# Retrieves a sorted list of all the characters in the data file
names = entries.keys()
names.sort()

# Generate the HTML page
if os.path.exists(dest):
    shutil.rmtree(dest)

shutil.copytree(os.path.join(os.path.dirname(sys.argv[0]), 'templates'), dest)

input_file = open(os.path.join(dest, 'index.html'), 'r')
content = input_file.read()
input_file.close()

content = content.replace('$ENTRIES$', entries_stringified). \
                  replace('$TIMETAMPS$', str(timestamps)).   \
                  replace('$CHARACTER_NAMES$', str(map(lambda x: str(x), names))). \
                  replace('$CHARACTERS_TO_IGNORE$', str(CHARACTERS_TO_IGNORE)). \
                  replace('$CHARACTERS_COLORS$', str(CHARACTERS_COLORS))

output_file = open(os.path.join(dest, 'index.html'), 'w')
output_file.write(content)
output_file.close()
<|endoftext|>