<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
run.py
------

'''

from __future__ import division, print_function, absolute_import, unicode_literals
from .plot import Plot
from .data import GetK2Stars, GetK2Data, GetK2Planets, GetK2InjectionTestStars
from .compute import Compute
from .utils import ExceptionHook, ExceptionHookPDB, FunctionWrapper
from .pool import Pool
import os
EVEREST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from kplr.config import KPLR_ROOT
import sys
import subprocess
import imp
import time

def DownloadCampaign(campaign, queue = 'build', email = 'rodluger@gmail.com', walltime = 8):
  '''
  Submits a cluster job to the build queue to download all TPFs for a given
  campaign.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'download.pbs')
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,CAMPAIGN=%d' % (EVEREST_ROOT, campaign)
  str_out = os.path.join(EVEREST_ROOT, 'DOWNLOAD_C%02d.log' % campaign)
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'DOWNLOAD_C%02d' % campaign,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def Run(EPIC, **kwargs):
  '''
  Wrapper around ``Compute()`` and ``Plot()``.

  '''

  data = Compute(EPIC, **kwargs)
  if data is not None:
    Plot(data)
    return True
  else:
    return False

def RunSingle(EPIC, debug = False, kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Compute and plot data for a given target.

  '''

  # Get the kwargs
  kwargs = imp.load_source("kwargs", kwargs_file).kwargs

  # Set up our custom exception handlers
  if debug:
    sys.excepthook = ExceptionHookPDB
  else:
    sys.excepthook = ExceptionHook

  # Run
  Run(EPIC, **kwargs)

def RunInjections(depth = 0.01, mask = False, queue = 'vsm',
                  nodes = 5, ppn = 12, walltime = 100,
                  email = 'rodluger@gmail.com',
                  kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for a sample
  of targets chosen for transit injection tests.

  '''

  # Submit the cluster job
  name = 'inject_%.4f%s' % (depth, ('m' if mask else 'u'))
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runinjections.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,MASK=%d,DEPTH=%0.4f,KWARGS_FILE=%s' % (EVEREST_ROOT,
          nodes, int(mask), depth, os.path.abspath(kwargs_file))
  str_out = os.path.join(EVEREST_ROOT, '%s.log' % name)
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', name,
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def RunCandidates(nodes = 5, ppn = 12, walltime = 100, queue = 'vsm',
                  email = 'rodluger@gmail.com',
                  kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for all targets hosting
  confirmed planets or planet candidates.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runcandidates.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,KWARGS_FILE=%s' % (EVEREST_ROOT,
          nodes, os.path.abspath(kwargs_file))
  str_out = os.path.join(EVEREST_ROOT, 'candidates.log')
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'candidates',
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def RunCampaign(campaign, nodes = 5, ppn = 12, walltime = 100,
                email = 'rodluger@gmail.com', queue = 'vsm',
                kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for all targets in a given campaign.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runcampaign.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,KWARGS_FILE=%s,CAMPAIGN=%d' % (EVEREST_ROOT,
          nodes, os.path.abspath(kwargs_file), campaign)
  str_out = os.path.join(EVEREST_ROOT, 'C%02d.log' % campaign)
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'C%02d' % campaign,
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

# ---- PBS routines ---- #

def _RunCandidates(kwargs_file):
  '''
  The actual function that runs all candidates; this must
  be called from ``runcandidates.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Override ``mask_candidates``
    kwargs['mask_candidates'] = True

    # Get all the stars
    stars = [int(p.epic_name[5:]) for p in GetK2Planets()]
    new = [int(f[:9]) for f in os.listdir(os.path.join(EVEREST_ROOT, 'new')) if f.endswith('.npz')]
    stars = list(set(stars + new))

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _RunInjections(kwargs_file, depth, mask):
  '''
  The actual function that runs injection tests; this must
  be called from ``runinjections.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Get all the stars
    stars = [int(s) for s in GetK2InjectionTestStars()]

    # Override ``inject`` and ``run_name``
    kwargs['inject'] = dict(t0 = 0.,
                            per = 3.56789,
                            depth = depth,
                            dur = 0.1,
                            mask = mask)
    kwargs['run_name'] = 'inject_%.4f%s' % (depth, ('m' if mask else 'u'))

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _RunCampaign(campaign, kwargs_file):
  '''
  The actual function that runs a given campaign; this must
  be called from ``runcampaign.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Get all the stars
    stars = GetK2Stars()[campaign]

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _DownloadCampaign(campaign):
  '''
  Download all stars from a given campaign. This is
  called from ``download.pbs``

  '''

  # Get all star IDs for this campaign
  stars = GetK2Stars()[campaign]
  nstars = len(stars)

  # Download the TPF data for each one
  for i, EPIC in enumerate(stars):
    print("Downloading data for EPIC %d (%d/%d)..." % (EPIC, i + 1, nstars))
    if not os.path.exists(os.path.join(KPLR_ROOT, 'data', 'everest',
                          str(EPIC), str(EPIC) + '.npz')):
      try:
        GetK2Data(EPIC)
      except:
        # Some targets could be corrupted
        continue  <commit_msg>bugfix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
run.py
------

'''

from __future__ import division, print_function, absolute_import, unicode_literals
from .plot import Plot
from .data import GetK2Stars, GetK2Data, GetK2Planets, GetK2InjectionTestStars
from .compute import Compute
from .utils import ExceptionHook, ExceptionHookPDB, FunctionWrapper
from .pool import Pool
import os
EVEREST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from kplr.config import KPLR_ROOT
import sys
import subprocess
import imp
import time

def DownloadCampaign(campaign, queue = 'build', email = 'rodluger@gmail.com', walltime = 8):
  '''
  Submits a cluster job to the build queue to download all TPFs for a given
  campaign.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'download.pbs')
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,CAMPAIGN=%d' % (EVEREST_ROOT, campaign)
  str_out = os.path.join(EVEREST_ROOT, 'DOWNLOAD_C%02d.log' % campaign)
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'DOWNLOAD_C%02d' % campaign,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def Run(EPIC, **kwargs):
  '''
  Wrapper around ``Compute()`` and ``Plot()``.

  '''

  data = Compute(EPIC, **kwargs)
  if data is not None:
    Plot(data)
    return True
  else:
    return False

def RunSingle(EPIC, debug = False, kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Compute and plot data for a given target.

  '''

  # Get the kwargs
  kwargs = imp.load_source("kwargs", kwargs_file).kwargs

  # Set up our custom exception handlers
  if debug:
    sys.excepthook = ExceptionHookPDB
  else:
    sys.excepthook = ExceptionHook

  # Run
  Run(EPIC, **kwargs)

def RunInjections(depth = 0.01, mask = False, queue = None,
                  nodes = 5, ppn = 12, walltime = 100,
                  email = 'rodluger@gmail.com',
                  kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for a sample
  of targets chosen for transit injection tests.

  '''

  # Submit the cluster job
  name = 'inject_%.4f%s' % (depth, ('m' if mask else 'u'))
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runinjections.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,MASK=%d,DEPTH=%0.4f,KWARGS_FILE=%s' % (EVEREST_ROOT,
          nodes, int(mask), depth, os.path.abspath(kwargs_file))
  str_out = os.path.join(EVEREST_ROOT, '%s.log' % name)
  qsub_args = ['qsub', pbsfile,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', name,
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']
  if queue is not None:
    qsub_args += ['-q', queue]
  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def RunCandidates(nodes = 5, ppn = 12, walltime = 100, queue = 'vsm',
                  email = 'rodluger@gmail.com',
                  kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for all targets hosting
  confirmed planets or planet candidates.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runcandidates.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,KWARGS_FILE=%s' % (EVEREST_ROOT,
          nodes, os.path.abspath(kwargs_file))
  str_out = os.path.join(EVEREST_ROOT, 'candidates.log')
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'candidates',
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

def RunCampaign(campaign, nodes = 5, ppn = 12, walltime = 100,
                email = 'rodluger@gmail.com', queue = 'vsm',
                kwargs_file = os.path.join(EVEREST_ROOT, 'scripts', 'kwargs.py')):
  '''
  Submits a cluster job to compute and plot data for all targets in a given campaign.

  '''

  # Submit the cluster job
  pbsfile = os.path.join(EVEREST_ROOT, 'everest', 'runcampaign.pbs')
  str_n = 'nodes=%d:ppn=%d,feature=%dcore' % (nodes, ppn, ppn)
  str_w = 'walltime=%d:00:00' % walltime
  str_v = 'EVEREST_ROOT=%s,NODES=%d,KWARGS_FILE=%s,CAMPAIGN=%d' % (EVEREST_ROOT,
          nodes, os.path.abspath(kwargs_file), campaign)
  str_out = os.path.join(EVEREST_ROOT, 'C%02d.log' % campaign)
  qsub_args = ['qsub', pbsfile,
               '-q', queue,
               '-v', str_v,
               '-o', str_out,
               '-j', 'oe',
               '-N', 'C%02d' % campaign,
               '-l', str_n,
               '-l', str_w,
               '-M', email,
               '-m', 'ae']

  # Now we submit the job
  print("Submitting the job...")
  subprocess.call(qsub_args)

# ---- PBS routines ---- #

def _RunCandidates(kwargs_file):
  '''
  The actual function that runs all candidates; this must
  be called from ``runcandidates.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Override ``mask_candidates``
    kwargs['mask_candidates'] = True

    # Get all the stars
    stars = [int(p.epic_name[5:]) for p in GetK2Planets()]
    new = [int(f[:9]) for f in os.listdir(os.path.join(EVEREST_ROOT, 'new')) if f.endswith('.npz')]
    stars = list(set(stars + new))

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _RunInjections(kwargs_file, depth, mask):
  '''
  The actual function that runs injection tests; this must
  be called from ``runinjections.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Get all the stars
    stars = [int(s) for s in GetK2InjectionTestStars()]

    # Override ``inject`` and ``run_name``
    kwargs['inject'] = dict(t0 = 0.,
                            per = 3.56789,
                            depth = depth,
                            dur = 0.1,
                            mask = mask)
    kwargs['run_name'] = 'inject_%.4f%s' % (depth, ('m' if mask else 'u'))

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _RunCampaign(campaign, kwargs_file):
  '''
  The actual function that runs a given campaign; this must
  be called from ``runcampaign.pbs``.

  '''

  # Set up our custom exception handler
  sys.excepthook = ExceptionHook

  # Initialize our multiprocessing pool
  with Pool() as pool:

    # Get the kwargs
    kwargs = imp.load_source("kwargs", kwargs_file).kwargs

    # Get all the stars
    stars = GetK2Stars()[campaign]

    # Compute and plot
    C = FunctionWrapper(Run, **kwargs)
    pool.map(C, stars)

def _DownloadCampaign(campaign):
  '''
  Download all stars from a given campaign. This is
  called from ``download.pbs``

  '''

  # Get all star IDs for this campaign
  stars = GetK2Stars()[campaign]
  nstars = len(stars)

  # Download the TPF data for each one
  for i, EPIC in enumerate(stars):
    print("Downloading data for EPIC %d (%d/%d)..." % (EPIC, i + 1, nstars))
    if not os.path.exists(os.path.join(KPLR_ROOT, 'data', 'everest',
                          str(EPIC), str(EPIC) + '.npz')):
      try:
        GetK2Data(EPIC)
      except:
        # Some targets could be corrupted
        continue  <|endoftext|>


<commit_before>
# Make histogram of a_world values and cut below peak

output = open("shapecutCatalog.txt" , "w")

with open('temp.txt', 'r') as f:
    for object in (raw.strip().split() for raw in f):
        if float(object[11]) < 2e-4:
            output.write("%10s" % object[0] + "%12s" % object[1] +  "%12s" % object[2]
            + "%10s" % object[3] + "%13s" % object[4] + "%10s" % object[5]
            + "%13s" % object[6]+ "%10s" % object[7] + "%13s" % object[8]
            + "%10s" % object[9] + "%13s" % object[10] + "%13s" % object[11] +
            "%13s" % object[12] + "\n")

<commit_msg>Adding documentation. Still a very incomplete program<commit_after>'''
AUTHOR
    Alexa Villaume, UCSC

PURPOSE
    To find the point-source objects in a catalog and make a
    cut to only include them

INPUT PARAMETERS
    The catalog for the cuts to be made ont

FILES CREATED
    Shape cut catalog

NOTES
    This is very incomplete. As of now you need to know where to make the
    cut. Plan to make this more complete and interactive.

'''

# Make histogram of a_world values and cut below peak

output = open("shapecutCatalog.txt" , "w")

with open('temp.txt', 'r') as f:
    for object in (raw.strip().split() for raw in f):
        if float(object[11]) < 2e-4:
            output.write("%10s" % object[0] + "%12s" % object[1] +  "%12s" % object[2]
            + "%10s" % object[3] + "%13s" % object[4] + "%10s" % object[5]
            + "%13s" % object[6]+ "%10s" % object[7] + "%13s" % object[8]
            + "%10s" % object[9] + "%13s" % object[10] + "%13s" % object[11] +
            "%13s" % object[12] + "\n")

<|endoftext|>


<commit_before>tc = 1
N = int(raw_input())
while N:
    content = []
    for line in range(N):
        content.append(raw_input())
    input_html = "".join(content)
    print "Case #%d: %s" % (tc, input_html)
    tc += 1
    N = int(raw_input())

<commit_msg>formulation<commit_after>import re

def solve(text):
    # greedy regex search
    tags = re.compile(r'<.*?>')
    for tag in tags.finditer(text):
        # if not find upper case alphabetic character
        # bad character in tag name
        # if more than 10 characters in tag.
        # too many/few characters in tag name
        # No ending tag
        # expected </xxxxx>
        # No begin tag
        # no matching begin tag.
        if "/" in tag:
            if (len(tag[2:-1]) > 10):
                return "too many/few characters in tag name"
        else:
            if len(tag[1:-1] > 10):
                return "too many/few characters in tag name"
    return "0"

tc = 1
N = int(raw_input())
while N:
    content = []
    for line in range(N):
        content.append(raw_input())
    input_html = "".join(content)
    result = solve(input_html)
    print "Case #%d: %s" % (tc, input_html)
    tc += 1
    N = int(raw_input())

<|endoftext|>


<commit_before>from __future__ import unicode_literals

from waldur_core.core import WaldurExtension

class AuthSocialExtension(WaldurExtension):

    class Settings:
        # wiki: https://opennode.atlassian.net/wiki/display/WD/AuthSocial+plugin+configuration
        WALDUR_AUTH_SOCIAL = {
            'GOOGLE_SECRET': '',
            'GOOGLE_CLIENT_ID': '',

            'FACEBOOK_SECRET': '',
            'FACEBOOK_CLIENT_ID': '',

            'SMARTIDEE_SECRET': '',
            'SMARTIDEE_CLIENT_ID': '',

            'TARA_SECRET': '',
            'TARA_CLIENT_ID': '',
            'TARA_SANDBOX': True,

            'USER_ACTIVATION_URL_TEMPLATE': 'http://example.com/#/activate/{user_uuid}/{token}/',
        }

    @staticmethod
    def get_public_settings():
        return [
            'GOOGLE_CLIENT_ID',
            'FACEBOOK_CLIENT_ID',
            'SMARTIDEE_CLIENT_ID',
            'TARA_CLIENT_ID',
            'TARA_SANDBOX',
        ]

    @staticmethod
    def django_app():
        return 'waldur_auth_social'

    @staticmethod
    def django_urls():
        from .urls import urlpatterns
        return urlpatterns
<commit_msg>Expose TARA label via REST API [WAL-2216]<commit_after>from __future__ import unicode_literals

from waldur_core.core import WaldurExtension

class AuthSocialExtension(WaldurExtension):

    class Settings:
        # wiki: https://opennode.atlassian.net/wiki/display/WD/AuthSocial+plugin+configuration
        WALDUR_AUTH_SOCIAL = {
            'GOOGLE_SECRET': '',
            'GOOGLE_CLIENT_ID': '',

            'FACEBOOK_SECRET': '',
            'FACEBOOK_CLIENT_ID': '',

            'SMARTIDEE_SECRET': '',
            'SMARTIDEE_CLIENT_ID': '',

            'TARA_SECRET': '',
            'TARA_CLIENT_ID': '',
            'TARA_SANDBOX': True,
            'TARA_LABEL': 'Riigi Autentimisteenus',

            'USER_ACTIVATION_URL_TEMPLATE': 'http://example.com/#/activate/{user_uuid}/{token}/',
        }

    @staticmethod
    def get_public_settings():
        return [
            'GOOGLE_CLIENT_ID',
            'FACEBOOK_CLIENT_ID',
            'SMARTIDEE_CLIENT_ID',
            'TARA_CLIENT_ID',
            'TARA_SANDBOX',
            'TARA_LABEL',
        ]

    @staticmethod
    def django_app():
        return 'waldur_auth_social'

    @staticmethod
    def django_urls():
        from .urls import urlpatterns
        return urlpatterns
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

"""
   File Name: eval_indexer.py
      Author: Wan Ji
      E-mail: wanji@live.com
  Created on: Tue Jul 28 13:31:06 2015 CST
"""
DESCRIPTION = """
"""

import os
import argparse
import logging
import time
import tempfile

import numpy as np
from scipy.io import loadmat, savemat

from hdidx import indexer
from hdidx import util

def runcmd(cmd):
    """ Run command.
    """

    logging.info("%s" % cmd)
    os.system(cmd)

def getargs():
    """ Parse program arguments.
    """

    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('dataset', type=str,
                        help='path of the dataset')
    parser.add_argument('--exp_dir', type=str,
                        help='directory for saving experimental results')
    parser.add_argument("--nbits", type=int, nargs='+', default=[64],
                        help="number of bits")
    parser.add_argument("--topk", type=int, default=100,
                        help="retrieval `topk` nearest neighbors")
    parser.add_argument("--log", type=str, default="INFO",
                        help="log level")

    return parser.parse_args()

class Dataset(object):
    def __init__(self, data_path):
        data_dic = loadmat(data_path)
        for key, val in data_dic.iteritems():
            if not key.startswith("__"):
                setattr(self, key, val.T)
        self.groundtruth = self.groundtruth[:, :1]
        self.nlrn = self.learn.shape[0]
        self.nbae = self.base.shape[0]
        self.nqry = self.query.shape[0]
        self.name = data_path.split("/")[-1].split(".")[0]

def compute_stats(ids_gnd, ids_qry, k):
    nquery = ids_qry.shape[0]
    nn_ranks_pqc = np.zeros(nquery)
    for i in xrange(nquery):
        ret_lst = ids_qry[i, :].tolist()
        nn_pos = []
        for gnd_id in ids_gnd[i, :]:
            try:
                nn_pos.append(ret_lst.index(gnd_id))
            except ValueError:
                pass

        if len(nn_pos) == 1:
            nn_ranks_pqc[i] = nn_pos[0]
        else:
            nn_ranks_pqc[i] = k + 1

    nn_ranks_pqc.sort()

    v_recall = []
    for i in [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000,
              10000, 20000, 50000, 100000, 200000, 500000, 1000000]:
        if i > k:
            break
        r_at_i = (nn_ranks_pqc < i).sum().astype(np.float) / nquery
        logging.info('recall@%3d = %.4f' % (i, r_at_i))
        v_recall.append((i, r_at_i))
    return v_recall

BLOCK_SIZE = 1000

def save_result(rslt_path, ids, dis):
    nret = ids.shape[1]
    if nret > BLOCK_SIZE:
        os.makedirs(rslt_path)
        for i in xrange(0, nret, BLOCK_SIZE):
            savemat(os.path.join(rslt_path, "%d.mat" % i),
                    {'ids': ids[:, i:i+BLOCK_SIZE],
                     'dis': dis[:, i:i+BLOCK_SIZE]})
    else:
        savemat(rslt_path, {'ids': ids, 'dis': dis})

def load_result(rslt_path):
    if os.path.isdir(rslt_path):
        v_ids = []
        v_dis = []
        files = sorted([rslt_file for rslt_file in os.listdir(rslt_path)
                        if rslt_file.endseith(".mat")],
                       key=lambda x: int(x.split(".")[0]))
        for rslt_file in files:
            rslt = loadmat(rslt_path)
            v_ids.append(rslt['ids'])
            v_dis.append(rslt['dis'])
        return np.hstack(v_ids), np.hstack(v_dis)
    else:
        rslt = loadmat(rslt_path)
        ids = rslt['ids']
        dis = rslt['dis']
    return ids, dis

def eval_indexer(data, indexer_param, dsname, topk):
    CurIndexer = indexer_param['indexer']
    build_param = indexer_param['build_param']
    index_prefix = indexer_param['index_prefix']
    info_path = index_prefix + ".info"
    lmdb_path = index_prefix + ".idx"
    rslt_path = index_prefix + "-result.mat"

    build_param['vals'] = data.learn

    idx = CurIndexer()
    if os.path.exists(info_path):
        idx.load(info_path)
    else:
        idx.build(build_param)
        idx.save(info_path)

    do_add = not os.path.exists(lmdb_path)

    idx.set_storage('lmdb', {
        'path': lmdb_path,
        'clear': do_add,
    })
    if do_add:
        idx.add(data.base)

    if os.path.exists(rslt_path):
        logging.info("Loading saved retrieval results ...")
        ids, dis = load_result(rslt_path)
        logging.info("\tDone!")
    else:
        logging.info("Searching ...")
        ids, dis = idx.search(data.query, topk=topk)
        save_result(rslt_path, ids, dis)
        logging.info("\tDone!")
    return compute_stats(data.groundtruth, ids, topk)

def main(args):
    """ Main entry.
    """
    exp_dir = args.exp_dir if args.exp_dir \
        else tempfile.mkdtemp(prefix="hdidx-eval-")
    logging.info("saving experimental results to: %s" % exp_dir)
    if not os.path.exists(exp_dir):
        os.makedirs(exp_dir)
    report = os.path.join(exp_dir, "report.txt")

    if os.path.exists(report):
        logging.warning("report file already exists")

    with open(report, "a") as rptf:
        rptf.write("*" * 64 + "\n")
        rptf.write("* %s\n" % time.asctime())
        rptf.write("*" * 64 + "\n")

    data = Dataset(args.dataset)
    logging.info("learn/base/query: %d/%d/%d" %
                 (data.nlrn, data.nbae, data.nqry))

    for nbits in args.nbits:
        if nbits % 8 != 0:
            raise util.HDIdxException("`nbits` must be multiple of 8")
        nsubq = nbits / 8
        v_indexer_param = [
            {
                'indexer': indexer.PQIndexer,
                'build_param': {
                    'nsubq': nsubq,
                    'nsubqbits': 8,
                },
                'index_prefix': '%s/%s_%s_nsubq%d' % (
                    exp_dir, data.name, 'pq', nsubq),
            },
            {
                'indexer': indexer.SHIndexer,
                'build_param': {
                    'nbits': nbits,
                },
                'index_prefix': '%s/%s_%s_nbits%d' % (
                    exp_dir, data.name, 'sh', nbits),
            },
        ]

        for indexer_param in v_indexer_param:
            v_recall = eval_indexer(data, indexer_param, data.name, args.topk)
            with open(report, "a") as rptf:
                rptf.write("=" * 64 + "\n")
                rptf.write(str(indexer_param) + "\n")
                rptf.write("-" * 64 + "\n")
                for recall in v_recall:
                    rptf.write("recall@%-8d%.4f\n" % (recall[0], recall[1]))
    logging.info("All done! You can check the report in: %s" % report)

if __name__ == '__main__':
    args = getargs()
    numeric_level = getattr(logging, args.log.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError("Invalid log level: " + args.log)
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                        level=numeric_level)
    main(args)
<commit_msg>simplify the report file<commit_after>#!/usr/bin/env python
# coding: utf-8

"""
   File Name: eval_indexer.py
      Author: Wan Ji
      E-mail: wanji@live.com
  Created on: Tue Jul 28 13:31:06 2015 CST
"""
DESCRIPTION = """
"""

import os
import argparse
import logging
import time
import tempfile

import numpy as np
from scipy.io import loadmat, savemat

from hdidx import indexer
from hdidx import util

def runcmd(cmd):
    """ Run command.
    """

    logging.info("%s" % cmd)
    os.system(cmd)

def getargs():
    """ Parse program arguments.
    """

    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('dataset', type=str,
                        help='path of the dataset')
    parser.add_argument('--exp_dir', type=str,
                        help='directory for saving experimental results')
    parser.add_argument("--nbits", type=int, nargs='+', default=[64],
                        help="number of bits")
    parser.add_argument("--topk", type=int, default=100,
                        help="retrieval `topk` nearest neighbors")
    parser.add_argument("--log", type=str, default="INFO",
                        help="log level")

    return parser.parse_args()

class Dataset(object):
    def __init__(self, data_path):
        data_dic = loadmat(data_path)
        for key, val in data_dic.iteritems():
            if not key.startswith("__"):
                setattr(self, key, val.T)
        self.groundtruth = self.groundtruth[:, :1]
        self.nlrn = self.learn.shape[0]
        self.nbae = self.base.shape[0]
        self.nqry = self.query.shape[0]
        self.name = data_path.split("/")[-1].split(".")[0]

def compute_stats(ids_gnd, ids_qry, k):
    nquery = ids_qry.shape[0]
    nn_ranks_pqc = np.zeros(nquery)
    for i in xrange(nquery):
        ret_lst = ids_qry[i, :].tolist()
        nn_pos = []
        for gnd_id in ids_gnd[i, :]:
            try:
                nn_pos.append(ret_lst.index(gnd_id))
            except ValueError:
                pass

        if len(nn_pos) == 1:
            nn_ranks_pqc[i] = nn_pos[0]
        else:
            nn_ranks_pqc[i] = k + 1

    nn_ranks_pqc.sort()

    v_recall = []
    for i in [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000,
              10000, 20000, 50000, 100000, 200000, 500000, 1000000]:
        if i > k:
            break
        r_at_i = (nn_ranks_pqc < i).sum().astype(np.float) / nquery
        logging.info('recall@%3d = %.4f' % (i, r_at_i))
        v_recall.append((i, r_at_i))
    return v_recall

BLOCK_SIZE = 1000

def save_result(rslt_path, ids, dis):
    nret = ids.shape[1]
    if nret > BLOCK_SIZE:
        os.makedirs(rslt_path)
        for i in xrange(0, nret, BLOCK_SIZE):
            savemat(os.path.join(rslt_path, "%d.mat" % i),
                    {'ids': ids[:, i:i+BLOCK_SIZE],
                     'dis': dis[:, i:i+BLOCK_SIZE]})
    else:
        savemat(rslt_path, {'ids': ids, 'dis': dis})

def load_result(rslt_path):
    if os.path.isdir(rslt_path):
        v_ids = []
        v_dis = []
        files = sorted([rslt_file for rslt_file in os.listdir(rslt_path)
                        if rslt_file.endswith(".mat")],
                       key=lambda x: int(x.split(".")[0]))
        for rslt_file in files:
            rslt = loadmat(os.path.join(rslt_path, rslt_file))
            v_ids.append(rslt['ids'])
            v_dis.append(rslt['dis'])
        return np.hstack(v_ids), np.hstack(v_dis)
    else:
        rslt = loadmat(rslt_path)
        ids = rslt['ids']
        dis = rslt['dis']
    return ids, dis

def eval_indexer(data, indexer_param, dsname, topk):
    CurIndexer = indexer_param['indexer']
    build_param = indexer_param['build_param']
    index_prefix = indexer_param['index_prefix']
    info_path = index_prefix + ".info"
    lmdb_path = index_prefix + ".idx"
    rslt_path = index_prefix + "-result.mat"

    build_param['vals'] = data.learn

    idx = CurIndexer()
    if os.path.exists(info_path):
        idx.load(info_path)
    else:
        idx.build(build_param)
        idx.save(info_path)

    do_add = not os.path.exists(lmdb_path)

    idx.set_storage('lmdb', {
        'path': lmdb_path,
        'clear': do_add,
    })
    if do_add:
        idx.add(data.base)

    if os.path.exists(rslt_path):
        logging.info("Loading saved retrieval results ...")
        ids, dis = load_result(rslt_path)
        logging.info("\tDone!")
    else:
        logging.info("Searching ...")
        ids, dis = idx.search(data.query, topk=topk)
        save_result(rslt_path, ids, dis)
        logging.info("\tDone!")
    return compute_stats(data.groundtruth, ids, topk)

def main(args):
    """ Main entry.
    """
    exp_dir = args.exp_dir if args.exp_dir \
        else tempfile.mkdtemp(prefix="hdidx-eval-")
    logging.info("saving experimental results to: %s" % exp_dir)
    if not os.path.exists(exp_dir):
        os.makedirs(exp_dir)
    report = os.path.join(exp_dir, "report.txt")

    if os.path.exists(report):
        logging.warning("report file already exists")

    with open(report, "a") as rptf:
        rptf.write("*" * 64 + "\n")
        rptf.write("* %s\n" % time.asctime())
        rptf.write("*" * 64 + "\n")

    data = Dataset(args.dataset)
    logging.info("learn/base/query: %d/%d/%d" %
                 (data.nlrn, data.nbae, data.nqry))

    for nbits in args.nbits:
        if nbits % 8 != 0:
            raise util.HDIdxException("`nbits` must be multiple of 8")
        nsubq = nbits / 8
        v_indexer_param = [
            {
                'indexer': indexer.PQIndexer,
                'build_param': {
                    'nsubq': nsubq,
                    'nsubqbits': 8,
                },
                'index_prefix': '%s/%s_%s_nsubq%d' % (
                    exp_dir, data.name, 'pq', nsubq),
            },
            {
                'indexer': indexer.SHIndexer,
                'build_param': {
                    'nbits': nbits,
                },
                'index_prefix': '%s/%s_%s_nbits%d' % (
                    exp_dir, data.name, 'sh', nbits),
            },
        ]

        for indexer_param in v_indexer_param:
            v_recall = eval_indexer(data, indexer_param, data.name, args.topk)
            with open(report, "a") as rptf:
                rptf.write("=" * 64 + "\n")
                # rptf.write(str(indexer_param) + "\n")
                rptf.write("index_%s-nbits_%s\n" % (
                    indexer_param['indexer'].__name__, nbits))
                rptf.write("-" * 64 + "\n")
                for recall in v_recall:
                    rptf.write("recall@%-8d%.4f\n" % (recall[0], recall[1]))
    logging.info("All done! You can check the report in: %s" % report)

if __name__ == '__main__':
    args = getargs()
    numeric_level = getattr(logging, args.log.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError("Invalid log level: " + args.log)
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                        level=numeric_level)
    main(args)
<|endoftext|>


<commit_before># coding: utf-8

from .util import LowerCaseDict, log_exceptions
from .exceptions import *

from typing import Undefined, Optional
import logging

logger = logging.getLogger('tornado.general')

class Channel(object):
    '''IRC channel'''
    _name = Undefined(str)
    catalog = Undefined('ChannelCatalog')
    topic = Undefined(Optional[str])
    users = Undefined(LowerCaseDict) # users = {'nick': {'user': user}}
    key = Undefined(Optional[str])
    _limit = Undefined(int)

    @property
    def channel(self):
        return self._channel

    @channel.setter
    def channel(self, value):
        self._channel = value

    @property
    def limit(self):
        return self._limit

    @limit.setter
    def limit(self, value):
        chanlimit = self.catalog.server.settings['chanlimit']
        self._limit = min(self.server_chanlimit, value)

    def __init__(self, name: str, catalog: 'ChannelCatalog'):
        if not name or name[0] != '#':
            raise NoSuchChannelError(channel = name)

        self.name = name
        self.catalog = catalog
        self.topic = None
        self.users = LowerCaseDict()
        self.key = None
        self._limit = catalog.server.settings['chanlimit']

    def broadcast_message(self, *args, **kwargs):
        '''Broadcast message to all channel members.'''
        for nick in self.users:
            target = self.users[nick]['user']
            target.send_message(*args, **kwargs)

    def join(self, user: 'User', key: Optional[str] = None):
        '''Joins user to this channel.'''
        if user.nick in self.users:
            return
        if self.key and key and self.key != key:
            raise BadChannelKeyError()
        if len(self.users) >= self.limit:
            raise ChannelIsFullError(channel = self.name)

        self.users[user.nick] = {'user': user}
        user.channels[self.name] = self
        self.broadcast_message('CMD_JOIN',
                               useraddr = user.address,
                               channel = self.name)
        self.send_topic(user)
        self.send_names(user)

    def part(self, user: 'User', message: str = None):
        '''Parts user from this channel.'''
        if user.nick not in self.users:
            raise NotOnChannelError()
        if not message:
            message = ''
        self.broadcast_message('CMD_PART',
                               useraddr = user.address,
                               channel = self.name,
                               message = message)
        del self.users[user.nick]
        del user.channels[self.name]

        if len(self.users) == 0:
            catalog = self.catalog
            self.catalog = None
            del catalog[self.name]
            self.server = None

    def quit(self, user: 'User', message: str = None):
        '''Parts user from this channel.'''
        if not message:
            message = ''
        self.broadcast_message('CMD_QUIT',
                               useraddr = user.address,
                               message = message)
        if user.nick in self.users:
            del self.users[user.nick]
        if self.name in user.channels:
            del user.channels[self.name]

    def set_topic(self, user: 'User', topic: str = ''):
        '''Sets channel topic.'''
        self.topic = topic
        self.broadcast_message('CMD_TOPIC',
                               useraddr = user.address,
                               channel = self.name,
                               topic = topic)

    def send_topic(self, user: 'User'):
        '''Send channel topic to user.'''
        if self.topic:
            user.send_message('RPL_TOPIC',
                              channel = self.name, topic = self.topic)
        else:
            user.send_message('RPL_NOTOPIC', channel = self.name)

    @log_exceptions
    def send_names(self, user: 'User', suppress_end = False):
        '''Send current channel members nicks to user.'''
        users = self.users
        nicklist = [users[nick]['user'].nick for nick in users]
        nicklist_str = ' '.join(nicklist)
        try:
            user.send_message('RPL_NAMREPLY',
                              channel = self.name,
                              chantype = '=',
                              nicklist = nicklist_str)
        except TooLongMessageException as e:
            length = e.length
            baselen = length - len(nicklist_str.encode('utf-8'))
            maxnicks = (512 - baselen) // user.server.settings['nicklen']
            if maxnicks <= 0:
                raise
            for i in range(0, len(nicklist) // maxnicks + 1):
                nicklist_str = ' '.join(nicklist[maxnicks * i:maxnicks * i + maxnicks])
                user.send_message('RPL_NAMREPLY',
                                  channel = self.name,
                                  chantype = '=',
                                  nicklist = nicklist_str)
                start = maxnicks

        if not suppress_end:
            user.send_message('RPL_ENDOFNAMES', channel = self.name)

    def send_privmsg(self, sender: str, recipient: str, text: str):
        '''Send PRIVMSG command to channel\'s users.'''
        sendernick = sender.split('!')[0]
        if sendernick not in self.users:
            raise CannotSendToChanError()

        for nick in self.users:
            target = self.users[nick]['user']
            if target.address == sender:
                continue
            target.send_privmsg(sender = sender, recipient = recipient,
                                text = text)

    def send_notice(self, sender: str, recipient: str, text: str):
        '''Send NOTICE command to channel\'s users.'''
        sendernick = sender.split('!')[0]
        if sendernick not in self.users:
            return

        for nick in self.users:
            target = self.users[nick]['user']
            if target.address == sender:
                continue
            target.send_notice(sender = sender, recipient = recipient,
                               text = text)

class ChannelCatalog(LowerCaseDict):
    '''Catalog with all channels within an IRC server/network.'''
    server = Undefined('Server')

    def __init__(self, server: 'Server', *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.server = server

    def join(self, user: 'User', name: str, key: Optional[str] = None):
        '''Add an user to a channel. Returns channel.'''
        # Create or get channel
        if name in self:
            channel = self[name]
        else:
            channel = Channel(name = name, catalog = self)
            self[name] = channel

        # Join
        if key:
            channel.join(user = user, key = key)
        else:
            channel.join(user = user)
<commit_msg>channel: bug fix<commit_after># coding: utf-8

from .util import LowerCaseDict, log_exceptions
from .exceptions import *

from typing import Undefined, Optional
import logging

logger = logging.getLogger('tornado.general')

class Channel(object):
    '''IRC channel'''
    _name = Undefined(str)
    catalog = Undefined('ChannelCatalog')
    topic = Undefined(Optional[str])
    users = Undefined(LowerCaseDict) # users = {'nick': {'user': user}}
    key = Undefined(Optional[str])
    _limit = Undefined(int)

    @property
    def channel(self):
        return self._channel

    @channel.setter
    def channel(self, value):
        self._channel = value

    @property
    def limit(self):
        return self._limit

    @limit.setter
    def limit(self, value):
        chanlimit = self.catalog.server.settings['chanlimit']
        self._limit = min(chanlimit, value)

    def __init__(self, name: str, catalog: 'ChannelCatalog'):
        if not name or name[0] != '#':
            raise NoSuchChannelError(channel = name)

        self.name = name
        self.catalog = catalog
        self.topic = None
        self.users = LowerCaseDict()
        self.key = None
        self._limit = catalog.server.settings['chanlimit']

    def broadcast_message(self, *args, **kwargs):
        '''Broadcast message to all channel members.'''
        for nick in self.users:
            target = self.users[nick]['user']
            target.send_message(*args, **kwargs)

    def join(self, user: 'User', key: Optional[str] = None):
        '''Joins user to this channel.'''
        if user.nick in self.users:
            return
        if self.key and key and self.key != key:
            raise BadChannelKeyError()
        if len(self.users) >= self.limit:
            raise ChannelIsFullError(channel = self.name)

        self.users[user.nick] = {'user': user}
        user.channels[self.name] = self
        self.broadcast_message('CMD_JOIN',
                               useraddr = user.address,
                               channel = self.name)
        self.send_topic(user)
        self.send_names(user)

    def part(self, user: 'User', message: str = None):
        '''Parts user from this channel.'''
        if user.nick not in self.users:
            raise NotOnChannelError()
        if not message:
            message = ''
        self.broadcast_message('CMD_PART',
                               useraddr = user.address,
                               channel = self.name,
                               message = message)
        del self.users[user.nick]
        del user.channels[self.name]

        if len(self.users) == 0:
            catalog = self.catalog
            self.catalog = None
            del catalog[self.name]
            self.server = None

    def quit(self, user: 'User', message: str = None):
        '''Parts user from this channel.'''
        if not message:
            message = ''
        self.broadcast_message('CMD_QUIT',
                               useraddr = user.address,
                               message = message)
        if user.nick in self.users:
            del self.users[user.nick]
        if self.name in user.channels:
            del user.channels[self.name]

    def set_topic(self, user: 'User', topic: str = ''):
        '''Sets channel topic.'''
        self.topic = topic
        self.broadcast_message('CMD_TOPIC',
                               useraddr = user.address,
                               channel = self.name,
                               topic = topic)

    def send_topic(self, user: 'User'):
        '''Send channel topic to user.'''
        if self.topic:
            user.send_message('RPL_TOPIC',
                              channel = self.name, topic = self.topic)
        else:
            user.send_message('RPL_NOTOPIC', channel = self.name)

    @log_exceptions
    def send_names(self, user: 'User', suppress_end = False):
        '''Send current channel members nicks to user.'''
        users = self.users
        nicklist = [users[nick]['user'].nick for nick in users]
        nicklist_str = ' '.join(nicklist)
        try:
            user.send_message('RPL_NAMREPLY',
                              channel = self.name,
                              chantype = '=',
                              nicklist = nicklist_str)
        except TooLongMessageException as e:
            length = e.length
            baselen = length - len(nicklist_str.encode('utf-8'))
            maxnicks = (512 - baselen) // user.server.settings['nicklen']
            if maxnicks <= 0:
                raise
            for i in range(0, len(nicklist) // maxnicks + 1):
                nicklist_str = ' '.join(nicklist[maxnicks * i:maxnicks * i + maxnicks])
                user.send_message('RPL_NAMREPLY',
                                  channel = self.name,
                                  chantype = '=',
                                  nicklist = nicklist_str)
                start = maxnicks

        if not suppress_end:
            user.send_message('RPL_ENDOFNAMES', channel = self.name)

    def send_privmsg(self, sender: str, recipient: str, text: str):
        '''Send PRIVMSG command to channel\'s users.'''
        sendernick = sender.split('!')[0]
        if sendernick not in self.users:
            raise CannotSendToChanError()

        for nick in self.users:
            target = self.users[nick]['user']
            if target.address == sender:
                continue
            target.send_privmsg(sender = sender, recipient = recipient,
                                text = text)

    def send_notice(self, sender: str, recipient: str, text: str):
        '''Send NOTICE command to channel\'s users.'''
        sendernick = sender.split('!')[0]
        if sendernick not in self.users:
            return

        for nick in self.users:
            target = self.users[nick]['user']
            if target.address == sender:
                continue
            target.send_notice(sender = sender, recipient = recipient,
                               text = text)

class ChannelCatalog(LowerCaseDict):
    '''Catalog with all channels within an IRC server/network.'''
    server = Undefined('Server')

    def __init__(self, server: 'Server', *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.server = server

    def join(self, user: 'User', name: str, key: Optional[str] = None):
        '''Add an user to a channel. Returns channel.'''
        # Create or get channel
        if name in self:
            channel = self[name]
        else:
            channel = Channel(name = name, catalog = self)
            self[name] = channel

        # Join
        if key:
            channel.join(user = user, key = key)
        else:
            channel.join(user = user)
<|endoftext|>


<commit_before># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

from SCons.Action import Action  # pylint: disable=import-error
from SCons.Script import ARGUMENTS  # pylint: disable=import-error
from SCons.Script import AlwaysBuild  # pylint: disable=import-error

from platformio import compat, fs

def VerboseAction(_, act, actstr):
    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
        return act
    return Action(act, actstr)

def PioClean(env, clean_dir):
    def _relpath(path):
        if compat.WINDOWS:
            prefix = os.getcwd()[:2].lower()
            if ":" not in prefix or not path.lower().startswith(prefix):
                return path
        return os.path.relpath(path)

    if not os.path.isdir(clean_dir):
        print("Build environment is clean")
        env.Exit(0)
    clean_rel_path = _relpath(clean_dir)
    for root, _, files in os.walk(clean_dir):
        for f in files:
            dst = os.path.join(root, f)
            os.remove(dst)
            print(
                "Removed %s"
                % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
            )
    print("Done cleaning")
    fs.rmtree(clean_dir)
    env.Exit(0)

def AddTarget(  # pylint: disable=too-many-arguments
    env,
    name,
    dependencies,
    actions,
    title=None,
    description=None,
    group="Generic",
    always_build=True,
):
    if "__PIO_TARGETS" not in env:
        env["__PIO_TARGETS"] = {}
    assert name not in env["__PIO_TARGETS"]
    env["__PIO_TARGETS"][name] = dict(
        name=name, title=title, description=description, group=group
    )
    target = env.Alias(name, dependencies, actions)
    if always_build:
        AlwaysBuild(target)
    return target

def AddPlatformTarget(env, *args, **kwargs):
    return env.AddTarget(group="Platform", *args, **kwargs)

def AddCustomTarget(env, *args, **kwargs):
    return env.AddTarget(group="Custom", *args, **kwargs)

def DumpTargets(env):
    targets = env.get("__PIO_TARGETS") or {}
    # pre-fill default targets if embedded dev-platform
    if env.PioPlatform().is_embedded() and not any(
        t["group"] == "Platform" for t in targets.values()
    ):
        targets["upload"] = dict(name="upload", group="Platform", title="Upload")
    targets["compiledb"] = dict(
        name="compiledb",
        title="Compilation Database",
        description="Generate compilation database `compile_commands.json`",
        group="Advanced",
    )
    targets["clean"] = dict(name="clean", title="Clean", group="Generic")
    return list(targets.values())

def exists(_):
    return True

def generate(env):
    env.AddMethod(VerboseAction)
    env.AddMethod(PioClean)
    env.AddMethod(AddTarget)
    env.AddMethod(AddPlatformTarget)
    env.AddMethod(AddCustomTarget)
    env.AddMethod(DumpTargets)
    return env
<commit_msg>Don't print relative paths with double-dot<commit_after># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import os

from SCons.Action import Action  # pylint: disable=import-error
from SCons.Script import ARGUMENTS  # pylint: disable=import-error
from SCons.Script import AlwaysBuild  # pylint: disable=import-error

from platformio import compat, fs

def VerboseAction(_, act, actstr):
    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
        return act
    return Action(act, actstr)

def PioClean(env, clean_dir):
    def _relpath(path):
        if compat.WINDOWS:
            prefix = os.getcwd()[:2].lower()
            if (
                ":" not in prefix
                or not path.lower().startswith(prefix)
                or os.path.relpath(path).startswith("..")
            ):
                return path
        return os.path.relpath(path)

    if not os.path.isdir(clean_dir):
        print("Build environment is clean")
        env.Exit(0)
    clean_rel_path = _relpath(clean_dir)
    for root, _, files in os.walk(clean_dir):
        for f in files:
            dst = os.path.join(root, f)
            os.remove(dst)
            print(
                "Removed %s"
                % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
            )
    print("Done cleaning")
    fs.rmtree(clean_dir)
    env.Exit(0)

def AddTarget(  # pylint: disable=too-many-arguments
    env,
    name,
    dependencies,
    actions,
    title=None,
    description=None,
    group="Generic",
    always_build=True,
):
    if "__PIO_TARGETS" not in env:
        env["__PIO_TARGETS"] = {}
    assert name not in env["__PIO_TARGETS"]
    env["__PIO_TARGETS"][name] = dict(
        name=name, title=title, description=description, group=group
    )
    target = env.Alias(name, dependencies, actions)
    if always_build:
        AlwaysBuild(target)
    return target

def AddPlatformTarget(env, *args, **kwargs):
    return env.AddTarget(group="Platform", *args, **kwargs)

def AddCustomTarget(env, *args, **kwargs):
    return env.AddTarget(group="Custom", *args, **kwargs)

def DumpTargets(env):
    targets = env.get("__PIO_TARGETS") or {}
    # pre-fill default targets if embedded dev-platform
    if env.PioPlatform().is_embedded() and not any(
        t["group"] == "Platform" for t in targets.values()
    ):
        targets["upload"] = dict(name="upload", group="Platform", title="Upload")
    targets["compiledb"] = dict(
        name="compiledb",
        title="Compilation Database",
        description="Generate compilation database `compile_commands.json`",
        group="Advanced",
    )
    targets["clean"] = dict(name="clean", title="Clean", group="Generic")
    return list(targets.values())

def exists(_):
    return True

def generate(env):
    env.AddMethod(VerboseAction)
    env.AddMethod(PioClean)
    env.AddMethod(AddTarget)
    env.AddMethod(AddPlatformTarget)
    env.AddMethod(AddCustomTarget)
    env.AddMethod(DumpTargets)
    return env
<|endoftext|>


<commit_before>"""
A rating system for player performance
"""

_RATINGS = dict()
_DEFAULT_RATING = 1.0
_MINIMUM_RATING = 0.001
_FRAGS_PER_SECOND = 0.016
_ABSORBER = 0.3

def get_rating(nick):
	if not nick in _RATINGS:
		_RATINGS[nick] = _DEFAULT_RATING
	return _RATINGS[nick]

def set_ratings(timelines):
	for timeline in timelines:
		for i in xrange(len(timeline)):
			p = timeline[i]
			if p.elo > 0:
				_RATINGS[p.nick] = p.elo

def team_average(game, team_id):
	avg = (0.0, 0)
	for pid, p in game.players.items():
		if not hasattr(p, 'team_id'):
			continue
		elif p.team_id == team_id:
			avg = (avg[0] + get_rating(p.nick), avg[1]+1)
	return avg[0] / avg[1]

def other_team(team_id):
	if team_id == 1:
		return 2
	elif team_id == 2:
		return 1
	else:
		return team_id

def predict_player(player, game):
	avg = team_average(game, other_team(player.team_id))
	rel_frags = (avg + get_rating(player.nick)) / avg
	# TODO time could be wrong if the player joined later
	return game.game_duration * _FRAGS_PER_SECOND * rel_frags

def rating_adaption(player, game):
	pred = predict_player(player, game)
	adapt = _ABSORBER * ((player.kill_count - pred) / game.frag_count)
	if get_rating(player.nick) - adapt < _MINIMUM_RATING:
		adapt = _MINIMUM_RATING - get_rating(player.nick)
	return adapt

def adapt_ratings(game):
	adaptions = dict()
	for pid, p in game.players.items():
		if not hasattr(p, 'team_id'):
			continue
		p.elo = get_rating(p.nick) + rating_adaption(p, game)
		_RATINGS[p.nick] = p.elo

def rate(game):
	adapt_ratings(game)

<commit_msg>Adapt elo calculation to documentation<commit_after>"""
A rating system for player performance
"""

_RATINGS = dict()
_DEFAULT_RATING = 1.0
_MINIMUM_RATING = 0.001
_FRAGS_PER_SECOND = 0.016
_ABSORBER = 0.2

def get_rating(nick):
	if not nick in _RATINGS:
		_RATINGS[nick] = _DEFAULT_RATING
	return _RATINGS[nick]

def set_ratings(timelines):
	for timeline in timelines:
		for i in xrange(len(timeline)):
			p = timeline[i]
			if p.elo > 0:
				_RATINGS[p.nick] = p.elo

def elo_sum(game):
	elo_sum = 0
	for p in game.players.values():
		if not hasattr(p, 'team_id'):
			continue
		elo_sum += get_rating(p.nick)
	assert (elo_sum > 0), elo_sum
	return elo_sum

def predict_player(player, game, R):
	return get_rating(player.nick) / R

def rating_adaption(player, game, R):
	pred = predict_player(player, game, R)
	actual = float(player.kill_count) / game.frag_count
	d = actual - pred
	n = len(game.players) - 1 # subtract <world>
	adapt = d * _ABSORBER * n
	print "adapt %s %df %.3f %.3f %.3f" % (player.nick, player.kill_count, d, pred, actual)
	return adapt

def adapt_ratings(game):
	R = elo_sum(game)
	print "-"*20
	if game.frag_count == 0:
		return # nothing to adapt
	adaptions = dict()
	for pid, p in game.players.items():
		if not hasattr(p, 'team_id'):
			continue
		p.elo = get_rating(p.nick) + rating_adaption(p, game, R)
		_RATINGS[p.nick] = p.elo

def rate(game):
	adapt_ratings(game)

<|endoftext|>


<commit_before>###########################################################
#
# Copyright (c) 2005-2008, Southpaw Technology
#                     All Rights Reserved
#
# PROPRIETARY INFORMATION.  This software is proprietary to
# Southpaw Technology, and is not to be reproduced, transmitted,
# or disclosed in any way without written permission.
#
#
#

__all__ = ['FileDetailWdg']

from tactic.ui.common import BaseRefreshWdg

from pyasm.web import DivWdg, WebContainer, Table, WebState, HtmlElement
from pyasm.search import Search, SearchType, SearchKey
from pyasm.biz import Snapshot
from pyasm.widget import TextAreaWdg
from tactic.ui.panel import TableLayoutWdg

from pyasm.widget import ThumbWdg

import os

class FileDetailWdg(BaseRefreshWdg):
    '''Single File Widget'''

    def get_display(my):

        my.search_key = my.kwargs.get("search_key")
        sobject = Search.get_by_search_key(my.search_key)

        if sobject.get_base_search_type() == "sthpw/snapshot":
            snapshot = sobject
        elif sobject.get_base_search_type() == "sthpw/file":
            # if it is a file object
            snapshot = sobject.get_parent()
        else:
            snapshots = Snapshot.get_by_sobject(sobject)
            snapshot = snapshots[0]

        #parent = snapshot.get_parent()

        top = my.top

        from tactic.ui.container import ResizableTableWdg
        table = ResizableTableWdg()
        top.add(table)
        table.add_row()
        table.add_style("width: 100%")
        table.add_style("text-align", "center")

        from tactic.ui.widget import EmbedWdg
        td = table.add_cell()
        td.add_color("background", "background",)
        td.add_style("vertical-align: middle")
        td.add_style("height: 200px")
        td.add_style("overflow-x: auto")

        file_type = "icon"
        thumb_path = snapshot.get_web_path_by_type(file_type)

        # Extension determine UI class for preview
        from pyasm.biz import File
        file_type = "main"
        lib_path = snapshot.get_lib_path_by_type(file_type)
        src = snapshot.get_web_path_by_type(file_type)

        parts = os.path.splitext(src)
        ext = parts[1]
        ext = ext.lstrip(".")
        ext = ext.lower()

        if ext in ['txt','html', 'ini']:
            content_div = DivWdg()
            f = open(lib_path, 'r')
            content = f.read(10000)
            f.close()
            if not content:
                text = "No Content"
            else:

                size = os.path.getsize(lib_path)

                from pyasm.common import FormatValue
                value = FormatValue().get_format_value(size, "KB")

                content_div.add("Showing first 10K of %s
" % value)

                text = TextAreaWdg()
                text.add(content)
                text.add_style("width: 100%")
                text.add_style("height: 300px")
                text.add_style("padding: 10px")
                text.add_style("border: none")
                text.add_attr("readonly", "true")

            content_div.add(text)
            td.add(content_div)
            content_div.add_style("color", "#000")
            content_div.add_style("width", "auto")
            content_div.add_style("margin", "20px")

        elif ext in "gif":
            img = HtmlElement.img(src=src)
            td.add(img)
        elif ext in ["tif", "tiff"]:
            img = HtmlElement.img(src=thumb_path)
            td.add(img)
        elif ext in File.VIDEO_EXT or ext in File.IMAGE_EXT:
            if ext in File.VIDEO_EXT:
                embed_wdg = EmbedWdg(src=src, thumb_path=thumb_path, preload="auto", controls=True)
            elif ext in File.IMAGE_EXT:
                embed_wdg = EmbedWdg(src=src, thumb_path=thumb_path, height='200')

            td.add(embed_wdg)

            # 100% width is default in EmbedWdg
            embed_wdg.add_style("margin: auto auto")
            embed_wdg.add_class("spt_resizable")
            #embed_wdg.add_style("width: 100%")

            embed_wdg.add_behavior( {
                'type': 'load',
                'cbjs_action': '''
                var last_height = spt.container.get_value("last_img_height");
                if (last_height) {
                    bvr.src_el.setStyle("height", last_height);
                }
                '''
            } )

            embed_wdg.add_behavior( {
                'type': 'unload',
                'cbjs_action': '''
                var last_height = bvr.src_el.getStyle("height");
                spt.container.set_value("last_img_height", last_height);
                '''
            } )

        else:
            thumb_table = DivWdg()
            td.add(thumb_table)

            thumb_table.add_behavior( {
                'type': 'click_up',
                'src': src,
                'cbjs_action': '''
                window.open(bvr.src);
                '''
            } )
            thumb_table.add_class("hand")
            thumb_table.add_style("width: 200px")
            thumb_table.add_style("height: 125px")
            thumb_table.add_style("padding: 5px")
            thumb_table.add_style("margin-left: 20px")
            thumb_table.add_style("display: inline-block")
            thumb_table.add_style("vertical-align: top")
            thumb_table.add_style("overflow-y: hidden")

            from tactic.ui.panel import ThumbWdg2
            thumb = ThumbWdg2()
            thumb_table.add(thumb)
            thumb.set_sobject(snapshot)

        table.add_row()
        td = table.add_cell()

        from tactic.ui.checkin import PathMetadataWdg
        from tactic.ui.checkin import SnapshotMetadataWdg

        metadata_div = DivWdg()
        td.add(metadata_div)
        metadata_div.add_style("max-height: 400px")
        metadata_div.add_style("overflow-y: auto")
        metadata_div.add_style("overflow-x: hidden")

        parser = my.kwargs.get("parser")
        use_tactic_tags = my.kwargs.get("use_tactic_tags")

        file_type = "main"
        server_src = snapshot.get_lib_path_by_type(file_type)

        # get it dynamically by path
        metadata_wdg = PathMetadataWdg(path=server_src, parser=parser, use_tactic_tags=use_tactic_tags)
        metadata_div.add(metadata_wdg)

        #else:
        #    metadata_wdg = SnapshotMetadataWdg(snapshot=snapshot)
        #    metadata_div.add(metadata_wdg)

        top.add("
")

        return top

<commit_msg>FileDetailWdg: Added PDF preview.<commit_after>###########################################################
#
# Copyright (c) 2005-2008, Southpaw Technology
#                     All Rights Reserved
#
# PROPRIETARY INFORMATION.  This software is proprietary to
# Southpaw Technology, and is not to be reproduced, transmitted,
# or disclosed in any way without written permission.
#
#
#

__all__ = ['FileDetailWdg']

from tactic.ui.common import BaseRefreshWdg

from pyasm.web import DivWdg, WebContainer, Table, WebState, HtmlElement
from pyasm.search import Search, SearchType, SearchKey
from pyasm.biz import Snapshot
from pyasm.widget import TextAreaWdg
from tactic.ui.panel import TableLayoutWdg

from pyasm.widget import ThumbWdg

import os

class FileDetailWdg(BaseRefreshWdg):
    '''Single File Widget'''

    def get_display(my):

        my.search_key = my.kwargs.get("search_key")
        sobject = Search.get_by_search_key(my.search_key)

        if sobject.get_base_search_type() == "sthpw/snapshot":
            snapshot = sobject
        elif sobject.get_base_search_type() == "sthpw/file":
            # if it is a file object
            snapshot = sobject.get_parent()
        else:
            snapshots = Snapshot.get_by_sobject(sobject)
            snapshot = snapshots[0]

        # Extension determine UI class for preview
        thumb_path = snapshot.get_web_path_by_type("icon")
        web_src = snapshot.get_web_path_by_type("web")

        from pyasm.biz import File
        file_type = "main"
        lib_path = snapshot.get_lib_path_by_type(file_type)
        src = snapshot.get_web_path_by_type(file_type)
        if not web_src:
            web_src = src

        parts = os.path.splitext(src)
        ext = parts[1]
        ext = ext.lstrip(".")
        ext = ext.lower()

        #parent = snapshot.get_parent()

        top = my.top

        if ext == "pdf":
            iframe = HtmlElement.iframe()
            iframe.set_attr('src', src)
            iframe.add_style("width: 100%")
            iframe.add_style("height: 800px")
            top.add(iframe)
            return top

        from tactic.ui.container import ResizableTableWdg
        table = ResizableTableWdg()
        top.add(table)
        tr = table.add_row()

        # These bvrs allow for smooth switching if switching between files
        # like in the RepoBrowserWdg
        tr.add_style("height: 200px")
        load_height_bvr = {
            'type': 'load',
            'cbjs_action': '''
            var last_height = spt.container.get_value("last_img_height");
            if (last_height) {
                bvr.src_el.setStyle("height", last_height);
            }
            '''
        }
        tr.add_behavior(load_height_bvr)

        unload_height_bvr = {
            'type': 'unload',
            'cbjs_action': '''
            var last_height = bvr.src_el.getStyle("height");
            spt.container.set_value("last_img_height", last_height);
            '''
        }
        tr.add_behavior(unload_height_bvr)

        table.add_style("width: 100%")
        table.add_style("text-align", "center")

        from tactic.ui.widget import EmbedWdg
        td = table.add_cell()
        td.add_color("background", "background",)
        td.add_style("vertical-align: middle")
        td.add_style("height: inherit")
        td.add_style("overflow-x: auto")

        if ext in ['txt','html', 'ini']:
            content_div = DivWdg()
            f = open(lib_path, 'r')
            content = f.read(10000)
            f.close()
            if not content:
                text = "No Content"
            else:

                size = os.path.getsize(lib_path)

                from pyasm.common import FormatValue
                value = FormatValue().get_format_value(size, "KB")

                content_div.add("Showing first 10K of %s
" % value)

                text = TextAreaWdg()
                text.add(content)
                text.add_style("width: 100%")
                text.add_style("height: 100%")
                text.add_style("padding: 10px")
                text.add_style("border: none")
                text.add_attr("readonly", "true")

            content_div.add(text)
            td.add(content_div)
            content_div.add_style("color", "#000")
            content_div.add_style("width", "auto")
            content_div.add_style("margin", "20px")
            content_div.add_style("height", "100%")

        elif ext in File.IMAGE_EXT or ext == "gif":
            if ext == "gif":
                img = HtmlElement.img(src=src)
            else:
                img = HtmlElement.img(src=web_src)
            img.add_style("height: inherit")
            img.add_style("width: auto")
            td.add(img)
        elif ext in File.VIDEO_EXT:
            embed_wdg = EmbedWdg(src=src, thumb_path=thumb_path, preload="auto", controls=True)
            td.add(embed_wdg)

            embed_wdg.add_style("margin: auto auto")
            embed_wdg.add_class("spt_resizable")

            embed_wdg.add_behavior(load_height_bvr)
            embed_wdg.add_behavior(unload_height_bvr)

        else:
            thumb_table = DivWdg()
            td.add(thumb_table)

            thumb_table.add_behavior( {
                'type': 'click_up',
                'src': src,
                'cbjs_action': '''
                window.open(bvr.src);
                '''
            } )
            thumb_table.add_class("hand")
            thumb_table.add_style("width: 200px")
            thumb_table.add_style("height: 125px")
            thumb_table.add_style("padding: 5px")
            thumb_table.add_style("margin-left: 20px")
            thumb_table.add_style("display: inline-block")
            thumb_table.add_style("vertical-align: top")
            thumb_table.add_style("overflow-y: hidden")
            from tactic.ui.panel import ThumbWdg2
            thumb = ThumbWdg2()
            thumb_table.add(thumb)
            thumb.set_sobject(snapshot)

        table.add_row()
        td = table.add_cell()

        from tactic.ui.checkin import PathMetadataWdg
        from tactic.ui.checkin import SnapshotMetadataWdg

        metadata_div = DivWdg()
        td.add(metadata_div)
        metadata_div.add_style("max-height: 400px")
        metadata_div.add_style("overflow-y: auto")
        metadata_div.add_style("overflow-x: hidden")

        parser = my.kwargs.get("parser")
        use_tactic_tags = my.kwargs.get("use_tactic_tags")

        file_type = "main"
        server_src = snapshot.get_lib_path_by_type(file_type)

        # get it dynamically by path
        metadata_wdg = PathMetadataWdg(path=server_src, parser=parser, use_tactic_tags=use_tactic_tags)
        metadata_div.add(metadata_wdg)

        #else:
        #    metadata_wdg = SnapshotMetadataWdg(snapshot=snapshot)
        #    metadata_div.add(metadata_wdg)

        top.add("
")

        return top

<|endoftext|>


<commit_before>import logging
log = logging.getLogger(__name__)

import threading

try:
	import html
	unescape_html = html.unescape
except (ImportError, AttributeError):
	import HTMLParser
	html = HTMLParser.HTMLParser()
	unescape_html = html.unescape

def decode(bytestring):
	"""Attempt to decode a byte string into a UTF-8 string.

	Because IRC doesn't enforce any particular encoding, we have to either guess
	what encoding is being used, or keep trying until it works. UTF-8 and
	ISO-8859-1 (Windows) are the most common, so we just try those two.
	"""
	try:
		return bytestring.decode('utf-8').strip()
	except UnicodeDecodeError:
		try:
			return bytestring.decode('iso-8859-1').strip()
		except:
			log.error('Could not decode string: '+repr(bytestring))
			return None

class ErrorProneThread(threading.Thread):
	def __init__(self, *args, error_handler=None, **kwargs):
		self.error_handler = error_handler
		super().__init__(*args, **kwargs)

	def run(self):
		try:
			super().run()
		except Exception as exception:
			if self.error_handler:
				self.error_handler(exception)
			else:
				raise
<commit_msg>fix HTMLParser module name, tweak exception types<commit_after>import logging
log = logging.getLogger(__name__)

import threading

import html
try:
	unescape_html = html.unescape
except AttributeError:
	import html.parser
	html = html.parser.HTMLParser()
	unescape_html = html.unescape

def decode(bytestring):
	"""Attempt to decode a byte string into a UTF-8 string.

	Because IRC doesn't enforce any particular encoding, we have to either guess
	what encoding is being used, or keep trying until it works. UTF-8 and
	ISO-8859-1 (Windows) are the most common, so we just try those two.
	"""
	try:
		return bytestring.decode('utf-8').strip()
	except UnicodeDecodeError:
		try:
			return bytestring.decode('iso-8859-1').strip()
		except:
			log.error('Could not decode string: '+repr(bytestring))
			return None

class ErrorProneThread(threading.Thread):
	def __init__(self, *args, error_handler=None, **kwargs):
		self.error_handler = error_handler
		super().__init__(*args, **kwargs)

	def run(self):
		try:
			super().run()
		except Exception as exception:
			if self.error_handler:
				self.error_handler(exception)
			else:
				raise
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import argparse
import json

import requests
import six

_DEFAULT_URL = 'http://127.0.0.1:5050/v1'

def _prepare(base_url, auth_token):
    base_url = base_url.rstrip('/')
    if not base_url.endswith('v1'):
        base_url += '/v1'
    headers = {'X-Auth-Token': auth_token}
    return base_url, headers

def introspect(uuid, base_url=_DEFAULT_URL, auth_token=''):
    """Start introspection for a node.

    :param uuid: node uuid
    :param base_url: *ironic-discoverd* URL in form: http://host:port[/ver],
                     defaults to ``http://127.0.0.1:5050/v1``.
    :param auth_token: Keystone authentication token.
    """
    if not isinstance(uuid, six.string_types):
        raise TypeError("Expected string for uuid argument, got %r" % uuid)

    base_url, headers = _prepare(base_url, auth_token)
    res = requests.post("%s/introspection/%s" % (base_url, uuid),
                        headers=headers)
    res.raise_for_status()

def get_status(uuid, base_url=_DEFAULT_URL, auth_token=''):
    """Get introspection status for a node.

    New in ironic-discoverd version 1.0.0.
    :param uuid: node uuid.
    :param base_url: *ironic-discoverd* URL in form: http://host:port[/ver],
                     defaults to ``http://127.0.0.1:5050/v1``.
    :param auth_token: Keystone authentication token.
    :raises: *requests* library HTTP errors.
    """
    if not isinstance(uuid, six.string_types):
        raise TypeError("Expected string for uuid argument, got %r" % uuid)

    base_url, headers = _prepare(base_url, auth_token)
    res = requests.get("%s/introspection/%s" % (base_url, uuid),
                       headers=headers)
    res.raise_for_status()
    return res.json()

def discover(uuids, base_url=_DEFAULT_URL, auth_token=''):
    """Post node UUID's for discovery.

    DEPRECATED. Use introspect instead.
    """
    if not all(isinstance(s, six.string_types) for s in uuids):
        raise TypeError("Expected list of strings for uuids argument, got %s" %
                        uuids)

    base_url, headers = _prepare(base_url, auth_token)
    headers['Content-Type'] = 'application/json'
    res = requests.post(base_url + "/discover",
                        data=json.dumps(uuids), headers=headers)
    res.raise_for_status()

if __name__ == '__main__':  # pragma: no cover
    parser = argparse.ArgumentParser(description='Discover nodes.')
    parser.add_argument('cmd', metavar='cmd',
                        choices=['introspect', 'get_status'],
                        help='command: introspect or get_status.')
    parser.add_argument('uuid', metavar='UUID', type=str,
                        help='node UUID.')
    parser.add_argument('--base-url', dest='base_url', action='store',
                        default=_DEFAULT_URL,
                        help='base URL, default to localhost.')
    parser.add_argument('--auth-token', dest='auth_token', action='store',
                        default='',
                        help='Keystone token.')
    args = parser.parse_args()
    func = globals()[args.cmd]
    try:
        res = func(uuid=args.uuid, base_url=args.base_url,
                   auth_token=args.auth_token)
    except Exception as exc:
        print('Error:', exc)
    else:
        if res:
            print(res)
<commit_msg>Dump test CLI output as JSON, not as Python repr<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import argparse
import json

import requests
import six

_DEFAULT_URL = 'http://127.0.0.1:5050/v1'

def _prepare(base_url, auth_token):
    base_url = base_url.rstrip('/')
    if not base_url.endswith('v1'):
        base_url += '/v1'
    headers = {'X-Auth-Token': auth_token}
    return base_url, headers

def introspect(uuid, base_url=_DEFAULT_URL, auth_token=''):
    """Start introspection for a node.

    :param uuid: node uuid
    :param base_url: *ironic-discoverd* URL in form: http://host:port[/ver],
                     defaults to ``http://127.0.0.1:5050/v1``.
    :param auth_token: Keystone authentication token.
    """
    if not isinstance(uuid, six.string_types):
        raise TypeError("Expected string for uuid argument, got %r" % uuid)

    base_url, headers = _prepare(base_url, auth_token)
    res = requests.post("%s/introspection/%s" % (base_url, uuid),
                        headers=headers)
    res.raise_for_status()

def get_status(uuid, base_url=_DEFAULT_URL, auth_token=''):
    """Get introspection status for a node.

    New in ironic-discoverd version 1.0.0.
    :param uuid: node uuid.
    :param base_url: *ironic-discoverd* URL in form: http://host:port[/ver],
                     defaults to ``http://127.0.0.1:5050/v1``.
    :param auth_token: Keystone authentication token.
    :raises: *requests* library HTTP errors.
    """
    if not isinstance(uuid, six.string_types):
        raise TypeError("Expected string for uuid argument, got %r" % uuid)

    base_url, headers = _prepare(base_url, auth_token)
    res = requests.get("%s/introspection/%s" % (base_url, uuid),
                       headers=headers)
    res.raise_for_status()
    return res.json()

def discover(uuids, base_url=_DEFAULT_URL, auth_token=''):
    """Post node UUID's for discovery.

    DEPRECATED. Use introspect instead.
    """
    if not all(isinstance(s, six.string_types) for s in uuids):
        raise TypeError("Expected list of strings for uuids argument, got %s" %
                        uuids)

    base_url, headers = _prepare(base_url, auth_token)
    headers['Content-Type'] = 'application/json'
    res = requests.post(base_url + "/discover",
                        data=json.dumps(uuids), headers=headers)
    res.raise_for_status()

if __name__ == '__main__':  # pragma: no cover
    parser = argparse.ArgumentParser(description='Discover nodes.')
    parser.add_argument('cmd', metavar='cmd',
                        choices=['introspect', 'get_status'],
                        help='command: introspect or get_status.')
    parser.add_argument('uuid', metavar='UUID', type=str,
                        help='node UUID.')
    parser.add_argument('--base-url', dest='base_url', action='store',
                        default=_DEFAULT_URL,
                        help='base URL, default to localhost.')
    parser.add_argument('--auth-token', dest='auth_token', action='store',
                        default='',
                        help='Keystone token.')
    args = parser.parse_args()
    func = globals()[args.cmd]
    try:
        res = func(uuid=args.uuid, base_url=args.base_url,
                   auth_token=args.auth_token)
    except Exception as exc:
        print('Error:', exc)
    else:
        if res:
            print(json.dumps(res))
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unit tests for gcl.py."""

import StringIO
import os
import sys
import unittest

# Local imports
import gcl

class GclTestsBase(unittest.TestCase):
  """Setups and tear downs the mocks but doesn't test anything as-is."""
  def setUp(self):
    def RunShellMock(filename):
      return filename
    self._RunShell = gcl.RunShell
    gcl.RunShell = RunShellMock

  def tearDown(self):
    gcl.RunShell = self._RunShell

  def compareMembers(self, object, members):
    """If you add a member, be sure to add the relevant test!"""
    # Skip over members starting with '_' since they are usually not meant to
    # be for public use.
    actual_members = [x for x in sorted(dir(object))
                      if not x.startswith('_')]
    expected_members = sorted(members)
    if actual_members != expected_members:
      diff = ([i for i in actual_members if i not in expected_members] +
              [i for i in expected_members if i not in actual_members])
      print diff
    self.assertEqual(actual_members, expected_members)

class GclUnittest(GclTestsBase):
  """General gcl.py tests."""
  def testMembersChanged(self):
    members = [
      'CODEREVIEW_SETTINGS', 'CODEREVIEW_SETTINGS_FILE', 'CPP_EXTENSIONS',
      'Change', 'ChangeInfo', 'Changes', 'Commit', 'DoPresubmitChecks',
      'ErrorExit', 'GenerateChangeName', 'GenerateDiff', 'GetCLs',
      'GetChangelistInfoFile', 'GetCodeReviewSetting', 'GetEditor',
      'GetFilesNotInCL', 'GetInfoDir', 'GetIssueDescription',
      'GetModifiedFiles', 'GetRepositoryRoot',
      'GetSVNFileProperty', 'Help', 'IGNORE_PATHS', 'IsSVNMoved', 'IsTreeOpen',
      'Lint', 'LoadChangelistInfo', 'LoadChangelistInfoForMultiple',
      'MISSING_TEST_MSG', 'Opened', 'PresubmitCL', 'ReadFile',
      'RunShell',
      'RunShellWithReturnCode', 'SEPARATOR', 'SendToRietveld', 'TryChange',
      'UnknownFiles', 'UploadCL', 'Warn', 'WriteFile', 'gclient',
      'gcl_info_dir', 'getpass', 'main', 'os', 'random', 're', 'read_gcl_info',
      'repository_root', 'string', 'subprocess', 'sys', 'tempfile', 'upload',
      'urllib2', 'xml',
    ]
    # If this test fails, you should add the relevant test.
    self.compareMembers(gcl, members)

  def testHelp(self):
    old_stdout = sys.stdout
    try:
      dummy = StringIO.StringIO()
      gcl.sys.stdout = dummy
      gcl.Help()
      self.assertEquals(len(dummy.getvalue()), 1718)
    finally:
      gcl.sys.stdout = old_stdout

  def testGetRepositoryRoot(self):
    try:
      def RunShellMock(filename):
        return '<?xml version="1.0"?>\n<info>'
      gcl.RunShell = RunShellMock
      gcl.GetRepositoryRoot()
    except Exception,e:
      self.assertEquals(e.args[0], "gcl run outside of repository")
    pass

class ChangeInfoUnittest(GclTestsBase):
  def testChangeInfoMembers(self):
    members = [
      'CloseIssue', 'Delete', 'FileList', 'MissingTests', 'Save',
      'UpdateRietveldDescription', 'description', 'files', 'issue', 'name',
      'patch'
    ]
    # If this test fails, you should add the relevant test.
    self.compareMembers(gcl.ChangeInfo(), members)

  def testChangeInfoBase(self):
    files = [('M', 'foo'), ('A', 'bar')]
    o = gcl.ChangeInfo('name2', 'issue2', 'description2', files)
    self.assertEquals(o.name, 'name2')
    self.assertEquals(o.issue, 'issue2')
    self.assertEquals(o.description, 'description2')
    self.assertEquals(o.files, files)
    self.assertEquals(o.patch, None)
    self.assertEquals(o.FileList(), ['foo', 'bar'])

if __name__ == '__main__':
  unittest.main()
<commit_msg>Fix unit test due to change r15911.<commit_after>#!/usr/bin/python
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unit tests for gcl.py."""

import StringIO
import os
import sys
import unittest

# Local imports
import gcl

class GclTestsBase(unittest.TestCase):
  """Setups and tear downs the mocks but doesn't test anything as-is."""
  def setUp(self):
    def RunShellMock(filename):
      return filename
    self._RunShell = gcl.RunShell
    gcl.RunShell = RunShellMock

  def tearDown(self):
    gcl.RunShell = self._RunShell

  def compareMembers(self, object, members):
    """If you add a member, be sure to add the relevant test!"""
    # Skip over members starting with '_' since they are usually not meant to
    # be for public use.
    actual_members = [x for x in sorted(dir(object))
                      if not x.startswith('_')]
    expected_members = sorted(members)
    if actual_members != expected_members:
      diff = ([i for i in actual_members if i not in expected_members] +
              [i for i in expected_members if i not in actual_members])
      print diff
    self.assertEqual(actual_members, expected_members)

class GclUnittest(GclTestsBase):
  """General gcl.py tests."""
  def testMembersChanged(self):
    members = [
      'CODEREVIEW_SETTINGS', 'CODEREVIEW_SETTINGS_FILE', 'CPP_EXTENSIONS',
      'Change', 'ChangeInfo', 'Changes', 'Commit', 'DoPresubmitChecks',
      'ErrorExit', 'GenerateChangeName', 'GenerateDiff', 'GetCLs',
      'GetChangelistInfoFile', 'GetCodeReviewSetting', 'GetEditor',
      'GetFilesNotInCL', 'GetInfoDir', 'GetIssueDescription',
      'GetModifiedFiles', 'GetRepositoryRoot',
      'GetSVNFileProperty', 'Help', 'IGNORE_PATHS', 'IsSVNMoved', 'IsTreeOpen',
      'Lint', 'LoadChangelistInfo', 'LoadChangelistInfoForMultiple',
      'MISSING_TEST_MSG', 'Opened', 'PresubmitCL', 'ReadFile',
      'RunShell',
      'RunShellWithReturnCode', 'SEPARATOR', 'SendToRietveld', 'TryChange',
      'UnknownFiles', 'UploadCL', 'Warn', 'WriteFile', 'gclient',
      'gcl_info_dir', 'getpass', 'main', 'os', 'random', 're', 'read_gcl_info',
      'repository_root', 'string', 'subprocess', 'sys', 'tempfile', 'upload',
      'urllib2', 'xml',
    ]
    # If this test fails, you should add the relevant test.
    self.compareMembers(gcl, members)

  def testHelp(self):
    old_stdout = sys.stdout
    try:
      dummy = StringIO.StringIO()
      gcl.sys.stdout = dummy
      gcl.Help()
      self.assertEquals(len(dummy.getvalue()), 1813)
    finally:
      gcl.sys.stdout = old_stdout

  def testGetRepositoryRoot(self):
    try:
      def RunShellMock(filename):
        return '<?xml version="1.0"?>\n<info>'
      gcl.RunShell = RunShellMock
      gcl.GetRepositoryRoot()
    except Exception,e:
      self.assertEquals(e.args[0], "gcl run outside of repository")
    pass

class ChangeInfoUnittest(GclTestsBase):
  def testChangeInfoMembers(self):
    members = [
      'CloseIssue', 'Delete', 'FileList', 'MissingTests', 'Save',
      'UpdateRietveldDescription', 'description', 'files', 'issue', 'name',
      'patch'
    ]
    # If this test fails, you should add the relevant test.
    self.compareMembers(gcl.ChangeInfo(), members)

  def testChangeInfoBase(self):
    files = [('M', 'foo'), ('A', 'bar')]
    o = gcl.ChangeInfo('name2', 'issue2', 'description2', files)
    self.assertEquals(o.name, 'name2')
    self.assertEquals(o.issue, 'issue2')
    self.assertEquals(o.description, 'description2')
    self.assertEquals(o.files, files)
    self.assertEquals(o.patch, None)
    self.assertEquals(o.FileList(), ['foo', 'bar'])

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, tarfile, hashlib, re, shutil
from snapshot import *

def snap_filename_hash_part(snap):
  match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
  if not match:
    raise Exception("unable to find hash in filename: " + snap)
  return match.group(1)

def unpack_snapshot(snap):
  dl_path = os.path.join(download_dir_base, snap)
  print("opening snapshot " + dl_path)
  tar = tarfile.open(dl_path)
  kernel = get_kernel()
  for name in snapshot_files[kernel]:
    p = os.path.join("rust-stage0", name)
    fp = os.path.join("stage0", name)
    print("extracting " + fp)
    tar.extract(p, download_unpack_base)
    tp = os.path.join(download_unpack_base, p)
    shutil.move(tp, fp)
  tar.close()
  shutil.rmtree(download_unpack_base)

def determine_curr_snapshot_for_platform():

  i = 0
  platform = get_platform()

  found_file = False
  found_snap = False
  hsh = None
  date = None
  rev = None

  with open(snapshotfile) as f:
    for line in f.xreadlines():
      i += 1
      parsed = parse_line(i, line)
      if (not parsed): continue

      if parsed["type"] == "transition":
        raise Exception("working on a transition, not updating stage0")

      if found_snap and parsed["type"] == "file":
        if parsed["platform"] == platform:
          hsh = parsed["hash"]
          found_file = True
          break;
      elif parsed["type"] == "snapshot":
        date = parsed["date"]
        rev = parsed["rev"]
        found_snap = True

  if not found_snap:
    raise Exception("no snapshot entries in file")

  if not found_file:
    raise Exception("no snapshot file found for platform %s, rev %s" %
                    (platform, rev))

  return full_snapshot_name(date, rev, get_platform(), hsh)

# Main

snap = determine_curr_snapshot_for_platform()
dl = os.path.join(download_dir_base, snap)
url = download_url_base + "/" + snap
print("determined most recent snapshot: " + snap)

if (not os.path.exists(dl)):
  get_url_to_file(url, dl)

if (snap_filename_hash_part(snap) == hash_file(dl)):
  print("got download with ok hash")
else:
  raise Exception("bad hash on download")

unpack_snapshot(snap)
<commit_msg>Attempt to use more portable python<commit_after>#!/usr/bin/env python

import os, tarfile, hashlib, re, shutil
from snapshot import *

def snap_filename_hash_part(snap):
  match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
  if not match:
    raise Exception("unable to find hash in filename: " + snap)
  return match.group(1)

def unpack_snapshot(snap):
  dl_path = os.path.join(download_dir_base, snap)
  print("opening snapshot " + dl_path)
  tar = tarfile.open(dl_path)
  kernel = get_kernel()
  for name in snapshot_files[kernel]:
    p = os.path.join("rust-stage0", name)
    fp = os.path.join("stage0", name)
    print("extracting " + fp)
    tar.extract(p, download_unpack_base)
    tp = os.path.join(download_unpack_base, p)
    shutil.move(tp, fp)
  tar.close()
  shutil.rmtree(download_unpack_base)

def determine_curr_snapshot_for_platform():

  i = 0
  platform = get_platform()

  found_file = False
  found_snap = False
  hsh = None
  date = None
  rev = None

  f = open(snapshotfile)
  for line in f.readlines():
    i += 1
    parsed = parse_line(i, line)
    if (not parsed): continue

    if parsed["type"] == "transition":
      raise Exception("working on a transition, not updating stage0")

    if found_snap and parsed["type"] == "file":
      if parsed["platform"] == platform:
        hsh = parsed["hash"]
        found_file = True
        break;
    elif parsed["type"] == "snapshot":
      date = parsed["date"]
      rev = parsed["rev"]
      found_snap = True

  if not found_snap:
    raise Exception("no snapshot entries in file")

  if not found_file:
    raise Exception("no snapshot file found for platform %s, rev %s" %
                    (platform, rev))

  return full_snapshot_name(date, rev, get_platform(), hsh)

# Main

snap = determine_curr_snapshot_for_platform()
dl = os.path.join(download_dir_base, snap)
url = download_url_base + "/" + snap
print("determined most recent snapshot: " + snap)

if (not os.path.exists(dl)):
  get_url_to_file(url, dl)

if (snap_filename_hash_part(snap) == hash_file(dl)):
  print("got download with ok hash")
else:
  raise Exception("bad hash on download")

unpack_snapshot(snap)
<|endoftext|>