<commit_before>'''Analysis datatype class'''

# General Import
import sys

# App Import
from basepair.helpers import eprint
from bin.common_parser import add_common_args, add_single_uid_parser, add_uid_parser, add_json_parser, \
  add_tags_parser, add_outdir_parser, valid_uid, validate_analysis_yaml

class Analysis:
  '''Analysis action methods'''

  @staticmethod
  def create_analysis(bp_api, args):
    '''Create and submit an analysis'''
    params = {'node': {}, 'info': {}}
    if args.instance:
      try:
        instance_choices = bp_api.get_instances()
      except KeyError:
        sys.exit('ERROR: Failed to get instance data.')
      if args.instance not in instance_choices:
        sys.exit('ERROR: invalid instance_type available instances - {}'.format(' '.join(instance_choices)))
      params['info']['instance_type'] = args.instance
    if args.custom_pipeline or args.custom_modules:
      validate_analysis_yaml(args.custom_pipeline if args.custom_pipeline else args.custom_modules)
    if args.params:
      for param in args.params:
        try:
          node_id, arg, val = param.split(':')
          if node_id not in params['node']:
            params['node'][node_id] = {}
          params['node'][node_id][arg] = val
        except ValueError:
          sys.exit('ERROR: Missing : in params values.')
    else:
      eprint('You specified no parameters, submitting with default ones.')
    return bp_api.create_analysis(
        control_ids=args.control or [],
        ignore_validation_warnings=args.ignore_warning,
        params=params,
        project_id=args.project,
        sample_ids=args.sample,
        workflow_id=args.pipeline,
        pipeline_yaml=args.custom_pipeline,
        module_yaml=args.custom_modules
      )

  @staticmethod
  def delete_analysis(bp_api, args):
    '''Delete analysis'''
    uids = args.uid
    all_fail = True
    for uid in uids:
      answer = bp_api.yes_or_no('Are you sure you want to delete {}?'.format(uid))
      if answer:
        all_fail = not bool(bp_api.delete_analysis(uid)) and all_fail
    if all_fail:
      sys.exit('ERROR: Deleting analysis failed.')

  @staticmethod
  def download_analysis(bp_api, args):
    '''Download analysis'''
    # download a file from an analysis by tags
    all_fail = True
    for each_uid in args.uid:
      analysis = bp_api.get_analysis(each_uid)
      all_fail = not (bool(analysis.get('id')) \
        and bool(bp_api.download_analysis(each_uid, analysis=analysis, outdir=args.outdir, tagkind=args.tagkind, tags=args.tags))) \
        and all_fail
    if all_fail:
      sys.exit('ERROR: Downloading analysis failed.')
    eprint('All analysis files have been downloaded successfully.')

  @staticmethod
  def download_log_analysis(bp_api, args):
    '''Download analysis log'''
    all_fail = True
    for uid in args.uid:
      analysis = bp_api.get_analysis(uid)  # check analysis id is valid
      all_fail = not (bool(analysis.get('id')) and bool(bp_api.get_log(uid, args.outdir))) and all_fail
    if all_fail:
      sys.exit('ERROR: Downloading logs failed.')

  @staticmethod
  def get_analysis(bp_api, args):
    '''Get analysis'''
    all_fail = True
    for uid in args.uid:
      all_fail = not bool(bp_api.print_data(data_type='analysis', uid=uid, is_json=args.json)) and all_fail
    if all_fail:
      sys.exit('ERROR: Analyses data not found.')

  @staticmethod
  def list_analysis(bp_api, args):
    '''List analyses'''
    bp_api.print_data(data_type='analyses', is_json=args.json, project=args.project)

  @staticmethod
  def reanalyze_analysis(bp_api, args):
    '''Restart analysis'''
    if args.instance:
      try:
        instance_choices = bp_api.get_instances()
      except KeyError:
        sys.exit('ERROR: Failed to get instance data.')
      if args.instance not in instance_choices:
        sys.exit('ERROR: invalid instance_type available instances - {}'.format(' '.join(instance_choices)))

    all_fail = True
    for each_uid in args.uid:
      all_fail = not bool(bp_api.restart_analysis(each_uid, args.instance)) and all_fail
    if all_fail:
      sys.exit('ERROR: while re-analyze the analysis data.')

  @staticmethod
  def terminate_analysis(bp_api, args):
    '''Terminate analysis'''
    for each_uid in args.uid:
      bp_api.terminate_analysis(each_uid, args.instance)

  @staticmethod
  def update_analysis(bp_api, args):
    '''Update analysis'''
    data = {}
    if args.key and args.val:
      for key, val in zip(args.key, args.val):
        data[key] = val
    bp_api.update_analysis(args.uid, data)

  @staticmethod
  def analysis_action_parser(action_parser):
    '''Analysis parser'''

    # create analysis parser
    create_analysis_p = action_parser.add_parser(
      'create',
      help='Create and run an analysis.'
    )
    create_analysis_p = add_common_args(create_analysis_p)
    create_analysis_p.add_argument(
      '--control', nargs='+', help='Control id', type=valid_uid
    )
    create_analysis_p.add_argument(
      '--custom-modules', dest='custom_modules', help='Path to custom module yaml files', nargs='+'
    )
    create_analysis_p.add_argument(
      '--custom-pipeline', dest='custom_pipeline', help='Path to custom pipeline yaml file',
    )
    create_analysis_p.add_argument(
      '--ignore-warning',
      action='store_true',
      default=False,
      help='Ignore validation warnings',
    )
    create_analysis_p.add_argument(
      '--instance', help='instance_type for analysis'
    )
    create_analysis_p.add_argument('--params', nargs='+')
    create_analysis_p.add_argument('--project', help='Project id', type=valid_uid)
    create_analysis_p.add_argument(
      '--pipeline', help='Pipeline id', type=valid_uid
    )
    create_analysis_p.add_argument(
      '--sample', nargs='+', help='Sample id', required=True, type=valid_uid
    )

    # delete analysis parser
    delete_analysis_p = action_parser.add_parser(
      'delete',
      help='delete an analysis.'
    )
    delete_analysis_p = add_common_args(delete_analysis_p)
    delete_analysis_p = add_uid_parser(delete_analysis_p, 'analysis')

    # download analysis parser
    download_analysis_p = action_parser.add_parser(
      'download',
      help='Download files for one or more analyses. Can filter by file tags.'
    )
    download_analysis_p = add_uid_parser(download_analysis_p, 'analysis')
    download_analysis_p = add_tags_parser(download_analysis_p)
    download_analysis_p = add_outdir_parser(download_analysis_p)
    download_analysis_p = add_common_args(download_analysis_p)

    # download analysis log parser
    download_log_p = action_parser.add_parser(
      'download-log',
      help='Download analysis logs for one or more analyses.'
    )
    download_log_p = add_uid_parser(download_log_p, 'analysis')
    download_log_p = add_outdir_parser(download_log_p)
    download_log_p = add_common_args(download_log_p)

    # get analysis parser
    get_analysis_p = action_parser.add_parser(
      'get',
      help='Get details of an analysis.'
    )
    get_analysis_p = add_common_args(get_analysis_p)
    get_analysis_p = add_uid_parser(get_analysis_p, 'analysis')
    get_analysis_p = add_json_parser(get_analysis_p)

    # list analysis parser
    list_analyses_p = action_parser.add_parser(
      'list',
      help='List basic info about your analyses.'
    )
    list_analyses_p.add_argument(
      '--project',
      help='List analyses of a project.',
      required=True,
      type=valid_uid
    )
    list_analyses_p = add_common_args(list_analyses_p)
    list_analyses_p = add_json_parser(list_analyses_p)

    # reanalyze parser
    reanalyze_p = action_parser.add_parser(
      'reanalyze',
      help='Reanalyze analyses.'
    )
    reanalyze_p.add_argument(
      '--instance', help='instance_type for analysis'
    )
    reanalyze_p = add_common_args(reanalyze_p)
    reanalyze_p = add_uid_parser(reanalyze_p, 'analysis')

    # terminate parser
    terminate_p = action_parser.add_parser(
      'terminate',
      help='Terminate analyses.'
    )
    terminate_p = add_common_args(terminate_p)
    terminate_p = add_uid_parser(terminate_p, 'analysis')

    # update an analysis parser
    update_analysis_parser = action_parser.add_parser(
      'update',
      help='Update information associated with an analysis.'
    )
    update_analysis_parser.add_argument('--key', action='append', required=True)
    update_analysis_parser.add_argument('--val', action='append', required=True)
    update_analysis_parser = add_common_args(update_analysis_parser)
    update_analysis_parser = add_single_uid_parser(update_analysis_parser, 'analysis')

    return action_parser
<commit_msg>BugFix: Remove not needed instance param for analysis terminate API.<commit_after>'''Analysis datatype class'''

# General Import
import sys

# App Import
from basepair.helpers import eprint
from bin.common_parser import add_common_args, add_single_uid_parser, add_uid_parser, add_json_parser, \
  add_tags_parser, add_outdir_parser, valid_uid, validate_analysis_yaml

class Analysis:
  '''Analysis action methods'''

  @staticmethod
  def create_analysis(bp_api, args):
    '''Create and submit an analysis'''
    params = {'node': {}, 'info': {}}
    if args.instance:
      try:
        instance_choices = bp_api.get_instances()
      except KeyError:
        sys.exit('ERROR: Failed to get instance data.')
      if args.instance not in instance_choices:
        sys.exit('ERROR: invalid instance_type available instances - {}'.format(' '.join(instance_choices)))
      params['info']['instance_type'] = args.instance
    if args.custom_pipeline or args.custom_modules:
      validate_analysis_yaml(args.custom_pipeline if args.custom_pipeline else args.custom_modules)
    if args.params:
      for param in args.params:
        try:
          node_id, arg, val = param.split(':')
          if node_id not in params['node']:
            params['node'][node_id] = {}
          params['node'][node_id][arg] = val
        except ValueError:
          sys.exit('ERROR: Missing : in params values.')
    else:
      eprint('You specified no parameters, submitting with default ones.')
    return bp_api.create_analysis(
        control_ids=args.control or [],
        ignore_validation_warnings=args.ignore_warning,
        params=params,
        project_id=args.project,
        sample_ids=args.sample,
        workflow_id=args.pipeline,
        pipeline_yaml=args.custom_pipeline,
        module_yaml=args.custom_modules
      )

  @staticmethod
  def delete_analysis(bp_api, args):
    '''Delete analysis'''
    uids = args.uid
    all_fail = True
    for uid in uids:
      answer = bp_api.yes_or_no('Are you sure you want to delete {}?'.format(uid))
      if answer:
        all_fail = not bool(bp_api.delete_analysis(uid)) and all_fail
    if all_fail:
      sys.exit('ERROR: Deleting analysis failed.')

  @staticmethod
  def download_analysis(bp_api, args):
    '''Download analysis'''
    # download a file from an analysis by tags
    all_fail = True
    for each_uid in args.uid:
      analysis = bp_api.get_analysis(each_uid)
      all_fail = not (bool(analysis.get('id')) \
        and bool(bp_api.download_analysis(each_uid, analysis=analysis, outdir=args.outdir, tagkind=args.tagkind, tags=args.tags))) \
        and all_fail
    if all_fail:
      sys.exit('ERROR: Downloading analysis failed.')
    eprint('All analysis files have been downloaded successfully.')

  @staticmethod
  def download_log_analysis(bp_api, args):
    '''Download analysis log'''
    all_fail = True
    for uid in args.uid:
      analysis = bp_api.get_analysis(uid)  # check analysis id is valid
      all_fail = not (bool(analysis.get('id')) and bool(bp_api.get_log(uid, args.outdir))) and all_fail
    if all_fail:
      sys.exit('ERROR: Downloading logs failed.')

  @staticmethod
  def get_analysis(bp_api, args):
    '''Get analysis'''
    all_fail = True
    for uid in args.uid:
      all_fail = not bool(bp_api.print_data(data_type='analysis', uid=uid, is_json=args.json)) and all_fail
    if all_fail:
      sys.exit('ERROR: Analyses data not found.')

  @staticmethod
  def list_analysis(bp_api, args):
    '''List analyses'''
    bp_api.print_data(data_type='analyses', is_json=args.json, project=args.project)

  @staticmethod
  def reanalyze_analysis(bp_api, args):
    '''Restart analysis'''
    if args.instance:
      try:
        instance_choices = bp_api.get_instances()
      except KeyError:
        sys.exit('ERROR: Failed to get instance data.')
      if args.instance not in instance_choices:
        sys.exit('ERROR: invalid instance_type available instances - {}'.format(' '.join(instance_choices)))

    all_fail = True
    for each_uid in args.uid:
      all_fail = not bool(bp_api.restart_analysis(each_uid, args.instance)) and all_fail
    if all_fail:
      sys.exit('ERROR: while re-analyze the analysis data.')

  @staticmethod
  def terminate_analysis(bp_api, args):
    '''Terminate analysis'''
    for each_uid in args.uid:
      bp_api.terminate_analysis(each_uid)

  @staticmethod
  def update_analysis(bp_api, args):
    '''Update analysis'''
    data = {}
    if args.key and args.val:
      for key, val in zip(args.key, args.val):
        data[key] = val
    bp_api.update_analysis(args.uid, data)

  @staticmethod
  def analysis_action_parser(action_parser):
    '''Analysis parser'''

    # create analysis parser
    create_analysis_p = action_parser.add_parser(
      'create',
      help='Create and run an analysis.'
    )
    create_analysis_p = add_common_args(create_analysis_p)
    create_analysis_p.add_argument(
      '--control', nargs='+', help='Control id', type=valid_uid
    )
    create_analysis_p.add_argument(
      '--custom-modules', dest='custom_modules', help='Path to custom module yaml files', nargs='+'
    )
    create_analysis_p.add_argument(
      '--custom-pipeline', dest='custom_pipeline', help='Path to custom pipeline yaml file',
    )
    create_analysis_p.add_argument(
      '--ignore-warning',
      action='store_true',
      default=False,
      help='Ignore validation warnings',
    )
    create_analysis_p.add_argument(
      '--instance', help='instance_type for analysis'
    )
    create_analysis_p.add_argument('--params', nargs='+')
    create_analysis_p.add_argument('--project', help='Project id', type=valid_uid)
    create_analysis_p.add_argument(
      '--pipeline', help='Pipeline id', type=valid_uid
    )
    create_analysis_p.add_argument(
      '--sample', nargs='+', help='Sample id', required=True, type=valid_uid
    )

    # delete analysis parser
    delete_analysis_p = action_parser.add_parser(
      'delete',
      help='delete an analysis.'
    )
    delete_analysis_p = add_common_args(delete_analysis_p)
    delete_analysis_p = add_uid_parser(delete_analysis_p, 'analysis')

    # download analysis parser
    download_analysis_p = action_parser.add_parser(
      'download',
      help='Download files for one or more analyses. Can filter by file tags.'
    )
    download_analysis_p = add_uid_parser(download_analysis_p, 'analysis')
    download_analysis_p = add_tags_parser(download_analysis_p)
    download_analysis_p = add_outdir_parser(download_analysis_p)
    download_analysis_p = add_common_args(download_analysis_p)

    # download analysis log parser
    download_log_p = action_parser.add_parser(
      'download-log',
      help='Download analysis logs for one or more analyses.'
    )
    download_log_p = add_uid_parser(download_log_p, 'analysis')
    download_log_p = add_outdir_parser(download_log_p)
    download_log_p = add_common_args(download_log_p)

    # get analysis parser
    get_analysis_p = action_parser.add_parser(
      'get',
      help='Get details of an analysis.'
    )
    get_analysis_p = add_common_args(get_analysis_p)
    get_analysis_p = add_uid_parser(get_analysis_p, 'analysis')
    get_analysis_p = add_json_parser(get_analysis_p)

    # list analysis parser
    list_analyses_p = action_parser.add_parser(
      'list',
      help='List basic info about your analyses.'
    )
    list_analyses_p.add_argument(
      '--project',
      help='List analyses of a project.',
      required=True,
      type=valid_uid
    )
    list_analyses_p = add_common_args(list_analyses_p)
    list_analyses_p = add_json_parser(list_analyses_p)

    # reanalyze parser
    reanalyze_p = action_parser.add_parser(
      'reanalyze',
      help='Reanalyze analyses.'
    )
    reanalyze_p.add_argument(
      '--instance', help='instance_type for analysis'
    )
    reanalyze_p = add_common_args(reanalyze_p)
    reanalyze_p = add_uid_parser(reanalyze_p, 'analysis')

    # terminate parser
    terminate_p = action_parser.add_parser(
      'terminate',
      help='Terminate analyses.'
    )
    terminate_p = add_common_args(terminate_p)
    terminate_p = add_uid_parser(terminate_p, 'analysis')

    # update an analysis parser
    update_analysis_parser = action_parser.add_parser(
      'update',
      help='Update information associated with an analysis.'
    )
    update_analysis_parser.add_argument('--key', action='append', required=True)
    update_analysis_parser.add_argument('--val', action='append', required=True)
    update_analysis_parser = add_common_args(update_analysis_parser)
    update_analysis_parser = add_single_uid_parser(update_analysis_parser, 'analysis')

    return action_parser
<|endoftext|>


<commit_before>import optparse
import os

import vdj.analysis

option_parser = optparse.OptionParser()
option_parser.add_option('-m','--mapping_file')
(options,args) = option_parser.parse_args()

if len(args) != 2:
    raise ValueError, "need input and output filenames"

# Read sample mapping file
mapping_handle = open(options.mapping_file,'r')
samples = [(line.split('\t')[0].strip(),line.split('\t')[2].strip()) for line in mapping_handle if not line.startswith('#')]
mapping_handle.close()

# Load count data
infilename = args[0]
inhandle = open(infilename,'r')
(uniq_feature_values,countdict) = vdj.analysis.vdjxml2countdict(inhandle,['barcode','clone'])
inhandle.close()

# Convert to matrix form
countmatrix = vdj.analysis.countdict2matrix(['barcode','clone'],uniq_feature_values,countdict).transpose()

# Reorder columns to correspond to mapping file order
sample_idxs = dict([(v,i) for (i,v) in enumerate(uniq_feature_values['barcode'])])
argsort = [sample_idxs[sample] for (sample,descr) in samples]
countmatrix = countmatrix[:,argsort]

# Dump OTU table
basename = os.path.basename(args[0])
outfilename = args[1]
outhandle = open(outfilename,'w')

print >>outhandle, "#OTU counts %s" % basename
header = "OTU ID"
for (sample,descr) in samples: header += "\t%s" % ('_'.join([sample,descr]))
print >>outhandle, header

for (label,countvector) in zip(uniq_feature_values['clone'],countmatrix):
    line = label
    for count in countvector:
        line += "\t%i" % int(count)
    print >>outhandle, line

outhandle.close()
<commit_msg>Added shebang line<commit_after>#! /usr/bin/env python

import optparse
import os

import vdj.analysis

option_parser = optparse.OptionParser()
option_parser.add_option('-m','--mapping_file')
(options,args) = option_parser.parse_args()

if len(args) != 2:
    raise ValueError, "need input and output filenames"

# Read sample mapping file
mapping_handle = open(options.mapping_file,'r')
samples = [(line.split('\t')[0].strip(),line.split('\t')[2].strip()) for line in mapping_handle if not line.startswith('#')]
mapping_handle.close()

# Load count data
infilename = args[0]
inhandle = open(infilename,'r')
(uniq_feature_values,countdict) = vdj.analysis.vdjxml2countdict(inhandle,['barcode','clone'])
inhandle.close()

# Convert to matrix form
countmatrix = vdj.analysis.countdict2matrix(['barcode','clone'],uniq_feature_values,countdict).transpose()

# Reorder columns to correspond to mapping file order
sample_idxs = dict([(v,i) for (i,v) in enumerate(uniq_feature_values['barcode'])])
argsort = [sample_idxs[sample] for (sample,descr) in samples]
countmatrix = countmatrix[:,argsort]

# Dump OTU table
basename = os.path.basename(args[0])
outfilename = args[1]
outhandle = open(outfilename,'w')

print >>outhandle, "#OTU counts %s" % basename
header = "OTU ID"
for (sample,descr) in samples: header += "\t%s" % ('_'.join([sample,descr]))
print >>outhandle, header

for (label,countvector) in zip(uniq_feature_values['clone'],countmatrix):
    line = label
    for count in countvector:
        line += "\t%i" % int(count)
    print >>outhandle, line

outhandle.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
gftools fix-font

Update a font so it conforms to the Google Fonts specification
https://github.com/googlefonts/gf-docs/tree/master/Spec

Usage:

gftools fix-font font.ttf

# Fix font issues that should be fixed in the source files
gftools fix-font font.ttf --include-source-fixes
"""
import argparse
import logging
import os
from fontTools.ttLib import TTFont
from gftools.fix import *

log = logging.getLogger(__name__)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("fonts", nargs="+", help="Font family paths")
    parser.add_argument("--inplace", action='store_true', default=False, help="Save fixed fonts inplace")
    parser.add_argument("-o", "--out", help="Output dir for fixed fonts")
    parser.add_argument(
        "--include-source-fixes",
        action="store_true",
        help="Fix font issues that should be fixed in the source files.",
    )
    args = parser.parse_args()

    fonts = [TTFont(f) for f in args.fonts]
    fix_family(fonts, args.include_source_fixes)

    if args.inplace:
        for font in fonts:
            font.save(font.reader.file.name)
    elif args.out:
        if not os.path.isdir(args.out):
            os.mkdir(args.out)
            for font in fonts:
                out_path = os.path.join(
                    args.out, os.path.basename(font.reader.file.name)
                )
                font.save(out_path)
    else:
        for font in fonts:
            font.save(font.reader.file.name + ".fix")

if __name__ == "__main__":
    main()
<commit_msg>Update docstring<commit_after>#!/usr/bin/env python3
"""
gftools fix-family

Update a family so it conforms to the Google Fonts specification
https://github.com/googlefonts/gf-docs/tree/master/Spec

Usage:

gftools fix-family fonts1.ttf fonts2.ttf

# Fix font issues that should be fixed in the source files
gftools fix-family fonts1.ttf --include-source-fixes
"""
import argparse
import logging
import os
from fontTools.ttLib import TTFont
from gftools.fix import *

log = logging.getLogger(__name__)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("fonts", nargs="+", help="Font family paths")
    parser.add_argument("--inplace", action='store_true', default=False, help="Save fixed fonts inplace")
    parser.add_argument("-o", "--out", help="Output dir for fixed fonts")
    parser.add_argument(
        "--include-source-fixes",
        action="store_true",
        help="Fix font issues that should be fixed in the source files.",
    )
    args = parser.parse_args()

    fonts = [TTFont(f) for f in args.fonts]
    fix_family(fonts, args.include_source_fixes)

    if args.inplace:
        for font in fonts:
            font.save(font.reader.file.name)
    elif args.out:
        if not os.path.isdir(args.out):
            os.mkdir(args.out)
            for font in fonts:
                out_path = os.path.join(
                    args.out, os.path.basename(font.reader.file.name)
                )
                font.save(out_path)
    else:
        for font in fonts:
            font.save(font.reader.file.name + ".fix")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright P. Christeas <p_christ@hol.gr> 2008,2009
# A part of the code comes from the ganeti project:  http://www.mail-archive.com/ganeti-devel@googlegroups.com/msg00713.html#
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

""" Framework for generic http servers

"""

import socket
import base64
import SocketServer
from BaseHTTPServer import *
from SimpleHTTPServer import SimpleHTTPRequestHandler

class AuthRequiredExc(Exception):
    def __init__(self,atype,realm):
        Exception.__init__(self)
        self.atype = atype
        self.realm = realm

class AuthRejectedExc(Exception):
    pass

class AuthProvider:
    def __init__(self,realm):
        self.realm = realm

    def setupAuth(self, multi,handler):
        """ Attach an AuthProxy object to handler
        """
        pass

    def authenticate(self, user, passwd, client_address):
        return False

    def log(self, msg):
        print msg

class BasicAuthProvider(AuthProvider):
    def setupAuth(self, multi, handler):
        if not multi.sec_realms.has_key(self.realm):
            multi.sec_realms[self.realm] = BasicAuthProxy(self)

class AuthProxy:
    """ This class will hold authentication information for a handler,
        i.e. a connection
    """
    def __init__(self, provider):
        self.provider = provider

    def checkRequest(self,handler,path = '/'):
        """ Check if we are allowed to process that request
        """
        pass

class BasicAuthProxy(AuthProxy):
    """ Require basic authentication..
    """
    def __init__(self,provider):
        AuthProxy.__init__(self,provider)
        self.auth_creds = None
        self.auth_tries = 0

    def checkRequest(self,handler,path = '/'):
        if self.auth_creds:
            return True
        auth_str = handler.headers.get('Authorization',False)
        if auth_str and auth_str.startswith('Basic '):
            auth_str=auth_str[len('Basic '):]
            (user,passwd) = base64.decodestring(auth_str).split(':')
            self.provider.log("Found user=\"%s\", passwd=\"%s\"" %(user,passwd))
            self.auth_creds = self.provider.authenticate(user,passwd,handler.client_address)
            if self.auth_creds:
                return True
        if self.auth_tries > 5:
            self.provider.log("Failing authorization after 5 requests w/o password")
            raise AuthRejectedExc("Authorization failed.")
        self.auth_tries += 1
        raise AuthRequiredExc(atype = 'Basic', realm=self.provider.realm)

class HTTPHandler(SimpleHTTPRequestHandler):
    def __init__(self,request, client_address, server):
        SimpleHTTPRequestHandler.__init__(self,request,client_address,server)
        # print "Handler for %s inited" % str(client_address)
        self.protocol_version = 'HTTP/1.1'
        self.connection = dummyconn()

    def handle(self):
        """ Classes here should NOT handle inside their constructor
        """
        pass

    def finish(self):
        pass

    def setup(self):
        pass

class HTTPDir:
    """ A dispatcher class, like a virtual folder in httpd
    """
    def __init__(self,path,handler, auth_provider = None):
        self.path = path
        self.handler = handler
        self.auth_provider = auth_provider

    def matches(self, request):
        """ Test if some request matches us. If so, return
            the matched path. """
        if request.startswith(self.path):
            return self.path
        return False

class noconnection:
    """ a class to use instead of the real connection
    """
    def makefile(self, mode, bufsize):
        return None

class dummyconn:
    def shutdown(self, tru):
        pass

def _quote_html(html):
    return html.replace("&", "&").replace("<", "<").replace(">", ">")

class FixSendError:
    #error_message_format = """ """
    def send_error(self, code, message=None):
        #overriden from BaseHTTPRequestHandler, we also send the content-length
        try:
            short, long = self.responses[code]
        except KeyError:
            short, long = '???', '???'
        if message is None:
            message = short
        explain = long
        self.log_error("code %d, message %s", code, message)
        # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
        content = (self.error_message_format %
                   {'code': code, 'message': _quote_html(message), 'explain': explain})
        self.send_response(code, message)
        self.send_header("Content-Type", self.error_content_type)
        self.send_header('Connection', 'close')
        self.send_header('Content-Length', len(content) or 0)
        self.end_headers()
        if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
            self.wfile.write(content)

class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
    """ this is a multiple handler, that will dispatch each request
        to a nested handler, iff it matches

        The handler will also have *one* dict of authentication proxies,
        groupped by their realm.
    """

    protocol_version = "HTTP/1.1"
    default_request_version = "HTTP/0.9"    # compatibility with py2.5

    auth_required_msg = """ <title>Authorization required</title>
    You must authenticate to use this service\r\r"""

    def __init__(self, request, client_address, server):
        self.in_handlers = {}
        self.sec_realms = {}
        SocketServer.StreamRequestHandler.__init__(self,request,client_address,server)
        self.log_message("MultiHttpHandler init for %s" %(str(client_address)))

    def _handle_one_foreign(self,fore, path, auth_provider):
        """ This method overrides the handle_one_request for *children*
            handlers. It is required, since the first line should not be
            read again..

        """
        fore.raw_requestline = "%s %s %s\n" % (self.command, path, self.version)
        if not fore.parse_request(): # An error code has been sent, just exit
            return
        self.request_version = fore.request_version
        if auth_provider and auth_provider.realm:
            try:
                self.sec_realms[auth_provider.realm].checkRequest(fore,path)
            except AuthRequiredExc,ae:
                if self.request_version != 'HTTP/1.1':
                    self.log_error("Cannot require auth at %s",self.request_version)
                    self.send_error(403)
                    return
                self._get_ignore_body(fore) # consume any body that came, not loose sync with input
                self.send_response(401,'Authorization required')
                self.send_header('WWW-Authenticate','%s realm="%s"' % (ae.atype,ae.realm))
                self.send_header('Connection', 'keep-alive')
                self.send_header('Content-Type','text/html')
                self.send_header('Content-Length',len(self.auth_required_msg))
                self.end_headers()
                self.wfile.write(self.auth_required_msg)
                return
            except AuthRejectedExc,e:
                self.log_error("Rejected auth: %s" % e.args[0])
                self.send_error(403,e.args[0])
                self.close_connection = 1
                return
        mname = 'do_' + fore.command
        if not hasattr(fore, mname):
            fore.send_error(501, "Unsupported method (%r)" % fore.command)
            return
        fore.close_connection = 0
        method = getattr(fore, mname)
        method()
        if fore.close_connection:
            # print "Closing connection because of handler"
            self.close_connection = fore.close_connection
        if hasattr(fore, '_flush'):
            fore._flush()

    def parse_rawline(self):
        """Parse a request (internal).

        The request should be stored in self.raw_requestline; the results
        are in self.command, self.path, self.request_version and
        self.headers.

        Return True for success, False for failure; on failure, an
        error is sent back.

        """
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version
        self.close_connection = 1
        requestline = self.raw_requestline
        if requestline[-2:] == '\r\n':
            requestline = requestline[:-2]
        elif requestline[-1:] == '\n':
            requestline = requestline[:-1]
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            [command, path, version] = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            [command, path] = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            #self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.request_version = version
        self.command, self.path, self.version = command, path, version
        return True

    def handle_one_request(self):
        """Handle a single HTTP request.
           Dispatch to the correct handler.
        """
        self.request.setblocking(True)
        self.raw_requestline = self.rfile.readline()
        if not self.raw_requestline:
            self.close_connection = 1
            # self.log_message("no requestline, connection closed?")
            return
        if not self.parse_rawline():
            self.log_message("Could not parse rawline.")
            return
        # self.parse_request(): # Do NOT parse here. the first line should be the only
        for vdir in self.server.vdirs:
            p = vdir.matches(self.path)
            if p == False:
                continue
            npath = self.path[len(p):]
            if not npath.startswith('/'):
                npath = '/' + npath

            if not self.in_handlers.has_key(p):
                self.in_handlers[p] = vdir.handler(noconnection(),self.client_address,self.server)
                if vdir.auth_provider:
                    vdir.auth_provider.setupAuth(self, self.in_handlers[p])
            hnd = self.in_handlers[p]
            hnd.rfile = self.rfile
            hnd.wfile = self.wfile
            self.rlpath = self.raw_requestline
            self._handle_one_foreign(hnd,npath, vdir.auth_provider)
            # print "Handled, closing = ", self.close_connection
            return
        # if no match:
        self.send_error(404, "Path not found: %s" % self.path)
        return

    def _get_ignore_body(self,fore):
        if not fore.headers.has_key("content-length"):
            return
        max_chunk_size = 10*1024*1024
        size_remaining = int(fore.headers["content-length"])
        got = ''
        while size_remaining:
            chunk_size = min(size_remaining, max_chunk_size)
            got = fore.rfile.read(chunk_size)
            size_remaining -= len(got)

class SecureMultiHTTPHandler(MultiHTTPHandler):
    def getcert_fnames(self):
        """ Return a pair with the filenames of ssl cert,key

            Override this to direct to other filenames
        """
        return ('server.cert','server.key')

    def setup(self):
        import ssl
        certfile, keyfile = self.getcert_fnames()
        try:
            self.connection = ssl.wrap_socket(self.request,
                                server_side=True,
                                certfile=certfile,
                                keyfile=keyfile,
                                ssl_version=ssl.PROTOCOL_SSLv23)
            self.rfile = self.connection.makefile('rb', self.rbufsize)
            self.wfile = self.connection.makefile('wb', self.wbufsize)
            self.log_message("Secure %s connection from %s",self.connection.cipher(),self.client_address)
        except Exception:
            self.request.shutdown(socket.SHUT_RDWR)
            raise

    def finish(self):
        # With ssl connections, closing the filehandlers alone may not
        # work because of ref counting. We explicitly tell the socket
        # to shutdown.
        MultiHTTPHandler.finish(self)
        try:
            self.connection.shutdown(socket.SHUT_RDWR)
        except Exception:
            pass

import threading
class ConnThreadingMixIn:
    """Mix-in class to handle each _connection_ in a new thread.

       This is necessary for persistent connections, where multiple
       requests should be handled synchronously at each connection, but
       multiple connections can run in parallel.
    """

    # Decides how threads will act upon termination of the
    # main process
    daemon_threads = False

    def _handle_request_noblock(self):
        """Start a new thread to process the request."""
        t = threading.Thread(target = self._handle_request2)
        if self.daemon_threads:
            t.setDaemon (1)
        t.start()

    def _handle_request2(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except Exception:
                self.handle_error(request, client_address)
                self.close_request(request)

#eof
<commit_msg>websrv_lib: one case of cleaner exit.<commit_after># -*- coding: utf-8 -*-
#
# Copyright P. Christeas <p_christ@hol.gr> 2008,2009
# A part of the code comes from the ganeti project:  http://www.mail-archive.com/ganeti-devel@googlegroups.com/msg00713.html#
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

""" Framework for generic http servers

"""

import socket
import base64
import SocketServer
from BaseHTTPServer import *
from SimpleHTTPServer import SimpleHTTPRequestHandler

class AuthRequiredExc(Exception):
    def __init__(self,atype,realm):
        Exception.__init__(self)
        self.atype = atype
        self.realm = realm

class AuthRejectedExc(Exception):
    pass

class AuthProvider:
    def __init__(self,realm):
        self.realm = realm

    def setupAuth(self, multi,handler):
        """ Attach an AuthProxy object to handler
        """
        pass

    def authenticate(self, user, passwd, client_address):
        return False

    def log(self, msg):
        print msg

class BasicAuthProvider(AuthProvider):
    def setupAuth(self, multi, handler):
        if not multi.sec_realms.has_key(self.realm):
            multi.sec_realms[self.realm] = BasicAuthProxy(self)

class AuthProxy:
    """ This class will hold authentication information for a handler,
        i.e. a connection
    """
    def __init__(self, provider):
        self.provider = provider

    def checkRequest(self,handler,path = '/'):
        """ Check if we are allowed to process that request
        """
        pass

class BasicAuthProxy(AuthProxy):
    """ Require basic authentication..
    """
    def __init__(self,provider):
        AuthProxy.__init__(self,provider)
        self.auth_creds = None
        self.auth_tries = 0

    def checkRequest(self,handler,path = '/'):
        if self.auth_creds:
            return True
        auth_str = handler.headers.get('Authorization',False)
        if auth_str and auth_str.startswith('Basic '):
            auth_str=auth_str[len('Basic '):]
            (user,passwd) = base64.decodestring(auth_str).split(':')
            self.provider.log("Found user=\"%s\", passwd=\"%s\"" %(user,passwd))
            self.auth_creds = self.provider.authenticate(user,passwd,handler.client_address)
            if self.auth_creds:
                return True
        if self.auth_tries > 5:
            self.provider.log("Failing authorization after 5 requests w/o password")
            raise AuthRejectedExc("Authorization failed.")
        self.auth_tries += 1
        raise AuthRequiredExc(atype = 'Basic', realm=self.provider.realm)

class HTTPHandler(SimpleHTTPRequestHandler):
    def __init__(self,request, client_address, server):
        SimpleHTTPRequestHandler.__init__(self,request,client_address,server)
        # print "Handler for %s inited" % str(client_address)
        self.protocol_version = 'HTTP/1.1'
        self.connection = dummyconn()

    def handle(self):
        """ Classes here should NOT handle inside their constructor
        """
        pass

    def finish(self):
        pass

    def setup(self):
        pass

class HTTPDir:
    """ A dispatcher class, like a virtual folder in httpd
    """
    def __init__(self,path,handler, auth_provider = None):
        self.path = path
        self.handler = handler
        self.auth_provider = auth_provider

    def matches(self, request):
        """ Test if some request matches us. If so, return
            the matched path. """
        if request.startswith(self.path):
            return self.path
        return False

class noconnection:
    """ a class to use instead of the real connection
    """
    def makefile(self, mode, bufsize):
        return None

class dummyconn:
    def shutdown(self, tru):
        pass

def _quote_html(html):
    return html.replace("&", "&").replace("<", "<").replace(">", ">")

class FixSendError:
    #error_message_format = """ """
    def send_error(self, code, message=None):
        #overriden from BaseHTTPRequestHandler, we also send the content-length
        try:
            short, long = self.responses[code]
        except KeyError:
            short, long = '???', '???'
        if message is None:
            message = short
        explain = long
        self.log_error("code %d, message %s", code, message)
        # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
        content = (self.error_message_format %
                   {'code': code, 'message': _quote_html(message), 'explain': explain})
        self.send_response(code, message)
        self.send_header("Content-Type", self.error_content_type)
        self.send_header('Connection', 'close')
        self.send_header('Content-Length', len(content) or 0)
        self.end_headers()
        if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
            self.wfile.write(content)

class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
    """ this is a multiple handler, that will dispatch each request
        to a nested handler, iff it matches

        The handler will also have *one* dict of authentication proxies,
        groupped by their realm.
    """

    protocol_version = "HTTP/1.1"
    default_request_version = "HTTP/0.9"    # compatibility with py2.5

    auth_required_msg = """ <title>Authorization required</title>
    You must authenticate to use this service\r\r"""

    def __init__(self, request, client_address, server):
        self.in_handlers = {}
        self.sec_realms = {}
        SocketServer.StreamRequestHandler.__init__(self,request,client_address,server)
        self.log_message("MultiHttpHandler init for %s" %(str(client_address)))

    def _handle_one_foreign(self,fore, path, auth_provider):
        """ This method overrides the handle_one_request for *children*
            handlers. It is required, since the first line should not be
            read again..

        """
        fore.raw_requestline = "%s %s %s\n" % (self.command, path, self.version)
        if not fore.parse_request(): # An error code has been sent, just exit
            return
        self.request_version = fore.request_version
        if auth_provider and auth_provider.realm:
            try:
                self.sec_realms[auth_provider.realm].checkRequest(fore,path)
            except AuthRequiredExc,ae:
                if self.request_version != 'HTTP/1.1':
                    self.log_error("Cannot require auth at %s",self.request_version)
                    self.send_error(403)
                    return
                self._get_ignore_body(fore) # consume any body that came, not loose sync with input
                self.send_response(401,'Authorization required')
                self.send_header('WWW-Authenticate','%s realm="%s"' % (ae.atype,ae.realm))
                self.send_header('Connection', 'keep-alive')
                self.send_header('Content-Type','text/html')
                self.send_header('Content-Length',len(self.auth_required_msg))
                self.end_headers()
                self.wfile.write(self.auth_required_msg)
                return
            except AuthRejectedExc,e:
                self.log_error("Rejected auth: %s" % e.args[0])
                self.send_error(403,e.args[0])
                self.close_connection = 1
                return
        mname = 'do_' + fore.command
        if not hasattr(fore, mname):
            fore.send_error(501, "Unsupported method (%r)" % fore.command)
            return
        fore.close_connection = 0
        method = getattr(fore, mname)
        method()
        if fore.close_connection:
            # print "Closing connection because of handler"
            self.close_connection = fore.close_connection
        if hasattr(fore, '_flush'):
            fore._flush()

    def parse_rawline(self):
        """Parse a request (internal).

        The request should be stored in self.raw_requestline; the results
        are in self.command, self.path, self.request_version and
        self.headers.

        Return True for success, False for failure; on failure, an
        error is sent back.

        """
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version
        self.close_connection = 1
        requestline = self.raw_requestline
        if requestline[-2:] == '\r\n':
            requestline = requestline[:-2]
        elif requestline[-1:] == '\n':
            requestline = requestline[:-1]
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            [command, path, version] = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            [command, path] = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            #self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.request_version = version
        self.command, self.path, self.version = command, path, version
        return True

    def handle_one_request(self):
        """Handle a single HTTP request.
           Dispatch to the correct handler.
        """
        self.request.setblocking(True)
        self.raw_requestline = self.rfile.readline()
        if not self.raw_requestline:
            self.close_connection = 1
            # self.log_message("no requestline, connection closed?")
            return
        if not self.parse_rawline():
            self.log_message("Could not parse rawline.")
            return
        # self.parse_request(): # Do NOT parse here. the first line should be the only
        for vdir in self.server.vdirs:
            p = vdir.matches(self.path)
            if p == False:
                continue
            npath = self.path[len(p):]
            if not npath.startswith('/'):
                npath = '/' + npath

            if not self.in_handlers.has_key(p):
                self.in_handlers[p] = vdir.handler(noconnection(),self.client_address,self.server)
                if vdir.auth_provider:
                    vdir.auth_provider.setupAuth(self, self.in_handlers[p])
            hnd = self.in_handlers[p]
            hnd.rfile = self.rfile
            hnd.wfile = self.wfile
            self.rlpath = self.raw_requestline
            self._handle_one_foreign(hnd,npath, vdir.auth_provider)
            # print "Handled, closing = ", self.close_connection
            return
        # if no match:
        self.send_error(404, "Path not found: %s" % self.path)
        return

    def _get_ignore_body(self,fore):
        if not fore.headers.has_key("content-length"):
            return
        max_chunk_size = 10*1024*1024
        size_remaining = int(fore.headers["content-length"])
        got = ''
        while size_remaining:
            chunk_size = min(size_remaining, max_chunk_size)
            got = fore.rfile.read(chunk_size)
            size_remaining -= len(got)

class SecureMultiHTTPHandler(MultiHTTPHandler):
    def getcert_fnames(self):
        """ Return a pair with the filenames of ssl cert,key

            Override this to direct to other filenames
        """
        return ('server.cert','server.key')

    def setup(self):
        import ssl
        certfile, keyfile = self.getcert_fnames()
        try:
            self.connection = ssl.wrap_socket(self.request,
                                server_side=True,
                                certfile=certfile,
                                keyfile=keyfile,
                                ssl_version=ssl.PROTOCOL_SSLv23)
            self.rfile = self.connection.makefile('rb', self.rbufsize)
            self.wfile = self.connection.makefile('wb', self.wbufsize)
            self.log_message("Secure %s connection from %s",self.connection.cipher(),self.client_address)
        except Exception:
            self.request.shutdown(socket.SHUT_RDWR)
            raise

    def finish(self):
        # With ssl connections, closing the filehandlers alone may not
        # work because of ref counting. We explicitly tell the socket
        # to shutdown.
        MultiHTTPHandler.finish(self)
        try:
            self.connection.shutdown(socket.SHUT_RDWR)
        except Exception:
            pass

import threading
class ConnThreadingMixIn:
    """Mix-in class to handle each _connection_ in a new thread.

       This is necessary for persistent connections, where multiple
       requests should be handled synchronously at each connection, but
       multiple connections can run in parallel.
    """

    # Decides how threads will act upon termination of the
    # main process
    daemon_threads = False

    def _handle_request_noblock(self):
        """Start a new thread to process the request."""
        if not threading: # happens while quitting python
            return
        t = threading.Thread(target = self._handle_request2)
        if self.daemon_threads:
            t.setDaemon (1)
        t.start()

    def _handle_request2(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except Exception:
                self.handle_error(request, client_address)
                self.close_request(request)

#eof
<|endoftext|>


<commit_before># Lint as: python3
# Copyright 2018, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Library methods for working with centralized data used in simulation.

N.B. Federated Learning does not use client IDs or perform any tracking of
clients. However in simulation experiments using centralized test data the
experimenter may select specific clients to be processed per round. The concept
of a client ID is only available at the preprocessing stage when preparing input
data for the simulation and is not part of the TensorFlow Federated core APIs.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import numpy as np

import six
import tensorflow as tf
from tensorflow_federated.python.common_libs import py_typecheck

@six.add_metaclass(abc.ABCMeta)
class ClientData(object):
  """Object to hold a dataset and a mapping of clients to examples."""

  @abc.abstractproperty
  def client_ids(self):
    """The list of string identifiers for clients in this dataset."""
    pass

  @abc.abstractmethod
  def create_tf_dataset_for_client(self, client_id):
    """Creates a new `tf.data.Dataset` containing the client training examples.

    Args:
      client_id: The string client_id for the desired client.

    Returns:
      A `tf.data.Dataset` object.
    """
    pass

  @abc.abstractproperty
  def output_types(self):
    """Returns the type of each component of an element of the client datasets.

    Any `tf.data.Dataset` constructed by this class is expected have matching
    `output_types` properties when accessed via
    `tf.compat.v1.data.get_output_types(dataset)`.

    Returns:
      A nested structure of `tf.DType` objects corresponding to each component
      of an element of the client datasets.
    """
    pass

  @abc.abstractproperty
  def output_shapes(self):
    """Returns the shape of each component of an element of the client datasets.

    Any `tf.data.Dataset` constructed by this class is expected to have matching
    `output_shapes` properties when accessed via
    `tf.compat.v1.data.get_output_shapes(dataset)`.

    Returns:
      A nested structure of `tf.TensorShape` objects corresponding to each
      component of an element of the client datasets.
    """
    pass

  def create_tf_dataset_from_all_clients(self, seed=None):
    """Creates a new `tf.data.Dataset` containing _all_ client examples.

    NOTE: the returned `tf.data.Dataset` is not serializable and runnable on
    other devices, as it uses `tf.py_func` internally.

    Currently, the implementation produces a dataset that contains
    all examples from a single client in order, and so generally additional
    shuffling should be performed.

    Args:
      seed: Optional, a seed to determine the order in which clients are
        processed in the joined dataset.

    Returns:
      A `tf.data.Dataset` object.
    """

    # NOTE: simply calling Dataset.concatenate() will result in too deep
    # recursion depth.
    # NOTE: Tests are via the simple concrete from_tensor_slices_client_data.
    def _generator():
      client_ids = list(self.client_ids)
      np.random.RandomState(seed=seed).shuffle(client_ids)
      for client_id in client_ids:
        for example in self.create_tf_dataset_for_client(client_id):
          yield example

    return tf.data.Dataset.from_generator(_generator, self.output_types,
                                          self.output_shapes)

  def preprocess(self, preprocess_fn):
    """Applies `preprocess_fn` to each client's data."""
    py_typecheck.check_callable(preprocess_fn)

    def get_dataset(client_id):
      return preprocess_fn(self.create_tf_dataset_for_client(client_id))

    return ConcreteClientData(self.client_ids, get_dataset)

class ConcreteClientData(ClientData):
  """A generic `ClientData` object.

  This is a simple implementation of client_data, where Datasets are specified
  as a function from client_id to Dataset.

  The `ConcreteClientData.preprocess` classmethod is provided as a utility
  used to wrap another `ClientData` with an additional preprocessing function.
  """

  def __init__(self, client_ids, create_tf_dataset_for_client_fn):
    """Arguments correspond to the corresponding members of `ClientData`.

    Args:
      client_ids: A non-empty list of client_ids.
      create_tf_dataset_for_client_fn: A function that takes a client_id from
        the above list, and returns a `tf.data.Dataset`.
    """
    py_typecheck.check_type(client_ids, list)
    py_typecheck.check_callable(create_tf_dataset_for_client_fn)
    if not client_ids:
      raise ValueError('At least one client_id is required.')

    self._client_ids = client_ids
    self._create_tf_dataset_for_client_fn = create_tf_dataset_for_client_fn

    example_dataset = create_tf_dataset_for_client_fn(client_ids[0])
    self._output_types = example_dataset.output_types
    self._output_shapes = example_dataset.output_shapes

  @property
  def client_ids(self):
    return self._client_ids

  def create_tf_dataset_for_client(self, client_id):
    return self._create_tf_dataset_for_client_fn(client_id)

  @property
  def output_types(self):
    return self._output_types

  @property
  def output_shapes(self):
    return self._output_shapes
<commit_msg>Use compat.v1 api for getting tf.data.Dataset shapes and types for TF 2.0 compatibility.<commit_after># Lint as: python3
# Copyright 2018, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Library methods for working with centralized data used in simulation.

N.B. Federated Learning does not use client IDs or perform any tracking of
clients. However in simulation experiments using centralized test data the
experimenter may select specific clients to be processed per round. The concept
of a client ID is only available at the preprocessing stage when preparing input
data for the simulation and is not part of the TensorFlow Federated core APIs.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import abc
import numpy as np

import six
import tensorflow as tf
from tensorflow_federated.python.common_libs import py_typecheck

@six.add_metaclass(abc.ABCMeta)
class ClientData(object):
  """Object to hold a dataset and a mapping of clients to examples."""

  @abc.abstractproperty
  def client_ids(self):
    """The list of string identifiers for clients in this dataset."""
    pass

  @abc.abstractmethod
  def create_tf_dataset_for_client(self, client_id):
    """Creates a new `tf.data.Dataset` containing the client training examples.

    Args:
      client_id: The string client_id for the desired client.

    Returns:
      A `tf.data.Dataset` object.
    """
    pass

  @abc.abstractproperty
  def output_types(self):
    """Returns the type of each component of an element of the client datasets.

    Any `tf.data.Dataset` constructed by this class is expected have matching
    `output_types` properties when accessed via
    `tf.compat.v1.data.get_output_types(dataset)`.

    Returns:
      A nested structure of `tf.DType` objects corresponding to each component
      of an element of the client datasets.
    """
    pass

  @abc.abstractproperty
  def output_shapes(self):
    """Returns the shape of each component of an element of the client datasets.

    Any `tf.data.Dataset` constructed by this class is expected to have matching
    `output_shapes` properties when accessed via
    `tf.compat.v1.data.get_output_shapes(dataset)`.

    Returns:
      A nested structure of `tf.TensorShape` objects corresponding to each
      component of an element of the client datasets.
    """
    pass

  def create_tf_dataset_from_all_clients(self, seed=None):
    """Creates a new `tf.data.Dataset` containing _all_ client examples.

    NOTE: the returned `tf.data.Dataset` is not serializable and runnable on
    other devices, as it uses `tf.py_func` internally.

    Currently, the implementation produces a dataset that contains
    all examples from a single client in order, and so generally additional
    shuffling should be performed.

    Args:
      seed: Optional, a seed to determine the order in which clients are
        processed in the joined dataset.

    Returns:
      A `tf.data.Dataset` object.
    """

    # NOTE: simply calling Dataset.concatenate() will result in too deep
    # recursion depth.
    # NOTE: Tests are via the simple concrete from_tensor_slices_client_data.
    def _generator():
      client_ids = list(self.client_ids)
      np.random.RandomState(seed=seed).shuffle(client_ids)
      for client_id in client_ids:
        for example in self.create_tf_dataset_for_client(client_id):
          yield example

    return tf.data.Dataset.from_generator(_generator, self.output_types,
                                          self.output_shapes)

  def preprocess(self, preprocess_fn):
    """Applies `preprocess_fn` to each client's data."""
    py_typecheck.check_callable(preprocess_fn)

    def get_dataset(client_id):
      return preprocess_fn(self.create_tf_dataset_for_client(client_id))

    return ConcreteClientData(self.client_ids, get_dataset)

class ConcreteClientData(ClientData):
  """A generic `ClientData` object.

  This is a simple implementation of client_data, where Datasets are specified
  as a function from client_id to Dataset.

  The `ConcreteClientData.preprocess` classmethod is provided as a utility
  used to wrap another `ClientData` with an additional preprocessing function.
  """

  def __init__(self, client_ids, create_tf_dataset_for_client_fn):
    """Arguments correspond to the corresponding members of `ClientData`.

    Args:
      client_ids: A non-empty list of client_ids.
      create_tf_dataset_for_client_fn: A function that takes a client_id from
        the above list, and returns a `tf.data.Dataset`.
    """
    py_typecheck.check_type(client_ids, list)
    py_typecheck.check_callable(create_tf_dataset_for_client_fn)
    if not client_ids:
      raise ValueError('At least one client_id is required.')

    self._client_ids = client_ids
    self._create_tf_dataset_for_client_fn = create_tf_dataset_for_client_fn

    example_dataset = create_tf_dataset_for_client_fn(client_ids[0])
    self._output_types = tf.compat.v1.data.get_output_types(example_dataset)
    self._output_shapes = tf.compat.v1.data.get_output_shapes(example_dataset)

  @property
  def client_ids(self):
    return self._client_ids

  def create_tf_dataset_for_client(self, client_id):
    return self._create_tf_dataset_for_client_fn(client_id)

  @property
  def output_types(self):
    return self._output_types

  @property
  def output_shapes(self):
    return self._output_shapes
<|endoftext|>


<commit_before>import unittest
from tests.ggrc import TestCase
from ggrc.models import get_model
from ggrc.models import all_models
from tests.ggrc.api_helper import Api
from tests.ggrc.generator import Generator
from tests.ggrc.generator import GgrcGenerator

class TestCreator(TestCase):
  def setUp(self):
    TestCase.setUp(self)

    self.generator = Generator()
    self.api = Api()
    self.ggrc_generator = GgrcGenerator()
    self.maxDiff = None
    self._generate_users()

  def _generate_users(self):
    users = [('creator', 'Creator'), ('admin', 'gGRC Admin')]
    for (name, role) in users:
      _, user = self.ggrc_generator.generate_person(
          data={"name": name}, user_role=role)
      setattr(self, name, user)

  def test_creator_can_crud_basic_models(self):
    self.api.set_user(self.creator)
    all_errors = []
    base_models = set([
        'Control', 'ControlAssessment', 'DataAsset', 'Contract',
        'Policy', 'Regulation', 'Standard', 'Document', 'Facility',
        'Market', 'Objective', 'OrgGroup', 'Vendor', 'Product',
        'Clause', 'System', 'Process', 'Issue', 'Project'
    ])
    for model_singular in base_models:
      try:
        model = get_model(model_singular)
        table_singular = model._inflector.table_singular
        table_plural = model._inflector.table_plural
        # Test POST creation
        response = self.api.post(model, {
            table_singular: {
                "title": model_singular,
                "context": None,
                "reference_url": "ref",
                "contact": {
                    "type": "Person",
                    "id": self.creator.id,
                },
            },
        })
        if response.status_code != 201:
          all_errors.append("{} post creation failed {} {}".format(
              model_singular, response.status, response.data))
          continue

        # Test GET when not owner
        obj_id = response.json.get(table_singular).get('id')
        response = self.api.get(model, obj_id)
        if response.status_code != 403:  # we are not onwers yet
          all_errors.append(
              "{} can retrieve object if not owner".format(model_singular))
          continue
        response = self.api.get_collection(model, obj_id)
        collection = response.json.get(
            "{}_collection".format(table_plural)).get(table_plural)
        if len(collection) != 0:
          all_errors.append(
              "{} can retrieve object if not owner (collection)".format(model_singular))
          continue
        # Become an owner
        response = self.api.post(all_models.ObjectOwner, {"object_owner": {
            "person": {
                "id": self.creator.id,
                "type": "Person",
            }, "ownable": {
                "type": model_singular,
                "id": obj_id
            }, "context": None},
        })
        if response.status_code != 201:
          all_errors.append("{} can't create owner {}.".format(
              model_singular, response.status))
          continue

        # Test GET when owner
        response = self.api.get(model, obj_id)
        if response.status_code != 200:
          all_errors.append("{} can't GET object {}".format(
              model_singular, response.status))
          continue

        # Test GET collection when owner
        response = self.api.get_collection(model, obj_id)
        collection = response.json.get(
            "{}_collection".format(table_plural)).get(table_plural)
        if len(collection) == 0:
          all_errors.append(
              "{} cannot retrieve object even if owner (collection)".format(model_singular))
          continue
      except:
          all_errors.append("{} exception thrown".format(model_singular))
          raise
    self.assertEquals(all_errors, [])

  def test_creator_search(self):
    self.api.set_user(self.admin)
    self.api.post(all_models.Regulation, {
        "regulation": {"title": "Admin regulation", "context": None},
    })
    self.api.set_user(self.creator)
    response = self.api.post(all_models.Policy, {
        "policy": {"title": "Creator Policy", "context": None},
    })
    obj_id = response.json.get('policy').get('id')
    self.api.post(all_models.ObjectOwner, {"object_owner": {
        "person": {
            "id": self.creator.id,
            "type": "Person",
        }, "ownable": {
            "type": "Policy",
            "id": obj_id,
        }, "context": None},
    })
    response, _ = self.api.search('Regulation,Policy')
    entries = response.json['results']['entries']
    self.assertEquals(len(entries), 1)
    self.assertEquals(entries[0]['type'], "Policy")
    response, _ = self.api.search('Regulation,Policy', counts=True)
    self.assertEquals(response.json['results']['counts']['Policy'], 1)
    self.assertEquals(
        response.json['results']['counts'].get('Regulation'), None)

  def _get_count(self, obj):
    response, _ = self.api.search(obj, counts=True)
    return response.json['results']['counts'].get(obj)

  def test_creator_should_see_all_users(self):
    self.api.set_user(self.admin)
    admin_count = self._get_count('Person')
    self.api.set_user(self.creator)
    creator_count = self._get_count('Person')
    self.assertEquals(admin_count, creator_count)

  def test_creator_cannot_become_owner(self):
    self.api.set_user(self.admin)
    _, obj = self.generator.generate(all_models.Regulation, "regulation", {
        "regulation": {"title": "Test regulation", "context": None},
    })
    self.api.set_user(self.creator)
    response = self.api.post(all_models.ObjectOwner, {"object_owner": {
        "person": {
            "id": self.creator.id,
            "type": "Person",
        }, "ownable": {
            "type": "Regulation",
            "id": obj.id,
        }, "context": None},
    })
    self.assertEquals(response.status_code, 403)

if __name__ == '__main__':
  unittest.main()
<commit_msg>Fix TestCreator test Fix TestCreator by renaming GgrcGenerator to ObjectGenerator<commit_after>import unittest
from tests.ggrc import TestCase
from ggrc.models import get_model
from ggrc.models import all_models
from tests.ggrc.api_helper import Api
from tests.ggrc.generator import Generator
from tests.ggrc.generator import ObjectGenerator

class TestCreator(TestCase):
  def setUp(self):
    TestCase.setUp(self)

    self.generator = Generator()
    self.api = Api()
    self.object_generator = ObjectGenerator()
    self.maxDiff = None
    self._generate_users()

  def _generate_users(self):
    users = [('creator', 'Creator'), ('admin', 'gGRC Admin')]
    for (name, role) in users:
      _, user = self.object_generator.generate_person(
          data={"name": name}, user_role=role)
      setattr(self, name, user)

  def test_creator_can_crud_basic_models(self):
    self.api.set_user(self.creator)
    all_errors = []
    base_models = set([
        'Control', 'ControlAssessment', 'DataAsset', 'Contract',
        'Policy', 'Regulation', 'Standard', 'Document', 'Facility',
        'Market', 'Objective', 'OrgGroup', 'Vendor', 'Product',
        'Clause', 'System', 'Process', 'Issue', 'Project'
    ])
    for model_singular in base_models:
      try:
        model = get_model(model_singular)
        table_singular = model._inflector.table_singular
        table_plural = model._inflector.table_plural
        # Test POST creation
        response = self.api.post(model, {
            table_singular: {
                "title": model_singular,
                "context": None,
                "reference_url": "ref",
                "contact": {
                    "type": "Person",
                    "id": self.creator.id,
                },
            },
        })
        if response.status_code != 201:
          all_errors.append("{} post creation failed {} {}".format(
              model_singular, response.status, response.data))
          continue

        # Test GET when not owner
        obj_id = response.json.get(table_singular).get('id')
        response = self.api.get(model, obj_id)
        if response.status_code != 403:  # we are not onwers yet
          all_errors.append(
              "{} can retrieve object if not owner".format(model_singular))
          continue
        response = self.api.get_collection(model, obj_id)
        collection = response.json.get(
            "{}_collection".format(table_plural)).get(table_plural)
        if len(collection) != 0:
          all_errors.append(
              "{} can retrieve object if not owner (collection)".format(model_singular))
          continue
        # Become an owner
        response = self.api.post(all_models.ObjectOwner, {"object_owner": {
            "person": {
                "id": self.creator.id,
                "type": "Person",
            }, "ownable": {
                "type": model_singular,
                "id": obj_id
            }, "context": None},
        })
        if response.status_code != 201:
          all_errors.append("{} can't create owner {}.".format(
              model_singular, response.status))
          continue

        # Test GET when owner
        response = self.api.get(model, obj_id)
        if response.status_code != 200:
          all_errors.append("{} can't GET object {}".format(
              model_singular, response.status))
          continue

        # Test GET collection when owner
        response = self.api.get_collection(model, obj_id)
        collection = response.json.get(
            "{}_collection".format(table_plural)).get(table_plural)
        if len(collection) == 0:
          all_errors.append(
              "{} cannot retrieve object even if owner (collection)".format(model_singular))
          continue
      except:
          all_errors.append("{} exception thrown".format(model_singular))
          raise
    self.assertEquals(all_errors, [])

  def test_creator_search(self):
    self.api.set_user(self.admin)
    self.api.post(all_models.Regulation, {
        "regulation": {"title": "Admin regulation", "context": None},
    })
    self.api.set_user(self.creator)
    response = self.api.post(all_models.Policy, {
        "policy": {"title": "Creator Policy", "context": None},
    })
    obj_id = response.json.get('policy').get('id')
    self.api.post(all_models.ObjectOwner, {"object_owner": {
        "person": {
            "id": self.creator.id,
            "type": "Person",
        }, "ownable": {
            "type": "Policy",
            "id": obj_id,
        }, "context": None},
    })
    response, _ = self.api.search('Regulation,Policy')
    entries = response.json['results']['entries']
    self.assertEquals(len(entries), 1)
    self.assertEquals(entries[0]['type'], "Policy")
    response, _ = self.api.search('Regulation,Policy', counts=True)
    self.assertEquals(response.json['results']['counts']['Policy'], 1)
    self.assertEquals(
        response.json['results']['counts'].get('Regulation'), None)

  def _get_count(self, obj):
    response, _ = self.api.search(obj, counts=True)
    return response.json['results']['counts'].get(obj)

  def test_creator_should_see_all_users(self):
    self.api.set_user(self.admin)
    admin_count = self._get_count('Person')
    self.api.set_user(self.creator)
    creator_count = self._get_count('Person')
    self.assertEquals(admin_count, creator_count)

  def test_creator_cannot_become_owner(self):
    self.api.set_user(self.admin)
    _, obj = self.generator.generate(all_models.Regulation, "regulation", {
        "regulation": {"title": "Test regulation", "context": None},
    })
    self.api.set_user(self.creator)
    response = self.api.post(all_models.ObjectOwner, {"object_owner": {
        "person": {
            "id": self.creator.id,
            "type": "Person",
        }, "ownable": {
            "type": "Regulation",
            "id": obj.id,
        }, "context": None},
    })
    self.assertEquals(response.status_code, 403)

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for util."""
import tensorflow as tf
from tensorflow_model_analysis.notebook.colab import util

class UtilTest(tf.test.TestCase):

  def testGenerateSimpleHtmlForTfmaComponent(self):
    tfma_component_name = 'tfma-nb-slicing-metrics'
    html_code = util.generate_html_for_tfma_component(tfma_component_name, None,
                                                      None, '')
    # Count opening and closing tag
    self.assertEqual(html_code.count(tfma_component_name), 2)

  def testGenerateForNotTrustedTfmaComponent(self):
    tfma_component_name = 'my-component'
    with self.assertRaises(ValueError):
      util.generate_html_for_tfma_component(tfma_component_name, None, None, '')

if __name__ == '__main__':
  tf.test.main()
<commit_msg>Automated rollback of commit 7679915999905528cca912e7d824b1dc4f548d9e<commit_after><|endoftext|>