<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import copy
import os

import dataproperty

from .error import InvalidFilePathError
from .error import LoaderNotFoundError

from .csv.core import CsvTableFileLoader
from .html.core import HtmlTableFileLoader
from .json.core import JsonTableFileLoader
from .markdown.core import MarkdownTableFileLoader
from .mediawiki.core import MediaWikiTableFileLoader
from .spreadsheet.excelloader import ExcelTableFileLoader

class FileLoaderFactory(object):
    """
    :param str file_path: File path to loading.
    :raises pytablereader.InvalidFilePathError:
        If the ``file_path`` is a empty path.
    """

    __COMMON_LOADER_TABLE = {
        "csv": CsvTableFileLoader,
        "html": HtmlTableFileLoader,
        "json": JsonTableFileLoader,
    }

    @property
    def file_path(self):
        """
        :return: File path to loading.
        :rtype: str
        """

        return self.__file_path

    @property
    def file_extension(self):
        """
        :return: File extension of the :py:attr:`.file_path` (without period).
        :rtype: str
        """

        return self.__file_extension

    def __init__(self, file_path):
        self.__file_path = file_path

        self.__validate()

        self.__file_extension = os.path.splitext(self.file_path)[1].lstrip(".")

    def create_from_file_path(self):
        """
        Create a file loader from file extension to loading.

        :return: Loader that coincide with the :py:attr:`.file_path`.
        :raises pytablereader.LoaderNotFoundError:
            If appropriate file loader not found.
        """

        try:
            return self.__create_loader(
                self.get_extension_loader_mapping(), self.file_extension)
        except LoaderNotFoundError as e:
            raise LoaderNotFoundError("\n".join([
                "{:s} (unknown file extension).".format(e.args[0]),
                "",
                "acceptable file extensions are: {}.".format(
                    ", ".join(sorted(self.get_extension_loader_mapping()))),
                "",
                "or specify destination file format with '--from' option.",
                "acceptable file formats to '--from' options are: {}.".format(
                    ", ".join(sorted(self.get_format_name_loader_mapping()))),
            ]))

    def create_from_format_name(self, format_name):
        """
        Create a file loader from file extension to loading.

        :return:
            Loader that coincide with the ``format_name``:

            +-----------+---------------------------------------------------+
            |Format name|         Loader                                    |
            +===========+===================================================+
            |csv        |:py:class:`~pytablereader.CsvTableFileLoader`      |
            +-----------+---------------------------------------------------+
            |excel      |:py:class:`~pytablereader.ExcelTableFileLoader`    |
            +-----------+---------------------------------------------------+
            |html       |:py:class:`~pytablereader.HtmlTableFileLoader`     |
            +-----------+---------------------------------------------------+
            |json       |:py:class:`~pytablereader.JsonTableFileLoader`     |
            +-----------+---------------------------------------------------+
            |markdown   |:py:class:`~pytablereader.MarkdownTableFileLoader` |
            +-----------+---------------------------------------------------+
            |mediawiki  |:py:class:`~pytablereader.MediaWikiTableFileLoader`|
            +-----------+---------------------------------------------------+
        :raises |LoaderNotFoundError|: If appropriate file loader not found.
        """

        if format_name.lower() == "auto":
            return self.create_from_file_path()

        try:
            return self.__create_loader(
                self.get_format_name_loader_mapping(), format_name)
        except LoaderNotFoundError as e:
            raise LoaderNotFoundError("\n".join([
                "{:s} (unknown file format).".format(e.args[0]),
                "specify destination file format with '--to' option.",
                "acceptable file formats to '--to' options are: {}.".format(
                    ", ".join(sorted(self.get_format_name_loader_mapping()))),
            ]))

    def __validate(self):
        if dataproperty.is_empty_string(self.file_path):
            raise InvalidFilePathError("file path is empty")

    def __create_loader(self, loader_table, format_name):
        format_name = format_name.lower()

        try:
            return loader_table[format_name](self.file_path)
        except KeyError:
            raise LoaderNotFoundError(", ".join([
                "loader not found: format='{:s}'".format(format_name),
                "path='{:s}'".format(self.file_path),
            ]))

    @classmethod
    def get_format_name_loader_mapping(self):
        """
        :return: Mappings of format-name and loader class.
        :rtype: dict
        """

        loader_table = copy.deepcopy(self.__COMMON_LOADER_TABLE)
        loader_table.update({
            "excel": ExcelTableFileLoader,
            "markdown": MarkdownTableFileLoader,
            "mediawiki": MediaWikiTableFileLoader,
        })

        return loader_table

    @classmethod
    def get_extension_loader_mapping(self):
        """
        :return: Mappings of format-extension and loader class.
        :rtype: dict
        """

        loader_table = copy.deepcopy(
            self.__COMMON_LOADER_TABLE)
        loader_table .update({
            "htm": HtmlTableFileLoader,
            "md": MarkdownTableFileLoader,
            "xlsx": ExcelTableFileLoader,
            "xls": ExcelTableFileLoader,
        })

        return loader_table
<commit_msg>Update docstring<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import copy
import os

import dataproperty

from .error import InvalidFilePathError
from .error import LoaderNotFoundError

from .csv.core import CsvTableFileLoader
from .html.core import HtmlTableFileLoader
from .json.core import JsonTableFileLoader
from .markdown.core import MarkdownTableFileLoader
from .mediawiki.core import MediaWikiTableFileLoader
from .spreadsheet.excelloader import ExcelTableFileLoader

class FileLoaderFactory(object):
    """
    :param str file_path: File path to loading.
    :raises pytablereader.InvalidFilePathError:
        If the ``file_path`` is a empty path.
    """

    __COMMON_LOADER_TABLE = {
        "csv": CsvTableFileLoader,
        "html": HtmlTableFileLoader,
        "json": JsonTableFileLoader,
    }

    @property
    def file_path(self):
        """
        :return: File path to loading.
        :rtype: str
        """

        return self.__file_path

    @property
    def file_extension(self):
        """
        :return: File extension of the :py:attr:`.file_path` (without period).
        :rtype: str
        """

        return self.__file_extension

    def __init__(self, file_path):
        self.__file_path = file_path

        self.__validate()

        self.__file_extension = os.path.splitext(self.file_path)[1].lstrip(".")

    def create_from_file_path(self):
        """
        Create a file loader from the file extension to loading file.
        Supported file extensions are as follows:

            +----------------+--------------------------------------------------+
            |Format name     |         Loader                                   |
            +================+==================================================+
            |``csv``         |:py:class:`~pytablereader.CsvTableFileLoader`     |
            +----------------+--------------------------------------------------+
            |``xls``/``xlsx``|:py:class:`~pytablereader.ExcelTableFileLoader`   |
            +----------------+--------------------------------------------------+
            |``htm``/``html``|:py:class:`~pytablereader.HtmlTableFileLoader`    |
            +----------------+--------------------------------------------------+
            |``json``        |:py:class:`~pytablereader.JsonTableFileLoader`    |
            +----------------+--------------------------------------------------+
            |``md``          |:py:class:`~pytablereader.MarkdownTableFileLoader`|
            +----------------+--------------------------------------------------+

        :return:
            Loader that coincide with the file extesnion of
            :py:attr:`.file_path`.
        :raises pytablereader.LoaderNotFoundError:
            If appropriate file loader not found.
        """

        try:
            return self.__create_loader(
                self.get_extension_loader_mapping(), self.file_extension)
        except LoaderNotFoundError as e:
            raise LoaderNotFoundError("\n".join([
                "{:s} (unknown file extension).".format(e.args[0]),
                "",
                "acceptable file extensions are: {}.".format(
                    ", ".join(sorted(self.get_extension_loader_mapping()))),
                "",
                "or specify destination file format with '--from' option.",
                "acceptable file formats to '--from' options are: {}.".format(
                    ", ".join(sorted(self.get_format_name_loader_mapping()))),
            ]))

    def create_from_format_name(self, format_name):
        """
        Create a file loader from a format name.
        Supported file formats are as follows:

            +---------------+---------------------------------------------------+
            |Format name    |         Loader                                    |
            +===============+===================================================+
            |``"csv"``      |:py:class:`~pytablereader.CsvTableFileLoader`      |
            +---------------+---------------------------------------------------+
            |``"excel"``    |:py:class:`~pytablereader.ExcelTableFileLoader`    |
            +---------------+---------------------------------------------------+
            |``"html"``     |:py:class:`~pytablereader.HtmlTableFileLoader`     |
            +---------------+---------------------------------------------------+
            |``"json"``     |:py:class:`~pytablereader.JsonTableFileLoader`     |
            +---------------+---------------------------------------------------+
            |``"markdown"`` |:py:class:`~pytablereader.MarkdownTableFileLoader` |
            +---------------+---------------------------------------------------+
            |``"mediawiki"``|:py:class:`~pytablereader.MediaWikiTableFileLoader`|
            +---------------+---------------------------------------------------+

        This method will call :py:meth:`.create_from_file_path` method
        if the format name is ``"auto"``.

        :param str format_name:
            Format name string or ``"auto"`` (case insensitive).
        :return: Loader that coincide with the ``format_name``:
        :raises pytablereader.LoaderNotFoundError:
            If appropriate file loader not found.
        """

        if format_name.lower() == "auto":
            return self.create_from_file_path()

        try:
            return self.__create_loader(
                self.get_format_name_loader_mapping(), format_name)
        except LoaderNotFoundError as e:
            raise LoaderNotFoundError("\n".join([
                "{:s} (unknown file format).".format(e.args[0]),
                "specify destination file format with '--to' option.",
                "acceptable file formats to '--to' options are: {}.".format(
                    ", ".join(sorted(self.get_format_name_loader_mapping()))),
            ]))

    def __validate(self):
        if dataproperty.is_empty_string(self.file_path):
            raise InvalidFilePathError("file path is empty")

    def __create_loader(self, loader_table, format_name):
        format_name = format_name.lower()

        try:
            return loader_table[format_name](self.file_path)
        except KeyError:
            raise LoaderNotFoundError(", ".join([
                "loader not found: format='{:s}'".format(format_name),
                "path='{:s}'".format(self.file_path),
            ]))

    @classmethod
    def get_format_name_loader_mapping(self):
        """
        :return: Mappings of format-name and loader class.
        :rtype: dict
        """

        loader_table = copy.deepcopy(self.__COMMON_LOADER_TABLE)
        loader_table.update({
            "excel": ExcelTableFileLoader,
            "markdown": MarkdownTableFileLoader,
            "mediawiki": MediaWikiTableFileLoader,
        })

        return loader_table

    @classmethod
    def get_extension_loader_mapping(self):
        """
        :return: Mappings of format-extension and loader class.
        :rtype: dict
        """

        loader_table = copy.deepcopy(
            self.__COMMON_LOADER_TABLE)
        loader_table .update({
            "htm": HtmlTableFileLoader,
            "md": MarkdownTableFileLoader,
            "xlsx": ExcelTableFileLoader,
            "xls": ExcelTableFileLoader,
        })

        return loader_table
<|endoftext|>


<commit_before>import hashlib
import sha3
import argparse

def calculate_hash(input, hash_types=['sha1', 'sha256', 'sha512', 'sha3']):
    for hash_type in hash_types:
        if hash_type == 'sha1':
            hash_obj = hashlib.sha1()
        elif hash_type == 'sha256':
            hash_obj = hashlib.sha256()
        elif hash_type == 'sha512':
            hash_obj = hashlib.sha512()
        elif hash_type == 'sha384':
            hash_obj = hashlib.sha384()
        elif hash_type == 'sha3':
            hash_obj = sha3.keccak_512()

        hash_obj.update(input)
        print('{} \t {}'.format(hash_type, hash_obj.hexdigest()))

parser = argparse.ArgumentParser(description='Calculate hash of a file or string.', add_help=True)

group = parser.add_argument_group('input_group')
group.add_argument('-f', '--file', help='the input file to calculate the hash')
group.add_argument('-t', '--text', help='the input text to calculate the hash')
parser.add_argument('-ht', '--hash-type', help='the type of the hash to be calculated')

args = parser.parse_args()

if args.file is not None:
    with open(args.file, "rb") as binary_file:
        data = binary_file.read()
elif args.text is not None:
    data = str.encode(args.text)

if data is not None:
    calculate_hash(data)
else:
    parser.print_help()
<commit_msg>Add md5 and compare hashes<commit_after>import hashlib
import argparse

supported_hashes = ['md5', 'sha1', 'sha256', 'sha512', 'sha384']

def calculate_hash(input, hash_types=supported_hashes, compare_hash=None):
    for hash_type in hash_types:
        if hash_type == 'md5':
            hash_obj = hashlib.md5()
        elif hash_type == 'sha1':
            hash_obj = hashlib.sha1()
        elif hash_type == 'sha256':
            hash_obj = hashlib.sha256()
        elif hash_type == 'sha512':
            hash_obj = hashlib.sha512()
        elif hash_type == 'sha384':
            hash_obj = hashlib.sha384()

        hash_obj.update(input)
        current_hash = hash_obj.hexdigest()
        print('{} \t {}'.format(hash_type, current_hash))

        if compare_hash is not None:
            if compare_hash == current_hash:
                print('\x1b[6;30;42m' + 'Hash match!' + '\x1b[0m')
            else:
                print('\x1b[6;30;42m' + 'Hash do not match!' + '\x1b[0m')

parser = argparse.ArgumentParser(description='Calculate hash of a file or string.', add_help=True)

parser.add_argument('-f', '--file', help='the input file to calculate the hash')
parser.add_argument('-t', '--text', help='the input text to calculate the hash')
parser.add_argument('-ht', '--hash-type', help='the type of the hash to be calculated', choices=supported_hashes)
parser.add_argument('-c', '--compare', help='the hash to compare with')

args = parser.parse_args()

if args.file is not None:
    with open(args.file, "rb") as binary_file:
        data = binary_file.read()
elif args.text is not None:
    data = str.encode(args.text)
else:
    data = None

if data is not None:
    calculate_hash(data, compare_hash=args.compare)
else:
    parser.print_help()
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import sys
from shutil import copyfile, copytree, rmtree

from setuptools import setup

TEMP_PATH = "bigdl/share/dllib"
bigdl_home = os.path.abspath(__file__ + "/../../../..")

try:
    exec(open('bigdl/version.py').read())
except IOError:
    print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.")
    sys.exit(-1)

VERSION = __version__

building_error_msg = """
If you are packing python API from BigDL source, you must build BigDL first
and run sdist.
    To build BigDL with maven you can run:
      cd $BigDL_HOME
      ./make-dist.sh
    Building the source dist is done in the Python directory:
      cd python
      python setup.py sdist
      pip install dist/*.tar.gz"""

def build_from_source():
    code_path = bigdl_home + "/python/dllib/src/bigdl/dllib/utils/common.py"
    print("Checking: %s to see if build from source" % code_path)
    if os.path.exists(code_path):
        return True
    return False

def init_env():
    if build_from_source():
        print("Start to build distributed package")
        print("HOME OF BIGDL: " + bigdl_home)
        dist_source = bigdl_home + "/dist"
        if not os.path.exists(dist_source):
            print(building_error_msg)
            sys.exit(-1)
        if os.path.exists(TEMP_PATH):
            rmtree(TEMP_PATH)
        copytree(dist_source, TEMP_PATH)
        copyfile(bigdl_home + "/python/dllib/src/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py")
    else:
        print("Do nothing for release installation")

def get_bigdl_packages():
    bigdl_python_home = os.path.abspath(__file__)[:-8]
    bigdl_packages = ['bigdl.share.dllib']
    for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"):
        package = dirpath.split(bigdl_python_home)[1].replace('/', '.')
        if "__pycache__" not in package:
            bigdl_packages.append(package)
    print("=========================bigdl packages=========================")
    print("\n".join(bigdl_packages))
    print("================================================================")
    return bigdl_packages

def setup_package():
    metadata = dict(
        name='BigDL-dllib',
        version=VERSION,
        description='Distributed Deep Learning Library for Apache Spark',
        author='BigDL Authors',
        author_email='bigdl-user-group@googlegroups.com',
        license='Apache License, Version 2.0',
        url='https://github.com/intel-analytics/Bigdl',
        packages=get_bigdl_packages(),
        install_requires=['numpy>=1.7', 'pyspark==2.4.6', 'six>=1.10.0'],
        dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'],
        include_package_data=True,
        package_data={"bigdl.share.dllib": ['lib/bigdl-dllib*.jar', 'conf/*']},
        classifiers=[
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3',
            'Programming Language :: Python :: 3.4',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: Implementation :: CPython'],
        platforms=['mac', 'linux']
    )

    setup(**metadata)

if __name__ == '__main__':
    try:
        init_env()
        setup_package()
    except Exception as e:
        raise e
    finally:
        if build_from_source() and os.path.exists(TEMP_PATH):
             rmtree(TEMP_PATH)

<commit_msg>unify release bigdl2.0 PyPI project name (#4818)<commit_after>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import sys
from shutil import copyfile, copytree, rmtree

from setuptools import setup

TEMP_PATH = "bigdl/share/dllib"
bigdl_home = os.path.abspath(__file__ + "/../../../..")

try:
    exec(open('bigdl/version.py').read())
except IOError:
    print("Failed to load Bigdl version file for packaging. You must be in Bigdl's python dir.")
    sys.exit(-1)

VERSION = __version__

building_error_msg = """
If you are packing python API from BigDL source, you must build BigDL first
and run sdist.
    To build BigDL with maven you can run:
      cd $BigDL_HOME
      ./make-dist.sh
    Building the source dist is done in the Python directory:
      cd python
      python setup.py sdist
      pip install dist/*.tar.gz"""

def build_from_source():
    code_path = bigdl_home + "/python/dllib/src/bigdl/dllib/utils/common.py"
    print("Checking: %s to see if build from source" % code_path)
    if os.path.exists(code_path):
        return True
    return False

def init_env():
    if build_from_source():
        print("Start to build distributed package")
        print("HOME OF BIGDL: " + bigdl_home)
        dist_source = bigdl_home + "/dist"
        if not os.path.exists(dist_source):
            print(building_error_msg)
            sys.exit(-1)
        if os.path.exists(TEMP_PATH):
            rmtree(TEMP_PATH)
        copytree(dist_source, TEMP_PATH)
        copyfile(bigdl_home + "/python/dllib/src/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py")
    else:
        print("Do nothing for release installation")

def get_bigdl_packages():
    bigdl_python_home = os.path.abspath(__file__)[:-8]
    bigdl_packages = ['bigdl.share.dllib']
    for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"):
        package = dirpath.split(bigdl_python_home)[1].replace('/', '.')
        if "__pycache__" not in package:
            bigdl_packages.append(package)
    print("=========================bigdl packages=========================")
    print("\n".join(bigdl_packages))
    print("================================================================")
    return bigdl_packages

def setup_package():
    metadata = dict(
        name='bigdl-dllib',
        version=VERSION,
        description='Distributed Deep Learning Library for Apache Spark',
        author='BigDL Authors',
        author_email='bigdl-user-group@googlegroups.com',
        license='Apache License, Version 2.0',
        url='https://github.com/intel-analytics/Bigdl',
        packages=get_bigdl_packages(),
        install_requires=['numpy>=1.7', 'pyspark==2.4.6', 'six>=1.10.0'],
        dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'],
        include_package_data=True,
        package_data={"bigdl.share.dllib": ['lib/bigdl-dllib*.jar', 'conf/*']},
        classifiers=[
            'License :: OSI Approved :: Apache Software License',
            'Programming Language :: Python :: 2.7',
            'Programming Language :: Python :: 3',
            'Programming Language :: Python :: 3.4',
            'Programming Language :: Python :: 3.5',
            'Programming Language :: Python :: Implementation :: CPython'],
        platforms=['mac', 'linux']
    )

    setup(**metadata)

if __name__ == '__main__':
    try:
        init_env()
        setup_package()
    except Exception as e:
        raise e
    finally:
        if build_from_source() and os.path.exists(TEMP_PATH):
             rmtree(TEMP_PATH)

<|endoftext|>


<commit_before>'''
Created on Feb 28, 2013

@author: jmht
'''

# python imports
import logging
import os
import re
import shutil
import sys
import unittest

# our imports
import clusterize
import Final_display_results
import printTable
import run_mr_bump_shelx_parallel

def mrbump_ensemble_cluster( ensembles, amopt, clusterID="X" ):
    """
    Process the list of ensembles using MrBump on a cluster
    INPUTS:
    ensembles - list of pdb files, each file containing a group of ensembles
    amopt - amopt object
    clusterId: number/id of the cluster that is being processed
    """
    logger = logging.getLogger()
    logger.info("Running MR and model building on a cluster\n\n")

    mrBuild = clusterize.ClusterRun()
    mrBuild.QTYPE = "SGE"

    mrBuildClusterDir = os.path.join(amopt.d['mrbump_dir'], "cluster_run" + str(clusterID))
    os.mkdir(mrBuildClusterDir)
    mrBuild.getMTZInfo(amopt.d['mtz'], mrBuildClusterDir)

    # Reset the queue list
    mrBuild.qList = []
    jobID = 0
    for pdbfile in ensembles:
        mrBuild.mrBuildOnCluster(mrBuildClusterDir, pdbfile, jobID, amopt)
        jobID = jobID + 1
    mrBuild.monitorQueue()

    work_dir = amopt.d['work_dir']
    shutil.rmtree(work_dir + '/fine_cluster_' + str(clusterID))
    # shutil.rmtree(work_dir+'/pre_models')
    for l in os.listdir(work_dir + '/spicker_run'):
        if os.path.splitext(l)[1] == 'pdb':
            os.remove(work_dir + '/spicker_run/' + l)
    os.remove(work_dir + '/spicker_run/rep1.tra1')
    T = printTable.Table()

    T.bumppath = mrBuildClusterDir
    T.cluster = True
    table = T.maketable()
    out = sys.stdout
    T.pprint_table(out, table)

        # cleanup
    # for each_run in os.listdir(mrBuildClusterDir ):
        #   if os.path.isdir(  os.path.join(mrBuildClusterDir, each_run)):
        #      name=re.split('_', each_run)
        #      mrBuildOutputDir=os.path.join(bump_dir, "cluster_run"+str(cluster)+"result"+name[1])
        #      os.mkdir(mrBuildOutputDir)
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","phaser_shelx" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","molrep_shelx" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","data" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "logs" ),mrBuildOutputDir  )
        # shutil.rmtree(mrBuildClusterDir)

    # Monitor the cluster queue to see when all jobs have finished
##End mrbump_ensemble_cluster

def mrbump_ensemble_local( ensembles, amopt, clusterID="X" ):
    """
    Process the list of ensembles using MrBump on a local machine
    INPUTS:
    ensembles - list of pdb files, each file containing a group of ensembles
    amopt - amopt object
    clusterId: number/id of the cluster that is being processed
    """
    # Split ensembles and rum mrbump
    split_ensembles = run_mr_bump_shelx_parallel.split(ensembles, amopt.d['nproc'])
    run_mr_bump_shelx_parallel.split_into_runs(split_ensembles, amopt)

    bump_dir = amopt.d['mrbump_dir']
    # Process results
    if amopt.d['use_shelxe']:
        Final_display_results.make_log(bump_dir, os.path.join(amopt.d['work_dir'], 'Final_results.log'))
        # print '\n\nFinal Results:\n\n'
        # T=printTable.Table()
        # T.bumppath = work_dir +'/MRBUMP_cluster'+str(cluster)
        # T.cluster = False
        # table = T.maketable()
        # out = sys.stdout
        # T.pprint_table(out, table)
    else:
        resultslog = open(amopt.d['work_dir'] + os.sep + 'Results.log', "w")
        print 'getting results from: {}',format( bump_dir )
        for mrbumplog in os.listdir(bump_dir):
            if re.search('.log', mrbumplog):
                # print mrbumplog
                for line in open(mrbumplog):
                    if re.search('^(\d)\s*loc0_', line):
                        if not re.search('method', line):
                            print line
                            resultslog.write(line)
        resultslog.close()

##End mrbump_ensemble_local

class Test(unittest.TestCase):

    def testName(self):
        pass

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<commit_msg>Commented out unused code<commit_after>'''
Created on Feb 28, 2013

@author: jmht
'''

# python imports
import logging
import os
import re
import shutil
import sys
import unittest

# our imports
import clusterize
import Final_display_results
import printTable
import run_mr_bump_shelx_parallel

def mrbump_ensemble_cluster( ensembles, amopt, clusterID="X" ):
    """
    Process the list of ensembles using MrBump on a cluster
    INPUTS:
    ensembles - list of pdb files, each file containing a group of ensembles
    amopt - amopt object
    clusterId: number/id of the cluster that is being processed
    """
    logger = logging.getLogger()
    logger.info("Running MR and model building on a cluster\n\n")

    mrBuild = clusterize.ClusterRun()
    mrBuild.QTYPE = "SGE"

    mrBuildClusterDir = os.path.join(amopt.d['mrbump_dir'], "cluster_run" + str(clusterID))
    os.mkdir(mrBuildClusterDir)
    #mrBuild.getMTZInfo(amopt.d['mtz'], mrBuildClusterDir)

    # Reset the queue list
    mrBuild.qList = []
    jobID = 0
    for pdbfile in ensembles:
        mrBuild.mrBuildOnCluster(mrBuildClusterDir, pdbfile, jobID, amopt)
        jobID = jobID + 1
    mrBuild.monitorQueue()

    work_dir = amopt.d['work_dir']
    shutil.rmtree(work_dir + '/fine_cluster_' + str(clusterID))
    # shutil.rmtree(work_dir+'/pre_models')
    for l in os.listdir(work_dir + '/spicker_run'):
        if os.path.splitext(l)[1] == 'pdb':
            os.remove(work_dir + '/spicker_run/' + l)
    os.remove(work_dir + '/spicker_run/rep1.tra1')
    T = printTable.Table()

    T.bumppath = mrBuildClusterDir
    T.cluster = True
    table = T.maketable()
    out = sys.stdout
    T.pprint_table(out, table)

        # cleanup
    # for each_run in os.listdir(mrBuildClusterDir ):
        #   if os.path.isdir(  os.path.join(mrBuildClusterDir, each_run)):
        #      name=re.split('_', each_run)
        #      mrBuildOutputDir=os.path.join(bump_dir, "cluster_run"+str(cluster)+"result"+name[1])
        #      os.mkdir(mrBuildOutputDir)
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","phaser_shelx" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","molrep_shelx" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "search_"+name[1]+"_mrbump","data" ),mrBuildOutputDir  )
        #      shutil.move (os.path.join(mrBuildClusterDir, each_run, "logs" ),mrBuildOutputDir  )
        # shutil.rmtree(mrBuildClusterDir)

    # Monitor the cluster queue to see when all jobs have finished
##End mrbump_ensemble_cluster

def mrbump_ensemble_local( ensembles, amopt, clusterID="X" ):
    """
    Process the list of ensembles using MrBump on a local machine
    INPUTS:
    ensembles - list of pdb files, each file containing a group of ensembles
    amopt - amopt object
    clusterId: number/id of the cluster that is being processed
    """
    # Split ensembles and rum mrbump
    split_ensembles = run_mr_bump_shelx_parallel.split(ensembles, amopt.d['nproc'])
    run_mr_bump_shelx_parallel.split_into_runs(split_ensembles, amopt)

    bump_dir = amopt.d['mrbump_dir']
    # Process results
    if amopt.d['use_shelxe']:
        Final_display_results.make_log(bump_dir, os.path.join(amopt.d['work_dir'], 'Final_results.log'))
        # print '\n\nFinal Results:\n\n'
        # T=printTable.Table()
        # T.bumppath = work_dir +'/MRBUMP_cluster'+str(cluster)
        # T.cluster = False
        # table = T.maketable()
        # out = sys.stdout
        # T.pprint_table(out, table)
    else:
        resultslog = open(amopt.d['work_dir'] + os.sep + 'Results.log', "w")
        print 'getting results from: {}',format( bump_dir )
        for mrbumplog in os.listdir(bump_dir):
            if re.search('.log', mrbumplog):
                # print mrbumplog
                for line in open(mrbumplog):
                    if re.search('^(\d)\s*loc0_', line):
                        if not re.search('method', line):
                            print line
                            resultslog.write(line)
        resultslog.close()

##End mrbump_ensemble_local

class Test(unittest.TestCase):

    def testName(self):
        pass

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<|endoftext|>


<commit_before># Parallel processing can be done in single machines or distributed in a cluster
  # There are many packages for both.
  # Single machines include multiprocessing & the default package concurrent.futures which work very similarly
  # Clusters include parallel python and jug
  # More here: https://wiki.python.org/moin/ParallelProcessing

# More on multiprocessing vs multithreading
  # https://medium.com/towards-artificial-intelligence/the-why-when-and-how-of-using-python-multi-threading-and-multi-processing-afd1b8a8ecca
  # CPU-bound tasks > multiprocessing
      # calculations
  # IO-bound tasks > multithreading
      # network
      # read-write files

# MULTIPROCESSING
# check no. of cores
#---------------------------------------
import multiprocessing as mp
print(mp.cpu_count())

#---------------------------------------
import multiprocessing as mp

def worker(files):
    some task

def pooling(allfiles, name, newfolderpath, acctHeader, fieldnamefile):
    list_to_iterate = [file1, file2, file3, file4]

    process_spawn = min(mp.cpu_count(), len(allfiles))
    pool = mp.Pool(process_spawn)
    pool.map(worker, list_to_iterate, chunksize=1)
    pool.close()

#---------------------------------------
# for worker function that require more than one variable, use 'partial' function to 'group' them together
import multiprocessing as mp
from functools import partial

def fdtw(match, pattern, features, distances):
    do_something

def pooling(win_slices, pattern, features, cores=1):
    processes = round(min(mp.cpu_count(), len(win_slices))*cores)
    pool = mp.Pool(processes)
    fdtw_partial = partial(fdtw, pattern=pattern, features=features, distances=distances)
    pool.map(fdtw_partial, win_slices, chunksize=1)
    pool.close()

# use multiprocessing pool
# https://www.youtube.com/watch?v=s1SkCYMnfbY
#---------------------------------------
# EXAMPLE 1
# unzip files from different folders
import tarfile
import gzip
import os
import multiprocessing as mp
from time import time

# worker, task to process, i.e., unzipping
def unziptar(folder):
    """worker unzips one file"""
    for file in os.listdir(folder):
        filepath = os.path.join(folder, file)
        if file.endswith("tar.gz"):
            print('extracting... {}'.format(filepath))
            tar = tarfile.open(filepath, 'r:gz')
            tar.extractall(os.path.dirname(filepath))
            tar.close()

# spawn processes for each loop
def fanout_unziptar(path):
    """create pool to extract all"""
    # collect all paths of tar.gz
    my_files = []
    for root, dirs, files in os.walk(path):
        for i in files:
            if i.endswith("tar.gz"):
                my_files.append(root)
                # my_files.append(os.path.join(root, i))
    my_files = set(my_files) #remove duplicates

    # set number of workers
    # note, for unzipping its I/O intensive~ so allocating too many cores will burn the RAM and make it even slower.
    pool = mp.Pool(processes=4)
    # separate the list of files such that only 1 file take one core (chunksize)
    pool.map(unziptar, my_files, chunksize=1)
    pool.close()

# need this "if" else windows will have recursive error
# a python entry point for a function
if __name__ == "__main__":
    start = time()
    path = r"/Users/jake/Desktop/test"
    fanout_unziptar(path)
    end = time()
    print('script ended after {} mins'.format((end-start)/60))

# set special list or dict for results of multiprocess to pump within
#---------------------------------------
l = mp.Manager().list()

d = mp.Manager().dict()
# convert dict proxy to dict
d = json.dumps(d.copy())
d = json.loads(d)

# set error logging
import logging
mpl = mp.log_to_stderr()
mpl.setLevel(logging.INFO)

def find_background(img, imgfolder, color="white", threshold="0.3"):
    "worker that that delete images"
    os.remove("some images")
    return img

def find_background_pool(imgfolder, engine, keywords):
    """parallel processing"""
    processes = round(min(mp.cpu_count(), len(img_list_new)))
    pool = mp.Pool(processes)
    find_background_ = partial(find_background, imgfolder=imgfolder, color="white", threshold=0.3)
    # returned images will be all stored within a list
    deleted_imgs = pool.map(find_background_, img_list_new)
    pool.close()

# execute different functions concurrently -----
# v1
from functools import partial
from multiprocessing import Pool

def a(param1, param2, param3):
    return param1 + param2 + param3

def b(param1, param2):
    return param1 + param2

def smap(f):
    return f()

func1 = partial(a, 1, 2, 3)
func2 = partial(b, 1, 2)

pool = Pool(processes=2)
res = pool.map(smap, [func1, func2])
pool.close()
pool.join()
print(res)

# v2
from multiprocessing import Process
import os
import datetime

def func_1(title):
    now = datetime.datetime.now()
    print "hello, world"
    print "Current second: %d" % now.second
    print "Current microsecond: %d" % now.microsecond

def func_2(name):
    now = datetime.datetime.now()
    print "Bye, world"
    print "Current second: %d" % now.second
    print "Current microsecond: %d" % now.microsecond

if __name__ == '__main__':
    procs = []
    procs.append(Process(target=func_2, args=('bob',)))
    procs.append(Process(target=func_1, args=('sir',)))
    map(lambda x: x.start(), procs)
    map(lambda x: x.join(), procs)

# MULTI THREADING
# https://stackoverflow.com/questions/48994440/execute-a-function-after-flask-returns-response/51013358
from threading import Thread

def do_something(json_input):
    something = json_input
    return something

thread = Thread(target=do_something, kwargs={'json_input': json})
thread.start()

# https://www.digitalocean.com/community/tutorials/how-to-use-threadpoolexecutor-in-python-3
import requests
import concurrent.futures

def get_wiki_page_existence(wiki_page_url, timeout=10):
    response = requests.get(url=wiki_page_url, timeout=timeout)

    page_status = "unknown"
    if response.status_code == 200:
        page_status = "exists"
    elif response.status_code == 404:
        page_status = "does not exist"

    return wiki_page_url + " - " + page_status

wiki_page_urls = [
    "https://en.wikipedia.org/wiki/Ocean",
    "https://en.wikipedia.org/wiki/Island",
    "https://en.wikipedia.org/wiki/this_page_does_not_exist",
    "https://en.wikipedia.org/wiki/Shark",
]
with concurrent.futures.ThreadPoolExecutor() as executor:
    futures = []
    for url in wiki_page_urls:
        futures.append(executor.submit(get_wiki_page_existence, wiki_page_url=url))
    for future in concurrent.futures.as_completed(futures):
        print(future.result())

# RAY
#---------------------------------------
# use apache arrow backend, supposed much faster then multiprocessing
# https://towardsdatascience.com/10x-faster-parallel-python-without-python-multiprocessing-e5017c93cce1

# execute embarassingly parallel task -----
import ray
ray.init()

@ray.remote
def f(x):
    return x * x

futures = [f.remote(i) for i in range(4)]
print(ray.get(futures))

# execute different functions concurrently -----
import ray

ray.init(num_cpus=2)

# Define functions you want to execute in parallel using
# the ray.remote decorator.
@ray.remote
def func1():
    print("Working1")

@ray.remote
def func2():
    print("Working2")

# Execute func1 and func2 in parallel.
for i in range(20):
    ray.get([func1.remote(), func2.remote()])<commit_msg>Add joblib<commit_after># Parallel processing can be done in single machines or distributed in a cluster
  # There are many packages for both.
  # Single machines include multiprocessing & the default package concurrent.futures which work very similarly
  # Clusters include parallel python and jug
  # More here: https://wiki.python.org/moin/ParallelProcessing

# More on multiprocessing vs multithreading
  # https://medium.com/towards-artificial-intelligence/the-why-when-and-how-of-using-python-multi-threading-and-multi-processing-afd1b8a8ecca
  # CPU-bound tasks > multiprocessing
      # calculations
  # IO-bound tasks > multithreading
      # network
      # read-write files

# MULTIPROCESSING
# check no. of cores
#---------------------------------------
import multiprocessing as mp
print(mp.cpu_count())

#---------------------------------------
import multiprocessing as mp

def worker(files):
    some task

def pooling(allfiles, name, newfolderpath, acctHeader, fieldnamefile):
    list_to_iterate = [file1, file2, file3, file4]

    process_spawn = min(mp.cpu_count(), len(allfiles))
    pool = mp.Pool(process_spawn)
    pool.map(worker, list_to_iterate, chunksize=1)
    pool.close()

#---------------------------------------
# for worker function that require more than one variable, use 'partial' function to 'group' them together
import multiprocessing as mp
from functools import partial

def fdtw(match, pattern, features, distances):
    do_something

def pooling(win_slices, pattern, features, cores=1):
    processes = round(min(mp.cpu_count(), len(win_slices))*cores)
    pool = mp.Pool(processes)
    fdtw_partial = partial(fdtw, pattern=pattern, features=features, distances=distances)
    pool.map(fdtw_partial, win_slices, chunksize=1)
    pool.close()

# use multiprocessing pool
# https://www.youtube.com/watch?v=s1SkCYMnfbY
#---------------------------------------
# EXAMPLE 1
# unzip files from different folders
import tarfile
import gzip
import os
import multiprocessing as mp
from time import time

# worker, task to process, i.e., unzipping
def unziptar(folder):
    """worker unzips one file"""
    for file in os.listdir(folder):
        filepath = os.path.join(folder, file)
        if file.endswith("tar.gz"):
            print('extracting... {}'.format(filepath))
            tar = tarfile.open(filepath, 'r:gz')
            tar.extractall(os.path.dirname(filepath))
            tar.close()

# spawn processes for each loop
def fanout_unziptar(path):
    """create pool to extract all"""
    # collect all paths of tar.gz
    my_files = []
    for root, dirs, files in os.walk(path):
        for i in files:
            if i.endswith("tar.gz"):
                my_files.append(root)
                # my_files.append(os.path.join(root, i))
    my_files = set(my_files) #remove duplicates

    # set number of workers
    # note, for unzipping its I/O intensive~ so allocating too many cores will burn the RAM and make it even slower.
    pool = mp.Pool(processes=4)
    # separate the list of files such that only 1 file take one core (chunksize)
    pool.map(unziptar, my_files, chunksize=1)
    pool.close()

# need this "if" else windows will have recursive error
# a python entry point for a function
if __name__ == "__main__":
    start = time()
    path = r"/Users/jake/Desktop/test"
    fanout_unziptar(path)
    end = time()
    print('script ended after {} mins'.format((end-start)/60))

# set special list or dict for results of multiprocess to pump within
#---------------------------------------
l = mp.Manager().list()

d = mp.Manager().dict()
# convert dict proxy to dict
d = json.dumps(d.copy())
d = json.loads(d)

# set error logging
import logging
mpl = mp.log_to_stderr()
mpl.setLevel(logging.INFO)

def find_background(img, imgfolder, color="white", threshold="0.3"):
    "worker that that delete images"
    os.remove("some images")
    return img

def find_background_pool(imgfolder, engine, keywords):
    """parallel processing"""
    processes = round(min(mp.cpu_count(), len(img_list_new)))
    pool = mp.Pool(processes)
    find_background_ = partial(find_background, imgfolder=imgfolder, color="white", threshold=0.3)
    # returned images will be all stored within a list
    deleted_imgs = pool.map(find_background_, img_list_new)
    pool.close()

# execute different functions concurrently -----
# v1
from functools import partial
from multiprocessing import Pool

def a(param1, param2, param3):
    return param1 + param2 + param3

def b(param1, param2):
    return param1 + param2

def smap(f):
    return f()

func1 = partial(a, 1, 2, 3)
func2 = partial(b, 1, 2)

pool = Pool(processes=2)
res = pool.map(smap, [func1, func2])
pool.close()
pool.join()
print(res)

# v2
from multiprocessing import Process
import os
import datetime

def func_1(title):
    now = datetime.datetime.now()
    print "hello, world"
    print "Current second: %d" % now.second
    print "Current microsecond: %d" % now.microsecond

def func_2(name):
    now = datetime.datetime.now()
    print "Bye, world"
    print "Current second: %d" % now.second
    print "Current microsecond: %d" % now.microsecond

if __name__ == '__main__':
    procs = []
    procs.append(Process(target=func_2, args=('bob',)))
    procs.append(Process(target=func_1, args=('sir',)))
    map(lambda x: x.start(), procs)
    map(lambda x: x.join(), procs)

# MULTI THREADING
# https://stackoverflow.com/questions/48994440/execute-a-function-after-flask-returns-response/51013358
from threading import Thread

def do_something(json_input):
    something = json_input
    return something

thread = Thread(target=do_something, kwargs={'json_input': json})
thread.start()

# https://www.digitalocean.com/community/tutorials/how-to-use-threadpoolexecutor-in-python-3
import requests
import concurrent.futures

def get_wiki_page_existence(wiki_page_url, timeout=10):
    response = requests.get(url=wiki_page_url, timeout=timeout)

    page_status = "unknown"
    if response.status_code == 200:
        page_status = "exists"
    elif response.status_code == 404:
        page_status = "does not exist"

    return wiki_page_url + " - " + page_status

wiki_page_urls = [
    "https://en.wikipedia.org/wiki/Ocean",
    "https://en.wikipedia.org/wiki/Island",
    "https://en.wikipedia.org/wiki/this_page_does_not_exist",
    "https://en.wikipedia.org/wiki/Shark",
]
with concurrent.futures.ThreadPoolExecutor() as executor:
    futures = []
    for url in wiki_page_urls:
        futures.append(executor.submit(get_wiki_page_existence, wiki_page_url=url))
    for future in concurrent.futures.as_completed(futures):
        print(future.result())

# RAY
#---------------------------------------
# use apache arrow backend, supposed much faster then multiprocessing
# https://towardsdatascience.com/10x-faster-parallel-python-without-python-multiprocessing-e5017c93cce1

# execute embarassingly parallel task -----
import ray
ray.init()

@ray.remote
def f(x):
    return x * x

futures = [f.remote(i) for i in range(4)]
print(ray.get(futures))

# execute different functions concurrently -----
import ray

ray.init(num_cpus=2)

# Define functions you want to execute in parallel using
# the ray.remote decorator.
@ray.remote
def func1():
    print("Working1")

@ray.remote
def func2():
    print("Working2")

# Execute func1 and func2 in parallel.
for i in range(20):
    ray.get([func1.remote(), func2.remote()])

# Joblib
#---------------------------------------

# sequential
sentences = [preprocess(text) for text in pages]
# parallel
from joblib import Parallel, delayed
sentences = Parallel(n_jobs=5)(delayed(preprocess)(text) for text in pages)<|endoftext|>


<commit_before>import _pyjava

# Exception classes
from _pyjava import Error, ClassNotFound, NoMatchingOverload

__all__ = [
        'Error', 'ClassNotFound', 'NoMatchingOverload',
        'start', 'getclass']

# TODO : this could be a descriptor whose __get__ method return a
# _BoundJavaMethod... I think
class _UnboundJavaMethod(object):
    """An unbound Java method.

    You can call it like a regular method. It might raise NoMatchingOverload if
    the parameters number or types are not compatible with one of the
    overloads.

    Note that this type represents both object methods before they are bound
    and static methods.
    """
    def __init__(self, method):
        self.__method = method

    def __call__(self, jself, *args):
        return self.__method.call(jself, *args)
                # might raise NoMatchingOverload

class _BoundJavaMethod(object):
    """A bound Java method.

    You can call it like a regular method. It might raise NoMatchingOverload if
    the parameters number or types are not compatible with one of the
    overloads.
    """
    def __init__(self, method, obj):
        self.__method = method
        self.__obj = obj

    def __call__(self, *args):
        return self.__method.call(self.__obj, *args)
                # might raise NoMatchingOverload

# TODO : define __instancecheck__ and __subclasscheck__ to override Python's
# isinstance() and issubclass() builtins
# PEP 3119:
#   The primary mechanism proposed here is to allow overloading the built-in
#   functions isinstance() and issubclass(). The overloading works as follows:
#     The call isinstance(x, C) first checks whether C.__instancecheck__
#     exists, and if so, calls C.__instancecheck__(x) instead of its normal
#     implementation.
#     Similarly, the call issubclass(D, C) first checks whether
#     C.__subclasscheck__ exists, and if so, calls C.__subclasscheck__(D)
#     instead of its normal implementation.

# This is the "special" java.lang.Class class
class _JavaClass(type):
    _JavaObject_constructed = False

    def __new__(*args, **kwargs):
        if len(args) == 4 and not _JavaClass._JavaObject_constructed:
            # This class may only be used once as __metaclass__:
            # to construct _JavaObject (just below)
            _JavaClass._JavaObject_constructed = True
            args[3].update(dict(_pyjava__javaclass = None)) # initialization of
                    # this field happens in start(), as getclass() needs a
                    # running VM
            cls = type.__new__(
                    _JavaClass,
                    args[1],
                    (),
                    args[3])
        elif len(args) == 4 and _JavaClass._JavaObject_constructed:
            raise Error("_JavaClass shouldn't be used as __metaclass__")
        elif len(args) != 1:
            raise TypeError("_JavaClass only accepts keyword arguments "
                            "_javaclass and _classname")
        elif set(kwargs.keys()) == set(['_javaclass', '_classname']):
            # It is used with the keyword argument _javaclass to construct
            # classes
            cls = type.__new__(
                    _JavaClass,
                    kwargs['_classname'],
                    (),
                    dict(_pyjava__javaclass=kwargs['_javaclass']))
        else:
            raise TypeError("Invalid keyword arguments to _JavaClass")

        return cls

    def __init__(self, *args, **kwargs):
        # Will be called after __new__
        # We just override type.__init__ so that it accepts the arguments meant
        # for __new__
        pass

    def __getattr__(self, attr):
        # Method
        try:
            return _UnboundJavaMethod(
                    self.__dict__['_pyjava__javaclass'].getmethod(attr))
        except AttributeError:
            pass
        # TODO : static fields
        raise AttributeError

    def __setattr__(self, attr, value):
        # TODO : static fields
        pass

# This is the root Java class, java.lang.Object
# It is also instance of the meta-class java.lang.Class
class _JavaObject(object):
    __metaclass__ = _JavaClass

    def __init__(self, *args, **kwargs):
        # We accept a keyword-only argument _javaobject=None
        try:
            _javaobject = kwargs.pop('_javaobject')
        except KeyError:
            _javaobject = None
        if kwargs:
            raise TypeError('_JavaObject.__init__ got an unexpected '
                            'keyword argument %s' % kwargs.keys()[0])

        if _javaobject is not None:
            self._pyjava__javaobject = _javaobject
        else:
            # This might raise NoMatchingOverload
            self._pyjava__javaobject = (
                    self._pyjava__javaclass.create(*args))

    # This is called when an attribute is assigned
    # If it matches a Java field, update it, else call object.__setattr__
    def __setattr__(self, attr, value):
        # TODO : non-static fields
        pass

    # Contrary to __getattribute__, this will only be called if nothing is
    # found with the usual mechanism
    # We attempt to find a field or a method in the Java class or raise
    # AttributeError
    def __getattr__(self, attr):
        # Method
        try:
            return _BoundJavaMethod(
                    self._pyjava__javaclass.getmethod(attr),
                    self._pyjava__javaobject)
        except AttributeError:
            pass
        # TODO : non-static fields
        raise AttributeError

def start(path, *args):
    try:
        if len(args) == 0:
            _pyjava.start(path, [])
        elif len(args) == 1 and (isinstance(args[0], list)):
            _pyjava.start(path, args[0])
        else:
            _pyjava.start(path, args)
    except Error:
        raise Error("Unable to start Java VM with path %s" % path)

    _JavaObject._pyjava__javaclass = _pyjava.getclass('java/lang/Object')

_known_classes = {
        'java.lang.Object': _JavaObject,
        'java.lang.Class': _JavaClass}

def getclass(classname):
    try:
        return _known_classes[classname]
    except KeyError:
        # Convert from the 'usual' syntax to the 'JNI' syntax
        jni_classname = classname.replace('.', '/')

        cls = _pyjava.getclass(jni_classname) # might raise ClassNotFound
        cls = _JavaClass(_classname=classname, _javaclass=cls)
        _known_classes[classname] = cls
        return cls
<commit_msg>Rewrites Python side<commit_after>import _pyjava

# Exception classes
from _pyjava import Error, ClassNotFound, NoMatchingOverload

__all__ = [
        'Error', 'ClassNotFound', 'NoMatchingOverload',
        'start', 'getclass']

class _UnboundJavaMethod(object):
    """An unbound Java method.

    You can call it like a regular method. It might raise NoMatchingOverload if
    the parameters number or types are not compatible with one of the
    overloads.

    Note that this type represents both object methods before they are bound
    and static methods.
    """
    def __init__(self, method):
        self._pyjava_method = method

    def __call__(self, jself, *args):
        return self._pyjava_method.call(jself, *args)
                # might raise NoMatchingMethod

class _BoundJavaMethod(object):
    """A bound Java method.

    You can call it like a regular method. It might raise NoMatchingOverload if
    the parameters number or types are not compatible with one of the
    overloads.
    """
    def __init__(self, method, obj):
        self._pyjava_method = method
        self._pyjava_obj = obj

    def __call__(self, *args):
        return self._pyjava_method.call(self.__obj, *args)
                # might raise NoMatchingOverload

# Here we define __instancecheck__ and __subclasscheck__ to override Python's
# isinstance() and issubclass() builtins
#
# PEP 3119:
#   The primary mechanism proposed here is to allow overloading the built-in
#   functions isinstance() and issubclass(). The overloading works as follows:
#     The call isinstance(x, C) first checks whether C.__instancecheck__
#     exists, and if so, calls C.__instancecheck__(x) instead of its normal
#     implementation.
#     Similarly, the call issubclass(D, C) first checks whether
#     C.__subclasscheck__ exists, and if so, calls C.__subclasscheck__(D)
#     instead of its normal implementation.

# This is the metaclass, i.e. objects of this type represent Java classes
# It is NOT java.lang.Class however
class _JavaClass(object):
    def __init__(self, classname, javaclass):
        self.__dict__['_pyjava_classname'] = classname
        self.__dict__['_pyjava_javaclass'] = javaclass

    def __getattr__(self, attr):
        # Method
        try:
            # Use __dict__ to avoid infinite recursion
            return _UnboundJavaMethod(
                    self.__dict__['_pyjava_javaclass'].getmethod(attr))
        except AttributeError:
            pass
        # TODO : static fields
        raise AttributeError

    def __setattr__(self, attr, value):
        # TODO : static fields
        pass

    def __instancecheck__(self, inst):
        """Called to check if an object is an instance of this class.
        """
        if isinstance(inst, _JavaObject):
            return _pyjava.issubclass(
                    inst.__dict__['_pyjava_class'].__dict__['_pyjava_javaclass'],
                    self.__dict__['_pyjava_javaclass'])
        else:
            return False

    def __subclasscheck__(self, other):
        """Called to check if a class is a subclass of this class
        """
        if isinstance(other, _JavaClass):
            return _pyjava.issubclass(other.__dict__['_pyjava_javaclass'],
                                      self.__dict__['_pyjava_javaclass'])
        elif isinstance(other, type): # A non-Java class
            return False
        else: # Not a class
            raise TypeError("issubclass() arg 2 must be a class")

    def __call__(self, *args):
        """Called to instanciate this class.
        """
        return _JavaObject(
                self.__dict__['_pyjava_javaclass'],
                self.__dict__['_pyjava_javaclass'].create(*args))
                # This might raise NoMatchingOverload

# This is the real class of all Java objects, even though we pretend (by
# overriding isinstance() and issubclass() behaviors) that they are instances
# of instances of _JavaClass (which we pretend are classes)
class _JavaObject(object):
    def __init__(self, klass, javaobject):
        self.__dict__['_pyjava_class'] = klass
        self.__dict__['_pyjava_javaobject'] = javaobject

    # This is called when an attribute is assigned
    # If it matches a Java field, update it, else call object.__setattr__
    def __setattr__(self, attr, value):
        # TODO : non-static fields
        pass

    # Contrary to __getattribute__, this will only be called if nothing is
    # found with the usual mechanism
    # We attempt to find a field or a method in the Java class or raise
    # AttributeError
    def __getattr__(self, attr):
        # Method
        try:
            return _BoundJavaMethod(
                    self.__dict__['_pyjava__javaclass'].getmethod(attr),
                    self.__dict__['_pyjava__javaobject'])
        except AttributeError:
            pass
        # TODO : non-static fields
        raise AttributeError

def start(path, *args):
    try:
        if len(args) == 0:
            _pyjava.start(path, [])
        elif len(args) == 1 and (isinstance(args[0], list)):
            _pyjava.start(path, args[0])
        else:
            _pyjava.start(path, args)
    except Error:
        raise Error("Unable to start Java VM with path %s" % path)

_known_classes = dict()

def getclass(classname):
    try:
        return _known_classes[classname]
    except KeyError:
        # Convert from the 'usual' syntax to the 'JNI' syntax
        jni_classname = classname.replace('.', '/')

        cls = _pyjava.getclass(jni_classname) # might raise ClassNotFound
        cls = _JavaClass(classname, cls)
        _known_classes[classname] = cls
        return cls
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import shutil
import sys
from threading import Lock
from tempfile import NamedTemporaryFile

from pyspark import accumulators
from pyspark.accumulators import Accumulator
from pyspark.broadcast import Broadcast
from pyspark.files import SparkFiles
from pyspark.java_gateway import launch_gateway
from pyspark.serializers import dump_pickle, write_with_length, batched
from pyspark.rdd import RDD

from py4j.java_collections import ListConverter

class SparkContext(object):
    """
    Main entry point for Spark functionality. A SparkContext represents the
    connection to a Spark cluster, and can be used to create L{RDD}s and
    broadcast variables on that cluster.
    """

    _gateway = None
    _jvm = None
    _writeIteratorToPickleFile = None
    _takePartition = None
    _next_accum_id = 0
    _active_spark_context = None
    _lock = Lock()

    def __init__(self, master, jobName, sparkHome=None, pyFiles=None,
        environment=None, batchSize=1024):
        """
        Create a new SparkContext.

        @param master: Cluster URL to connect to
               (e.g. mesos://host:port, spark://host:port, local[4]).
        @param jobName: A name for your job, to display on the cluster web UI
        @param sparkHome: Location where Spark is installed on cluster nodes.
        @param pyFiles: Collection of .zip or .py files to send to the cluster
               and add to PYTHONPATH.  These can be paths on the local file
               system or HDFS, HTTP, HTTPS, or FTP URLs.
        @param environment: A dictionary of environment variables to set on
               worker nodes.
        @param batchSize: The number of Python objects represented as a single
               Java object.  Set 1 to disable batching or -1 to use an
               unlimited batch size.
        """
        with SparkContext._lock:
            if SparkContext._active_spark_context:
                raise ValueError("Cannot run multiple SparkContexts at once")
            else:
                SparkContext._active_spark_context = self
                if not SparkContext._gateway:
                    SparkContext._gateway = launch_gateway()
                    SparkContext._jvm = SparkContext._gateway.jvm
                    SparkContext._writeIteratorToPickleFile = \
                        SparkContext._jvm.PythonRDD.writeIteratorToPickleFile
                    SparkContext._takePartition = \
                        SparkContext._jvm.PythonRDD.takePartition
        self.master = master
        self.jobName = jobName
        self.sparkHome = sparkHome or None # None becomes null in Py4J
        self.environment = environment or {}
        self.batchSize = batchSize  # -1 represents a unlimited batch size

        # Create the Java SparkContext through Py4J
        empty_string_array = self._gateway.new_array(self._jvm.String, 0)
        self._jsc = self._jvm.JavaSparkContext(master, jobName, sparkHome,
                                              empty_string_array)

        # Create a single Accumulator in Java that we'll send all our updates through;
        # they will be passed back to us through a TCP server
        self._accumulatorServer = accumulators._start_update_server()
        (host, port) = self._accumulatorServer.server_address
        self._javaAccumulator = self._jsc.accumulator(
                self._jvm.java.util.ArrayList(),
                self._jvm.PythonAccumulatorParam(host, port))

        self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
        # Broadcast's __reduce__ method stores Broadcast instances here.
        # This allows other code to determine which Broadcast instances have
        # been pickled, so it can determine which Java broadcast objects to
        # send.
        self._pickled_broadcast_vars = set()

        # Deploy any code dependencies specified in the constructor
        for path in (pyFiles or []):
            self.addPyFile(path)
        SparkFiles._sc = self
        sys.path.append(SparkFiles.getRootDirectory())

        # Create a temporary directory inside spark.local.dir:
        local_dir = self._jvm.spark.Utils.getLocalDir()
        self._temp_dir = \
            self._jvm.spark.Utils.createTempDir(local_dir).getAbsolutePath()

    @property
    def defaultParallelism(self):
        """
        Default level of parallelism to use when not given by user (e.g. for
        reduce tasks)
        """
        return self._jsc.sc().defaultParallelism()

    def __del__(self):
        self.stop()

    def stop(self):
        """
        Shut down the SparkContext.
        """
        if self._jsc:
            self._jsc.stop()
            self._jsc = None
        if self._accumulatorServer:
            self._accumulatorServer.shutdown()
            self._accumulatorServer = None
        with SparkContext._lock:
            SparkContext._active_spark_context = None

    def parallelize(self, c, numSlices=None):
        """
        Distribute a local Python collection to form an RDD.
        """
        numSlices = numSlices or self.defaultParallelism
        # Calling the Java parallelize() method with an ArrayList is too slow,
        # because it sends O(n) Py4J commands.  As an alternative, serialized
        # objects are written to a file and loaded through textFile().
        tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir)
        if self.batchSize != 1:
            c = batched(c, self.batchSize)
        for x in c:
            write_with_length(dump_pickle(x), tempFile)
        tempFile.close()
        readRDDFromPickleFile = self._jvm.PythonRDD.readRDDFromPickleFile
        jrdd = readRDDFromPickleFile(self._jsc, tempFile.name, numSlices)
        return RDD(jrdd, self)

    def textFile(self, name, minSplits=None):
        """
        Read a text file from HDFS, a local file system (available on all
        nodes), or any Hadoop-supported file system URI, and return it as an
        RDD of Strings.
        """
        minSplits = minSplits or min(self.defaultParallelism, 2)
        jrdd = self._jsc.textFile(name, minSplits)
        return RDD(jrdd, self)

    def _checkpointFile(self, name):
        jrdd = self._jsc.checkpointFile(name)
        return RDD(jrdd, self)

    def union(self, rdds):
        """
        Build the union of a list of RDDs.
        """
        first = rdds[0]._jrdd
        rest = [x._jrdd for x in rdds[1:]]
        rest = ListConverter().convert(rest, self.gateway._gateway_client)
        return RDD(self._jsc.union(first, rest), self)

    def broadcast(self, value):
        """
        Broadcast a read-only variable to the cluster, returning a C{Broadcast}
        object for reading it in distributed functions. The variable will be
        sent to each cluster only once.
        """
        jbroadcast = self._jsc.broadcast(bytearray(dump_pickle(value)))
        return Broadcast(jbroadcast.id(), value, jbroadcast,
                         self._pickled_broadcast_vars)

    def accumulator(self, value, accum_param=None):
        """
        Create an L{Accumulator} with the given initial value, using a given
        L{AccumulatorParam} helper object to define how to add values of the
        data type if provided. Default AccumulatorParams are used for integers
        and floating-point numbers if you do not provide one. For other types,
        a custom AccumulatorParam can be used.
        """
        if accum_param == None:
            if isinstance(value, int):
                accum_param = accumulators.INT_ACCUMULATOR_PARAM
            elif isinstance(value, float):
                accum_param = accumulators.FLOAT_ACCUMULATOR_PARAM
            elif isinstance(value, complex):
                accum_param = accumulators.COMPLEX_ACCUMULATOR_PARAM
            else:
                raise Exception("No default accumulator param for type %s" % type(value))
        SparkContext._next_accum_id += 1
        return Accumulator(SparkContext._next_accum_id - 1, value, accum_param)

    def addFile(self, path):
        """
        Add a file to be downloaded with this Spark job on every node.
        The C{path} passed can be either a local file, a file in HDFS
        (or other Hadoop-supported filesystems), or an HTTP, HTTPS or
        FTP URI.

        To access the file in Spark jobs, use
        L{SparkFiles.get(path)<pyspark.files.SparkFiles.get>} to find its
        download location.

        >>> from pyspark import SparkFiles
        >>> path = os.path.join(tempdir, "test.txt")
        >>> with open(path, "w") as testFile:
        ...    testFile.write("100")
        >>> sc.addFile(path)
        >>> def func(iterator):
        ...    with open(SparkFiles.get("test.txt")) as testFile:
        ...        fileVal = int(testFile.readline())
        ...        return [x * 100 for x in iterator]
        >>> sc.parallelize([1, 2, 3, 4]).mapPartitions(func).collect()
        [100, 200, 300, 400]
        """
        self._jsc.sc().addFile(path)

    def clearFiles(self):
        """
        Clear the job's list of files added by L{addFile} or L{addPyFile} so
        that they do not get downloaded to any new nodes.
        """
        # TODO: remove added .py or .zip files from the PYTHONPATH?
        self._jsc.sc().clearFiles()

    def addPyFile(self, path):
        """
        Add a .py or .zip dependency for all tasks to be executed on this
        SparkContext in the future.  The C{path} passed can be either a local
        file, a file in HDFS (or other Hadoop-supported filesystems), or an
        HTTP, HTTPS or FTP URI.
        """
        self.addFile(path)
        filename = path.split("/")[-1]

    def setCheckpointDir(self, dirName, useExisting=False):
        """
        Set the directory under which RDDs are going to be checkpointed. The
        directory must be a HDFS path if running on a cluster.

        If the directory does not exist, it will be created. If the directory
        exists and C{useExisting} is set to true, then the exisiting directory
        will be used.  Otherwise an exception will be thrown to prevent
        accidental overriding of checkpoint files in the existing directory.
        """
        self._jsc.sc().setCheckpointDir(dirName, useExisting)

def _test():
    import atexit
    import doctest
    import tempfile
    globs = globals().copy()
    globs['sc'] = SparkContext('local[4]', 'PythonTest', batchSize=2)
    globs['tempdir'] = tempfile.mkdtemp()
    atexit.register(lambda: shutil.rmtree(globs['tempdir']))
    (failure_count, test_count) = doctest.testmod(globs=globs)
    globs['sc'].stop()
    if failure_count:
        exit(-1)

if __name__ == "__main__":
    _test()
<commit_msg>SPARK-815. Python parallelize() should split lists before batching<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import shutil
import sys
from threading import Lock
from tempfile import NamedTemporaryFile

from pyspark import accumulators
from pyspark.accumulators import Accumulator
from pyspark.broadcast import Broadcast
from pyspark.files import SparkFiles
from pyspark.java_gateway import launch_gateway
from pyspark.serializers import dump_pickle, write_with_length, batched
from pyspark.rdd import RDD

from py4j.java_collections import ListConverter

class SparkContext(object):
    """
    Main entry point for Spark functionality. A SparkContext represents the
    connection to a Spark cluster, and can be used to create L{RDD}s and
    broadcast variables on that cluster.
    """

    _gateway = None
    _jvm = None
    _writeIteratorToPickleFile = None
    _takePartition = None
    _next_accum_id = 0
    _active_spark_context = None
    _lock = Lock()

    def __init__(self, master, jobName, sparkHome=None, pyFiles=None,
        environment=None, batchSize=1024):
        """
        Create a new SparkContext.

        @param master: Cluster URL to connect to
               (e.g. mesos://host:port, spark://host:port, local[4]).
        @param jobName: A name for your job, to display on the cluster web UI
        @param sparkHome: Location where Spark is installed on cluster nodes.
        @param pyFiles: Collection of .zip or .py files to send to the cluster
               and add to PYTHONPATH.  These can be paths on the local file
               system or HDFS, HTTP, HTTPS, or FTP URLs.
        @param environment: A dictionary of environment variables to set on
               worker nodes.
        @param batchSize: The number of Python objects represented as a single
               Java object.  Set 1 to disable batching or -1 to use an
               unlimited batch size.
        """
        with SparkContext._lock:
            if SparkContext._active_spark_context:
                raise ValueError("Cannot run multiple SparkContexts at once")
            else:
                SparkContext._active_spark_context = self
                if not SparkContext._gateway:
                    SparkContext._gateway = launch_gateway()
                    SparkContext._jvm = SparkContext._gateway.jvm
                    SparkContext._writeIteratorToPickleFile = \
                        SparkContext._jvm.PythonRDD.writeIteratorToPickleFile
                    SparkContext._takePartition = \
                        SparkContext._jvm.PythonRDD.takePartition
        self.master = master
        self.jobName = jobName
        self.sparkHome = sparkHome or None # None becomes null in Py4J
        self.environment = environment or {}
        self.batchSize = batchSize  # -1 represents a unlimited batch size

        # Create the Java SparkContext through Py4J
        empty_string_array = self._gateway.new_array(self._jvm.String, 0)
        self._jsc = self._jvm.JavaSparkContext(master, jobName, sparkHome,
                                              empty_string_array)

        # Create a single Accumulator in Java that we'll send all our updates through;
        # they will be passed back to us through a TCP server
        self._accumulatorServer = accumulators._start_update_server()
        (host, port) = self._accumulatorServer.server_address
        self._javaAccumulator = self._jsc.accumulator(
                self._jvm.java.util.ArrayList(),
                self._jvm.PythonAccumulatorParam(host, port))

        self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
        # Broadcast's __reduce__ method stores Broadcast instances here.
        # This allows other code to determine which Broadcast instances have
        # been pickled, so it can determine which Java broadcast objects to
        # send.
        self._pickled_broadcast_vars = set()

        # Deploy any code dependencies specified in the constructor
        for path in (pyFiles or []):
            self.addPyFile(path)
        SparkFiles._sc = self
        sys.path.append(SparkFiles.getRootDirectory())

        # Create a temporary directory inside spark.local.dir:
        local_dir = self._jvm.spark.Utils.getLocalDir()
        self._temp_dir = \
            self._jvm.spark.Utils.createTempDir(local_dir).getAbsolutePath()

    @property
    def defaultParallelism(self):
        """
        Default level of parallelism to use when not given by user (e.g. for
        reduce tasks)
        """
        return self._jsc.sc().defaultParallelism()

    def __del__(self):
        self.stop()

    def stop(self):
        """
        Shut down the SparkContext.
        """
        if self._jsc:
            self._jsc.stop()
            self._jsc = None
        if self._accumulatorServer:
            self._accumulatorServer.shutdown()
            self._accumulatorServer = None
        with SparkContext._lock:
            SparkContext._active_spark_context = None

    def parallelize(self, c, numSlices=None):
        """
        Distribute a local Python collection to form an RDD.

        >>> sc.parallelize(range(5), 5).glom().collect()
        [[0], [1], [2], [3], [4]]
        """
        numSlices = numSlices or self.defaultParallelism
        # Calling the Java parallelize() method with an ArrayList is too slow,
        # because it sends O(n) Py4J commands.  As an alternative, serialized
        # objects are written to a file and loaded through textFile().
        tempFile = NamedTemporaryFile(delete=False, dir=self._temp_dir)
        # Make sure we distribute data evenly if it's smaller than self.batchSize
        if "__len__" not in dir(c):
            c = list(c)    # Make it a list so we can compute its length
        batchSize = min(len(c) // numSlices, self.batchSize)
        if batchSize > 1:
            c = batched(c, batchSize)
        for x in c:
            write_with_length(dump_pickle(x), tempFile)
        tempFile.close()
        readRDDFromPickleFile = self._jvm.PythonRDD.readRDDFromPickleFile
        jrdd = readRDDFromPickleFile(self._jsc, tempFile.name, numSlices)
        return RDD(jrdd, self)

    def textFile(self, name, minSplits=None):
        """
        Read a text file from HDFS, a local file system (available on all
        nodes), or any Hadoop-supported file system URI, and return it as an
        RDD of Strings.
        """
        minSplits = minSplits or min(self.defaultParallelism, 2)
        jrdd = self._jsc.textFile(name, minSplits)
        return RDD(jrdd, self)

    def _checkpointFile(self, name):
        jrdd = self._jsc.checkpointFile(name)
        return RDD(jrdd, self)

    def union(self, rdds):
        """
        Build the union of a list of RDDs.
        """
        first = rdds[0]._jrdd
        rest = [x._jrdd for x in rdds[1:]]
        rest = ListConverter().convert(rest, self.gateway._gateway_client)
        return RDD(self._jsc.union(first, rest), self)

    def broadcast(self, value):
        """
        Broadcast a read-only variable to the cluster, returning a C{Broadcast}
        object for reading it in distributed functions. The variable will be
        sent to each cluster only once.
        """
        jbroadcast = self._jsc.broadcast(bytearray(dump_pickle(value)))
        return Broadcast(jbroadcast.id(), value, jbroadcast,
                         self._pickled_broadcast_vars)

    def accumulator(self, value, accum_param=None):
        """
        Create an L{Accumulator} with the given initial value, using a given
        L{AccumulatorParam} helper object to define how to add values of the
        data type if provided. Default AccumulatorParams are used for integers
        and floating-point numbers if you do not provide one. For other types,
        a custom AccumulatorParam can be used.
        """
        if accum_param == None:
            if isinstance(value, int):
                accum_param = accumulators.INT_ACCUMULATOR_PARAM
            elif isinstance(value, float):
                accum_param = accumulators.FLOAT_ACCUMULATOR_PARAM
            elif isinstance(value, complex):
                accum_param = accumulators.COMPLEX_ACCUMULATOR_PARAM
            else:
                raise Exception("No default accumulator param for type %s" % type(value))
        SparkContext._next_accum_id += 1
        return Accumulator(SparkContext._next_accum_id - 1, value, accum_param)

    def addFile(self, path):
        """
        Add a file to be downloaded with this Spark job on every node.
        The C{path} passed can be either a local file, a file in HDFS
        (or other Hadoop-supported filesystems), or an HTTP, HTTPS or
        FTP URI.

        To access the file in Spark jobs, use
        L{SparkFiles.get(path)<pyspark.files.SparkFiles.get>} to find its
        download location.

        >>> from pyspark import SparkFiles
        >>> path = os.path.join(tempdir, "test.txt")
        >>> with open(path, "w") as testFile:
        ...    testFile.write("100")
        >>> sc.addFile(path)
        >>> def func(iterator):
        ...    with open(SparkFiles.get("test.txt")) as testFile:
        ...        fileVal = int(testFile.readline())
        ...        return [x * 100 for x in iterator]
        >>> sc.parallelize([1, 2, 3, 4]).mapPartitions(func).collect()
        [100, 200, 300, 400]
        """
        self._jsc.sc().addFile(path)

    def clearFiles(self):
        """
        Clear the job's list of files added by L{addFile} or L{addPyFile} so
        that they do not get downloaded to any new nodes.
        """
        # TODO: remove added .py or .zip files from the PYTHONPATH?
        self._jsc.sc().clearFiles()

    def addPyFile(self, path):
        """
        Add a .py or .zip dependency for all tasks to be executed on this
        SparkContext in the future.  The C{path} passed can be either a local
        file, a file in HDFS (or other Hadoop-supported filesystems), or an
        HTTP, HTTPS or FTP URI.
        """
        self.addFile(path)
        filename = path.split("/")[-1]

    def setCheckpointDir(self, dirName, useExisting=False):
        """
        Set the directory under which RDDs are going to be checkpointed. The
        directory must be a HDFS path if running on a cluster.

        If the directory does not exist, it will be created. If the directory
        exists and C{useExisting} is set to true, then the exisiting directory
        will be used.  Otherwise an exception will be thrown to prevent
        accidental overriding of checkpoint files in the existing directory.
        """
        self._jsc.sc().setCheckpointDir(dirName, useExisting)

def _test():
    import atexit
    import doctest
    import tempfile
    globs = globals().copy()
    globs['sc'] = SparkContext('local[4]', 'PythonTest', batchSize=2)
    globs['tempdir'] = tempfile.mkdtemp()
    atexit.register(lambda: shutil.rmtree(globs['tempdir']))
    (failure_count, test_count) = doctest.testmod(globs=globs)
    globs['sc'].stop()
    if failure_count:
        exit(-1)

if __name__ == "__main__":
    _test()
<|endoftext|>