<commit_before>import shutil
import tarfile
from abc import ABC, abstractmethod
from enum import Enum
from pathlib import Path
from tempfile import TemporaryFile
from platform import architecture, machine
from sys import platform
from typing import Optional, Any
from urllib.request import urlopen

try:
    from invoke import task, Context
except ImportError:
    # This will happen when doing pip install nassl in an environment that does not have invoke pre-installed
    # We still want this script to be usable directly by pip
    def task(*args, **kwargs):  # type: ignore
        # Return a function - anything will do
        return repr

    Context = Any

# Where we store the src packages of OpenSSL and Zlib
_DEPS_PATH = Path(__file__).parent.absolute() / "deps"

class SupportedPlatformEnum(Enum):
    """Platforms supported by nassl."""

    OSX_64 = 1
    LINUX_64 = 2
    LINUX_32 = 3
    WINDOWS_32 = 4
    WINDOWS_64 = 5
    OPENBSD_64 = 6
    OSX_ARM64 = 7
    LINUX_ARM64 = 8
    LINUX_ARM32 = 9

CURRENT_PLATFORM = None
if architecture()[0] == "64bit":
    if platform == "darwin":
        if machine() == "x86_64":
            CURRENT_PLATFORM = SupportedPlatformEnum.OSX_64
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.OSX_ARM64
    elif platform in ["linux", "linux2"]:
        if machine() == "aarch64":
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_ARM64
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_64
    elif platform == "win32":
        CURRENT_PLATFORM = SupportedPlatformEnum.WINDOWS_64
    elif platform == "openbsd5":
        CURRENT_PLATFORM = SupportedPlatformEnum.OPENBSD_64
elif architecture()[0] == "32bit":
    if platform in ["linux", "linux2"]:
        if machine() == "armv7l":
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_ARM32
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_32
    elif platform == "win32":
        CURRENT_PLATFORM = SupportedPlatformEnum.WINDOWS_32

class BuildConfig(ABC):
    """Base class we use to configure and build Zlib and OpenSSL."""

    def __init__(self, platform: SupportedPlatformEnum) -> None:
        self.platform = platform

    @property
    @abstractmethod
    def src_path(self) -> Path:
        """Where the src package is extracted to before trying to build it."""
        pass

    def clean(self) -> None:
        shutil.rmtree(self.src_path, ignore_errors=True)

    @property
    @abstractmethod
    def src_tar_gz_url(self) -> str:
        """Where to download src package from."""
        pass

    def fetch_source(self) -> None:
        """Download the tar archive that contains the source code for the library."""
        with TemporaryFile() as temp_file:
            # Download the source archive
            with urlopen(self.src_tar_gz_url) as src_tar_gz_response:
                temp_file.write(src_tar_gz_response.read())

            # Rewind the file
            temp_file.seek(0)

            # Extract the content of the archive
            tar_file = tarfile.open(fileobj=temp_file)
            tar_file.extractall(path=_DEPS_PATH)

    @abstractmethod
    def build(self, ctx: Context) -> None:
        pass

    @property
    @abstractmethod
    def include_path(self) -> Path:
        pass

class OpenSslBuildConfig(BuildConfig, ABC):
    @property
    @abstractmethod
    def _openssl_git_tag(self) -> str:
        pass

    @property
    def src_tar_gz_url(self) -> str:
        return f"https://github.com/openssl/openssl/archive/{self._openssl_git_tag}.tar.gz"

    @property
    def src_path(self) -> Path:
        return _DEPS_PATH / f"openssl-{self._openssl_git_tag}"

    @property
    @abstractmethod
    def libcrypto_path(self) -> Path:
        pass

    @property
    @abstractmethod
    def libssl_path(self) -> Path:
        pass

    @property
    @abstractmethod
    def exe_path(self) -> Path:
        pass

    def _get_build_target(self, should_build_for_debug: bool) -> str:
        if self.platform == SupportedPlatformEnum.WINDOWS_32:
            openssl_target = "VC-WIN32"
        elif self.platform == SupportedPlatformEnum.WINDOWS_64:
            openssl_target = "VC-WIN64A"
        elif self.platform == SupportedPlatformEnum.OSX_64:
            openssl_target = "darwin64-x86_64-cc"
        elif self.platform == SupportedPlatformEnum.OSX_ARM64:
            openssl_target = "arm64-x86_64-cc"
        elif self.platform == SupportedPlatformEnum.LINUX_64:
            openssl_target = "linux-x86_64"
        elif self.platform == SupportedPlatformEnum.LINUX_32:
            openssl_target = "linux-elf"
        elif self.platform == SupportedPlatformEnum.LINUX_ARM64:
            openssl_target = "linux-aarch64"
        elif self.platform == SupportedPlatformEnum.LINUX_ARM32:
            openssl_target = "linux-armv4"
        else:
            raise ValueError("Unknown platform")

        if should_build_for_debug:
            if self.platform not in [
                SupportedPlatformEnum.WINDOWS_32,
                SupportedPlatformEnum.WINDOWS_64,
            ]:
                raise ValueError("Debug builds only supported for Windows")
            else:
                openssl_target = f"debug-{openssl_target}"

        return openssl_target

    def build(
        self,
        ctx: Context,
        zlib_lib_path: Optional[Path] = None,
        zlib_include_path: Optional[Path] = None,
        should_build_for_debug: bool = False,
    ) -> None:
        if not zlib_lib_path or not zlib_include_path:
            raise ValueError("Missing argument")

        # Build OpenSSL
        openssl_target = self._get_build_target(should_build_for_debug)
        with ctx.cd(str(self.src_path)):
            self._run_configure_command(ctx, openssl_target, zlib_lib_path, zlib_include_path)
            self._run_build_steps(ctx)

    # To be defined in subclasses
    _OPENSSL_CONF_CMD: str = None

    def _run_configure_command(
        self, ctx: Context, openssl_target: str, zlib_lib_path: Path, zlib_include_path: Path
    ) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            extra_args = "-no-asm -DZLIB_WINAPI"  # *hate* zlib
            # On Windows OpenSSL wants the full path to the lib file
            final_zlib_path = zlib_lib_path
        else:
            extra_args = " -fPIC"
            # On Unix OpenSSL wants the path to the folder where the lib is
            final_zlib_path = zlib_lib_path.parent

        ctx.run(
            self._OPENSSL_CONF_CMD.format(
                target=openssl_target,
                zlib_lib_path=final_zlib_path,
                zlib_include_path=zlib_include_path,
                extra_args=extra_args,
            )
        )

    def _run_build_steps(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            if self.platform == SupportedPlatformEnum.WINDOWS_32:
                ctx.run("ms\\do_ms")
            else:
                ctx.run("ms\\do_win64a.bat")

            ctx.run(
                "nmake -f ms\\nt.mak clean", warn=True
            )  # Does not work if tmp32 does not exist (fresh build)
            ctx.run("nmake -f ms\\nt.mak")

        else:
            ctx.run("make clean", warn=True)
            ctx.run("make")  # Only build the libs as it is faster - not available on Windows

class LegacyOpenSslBuildConfig(OpenSslBuildConfig):
    @property
    def _openssl_git_tag(self) -> str:
        return "OpenSSL_1_0_2e"

    _OPENSSL_CONF_CMD = (
        "perl Configure {target} zlib no-zlib-dynamic no-shared enable-rc5 enable-md2 enable-gost "
        "enable-cast enable-idea enable-ripemd enable-mdc2 --with-zlib-include={zlib_include_path} "
        "--with-zlib-lib={zlib_lib_path} {extra_args}"
    )

    @property
    def include_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "inc32"
        else:
            return self.src_path / "include"

    @property
    def libcrypto_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "libeay32.lib"
        else:
            return self.src_path / "libcrypto.a"

    @property
    def libssl_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "ssleay32.lib"
        else:
            return self.src_path / "libssl.a"

    @property
    def exe_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "openssl.exe"
        else:
            return self.src_path / "apps" / "openssl"

class ModernOpenSslBuildConfig(OpenSslBuildConfig):
    @property
    def _openssl_git_tag(self) -> str:
        return "OpenSSL_1_1_1h"

    _OPENSSL_CONF_CMD = (
        "perl Configure {target} zlib no-zlib-dynamic no-shared enable-rc5 enable-md2 enable-gost "
        "enable-cast enable-idea enable-ripemd enable-mdc2 --with-zlib-include={zlib_include_path} "
        "--with-zlib-lib={zlib_lib_path} enable-weak-ssl-ciphers enable-tls1_3 {extra_args} no-async"
    )

    def _run_build_steps(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            ctx.run("nmake clean", warn=True)
            ctx.run("nmake")
        else:
            return super()._run_build_steps(ctx)

    @property
    def libcrypto_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "libcrypto.lib"
        else:
            return self.src_path / "libcrypto.a"

    @property
    def libssl_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "libssl.lib"
        else:
            return self.src_path / "libssl.a"

    @property
    def include_path(self) -> Path:
        return self.src_path / "include"

    @property
    def exe_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "apps" / "openssl.exe"
        else:
            return self.src_path / "apps" / "openssl"

class ZlibBuildConfig(BuildConfig):
    @property
    def src_tar_gz_url(self) -> str:
        return "https://zlib.net/zlib-1.2.11.tar.gz"

    @property
    def src_path(self) -> Path:
        return _DEPS_PATH / "zlib-1.2.11"

    def build(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            if self.platform == SupportedPlatformEnum.WINDOWS_32:
                arch = "x86"
                build_script = "bld_ml32.bat"
                build_platform = "Win32"
            else:
                arch = "x64"
                build_script = "bld_ml64.bat"
                build_platform = "x64"

            masm_path = self.src_path / "contrib" / f"masm{arch}"
            with ctx.cd(str(masm_path)):
                ctx.run(build_script)
                ctx.run(
                    f"msbuild ..\\vstudio\\vc14\\zlibvc.sln /P:Configuration=Release /P:Platform={build_platform}"
                )

        else:
            # Linux/macOS build
            with ctx.cd(str(self.src_path)):
                ctx.run('CFLAGS="-fPIC" ./configure -static')
                ctx.run("make clean")
                ctx.run("make")

    @property
    def libz_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            arch = "x86" if self.platform == SupportedPlatformEnum.WINDOWS_32 else "x64"
            zlib_lib_path = (
                self.src_path / "contrib" / "vstudio" / "vc14" / arch / "ZlibStatRelease" / "zlibstat.lib"
            )
        else:
            zlib_lib_path = self.src_path / "libz.a"
        return zlib_lib_path

    @property
    def include_path(self) -> Path:
        return self.src_path

@task
def build_zlib(ctx, do_not_clean=False):
    print("ZLIB: Starting...")
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        zlib_cfg.clean()
        zlib_cfg.fetch_source()
    zlib_cfg.build(ctx)
    print("ZLIB: All done")

@task
def build_legacy_openssl(ctx, do_not_clean=False):
    print("OPENSSL LEGACY: Starting...")
    ssl_legacy_cfg = LegacyOpenSslBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        ssl_legacy_cfg.clean()
        ssl_legacy_cfg.fetch_source()
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    ssl_legacy_cfg.build(ctx, zlib_lib_path=zlib_cfg.libz_path, zlib_include_path=zlib_cfg.include_path)
    print("OPENSSL LEGACY: All done")

@task
def build_modern_openssl(ctx, do_not_clean=False):
    print("OPENSSL MODERN: Starting...")
    ssl_modern_cfg = ModernOpenSslBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        ssl_modern_cfg.clean()
        ssl_modern_cfg.fetch_source()
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    ssl_modern_cfg.build(ctx, zlib_lib_path=zlib_cfg.libz_path, zlib_include_path=zlib_cfg.include_path)
    print("OPENSSL MODERN: All done")

@task
def build_nassl(ctx):
    """Build the nassl C extension."""
    extra_args = ""
    if CURRENT_PLATFORM == SupportedPlatformEnum.WINDOWS_32:
        extra_args = "--plat-name=win32"
    elif CURRENT_PLATFORM == SupportedPlatformEnum.WINDOWS_64:
        extra_args = "--plat-name=win-amd64"

    # Reset the ./build folder if there was a previous version of nassl
    build_path = Path(__file__).parent.absolute() / "build"
    if build_path.exists():
        shutil.rmtree(build_path)

    ctx.run(f"python setup.py build_ext -i {extra_args}")

@task
def build_deps(ctx, do_not_clean=False):
    """Build the C libraries the nassl C extension depends on."""
    build_zlib(ctx, do_not_clean)
    build_legacy_openssl(ctx, do_not_clean)
    build_modern_openssl(ctx, do_not_clean)

@task
def build_all(ctx, do_not_clean=False):
    """Build the nassl C extension and the C libraries from scratch."""
    build_deps(ctx, do_not_clean)
    build_nassl(ctx)
<commit_msg>update zlib to 1.2.12<commit_after>import shutil
import tarfile
from abc import ABC, abstractmethod
from enum import Enum
from pathlib import Path
from tempfile import TemporaryFile
from platform import architecture, machine
from sys import platform
from typing import Optional, Any
from urllib.request import urlopen

try:
    from invoke import task, Context
except ImportError:
    # This will happen when doing pip install nassl in an environment that does not have invoke pre-installed
    # We still want this script to be usable directly by pip
    def task(*args, **kwargs):  # type: ignore
        # Return a function - anything will do
        return repr

    Context = Any

# Where we store the src packages of OpenSSL and Zlib
_DEPS_PATH = Path(__file__).parent.absolute() / "deps"

class SupportedPlatformEnum(Enum):
    """Platforms supported by nassl."""

    OSX_64 = 1
    LINUX_64 = 2
    LINUX_32 = 3
    WINDOWS_32 = 4
    WINDOWS_64 = 5
    OPENBSD_64 = 6
    OSX_ARM64 = 7
    LINUX_ARM64 = 8
    LINUX_ARM32 = 9

CURRENT_PLATFORM = None
if architecture()[0] == "64bit":
    if platform == "darwin":
        if machine() == "x86_64":
            CURRENT_PLATFORM = SupportedPlatformEnum.OSX_64
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.OSX_ARM64
    elif platform in ["linux", "linux2"]:
        if machine() == "aarch64":
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_ARM64
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_64
    elif platform == "win32":
        CURRENT_PLATFORM = SupportedPlatformEnum.WINDOWS_64
    elif platform == "openbsd5":
        CURRENT_PLATFORM = SupportedPlatformEnum.OPENBSD_64
elif architecture()[0] == "32bit":
    if platform in ["linux", "linux2"]:
        if machine() == "armv7l":
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_ARM32
        else:
            CURRENT_PLATFORM = SupportedPlatformEnum.LINUX_32
    elif platform == "win32":
        CURRENT_PLATFORM = SupportedPlatformEnum.WINDOWS_32

class BuildConfig(ABC):
    """Base class we use to configure and build Zlib and OpenSSL."""

    def __init__(self, platform: SupportedPlatformEnum) -> None:
        self.platform = platform

    @property
    @abstractmethod
    def src_path(self) -> Path:
        """Where the src package is extracted to before trying to build it."""
        pass

    def clean(self) -> None:
        shutil.rmtree(self.src_path, ignore_errors=True)

    @property
    @abstractmethod
    def src_tar_gz_url(self) -> str:
        """Where to download src package from."""
        pass

    def fetch_source(self) -> None:
        """Download the tar archive that contains the source code for the library."""
        with TemporaryFile() as temp_file:
            # Download the source archive
            with urlopen(self.src_tar_gz_url) as src_tar_gz_response:
                temp_file.write(src_tar_gz_response.read())

            # Rewind the file
            temp_file.seek(0)

            # Extract the content of the archive
            tar_file = tarfile.open(fileobj=temp_file)
            tar_file.extractall(path=_DEPS_PATH)

    @abstractmethod
    def build(self, ctx: Context) -> None:
        pass

    @property
    @abstractmethod
    def include_path(self) -> Path:
        pass

class OpenSslBuildConfig(BuildConfig, ABC):
    @property
    @abstractmethod
    def _openssl_git_tag(self) -> str:
        pass

    @property
    def src_tar_gz_url(self) -> str:
        return f"https://github.com/openssl/openssl/archive/{self._openssl_git_tag}.tar.gz"

    @property
    def src_path(self) -> Path:
        return _DEPS_PATH / f"openssl-{self._openssl_git_tag}"

    @property
    @abstractmethod
    def libcrypto_path(self) -> Path:
        pass

    @property
    @abstractmethod
    def libssl_path(self) -> Path:
        pass

    @property
    @abstractmethod
    def exe_path(self) -> Path:
        pass

    def _get_build_target(self, should_build_for_debug: bool) -> str:
        if self.platform == SupportedPlatformEnum.WINDOWS_32:
            openssl_target = "VC-WIN32"
        elif self.platform == SupportedPlatformEnum.WINDOWS_64:
            openssl_target = "VC-WIN64A"
        elif self.platform == SupportedPlatformEnum.OSX_64:
            openssl_target = "darwin64-x86_64-cc"
        elif self.platform == SupportedPlatformEnum.OSX_ARM64:
            openssl_target = "arm64-x86_64-cc"
        elif self.platform == SupportedPlatformEnum.LINUX_64:
            openssl_target = "linux-x86_64"
        elif self.platform == SupportedPlatformEnum.LINUX_32:
            openssl_target = "linux-elf"
        elif self.platform == SupportedPlatformEnum.LINUX_ARM64:
            openssl_target = "linux-aarch64"
        elif self.platform == SupportedPlatformEnum.LINUX_ARM32:
            openssl_target = "linux-armv4"
        else:
            raise ValueError("Unknown platform")

        if should_build_for_debug:
            if self.platform not in [
                SupportedPlatformEnum.WINDOWS_32,
                SupportedPlatformEnum.WINDOWS_64,
            ]:
                raise ValueError("Debug builds only supported for Windows")
            else:
                openssl_target = f"debug-{openssl_target}"

        return openssl_target

    def build(
        self,
        ctx: Context,
        zlib_lib_path: Optional[Path] = None,
        zlib_include_path: Optional[Path] = None,
        should_build_for_debug: bool = False,
    ) -> None:
        if not zlib_lib_path or not zlib_include_path:
            raise ValueError("Missing argument")

        # Build OpenSSL
        openssl_target = self._get_build_target(should_build_for_debug)
        with ctx.cd(str(self.src_path)):
            self._run_configure_command(ctx, openssl_target, zlib_lib_path, zlib_include_path)
            self._run_build_steps(ctx)

    # To be defined in subclasses
    _OPENSSL_CONF_CMD: str = None

    def _run_configure_command(
        self, ctx: Context, openssl_target: str, zlib_lib_path: Path, zlib_include_path: Path
    ) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            extra_args = "-no-asm -DZLIB_WINAPI"  # *hate* zlib
            # On Windows OpenSSL wants the full path to the lib file
            final_zlib_path = zlib_lib_path
        else:
            extra_args = " -fPIC"
            # On Unix OpenSSL wants the path to the folder where the lib is
            final_zlib_path = zlib_lib_path.parent

        ctx.run(
            self._OPENSSL_CONF_CMD.format(
                target=openssl_target,
                zlib_lib_path=final_zlib_path,
                zlib_include_path=zlib_include_path,
                extra_args=extra_args,
            )
        )

    def _run_build_steps(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            if self.platform == SupportedPlatformEnum.WINDOWS_32:
                ctx.run("ms\\do_ms")
            else:
                ctx.run("ms\\do_win64a.bat")

            ctx.run(
                "nmake -f ms\\nt.mak clean", warn=True
            )  # Does not work if tmp32 does not exist (fresh build)
            ctx.run("nmake -f ms\\nt.mak")

        else:
            ctx.run("make clean", warn=True)
            ctx.run("make")  # Only build the libs as it is faster - not available on Windows

class LegacyOpenSslBuildConfig(OpenSslBuildConfig):
    @property
    def _openssl_git_tag(self) -> str:
        return "OpenSSL_1_0_2e"

    _OPENSSL_CONF_CMD = (
        "perl Configure {target} zlib no-zlib-dynamic no-shared enable-rc5 enable-md2 enable-gost "
        "enable-cast enable-idea enable-ripemd enable-mdc2 --with-zlib-include={zlib_include_path} "
        "--with-zlib-lib={zlib_lib_path} {extra_args}"
    )

    @property
    def include_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "inc32"
        else:
            return self.src_path / "include"

    @property
    def libcrypto_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "libeay32.lib"
        else:
            return self.src_path / "libcrypto.a"

    @property
    def libssl_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "ssleay32.lib"
        else:
            return self.src_path / "libssl.a"

    @property
    def exe_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "out32" / "openssl.exe"
        else:
            return self.src_path / "apps" / "openssl"

class ModernOpenSslBuildConfig(OpenSslBuildConfig):
    @property
    def _openssl_git_tag(self) -> str:
        return "OpenSSL_1_1_1h"

    _OPENSSL_CONF_CMD = (
        "perl Configure {target} zlib no-zlib-dynamic no-shared enable-rc5 enable-md2 enable-gost "
        "enable-cast enable-idea enable-ripemd enable-mdc2 --with-zlib-include={zlib_include_path} "
        "--with-zlib-lib={zlib_lib_path} enable-weak-ssl-ciphers enable-tls1_3 {extra_args} no-async"
    )

    def _run_build_steps(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            ctx.run("nmake clean", warn=True)
            ctx.run("nmake")
        else:
            return super()._run_build_steps(ctx)

    @property
    def libcrypto_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "libcrypto.lib"
        else:
            return self.src_path / "libcrypto.a"

    @property
    def libssl_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "libssl.lib"
        else:
            return self.src_path / "libssl.a"

    @property
    def include_path(self) -> Path:
        return self.src_path / "include"

    @property
    def exe_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            return self.src_path / "apps" / "openssl.exe"
        else:
            return self.src_path / "apps" / "openssl"

class ZlibBuildConfig(BuildConfig):
    @property
    def src_tar_gz_url(self) -> str:
        return "https://zlib.net/zlib-1.2.12.tar.gz"

    @property
    def src_path(self) -> Path:
        return _DEPS_PATH / "zlib-1.2.12"

    def build(self, ctx: Context) -> None:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            if self.platform == SupportedPlatformEnum.WINDOWS_32:
                arch = "x86"
                build_script = "bld_ml32.bat"
                build_platform = "Win32"
            else:
                arch = "x64"
                build_script = "bld_ml64.bat"
                build_platform = "x64"

            masm_path = self.src_path / "contrib" / f"masm{arch}"
            with ctx.cd(str(masm_path)):
                ctx.run(build_script)
                ctx.run(
                    f"msbuild ..\\vstudio\\vc14\\zlibvc.sln /P:Configuration=Release /P:Platform={build_platform}"
                )

        else:
            # Linux/macOS build
            with ctx.cd(str(self.src_path)):
                ctx.run('CFLAGS="-fPIC" ./configure -static')
                ctx.run("make clean")
                ctx.run("make")

    @property
    def libz_path(self) -> Path:
        if self.platform in [SupportedPlatformEnum.WINDOWS_32, SupportedPlatformEnum.WINDOWS_64]:
            arch = "x86" if self.platform == SupportedPlatformEnum.WINDOWS_32 else "x64"
            zlib_lib_path = (
                self.src_path / "contrib" / "vstudio" / "vc14" / arch / "ZlibStatRelease" / "zlibstat.lib"
            )
        else:
            zlib_lib_path = self.src_path / "libz.a"
        return zlib_lib_path

    @property
    def include_path(self) -> Path:
        return self.src_path

@task
def build_zlib(ctx, do_not_clean=False):
    print("ZLIB: Starting...")
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        zlib_cfg.clean()
        zlib_cfg.fetch_source()
    zlib_cfg.build(ctx)
    print("ZLIB: All done")

@task
def build_legacy_openssl(ctx, do_not_clean=False):
    print("OPENSSL LEGACY: Starting...")
    ssl_legacy_cfg = LegacyOpenSslBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        ssl_legacy_cfg.clean()
        ssl_legacy_cfg.fetch_source()
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    ssl_legacy_cfg.build(ctx, zlib_lib_path=zlib_cfg.libz_path, zlib_include_path=zlib_cfg.include_path)
    print("OPENSSL LEGACY: All done")

@task
def build_modern_openssl(ctx, do_not_clean=False):
    print("OPENSSL MODERN: Starting...")
    ssl_modern_cfg = ModernOpenSslBuildConfig(CURRENT_PLATFORM)
    if not do_not_clean:
        ssl_modern_cfg.clean()
        ssl_modern_cfg.fetch_source()
    zlib_cfg = ZlibBuildConfig(CURRENT_PLATFORM)
    ssl_modern_cfg.build(ctx, zlib_lib_path=zlib_cfg.libz_path, zlib_include_path=zlib_cfg.include_path)
    print("OPENSSL MODERN: All done")

@task
def build_nassl(ctx):
    """Build the nassl C extension."""
    extra_args = ""
    if CURRENT_PLATFORM == SupportedPlatformEnum.WINDOWS_32:
        extra_args = "--plat-name=win32"
    elif CURRENT_PLATFORM == SupportedPlatformEnum.WINDOWS_64:
        extra_args = "--plat-name=win-amd64"

    # Reset the ./build folder if there was a previous version of nassl
    build_path = Path(__file__).parent.absolute() / "build"
    if build_path.exists():
        shutil.rmtree(build_path)

    ctx.run(f"python setup.py build_ext -i {extra_args}")

@task
def build_deps(ctx, do_not_clean=False):
    """Build the C libraries the nassl C extension depends on."""
    build_zlib(ctx, do_not_clean)
    build_legacy_openssl(ctx, do_not_clean)
    build_modern_openssl(ctx, do_not_clean)

@task
def build_all(ctx, do_not_clean=False):
    """Build the nassl C extension and the C libraries from scratch."""
    build_deps(ctx, do_not_clean)
    build_nassl(ctx)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from setuptools import setup, find_packages

setup(name="opencivicdata-django",
      version='0.7.4',
      author="James Turk",
      author_email='james.p.turk@gmail.com',
      license="BSD",
      description="python opencivicdata library",
      long_description="",
      url="",
      py_modules=['opencivicdata.apps', 'opencivicdata.common'],
      packages=['opencivicdata.admin',
                'opencivicdata.management',
                'opencivicdata.management.commands',
                'opencivicdata.migrations',
                'opencivicdata.models',
                'opencivicdata.tests'],
      install_requires=[
          'Django>=1.9b1',
          'psycopg2',
          'opencivicdata-divisions',
      ],
      platforms=["any"],
      classifiers=["Development Status :: 4 - Beta",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: BSD License",
                   "Natural Language :: English",
                   "Operating System :: OS Independent",
                   "Programming Language :: Python :: 3.4",
                   ],
      )
<commit_msg>bump to 0.8<commit_after>#!/usr/bin/env python
from setuptools import setup, find_packages

setup(name="opencivicdata-django",
      version='0.8.0',
      author="James Turk",
      author_email='james.p.turk@gmail.com',
      license="BSD",
      description="python opencivicdata library",
      long_description="",
      url="",
      py_modules=['opencivicdata.apps', 'opencivicdata.common'],
      packages=['opencivicdata.admin',
                'opencivicdata.management',
                'opencivicdata.management.commands',
                'opencivicdata.migrations',
                'opencivicdata.models',
                'opencivicdata.tests'],
      install_requires=[
          'Django>=1.9b1',
          'psycopg2',
          'opencivicdata-divisions',
      ],
      platforms=["any"],
      classifiers=["Development Status :: 4 - Beta",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: BSD License",
                   "Natural Language :: English",
                   "Operating System :: OS Independent",
                   "Programming Language :: Python :: 3.4",
                   ],
      )
<|endoftext|>


<commit_before>from calendar import mdays
from decimal import Decimal

from publican.engine.kit import Date, cents, quarters_range, zero

_arbitrary = Decimal('0.1195')
_point104 = Decimal('.104')
_point029 = Decimal('.029')

title = u"Employer's QUARTERLY Federal Tax Return"
filename = 'f941--2012.pdf'

def periods(company):
    december = Date(company.today.year, 12, 1)
    return list(quarters_range(company.incorporation_date, december))

def tally(company, period, filing):

    tlist = list(company.transactions(
        within=period,
        debit_type='business',
        credit_type='employee',
        ))
    number_of_employees = len(set(t.credit_account.id for t in tlist))
    wages = sum(t.amount for t in tlist) or zero  # "or zero" keeps it Decimal

    p = filing.new_page(1)

    p.ein = company.ein
    p.name = company.name

    p.quarter1 = 'X' if period.number == 1 else ''
    p.quarter2 = 'X' if period.number == 2 else ''
    p.quarter3 = 'X' if period.number == 3 else ''
    p.quarter4 = 'X' if period.number == 4 else ''

    p.line1 = number_of_employees
    p.line2 = wages
    p.line3 = cents(wages * _arbitrary)           # TODO

    p.line5a1 = wages                             # TODO
    p.line5b1 = zero                              # TODO
    p.line5c1 = wages                             # TODO

    p.line5a2 = cents(p.line5a1 * _point104)
    p.line5b2 = cents(p.line5b1 * _point104)
    p.line5c2 = cents(p.line5c1 * _point029)

    p.line5d = p.line5a2 + p.line5b2 + p.line5c2
    p.line5e = zero

    p.line6e = p.line3 + p.line5d + p.line5e

    p.line10 = p.line6e

    p.line14 = p.line10

    # TODO: page 2

    filing.balance_due = p.line14

    y = period.end.year
    m = period.end.month
    y, m = (y + 1, 1) if m == 12 else (y, m + 1)
    filing.due_date = Date(y, m, mdays[m]).next_business_day()

    return filing

grids = {
 1: """
  ein-
  name-
  x x line1
  x x line2
  x x line3
  line5a1 line5a2
  line5b1 line5b2
  line5c1 line5c2
  x x line5d
  x x line5e
  x x line6e
  x x line10
  x x line14
  """,
 }

pdf_fields = [
 ]
<commit_msg>The app can now fill in a rudiementary 941 PDF<commit_after>from calendar import mdays
from decimal import Decimal

from publican.engine.kit import Date, cents, quarters_range, zero

_arbitrary = Decimal('0.1195')
_point104 = Decimal('.104')
_point029 = Decimal('.029')

title = u"Employer's QUARTERLY Federal Tax Return"
filename = 'f941--2012.pdf'

def periods(company):
    december = Date(company.today.year, 12, 1)
    return list(quarters_range(company.incorporation_date, december))

def tally(company, period, filing):

    tlist = list(company.transactions(
        within=period,
        debit_type='business',
        credit_type='employee',
        ))
    number_of_employees = len(set(t.credit_account.id for t in tlist))
    wages = sum(t.amount for t in tlist) or zero  # "or zero" keeps it Decimal

    p = filing.new_page(1)

    p.ein = company.ein
    p.name = company.name

    p.quarter1 = 'X' if period.number == 1 else ''
    p.quarter2 = 'X' if period.number == 2 else ''
    p.quarter3 = 'X' if period.number == 3 else ''
    p.quarter4 = 'X' if period.number == 4 else ''

    p.line1 = number_of_employees
    p.line2 = wages
    p.line3 = cents(wages * _arbitrary)           # TODO

    p.line5a1 = wages                             # TODO
    p.line5b1 = zero                              # TODO
    p.line5c1 = wages                             # TODO

    p.line5a2 = cents(p.line5a1 * _point104)
    p.line5b2 = cents(p.line5b1 * _point104)
    p.line5c2 = cents(p.line5c1 * _point029)

    p.line5d = p.line5a2 + p.line5b2 + p.line5c2
    p.line5e = zero

    p.line6 = p.line3 + p.line5d + p.line5e

    p.line10 = p.line6

    p.line14 = p.line10

    # TODO: page 2

    filing.balance_due = p.line14

    y = period.end.year
    m = period.end.month
    y, m = (y + 1, 1) if m == 12 else (y, m + 1)
    filing.due_date = Date(y, m, mdays[m]).next_business_day()

    return filing

grids = {
 1: """
  ein-
  name-
  x x line1
  x x line2
  x x line3
  line5a1 line5a2
  line5b1 line5b2
  line5c1 line5c2
  x x line5d
  x x line5e
  x x line6
  x x line10
  x x line14
  """,
 }

stride = 18

pdf_fields = [
 (140,688, 'name'),
 (158,712, 'ein', 24),
 (460,552, 'line1'),
 (460,552 - stride, 'line2'),
 (460,552 - 2 * stride, 'line3'),
 (212,462, 'line5a1'), (350,462, 'line5a2'),
 (212,462 - stride, 'line5b1'), (350,462 - stride, 'line5b2'),
 (212,462 - 2 * stride, 'line5c1'), (350,462 - 2 * stride, 'line5c2'),
 (460,402, 'line5d'),
 (460,402 - stride, 'line5e'),
 (460,402 - 2 * stride, 'line6'),
 (460,402 - 6 * stride, 'line10'),
 (460,192, 'line14'),
 ]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
__author__ = 'nikolojedison'
#This code was written to test some things and may end up being used
#on the robot, but I dunno. So far it seems to work fine. Change what
#needs to be changed. Will evolve as time goes on.
#-Nik Mal

from networktables import NetworkTable
import wpilib
import logging
from autonomous_utilities import Auto
from drive_control import *

from subsystems.camera import Camera
from subsystems.derailer import Derailer
from subsystems.drivetrain import Drivetrain
from subsystems.lift import Lift
from subsystems.grabber import Grabber
from subsystems.mast import Mast

class Lopez_Jr(wpilib.SampleRobot):
    def robotInit(self):
        """initialises robot as a mecanum drive bot w/ 2 joysticks and a camera"""

        self.camera = Camera(self)
        self.derailer = Derailer(self)
        self.drivetrain = Drivetrain(self)
        self.lift = Lift(self)
        self.grabber = Grabber(self)
        self.mast = Mast(self)
        self.oi = OI(self)

        self.autonomousCommand = Autonomous(self)

    def autonomous(self):
        """Woo, auton code. Needs to be tested."""

        self.autonomousCommand.start()

        while self.isAutonomous() and self.isEnabled():
            Scheduler.getInstance().run()
            wpilib.Timer.delay(.005)    # don't burn up the cpu

    def operatorControl(self):
        """Runs the drive with mecanum steering. Other motors added as needed."""
        self.drive.setSafetyEnabled(True)
        wpilib.Timer.delay(.005)    # don't burn up the cpu

    def disabled(self):
        pass

    def test(self):
        """no tests yet, woo"""
        pass

    def log(self):
        self.drivetrain.log

if __name__ == "__main__":
    wpilib.run(Lopez_Jr)
<commit_msg>Fixed a derp.<commit_after>#!/usr/bin/env python3
__author__ = 'nikolojedison'
#This code was written to test some things and may end up being used
#on the robot, but I dunno. So far it seems to work fine. Change what
#needs to be changed. Will evolve as time goes on.
#-Nik Mal

from networktables import NetworkTable
import wpilib
import logging
from autonomous_utilities import Auto
from drive_control import *

from subsystems.camera import Camera
from subsystems.derailer import Derailer
from subsystems.drivetrain import Drivetrain
from subsystems.lift import Lift
from subsystems.grabber import Grabber
from subsystems.mast import Mast

class Lopez_Jr(wpilib.SampleRobot):
    def robotInit(self):
        """initialises robot as a mecanum drive bot w/ 2 joysticks and a camera"""

        self.camera = Camera(self)
        self.derailer = Derailer(self)
        self.drivetrain = Drivetrain(self)
        self.lift = Lift(self)
        self.grabber = Grabber(self)
        self.mast = Mast(self)
        self.oi = OI(self)

        self.autonomousCommand = Autonomous(self)

    def autonomous(self):
        """Woo, auton code. Needs to be tested."""

        self.autonomousCommand.start()

        while self.isAutonomous() and self.isEnabled():
            Scheduler.getInstance().run()
            wpilib.Timer.delay(.005)    # don't burn up the cpu

    def operatorControl(self):
        """Runs the drive with mecanum steering. Other motors added as needed."""
        self.drivetrain.drive.setSafetyEnabled(True)
        while self.isAutonomous() and self.isEnabled():
            Scheduler.getInstance().run()
            wpilib.Timer.delay(.005)    # don't burn up the cpu

    def disabled(self):
        pass

    def test(self):
        """no tests yet, woo"""
        pass

    def log(self):
        self.drivetrain.log

if __name__ == "__main__":
    wpilib.run(Lopez_Jr)
<|endoftext|>


<commit_before># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 3 clause
from __future__ import print_function

import numpy as np
import numba

from umap.utils import (
    tau_rand,
    make_heap,
    heap_push,
    unchecked_heap_push,
    smallest_flagged,
    rejection_sample,
    build_candidates,
    new_build_candidates,
    deheap_sort,
)

from umap.rp_tree import search_flat_tree

# def make_nn_descent(dist, dist_args):
#     """Create a numba accelerated version of nearest neighbor descent
#     specialised for the given distance metric and metric arguments. Numba
#     doesn't support higher order functions directly, but we can instead JIT
#     compile the version of NN-descent for any given metric.
#
#     Parameters
#     ----------
#     dist: function
#         A numba JITd distance function which, given two arrays computes a
#         dissimilarity between them.
#
#     dist_args: tuple
#         Any extra arguments that need to be passed to the distance function
#         beyond the two arrays to be compared.
#
#     Returns
#     -------
#     A numba JITd function for nearest neighbor descent computation that is
#     specialised to the given metric.
#     """

@numba.njit(parallel=True)
def nn_descent(
        data,
        n_neighbors,
        rng_state,
        dist,
        dist_args=(),
        max_candidates=50,
        n_iters=10,
        delta=0.001,
        rho=0.5,
        rp_tree_init=True,
        leaf_array=None,
        verbose=False,
):
    n_vertices = data.shape[0]

    current_graph = make_heap(data.shape[0], n_neighbors)
    for i in range(data.shape[0]):
        indices = rejection_sample(n_neighbors, data.shape[0], rng_state)
        for j in range(indices.shape[0]):
            d = dist(data[i], data[indices[j]], *dist_args)
            heap_push(current_graph, i, d, indices[j], 1)
            heap_push(current_graph, indices[j], d, i, 1)

    if rp_tree_init:
        for n in range(leaf_array.shape[0]):
            for i in range(leaf_array.shape[1]):
                if leaf_array[n, i] < 0:
                    break
                for j in range(i + 1, leaf_array.shape[1]):
                    if leaf_array[n, j] < 0:
                        break
                    d = dist(
                        data[leaf_array[n, i]], data[leaf_array[n, j]], *dist_args
                    )
                    heap_push(
                        current_graph, leaf_array[n, i], d, leaf_array[n, j], 1
                    )
                    heap_push(
                        current_graph, leaf_array[n, j], d, leaf_array[n, i], 1
                    )

    # for n in range(n_iters):
    #     if verbose:
    #         print("\t", n, " / ", n_iters)
    #
    #     candidate_neighbors = build_candidates(
    #         current_graph, n_vertices, n_neighbors, max_candidates, rng_state
    #     )
    #
    #     c = 0
    #     for i in range(n_vertices):
    #         for j in range(max_candidates):
    #             p = int(candidate_neighbors[0, i, j])
    #             if p < 0 or tau_rand(rng_state) < rho:
    #                 continue
    #             for k in range(max_candidates):
    #                 q = int(candidate_neighbors[0, i, k])
    #                 if (
    #                     q < 0
    #                     or not candidate_neighbors[2, i, j]
    #                     and not candidate_neighbors[2, i, k]
    #                 ):
    #                     continue
    #
    #                 d = dist(data[p], data[q], *dist_args)
    #                 c += heap_push(current_graph, p, d, q, 1)
    #                 c += heap_push(current_graph, q, d, p, 1)
    #
    #     if c <= delta * n_neighbors * data.shape[0]:
    #         break
    for n in range(n_iters):

        (new_candidate_neighbors,
         old_candidate_neighbors) = new_build_candidates(current_graph,
                                                         n_vertices,
                                                         n_neighbors,
                                                         max_candidates,
                                                         rng_state, rho)

        c = 0
        for i in range(n_vertices):
            for j in range(max_candidates):
                p = int(new_candidate_neighbors[0, i, j])
                if p < 0:
                    continue
                for k in range(j, max_candidates):
                    q = int(new_candidate_neighbors[0, i, k])
                    if q < 0:
                        continue

                    d = dist(data[p], data[q], *dist_args)
                    c += heap_push(current_graph, p, d, q, 1)
                    c += heap_push(current_graph, q, d, p, 1)

                for k in range(max_candidates):
                    q = int(old_candidate_neighbors[0, i, k])
                    if q < 0:
                        continue

                    d = dist(data[p], data[q], *dist_args)
                    c += heap_push(current_graph, p, d, q, 1)
                    c += heap_push(current_graph, q, d, p, 1)

        if c <= delta * n_neighbors * data.shape[0]:
            break

    return deheap_sort(current_graph)

    # return nn_descent

# def make_initialisations(dist, dist_args):
@numba.njit(parallel=True)
def init_from_random(n_neighbors, data, query_points, heap, rng_state, dist, dist_args):
    for i in range(query_points.shape[0]):
        indices = rejection_sample(n_neighbors, data.shape[0], rng_state)
        for j in range(indices.shape[0]):
            if indices[j] < 0:
                continue
            d = dist(data[indices[j]], query_points[i], *dist_args)
            heap_push(heap, i, d, indices[j], 1)
    return

@numba.njit(parallel=True)
def init_from_tree(tree, data, query_points, heap, rng_state, dist, dist_args):
    for i in range(query_points.shape[0]):
        indices = search_flat_tree(
            query_points[i],
            tree.hyperplanes,
            tree.offsets,
            tree.children,
            tree.indices,
            rng_state,
        )

        for j in range(indices.shape[0]):
            if indices[j] < 0:
                continue
            d = dist(data[indices[j]], query_points[i], *dist_args)
            heap_push(heap, i, d, indices[j], 1)

    return

    # return init_from_random, init_from_tree

def initialise_search(
        forest, data, query_points, n_neighbors, rng_state, dist, dist_args
):
    results = make_heap(query_points.shape[0], n_neighbors)
    init_from_random(n_neighbors, data, query_points, results, rng_state, dist, dist_args)
    if forest is not None:
        for tree in forest:
            init_from_tree(tree, data, query_points, results, rng_state, dist, dist_args)

    return results

# def make_initialized_nnd_search(dist, dist_args):
@numba.njit(parallel=True)
def initialized_nnd_search(data, indptr, indices, initialization, query_points, dist, dist_args):
    for i in numba.prange(query_points.shape[0]):

        tried = set(initialization[0, i])

        while True:

            # Find smallest flagged vertex
            vertex = smallest_flagged(initialization, i)

            if vertex == -1:
                break
            candidates = indices[indptr[vertex]: indptr[vertex + 1]]
            for j in range(candidates.shape[0]):
                if (
                        candidates[j] == vertex
                        or candidates[j] == -1
                        or candidates[j] in tried
                ):
                    continue
                d = dist(data[candidates[j]], query_points[i], *dist_args)
                unchecked_heap_push(initialization, i, d, candidates[j], 1)
                tried.add(candidates[j])

    return initialization

    # return initialized_nnd_search
<commit_msg>Can't use parallel?<commit_after># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 3 clause
from __future__ import print_function

import numpy as np
import numba

from umap.utils import (
    tau_rand,
    make_heap,
    heap_push,
    unchecked_heap_push,
    smallest_flagged,
    rejection_sample,
    build_candidates,
    new_build_candidates,
    deheap_sort,
)

from umap.rp_tree import search_flat_tree

# def make_nn_descent(dist, dist_args):
#     """Create a numba accelerated version of nearest neighbor descent
#     specialised for the given distance metric and metric arguments. Numba
#     doesn't support higher order functions directly, but we can instead JIT
#     compile the version of NN-descent for any given metric.
#
#     Parameters
#     ----------
#     dist: function
#         A numba JITd distance function which, given two arrays computes a
#         dissimilarity between them.
#
#     dist_args: tuple
#         Any extra arguments that need to be passed to the distance function
#         beyond the two arrays to be compared.
#
#     Returns
#     -------
#     A numba JITd function for nearest neighbor descent computation that is
#     specialised to the given metric.
#     """

@numba.njit()
def nn_descent(
        data,
        n_neighbors,
        rng_state,
        dist,
        dist_args=(),
        max_candidates=50,
        n_iters=10,
        delta=0.001,
        rho=0.5,
        rp_tree_init=True,
        leaf_array=None,
        verbose=False,
):
    n_vertices = data.shape[0]

    current_graph = make_heap(data.shape[0], n_neighbors)
    for i in range(data.shape[0]):
        indices = rejection_sample(n_neighbors, data.shape[0], rng_state)
        for j in range(indices.shape[0]):
            d = dist(data[i], data[indices[j]], *dist_args)
            heap_push(current_graph, i, d, indices[j], 1)
            heap_push(current_graph, indices[j], d, i, 1)

    if rp_tree_init:
        for n in range(leaf_array.shape[0]):
            for i in range(leaf_array.shape[1]):
                if leaf_array[n, i] < 0:
                    break
                for j in range(i + 1, leaf_array.shape[1]):
                    if leaf_array[n, j] < 0:
                        break
                    d = dist(
                        data[leaf_array[n, i]], data[leaf_array[n, j]], *dist_args
                    )
                    heap_push(
                        current_graph, leaf_array[n, i], d, leaf_array[n, j], 1
                    )
                    heap_push(
                        current_graph, leaf_array[n, j], d, leaf_array[n, i], 1
                    )

    # for n in range(n_iters):
    #     if verbose:
    #         print("\t", n, " / ", n_iters)
    #
    #     candidate_neighbors = build_candidates(
    #         current_graph, n_vertices, n_neighbors, max_candidates, rng_state
    #     )
    #
    #     c = 0
    #     for i in range(n_vertices):
    #         for j in range(max_candidates):
    #             p = int(candidate_neighbors[0, i, j])
    #             if p < 0 or tau_rand(rng_state) < rho:
    #                 continue
    #             for k in range(max_candidates):
    #                 q = int(candidate_neighbors[0, i, k])
    #                 if (
    #                     q < 0
    #                     or not candidate_neighbors[2, i, j]
    #                     and not candidate_neighbors[2, i, k]
    #                 ):
    #                     continue
    #
    #                 d = dist(data[p], data[q], *dist_args)
    #                 c += heap_push(current_graph, p, d, q, 1)
    #                 c += heap_push(current_graph, q, d, p, 1)
    #
    #     if c <= delta * n_neighbors * data.shape[0]:
    #         break
    for n in range(n_iters):

        (new_candidate_neighbors,
         old_candidate_neighbors) = new_build_candidates(current_graph,
                                                         n_vertices,
                                                         n_neighbors,
                                                         max_candidates,
                                                         rng_state, rho)

        c = 0
        for i in range(n_vertices):
            for j in range(max_candidates):
                p = int(new_candidate_neighbors[0, i, j])
                if p < 0:
                    continue
                for k in range(j, max_candidates):
                    q = int(new_candidate_neighbors[0, i, k])
                    if q < 0:
                        continue

                    d = dist(data[p], data[q], *dist_args)
                    c += heap_push(current_graph, p, d, q, 1)
                    c += heap_push(current_graph, q, d, p, 1)

                for k in range(max_candidates):
                    q = int(old_candidate_neighbors[0, i, k])
                    if q < 0:
                        continue

                    d = dist(data[p], data[q], *dist_args)
                    c += heap_push(current_graph, p, d, q, 1)
                    c += heap_push(current_graph, q, d, p, 1)

        if c <= delta * n_neighbors * data.shape[0]:
            break

    return deheap_sort(current_graph)

    # return nn_descent

# def make_initialisations(dist, dist_args):
@numba.njit(parallel=True)
def init_from_random(n_neighbors, data, query_points, heap, rng_state, dist, dist_args):
    for i in range(query_points.shape[0]):
        indices = rejection_sample(n_neighbors, data.shape[0], rng_state)
        for j in range(indices.shape[0]):
            if indices[j] < 0:
                continue
            d = dist(data[indices[j]], query_points[i], *dist_args)
            heap_push(heap, i, d, indices[j], 1)
    return

@numba.njit(parallel=True)
def init_from_tree(tree, data, query_points, heap, rng_state, dist, dist_args):
    for i in range(query_points.shape[0]):
        indices = search_flat_tree(
            query_points[i],
            tree.hyperplanes,
            tree.offsets,
            tree.children,
            tree.indices,
            rng_state,
        )

        for j in range(indices.shape[0]):
            if indices[j] < 0:
                continue
            d = dist(data[indices[j]], query_points[i], *dist_args)
            heap_push(heap, i, d, indices[j], 1)

    return

    # return init_from_random, init_from_tree

def initialise_search(
        forest, data, query_points, n_neighbors, rng_state, dist, dist_args
):
    results = make_heap(query_points.shape[0], n_neighbors)
    init_from_random(n_neighbors, data, query_points, results, rng_state, dist, dist_args)
    if forest is not None:
        for tree in forest:
            init_from_tree(tree, data, query_points, results, rng_state, dist, dist_args)

    return results

# def make_initialized_nnd_search(dist, dist_args):
@numba.njit(parallel=True)
def initialized_nnd_search(data, indptr, indices, initialization, query_points, dist, dist_args):
    for i in numba.prange(query_points.shape[0]):

        tried = set(initialization[0, i])

        while True:

            # Find smallest flagged vertex
            vertex = smallest_flagged(initialization, i)

            if vertex == -1:
                break
            candidates = indices[indptr[vertex]: indptr[vertex + 1]]
            for j in range(candidates.shape[0]):
                if (
                        candidates[j] == vertex
                        or candidates[j] == -1
                        or candidates[j] in tried
                ):
                    continue
                d = dist(data[candidates[j]], query_points[i], *dist_args)
                unchecked_heap_push(initialization, i, d, candidates[j], 1)
                tried.add(candidates[j])

    return initialization

    # return initialized_nnd_search
<|endoftext|>


<commit_before>#!/usr/bin/python
# coding: utf-8

'''pageit setup file.'''
from setuptools import setup
import os.path
import sys

sys.path.insert(0, '')
import pageit
from pageit.lib import Namespace

def check_scripts(scripts):
    '''Add Windows scripts when needed.'''
    if sys.platform.startswith('win'):
        scripts += [script + '.bat' for script in scripts]
    return scripts

OPTS = Namespace(
    name='pageit',
    packages=['pageit'],
    install_requires=open('requirements.txt').read().strip().split('\n'),
    scripts=check_scripts(['scripts/pageit']),
    entry_points={'console_scripts': ['pageit = pageit.pageit:main']},
    version=pageit.__version__.replace('pre', ''),
    author=pageit.__author__,
    author_email=pageit.__email__,
    url='https://github.com/metaist/pageit',
    download_url='https://github.com/metaist/pageit',
    description=pageit.__doc__.split('\n')[0],
    long_description=pageit.__doc__,
    keywords='static website',
    license=pageit.__license__,
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2',
        'License :: OSI Approved :: MIT License',
        'Topic :: Software Development :: Libraries'
    ]
)

if sys.version_info >= (3,):
    OPTS['use_2to3'] = True

setup(**OPTS)
<commit_msg>cleaned up setup.py<commit_after>#!/usr/bin/python
# coding: utf-8

'''pageit setup file.'''
from setuptools import setup
import os.path
import sys

sys.path.insert(0, '')
import pageit
from pageit.lib import Namespace

def check_scripts(scripts):
    '''Add Windows scripts when needed.'''
    if sys.platform.startswith('win'):
        scripts += [script + '.bat' for script in scripts]
    return scripts

OPTS = Namespace(
    name='pageit',
    description=pageit.pageit.__doc__.split('\n')[0],
    long_description=pageit.pageit.__doc__,

    version=pageit.__version__.replace('pre', ''),
    packages=['pageit'],
    provides=['pageit'],

    install_requires=open('requirements.txt').read().strip().split('\n'),
    scripts=check_scripts(['scripts/pageit']),
    entry_points={'console_scripts': ['pageit = pageit.pageit:main']},

    author=pageit.__author__,
    author_email=pageit.__email__,
    license=pageit.__license__,

    url='https://github.com/metaist/pageit',
    download_url='https://github.com/metaist/pageit',

    keywords='static website generator',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2',
        'License :: OSI Approved :: MIT License',
        'Topic :: Software Development :: Libraries'
    ]
)

if sys.version_info >= (3,):
    OPTS['use_2to3'] = True

setup(**OPTS)
<|endoftext|>


<commit_before>from Lifecyle import Lifecycle
from ButtonLayout import ButtonLayout
from Controller import Controller
from Navigator import Navigator
from Settings import Settings
from App import App
from Samples import *
<commit_msg>fixed a silly mistake...<commit_after>from Lifecycle import Lifecycle
from ButtonLayout import ButtonLayout
from Controller import Controller
from Navigator import Navigator
from Settings import Settings
from App import App
from Samples import *
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Sentry
======

Sentry is a realtime event logging and aggregation platform. It specializes
in monitoring errors and extracting all the information needed to do a proper
post-mortem without any of the hassle of the standard user feedback loop.

Sentry is a Server
------------------

The Sentry package, at its core, is just a simple server and web UI. It will
handle authentication clients (such as `Raven <https://github.com/getsentry/raven-python>`_)
and all of the logic behind storage and aggregation.

That said, Sentry is not limited to Python. The primary implementation is in
Python, but it contains a full API for sending events from any language, in
any application.

:copyright: (c) 2011-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os.path

from distutils import log
from distutils.core import Command
from setuptools.command.develop import develop
from setuptools.command.sdist import sdist
from setuptools import setup, find_packages
from subprocess import check_output

# Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error
# in multiprocessing/util.py _exit_function when running `python
# setup.py test` (see
# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
for m in ('multiprocessing', 'billiard'):
    try:
        __import__(m)
    except ImportError:
        pass

ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__)))

dev_requires = [
    'flake8>=2.0,<2.1',
]

tests_require = [
    'blist',  # used by cassandra
    'casscache',
    'cqlsh',
    'elasticsearch',
    'httpretty',
    'pytest-cov>=1.4',
    'pytest-timeout',
    'python-coveralls',
    'responses',
    'riak',
]

install_requires = [
    'BeautifulSoup>=3.2.1,<3.3.0',
    'celery>=3.1.7,<3.2.0',
    'cssutils>=0.9.9,<0.10.0',
    'Django>=1.6.0,<1.7',
    'django-bitfield>=1.7.0,<1.8.0',
    'django-crispy-forms>=1.4.0,<1.5.0',
    'django-paging>=0.2.5,<0.3.0',
    'django-jsonfield>=0.9.13,<0.10.0',
    'django-picklefield>=0.3.0,<0.4.0',
    'django-recaptcha>=1.0.0,<1.1.0',
    'django-social-auth>=0.7.28,<0.8.0',
    'django-statsd-mozilla>=0.3.14.0,<0.3.15.0',
    'django-sudo>=1.1.3,<1.2.0',
    'django-templatetag-sugar>=0.1.0',
    'djangorestframework>=2.3.8,<2.4.0',
    'email-reply-parser>=0.2.0,<0.3.0',
    'enum34>=0.9.18,<0.10.0',
    'exam>=0.5.1',
    'gunicorn>=19.2.1,<20.0.0',
    'ipaddr>=2.1.11,<2.2.0',
    'logan>=0.5.8.2,<0.6.0',
    'lxml>=3.4.1',
    'mock>=0.8.0',
    'nydus>=0.11.0,<0.12.0',
    'markdown>=2.4.1,<2.5.0',
    'progressbar>=2.2,<2.4',
    'Pygments>=1.6.0,<1.7.0',
    'pytest',
    'pytest-django',
    'python-dateutil>=2.0.0,<3.0.0',
    'python-memcached>=1.53,<2.0.0',
    'raven>=5.0.0',
    'redis>=2.7.0,<2.11.0',
    'requests[security]>=2.5.1,<2.6.0',
    'simplejson>=3.1.0,<3.4.0',
    'six>=1.6.0,<2.0.0',
    'setproctitle>=1.1.7,<1.2.0',
    'statsd>=3.1.0,<3.2.0',
    'South==1.0.1',
    'toronado>=0.0.4,<0.1.0',
    'ua-parser>=0.3.5',
    'urllib3>=1.7.1,<1.8.0',
]

postgres_requires = [
    'psycopg2>=2.5.0,<2.6.0',
]

postgres_pypy_requires = [
    'psycopg2cffi',
]

mysql_requires = [
    'MySQL-python>=1.2.0,<1.3.0',
]

class DevelopWithBuildStatic(develop):
    def install_for_development(self):
        self.run_command('build_static')
        return develop.install_for_development(self)

class SdistWithBuildStatic(sdist):
    def make_distribution(self):
        self.run_command('build_static')
        return sdist.make_distribution(self)

class BuildStatic(Command):
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        log.info("running [npm install --quiet]")
        check_output(['npm', 'install', '--quiet'], cwd=ROOT)

        log.info("running [gulp dist]")
        check_output([os.path.join(ROOT, 'node_modules', '.bin', 'gulp'), 'dist'], cwd=ROOT)

setup(
    name='sentry',
    version='7.5.0.dev0',
    author='David Cramer',
    author_email='dcramer@gmail.com',
    url='https://www.getsentry.com',
    description='A realtime logging and aggregation server.',
    long_description=open('README.rst').read(),
    package_dir={'': 'src'},
    packages=find_packages('src'),
    zip_safe=False,
    install_requires=install_requires,
    extras_require={
        'tests': tests_require,
        'dev': dev_requires,
        'postgres': install_requires + postgres_requires,
        'postgres_pypy': install_requires + postgres_pypy_requires,
        'mysql': install_requires + mysql_requires,
    },
    cmdclass={
        'build_static': BuildStatic,
        'develop': DevelopWithBuildStatic,
        'sdist': SdistWithBuildStatic,
    },
    license='BSD',
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'sentry = sentry.utils.runner:main',
        ],
    },
    classifiers=[
        'Framework :: Django',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Operating System :: OS Independent',
        'Topic :: Software Development'
    ],
)
<commit_msg>Bump raven requirement to 5.3.0<commit_after>#!/usr/bin/env python
"""
Sentry
======

Sentry is a realtime event logging and aggregation platform. It specializes
in monitoring errors and extracting all the information needed to do a proper
post-mortem without any of the hassle of the standard user feedback loop.

Sentry is a Server
------------------

The Sentry package, at its core, is just a simple server and web UI. It will
handle authentication clients (such as `Raven <https://github.com/getsentry/raven-python>`_)
and all of the logic behind storage and aggregation.

That said, Sentry is not limited to Python. The primary implementation is in
Python, but it contains a full API for sending events from any language, in
any application.

:copyright: (c) 2011-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

import os.path

from distutils import log
from distutils.core import Command
from setuptools.command.develop import develop
from setuptools.command.sdist import sdist
from setuptools import setup, find_packages
from subprocess import check_output

# Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error
# in multiprocessing/util.py _exit_function when running `python
# setup.py test` (see
# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
for m in ('multiprocessing', 'billiard'):
    try:
        __import__(m)
    except ImportError:
        pass

ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__)))

dev_requires = [
    'flake8>=2.0,<2.1',
]

tests_require = [
    'blist',  # used by cassandra
    'casscache',
    'cqlsh',
    'elasticsearch',
    'httpretty',
    'pytest-cov>=1.4',
    'pytest-timeout',
    'python-coveralls',
    'responses',
    'riak',
]

install_requires = [
    'BeautifulSoup>=3.2.1,<3.3.0',
    'celery>=3.1.7,<3.2.0',
    'cssutils>=0.9.9,<0.10.0',
    'Django>=1.6.0,<1.7',
    'django-bitfield>=1.7.0,<1.8.0',
    'django-crispy-forms>=1.4.0,<1.5.0',
    'django-paging>=0.2.5,<0.3.0',
    'django-jsonfield>=0.9.13,<0.10.0',
    'django-picklefield>=0.3.0,<0.4.0',
    'django-recaptcha>=1.0.0,<1.1.0',
    'django-social-auth>=0.7.28,<0.8.0',
    'django-statsd-mozilla>=0.3.14.0,<0.3.15.0',
    'django-sudo>=1.1.3,<1.2.0',
    'django-templatetag-sugar>=0.1.0',
    'djangorestframework>=2.3.8,<2.4.0',
    'email-reply-parser>=0.2.0,<0.3.0',
    'enum34>=0.9.18,<0.10.0',
    'exam>=0.5.1',
    'gunicorn>=19.2.1,<20.0.0',
    'ipaddr>=2.1.11,<2.2.0',
    'logan>=0.5.8.2,<0.6.0',
    'lxml>=3.4.1',
    'mock>=0.8.0',
    'nydus>=0.11.0,<0.12.0',
    'markdown>=2.4.1,<2.5.0',
    'progressbar>=2.2,<2.4',
    'Pygments>=1.6.0,<1.7.0',
    'pytest',
    'pytest-django',
    'python-dateutil>=2.0.0,<3.0.0',
    'python-memcached>=1.53,<2.0.0',
    'raven>=5.3.0',
    'redis>=2.7.0,<2.11.0',
    'requests[security]>=2.5.1,<2.6.0',
    'simplejson>=3.1.0,<3.4.0',
    'six>=1.6.0,<2.0.0',
    'setproctitle>=1.1.7,<1.2.0',
    'statsd>=3.1.0,<3.2.0',
    'South==1.0.1',
    'toronado>=0.0.4,<0.1.0',
    'ua-parser>=0.3.5',
    'urllib3>=1.7.1,<1.8.0',
]

postgres_requires = [
    'psycopg2>=2.5.0,<2.6.0',
]

postgres_pypy_requires = [
    'psycopg2cffi',
]

mysql_requires = [
    'MySQL-python>=1.2.0,<1.3.0',
]

class DevelopWithBuildStatic(develop):
    def install_for_development(self):
        self.run_command('build_static')
        return develop.install_for_development(self)

class SdistWithBuildStatic(sdist):
    def make_distribution(self):
        self.run_command('build_static')
        return sdist.make_distribution(self)

class BuildStatic(Command):
    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        log.info("running [npm install --quiet]")
        check_output(['npm', 'install', '--quiet'], cwd=ROOT)

        log.info("running [gulp dist]")
        check_output([os.path.join(ROOT, 'node_modules', '.bin', 'gulp'), 'dist'], cwd=ROOT)

setup(
    name='sentry',
    version='7.5.0.dev0',
    author='David Cramer',
    author_email='dcramer@gmail.com',
    url='https://www.getsentry.com',
    description='A realtime logging and aggregation server.',
    long_description=open('README.rst').read(),
    package_dir={'': 'src'},
    packages=find_packages('src'),
    zip_safe=False,
    install_requires=install_requires,
    extras_require={
        'tests': tests_require,
        'dev': dev_requires,
        'postgres': install_requires + postgres_requires,
        'postgres_pypy': install_requires + postgres_pypy_requires,
        'mysql': install_requires + mysql_requires,
    },
    cmdclass={
        'build_static': BuildStatic,
        'develop': DevelopWithBuildStatic,
        'sdist': SdistWithBuildStatic,
    },
    license='BSD',
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'sentry = sentry.utils.runner:main',
        ],
    },
    classifiers=[
        'Framework :: Django',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Operating System :: OS Independent',
        'Topic :: Software Development'
    ],
)
<|endoftext|>


<commit_before>from distutils.core import setup

setup(
    name = 'py-web-search',
    packages = ['pws'],
    version = '0.1.1.3',
    description = 'A Python module to fetch and parse results from different search engines.',
    author='Rohith PR',
    author_email='praroh2@gmail.com',
    url='https://github.com/rohithpr/py-web-search',
    download_url = 'https://github.com/rohithpr/py-web-search/tarball/0.1.1.3',
    keywords = ['web search', 'web scraping'],
    classifiers=[
    'Intended Audience :: Developers',
    'Intended Audience :: Education',
    'Intended Audience :: End Users/Desktop',
    'Intended Audience :: Information Technology',
    'Intended Audience :: Science/Research',
    'License :: OSI Approved :: MIT License',
    'Natural Language :: English',
    'Programming Language :: Python',
    'Programming Language :: Python :: 3',
    'Topic :: Internet',
    'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
    'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator',],
)<commit_msg>New version: news added<commit_after>from distutils.core import setup

setup(
    name = 'py-web-search',
    packages = ['pws'],
    version = '0.1.2',
    description = 'A Python module to fetch and parse results from different search engines.',
    author='Rohith PR',
    author_email='praroh2@gmail.com',
    url='https://github.com/rohithpr/py-web-search',
    download_url = 'https://github.com/rohithpr/py-web-search/tarball/0.1.2',
    keywords = ['web search', 'web scraping'],
    classifiers=[
    'Intended Audience :: Developers',
    'Intended Audience :: Education',
    'Intended Audience :: End Users/Desktop',
    'Intended Audience :: Information Technology',
    'Intended Audience :: Science/Research',
    'License :: OSI Approved :: MIT License',
    'Natural Language :: English',
    'Programming Language :: Python',
    'Programming Language :: Python :: 3',
    'Topic :: Internet',
    'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
    'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator',],
)<|endoftext|>


<commit_before>import sys
import os

"""
This class is used to load sparkling water JAR into spark environment.
It is called when H2OConf or H2OContext is used
to ensure the required java classes are available in Spark.
"""
class Initializer(object):

    # Flag to inform us whether sparkling jar has been already loaded or not.
    # We don't want to load it more than once.
    __sparkling_jar_loaded = False
    __extracted_jar_dir = None

    @staticmethod
    def load_sparkling_jar(spark_context):
        if not Initializer.__sparkling_jar_loaded:
            sys.path.append(".")
            Initializer.__add_sparkling_jar_to_spark(spark_context)
            Initializer.__sparkling_jar_loaded = True

    @staticmethod
    def __add_sparkling_jar_to_spark(sc):
        jsc = sc._jsc
        jvm = sc._jvm
        # Add Sparkling water assembly JAR to driver
        sw_jar_file = Initializer.__get_sw_jar()

        #SW-593 - adding an extra / to fix a windows shell issue creating malform url
        if not sw_jar_file.startswith('/'):
            sw_jar_file = '/' + sw_jar_file

        url = jvm.java.net.URL("file://{0}".format(sw_jar_file))
        # Assuming that context class loader is always instance of URLClassLoader
        # ( which should be always true)
        cl = jvm.Thread.currentThread().getContextClassLoader()

        # Explicitly check if we run on databricks cloud since there we must add the jar to
        # the parent of context class loader
        if cl.getClass().getName()=='com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader':
            cl.getParent().getParent().addURL(url)
        else:
            spark_cl = Initializer.__find_spark_cl(cl, 'org.apache.spark.util.MutableURLClassLoader')
            spark_cl.addURL(url)

        # Add Sparkling Water Assembly JAR to Spark's file server so executors can fetch it
        # when they need to use the dependency.
        jsc.addJar(sw_jar_file)

        # SW-272: For yarn and cluster deployments we need to modify location of the jar file
        # to a driver cache location
        #
        # WARNING: This is low-level code accessing internal Spark API to enable SW2.0
        #          on Azure/CDH clusters.
        #          It needs to be properly tested and verified on different platforms!
        if jsc.sc().master() == "yarn" and jsc.sc().deployMode() == "cluster":
            file_server = jsc.env().rpcEnv().fileServer()
            field_jars = file_server.getClass().getDeclaredField("jars")
            if field_jars:
                field_jars.setAccessible(True)
                field_jars_value = field_jars.get(file_server)
                field_jars_value.put('sparkling_water_assembly.jar', jvm.java.io.File(sw_jar_file))

    @staticmethod
    def __extracted_jar_path():
        import sparkling_water
        sw_pkg_file = sparkling_water.__file__

        if Initializer.__extracted_jar_dir is None:
            zip_file = sw_pkg_file[:-len('/sparkling_water/__init__.py')]
            import tempfile
            Initializer.__extracted_jar_dir = tempfile.mkdtemp()
            import zipfile
            with zipfile.ZipFile(zip_file) as fzip:
                fzip.extract('sparkling_water/sparkling_water_assembly.jar', path = Initializer.__extracted_jar_dir)

        return os.path.abspath("{}/sparkling_water/sparkling_water_assembly.jar".format(Initializer.__extracted_jar_dir))

    @staticmethod
    def __get_sw_jar():
        import sparkling_water
        sw_pkg_file = sparkling_water.__file__
        # Extract jar file from zip
        if '.zip' in sw_pkg_file:
            return Initializer.__extracted_jar_path()
        else:
            from pkg_resources import resource_filename
            return os.path.abspath(resource_filename("sparkling_water", 'sparkling_water_assembly.jar'))

    @staticmethod
    def __find_spark_cl(start_cl, cl_name):
        cl = start_cl
        while cl:
            name = cl.getClass().getName()
            if (name == cl_name):
                return cl
            cl = cl.getParent()
        return None
<commit_msg>[SW-273] Remove the workaround introduced by SW-272 as it's already fixed in Spark 2.1.0 and later (#529)<commit_after>import sys
import os

"""
This class is used to load sparkling water JAR into spark environment.
It is called when H2OConf or H2OContext is used
to ensure the required java classes are available in Spark.
"""
class Initializer(object):

    # Flag to inform us whether sparkling jar has been already loaded or not.
    # We don't want to load it more than once.
    __sparkling_jar_loaded = False
    __extracted_jar_dir = None

    @staticmethod
    def load_sparkling_jar(spark_context):
        if not Initializer.__sparkling_jar_loaded:
            sys.path.append(".")
            Initializer.__add_sparkling_jar_to_spark(spark_context)
            Initializer.__sparkling_jar_loaded = True

    @staticmethod
    def __add_sparkling_jar_to_spark(sc):
        jsc = sc._jsc
        jvm = sc._jvm
        # Add Sparkling water assembly JAR to driver
        sw_jar_file = Initializer.__get_sw_jar()

        #SW-593 - adding an extra / to fix a windows shell issue creating malform url
        if not sw_jar_file.startswith('/'):
            sw_jar_file = '/' + sw_jar_file

        url = jvm.java.net.URL("file://{0}".format(sw_jar_file))
        # Assuming that context class loader is always instance of URLClassLoader
        # ( which should be always true)
        cl = jvm.Thread.currentThread().getContextClassLoader()

        # Explicitly check if we run on databricks cloud since there we must add the jar to
        # the parent of context class loader
        if cl.getClass().getName()=='com.databricks.backend.daemon.driver.DriverLocal$DriverLocalClassLoader':
            cl.getParent().getParent().addURL(url)
        else:
            spark_cl = Initializer.__find_spark_cl(cl, 'org.apache.spark.util.MutableURLClassLoader')
            spark_cl.addURL(url)

        # Add Sparkling Water Assembly JAR to Spark's file server so executors can fetch it
        # when they need to use the dependency.
        jsc.addJar(sw_jar_file)

    @staticmethod
    def __extracted_jar_path():
        import sparkling_water
        sw_pkg_file = sparkling_water.__file__

        if Initializer.__extracted_jar_dir is None:
            zip_file = sw_pkg_file[:-len('/sparkling_water/__init__.py')]
            import tempfile
            Initializer.__extracted_jar_dir = tempfile.mkdtemp()
            import zipfile
            with zipfile.ZipFile(zip_file) as fzip:
                fzip.extract('sparkling_water/sparkling_water_assembly.jar', path = Initializer.__extracted_jar_dir)

        return os.path.abspath("{}/sparkling_water/sparkling_water_assembly.jar".format(Initializer.__extracted_jar_dir))

    @staticmethod
    def __get_sw_jar():
        import sparkling_water
        sw_pkg_file = sparkling_water.__file__
        # Extract jar file from zip
        if '.zip' in sw_pkg_file:
            return Initializer.__extracted_jar_path()
        else:
            from pkg_resources import resource_filename
            return os.path.abspath(resource_filename("sparkling_water", 'sparkling_water_assembly.jar'))

    @staticmethod
    def __find_spark_cl(start_cl, cl_name):
        cl = start_cl
        while cl:
            name = cl.getClass().getName()
            if (name == cl_name):
                return cl
            cl = cl.getParent()
        return None
<|endoftext|>


<commit_before>
from setuptools import setup, find_packages

setup(
    name='munge',
    version=open('config/VERSION').read().rstrip(),
    author='Twentieth Century',
    author_email='code@20c.com',
    description='data manipulation client / library',
    long_description=open('README.md').read(),
    license='LICENSE.txt',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages = find_packages(),
    scripts=['munge/bin/munge'],
    url = 'https://github.com/20c/munge',
    download_url = 'https://github.com/20c/munge/%s' % version,
    include_package_data=True,
    zip_safe=False
)
<commit_msg>fix setup.py<commit_after>
from setuptools import setup, find_packages

version = open('config/VERSION').read().rstrip()

setup(
    name='munge',
    version=version,
    author='Twentieth Century',
    author_email='code@20c.com',
    description='data manipulation client / library',
    long_description=open('README.md').read(),
    license='LICENSE.txt',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: Apache Software License',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages = find_packages(),
    scripts=['munge/bin/munge'],
    url = 'https://github.com/20c/munge',
    download_url = 'https://github.com/20c/munge/%s' % version,
    include_package_data=True,
    zip_safe=False
)
<|endoftext|>