<commit_before># Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import dataclasses
import itertools
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
from pathlib import PurePath
from typing import ClassVar, Dict, Iterable, Optional, Tuple, Type

from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE
from pants.base.specs import OriginSpec
from pants.build_graph.address import Address
from pants.engine import desktop
from pants.engine.console import Console
from pants.engine.fs import Digest, DirectoryToMaterialize, Workspace
from pants.engine.goal import Goal, GoalSubsystem
from pants.engine.interactive_runner import InteractiveProcessRequest, InteractiveRunner
from pants.engine.isolated_process import FallibleExecuteProcessResult
from pants.engine.objects import union
from pants.engine.rules import UnionMembership, goal_rule, rule
from pants.engine.selectors import Get, MultiGet
from pants.engine.target import (
    Field,
    HydratedSources,
    HydrateSourcesRequest,
    RegisteredTargetTypes,
    Sources,
    Target,
    TargetsWithOrigins,
    TargetWithOrigin,
)

# TODO(#6004): use proper Logging singleton, rather than static logger.
logger = logging.getLogger(__name__)

class Status(Enum):
    SUCCESS = "SUCCESS"
    FAILURE = "FAILURE"

@dataclass(frozen=True)
class TestResult:
    status: Status
    stdout: str
    stderr: str
    coverage_data: Optional["CoverageData"] = None

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

    @staticmethod
    def from_fallible_execute_process_result(
        process_result: FallibleExecuteProcessResult,
        *,
        coverage_data: Optional["CoverageData"] = None,
    ) -> "TestResult":
        return TestResult(
            status=Status.SUCCESS if process_result.exit_code == 0 else Status.FAILURE,
            stdout=process_result.stdout.decode(),
            stderr=process_result.stderr.decode(),
            coverage_data=coverage_data,
        )

@dataclass(frozen=True)
class TestDebugRequest:
    ipr: InteractiveProcessRequest

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

# TODO: Factor this out once porting fmt.py and lint.py to the Target API. See `binary.py` for a
#  similar implementation (that one doesn't keep the `OriginSpec`).
@union
@dataclass(frozen=True)
class TestConfiguration(ABC):
    """An ad hoc collection of the fields necessary to run tests on a target."""

    required_fields: ClassVar[Tuple[Type[Field], ...]]

    address: Address
    origin: OriginSpec

    sources: Sources

    __test__ = False

    @classmethod
    def is_valid(cls, tgt: Target) -> bool:
        return tgt.has_fields(cls.required_fields)

    @classmethod
    def valid_target_types(
        cls, target_types: Iterable[Type[Target]], *, union_membership: UnionMembership
    ) -> Tuple[Type[Target], ...]:
        return tuple(
            target_type
            for target_type in target_types
            if target_type.class_has_fields(cls.required_fields, union_membership=union_membership)
        )

    @classmethod
    def create(cls, target_with_origin: TargetWithOrigin) -> "TestConfiguration":
        all_expected_fields: Dict[str, Type[Field]] = {
            dataclass_field.name: dataclass_field.type
            for dataclass_field in dataclasses.fields(cls)
            if issubclass(dataclass_field.type, Field)
        }
        tgt = target_with_origin.target
        return cls(
            address=tgt.address,
            origin=target_with_origin.origin,
            **{  # type: ignore[arg-type]
                dataclass_field_name: (
                    tgt[field_cls] if field_cls in cls.required_fields else tgt.get(field_cls)
                )
                for dataclass_field_name, field_cls in all_expected_fields.items()
            },
        )

# NB: This is only used for the sake of coordinator_of_tests. Consider inlining that rule so that
# we can remove this wrapper type.
@dataclass(frozen=True)
class WrappedTestConfiguration:
    config: TestConfiguration

@dataclass(frozen=True)
class AddressAndTestResult:
    address: Address
    test_result: TestResult

class CoverageData(ABC):
    """Base class for inputs to a coverage report.

    Subclasses should add whichever fields they require - snapshots of coverage output or xml files, etc.
    """

    @property
    @abstractmethod
    def batch_cls(self) -> Type["CoverageDataBatch"]:
        pass

@union
class CoverageDataBatch:
    pass

class CoverageReport(ABC):
    """Represents a code coverage report that can be materialized to the terminal or disk."""

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        """Materialize this code coverage report to the terminal or disk.

        :param console: A handle to the terminal.
        :param workspace: A handle to local disk.
        :return: If a report was materialized to disk, the path of the file in the report one might
                 open first to start examining the report.
        """
        ...

@dataclass(frozen=True)
class ConsoleCoverageReport(CoverageReport):
    """Materializes a code coverage report to the terminal."""

    report: str

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        console.print_stdout(f"\n{self.report}")
        return None

@dataclass(frozen=True)
class FilesystemCoverageReport(CoverageReport):
    """Materializes a code coverage report to disk."""

    result_digest: Digest
    directory_to_materialize_to: PurePath
    report_file: Optional[PurePath]

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        workspace.materialize_directory(
            DirectoryToMaterialize(
                self.result_digest, path_prefix=str(self.directory_to_materialize_to),
            )
        )
        console.print_stdout(f"\nWrote coverage report to `{self.directory_to_materialize_to}`")
        return self.report_file

class TestOptions(GoalSubsystem):
    """Runs tests."""

    name = "test"

    required_union_implementations = (TestConfiguration,)

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

    @classmethod
    def register_options(cls, register) -> None:
        super().register_options(register)
        register(
            "--debug",
            type=bool,
            default=False,
            help="Run a single test target in an interactive process. This is necessary, for "
            "example, when you add breakpoints in your code.",
        )
        register(
            "--run-coverage",
            type=bool,
            default=False,
            help="Generate a coverage report for this test run.",
        )
        register(
            "--open-coverage",
            type=bool,
            default=False,
            help="If a coverage report file is generated, open it on the local system if the "
            "system supports this.",
        )

class Test(Goal):
    subsystem_cls = TestOptions

    __test__ = False

@goal_rule
async def run_tests(
    console: Console,
    options: TestOptions,
    interactive_runner: InteractiveRunner,
    targets_with_origins: TargetsWithOrigins,
    workspace: Workspace,
    union_membership: UnionMembership,
    registered_target_types: RegisteredTargetTypes,
) -> Test:
    config_types: Iterable[Type[TestConfiguration]] = union_membership.union_rules[
        TestConfiguration
    ]

    if options.values.debug:
        target_with_origin = targets_with_origins.expect_single()
        target = target_with_origin.target
        valid_config_types = [
            config_type for config_type in config_types if config_type.is_valid(target)
        ]
        if not valid_config_types:
            all_valid_target_types = itertools.chain.from_iterable(
                config_type.valid_target_types(
                    registered_target_types.types, union_membership=union_membership
                )
                for config_type in config_types
            )
            formatted_target_types = sorted(
                target_type.alias for target_type in all_valid_target_types
            )
            raise ValueError(
                f"The `test` goal only works with the following target types: "
                f"{formatted_target_types}\n\nYou used {target.address} with target "
                f"type {repr(target.alias)}."
            )
        if len(valid_config_types) > 1:
            possible_config_types = sorted(
                config_type.__name__ for config_type in valid_config_types
            )
            raise ValueError(
                f"Multiple of the registered test implementations work for {target.address} "
                f"(target type {repr(target.alias)}). It is ambiguous which implementation to use. "
                f"Possible implementations: {possible_config_types}."
            )
        config_type = valid_config_types[0]
        logger.info(f"Starting test in debug mode: {target.address.reference()}")
        request = await Get[TestDebugRequest](
            TestConfiguration, config_type.create(target_with_origin)
        )
        debug_result = interactive_runner.run_local_interactive_process(request.ipr)
        return Test(debug_result.process_exit_code)

    # TODO: possibly factor out this filtering out of empty `sources`. We do this at this level of
    #  abstraction, rather than in the test runners, because the test runners often will use
    #  auto-discovery when given no input files.
    configs = tuple(
        config_type.create(target_with_origin)
        for target_with_origin in targets_with_origins
        for config_type in config_types
        if config_type.is_valid(target_with_origin.target)
    )
    all_hydrated_sources = await MultiGet(
        Get[HydratedSources](HydrateSourcesRequest, test_target.sources.request)
        for test_target in configs
    )

    results = await MultiGet(
        Get[AddressAndTestResult](WrappedTestConfiguration(config))
        for config, hydrated_sources in zip(configs, all_hydrated_sources)
        if hydrated_sources.snapshot.files
    )

    did_any_fail = False
    for result in results:
        if result.test_result.status == Status.FAILURE:
            did_any_fail = True
        if result.test_result.stdout:
            console.write_stdout(
                f"{result.address.reference()} stdout:\n{result.test_result.stdout}\n"
            )
        if result.test_result.stderr:
            # NB: we write to stdout, rather than to stderr, to avoid potential issues interleaving
            # the two streams.
            console.write_stdout(
                f"{result.address.reference()} stderr:\n{result.test_result.stderr}\n"
            )

    console.write_stdout("\n")

    for result in results:
        console.print_stdout(
            f"{result.address.reference():80}.....{result.test_result.status.value:>10}"
        )

    if did_any_fail:
        console.print_stderr(console.red("\nTests failed"))
        exit_code = PANTS_FAILED_EXIT_CODE
    else:
        exit_code = PANTS_SUCCEEDED_EXIT_CODE

    if options.values.run_coverage:
        # TODO: consider warning if a user uses `--coverage` but the language backend does not
        # provide coverage support. This might be too chatty to be worth doing?
        results_with_coverage = [x for x in results if x.test_result.coverage_data is not None]
        coverage_data_collections = itertools.groupby(
            results_with_coverage,
            lambda address_and_test_result: (
                address_and_test_result.test_result.coverage_data.batch_cls  # type: ignore[union-attr]
            ),
        )

        coverage_reports = await MultiGet(
            Get[CoverageReport](
                CoverageDataBatch,
                coverage_batch_cls(tuple(addresses_and_test_results)),  # type: ignore[call-arg]
            )
            for coverage_batch_cls, addresses_and_test_results in coverage_data_collections
        )

        coverage_report_files = []
        for report in coverage_reports:
            report_file = report.materialize(console, workspace)
            if report_file is not None:
                coverage_report_files.append(report_file)

        if coverage_report_files and options.values.open_coverage:
            desktop.ui_open(console, interactive_runner, coverage_report_files)

    return Test(exit_code)

@rule
async def coordinator_of_tests(wrapped_config: WrappedTestConfiguration) -> AddressAndTestResult:
    config = wrapped_config.config

    # TODO(#6004): when streaming to live TTY, rely on V2 UI for this information. When not a
    # live TTY, periodically dump heavy hitters to stderr. See
    # https://github.com/pantsbuild/pants/issues/6004#issuecomment-492699898.
    logger.info(f"Starting tests: {config.address.reference()}")
    result = await Get[TestResult](TestConfiguration, config)
    logger.info(
        f"Tests {'succeeded' if result.status == Status.SUCCESS else 'failed'}: "
        f"{config.address.reference()}"
    )
    return AddressAndTestResult(config.address, result)

def rules():
    return [coordinator_of_tests, run_tests]
<commit_msg>Fix 3.8 problem (#9502)<commit_after># Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import dataclasses
import itertools
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
from pathlib import PurePath
from typing import ClassVar, Dict, Iterable, Optional, Tuple, Type

from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE
from pants.base.specs import OriginSpec
from pants.build_graph.address import Address
from pants.engine import desktop
from pants.engine.console import Console
from pants.engine.fs import Digest, DirectoryToMaterialize, Workspace
from pants.engine.goal import Goal, GoalSubsystem
from pants.engine.interactive_runner import InteractiveProcessRequest, InteractiveRunner
from pants.engine.isolated_process import FallibleExecuteProcessResult
from pants.engine.objects import union
from pants.engine.rules import UnionMembership, goal_rule, rule
from pants.engine.selectors import Get, MultiGet
from pants.engine.target import (
    Field,
    HydratedSources,
    HydrateSourcesRequest,
    RegisteredTargetTypes,
    Sources,
    Target,
    TargetsWithOrigins,
    TargetWithOrigin,
)

# TODO(#6004): use proper Logging singleton, rather than static logger.
logger = logging.getLogger(__name__)

class Status(Enum):
    SUCCESS = "SUCCESS"
    FAILURE = "FAILURE"

@dataclass(frozen=True)
class TestResult:
    status: Status
    stdout: str
    stderr: str
    coverage_data: Optional["CoverageData"] = None

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

    @staticmethod
    def from_fallible_execute_process_result(
        process_result: FallibleExecuteProcessResult,
        *,
        coverage_data: Optional["CoverageData"] = None,
    ) -> "TestResult":
        return TestResult(
            status=Status.SUCCESS if process_result.exit_code == 0 else Status.FAILURE,
            stdout=process_result.stdout.decode(),
            stderr=process_result.stderr.decode(),
            coverage_data=coverage_data,
        )

@dataclass(frozen=True)
class TestDebugRequest:
    ipr: InteractiveProcessRequest

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

# TODO: Factor this out once porting fmt.py and lint.py to the Target API. See `binary.py` for a
#  similar implementation (that one doesn't keep the `OriginSpec`).
@union
@dataclass(frozen=True)
class TestConfiguration(ABC):
    """An ad hoc collection of the fields necessary to run tests on a target."""

    required_fields: ClassVar[Tuple[Type[Field], ...]]

    address: Address
    origin: OriginSpec

    sources: Sources

    __test__ = False

    @classmethod
    def is_valid(cls, tgt: Target) -> bool:
        return tgt.has_fields(cls.required_fields)

    @classmethod
    def valid_target_types(
        cls, target_types: Iterable[Type[Target]], *, union_membership: UnionMembership
    ) -> Tuple[Type[Target], ...]:
        return tuple(
            target_type
            for target_type in target_types
            if target_type.class_has_fields(cls.required_fields, union_membership=union_membership)
        )

    @classmethod
    def create(cls, target_with_origin: TargetWithOrigin) -> "TestConfiguration":
        all_expected_fields: Dict[str, Type[Field]] = {
            dataclass_field.name: dataclass_field.type
            for dataclass_field in dataclasses.fields(cls)
            if isinstance(dataclass_field.type, type) and issubclass(dataclass_field.type, Field)  # type: ignore[misc]
        }
        tgt = target_with_origin.target
        return cls(
            address=tgt.address,
            origin=target_with_origin.origin,
            **{  # type: ignore[arg-type]
                dataclass_field_name: (
                    tgt[field_cls] if field_cls in cls.required_fields else tgt.get(field_cls)
                )
                for dataclass_field_name, field_cls in all_expected_fields.items()
            },
        )

# NB: This is only used for the sake of coordinator_of_tests. Consider inlining that rule so that
# we can remove this wrapper type.
@dataclass(frozen=True)
class WrappedTestConfiguration:
    config: TestConfiguration

@dataclass(frozen=True)
class AddressAndTestResult:
    address: Address
    test_result: TestResult

class CoverageData(ABC):
    """Base class for inputs to a coverage report.

    Subclasses should add whichever fields they require - snapshots of coverage output or xml files, etc.
    """

    @property
    @abstractmethod
    def batch_cls(self) -> Type["CoverageDataBatch"]:
        pass

@union
class CoverageDataBatch:
    pass

class CoverageReport(ABC):
    """Represents a code coverage report that can be materialized to the terminal or disk."""

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        """Materialize this code coverage report to the terminal or disk.

        :param console: A handle to the terminal.
        :param workspace: A handle to local disk.
        :return: If a report was materialized to disk, the path of the file in the report one might
                 open first to start examining the report.
        """
        ...

@dataclass(frozen=True)
class ConsoleCoverageReport(CoverageReport):
    """Materializes a code coverage report to the terminal."""

    report: str

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        console.print_stdout(f"\n{self.report}")
        return None

@dataclass(frozen=True)
class FilesystemCoverageReport(CoverageReport):
    """Materializes a code coverage report to disk."""

    result_digest: Digest
    directory_to_materialize_to: PurePath
    report_file: Optional[PurePath]

    def materialize(self, console: Console, workspace: Workspace) -> Optional[PurePath]:
        workspace.materialize_directory(
            DirectoryToMaterialize(
                self.result_digest, path_prefix=str(self.directory_to_materialize_to),
            )
        )
        console.print_stdout(f"\nWrote coverage report to `{self.directory_to_materialize_to}`")
        return self.report_file

class TestOptions(GoalSubsystem):
    """Runs tests."""

    name = "test"

    required_union_implementations = (TestConfiguration,)

    # Prevent this class from being detected by pytest as a test class.
    __test__ = False

    @classmethod
    def register_options(cls, register) -> None:
        super().register_options(register)
        register(
            "--debug",
            type=bool,
            default=False,
            help="Run a single test target in an interactive process. This is necessary, for "
            "example, when you add breakpoints in your code.",
        )
        register(
            "--run-coverage",
            type=bool,
            default=False,
            help="Generate a coverage report for this test run.",
        )
        register(
            "--open-coverage",
            type=bool,
            default=False,
            help="If a coverage report file is generated, open it on the local system if the "
            "system supports this.",
        )

class Test(Goal):
    subsystem_cls = TestOptions

    __test__ = False

@goal_rule
async def run_tests(
    console: Console,
    options: TestOptions,
    interactive_runner: InteractiveRunner,
    targets_with_origins: TargetsWithOrigins,
    workspace: Workspace,
    union_membership: UnionMembership,
    registered_target_types: RegisteredTargetTypes,
) -> Test:
    config_types: Iterable[Type[TestConfiguration]] = union_membership.union_rules[
        TestConfiguration
    ]

    if options.values.debug:
        target_with_origin = targets_with_origins.expect_single()
        target = target_with_origin.target
        valid_config_types = [
            config_type for config_type in config_types if config_type.is_valid(target)
        ]
        if not valid_config_types:
            all_valid_target_types = itertools.chain.from_iterable(
                config_type.valid_target_types(
                    registered_target_types.types, union_membership=union_membership
                )
                for config_type in config_types
            )
            formatted_target_types = sorted(
                target_type.alias for target_type in all_valid_target_types
            )
            raise ValueError(
                f"The `test` goal only works with the following target types: "
                f"{formatted_target_types}\n\nYou used {target.address} with target "
                f"type {repr(target.alias)}."
            )
        if len(valid_config_types) > 1:
            possible_config_types = sorted(
                config_type.__name__ for config_type in valid_config_types
            )
            raise ValueError(
                f"Multiple of the registered test implementations work for {target.address} "
                f"(target type {repr(target.alias)}). It is ambiguous which implementation to use. "
                f"Possible implementations: {possible_config_types}."
            )
        config_type = valid_config_types[0]
        logger.info(f"Starting test in debug mode: {target.address.reference()}")
        request = await Get[TestDebugRequest](
            TestConfiguration, config_type.create(target_with_origin)
        )
        debug_result = interactive_runner.run_local_interactive_process(request.ipr)
        return Test(debug_result.process_exit_code)

    # TODO: possibly factor out this filtering out of empty `sources`. We do this at this level of
    #  abstraction, rather than in the test runners, because the test runners often will use
    #  auto-discovery when given no input files.
    configs = tuple(
        config_type.create(target_with_origin)
        for target_with_origin in targets_with_origins
        for config_type in config_types
        if config_type.is_valid(target_with_origin.target)
    )
    all_hydrated_sources = await MultiGet(
        Get[HydratedSources](HydrateSourcesRequest, test_target.sources.request)
        for test_target in configs
    )

    results = await MultiGet(
        Get[AddressAndTestResult](WrappedTestConfiguration(config))
        for config, hydrated_sources in zip(configs, all_hydrated_sources)
        if hydrated_sources.snapshot.files
    )

    did_any_fail = False
    for result in results:
        if result.test_result.status == Status.FAILURE:
            did_any_fail = True
        if result.test_result.stdout:
            console.write_stdout(
                f"{result.address.reference()} stdout:\n{result.test_result.stdout}\n"
            )
        if result.test_result.stderr:
            # NB: we write to stdout, rather than to stderr, to avoid potential issues interleaving
            # the two streams.
            console.write_stdout(
                f"{result.address.reference()} stderr:\n{result.test_result.stderr}\n"
            )

    console.write_stdout("\n")

    for result in results:
        console.print_stdout(
            f"{result.address.reference():80}.....{result.test_result.status.value:>10}"
        )

    if did_any_fail:
        console.print_stderr(console.red("\nTests failed"))
        exit_code = PANTS_FAILED_EXIT_CODE
    else:
        exit_code = PANTS_SUCCEEDED_EXIT_CODE

    if options.values.run_coverage:
        # TODO: consider warning if a user uses `--coverage` but the language backend does not
        # provide coverage support. This might be too chatty to be worth doing?
        results_with_coverage = [x for x in results if x.test_result.coverage_data is not None]
        coverage_data_collections = itertools.groupby(
            results_with_coverage,
            lambda address_and_test_result: (
                address_and_test_result.test_result.coverage_data.batch_cls  # type: ignore[union-attr]
            ),
        )

        coverage_reports = await MultiGet(
            Get[CoverageReport](
                CoverageDataBatch,
                coverage_batch_cls(tuple(addresses_and_test_results)),  # type: ignore[call-arg]
            )
            for coverage_batch_cls, addresses_and_test_results in coverage_data_collections
        )

        coverage_report_files = []
        for report in coverage_reports:
            report_file = report.materialize(console, workspace)
            if report_file is not None:
                coverage_report_files.append(report_file)

        if coverage_report_files and options.values.open_coverage:
            desktop.ui_open(console, interactive_runner, coverage_report_files)

    return Test(exit_code)

@rule
async def coordinator_of_tests(wrapped_config: WrappedTestConfiguration) -> AddressAndTestResult:
    config = wrapped_config.config

    # TODO(#6004): when streaming to live TTY, rely on V2 UI for this information. When not a
    # live TTY, periodically dump heavy hitters to stderr. See
    # https://github.com/pantsbuild/pants/issues/6004#issuecomment-492699898.
    logger.info(f"Starting tests: {config.address.reference()}")
    result = await Get[TestResult](TestConfiguration, config)
    logger.info(
        f"Tests {'succeeded' if result.status == Status.SUCCESS else 'failed'}: "
        f"{config.address.reference()}"
    )
    return AddressAndTestResult(config.address, result)

def rules():
    return [coordinator_of_tests, run_tests]
<|endoftext|>


<commit_before>import collections
from math import ceil
from gen.twitter.mesos.ttypes import *
from twitter.common import app, log

app.add_option('--mesos_updater_status_check_interval',
  dest='mesos_updater_status_check_interval',
  default=3,
  type='int',
  help='How often Mesos runs update loop...')

def debug_if(condition):
  return log.DEBUG if condition else log.ERROR

class Updater(object):
  """Update the shards of a job in batches."""

  class InvalidConfigError(Exception): pass

  def __init__(self, role, job_name, scheduler, clock, update_token, session):
    self._role = role
    self._job_name = job_name
    self._scheduler = scheduler
    self._clock = clock
    self._update_token = update_token
    self._session = session
    self._total_fail_count = 0
    self._max_total_failures = 0
    self._failures_by_shard = collections.defaultdict(int)

  @staticmethod
  def validate_config(update_config):
    """Perform sanity test on update_config."""
    if update_config.batchSize < 1:
      raise Updater.InvalidConfigError('Batch size should be greater than 0')
    if update_config.restartThreshold < 1:
      raise Updater.InvalidConfigError('Restart Threshold should be greater than 0')
    if update_config.watchSecs < 1:
      raise Updater.InvalidConfigError('Watch seconds should be greater than 0')

  def update_failure_counts(self, failed_shards):
    """Update the failure counts metrics based upon a batch of failed shards."""
    self._total_fail_count += len(failed_shards)
    for shard in failed_shards:
      self._failures_by_shard[shard] += 1

  def exceeded_total_fail_count(self):
    """Checks if the total number of failures is greater than a threshold."""
    return self._total_fail_count > self._max_total_failures

  def exceeded_shard_fail_count(self):
    """Checks if the per shard failure is greater than a threshold."""
    return any(map(lambda num_failures: num_failures > self._max_shard_failures,
      self._failures_by_shard.values()))

  def is_failed_update(self, failed_shards):
    is_failed = self.exceeded_total_fail_count() or self.exceeded_shard_fail_count()

    if is_failed:
      log.error('%s failures observed, maximum allowed is %s' % (self._total_fail_count,
          self._max_total_failures))
      for shard in self._failures_by_shard:
        if self._failures_by_shard[shard] > self._max_shard_failures:
          log.error('%s shard failures for shard %s, maximum allowed is %s' %
              (self._failures_by_shard[shard], shard, self._max_shard_failures))
    return is_failed

  def update(self, job_config):
    """Performs the job update, blocking until it completes.
    A rollback will be performed if the update was considered a failure based on the
    update configuration.

    Arguments:
    job_config -- job configuration object.

    Returns the set of shards that failed to update.
    """
    Updater.validate_config(job_config.updateConfig)

    update_config = job_config.updateConfig
    failed_shards = []
    self._max_total_failures = update_config.maxTotalFailures
    self._max_shard_failures = update_config.maxPerShardFailures
    initial_shards = sorted(map(lambda config: config.shardId, job_config.taskConfigs))
    remaining_shards = initial_shards[:]
    update_in_progress = True

    log.info('Starting job update...')
    while update_in_progress:
      batch_shards = remaining_shards[0 : update_config.batchSize]
      remaining_shards = list(set(remaining_shards) - set(batch_shards))
      resp = self.restart_shards(batch_shards, update_config)
      log.log(debug_if(resp.responseCode == UpdateResponseCode.OK),
        'Response from scheduler: %s (message: %s)' % (
          UpdateResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
      failed_shards = self.watch_tasks(batch_shards, update_config.restartThreshold,
          update_config.watchSecs)
      log.log(debug_if(failed_shards == []),
        'Failed_tasks : %s' % (failed_shards if failed_shards != [] else 'None'))
      remaining_shards += failed_shards
      remaining_shards.sort()
      self.update_failure_counts(failed_shards)
      update_in_progress = len(remaining_shards) == 0 and not self.is_failed_update()

    if failed_shards:
      shards_to_rollback = [shard for shard in set(initial_shards).difference(remaining_shards)] + (
          failed_shards)
      self.rollback(update_config, shards_to_rollback)

    return failed_shards

  def rollback(self, update_config, shards_to_rollback):
    """Performs the job rollback.

    Arguments:
    update_config -- update configuration object that describes how the rollback is performed.
    shards_to_rollback -- shard ids.
    """
    log.info('Reverting update for %s' % shards_to_rollback)
    shards_to_rollback.sort()
    while len(shards_to_rollback) > 0:
      batch_shards = shards_to_rollback[0 : update_config.batchSize]
      shards_to_rollback = list(set(shards_to_rollback) - set(batch_shards))
      resp = self._scheduler.rollbackShards(self._role, self._job_name, batch_shards,
          self._update_token, self._session)
      self.watch_tasks(batch_shards, update_config.restartThreshold, update_config.watchSecs)
      log.log(debug_if(resp == UpdateResponseCode.OK),
        'Response from scheduler: %s (message: %s)'
          % (UpdateResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))

  def restart_shards(self, shard_ids, update_config):
    """Performs a scheduler call for restart.

    Arguments:
    shard_ids -- set of shards to be restarted by the scheduler.
    update_config -- update configuration object that describes how the restart is performed.

    Returns a map of the current status of the restarted shards as returned by the scheduler.
    """
    log.info('Restarting shards: %s' % shard_ids)
    return self._scheduler.updateShards(self._role, self._job_name, shard_ids,
        self._update_token, self._session)

  def watch_tasks(self, task_ids, restart_threshold, watch_secs):
    """Monitors the restarted shards.

    Arguments:
    task_ids -- set of shards to watch.
    restart_threshold -- Maximum number of seconds before which a task must move
                         to the RUNNING state.
    watch_secs -- Number of seconds to watch the task once it is RUNNING.

    Returns a set of tasks that failed to meet the following criteria,
    1. Failed to move to RUNNING state before restart_threshold from the time of restart.
    2. Failed to stay in the RUNNING state before watch_secs expire.
    """
    # TODO(Sathya): Consider adding PREEMPTING as an active state.
    ACTIVE_STATES = set([ScheduleStatus.PENDING, ScheduleStatus.STARTING, ScheduleStatus.RUNNING,
        ScheduleStatus.ASSIGNED, ScheduleStatus.UPDATING, ScheduleStatus.ROLLBACK])
    start_time = self._clock.time()
    expected_running_by = start_time + restart_threshold
    statuses = {}
    running_state_times = {}
    healthy_tasks = set()
    failed_shards = set()
    while True:
      log.debug('Getting status...')
      query = TaskQuery()
      query.owner = Identity(role = self._role)
      query.jobName = self._job_name
      resp = self._scheduler.getTasksStatus(query)
      log.debug('Response from scheduler: %s (message: %s)'
          % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
      if resp.tasks:
        for task in resp.tasks:
          statuses[task.assignedTask.task.shardId] = task.status
      else:
        log.debug('No tasks found.')
      log.debug('Got statuses: %s' %
          dict([(task,ScheduleStatus._VALUES_TO_NAMES[status])
          for task,status in statuses.iteritems()]))
      now = self._clock.time()
      for task_id, status in statuses.items():
        if status is ScheduleStatus.RUNNING and task_id not in (
          failed_shards.union(running_state_times.keys())):
          running_state_times[task_id] = now
          log.debug('Adding task-%s to running tasks' % task_id)
      if now > expected_running_by:
        non_running_tasks = [id for id in task_ids if id not in running_state_times]
        log.log(debug_if(non_running_tasks == []),
          'Tasks failed to move into running: %s' %
            (non_running_tasks if non_running_tasks != [] else 'None'))
        failed_shards.update(non_running_tasks)
      healthy_tasks.update(id for id in statuses
          if id in running_state_times and now > running_state_times[id] + watch_secs)
      failed_shards.update(id for id in statuses
          if id not in healthy_tasks and statuses[id] not in ACTIVE_STATES)
      if healthy_tasks.union(failed_shards) == set(task_ids):
        return [shard for shard in failed_shards]
      elif now > (start_time + restart_threshold + watch_secs):
        return [shard for shard in set(task_ids).difference(healthy_tasks)]
      self._clock.sleep(app.get_options().mesos_updater_status_check_interval)
<commit_msg>Fixed bug in updater.py<commit_after>import collections
from math import ceil
from gen.twitter.mesos.ttypes import *
from twitter.common import app, log

app.add_option('--mesos_updater_status_check_interval',
  dest='mesos_updater_status_check_interval',
  default=3,
  type='int',
  help='How often Mesos runs update loop...')

def debug_if(condition):
  return log.DEBUG if condition else log.ERROR

class Updater(object):
  """Update the shards of a job in batches."""

  class InvalidConfigError(Exception): pass

  def __init__(self, role, job_name, scheduler, clock, update_token, session):
    self._role = role
    self._job_name = job_name
    self._scheduler = scheduler
    self._clock = clock
    self._update_token = update_token
    self._session = session
    self._total_fail_count = 0
    self._max_total_failures = 0
    self._failures_by_shard = collections.defaultdict(int)

  @staticmethod
  def validate_config(update_config):
    """Perform sanity test on update_config."""
    if update_config.batchSize < 1:
      raise Updater.InvalidConfigError('Batch size should be greater than 0')
    if update_config.restartThreshold < 1:
      raise Updater.InvalidConfigError('Restart Threshold should be greater than 0')
    if update_config.watchSecs < 1:
      raise Updater.InvalidConfigError('Watch seconds should be greater than 0')

  def update_failure_counts(self, failed_shards):
    """Update the failure counts metrics based upon a batch of failed shards."""
    self._total_fail_count += len(failed_shards)
    for shard in failed_shards:
      self._failures_by_shard[shard] += 1

  def exceeded_total_fail_count(self):
    """Checks if the total number of failures is greater than a threshold."""
    return self._total_fail_count > self._max_total_failures

  def exceeded_shard_fail_count(self):
    """Checks if the per shard failure is greater than a threshold."""
    return any(map(lambda num_failures: num_failures > self._max_shard_failures,
      self._failures_by_shard.values()))

  def is_failed_update(self, failed_shards):
    is_failed = self.exceeded_total_fail_count() or self.exceeded_shard_fail_count()

    if is_failed:
      log.error('%s failures observed, maximum allowed is %s' % (self._total_fail_count,
          self._max_total_failures))
      for shard in self._failures_by_shard:
        if self._failures_by_shard[shard] > self._max_shard_failures:
          log.error('%s shard failures for shard %s, maximum allowed is %s' %
              (self._failures_by_shard[shard], shard, self._max_shard_failures))
    return is_failed

  def update(self, job_config):
    """Performs the job update, blocking until it completes.
    A rollback will be performed if the update was considered a failure based on the
    update configuration.

    Arguments:
    job_config -- job configuration object.

    Returns the set of shards that failed to update.
    """
    Updater.validate_config(job_config.updateConfig)

    update_config = job_config.updateConfig
    failed_shards = []
    self._max_total_failures = update_config.maxTotalFailures
    self._max_shard_failures = update_config.maxPerShardFailures
    initial_shards = sorted(map(lambda config: config.shardId, job_config.taskConfigs))
    remaining_shards = initial_shards[:]
    update_in_progress = True

    log.info('Starting job update...')
    while update_in_progress:
      batch_shards = remaining_shards[0 : update_config.batchSize]
      remaining_shards = list(set(remaining_shards) - set(batch_shards))
      resp = self.restart_shards(batch_shards, update_config)
      log.log(debug_if(resp.responseCode == UpdateResponseCode.OK),
        'Response from scheduler: %s (message: %s)' % (
          UpdateResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
      failed_shards = self.watch_tasks(batch_shards, update_config.restartThreshold,
          update_config.watchSecs)
      log.log(debug_if(failed_shards == []),
        'Failed_tasks : %s' % (failed_shards if failed_shards != [] else 'None'))
      remaining_shards += failed_shards
      remaining_shards.sort()
      self.update_failure_counts(failed_shards)
      update_in_progress = len(remaining_shards) == 0 and not self.is_failed_update(failed_shards)

    if failed_shards:
      shards_to_rollback = [shard for shard in set(initial_shards).difference(remaining_shards)] + (
          failed_shards)
      self.rollback(update_config, shards_to_rollback)

    return failed_shards

  def rollback(self, update_config, shards_to_rollback):
    """Performs the job rollback.

    Arguments:
    update_config -- update configuration object that describes how the rollback is performed.
    shards_to_rollback -- shard ids.
    """
    log.info('Reverting update for %s' % shards_to_rollback)
    shards_to_rollback.sort()
    while len(shards_to_rollback) > 0:
      batch_shards = shards_to_rollback[0 : update_config.batchSize]
      shards_to_rollback = list(set(shards_to_rollback) - set(batch_shards))
      resp = self._scheduler.rollbackShards(self._role, self._job_name, batch_shards,
          self._update_token, self._session)
      self.watch_tasks(batch_shards, update_config.restartThreshold, update_config.watchSecs)
      log.log(debug_if(resp == UpdateResponseCode.OK),
        'Response from scheduler: %s (message: %s)'
          % (UpdateResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))

  def restart_shards(self, shard_ids, update_config):
    """Performs a scheduler call for restart.

    Arguments:
    shard_ids -- set of shards to be restarted by the scheduler.
    update_config -- update configuration object that describes how the restart is performed.

    Returns a map of the current status of the restarted shards as returned by the scheduler.
    """
    log.info('Restarting shards: %s' % shard_ids)
    return self._scheduler.updateShards(self._role, self._job_name, shard_ids,
        self._update_token, self._session)

  def watch_tasks(self, task_ids, restart_threshold, watch_secs):
    """Monitors the restarted shards.

    Arguments:
    task_ids -- set of shards to watch.
    restart_threshold -- Maximum number of seconds before which a task must move
                         to the RUNNING state.
    watch_secs -- Number of seconds to watch the task once it is RUNNING.

    Returns a set of tasks that failed to meet the following criteria,
    1. Failed to move to RUNNING state before restart_threshold from the time of restart.
    2. Failed to stay in the RUNNING state before watch_secs expire.
    """
    # TODO(Sathya): Consider adding PREEMPTING as an active state.
    ACTIVE_STATES = set([ScheduleStatus.PENDING, ScheduleStatus.STARTING, ScheduleStatus.RUNNING,
        ScheduleStatus.ASSIGNED, ScheduleStatus.UPDATING, ScheduleStatus.ROLLBACK])
    start_time = self._clock.time()
    expected_running_by = start_time + restart_threshold
    statuses = {}
    running_state_times = {}
    healthy_tasks = set()
    failed_shards = set()
    while True:
      log.debug('Getting status...')
      query = TaskQuery()
      query.owner = Identity(role=self._role)
      query.jobName = self._job_name
      resp = self._scheduler.getTasksStatus(query)
      log.debug('Response from scheduler: %s (message: %s)'
          % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
      if resp.tasks:
        for task in resp.tasks:
          statuses[task.assignedTask.task.shardId] = task.status
      else:
        log.debug('No tasks found.')
      log.debug('Got statuses: %s' %
          dict([(task, ScheduleStatus._VALUES_TO_NAMES[status])
          for task, status in statuses.iteritems()]))
      now = self._clock.time()
      for task_id, status in statuses.items():
        if status is ScheduleStatus.RUNNING and task_id not in (
          failed_shards.union(running_state_times.keys())):
          running_state_times[task_id] = now
          log.debug('Adding task-%s to running tasks' % task_id)
      if now > expected_running_by:
        non_running_tasks = [id for id in task_ids if id not in running_state_times]
        log.log(debug_if(non_running_tasks == []),
          'Tasks failed to move into running: %s' %
            (non_running_tasks if non_running_tasks != [] else 'None'))
        failed_shards.update(non_running_tasks)
      healthy_tasks.update(id for id in statuses
          if id in running_state_times and now > running_state_times[id] + watch_secs)
      failed_shards.update(id for id in statuses
          if id not in healthy_tasks and statuses[id] not in ACTIVE_STATES)
      if healthy_tasks.union(failed_shards) == set(task_ids):
        return [shard for shard in failed_shards]
      elif now > (start_time + restart_threshold + watch_secs):
        return [shard for shard in set(task_ids).difference(healthy_tasks)]
      self._clock.sleep(app.get_options().mesos_updater_status_check_interval)
<|endoftext|>


<commit_before>import getpass
import os

""" Constants """

'''configurations'''
IS_SERVER = False
DUMPING = True

'''file dir'''
FILE_CONTENTS_FOLDER = 'filecontents/'
RESULTS_FOLDER = 'analysis_results/'
WORKSPACE_DIR = 'workspace/'
# handle the temp dir in windows
TMP_FOLDER = os.path.expanduser(
    '~\AppData\Local\Temp') if os.name == 'nt' else '/tmp/'
UPLOAD_FOLDER = os.path.join(TMP_FOLDER, 'Lexos_' + str(getpass.getuser()))
CACHE_FOLDER = os.path.join(UPLOAD_FOLDER, 'cache/')
RESOURCE_DIR = "resources/"

'''file name'''
FILEMANAGER_FILENAME = 'filemanager.p'
SESSION_FILENAME = 'session.p'
DENDROGRAM_NEWICK_FILENAME = 'newNewickStr.txt'
FILE_INFORMATION_FIGNAME = 'statistic.svg'
CORPUS_INFORMATION_FIGNAME = 'corpus_statistic.svg'
WORKSPACE_FILENAME = 'workspace.lexos'
PCA_SMALL_GRAPH_FILENAME = 'small_PCA.html'
PCA_BIG_GRAPH_FILENAME = 'big_PCA.html'
KMEANS_GRAPH_FILENAME = 'kmeans.svg'
MALLET_INPUT_FILE_NAME = 'topicfile'
MALLET_OUTPUT_FILE_NAME = 'topicfile_for_json'
TOPWORD_CSV_FILE_NAME = 'topwordResult.csv'
DEBUG_LOG_FILE_NAME = 'debug.log'
MUFI_3_FILENAME = 'MUFI_3_DICT.tsv'
MUFI_4_FILENAME = 'MUFI_4_DICT.tsv'
STOPWORD_FILENAME = 'stopwords.p'
LEMMA_FILENAME = 'lemmas.p'
CONSOLIDATION_FILENAME = 'consolidations.p'
SPECIAL_CHAR_FILENAME = 'specialchars.p'
DIGIT_MAP_FILENAME = 'digitmap.p'
PUNCTUATION_MAP_FILENAME = 'punctuationmap.p'

'''constant numbers'''
MAX_FILE_SIZE = 250 * 1024 * 1024  # 250 MB
MAX_FILE_SIZE_INT = 250
MAX_FILE_SIZE_UNITS = "M"
# note: number of characters in a preview screen (e.g., on Select page)
PREVIEW_SIZE = 500
# minimum number of characters used to detect a file's encoding scheme
# upon upload
MIN_ENCODING_DETECT = 10000
MIN_NEWLINE_DETECT = 1000
CHARACTERS_PER_LINE_IN_LEGEND = 100
ROUND_DIGIT = 4
DENDRO_TITLE_FONT_SIZE = 15
DENDRO_LEGEND_FONT_SIZE = 10
DENDRO_LEGEND_X = 0
DENDRO_LEGEND_Y = 1.05
DENDRO_CHARACTERS_PER_LINE_IN_TITLE = 80
DENDRO_MAX_LINES_PER_PAGE = 80
DENDRO_MAX_LABELS_LENGTH = 15

'''secret key <not functional for now>'''
FILEMANAGER_KEY = ''  # the key you use to encrypt your file manager
SESSION_KEY = ''  # the key you use to encrypt your session
FILE_CONTENT_KEY = ''  # the key you use to encrypt you file content

'''session caching option'''
# for general settings
GENERALSETTINGS = ('beta_onbox')

# for scrub
SCRUBBOXES = (
    'ampersandbox',
    'aposbox',
    'digitsbox',
    'hyphensbox',
    'lowercasebox',
    'newlinesbox',
    'punctuationbox',
    'spacesbox',
    'tabsbox',
    'tagbox',
    'whitespacebox',
    'sw_option',
)
SCRUBINPUTS = (
    'manualstopwords',
    'manualspecialchars',
    'manualconsolidations',
    'manuallemmas')
OPTUPLOADNAMES = (
    'swfileselect[]',
    'lemfileselect[]',
    'consfileselect[]',
    'scfileselect[]')

# for xml handling

# for cut
WHITESPACE = ['\n', '\t', ' ', '', '\u3000']

CUTINPUTAREAS = (
    'cut_type',
    'lastprop',
    'overlap',
    'cutting_value',
    'cutsetnaming')

# for base analyze
ANALYZEBOXES = ('mfwcheckbox', 'cullcheckbox', 'greyword', 'inWordsOnly')
ANALYZEINPUTS = (
    'tokenSize',
    'tokenType',
    'normalizeType',
    'norm',
    'mfwnumber',
    'cullnumber')

# for rolling window
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
RWBOXES = ('rollinghasmilestone', 'hideDots', 'BWoutput')
RWINPUTS = (
    'filetorollinganalyze',
    'counttype',
    'windowtype',
    'inputtype',
    'rollingsearchword',
    'rollingsearchwordopt',
    'rollingwindowsize',
    'rollingmilestonetype')

# for word cloud and multicloud and bubbleviz
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
CLOUDLIST = ('segmentlist',)

# for word cloud
# for multicloud
MULTICLOUDINPUTS = ('analysistype',)
MULTICLOUDFILES = ('optuploadname',)

# for BubbleViz
BUBBLEVIZBOX = ('vizmaxwords',)
BUBBLEVIZINPUT = ('minlength', 'graphsize', 'maxwords')

# for hierarchical Clustering
HIERARCHICALBOX = ('augmented', 'dendroLegends')
HIERARCHICALINPUT = (
    'metric',
    'linkage',
    'title',
    'orientation',
    'pruning',
    'criterion')

# for kmeans Clustering
KMEANINPUT = (
    'nclusters',
    'max_iter',
    'init',
    'n_init',
    'tolerance',
    'KMeans_metric',
    'viz')
# for similarity query
SIMINPUT = ('uploadname',)
SIMBOX = ('simsuniquetokens',)

# for topword
TOPWORDINPUT = (
    'testInput',
    'groupOptionType',
    'outlierMethodType',
    "outlierTypeStd",
    "outlierTypeIQR",
    'lowerboundPC',
    'upperboundPC',
    'lowerboundRC',
    'upperboundRC',
    'useFreq')

# for statistic
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
STATISTIC_LIST = ('segmentlist',)

'''the request form default value'''
DEFAULT_GENERALSETTINGS_OPTIONS = {'beta_onbox': False}

DEFAULT_SCRUB_OPTIONS = {
    'aposbox': False,
    'ampersandbox': False,
    'digitsbox': True,
    'hyphensbox': False,
    'lowercasebox': True,
    'newlinesbox': True,
    'punctuationbox': True,
    'tabsbox': True,
    'spacesbox': True,
    'whitespacebox': False,
    'tagbox': False,
    'manualstopwords': '',
    'sw_option': 'off',
    'manualspecialchars': '',
    'manualconsolidations': '',
    'manuallemmas': '',
    'entityrules': 'default',
    'optuploadnames': {
        'swfileselect[]': '',
        'lemfileselect[]': '',
        'consfileselect[]': '',
        'scfileselect[]': ''}}

DEFAULT_CUT_OPTIONS = {
    'cutType': 'words', 'cutValue': '', 'cutOverlap': '0', 'cutLastProp': '50'
}

DEFAULT_TOKENIZER_OPTIONS = {
    'tokenizerOption': 'fileColumn'
}

DEFAULT_ROLLINGWINDOW_OPTIONS = {
    'rollinghasmilestone': False,
    'hideDots': False,
    'BWoutput': False,
    'filetorollinganalyze': '',
    'counttype': 'average',
    'windowtype': 'letter',
    'inputtype': 'string',
    'rollingsearchword': '',
    'rollingsearchwordopt': '',
    'rollingwindowsize': '',
    'rollingmilestonetype': ''}

DEFAULT_ANALYZE_OPTIONS = {
    'tokenSize': '1',
    'tokenType': 'word',
    'normalizeType': 'freq',
    'norm': 'l0',
    'mfwcheckbox': False,
    'mfwnumber': '100',
    'cullcheckbox': False,
    'cullnumber': '1',
    'greyword': False}

DEFAULT_CLOUD_OPTIONS = {'segmentlist': []}

DEFAULT_MULTICLOUD_OPTIONS = {'optuploadname': '', 'analysistype': 'userfiles'}

DEFAULT_BUBBLEVIZ_OPTIONS = {
    'minlength': '0',
    'graphsize': '800',
    'maxwords': '100'}

DEFAULT_HIERARCHICAL_OPTIONS = {
    'metric': 'euclidean',
    'linkage': 'average',
    'title': '',
    'orientation': 'bottom',
    'pruning': '',
    'augmented': True,
    'criterion': 'maxclust',
    'threshold': '',
    'dendroLegends': False,
    'degenerated': False}

DEFAULT_KMEAN_OPTIONS = {
    'nclusters': '',
    'max_iter': '',
    'init': 'k-means++',
    'n_init': '',
    'tolerance': '',
    'KMeans_metric': '',
    'viz': 'Voronoi'}

DEFAULT_SIM_OPTIONS = {'uploadname': '', 'simsuniquetokens': True}

DEFAULT_TOPWORD_OPTIONS = {
    'testInput': 'classToPara',
    'groupOptionType': 'all',
    'outlierMethodType': 'stdErr',
    "outlierTypeStd": 'top',
    "outlierTypeIQR": 'top',
    'lowerboundPC': '0',
    'upperboundPC': '1',
    'lowerboundRC': '0',
    'upperboundRC': '0',
    'useFreq': 'PC'}

# never used, go to lexos_core.py `statistic()` function to see why
DEFAULT_STATISTICS_OPTIONS = {}

DEFAULT_XMLHANDLING_OPTION = {'Remove Tag Only': 'foo'}

'''do not cache options'''
SESSION_DO_NOT_CACHE = {}
WORKSPACE_DO_NOT_CACHE = {}
<commit_msg>Constant name fixing<commit_after>import getpass
import os

""" Constants """

'''configurations'''
IS_SERVER = False
DUMPING = True

'''file dir'''
FILE_CONTENTS_FOLDER = 'filecontents/'
RESULTS_FOLDER = 'analysis_results/'
WORKSPACE_DIR = 'workspace/'
# handle the temp dir in windows
TMP_FOLDER = os.path.expanduser(
    '~\AppData\Local\Temp') if os.name == 'nt' else '/tmp/'
UPLOAD_FOLDER = os.path.join(TMP_FOLDER, 'Lexos_' + str(getpass.getuser()))
CACHE_FOLDER = os.path.join(UPLOAD_FOLDER, 'cache/')
RESOURCE_DIR = "resources/"

'''file name'''
FILEMANAGER_FILENAME = 'filemanager.p'
SESSION_FILENAME = 'session.p'
DENDROGRAM_NEWICK_FILENAME = 'newNewickStr.txt'
FILE_INFORMATION_FIGNAME = 'statistic.svg'
CORPUS_INFORMATION_FIGNAME = 'corpus_statistic.svg'
WORKSPACE_FILENAME = 'workspace.lexos'
PCA_SMALL_GRAPH_FILENAME = 'small_PCA.html'
PCA_BIG_GRAPH_FILENAME = 'big_PCA.html'
KMEANS_GRAPH_FILENAME = 'kmeans.svg'
MALLET_INPUT_FILE_NAME = 'topicfile'
MALLET_OUTPUT_FILE_NAME = 'topicfile_for_json'
TOPWORD_CSV_FILE_NAME = 'topwordResult.csv'
DEBUG_LOG_FILE_NAME = 'debug.log'
MUFI_3_FILENAME = 'MUFI_3_DICT.tsv'
MUFI_4_FILENAME = 'MUFI_4_DICT.tsv'
STOPWORD_FILENAME = 'stopwords.p'
LEMMA_FILENAME = 'lemmas.p'
CONSOLIDATION_FILENAME = 'consolidations.p'
SPECIAL_CHAR_FILENAME = 'specialchars.p'
DIGIT_MAP_FILENAME = 'digitmap.p'
PUNCTUATION_MAP_FILENAME = 'punctuationmap.p'

'''constant numbers'''
MAX_FILE_SIZE = 250 * 1024 * 1024  # 250 MB
MAX_FILE_SIZE_INT = 250
MAX_FILE_SIZE_UNITS = "M"
# note: number of characters in a preview screen (e.g., on Select page)
PREVIEW_SIZE = 500
# minimum number of characters used to detect a file's encoding scheme
# upon upload
MIN_ENCODING_DETECT = 10000
MIN_NEWLINE_DETECT = 1000
CHARACTERS_PER_LINE_IN_LEGEND = 100
ROUND_DIGIT = 4
DENDRO_TITLE_FONT_SIZE = 15
DENDRO_LEGEND_FONT_SIZE = 10
DENDRO_LEGEND_X = 0
DENDRO_LEGEND_Y = 1.05
DENDRO_CHARACTERS_PER_LINE_IN_TITLE = 80
DENDRO_MAX_LINES_PER_PAGE = 80
DENDRO_MAX_LABELS_LENGTH = 15

'''secret key <not functional for now>'''
FILEMANAGER_KEY = ''  # the key you use to encrypt your file manager
SESSION_KEY = ''  # the key you use to encrypt your session
FILE_CONTENT_KEY = ''  # the key you use to encrypt you file content

'''session caching option'''
# for general settings
GENERALSETTINGS = ('beta_onbox')

# for scrub
SCRUBBOXES = (
    'ampersandbox',
    'aposbox',
    'digitsbox',
    'hyphensbox',
    'lowercasebox',
    'newlinesbox',
    'punctuationbox',
    'spacesbox',
    'tabsbox',
    'tagbox',
    'whitespacebox',
    'sw_option',
)
SCRUBINPUTS = (
    'manualstopwords',
    'manualspecialchars',
    'manualconsolidations',
    'manuallemmas')
OPTUPLOADNAMES = (
    'swfileselect[]',
    'lemfileselect[]',
    'consfileselect[]',
    'scfileselect[]')

# for xml handling

# for cut
WHITESPACE = ['\n', '\t', ' ', '', '\u3000']

CUTINPUTAREAS = (
    'cut_type',
    'lastprop',
    'overlap',
    'cutting_value',
    'cutsetnaming')

# for base analyze
ANALYZEBOXES = ('mfwcheckbox', 'cullcheckbox', 'greyword', 'inWordsOnly')
ANALYZEINPUTS = (
    'tokenSize',
    'tokenType',
    'normalizeType',
    'norm',
    'mfwnumber',
    'cullnumber')

# for rolling window
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
RWBOXES = ('rollinghasmilestone', 'hideDots', 'BWoutput')
RWINPUTS = (
    'filetorollinganalyze',
    'counttype',
    'windowtype',
    'inputtype',
    'rollingsearchword',
    'rollingsearchwordopt',
    'rollingwindowsize',
    'rollingmilestonetype')

# for word cloud and multicloud and bubbleviz
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
CLOUDLIST = ('segmentlist',)

# for word cloud
# for multicloud
MULTICLOUDINPUTS = ('analysistype',)
MULTICLOUDFILES = ('optuploadname',)

# for BubbleViz
BUBBLEVIZBOX = ('vizmaxwords',)
BUBBLEVIZINPUT = ('minlength', 'graphsize', 'maxwords')

# for hierarchical Clustering
HIERARCHICALBOX = ('augmented', 'dendroLegends')
HIERARCHICALINPUT = (
    'metric',
    'linkage',
    'title',
    'orientation',
    'pruning',
    'criterion')

# for kmeans Clustering
KMEANINPUT = (
    'nclusters',
    'max_iter',
    'init',
    'n_init',
    'tolerance',
    'KMeans_metric',
    'viz')
# for similarity query
SIMINPUT = ('uploadname',)
SIMBOX = ('simsuniquetokens',)

# for topword
TOPWORDINPUT = (
    'testInput',
    'groupOptionType',
    'outlierMethodType',
    "outlierTypeStd",
    "outlierTypeIQR",
    'lowerboundPC',
    'upperboundPC',
    'lowerboundRC',
    'upperboundRC',
    'useFreq')

# for statistic
# if there is no comma in the end, python recognize this var as a string
# instead of a tuple
STATISTIC_LIST = ('segmentlist',)

'''the request form default value'''
DEFAULT_GENERALSETTINGS_OPTIONS = {'beta_onbox': False}

DEFAULT_SCRUB_OPTIONS = {
    'aposbox': False,
    'ampersandbox': False,
    'digitsbox': True,
    'hyphensbox': False,
    'lowercasebox': True,
    'newlinesbox': True,
    'punctuationbox': True,
    'tabsbox': True,
    'spacesbox': True,
    'whitespacebox': False,
    'tagbox': False,
    'manualstopwords': '',
    'sw_option': 'off',
    'manualspecialchars': '',
    'manualconsolidations': '',
    'manuallemmas': '',
    'entityrules': 'default',
    'optuploadnames': {
        'swfileselect[]': '',
        'lemfileselect[]': '',
        'consfileselect[]': '',
        'scfileselect[]': ''}}

DEFAULT_CUT_OPTIONS = {
    'cutType': 'words', 'cutValue': '', 'cutOverlap': '0', 'cutLastProp': '50'
}

DEFAULT_TOKENIZER_OPTIONS = {
    'tableOrientation': 'fileColumn'
}

DEFAULT_ROLLINGWINDOW_OPTIONS = {
    'rollinghasmilestone': False,
    'hideDots': False,
    'BWoutput': False,
    'filetorollinganalyze': '',
    'counttype': 'average',
    'windowtype': 'letter',
    'inputtype': 'string',
    'rollingsearchword': '',
    'rollingsearchwordopt': '',
    'rollingwindowsize': '',
    'rollingmilestonetype': ''}

DEFAULT_ANALYZE_OPTIONS = {
    'tokenSize': '1',
    'tokenType': 'word',
    'normalizeType': 'freq',
    'norm': 'l0',
    'mfwcheckbox': False,
    'mfwnumber': '100',
    'cullcheckbox': False,
    'cullnumber': '1',
    'greyword': False}

DEFAULT_CLOUD_OPTIONS = {'segmentlist': []}

DEFAULT_MULTICLOUD_OPTIONS = {'optuploadname': '', 'analysistype': 'userfiles'}

DEFAULT_BUBBLEVIZ_OPTIONS = {
    'minlength': '0',
    'graphsize': '800',
    'maxwords': '100'}

DEFAULT_HIERARCHICAL_OPTIONS = {
    'metric': 'euclidean',
    'linkage': 'average',
    'title': '',
    'orientation': 'bottom',
    'pruning': '',
    'augmented': True,
    'criterion': 'maxclust',
    'threshold': '',
    'dendroLegends': False,
    'degenerated': False}

DEFAULT_KMEAN_OPTIONS = {
    'nclusters': '',
    'max_iter': '',
    'init': 'k-means++',
    'n_init': '',
    'tolerance': '',
    'KMeans_metric': '',
    'viz': 'Voronoi'}

DEFAULT_SIM_OPTIONS = {'uploadname': '', 'simsuniquetokens': True}

DEFAULT_TOPWORD_OPTIONS = {
    'testInput': 'classToPara',
    'groupOptionType': 'all',
    'outlierMethodType': 'stdErr',
    "outlierTypeStd": 'top',
    "outlierTypeIQR": 'top',
    'lowerboundPC': '0',
    'upperboundPC': '1',
    'lowerboundRC': '0',
    'upperboundRC': '0',
    'useFreq': 'PC'}

# never used, go to lexos_core.py `statistic()` function to see why
DEFAULT_STATISTICS_OPTIONS = {}

DEFAULT_XMLHANDLING_OPTION = {'Remove Tag Only': 'foo'}

'''do not cache options'''
SESSION_DO_NOT_CACHE = {}
WORKSPACE_DO_NOT_CACHE = {}
<|endoftext|>


<commit_before>import numpy as np
import libsvm

_kernel_types = ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']
_svm_types = ['c_svc', 'nu_svc', 'one_class', 'epsilon_svr', 'nu_svr']

class BaseSVM(object):
    """
    Base class for classifiers that use support vector machine.

    Should not be used directly, use derived classes instead

    Parameters
    ----------
    X : array-like, shape = [N, D]
        It will be converted to a floating-point array.
    y : array, shape = [N]
        target vector relative to X
        It will be converted to a floating-point array.
    """

    def __init__(self, svm, kernel, degree, gamma, coef0, cache_size,
                 eps, C, nr_weight, nu, p, shrinking, probability,
                 scale):
        self.svm = _svm_types.index(svm)
        self.kernel = _kernel_types.index(kernel)
        self.degree = degree
        self.gamma = gamma
        self.coef0 = coef0
        self.cache_size = cache_size
        self.eps = eps
        self.C = C
        self.nr_weight = 0
        self.nu = nu
        self.p = p
        self.shrinking = shrinking
        self.probability = probability
        self.scale = scale

    def fit(self, X, y):
        """
        should empty arrays created be order='C' ?
        """
        X = np.asanyarray(X, dtype=np.float, order='C')
        y = np.asanyarray(y, dtype=np.float, order='C')

        if self.scale:
            self.mean = X.mean(0)
            self.std = X.std(0)
            X = (X - self.mean) / self.std

        # check dimensions
        if X.shape[0] != y.shape[0]: raise ValueError("Incompatible shapes")

        if (self.gamma == 0): self.gamma = 1.0/X.shape[0]
        self.coef_, self.rho_, self.support_, self.nclass_, self.nSV_, self.label_  = \
             libsvm.train_wrap(X, y, self.svm, self.kernel, self.degree,
                 self.gamma, self.coef0, self.eps, self.C, self.nr_weight,
                 np.empty(0, dtype=np.int), np.empty(0, dtype=np.float), self.nu,
                 self.cache_size, self.p, self.shrinking, self.probability)
        return self

    def predict(self, T):
        T = np.asanyarray(T, dtype=np.float, order='C')
        if self.scale: T = (T - self.mean) / self.std
        return libsvm.predict_from_model_wrap(T, self.support_,
                      self.coef_, self.rho_, self.svm,
                      self.kernel, self.degree, self.gamma,
                      self.coef0, self.eps, self.C, self.nr_weight,
                      np.empty(0, dtype=np.int), np.empty(0,
                      dtype=np.float), self.nu, self.cache_size,
                      self.p, self.shrinking, self.probability,
                      self.nclass_, self.nSV_, self.label_)

def predict(X, y, T, svm='c_svc', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0):
    """
    Shortcut that does fit and predict in a single step.

    Should be faster than instatating the object, since less copying is done.

    Parameters
    ----------
    X : array-like
        data points
    y : array
        targets
    T : array
        test points

    Optional Parameters
    -------------------
    TODO

    Examples
    --------
    """
    X = np.asanyarray(X, dtype=np.float, order='C')
    y = np.asanyarray(y, dtype=np.float, order='C')
    T = np.asanyarray(T, dtype=np.float, order='C')
    if X.shape[0] != y.shape[0]: raise ValueError("Incompatible shapes")
    return libsvm.predict_wrap(X, y, T, _svm_types.index(svm),
                               _kernel_types.index(kernel),
                               degree, gamma, coef0, eps, C,
                               nr_weight, np.empty(0, dtype=np.int),
                               np.empty(0, dtype=np.float), nu,
                               cache_size, p, shrinking, probability)

###
# Public API
# No processing should go into these classes

class SVC(BaseSVM):
    """
    Support Vector Classification

    Implementats C-SVC, nu-SVC

    Parameters
    ----------
    X : array-like, shape = [nsamples, nfeatures]
        Training vector, where nsamples in the number of samples and
        nfeatures is the number of features.
    Y : array, shape = [nsamples]
        Target vector relative to X

    Optional Parameters
    -------------------
    impl : string
        SVM implementation to choose from. This refers to different
        formulations of the SVM optimization problem.
        Can be one of 'c_svc', 'nu_svc'. By default 'c_svc' will be chosen.

    scale : boolean
        Scale data before fitting the model. Caution: if activated,
        estimated parameters will also be scaled.
        Default is True.

    nu: float
        An upper bound on the fraction of training errors and a lower
        bound of the fraction of support vectors. Should be in the
        interval (0, 1].
        By default 0.5 will be taken.
        Only available is impl is set to 'nu_svc'

    kernel : string
         Specifies the kernel type to be used in the algorithm.
         one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'.
         If none is given 'rbf' will be used.

    degree : int
        degree of kernel function
        is significant only in POLY, RBF, SIGMOID

    Members
    -------
    support_ : array-like, shape = [nSV, D]
        estimated support vectors.
        where nSV is the number of support vectors, D is the dimension
        of the underlying space.

    coef_ : array
        coefficient of the support vector in the decission function.

    rho_ : array
        constants in decision function

    Examples
    --------
    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> y = np.array([1, 1, 2, 2])
    >>> clf = SVM()
    >>> clf.fit(X, y)    #doctest: +ELLIPSIS
    <scikits.learn.svm.svm.SVM object at 0x...>
    >>> print clf.predict([[-0.8, -1]])
    [ 1.]

    References
    ----------
    http://scikit-learn.sourceforge.net/doc/modules/svm.html
    http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf
    """
    def __init__(self, impl='c_svc', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0, scale=True):

        BaseSVM.__init__(self, impl, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability, scale)

class SVR(BaseSVM):
    """
    Support Vector Regression.

    Parameters
    ----------
    X : array-like, shape = [N, D]
        Training vector
    Y : array, shape = [N]
        Target vector relative to X

    """
    def __init__(self, svm='epsilon_svr', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0, scale=True):
        BaseSVM.__init__(self, svm, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability, scale)

class OneClassSVM(BaseSVM):
    """
    Outlayer detection
    """
    def __init__(self, kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0, scale=True):
        svm = 'one_class'
        BaseSVM.__init__(self, svm, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability, scale)
<commit_msg>SVM refactoring<commit_after>import numpy as np
import libsvm

_kernel_types = ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']
_svm_types = ['c_svc', 'nu_svc', 'one_class', 'epsilon_svr', 'nu_svr']

class BaseSVM(object):
    """
    Base class for classifiers that use support vector machine.

    Should not be used directly, use derived classes instead

    Parameters
    ----------
    X : array-like, shape = [N, D]
        It will be converted to a floating-point array.
    y : array, shape = [N]
        target vector relative to X
        It will be converted to a floating-point array.
    """

    def __init__(self, svm, kernel, degree, gamma, coef0, cache_size,
                 eps, C, nr_weight, nu, p, shrinking, probability):
        self.svm = _svm_types.index(svm)
        self.kernel = _kernel_types.index(kernel)
        self.degree = degree
        self.gamma = gamma
        self.coef0 = coef0
        self.cache_size = cache_size
        self.eps = eps
        self.C = C
        self.nr_weight = 0
        self.nu = nu
        self.p = p
        self.shrinking = shrinking
        self.probability = probability

    def fit(self, X, y):
        """
        should empty arrays created be order='C' ?
        """
        X = np.asanyarray(X, dtype=np.float, order='C')
        y = np.asanyarray(y, dtype=np.float, order='C')

        # check dimensions
        if X.shape[0] != y.shape[0]: raise ValueError("Incompatible shapes")

        if (self.gamma == 0): self.gamma = 1.0/X.shape[0]
        self.coef_, self.rho_, self.support_, self.nclass_, self.nSV_, self.label_  = \
             libsvm.train_wrap(X, y, self.svm, self.kernel, self.degree,
                 self.gamma, self.coef0, self.eps, self.C, self.nr_weight,
                 np.empty(0, dtype=np.int), np.empty(0, dtype=np.float), self.nu,
                 self.cache_size, self.p, self.shrinking, self.probability)
        return self

    def predict(self, T):
        T = np.asanyarray(T, dtype=np.float, order='C')
        return libsvm.predict_from_model_wrap(T, self.support_,
                      self.coef_, self.rho_, self.svm,
                      self.kernel, self.degree, self.gamma,
                      self.coef0, self.eps, self.C, self.nr_weight,
                      np.empty(0, dtype=np.int), np.empty(0,
                      dtype=np.float), self.nu, self.cache_size,
                      self.p, self.shrinking, self.probability,
                      self.nclass_, self.nSV_, self.label_)

def predict(X, y, T, svm='c_svc', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0):
    """
    Shortcut that does fit and predict in a single step.

    Should be faster than instatating the object, since less copying is done.

    Parameters
    ----------
    X : array-like
        data points
    y : array
        targets
    T : array
        test points

    Optional Parameters
    -------------------
    TODO

    Examples
    --------
    """
    X = np.asanyarray(X, dtype=np.float, order='C')
    y = np.asanyarray(y, dtype=np.float, order='C')
    T = np.asanyarray(T, dtype=np.float, order='C')
    if X.shape[0] != y.shape[0]: raise ValueError("Incompatible shapes")
    return libsvm.predict_wrap(X, y, T, _svm_types.index(svm),
                               _kernel_types.index(kernel),
                               degree, gamma, coef0, eps, C,
                               nr_weight, np.empty(0, dtype=np.int),
                               np.empty(0, dtype=np.float), nu,
                               cache_size, p, shrinking, probability)

###
# Public API
# No processing should go into these classes

class SVC(BaseSVM):
    """
    Support Vector Classification

    Implementats C-SVC, nu-SVC

    Parameters
    ----------
    X : array-like, shape = [nsamples, nfeatures]
        Training vector, where nsamples in the number of samples and
        nfeatures is the number of features.
    Y : array, shape = [nsamples]
        Target vector relative to X

    impl : string, optional
        SVM implementation to choose from. This refers to different
        formulations of the SVM optimization problem.
        Can be one of 'c_svc', 'nu_svc'. By default 'c_svc' will be chosen.

    nu: float, optional
        An upper bound on the fraction of training errors and a lower
        bound of the fraction of support vectors. Should be in the
        interval (0, 1].
        By default 0.5 will be taken.
        Only available is impl is set to 'nu_svc'

    kernel : string, optional
         Specifies the kernel type to be used in the algorithm.
         one of 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'.
         If none is given 'rbf' will be used.

    degree : int, optional
        degree of kernel function
        is significant only in POLY, RBF, SIGMOID

    Members
    -------
    support_ : array-like, shape = [nSV, D]
        estimated support vectors.
        where nSV is the number of support vectors, D is the dimension
        of the underlying space.

    coef_ : array
        coefficient of the support vector in the decission function.

    rho_ : array
        constants in decision function

    Examples
    --------
    >>> X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
    >>> y = np.array([1, 1, 2, 2])
    >>> clf = SVM()
    >>> clf.fit(X, y)    #doctest: +ELLIPSIS
    <scikits.learn.svm.svm.SVM object at 0x...>
    >>> print clf.predict([[-0.8, -1]])
    [ 1.]

    See also
    --------
    http://scikit-learn.sourceforge.net/doc/modules/svm.html

    http://www.csie.ntu.edu.tw/~cjlin/papers/libsvm.pdf
    """
    def __init__(self, impl='c_svc', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0):

        BaseSVM.__init__(self, impl, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability)

class SVR(BaseSVM):
    """
    Support Vector Regression.

    Parameters
    ----------
    X : array-like, shape = [N, D]
        Training vector
    Y : array, shape = [N]
        Target vector relative to X

    """
    def __init__(self, svm='epsilon_svr', kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0):
        BaseSVM.__init__(self, svm, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability)

class OneClassSVM(BaseSVM):
    """
    Outlayer detection
    """
    def __init__(self, kernel='rbf', degree=3,
                 gamma=0.0, coef0=0.0, cache_size=100.0, eps=1e-3,
                 C=1.0, nr_weight=0, nu=0.5, p=0.1, shrinking=1,
                 probability=0):
        svm = 'one_class'
        BaseSVM.__init__(self, svm, kernel, degree, gamma, coef0,
                         cache_size, eps, C, nr_weight, nu, p,
                         shrinking, probability)
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.mail import EmailMessage, EmailMultiAlternatives
from django.template.loader import render_to_string
from django.utils import six

try:
    # Django >=1.5
    from django.contrib.auth import get_user_model
except ImportError:
    # Django < 1.5
    from django.contrib.auth.models import User as ContribAuthUserModel

def get_manager_emails():
    """Get a list of the managers' email addresses."""
    try:
        # Django >=1.5
        user_model = get_user_model()
    except NameError:
        # Django < 1.5
        user_model = ContribAuthUserModel

    staff = user_model.objects.filter(is_staff=True)
    manager_emails = staff.exclude(email='').distinct().values_list('email', flat=True)
    return manager_emails or [m[1] for m in settings.MANAGERS]

def send(sender=None, to=(), cc=(), bcc=(), subject='mail',
         attachments=(), template_name=(), text_template_name=(),
         context=None, **kwargs):
    """
    Render and send a (mail) template.
    if text_template_name is not None then a multipart email will be sent using
    template for the html part and text_template_name for the plain part.
    The context will include any extra_context specified.
    If no sender is specified then the DEFAULT_FROM_EMAIL or SERVER_EMAIL setting will be used.
    Any extra items passed in with kwargs will be added to the email headers.
    """
    to, cc, bcc = map(lambda v: [v] if isinstance(v, six.string_types) else v, [to, cc, bcc])

    if sender is None:
        sender = getattr(settings, 'DEFAULT_FROM_EMAIL', settings.SERVER_EMAIL)

    attachment_list = [[a.name, a.read(), a.content_type] for a in attachments]

    email_kwargs = {
        'from_email': sender,
        'to': to,
        'cc': cc,
        'bcc': bcc,
        'subject': six.text_type(subject),
        'attachments': attachment_list,
        'headers': kwargs,
    }
    if not text_template_name:
        email_kwargs['body'] = render_to_string(template_name, context)
        msg = EmailMessage(**email_kwargs)
    else:
        email_kwargs['body'] = render_to_string(text_template_name, context)
        msg = EmailMultiAlternatives(**email_kwargs)

        html_content = render_to_string(template_name, context)
        msg.attach_alternative(html_content, 'text/html')

    msg.send()
<commit_msg>Clean up the Django <1.5 compatibility code.<commit_after>from django.conf import settings
from django.core.mail import EmailMessage, EmailMultiAlternatives
from django.template.loader import render_to_string
from django.utils import six

try:
    # Django >=1.5
    from django.contrib.auth import get_user_model
except ImportError:
    # Django < 1.5
    from django.contrib.auth.models import User
else:
    User = get_user_model()

def get_manager_emails():
    """Get a list of the managers' email addresses."""
    staff = User.objects.filter(is_staff=True)
    manager_emails = staff.exclude(email='').distinct().values_list('email', flat=True)
    return manager_emails or [m[1] for m in settings.MANAGERS]

def send(sender=None, to=(), cc=(), bcc=(), subject='mail',
         attachments=(), template_name=(), text_template_name=(),
         context=None, **kwargs):
    """
    Render and send a (mail) template.
    if text_template_name is not None then a multipart email will be sent using
    template for the html part and text_template_name for the plain part.
    The context will include any extra_context specified.
    If no sender is specified then the DEFAULT_FROM_EMAIL or SERVER_EMAIL setting will be used.
    Any extra items passed in with kwargs will be added to the email headers.
    """
    to, cc, bcc = map(lambda v: [v] if isinstance(v, six.string_types) else v, [to, cc, bcc])

    if sender is None:
        sender = getattr(settings, 'DEFAULT_FROM_EMAIL', settings.SERVER_EMAIL)

    attachment_list = [[a.name, a.read(), a.content_type] for a in attachments]

    email_kwargs = {
        'from_email': sender,
        'to': to,
        'cc': cc,
        'bcc': bcc,
        'subject': six.text_type(subject),
        'attachments': attachment_list,
        'headers': kwargs,
    }
    if not text_template_name:
        email_kwargs['body'] = render_to_string(template_name, context)
        msg = EmailMessage(**email_kwargs)
    else:
        email_kwargs['body'] = render_to_string(text_template_name, context)
        msg = EmailMultiAlternatives(**email_kwargs)

        html_content = render_to_string(template_name, context)
        msg.attach_alternative(html_content, 'text/html')

    msg.send()
<|endoftext|>


<commit_before># django imports
from django import forms
from django.db import models

# django imports
from django import forms
from django import template
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.mail import send_mail
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_syncdb
from django.template import RequestContext
from django.template.loader import render_to_string
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

# lfc imports
import lfc.utils
from lfc.models import BaseContent

# lfc_contact_form imports
from lfc_contact_form.forms import ContactForm as DjangoContactForm

class ContactForm(BaseContent):
    """Contact form for LFC.
    """
    text = models.TextField(blank=True)
    thank_you_message = models.TextField(blank=True)

    def get_searchable_text(self):
        """Returns the searchable text of the contact form.
        """
        searchable_text = self.title + " " + self.description + " " + self.text
        return lfc.utils.html2text(searchable_text)

    def edit_form(self, **kwargs):
        """Returns the add/edit form of the Blog
        """
        return ContactFormForm(**kwargs)

    def render(self, request):
        """Renders the content of the contact form.
        """
        request_context = self.get_request_context(request)

        portal = lfc.utils.get_portal()
        if request.method == "POST":
            form = DjangoContactForm(data=request.POST)
            if form.is_valid():
                sent = True
                message = render_to_string("lfc_contact_form/mail.html", RequestContext(request, {
                    "form": form,
                }))
                send_mail(
                    subject=_("New mail from %s" % portal.title),
                    message=message,
                    from_email=portal.from_email,
                    recipient_list=portal.get_notification_emails()
                )
            else:
                sent = False
        else:
            form = DjangoContactForm()
            sent = False

        request_context["sent"] = sent
        request_context["form"] = form

        return self.render_to_string(request_context)

class ContactFormForm(forms.ModelForm):
    """The add/edit form of the ContactForm content object.
    """
    class Meta:
        model = ContactForm
        fields = ("title", "display_title", "slug", "description", "text", "thank_you_message")
<commit_msg>Renamed request_context to context<commit_after># django imports
from django import forms
from django.db import models

# django imports
from django import forms
from django import template
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.mail import send_mail
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models.signals import post_syncdb
from django.template import RequestContext
from django.template.loader import render_to_string
from django.utils import translation
from django.utils.translation import ugettext_lazy as _

# lfc imports
import lfc.utils
from lfc.models import BaseContent

# lfc_contact_form imports
from lfc_contact_form.forms import ContactForm as DjangoContactForm

class ContactForm(BaseContent):
    """Contact form for LFC.
    """
    text = models.TextField(blank=True)
    thank_you_message = models.TextField(blank=True)

    def get_searchable_text(self):
        """Returns the searchable text of the contact form.
        """
        searchable_text = self.title + " " + self.description + " " + self.text
        return lfc.utils.html2text(searchable_text)

    def edit_form(self, **kwargs):
        """Returns the add/edit form of the Blog
        """
        return ContactFormForm(**kwargs)

    def render(self, request):
        """Renders the content of the contact form.

        This adds the form and sent to the RequestContext.
        """
        portal = lfc.utils.get_portal()
        if request.method == "POST":
            form = DjangoContactForm(data=request.POST)
            if form.is_valid():
                sent = True
                message = render_to_string("lfc_contact_form/mail.html", RequestContext(request, {
                    "form": form,
                }))
                send_mail(
                    subject=_("New mail from %s" % portal.title),
                    message=message,
                    from_email=portal.from_email,
                    recipient_list=portal.get_notification_emails()
                )
            else:
                sent = False
        else:
            form = DjangoContactForm()
            sent = False

        self.context["sent"] = sent
        self.context["form"] = form

        return super(ContactForm, self).render(request)

class ContactFormForm(forms.ModelForm):
    """The add/edit form of the ContactForm content object.
    """
    class Meta:
        model = ContactForm
        fields = ("title", "display_title", "slug", "description", "text", "thank_you_message")
<|endoftext|>


<commit_before>from scrapers.las import get_las
from scrapers.nts import get_nts
from scrapers.imp import get_sas, get_taktic, get_apde, get_ammp, get_fd, get_ftmp
from scrapers.impprob import get_ipc
from scrapers.impgeom import get_ipgas, get_ipltgs
from scrapers.gcal import get_gcal
from scrapers.helpers import CachedObject, expired, EMPTY_CACHE

getter = {
    'las': get_las,
    'nts': get_nts,
    'ic/sas': get_sas,
    'ic/taktic': get_taktic,
    'ic/apde': get_apde,
    'ic/ammp': get_ammp,
    'ic/fd': get_fd,
    'ic/ftmp': get_ftmp,
    'ic/ipc': get_ipc,
    'ic/ipgas': get_ipgas,
    'ic/ipltgs': get_ipltgs
}

_gcal_cache = {}

def gcal(gcal_id):
    cached = _gcal_cache.get(gcal_id, CachedObject())
    if expired(cached.last_update) or cached.cache == EMPTY_CACHE:
        cached.cache = get_gcal(gcal_id, cached.last_update)
        _gcal_cache[gcal_id] = cached

    return cached.cache

_cache = {}

def custom(path):
    cached = _cache.get(path, CachedObject())
    if expired(cached.last_update) or cached.cache == EMPTY_CACHE:
        if path in getter:
            cached.cache = getter[path](cached.last_update)
            _cache[path] = cached

    return cached.cache
<commit_msg>add cache TODO in scrapers<commit_after>from scrapers.las import get_las
from scrapers.nts import get_nts
from scrapers.imp import get_sas, get_taktic, get_apde, get_ammp, get_fd, get_ftmp
from scrapers.impprob import get_ipc
from scrapers.impgeom import get_ipgas, get_ipltgs
from scrapers.gcal import get_gcal
from scrapers.helpers import CachedObject, expired, EMPTY_CACHE

# TODO: add cache check using sha1 hash of the raw data...

getter = {
    'las': get_las,
    'nts': get_nts,
    'ic/sas': get_sas,
    'ic/taktic': get_taktic,
    'ic/apde': get_apde,
    'ic/ammp': get_ammp,
    'ic/fd': get_fd,
    'ic/ftmp': get_ftmp,
    'ic/ipc': get_ipc,
    'ic/ipgas': get_ipgas,
    'ic/ipltgs': get_ipltgs
}

_gcal_cache = {}

def gcal(gcal_id):
    cached = _gcal_cache.get(gcal_id, CachedObject())
    if expired(cached.last_update) or cached.cache == EMPTY_CACHE:
        cached.cache = get_gcal(gcal_id, cached.last_update)
        _gcal_cache[gcal_id] = cached

    return cached.cache

_cache = {}

def custom(path):
    cached = _cache.get(path, CachedObject())
    if expired(cached.last_update) or cached.cache == EMPTY_CACHE:
        if path in getter:
            cached.cache = getter[path](cached.last_update)
            _cache[path] = cached

    return cached.cache
<|endoftext|>