<commit_before># Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import logging
import os
import traceback

from google.appengine.api import taskqueue
from google.appengine.ext import ndb

from dashboard.common import utils
from dashboard.pinpoint import mann_whitney_u
from dashboard.pinpoint.models import attempt as attempt_module
from dashboard.pinpoint.models import change as change_module
from dashboard.services import gitiles_service
from dashboard.services import issue_tracker_service

# We want this to be fast to minimize overhead while waiting for tasks to
# finish, but don't want to consume too many resources.
_TASK_INTERVAL = 10

_DEFAULT_REPEAT_COUNT = 10
_SIGNIFICANCE_LEVEL = 0.0005

_DIFFERENT = 'different'
_PENDING = 'pending'
_SAME = 'same'
_UNKNOWN = 'unknown'

_ROUND_PUSHPIN = u'\U0001f4cd'
_MIDDLE_DOT = u'\xb7'

def JobFromId(job_id):
  """Get a Job object from its ID. Its ID is just its key as a hex string.

  Users of Job should not have to import ndb. This function maintains an
  abstraction layer that separates users from the Datastore details.
  """
  job_key = ndb.Key('Job', int(job_id, 16))
  return job_key.get()

class Job(ndb.Model):
  """A Pinpoint job."""

  created = ndb.DateTimeProperty(required=True, auto_now_add=True)
  updated = ndb.DateTimeProperty(required=True, auto_now=True)

  # The name of the Task Queue task this job is running on. If it's present, the
  # job is running. The task is also None for Task Queue retries.
  task = ndb.StringProperty()

  # The string contents of any Exception that was thrown to the top level.
  # If it's present, the job failed.
  exception = ndb.StringProperty()

  # Request parameters.
  arguments = ndb.JsonProperty(required=True)

  repeat_count = ndb.IntegerProperty(required=True)

  # If True, the service should pick additional Changes to run (bisect).
  # If False, only run the Changes explicitly added by the user.
  auto_explore = ndb.BooleanProperty(required=True)

  # TODO: The bug id is only used for posting bug comments when a job starts and
  # completes. This probably should not be the responsibility of Pinpoint.
  bug_id = ndb.IntegerProperty()

  state = ndb.PickleProperty(required=True)

  @classmethod
  def New(cls, arguments, quests, auto_explore,
          repeat_count=_DEFAULT_REPEAT_COUNT, bug_id=None):
    repeat_count = repeat_count or _DEFAULT_REPEAT_COUNT
    # Create job.
    return cls(
        arguments=arguments,
        auto_explore=auto_explore,
        repeat_count=repeat_count,
        bug_id=bug_id,
        state=_JobState(quests, repeat_count))

  @property
  def job_id(self):
    return '%x' % self.key.id()

  @property
  def status(self):
    if self.task:
      return 'Running'

    if self.exception:
      return 'Failed'

    return 'Completed'

  @property
  def url(self):
    return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)

  def AddChange(self, change):
    self.state.AddChange(change)

  def Start(self):
    self.Schedule()
    self._PostBugComment('started')

  def Complete(self):
    self._PostBugComment('completed')

  def Fail(self):
    self.exception = traceback.format_exc()
    self._PostBugComment('stopped with an error')

  def Schedule(self):
    task = taskqueue.add(queue_name='job-queue', url='/api/run/' + self.job_id,
                         countdown=_TASK_INTERVAL)
    self.task = task.name

  def Run(self):
    self.exception = None  # In case the Job succeeds on retry.
    self.task = None  # In case an exception is thrown.

    try:
      if self.auto_explore:
        self.state.Explore()
      work_left = self.state.ScheduleWork()

      # Schedule moar task.
      if work_left:
        self.Schedule()
      else:
        self.Complete()
    except BaseException:
      self.Fail()
      raise

  def StatusDict(self):
    """A shorter version of AsDict() for faster Jobs list loading."""
    return {
        'job_id': self.job_id,

        'arguments': self.arguments,
        'auto_explore': self.auto_explore,

        'created': self.created.isoformat(),
        'updated': self.updated.isoformat(),
        'exception': self.exception,
        'status': self.status,
    }

  def AsDict(self):
    d = self.StatusDict()
    d['state'] = self.state.AsDict()
    return d

  def _PostBugComment(self, status):
    if not self.bug_id:
      return

    title = '%s Pinpoint job %s.' % (_ROUND_PUSHPIN, status)
    header = '\n'.join((title, self.url))

    # Include list of Changes.
    change_details = []
    for _, change in self.state.Differences():
      # TODO: Only show the most specific Dep.
      # TODO: Store the commit info in the Dep.
      for dep in change.all_deps:
        commit_info = gitiles_service.CommitInfo(dep.repository, dep.git_hash)
        subject = commit_info['message'].split('\n', 1)[0]
        author = commit_info['author']['email']
        time = commit_info['committer']['time']

        byline = 'By %s %s %s' % (author, _MIDDLE_DOT, time)
        git_link = dep.repository + '@' + dep.git_hash
        change_details.append('\n'.join((subject, byline, git_link)))

    comment = '\n\n'.join([header] + change_details)

    issue_tracker = issue_tracker_service.IssueTrackerService(
        utils.ServiceAccountHttp())
    issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)

class _JobState(object):
  """The internal state of a Job.

  Wrapping the entire internal state of a Job in a PickleProperty allows us to
  use regular Python objects, with constructors, dicts, and object references.

  We lose the ability to index and query the fields, but it's all internal
  anyway. Everything queryable should be on the Job object.
  """

  def __init__(self, quests, repeat_count):
    """Create a _JobState.

    Args:
      quests: A sequence of quests to run on each Change.
      repeat_count: The number of attempts to automatically run per Change.
    """
    # _quests is mutable. Any modification should mutate the existing list
    # in-place rather than assign a new list, because every Attempt references
    # this object and will be updated automatically if it's mutated.
    self._quests = list(quests)

    # _changes can be in arbitrary order. Client should not assume that the
    # list of Changes is sorted in any particular order.
    self._changes = []

    # A mapping from a Change to a list of Attempts on that Change.
    self._attempts = {}

    self._repeat_count = repeat_count

  def AddAttempt(self, change):
    assert change in self._attempts
    self._attempts[change].append(attempt_module.Attempt(self._quests, change))

  def AddChange(self, change, index=None):
    if index:
      self._changes.insert(index, change)
    else:
      self._changes.append(change)

    self._attempts[change] = []
    for _ in xrange(self._repeat_count):
      self.AddAttempt(change)

  def Explore(self):
    """Compare Changes and bisect by adding additional Changes as needed.

    For every pair of adjacent Changes, compare their results as probability
    distributions. If more information is needed to establish statistical
    confidence, add an additional Attempt. If the results are different, find
    the midpoint of the Changes and add it to the Job.

    The midpoint can only be added if the second Change represents a commit that
    comes after the first Change. Otherwise, this method won't explore further.
    For example, if Change A is repo@abc, and Change B is repo@abc + patch,
    there's no way to pick additional Changes to try.
    """
    # Compare every pair of Changes.
    # TODO: The list may Change while iterating through it.
    # TODO: Use JobState.Differences().
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]

      comparison_result = self._Compare(change_a, change_b)
      if comparison_result == _DIFFERENT:
        # Different: Bisect and add an additional Change to the job.
        try:
          midpoint = change_module.Change.Midpoint(change_a, change_b)
        except change_module.NonLinearError:
          midpoint = None
        if midpoint:
          logging.info('Adding Change %s.', midpoint)
          self.AddChange(midpoint, index)
      elif comparison_result == _SAME:
        # The same: Do nothing.
        continue
      elif comparison_result == _UNKNOWN:
        # Unknown: Add an Attempt to the Change with the fewest Attempts.
        change = min(change_a, change_b, key=lambda c: len(self._attempts[c]))
        self.AddAttempt(change)

  def ScheduleWork(self):
    work_left = False
    for attempts in self._attempts.itervalues():
      for attempt in attempts:
        if attempt.completed:
          continue

        attempt.ScheduleWork()
        work_left = True

    return work_left

  def Differences(self):
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]
      if self._Compare(change_a, change_b) == _DIFFERENT:
        yield index, change_b

  def AsDict(self):
    comparisons = []
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]
      comparisons.append(self._Compare(change_a, change_b))

    # result_values is a 3D array. result_values[change][quest] is a list of
    # all the result values for that Change and Quest.
    result_values = []
    for change in self._changes:
      change_result_values = []

      change_results_per_quest = _CombineResultsPerQuest(self._attempts[change])
      for quest in self._quests:
        change_result_values.append(change_results_per_quest[quest])

      result_values.append(change_result_values)

    attempts = []
    for c in self._changes:
      attempts.append([attempt.AsDict() for attempt in self._attempts[c]])

    return {
        'quests': map(str, self._quests),
        'changes': [change.AsDict() for change in self._changes],
        # TODO: Use JobState.Differences().
        'comparisons': comparisons,
        'result_values': result_values,
        'attempts': attempts,
    }

  def _Compare(self, change_a, change_b):
    attempts_a = self._attempts[change_a]
    attempts_b = self._attempts[change_b]

    if any(not attempt.completed for attempt in attempts_a + attempts_b):
      return _PENDING

    # Compare exceptions.
    exceptions_a = tuple(attempt.exception or '' for attempt in attempts_a)
    exceptions_b = tuple(attempt.exception or '' for attempt in attempts_b)

    if _CompareValues(exceptions_a, exceptions_b) == _DIFFERENT:
      return _DIFFERENT

    # Compare values.
    results_a = _CombineResultsPerQuest(attempts_a)
    results_b = _CombineResultsPerQuest(attempts_b)

    if any(_CompareValues(results_a[quest], results_b[quest]) == _DIFFERENT
           for quest in self._quests):
      return _DIFFERENT

    # Here, "the same" means that we fail to reject the null hypothesis. We can
    # never be completely sure that the two Changes have the same results, but
    # we've run everything that we planned to, and didn't detect any difference.
    if (len(attempts_a) >= self._repeat_count and
        len(attempts_b) >= self._repeat_count):
      return _SAME

    return _UNKNOWN

def _CombineResultsPerQuest(attempts):
  aggregate_results = collections.defaultdict(list)
  for attempt in attempts:
    if not attempt.completed:
      continue

    for quest, results in attempt.result_values.iteritems():
      aggregate_results[quest] += results

  return aggregate_results

def _CompareValues(values_a, values_b):
  if not (values_a and values_b):
    return _UNKNOWN

  try:
    p_value = mann_whitney_u.MannWhitneyU(values_a, values_b)
  except ValueError:
    return _UNKNOWN

  if p_value < _SIGNIFICANCE_LEVEL:
    return _DIFFERENT
  else:
    return _UNKNOWN
<commit_msg>[pinpoint] Change Job.exception from a StringProperty to a TextProperty.<commit_after># Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import logging
import os
import traceback

from google.appengine.api import taskqueue
from google.appengine.ext import ndb

from dashboard.common import utils
from dashboard.pinpoint import mann_whitney_u
from dashboard.pinpoint.models import attempt as attempt_module
from dashboard.pinpoint.models import change as change_module
from dashboard.services import gitiles_service
from dashboard.services import issue_tracker_service

# We want this to be fast to minimize overhead while waiting for tasks to
# finish, but don't want to consume too many resources.
_TASK_INTERVAL = 10

_DEFAULT_REPEAT_COUNT = 10
_SIGNIFICANCE_LEVEL = 0.0005

_DIFFERENT = 'different'
_PENDING = 'pending'
_SAME = 'same'
_UNKNOWN = 'unknown'

_ROUND_PUSHPIN = u'\U0001f4cd'
_MIDDLE_DOT = u'\xb7'

def JobFromId(job_id):
  """Get a Job object from its ID. Its ID is just its key as a hex string.

  Users of Job should not have to import ndb. This function maintains an
  abstraction layer that separates users from the Datastore details.
  """
  job_key = ndb.Key('Job', int(job_id, 16))
  return job_key.get()

class Job(ndb.Model):
  """A Pinpoint job."""

  created = ndb.DateTimeProperty(required=True, auto_now_add=True)
  updated = ndb.DateTimeProperty(required=True, auto_now=True)

  # The name of the Task Queue task this job is running on. If it's present, the
  # job is running. The task is also None for Task Queue retries.
  task = ndb.StringProperty()

  # The string contents of any Exception that was thrown to the top level.
  # If it's present, the job failed.
  exception = ndb.TextProperty()

  # Request parameters.
  arguments = ndb.JsonProperty(required=True)

  repeat_count = ndb.IntegerProperty(required=True)

  # If True, the service should pick additional Changes to run (bisect).
  # If False, only run the Changes explicitly added by the user.
  auto_explore = ndb.BooleanProperty(required=True)

  # TODO: The bug id is only used for posting bug comments when a job starts and
  # completes. This probably should not be the responsibility of Pinpoint.
  bug_id = ndb.IntegerProperty()

  state = ndb.PickleProperty(required=True)

  @classmethod
  def New(cls, arguments, quests, auto_explore,
          repeat_count=_DEFAULT_REPEAT_COUNT, bug_id=None):
    repeat_count = repeat_count or _DEFAULT_REPEAT_COUNT
    # Create job.
    return cls(
        arguments=arguments,
        auto_explore=auto_explore,
        repeat_count=repeat_count,
        bug_id=bug_id,
        state=_JobState(quests, repeat_count))

  @property
  def job_id(self):
    return '%x' % self.key.id()

  @property
  def status(self):
    if self.task:
      return 'Running'

    if self.exception:
      return 'Failed'

    return 'Completed'

  @property
  def url(self):
    return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)

  def AddChange(self, change):
    self.state.AddChange(change)

  def Start(self):
    self.Schedule()
    self._PostBugComment('started')

  def Complete(self):
    self._PostBugComment('completed')

  def Fail(self):
    self.exception = traceback.format_exc()
    self._PostBugComment('stopped with an error')

  def Schedule(self):
    task = taskqueue.add(queue_name='job-queue', url='/api/run/' + self.job_id,
                         countdown=_TASK_INTERVAL)
    self.task = task.name

  def Run(self):
    self.exception = None  # In case the Job succeeds on retry.
    self.task = None  # In case an exception is thrown.

    try:
      if self.auto_explore:
        self.state.Explore()
      work_left = self.state.ScheduleWork()

      # Schedule moar task.
      if work_left:
        self.Schedule()
      else:
        self.Complete()
    except BaseException:
      self.Fail()
      raise

  def StatusDict(self):
    """A shorter version of AsDict() for faster Jobs list loading."""
    return {
        'job_id': self.job_id,

        'arguments': self.arguments,
        'auto_explore': self.auto_explore,

        'created': self.created.isoformat(),
        'updated': self.updated.isoformat(),
        'exception': self.exception,
        'status': self.status,
    }

  def AsDict(self):
    d = self.StatusDict()
    d['state'] = self.state.AsDict()
    return d

  def _PostBugComment(self, status):
    if not self.bug_id:
      return

    title = '%s Pinpoint job %s.' % (_ROUND_PUSHPIN, status)
    header = '\n'.join((title, self.url))

    # Include list of Changes.
    change_details = []
    for _, change in self.state.Differences():
      # TODO: Only show the most specific Dep.
      # TODO: Store the commit info in the Dep.
      for dep in change.all_deps:
        commit_info = gitiles_service.CommitInfo(dep.repository, dep.git_hash)
        subject = commit_info['message'].split('\n', 1)[0]
        author = commit_info['author']['email']
        time = commit_info['committer']['time']

        byline = 'By %s %s %s' % (author, _MIDDLE_DOT, time)
        git_link = dep.repository + '@' + dep.git_hash
        change_details.append('\n'.join((subject, byline, git_link)))

    comment = '\n\n'.join([header] + change_details)

    issue_tracker = issue_tracker_service.IssueTrackerService(
        utils.ServiceAccountHttp())
    issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)

class _JobState(object):
  """The internal state of a Job.

  Wrapping the entire internal state of a Job in a PickleProperty allows us to
  use regular Python objects, with constructors, dicts, and object references.

  We lose the ability to index and query the fields, but it's all internal
  anyway. Everything queryable should be on the Job object.
  """

  def __init__(self, quests, repeat_count):
    """Create a _JobState.

    Args:
      quests: A sequence of quests to run on each Change.
      repeat_count: The number of attempts to automatically run per Change.
    """
    # _quests is mutable. Any modification should mutate the existing list
    # in-place rather than assign a new list, because every Attempt references
    # this object and will be updated automatically if it's mutated.
    self._quests = list(quests)

    # _changes can be in arbitrary order. Client should not assume that the
    # list of Changes is sorted in any particular order.
    self._changes = []

    # A mapping from a Change to a list of Attempts on that Change.
    self._attempts = {}

    self._repeat_count = repeat_count

  def AddAttempt(self, change):
    assert change in self._attempts
    self._attempts[change].append(attempt_module.Attempt(self._quests, change))

  def AddChange(self, change, index=None):
    if index:
      self._changes.insert(index, change)
    else:
      self._changes.append(change)

    self._attempts[change] = []
    for _ in xrange(self._repeat_count):
      self.AddAttempt(change)

  def Explore(self):
    """Compare Changes and bisect by adding additional Changes as needed.

    For every pair of adjacent Changes, compare their results as probability
    distributions. If more information is needed to establish statistical
    confidence, add an additional Attempt. If the results are different, find
    the midpoint of the Changes and add it to the Job.

    The midpoint can only be added if the second Change represents a commit that
    comes after the first Change. Otherwise, this method won't explore further.
    For example, if Change A is repo@abc, and Change B is repo@abc + patch,
    there's no way to pick additional Changes to try.
    """
    # Compare every pair of Changes.
    # TODO: The list may Change while iterating through it.
    # TODO: Use JobState.Differences().
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]

      comparison_result = self._Compare(change_a, change_b)
      if comparison_result == _DIFFERENT:
        # Different: Bisect and add an additional Change to the job.
        try:
          midpoint = change_module.Change.Midpoint(change_a, change_b)
        except change_module.NonLinearError:
          midpoint = None
        if midpoint:
          logging.info('Adding Change %s.', midpoint)
          self.AddChange(midpoint, index)
      elif comparison_result == _SAME:
        # The same: Do nothing.
        continue
      elif comparison_result == _UNKNOWN:
        # Unknown: Add an Attempt to the Change with the fewest Attempts.
        change = min(change_a, change_b, key=lambda c: len(self._attempts[c]))
        self.AddAttempt(change)

  def ScheduleWork(self):
    work_left = False
    for attempts in self._attempts.itervalues():
      for attempt in attempts:
        if attempt.completed:
          continue

        attempt.ScheduleWork()
        work_left = True

    return work_left

  def Differences(self):
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]
      if self._Compare(change_a, change_b) == _DIFFERENT:
        yield index, change_b

  def AsDict(self):
    comparisons = []
    for index in xrange(1, len(self._changes)):
      change_a = self._changes[index - 1]
      change_b = self._changes[index]
      comparisons.append(self._Compare(change_a, change_b))

    # result_values is a 3D array. result_values[change][quest] is a list of
    # all the result values for that Change and Quest.
    result_values = []
    for change in self._changes:
      change_result_values = []

      change_results_per_quest = _CombineResultsPerQuest(self._attempts[change])
      for quest in self._quests:
        change_result_values.append(change_results_per_quest[quest])

      result_values.append(change_result_values)

    attempts = []
    for c in self._changes:
      attempts.append([attempt.AsDict() for attempt in self._attempts[c]])

    return {
        'quests': map(str, self._quests),
        'changes': [change.AsDict() for change in self._changes],
        # TODO: Use JobState.Differences().
        'comparisons': comparisons,
        'result_values': result_values,
        'attempts': attempts,
    }

  def _Compare(self, change_a, change_b):
    attempts_a = self._attempts[change_a]
    attempts_b = self._attempts[change_b]

    if any(not attempt.completed for attempt in attempts_a + attempts_b):
      return _PENDING

    # Compare exceptions.
    exceptions_a = tuple(attempt.exception or '' for attempt in attempts_a)
    exceptions_b = tuple(attempt.exception or '' for attempt in attempts_b)

    if _CompareValues(exceptions_a, exceptions_b) == _DIFFERENT:
      return _DIFFERENT

    # Compare values.
    results_a = _CombineResultsPerQuest(attempts_a)
    results_b = _CombineResultsPerQuest(attempts_b)

    if any(_CompareValues(results_a[quest], results_b[quest]) == _DIFFERENT
           for quest in self._quests):
      return _DIFFERENT

    # Here, "the same" means that we fail to reject the null hypothesis. We can
    # never be completely sure that the two Changes have the same results, but
    # we've run everything that we planned to, and didn't detect any difference.
    if (len(attempts_a) >= self._repeat_count and
        len(attempts_b) >= self._repeat_count):
      return _SAME

    return _UNKNOWN

def _CombineResultsPerQuest(attempts):
  aggregate_results = collections.defaultdict(list)
  for attempt in attempts:
    if not attempt.completed:
      continue

    for quest, results in attempt.result_values.iteritems():
      aggregate_results[quest] += results

  return aggregate_results

def _CompareValues(values_a, values_b):
  if not (values_a and values_b):
    return _UNKNOWN

  try:
    p_value = mann_whitney_u.MannWhitneyU(values_a, values_b)
  except ValueError:
    return _UNKNOWN

  if p_value < _SIGNIFICANCE_LEVEL:
    return _DIFFERENT
  else:
    return _UNKNOWN
<|endoftext|>


<commit_before>import os
import logging
import requests
import pandas as pd
import time
import sys
import math
from datetime import datetime

from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactcore.models.domainModels import CFDAProgram
from dataactvalidator.health_check import create_app
from dataactvalidator.scripts.loader_utils import clean_data, insert_dataframe, format_date

logger = logging.getLogger(__name__)

S3_CFDA_FILE = 'https://files.usaspending.gov/reference_data/cfda.csv'

DATA_CLEANING_MAP = {
                 "program_title": "program_title",
                 "program_number": "program_number",
                 "popular_name_(020)": "popular_name",
                 "federal_agency_(030)": "federal_agency",
                 "authorization_(040)": "authorization",
                 "objectives_(050)": "objectives",
                 "types_of_assistance_(060)": "types_of_assistance",
                 "uses_and_use_restrictions_(070)": "uses_and_use_restrictions",
                 "applicant_eligibility_(081)": "applicant_eligibility",
                 "beneficiary_eligibility_(082)": "beneficiary_eligibility",
                 "credentials/documentation_(083)": "credentials_documentation",
                 "preapplication_coordination_(091)": "preapplication_coordination",
                 "application_procedures_(092)": "application_procedures",
                 "award_procedure_(093)": "award_procedure",
                 "deadlines_(094)": "deadlines",
                 "range_of_approval/disapproval_time_(095)": "range_of_approval_disapproval_time",
                 "appeals_(096)": "appeals",
                 "renewals_(097)": "renewals",
                 "formula_and_matching_requirements_(101)": "formula_and_matching_requirements",
                 "length_and_time_phasing_of_assistance_(102)": "length_and_time_phasing_of_assistance",
                 "reports_(111)": "reports",
                 "audits_(112)": "audits",
                 "records_(113)": "records",
                 "account_identification_(121)": "account_identification",
                 "obligations_(122)": "obligations",
                 "range_and_average_of_financial_assistance_(123)": "range_and_average_of_financial_assistance",
                 "program_accomplishments_(130)": "program_accomplishments",
                 "regulations__guidelines__and_literature_(140)": "regulations_guidelines_and_literature",
                 "regional_or__local_office_(151)": "regional_or_local_office",
                 "headquarters_office_(152)": "headquarters_office",
                 "website_address_(153)": "website_address",
                 "related_programs_(160)": "related_programs",
                 "examples_of_funded_projects_(170)": "examples_of_funded_projects",
                 "criteria_for_selecting_proposals_(180)": "criteria_for_selecting_proposals",
                 "url": "url",
                 "recovery": "recovery",
                 "omb_agency_code": "omb_agency_code",
                 "omb_bureau_code": "omb_bureau_code",
                 "published_date": "published_date",
                 "archived_date": "archived_date"
                 }

def load_cfda_program(base_path, load_local=False, local_file_name="cfda_program.csv"):
    """ Load cfda program.

        Args:
            base_path: directory that contains the cfda values files.
    """
    if not load_local:
        logger.info("Fetching CFDA file from {}".format(S3_CFDA_FILE))
        tmp_name = str(time.time()).replace(".", "")+"_cfda_program.csv"
        filename = os.path.join(base_path, tmp_name)
        r = requests.get(S3_CFDA_FILE, allow_redirects=True)
        open(filename, 'wb').write(r.content)
    else:
        filename = os.path.join(base_path, local_file_name)
    logger.info(base_path)
    logger.info('Loading CFDA program file: ' + filename)
    """Load country code lookup table."""
    model = CFDAProgram

    def fix_program_number(n, decimals=3):
        multiplier = 10 ** decimals
        value = math.floor(n*multiplier + 0.5) / multiplier
        return str(value).ljust(6, '0')

    with create_app().app_context():
        configure_logging()
        sess = GlobalDB.db().session

        now = datetime.utcnow()
        import_data = pd.read_csv(filename, dtype=str, encoding='latin1', na_filter=False)
        import_data = clean_data(
            import_data,
            model,
            DATA_CLEANING_MAP,
            {}
        )
        import_data["published_date"] = format_date(import_data["published_date"])
        import_data["archived_date"] = format_date(import_data["archived_date"])
        import_dataframe = import_data.copy(deep=True)
        # To do the comparison, first we need to mock the pk column that postgres creates. We'll set it universally to 1
        import_dataframe = import_dataframe.assign(cfda_program_id=1, created_at=now, updated_at=now)

        table_name = model.__table__.name
        current_data = pd.read_sql_table(table_name, sess.connection(), coerce_float=False)
        # Now we need to overwrite the db's audit dates in the created dataframe, and
        # also set all the  pks to 1, so they match
        current_data = current_data.assign(cfda_program_id=1, created_at=now, updated_at=now)
        # pandas comparison requires everything to be in the same order
        current_data.sort_values('program_number', inplace=True)
        import_dataframe.sort_values('program_number', inplace=True)

        # columns too
        cols = import_dataframe.columns.tolist()
        cols.sort()
        import_dataframe = import_dataframe[cols]

        cols = current_data.columns.tolist()
        cols.sort()
        current_data = current_data[cols]

        # need to reset the indexes now that we've done all this sorting, so that they match
        import_dataframe.reset_index(drop=True, inplace=True)
        current_data.reset_index(drop=True, inplace=True)
        # My favorite part: When pandas pulls the data out of postgres, the program_number column
        # is a Decimal. However, in adding it to the dataframe, this column loses precision. So for example, a program number of
        # 10.001 imports into the dataframe as 10.000999999999999. It also needs to be cast to a
        # string, and padded with the right number of zeroes, as needed.
        current_data['program_number'] = current_data['program_number'].apply(lambda x: fix_program_number(x))
        # Finally, you can execute this and get True back if the data truly has not changed from the last
        # time the CSV was loaded.
        new_data = not import_dataframe.equals(current_data)
        if new_data:
            # insert to db
            sess.query(model).delete()
            num = insert_dataframe(import_data, table_name, sess.connection())
            sess.commit()
    if not load_local:
        os.remove(filename)
    if new_data:
        logger.info('{} records inserted to {}'.format(num, table_name))
    else:
        logger.info("Skipped cfda load, no new data.")
        sys.exit(3)

def main():
    # Can be run from CLI as:  python load_cfda_data.py '/data-act/backend/dataactvalidator/config' True  'filename.csv'
    # (or similar)
    load_cfda_program(*sys.argv[1:])

if __name__ == '__main__':
    configure_logging()
    main()
<commit_msg>deflaking<commit_after>import os
import logging
import requests
import pandas as pd
import time
import sys
import math
from datetime import datetime

from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactcore.models.domainModels import CFDAProgram
from dataactvalidator.health_check import create_app
from dataactvalidator.scripts.loader_utils import clean_data, insert_dataframe, format_date

logger = logging.getLogger(__name__)

S3_CFDA_FILE = 'https://files.usaspending.gov/reference_data/cfda.csv'

DATA_CLEANING_MAP = {
                 "program_title": "program_title",
                 "program_number": "program_number",
                 "popular_name_(020)": "popular_name",
                 "federal_agency_(030)": "federal_agency",
                 "authorization_(040)": "authorization",
                 "objectives_(050)": "objectives",
                 "types_of_assistance_(060)": "types_of_assistance",
                 "uses_and_use_restrictions_(070)": "uses_and_use_restrictions",
                 "applicant_eligibility_(081)": "applicant_eligibility",
                 "beneficiary_eligibility_(082)": "beneficiary_eligibility",
                 "credentials/documentation_(083)": "credentials_documentation",
                 "preapplication_coordination_(091)": "preapplication_coordination",
                 "application_procedures_(092)": "application_procedures",
                 "award_procedure_(093)": "award_procedure",
                 "deadlines_(094)": "deadlines",
                 "range_of_approval/disapproval_time_(095)": "range_of_approval_disapproval_time",
                 "appeals_(096)": "appeals",
                 "renewals_(097)": "renewals",
                 "formula_and_matching_requirements_(101)": "formula_and_matching_requirements",
                 "length_and_time_phasing_of_assistance_(102)": "length_and_time_phasing_of_assistance",
                 "reports_(111)": "reports",
                 "audits_(112)": "audits",
                 "records_(113)": "records",
                 "account_identification_(121)": "account_identification",
                 "obligations_(122)": "obligations",
                 "range_and_average_of_financial_assistance_(123)": "range_and_average_of_financial_assistance",
                 "program_accomplishments_(130)": "program_accomplishments",
                 "regulations__guidelines__and_literature_(140)": "regulations_guidelines_and_literature",
                 "regional_or__local_office_(151)": "regional_or_local_office",
                 "headquarters_office_(152)": "headquarters_office",
                 "website_address_(153)": "website_address",
                 "related_programs_(160)": "related_programs",
                 "examples_of_funded_projects_(170)": "examples_of_funded_projects",
                 "criteria_for_selecting_proposals_(180)": "criteria_for_selecting_proposals",
                 "url": "url",
                 "recovery": "recovery",
                 "omb_agency_code": "omb_agency_code",
                 "omb_bureau_code": "omb_bureau_code",
                 "published_date": "published_date",
                 "archived_date": "archived_date"
                 }

def load_cfda_program(base_path, load_local=False, local_file_name="cfda_program.csv"):
    """ Load cfda program.

        Args:
            base_path: directory that contains the cfda values files.
    """
    if not load_local:
        logger.info("Fetching CFDA file from {}".format(S3_CFDA_FILE))
        tmp_name = str(time.time()).replace(".", "")+"_cfda_program.csv"
        filename = os.path.join(base_path, tmp_name)
        r = requests.get(S3_CFDA_FILE, allow_redirects=True)
        open(filename, 'wb').write(r.content)
    else:
        filename = os.path.join(base_path, local_file_name)
    logger.info(base_path)
    logger.info('Loading CFDA program file: ' + filename)
    """Load country code lookup table."""
    model = CFDAProgram

    def fix_program_number(n, decimals=3):
        multiplier = 10 ** decimals
        value = math.floor(n*multiplier + 0.5) / multiplier
        return str(value).ljust(6, '0')

    with create_app().app_context():
        configure_logging()
        sess = GlobalDB.db().session

        now = datetime.utcnow()
        import_data = pd.read_csv(filename, dtype=str, encoding='latin1', na_filter=False)
        import_data = clean_data(
            import_data,
            model,
            DATA_CLEANING_MAP,
            {}
        )
        import_data["published_date"] = format_date(import_data["published_date"])
        import_data["archived_date"] = format_date(import_data["archived_date"])
        import_dataframe = import_data.copy(deep=True)
        # To do the comparison, first we need to mock the pk column that postgres creates. We'll set it universally to 1
        import_dataframe = import_dataframe.assign(cfda_program_id=1, created_at=now, updated_at=now)

        table_name = model.__table__.name
        current_data = pd.read_sql_table(table_name, sess.connection(), coerce_float=False)
        # Now we need to overwrite the db's audit dates in the created dataframe, and
        # also set all the  pks to 1, so they match
        current_data = current_data.assign(cfda_program_id=1, created_at=now, updated_at=now)
        # pandas comparison requires everything to be in the same order
        current_data.sort_values('program_number', inplace=True)
        import_dataframe.sort_values('program_number', inplace=True)

        # columns too
        cols = import_dataframe.columns.tolist()
        cols.sort()
        import_dataframe = import_dataframe[cols]

        cols = current_data.columns.tolist()
        cols.sort()
        current_data = current_data[cols]

        # need to reset the indexes now that we've done all this sorting, so that they match
        import_dataframe.reset_index(drop=True, inplace=True)
        current_data.reset_index(drop=True, inplace=True)
        # My favorite part: When pandas pulls the data out of postgres, the program_number column
        # is a Decimal. However, in adding it to the dataframe, this column loses precision.
        # So for example, a program number  of 10.001 imports into the dataframe as 10.000999999999999.
        # It also needs to be cast to astring, and padded with the right number of zeroes, as needed.
        current_data['program_number'] = current_data['program_number'].apply(lambda x: fix_program_number(x))
        # Finally, you can execute this and get True back if the data truly has not changed from the last
        # time the CSV was loaded.
        new_data = not import_dataframe.equals(current_data)
        if new_data:
            # insert to db
            sess.query(model).delete()
            num = insert_dataframe(import_data, table_name, sess.connection())
            sess.commit()
    if not load_local:
        os.remove(filename)
    if new_data:
        logger.info('{} records inserted to {}'.format(num, table_name))
    else:
        logger.info("Skipped cfda load, no new data.")
        sys.exit(3)

def main():
    # Can be run from CLI as:  python load_cfda_data.py '/data-act/backend/dataactvalidator/config' True  'filename.csv'
    # (or similar)
    load_cfda_program(*sys.argv[1:])

if __name__ == '__main__':
    configure_logging()
    main()
<|endoftext|>


<commit_before># coding: utf-8
from datetime import datetime
from mock import patch
from unittest import TestCase
from notification.models import save_task, TaskHistory
from notification.tests.factory import TaskHistoryFactory

@patch('notification.models.get_redis_connection')
class SaveTaskTestCase(TestCase):

    def setUp(self):
        self.fake_task = TaskHistoryFactory.build()
        self.fake_task.user = 'admin'
        self.fake_task.task_name = 'notification.fake.fake_task'
        self.fake_task.id = 123
        self.fake_task.task_status = 'SUCCESS'
        self.fake_task.updated_at = datetime(2017, 7, 27, 14, 39, 22, 160307)
        self.fake_task.arguments = 'Database: database_fake, New Disk Offering: Micro'

        self.default_expected_params = {
            u'task_id': self.fake_task.id,
            u'task_name': u'fake_task',
            u'task_status': 'SUCCESS',
            u'user': 'admin', u'arguments': self.fake_task.arguments,
            u'updated_at': 1501177162,
            u'is_new': 1,
            u'read': 0,
        }

    def _simulate_signal(self):
        save_task(sender=TaskHistory, instance=self.fake_task)

    def test_dont_save_when_no_user(self, mock_conn):
        self.fake_task.user = None
        self._simulate_signal()

        self.assertFalse(mock_conn.hgetall.called)
        self.assertFalse(mock_conn.hmset.called)
        self.assertFalse(mock_conn.expire.called)

    def test_save(self, mock_conn):
        mock_conn = mock_conn()
        self._simulate_signal()

        self.assertTrue(mock_conn.hgetall.called)
        self.assertTrue(mock_conn.hmset.called)
        self.assertTrue(mock_conn.expire.called)

    def test_validate_params(self, mock_conn):
        # TODO: Verify why i must do that to work
        mock_conn = mock_conn()

        self._simulate_signal()

        self.assertTrue(mock_conn.hgetall.called)
        args = mock_conn.hmset.call_args[0]

        self.assertEqual(args[0], 'task_users:admin:123')
        self.assertDictEqual(args[1], self.default_expected_params)

    def test_is_new_1_when_status_equal(self, mock_conn):
        '''
        If the status stored on database is the same of new value
        is_new must be the old value
        '''

        mock_conn = mock_conn()
        self.default_expected_params['is_new'] = 0
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['is_new'], 0)

    def test_is_new_0_when_status_different(self, mock_conn):
        '''
        If the status stored on database is different of new value
        is_new must be 1
        '''

        mock_conn = mock_conn()
        self.default_expected_params.update({
            'is_new': 0,
            'task_status': 'OTHER'
        })
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['is_new'], 1)

    def test_read_1_when_status_equal(self, mock_conn):
        '''
        If the status stored on database is the same of new value
        read must be the old value
        '''

        mock_conn = mock_conn()
        self.default_expected_params['read'] = 1
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['read'], 1)

    def test_read_0_when_status_different(self, mock_conn):
        '''
        If the status stored on database is different of new value
        read must be 0
        '''

        mock_conn = mock_conn()
        self.default_expected_params.update({
            'read': 0,
            'task_status': 'OTHER'
        })
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['read'], 0)
<commit_msg>Add one test for database signal<commit_after># coding: utf-8
from datetime import datetime
from mock import patch
from unittest import TestCase
from notification.models import save_task, TaskHistory
from notification.tests.factory import TaskHistoryFactory

@patch('notification.models.get_redis_connection')
class SaveTaskTestCase(TestCase):

    def setUp(self):
        self.fake_task = TaskHistoryFactory.build()
        self.fake_task.user = 'admin'
        self.fake_task.task_name = 'notification.fake.fake_task'
        self.fake_task.id = 123
        self.fake_task.task_status = 'SUCCESS'
        self.fake_task.updated_at = datetime(2017, 7, 27, 14, 39, 22, 160307)
        self.fake_task.arguments = 'Database: database_fake, New Disk Offering: Micro'

        self.default_expected_params = {
            u'task_id': self.fake_task.id,
            u'task_name': u'fake_task',
            u'task_status': 'SUCCESS',
            u'user': 'admin', u'arguments': self.fake_task.arguments,
            u'updated_at': 1501177162,
            u'is_new': 1,
            u'read': 0,
        }

    def _simulate_signal(self):
        save_task(sender=TaskHistory, instance=self.fake_task)

    def test_dont_save_when_no_user(self, mock_conn):
        self.fake_task.user = None
        self._simulate_signal()

        self.assertFalse(mock_conn.hgetall.called)
        self.assertFalse(mock_conn.hmset.called)
        self.assertFalse(mock_conn.expire.called)

    def test_save(self, mock_conn):
        mock_conn = mock_conn()
        self._simulate_signal()

        self.assertTrue(mock_conn.hgetall.called)
        self.assertTrue(mock_conn.hmset.called)
        self.assertTrue(mock_conn.expire.called)

    def test_validate_params(self, mock_conn):
        # TODO: Verify why i must do that to work
        mock_conn = mock_conn()

        self._simulate_signal()

        self.assertTrue(mock_conn.hgetall.called)
        args = mock_conn.hmset.call_args[0]

        self.assertEqual(args[0], 'task_users:admin:123')
        self.assertDictEqual(args[1], self.default_expected_params)

    def test_is_new_1_when_status_equal(self, mock_conn):
        '''
        If the status stored on database is the same of new value
        is_new must be the old value
        '''

        mock_conn = mock_conn()
        self.default_expected_params['is_new'] = 0
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['is_new'], 0)

    def test_is_new_0_when_status_different(self, mock_conn):
        '''
        If the status stored on database is different of new value
        is_new must be 1
        '''

        mock_conn = mock_conn()
        self.default_expected_params.update({
            'is_new': 0,
            'task_status': 'OTHER'
        })
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['is_new'], 1)

    def test_read_1_when_status_equal(self, mock_conn):
        '''
        If the status stored on database is the same of new value
        read must be the old value
        '''

        mock_conn = mock_conn()
        self.default_expected_params['read'] = 1
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['read'], 1)

    def test_read_0_when_status_different(self, mock_conn):
        '''
        If the status stored on database is different of new value
        read must be 0
        '''

        mock_conn = mock_conn()
        self.default_expected_params.update({
            'read': 0,
            'task_status': 'OTHER'
        })
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['read'], 0)

    def test_change_task_status(self, mock_conn):

        mock_conn = mock_conn()
        self.default_expected_params['task_status'] = 'WAITING'
        mock_conn.hgetall.return_value = self.default_expected_params

        self._simulate_signal()

        args = mock_conn.hmset.call_args[0]
        self.assertEqual(args[1]['task_status'], 'SUCCESS')
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from collections import namedtuple

from distutils.version import StrictVersion
from django import get_version
from django.db import models
from nece.managers import TranslationManager, TranslationMixin
from nece.exceptions import NonTranslatableFieldError

if StrictVersion(get_version()) >= StrictVersion('1.9.0'):
    from django.contrib.postgres.fields import JSONField
else:
    from nece.fields.pgjson import JSONField

class TranslationModel(models.Model, TranslationMixin):
    translations = JSONField(null=True, blank=True)
    default_language = None
    _translated = None

    objects = TranslationManager()

    def __init__(self, *args, **kwargs):
        self.language_class = namedtuple('Language', self.translatable_fields)
        self._language_code = self._default_language_code
        return super(TranslationModel, self).__init__(*args, **kwargs)

    def __getattribute__(self, name):
        attr = object.__getattribute__(self, name)
        if name.startswith('__'):
            return attr
        translated = object.__getattribute__(self, '_translated')
        if translated:
            if hasattr(translated, name):
                return getattr(translated, name) or attr
        return attr

    def populate_translations(self, translations):
        for field in self.translatable_fields:
            if field not in translations:
                translations[field] = None
        return translations

    def translate(self, language_code=None, **kwargs):
        if language_code:
            self._language_code = language_code
        self.translations = self.translations or {}
        self.translations[self._language_code] = {}
        for name, value in kwargs.items():
            if name not in self.translatable_fields:
                raise NonTranslatableFieldError(name)
            if self.is_default_language(self._language_code):
                setattr(self, name, value)
            else:
                self.translations.get(self._language_code, {})[name] = value
        if language_code:
            self.language(language_code)

    def reset_language(self):
        self._translated = None
        self._language_code = self._default_language_code

    def language(self, language_code):
        self.reset_language()
        fields = self.translatable_fields
        self._language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return self
        self.default_language = self.language_class(
            **{i: getattr(self, i, None) for i in fields})
        translations = self.translations or {}
        if translations:
            translations = translations.get(self._language_code, {})
            if translations:
                translations = self.populate_translations(translations)
                self._translated = self.language_class(**translations)
        return self

    def language_or_none(self, language_code):
        language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return self.language(language_code)
        if not self.translations or not self.translations.get(language_code):
            return None
        return self.language(language_code)

    def language_as_dict(self, language_code=None):
        if not language_code:
            language_code = self._language_code
        tf = self.translatable_fields
        language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return {k: v for k, v in self.__dict__.items() if k in tf}

        translations = self.translations or {}
        if translations:
            translations = translations.get(language_code, {})
            return {k: v for k, v in translations.items() if v and k in tf}
        return {}

    def save(self, *args, **kwargs):
        language_code = self._language_code
        self.reset_language()
        if self.translations == '':
            self.translations = None
        super(TranslationModel, self).save(*args, **kwargs)
        self.language(language_code)

    class Meta:
        abstract = True
<commit_msg>prevent creating empty translations for default language<commit_after>from __future__ import unicode_literals
from collections import namedtuple

from distutils.version import StrictVersion
from django import get_version
from django.db import models
from nece.managers import TranslationManager, TranslationMixin
from nece.exceptions import NonTranslatableFieldError

if StrictVersion(get_version()) >= StrictVersion('1.9.0'):
    from django.contrib.postgres.fields import JSONField
else:
    from nece.fields.pgjson import JSONField

class TranslationModel(models.Model, TranslationMixin):
    translations = JSONField(null=True, blank=True)
    default_language = None
    _translated = None

    objects = TranslationManager()

    def __init__(self, *args, **kwargs):
        self.language_class = namedtuple('Language', self.translatable_fields)
        self._language_code = self._default_language_code
        return super(TranslationModel, self).__init__(*args, **kwargs)

    def __getattribute__(self, name):
        attr = object.__getattribute__(self, name)
        if name.startswith('__'):
            return attr
        translated = object.__getattribute__(self, '_translated')
        if translated:
            if hasattr(translated, name):
                return getattr(translated, name) or attr
        return attr

    def populate_translations(self, translations):
        for field in self.translatable_fields:
            if field not in translations:
                translations[field] = None
        return translations

    def translate(self, language_code=None, **kwargs):
        if language_code:
            self._language_code = language_code
        if not self.is_default_language(self._language_code):
            self.translations = self.translations or {}
            self.translations[self._language_code] = {}
        for name, value in kwargs.items():
            if name not in self.translatable_fields:
                raise NonTranslatableFieldError(name)
            if self.is_default_language(self._language_code):
                setattr(self, name, value)
            else:
                self.translations.get(self._language_code, {})[name] = value
        if language_code:
            self.language(language_code)

    def reset_language(self):
        self._translated = None
        self._language_code = self._default_language_code

    def language(self, language_code):
        self.reset_language()
        fields = self.translatable_fields
        self._language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return self
        self.default_language = self.language_class(
            **{i: getattr(self, i, None) for i in fields})
        translations = self.translations or {}
        if translations:
            translations = translations.get(self._language_code, {})
            if translations:
                translations = self.populate_translations(translations)
                self._translated = self.language_class(**translations)
        return self

    def language_or_none(self, language_code):
        language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return self.language(language_code)
        if not self.translations or not self.translations.get(language_code):
            return None
        return self.language(language_code)

    def language_as_dict(self, language_code=None):
        if not language_code:
            language_code = self._language_code
        tf = self.translatable_fields
        language_code = self.get_language_key(language_code)
        if self.is_default_language(language_code):
            return {k: v for k, v in self.__dict__.items() if k in tf}

        translations = self.translations or {}
        if translations:
            translations = translations.get(language_code, {})
            return {k: v for k, v in translations.items() if v and k in tf}
        return {}

    def save(self, *args, **kwargs):
        language_code = self._language_code
        self.reset_language()
        if self.translations == '':
            self.translations = None
        super(TranslationModel, self).save(*args, **kwargs)
        self.language(language_code)

    class Meta:
        abstract = True
<|endoftext|>


<commit_before>"""
Compute grey level co-occurrence matrices (GLCMs) and associated
properties to characterize image textures.
"""

import numpy as np
import skimage.util

from _greycomatrix import _glcm_loop

def greycomatrix(image, distances, angles, levels=256, symmetric=False,
                 normed=False):
    """Calculate the grey-level co-occurrence matrix.

    A grey level co-occurence matrix is a histogram of co-occuring
    greyscale values at a given offset over an image.

    Parameters
    ----------
    image : array_like of uint8
        Integer typed input image. The image will be cast to uint8, so
        the maximum value must be less than 256.
    distances : array_like
        List of pixel pair distance offsets.
    angles : array_like
        List of pixel pair angles in radians.
    levels : int, optional
        The input image should contain integers in [0, levels-1],
        where levels indicate the number of grey-levels counted
        (typically 256 for an 8-bit image). The maximum value is
        256.
    symmetric : bool, optional
        If True, the output matrix `P[:, :, d, theta]` is symmetric. This
        is accomplished by ignoring the order of value pairs, so both
        (i, j) and (j, i) are accumulated when (i, j) is encountered
        for a given offset. The default is False.
    normed : bool, optional
        If True, normalize each matrix `P[:, :, d, theta]` by dividing
        by the total number of accumulated co-occurrences for the given
        offset. The elements of the resulting matrix sum to 1. The
        default is False.

    Returns
    -------
    P : 4-D ndarray
        The grey-level co-occurrence histogram. The value
        `P[i,j,d,theta]` is the number of times that grey-level `j`
        occurs at a distance `d` and at an angle `theta` from
        grey-level `i`. If `normed` is `False`, the output is of
        type uint32, otherwise it is float64.

    References
    ----------
    .. [1] The GLCM Tutorial Home Page,
           http://www.fp.ucalgary.ca/mhallbey/tutorial.htm
    .. [2] Pattern Recognition Engineering, Morton Nadler & Eric P.
           Smith
    .. [3] Wikipedia, http://en.wikipedia.org/wiki/Co-occurrence_matrix

    Examples
    --------
    Compute 2 GLCMs: One for a 1-pixel offset to the right, and one
    for a 1-pixel offset upwards.

    >>> image = np.array([[0, 0, 1, 1],
    ...                   [0, 0, 1, 1],
    ...                   [0, 2, 2, 2],
    ...                   [2, 2, 3, 3]], dtype=np.uint8)
    >>> result = greycomatrix(image, [1], [0, np.pi/2], levels=4)
    >>> result[:, :, 0, 0]
    array([[2, 2, 1, 0],
           [0, 2, 0, 0],
           [0, 0, 3, 1],
           [0, 0, 0, 1]], dtype=uint32)
    >>> result[:, :, 0, 1]
    array([[3, 0, 2, 0],
           [0, 2, 2, 0],
           [0, 0, 1, 2],
           [0, 0, 0, 0]], dtype=uint32)

    """

    assert levels <= 256
    image = np.ascontiguousarray(image)
    assert image.ndim == 2
    assert image.min() >= 0
    assert image.max() < levels
    image = image.astype(np.uint8)
    distances = np.ascontiguousarray(distances, dtype=np.float64)
    angles = np.ascontiguousarray(angles, dtype=np.float64)
    assert distances.ndim == 1
    assert angles.ndim == 1

    P = np.zeros((levels, levels, len(distances), len(angles)),
                 dtype=np.uint32, order='C')

    # count co-occurences
    _glcm_loop(image, distances, angles, levels, P)

    # make each GLMC symmetric
    if symmetric:
        P += np.transpose(P, (1, 0, 2, 3))

    # normalize each GLMC
    if normed:
        P = P.astype(np.float64)
        glcm_sums = np.apply_over_axes(np.sum, P, axes=(0, 1))
        glcm_sums[glcm_sums == 0] = 1
        P /= glcm_sums

    return P

def greycoprops(P, prop='contrast'):
    """Calculate texture properties of a GLCM.

    Compute a feature of a grey level co-occurrence matrix to serve as
    a compact summary of the matrix. The properties are computed as
    follows:

    - 'contrast': :math:`\\sum_{i,j=0}^{levels-1} P_{i,j}(i-j)^2`
    - 'dissimilarity': :math:`\\sum_{i,j=0}^{levels-1}P_{i,j}|i-j|`
    - 'homogeneity': :math:`\\sum_{i,j=0}^{levels-1}\\frac{P_{i,j}}{1+(i-j)^2}`
    - 'ASM': :math:`\\sum_{i,j=0}^{levels-1} P_{i,j}^2`
    - 'energy': :math:`\\sqrt{ASM}`
    - 'correlation':
.. math:: \\sum_{i,j=0}^{levels-1} P_{i,j}\\left[\\frac{(i-\\mu_i) \\
          (j-\\mu_j)}{\\sqrt{(\\sigma_i^2)(\\sigma_j^2)}}\\right]

    Parameters
    ----------
    P : ndarray
        Input array. `P` is the grey-level co-occurrence histogram
        for which to compute the specified property. The value
        `P[i,j,d,theta]` is the number of times that grey-level j
        occurs at a distance d and at an angle theta from
        grey-level i.
    prop : {'contrast', 'dissimilarity', 'homogeneity', 'energy', \
            'correlation', 'ASM'}, optional
        The property of the GLCM to compute. The default is 'contrast'.

    Returns
    -------
    results : 2-D ndarray
        2-dimensional array. `results[d, a]` is the property 'prop' for
        the d'th distance and the a'th angle.

    References
    ----------
    .. [1] The GLCM Tutorial Home Page,
           http://www.fp.ucalgary.ca/mhallbey/tutorial.htm

    Examples
    --------
    Compute the contrast for GLCMs with distances [1, 2] and angles
    [0 degrees, 90 degrees]

    >>> image = np.array([[0, 0, 1, 1],
    ...                   [0, 0, 1, 1],
    ...                   [0, 2, 2, 2],
    ...                   [2, 2, 3, 3]], dtype=np.uint8)
    >>> g = greycomatrix(image, [1, 2], [0, np.pi/2], levels=4,
    ...                  normed=True, symmetric=True)
    >>> contrast = greycoprops(g, 'contrast')
    >>> contrast
    array([[ 0.58333333,  1.        ],
           [ 1.25      ,  2.75      ]])

    """

    assert P.ndim == 4
    (num_level, num_level2, num_dist, num_angle) = P.shape
    assert num_level == num_level2
    assert num_dist > 0
    assert num_angle > 0

    # create weights for specified property
    I, J = np.ogrid[0:num_level, 0:num_level]
    if prop == 'contrast':
        weights = (I - J) ** 2
    elif prop == 'dissimilarity':
        weights = np.abs(I - J)
    elif prop == 'homogeneity':
        weights = 1. / (1. + (I - J) ** 2)
    elif prop in ['ASM', 'energy', 'correlation']:
        pass
    else:
        raise ValueError('%s is an invalid property' % (prop))

    # compute property for each GLCM
    if prop == 'energy':
        asm = np.apply_over_axes(np.sum, (P ** 2), axes=(0, 1))[0, 0]
        results = np.sqrt(asm)
    elif prop == 'ASM':
        results = np.apply_over_axes(np.sum, (P ** 2), axes=(0, 1))[0, 0]
    elif prop == 'correlation':
        results = np.zeros((num_dist, num_angle), dtype=np.float64)
        I = np.array(range(num_level)).reshape((num_level, 1, 1, 1))
        J = np.array(range(num_level)).reshape((1, num_level, 1, 1))
        diff_i = I - np.apply_over_axes(np.sum, (I * P), axes=(0, 1))[0, 0]
        diff_j = J - np.apply_over_axes(np.sum, (J * P), axes=(0, 1))[0, 0]

        std_i = np.sqrt(np.apply_over_axes(np.sum, (P * (diff_i) ** 2),
                                           axes=(0, 1))[0, 0])
        std_j = np.sqrt(np.apply_over_axes(np.sum, (P * (diff_j) ** 2),
                                           axes=(0, 1))[0, 0])
        cov = np.apply_over_axes(np.sum, (P * (diff_i * diff_j)),
                                 axes=(0, 1))[0, 0]

        # handle the special case of standard deviations near zero
        mask_0 = std_i < 1e-15
        mask_0[std_j < 1e-15] = True
        results[mask_0] = 1

        # handle the standard case
        mask_1 = mask_0 == False
        results[mask_1] = cov[mask_1] / (std_i[mask_1] * std_j[mask_1])
    elif prop in ['contrast', 'dissimilarity', 'homogeneity']:
        weights = weights.reshape((num_level, num_level, 1, 1))
        results = np.apply_over_axes(np.sum, (P * weights), axes=(0, 1))[0, 0]

    return results
<commit_msg>Fix ImportError on Python 3<commit_after>"""
Compute grey level co-occurrence matrices (GLCMs) and associated
properties to characterize image textures.
"""

import numpy as np
import skimage.util

from ._greycomatrix import _glcm_loop

def greycomatrix(image, distances, angles, levels=256, symmetric=False,
                 normed=False):
    """Calculate the grey-level co-occurrence matrix.

    A grey level co-occurence matrix is a histogram of co-occuring
    greyscale values at a given offset over an image.

    Parameters
    ----------
    image : array_like of uint8
        Integer typed input image. The image will be cast to uint8, so
        the maximum value must be less than 256.
    distances : array_like
        List of pixel pair distance offsets.
    angles : array_like
        List of pixel pair angles in radians.
    levels : int, optional
        The input image should contain integers in [0, levels-1],
        where levels indicate the number of grey-levels counted
        (typically 256 for an 8-bit image). The maximum value is
        256.
    symmetric : bool, optional
        If True, the output matrix `P[:, :, d, theta]` is symmetric. This
        is accomplished by ignoring the order of value pairs, so both
        (i, j) and (j, i) are accumulated when (i, j) is encountered
        for a given offset. The default is False.
    normed : bool, optional
        If True, normalize each matrix `P[:, :, d, theta]` by dividing
        by the total number of accumulated co-occurrences for the given
        offset. The elements of the resulting matrix sum to 1. The
        default is False.

    Returns
    -------
    P : 4-D ndarray
        The grey-level co-occurrence histogram. The value
        `P[i,j,d,theta]` is the number of times that grey-level `j`
        occurs at a distance `d` and at an angle `theta` from
        grey-level `i`. If `normed` is `False`, the output is of
        type uint32, otherwise it is float64.

    References
    ----------
    .. [1] The GLCM Tutorial Home Page,
           http://www.fp.ucalgary.ca/mhallbey/tutorial.htm
    .. [2] Pattern Recognition Engineering, Morton Nadler & Eric P.
           Smith
    .. [3] Wikipedia, http://en.wikipedia.org/wiki/Co-occurrence_matrix

    Examples
    --------
    Compute 2 GLCMs: One for a 1-pixel offset to the right, and one
    for a 1-pixel offset upwards.

    >>> image = np.array([[0, 0, 1, 1],
    ...                   [0, 0, 1, 1],
    ...                   [0, 2, 2, 2],
    ...                   [2, 2, 3, 3]], dtype=np.uint8)
    >>> result = greycomatrix(image, [1], [0, np.pi/2], levels=4)
    >>> result[:, :, 0, 0]
    array([[2, 2, 1, 0],
           [0, 2, 0, 0],
           [0, 0, 3, 1],
           [0, 0, 0, 1]], dtype=uint32)
    >>> result[:, :, 0, 1]
    array([[3, 0, 2, 0],
           [0, 2, 2, 0],
           [0, 0, 1, 2],
           [0, 0, 0, 0]], dtype=uint32)

    """

    assert levels <= 256
    image = np.ascontiguousarray(image)
    assert image.ndim == 2
    assert image.min() >= 0
    assert image.max() < levels
    image = image.astype(np.uint8)
    distances = np.ascontiguousarray(distances, dtype=np.float64)
    angles = np.ascontiguousarray(angles, dtype=np.float64)
    assert distances.ndim == 1
    assert angles.ndim == 1

    P = np.zeros((levels, levels, len(distances), len(angles)),
                 dtype=np.uint32, order='C')

    # count co-occurences
    _glcm_loop(image, distances, angles, levels, P)

    # make each GLMC symmetric
    if symmetric:
        P += np.transpose(P, (1, 0, 2, 3))

    # normalize each GLMC
    if normed:
        P = P.astype(np.float64)
        glcm_sums = np.apply_over_axes(np.sum, P, axes=(0, 1))
        glcm_sums[glcm_sums == 0] = 1
        P /= glcm_sums

    return P

def greycoprops(P, prop='contrast'):
    """Calculate texture properties of a GLCM.

    Compute a feature of a grey level co-occurrence matrix to serve as
    a compact summary of the matrix. The properties are computed as
    follows:

    - 'contrast': :math:`\\sum_{i,j=0}^{levels-1} P_{i,j}(i-j)^2`
    - 'dissimilarity': :math:`\\sum_{i,j=0}^{levels-1}P_{i,j}|i-j|`
    - 'homogeneity': :math:`\\sum_{i,j=0}^{levels-1}\\frac{P_{i,j}}{1+(i-j)^2}`
    - 'ASM': :math:`\\sum_{i,j=0}^{levels-1} P_{i,j}^2`
    - 'energy': :math:`\\sqrt{ASM}`
    - 'correlation':
.. math:: \\sum_{i,j=0}^{levels-1} P_{i,j}\\left[\\frac{(i-\\mu_i) \\
          (j-\\mu_j)}{\\sqrt{(\\sigma_i^2)(\\sigma_j^2)}}\\right]

    Parameters
    ----------
    P : ndarray
        Input array. `P` is the grey-level co-occurrence histogram
        for which to compute the specified property. The value
        `P[i,j,d,theta]` is the number of times that grey-level j
        occurs at a distance d and at an angle theta from
        grey-level i.
    prop : {'contrast', 'dissimilarity', 'homogeneity', 'energy', \
            'correlation', 'ASM'}, optional
        The property of the GLCM to compute. The default is 'contrast'.

    Returns
    -------
    results : 2-D ndarray
        2-dimensional array. `results[d, a]` is the property 'prop' for
        the d'th distance and the a'th angle.

    References
    ----------
    .. [1] The GLCM Tutorial Home Page,
           http://www.fp.ucalgary.ca/mhallbey/tutorial.htm

    Examples
    --------
    Compute the contrast for GLCMs with distances [1, 2] and angles
    [0 degrees, 90 degrees]

    >>> image = np.array([[0, 0, 1, 1],
    ...                   [0, 0, 1, 1],
    ...                   [0, 2, 2, 2],
    ...                   [2, 2, 3, 3]], dtype=np.uint8)
    >>> g = greycomatrix(image, [1, 2], [0, np.pi/2], levels=4,
    ...                  normed=True, symmetric=True)
    >>> contrast = greycoprops(g, 'contrast')
    >>> contrast
    array([[ 0.58333333,  1.        ],
           [ 1.25      ,  2.75      ]])

    """

    assert P.ndim == 4
    (num_level, num_level2, num_dist, num_angle) = P.shape
    assert num_level == num_level2
    assert num_dist > 0
    assert num_angle > 0

    # create weights for specified property
    I, J = np.ogrid[0:num_level, 0:num_level]
    if prop == 'contrast':
        weights = (I - J) ** 2
    elif prop == 'dissimilarity':
        weights = np.abs(I - J)
    elif prop == 'homogeneity':
        weights = 1. / (1. + (I - J) ** 2)
    elif prop in ['ASM', 'energy', 'correlation']:
        pass
    else:
        raise ValueError('%s is an invalid property' % (prop))

    # compute property for each GLCM
    if prop == 'energy':
        asm = np.apply_over_axes(np.sum, (P ** 2), axes=(0, 1))[0, 0]
        results = np.sqrt(asm)
    elif prop == 'ASM':
        results = np.apply_over_axes(np.sum, (P ** 2), axes=(0, 1))[0, 0]
    elif prop == 'correlation':
        results = np.zeros((num_dist, num_angle), dtype=np.float64)
        I = np.array(range(num_level)).reshape((num_level, 1, 1, 1))
        J = np.array(range(num_level)).reshape((1, num_level, 1, 1))
        diff_i = I - np.apply_over_axes(np.sum, (I * P), axes=(0, 1))[0, 0]
        diff_j = J - np.apply_over_axes(np.sum, (J * P), axes=(0, 1))[0, 0]

        std_i = np.sqrt(np.apply_over_axes(np.sum, (P * (diff_i) ** 2),
                                           axes=(0, 1))[0, 0])
        std_j = np.sqrt(np.apply_over_axes(np.sum, (P * (diff_j) ** 2),
                                           axes=(0, 1))[0, 0])
        cov = np.apply_over_axes(np.sum, (P * (diff_i * diff_j)),
                                 axes=(0, 1))[0, 0]

        # handle the special case of standard deviations near zero
        mask_0 = std_i < 1e-15
        mask_0[std_j < 1e-15] = True
        results[mask_0] = 1

        # handle the standard case
        mask_1 = mask_0 == False
        results[mask_1] = cov[mask_1] / (std_i[mask_1] * std_j[mask_1])
    elif prop in ['contrast', 'dissimilarity', 'homogeneity']:
        weights = weights.reshape((num_level, num_level, 1, 1))
        results = np.apply_over_axes(np.sum, (P * weights), axes=(0, 1))[0, 0]

    return results
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
The :mod:`sklearn.kernel_approximation` module implements several
approximate kernel feature maps base on Fourier transforms.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD Style.

import numpy as np
from .base import BaseEstimator
from .base import TransformerMixin
from .utils import array2d, atleast2d_or_csr, check_random_state
from .utils.extmath import safe_sparse_dot

class RBFSampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of an RBF kernel by Monte Carlo approximation
    of its Fourier transform.

    Parameters
    ----------
    gamma: float
        parameter of RBF kernel: exp(-gamma * x**2)

    n_components: int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
    Benjamin Recht.
    """

    def __init__(self, gamma=1., n_components=100., random_state=None):
        self.gamma = gamma
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = atleast2d_or_csr(X)
        self.random_state = check_random_state(self.random_state)
        n_features = X.shape[1]

        self.random_weights_ = (np.sqrt(self.gamma) *
                self.random_state.normal(size=(n_features, self.n_components)))
        self.random_offset_ = self.random_state.uniform(0,
                2 * np.pi, size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = atleast2d_or_csr(X)
        projection = safe_sparse_dot(X, self.random_weights_)
        return (np.sqrt(2.) / np.sqrt(self.n_components)
                * np.cos(projection + self.random_offset_))

class SkewedChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of the "skewed chi-squared" kernel by Monte
    Carlo approximation of its Fourier transform.

    Parameters
    ----------
    skewedness: float
        "skewedness" parameter of the kernel. Needs to be cross-validated.

    n_components: int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Fourier Approximations for Skewed Multiplicative Histogram
    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.
    """

    def __init__(self, skewedness=1., n_components=100, random_state=None):
        self.skewedness = skewedness
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = array2d(X)
        self.random_state = check_random_state(self.random_state)
        n_features = X.shape[1]
        uniform = self.random_state.uniform(size=(n_features,
            self.n_components))
        # transform by inverse CDF of sech
        self.random_weights_ = (1. / np.pi
                * np.log(np.tan(np.pi / 2. * uniform)))
        self.random_offset_ = self.random_state.uniform(0,
                2 * np.pi, size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = array2d(X)
        if (X < 0).any():
            raise ValueError("X may not contain entries smaller than zero.")

        projection = safe_sparse_dot(np.log(X + self.skewedness),
                self.random_weights_)

        return (np.sqrt(2.) / np.sqrt(self.n_components)
                * np.cos(projection + self.random_offset_))

class AdditiveChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximate feature map for additive chi² kernel.

    Uses sampling the fourier transform of the kernel characteristic
    at regular intervals.

    Since the kernel that is to be approximated is additive, the components of
    the input vectors can be treated separately.  Each entry in the original
    space is transformed into 2×sample_steps+1 features, where sample_steps is
    a parameter of the method. Typical values of n include 1, 2 and 3.

    Optimal choices for the sampling interval for certain data ranges can be
    computed (see the reference). The default values should be reasonable.

    Parameters
    ----------
    sample_steps: int, optional
        Gives the number of (complex) sampling points.
    sample_interval: float, optional
        Sampling interval. Must be specified when sample_steps not in {1,2,3}.

    Notes
    -----
    See `"Efficient additive kernels via explicit feature maps"
    <http://eprints.pascal-network.org/archive/00006964/01/vedaldi10.pdf>`_
    Vedaldi, A. and Zisserman, A.
    - Computer Vision and Pattern Recognition 2010
    """

    def __init__(self, sample_steps=2, sample_interval=None):
        self.sample_steps = sample_steps
        self.sample_interval = sample_interval

    def fit(self, X, y=None):
        """Set parameters."""
        if self.sample_interval == None:
            # See reference, figure 2 c)
            if self.sample_steps == 1:
                self.sample_interval = 0.8
            elif self.sample_steps == 2:
                self.sample_interval = 0.5
            elif self.sample_steps == 3:
                self.sample_interval = 0.4
            else:
                raise ValueError("If sample_steps is not in [1, 2, 3],"
                    " you need to provide sample_interval")
        return self

    def transform(self, X, y=None):
        """Apply approximate feature map to X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_features * (2n + 1))
        """

        X = array2d(X)
        # check if X has negative values. Doesn't play well with np.log.
        if (X < 0).any():
            raise ValueError("Entries of X must be positive.")
        X_new = []
        # zeroth component
        # 1/cosh = sech
        # cosh(0) = 1.0

        non_zero = (X != 0.0)
        X_nz = X[non_zero]

        X_step = np.zeros_like(X)
        X_step[non_zero] = np.sqrt(X_nz * self.sample_interval)
        X_new.append(X_step.copy())

        log_step = np.zeros_like(X)
        step = np.zeros_like(X)

        log_step[non_zero] = self.sample_interval * np.log(X_nz)
        step[non_zero] = 2 * X_nz * self.sample_interval

        log_step_nz = log_step[non_zero]
        step_nz = step[non_zero]

        factor = np.zeros_like(X)

        for j in xrange(1, self.sample_steps):
            factor[non_zero] = np.sqrt(step_nz / \
                             np.cosh(np.pi * j * self.sample_interval))
            factor_nz = factor[non_zero]
            X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
            X_new.append(X_step.copy())
            X_step[non_zero] = factor_nz * np.sin(j * log_step_nz)
            X_new.append(X_step.copy())
        return np.hstack(X_new)
<commit_msg>kernel approximation: simplified mangement of non zero elements<commit_after># -*- coding: utf-8 -*-
"""
The :mod:`sklearn.kernel_approximation` module implements several
approximate kernel feature maps base on Fourier transforms.
"""

# Author: Andreas Mueller <amueller@ais.uni-bonn.de>
#
# License: BSD Style.

import numpy as np
from .base import BaseEstimator
from .base import TransformerMixin
from .utils import array2d, atleast2d_or_csr, check_random_state
from .utils.extmath import safe_sparse_dot

class RBFSampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of an RBF kernel by Monte Carlo approximation
    of its Fourier transform.

    Parameters
    ----------
    gamma: float
        parameter of RBF kernel: exp(-gamma * x**2)

    n_components: int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Features for Large-Scale Kernel Machines" by A. Rahimi and
    Benjamin Recht.
    """

    def __init__(self, gamma=1., n_components=100., random_state=None):
        self.gamma = gamma
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = atleast2d_or_csr(X)
        self.random_state = check_random_state(self.random_state)
        n_features = X.shape[1]

        self.random_weights_ = (np.sqrt(self.gamma) *
                self.random_state.normal(size=(n_features, self.n_components)))
        self.random_offset_ = self.random_state.uniform(0,
                2 * np.pi, size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X: {array-like, sparse matrix}, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = atleast2d_or_csr(X)
        projection = safe_sparse_dot(X, self.random_weights_)
        return (np.sqrt(2.) / np.sqrt(self.n_components)
                * np.cos(projection + self.random_offset_))

class SkewedChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximates feature map of the "skewed chi-squared" kernel by Monte
    Carlo approximation of its Fourier transform.

    Parameters
    ----------
    skewedness: float
        "skewedness" parameter of the kernel. Needs to be cross-validated.

    n_components: int
        number of Monte Carlo samples per original feature.
        Equals the dimensionality of the computed feature space.

    random_state : {int, RandomState}, optional
        If int, random_state is the seed used by the random number generator;
        if RandomState instance, random_state is the random number generator.

    Notes
    -----
    See "Random Fourier Approximations for Skewed Multiplicative Histogram
    Kernels" by Fuxin Li, Catalin Ionescu and Cristian Sminchisescu.
    """

    def __init__(self, skewedness=1., n_components=100, random_state=None):
        self.skewedness = skewedness
        self.n_components = n_components
        self.random_state = random_state

    def fit(self, X, y=None):
        """Fit the model with X.

        Samples random projection according to n_features.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            Training data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        self : object
            Returns the transformer.
        """

        X = array2d(X)
        self.random_state = check_random_state(self.random_state)
        n_features = X.shape[1]
        uniform = self.random_state.uniform(size=(n_features,
            self.n_components))
        # transform by inverse CDF of sech
        self.random_weights_ = (1. / np.pi
                * np.log(np.tan(np.pi / 2. * uniform)))
        self.random_offset_ = self.random_state.uniform(0,
                2 * np.pi, size=self.n_components)
        return self

    def transform(self, X, y=None):
        """Apply the approximate feature map to X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)
            New data, where n_samples in the number of samples
            and n_features is the number of features.

        Returns
        -------
        X_new: array-like, shape (n_samples, n_components)
        """
        X = array2d(X)
        if (X < 0).any():
            raise ValueError("X may not contain entries smaller than zero.")

        projection = safe_sparse_dot(np.log(X + self.skewedness),
                self.random_weights_)

        return (np.sqrt(2.) / np.sqrt(self.n_components)
                * np.cos(projection + self.random_offset_))

class AdditiveChi2Sampler(BaseEstimator, TransformerMixin):
    """Approximate feature map for additive chi² kernel.

    Uses sampling the fourier transform of the kernel characteristic
    at regular intervals.

    Since the kernel that is to be approximated is additive, the components of
    the input vectors can be treated separately.  Each entry in the original
    space is transformed into 2×sample_steps+1 features, where sample_steps is
    a parameter of the method. Typical values of n include 1, 2 and 3.

    Optimal choices for the sampling interval for certain data ranges can be
    computed (see the reference). The default values should be reasonable.

    Parameters
    ----------
    sample_steps: int, optional
        Gives the number of (complex) sampling points.
    sample_interval: float, optional
        Sampling interval. Must be specified when sample_steps not in {1,2,3}.

    Notes
    -----
    See `"Efficient additive kernels via explicit feature maps"
    <http://eprints.pascal-network.org/archive/00006964/01/vedaldi10.pdf>`_
    Vedaldi, A. and Zisserman, A.
    - Computer Vision and Pattern Recognition 2010
    """

    def __init__(self, sample_steps=2, sample_interval=None):
        self.sample_steps = sample_steps
        self.sample_interval = sample_interval

    def fit(self, X, y=None):
        """Set parameters."""
        if self.sample_interval == None:
            # See reference, figure 2 c)
            if self.sample_steps == 1:
                self.sample_interval = 0.8
            elif self.sample_steps == 2:
                self.sample_interval = 0.5
            elif self.sample_steps == 3:
                self.sample_interval = 0.4
            else:
                raise ValueError("If sample_steps is not in [1, 2, 3],"
                    " you need to provide sample_interval")
        return self

    def transform(self, X, y=None):
        """Apply approximate feature map to X.

        Parameters
        ----------
        X: array-like, shape (n_samples, n_features)

        Returns
        -------
        X_new: array-like, shape (n_samples, n_features * (2n + 1))
        """

        X = array2d(X)
        # check if X has negative values. Doesn't play well with np.log.
        if (X < 0).any():
            raise ValueError("Entries of X must be positive.")
        X_new = []
        # zeroth component
        # 1/cosh = sech
        # cosh(0) = 1.0

        non_zero = (X != 0.0)
        X_nz = X[non_zero]

        X_step = np.zeros_like(X)
        X_step[non_zero] = np.sqrt(X_nz * self.sample_interval)
        X_new.append(X_step.copy())

        log_step_nz = self.sample_interval * np.log(X_nz)
        step_nz = 2 * X_nz * self.sample_interval

        for j in xrange(1, self.sample_steps):
            factor_nz = np.sqrt(step_nz /
                             np.cosh(np.pi * j * self.sample_interval))
            X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
            X_new.append(X_step.copy())
            X_step[non_zero] = factor_nz * np.sin(j * log_step_nz)
            X_new.append(X_step.copy())
        return np.hstack(X_new)
<|endoftext|>


<commit_before>import Skype4Py
import config as gbconfig
import json
from common import get_project_path

# Get Skype class instance
SKYPE_OBJ = Skype4Py.Skype()

# Establish the connection from the Skype object to the Skype ddclient.
SKYPE_OBJ.Attach()

# Get all contact from object. This function might not be used in this case

def get_file():
    """Function to get file contains list of skype's contact"""
    returndata = {}
    try:
        root_path = get_project_path()
        # print root_path
        file_path = "%s/%s" % (root_path, gbconfig.FILE_CONTACT)
        filename = open(file_path, 'r')
        returndata = json.loads(filename.read())
        filename.close()
    except Exception as ex:
        print 'What the fuck? I could not load your file: %s - %s' % (gbconfig.FILE_CONTACT, ex)

    return returndata

def main_function():
    """Runable function"""
    get_file()

for contact, message in get_file().iteritems():
    SKYPE_OBJ.SendMessage(contact, message)
    print "Message has been sent"

if __name__ == "__main__":
    main_function()
<commit_msg>Update SController.py<commit_after>"""Import somes important package"""
import Skype4Py
import config as gbconfig
import json
from common import get_project_path

# Get Skype class instance
SKYPE_OBJ = Skype4Py.Skype()

# Establish the connection from the Skype object to the Skype ddclient.
SKYPE_OBJ.Attach()

# Get all contact from object. This function might not be used in this case

def get_file():
    """Function to get file contains list of skype's contact"""
    returndata = {}
    try:
        root_path = get_project_path()
        # print root_path
        file_path = "%s/%s" % (root_path, gbconfig.FILE_CONTACT)
        filename = open(file_path, 'r')
        returndata = json.loads(filename.read())
        filename.close()
    except Exception as ex:
        print 'What the fuck? I could not load your file: %s - %s' % (gbconfig.FILE_CONTACT, ex)

    return returndata

def main_function():
    """Runable function"""
    get_file()

for contact, message in get_file().iteritems():
    SKYPE_OBJ.SendMessage(contact, message)
    print "Message has been sent"

if __name__ == "__main__":
    main_function()
<|endoftext|>