<commit_before>#!/usr/bin/env python

# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A Test Script to be copied to the test VM to perform various tests to
   object storage via APIs exposed by the provider.

   Note: it is intentional that this test script is NOT dependant on the PKB
   package so we do not have to copy the entire PKB package to test VM just to
   run this script.
"""

import json
import logging
import sys
from threading import Thread
import time

import gflags as flags
import boto
import gcs_oauth2_boto_plugin  # noqa

FLAGS = flags.FLAGS

flags.DEFINE_enum(
    'storage', 'GCS', ['GCS', 'S3', 'AZURE'], 'The target storage to test.')

flags.DEFINE_string('host', None, 'The hostname of the storage endpoint.')

flags.DEFINE_string('bucket', None,
                    'The name of the bucket to test with. Caller is '
                    'responsible to create an empty bucket for a particular '
                    'invocation of this test and then clean-up the bucket '
                    'after this test returns.')

flags.DEFINE_enum(
    'scenario', 'OneByteRW', ['OneByteRW', 'ListConsistency',
                              'SingleStreamThroughput'],
    'The various scenarios to test. OneByteRW: read and write of single byte. '
    'ListConsistency: List-after-write and list-after-update consistency. '
    'SingleStreamThroughput: Throughput of single stream large object RW.')

flags.DEFINE_integer('iterations', 1, 'The number of iterations to run for the '
                     'particular test scenario. Currently only applicable to '
                     'the ListConsistency scenario, ignored in others.')

STORAGE_TO_SCHEMA_DICT = {'GCS': 'gs', 'S3': 's3'}

# If more than 5% of our upload or download operations fail for an iteration,
# there is an availability issue with the service provider or the connection
# between the test VM and the service provider. For this reason, we consider
# this particular iteration invalid, and we will print out an error message
# instead of a set of benchmark numbers (which will be incorrect).
FAILURE_TOLERANCE = 0.05

# When a storage provider fails more than a threshold number of requests, we
# stop the benchmarking tests and raise a low availability error back to the
# caller.
class LowAvailabilityError(Exception):
    pass

def _TpCalc(numbers):
  numbers_sorted = sorted(numbers)
  count = len(numbers_sorted)
  output = json.dumps({'tp50': numbers_sorted[int(count * 0.5)],
                       'tp90': numbers_sorted[int(count * 0.9)],
                       'tp99': numbers_sorted[int(count * 0.99)],
                       'tp99.9': numbers_sorted[int(count * 0.999)]
                       }, sort_keys=True)
  return output

def _ListObjects(storage_schema, bucket, prefix, host_to_connect=None):
  bucket_uri = boto.storage_uri(bucket, storage_schema)
  if host_to_connect is not None:
    bucket_uri.connect(host=host_to_connect)

  list_result = []
  for k in bucket_uri.list_bucket(prefix=prefix):
    list_result.append(k.name)

  return list_result

def WriteOneByteObjects(storage_schema, bucket, object_prefix, count,
                        objects_written, latency_results=None,
                        host_to_connect=None):
  """
  Write a number of one byte objects to a storage provider.

  :type storage_schema: string
  :param storage_schema: The address schema identifying a storage. e.g., "gs"

  :type bucket: string
  :param bucket: Name of the bucket to write to.

  :type object_prefix: string
  :param object_prefix: The prefix of names of objects to be written.

  :type count: integer
  :param count: The total number of objects that need to be written.

  :type objects_written: list of strings
  :param objects_written: A list of names of objects that have been successfully
      written by this function. Caller supplies the list and this function fills
      in the name of the objects.

  :type latency_results: list of floats
  :param latency_results: An optional parameter that caller can supply to hold
      a latency numbers, in seconds, for each object that is succeessfully
      written.

  :type host_to_connect: string
  :param host_to_connect: An optional endpoint string to connect to.
  """
  for i in range(count):
    object_name = '%s_%d' % (object_prefix, i)
    object_path = '%s/%s' % (bucket, object_name)
    object_uri = boto.storage_uri(object_path, storage_schema)
    # Note below, this does not really connect, it only sets the connection
    # property, the real http connnection is only established when doing the
    # actual upload.
    if host_to_connect is not None:
      object_uri.connect(host=host_to_connect)

    # Ready to go!
    start_time = time.time()

    # We write the object to storage provider and if successful we place the
    # name of the object to the objects_written list. When a write fails, the
    # object won't be placed into the objects_written list, and then it's up to
    # the caller to decide if they want to tolerate the failure and accept
    # the results, depending on test scenarios.
    try:
      object_uri.set_contents_from_string('f')

      if latency_results is not None:
        latency_results.append(time.time() - start_time)

      objects_written.append(object_name)
    except:
      logging.exception('Caught exception while writing object %s.' %
                        object_path)

def OneByteRWBenchmark(storage_schema, host_to_connect=None):
  """
  This is a benchmark test for one byte object read and write. It uploads and
  downloads ONE_BYTE_OBJECT_COUNT number of 1-byte objects to the storage
  provider, keeps track of the latency of these operations, and print out the
  result in JSON format at the end of the test.

  :type storage_schema: string
  :param storage_schema: The schema of the storage provider to use, e.g., "gs"

  :type host_to_connect: string
  :param host_to_connect: An optional host endpoint to connect to.
  """

  # One byte write
  object_prefix = 'pkb_one_byte_%f' % time.time()
  ONE_BYTE_OBJECT_COUNT = 1000

  one_byte_write_latency = []
  one_byte_objects_written = []

  WriteOneByteObjects(storage_schema, FLAGS.bucket, object_prefix,
                      ONE_BYTE_OBJECT_COUNT, one_byte_objects_written,
                      latency_results=one_byte_write_latency,
                      host_to_connect=host_to_connect)

  success_count = len(one_byte_objects_written)
  if success_count < ONE_BYTE_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to write required number of objects, '
                               'exiting.')

  print 'One byte upload - %s' % _TpCalc(one_byte_write_latency)

  # Now download these objects and measure the latencies.
  one_byte_read_latency = []

  for object_name in one_byte_objects_written:
    object_path = '%s/%s' % (FLAGS.bucket, object_name)
    object_uri = boto.storage_uri(object_path, storage_schema)
    object_uri.connect(host=host_to_connect)

    # Ready to go!
    start_time = time.time()

    try:
      object_uri.get_contents_as_string()

      one_byte_read_latency.append(time.time() - start_time)
    except:
      logging.exception('Failed to read object %s' % object_path)

  success_count = len(one_byte_read_latency)
  if success_count < ONE_BYTE_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to read required number of objects, '
                               'exiting.')

  print 'One byte download - %s' % _TpCalc(one_byte_read_latency)

def ListConsistencyBenchmark(storage_schema, host_to_connect=None):
  """
  This is a benchmark test to measure list-after-write consistency. It uploads
  a large number of 1-byte objects in a short amount of time, and then issues
  a list request. If the first list request returns all objects as expected,
  then the result is deemed consistent. Otherwise, it keeps issuing list request
  until all expected objects are returned, and the test reports the time
  it takes from the end of the last write to the time the list returns
  consistent result.

  :type storage_schema: string
  :param storage_schema: The schema of the storage provider to use, e.g., "gs"

  :type host_to_connect: string
  :param host_to_connect: An optional host endpoint to connect to.
  """

  # The maximum amount of seconds we are willing to wait for list results to be
  # consistent in this benchmark
  CONSISTENCY_WAIT_TIME_LIMIT = 300

  # Total number of objects we will provision before we do the list
  TOTAL_OBJECT_COUNT = 5000

  # Provisioning is done in parallel threads to reduce test iteration time!
  TOTAL_THREAD_COUNT = 100

  # Provision the test with tons of one byte objects. Write them all at once.
  object_prefix = 'pkb_list_consistency_%f' % time.time()
  final_objects_written = []

  per_thread_objects_written = [[] for i in range(TOTAL_THREAD_COUNT)]
  threads = []

  for i in range(TOTAL_THREAD_COUNT):
    my_prefix = '%s_%d' % (object_prefix, i)
    thread = Thread(target=WriteOneByteObjects,
                    args=(storage_schema, FLAGS.bucket, my_prefix,
                          TOTAL_OBJECT_COUNT / TOTAL_THREAD_COUNT,
                          per_thread_objects_written[i], None, host_to_connect))
    thread.daemon = True
    thread.start()
    threads.append(thread)

  print 'All threads started, waiting for them to end...'

  for i in range(TOTAL_THREAD_COUNT):
    try:
      threads[i].join()
      final_objects_written += per_thread_objects_written[i]
    except:
      logging.exception('Caught exception waiting for the %dth thread.' % i)
  print 'All threads ended...'

  write_finish_time = time.time()

  final_count = len(final_objects_written)
  if final_count < TOTAL_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to provision required number of '
                               'objects, exiting.')

  print ('Done provisioning the objects, objects written %d. Now start doing'
         ' the lists...') % final_count

  # Now list this bucket under this prefix, compare the list results with
  # objects_written. If they are not the same, keep doing it until they
  # are the same.

  total_wait_time = 0
  final_objects_written_set = set(final_objects_written)
  list_count = 0
  result_consistent = False
  list_latency = 0
  while total_wait_time < CONSISTENCY_WAIT_TIME_LIMIT:

    list_start_time = time.time()
    list_result = _ListObjects(storage_schema, FLAGS.bucket, object_prefix,
                               host_to_connect)
    list_count += 1
    list_latency = time.time() - list_start_time

    if final_objects_written_set.difference(set(list_result)):
      total_wait_time = time.time() - write_finish_time
      continue
    else:
      result_consistent = True
      break

  if result_consistent:
    print "Listed %d times until results are consistent." % list_count
    if list_count == 1:
      print "List latency is: %f" % list_latency
    else:
      print "list-after-write inconsistency window is: %f" % total_wait_time
  else:
    print "Results are still inconsistent after waiting for the max limit!"

def Main(argv=sys.argv):

  logging.basicConfig(level=logging.INFO)

  try:
    argv = FLAGS(argv)  # parse flags
  except flags.FlagsError as e:
    logging.error(
        '%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
    sys.exit(1)

  if FLAGS.bucket is None:
    raise ValueError('Must specify a valid bucket for this test.')

  print 'Storage is %s, bucket is %s, scenario is %s' % (FLAGS.storage,
                                                         FLAGS.bucket,
                                                         FLAGS.scenario)
  host_to_connect = None
  storage_schema = None

  if FLAGS.host is not None:
    print 'Will use user-specified host endpoint: %s' % FLAGS.host
    host_to_connect = FLAGS.host

  if FLAGS.storage == 'AZURE':
    raise NotImplementedError('Storage type Azure is not implemented yet.')

  storage_schema = STORAGE_TO_SCHEMA_DICT[FLAGS.storage]

  if FLAGS.scenario == 'OneByteRW':
    return OneByteRWBenchmark(storage_schema, host_to_connect)
  elif FLAGS.scenario == 'ListConsistency':
    for _ in range(FLAGS.iterations):
      ListConsistencyBenchmark(storage_schema, host_to_connect)
    return 0
  elif FLAGS.scenario == 'SingleStreamThroughput':
    raise NotImplementedError('Single Stream Throughput is not implemented.')

if __name__ == '__main__':
  sys.exit(Main())
<commit_msg>Address second round of CR comments.<commit_after>#!/usr/bin/env python

# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A Test Script to be copied to the test VM to perform various tests to
   object storage via APIs exposed by the provider.

   Note: it is intentional that this test script is NOT dependant on the PKB
   package so we do not have to copy the entire PKB package to test VM just to
   run this script.
"""

import json
import logging
import sys
from threading import Thread
import time

import gflags as flags
import boto
import gcs_oauth2_boto_plugin  # noqa

FLAGS = flags.FLAGS

flags.DEFINE_enum(
    'storage', 'GCS', ['GCS', 'S3', 'AZURE'], 'The target storage to test.')

flags.DEFINE_string('host', None, 'The hostname of the storage endpoint.')

flags.DEFINE_string('bucket', None,
                    'The name of the bucket to test with. Caller is '
                    'responsible to create an empty bucket for a particular '
                    'invocation of this test and then clean-up the bucket '
                    'after this test returns.')

flags.DEFINE_enum(
    'scenario', 'OneByteRW', ['OneByteRW', 'ListConsistency',
                              'SingleStreamThroughput'],
    'The various scenarios to test. OneByteRW: read and write of single byte. '
    'ListConsistency: List-after-write and list-after-update consistency. '
    'SingleStreamThroughput: Throughput of single stream large object RW. (not '
    'implemented yet)')

flags.DEFINE_integer('iterations', 1, 'The number of iterations to run for the '
                     'particular test scenario. Currently only applicable to '
                     'the ListConsistency scenario, ignored in others.')

STORAGE_TO_SCHEMA_DICT = {'GCS': 'gs', 'S3': 's3'}

# If more than 5% of our upload or download operations fail for an iteration,
# there is an availability issue with the service provider or the connection
# between the test VM and the service provider. For this reason, we consider
# this particular iteration invalid, and we will print out an error message
# instead of a set of benchmark numbers (which will be incorrect).
FAILURE_TOLERANCE = 0.05

# When a storage provider fails more than a threshold number of requests, we
# stop the benchmarking tests and raise a low availability error back to the
# caller.
class LowAvailabilityError(Exception):
    pass

def _PercentileCalculator(numbers):
  numbers_sorted = sorted(numbers)
  count = len(numbers_sorted)
  result = {}
  result['tp50'] = numbers_sorted[int(count * 0.5)]
  result['tp90'] = numbers_sorted[int(count * 0.9)]
  result['tp99'] = numbers_sorted[int(count * 0.99)]
  result['tp99.9'] = numbers_sorted[int(count * 0.999)]
  return result

def _ListObjects(storage_schema, bucket, prefix, host_to_connect=None):
  bucket_uri = boto.storage_uri(bucket, storage_schema)
  if host_to_connect is not None:
    bucket_uri.connect(host=host_to_connect)

  list_result = []
  for k in bucket_uri.list_bucket(prefix=prefix):
    list_result.append(k.name)

  return list_result

def WriteOneByteObjects(storage_schema, bucket, object_prefix, count,
                        objects_written, latency_results=None,
                        host_to_connect=None):
  """Write a number of one byte objects to a storage provider.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket to write to.
    object_prefix: The prefix of names of objects to be written.
    count: The total number of objects that need to be written.
    objects_written: A list of names of objects that have been successfully
        written by this function. Caller supplies the list and this function
        fills in the name of the objects.
    latency_results: An optional parameter that caller can supply to hold
        a latency numbers, in seconds, for each object that is successfully
        written.
    host_to_connect: An optional endpoint string to connect to.
  """

  for i in range(count):
    object_name = '%s_%d' % (object_prefix, i)
    object_path = '%s/%s' % (bucket, object_name)
    object_uri = boto.storage_uri(object_path, storage_schema)
    # Note below, this does not really connect, it only sets the connection
    # property, the real http connnection is only established when doing the
    # actual upload.
    if host_to_connect is not None:
      object_uri.connect(host=host_to_connect)

    # Ready to go!
    start_time = time.time()

    # We write the object to storage provider and if successful we place the
    # name of the object to the objects_written list. When a write fails, the
    # object won't be placed into the objects_written list, and then it's up to
    # the caller to decide if they want to tolerate the failure and accept
    # the results, depending on test scenarios.
    try:
      object_uri.set_contents_from_string('f')

      if latency_results is not None:
        latency_results.append(time.time() - start_time)

      objects_written.append(object_name)
    except:
      logging.exception('Caught exception while writing object %s.' %
                        object_path)

def OneByteRWBenchmark(storage_schema, host_to_connect=None):
  """ A benchmark test for one byte object read and write. It uploads and
  downloads ONE_BYTE_OBJECT_COUNT number of 1-byte objects to the storage
  provider, keeps track of the latency of these operations, and print out the
  result in JSON format at the end of the test.

  Args:
    storage_schema: The schema of the storage provider to use, e.g., "gs"
    host_to_connect: An optional host endpoint to connect to.

  Raises:
    LowAvailabilityError: when the storage provider has failed a high number of
        our RW requests that exceeds a threshold (>5%), we raise this error
        instead of collecting performance numbers from this run.
  """

  # One byte write
  object_prefix = 'pkb_one_byte_%f' % time.time()
  ONE_BYTE_OBJECT_COUNT = 1000

  one_byte_write_latency = []
  one_byte_objects_written = []

  WriteOneByteObjects(storage_schema, FLAGS.bucket, object_prefix,
                      ONE_BYTE_OBJECT_COUNT, one_byte_objects_written,
                      latency_results=one_byte_write_latency,
                      host_to_connect=host_to_connect)

  success_count = len(one_byte_objects_written)
  if success_count < ONE_BYTE_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to write required number of objects, '
                               'exiting.')

  logging.info('One byte upload - %s' %
               json.dumps(_PercentileCalculator(one_byte_write_latency),
                          sort_keys=True))

  # Now download these objects and measure the latencies.
  one_byte_read_latency = []

  for object_name in one_byte_objects_written:
    object_path = '%s/%s' % (FLAGS.bucket, object_name)
    object_uri = boto.storage_uri(object_path, storage_schema)
    object_uri.connect(host=host_to_connect)

    # Ready to go!
    start_time = time.time()

    try:
      object_uri.get_contents_as_string()

      one_byte_read_latency.append(time.time() - start_time)
    except:
      logging.exception('Failed to read object %s' % object_path)

  success_count = len(one_byte_read_latency)
  if success_count < ONE_BYTE_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to read required number of objects, '
                               'exiting.')

  logging.info('One byte download - %s' %
               json.dumps(_PercentileCalculator(one_byte_read_latency),
                          sort_keys=True))

def ListConsistencyBenchmark(storage_schema, host_to_connect=None):
  """ A benchmark test to measure list-after-write consistency. It uploads
  a large number of 1-byte objects in a short amount of time, and then issues
  a list request. If the first list request returns all objects as expected,
  then the result is deemed consistent. Otherwise, it keeps issuing list request
  until all expected objects are returned, and the test reports the time
  it takes from the end of the last write to the time the list returns
  consistent result.

  Args:
    storage_schema: The schema of the storage provider to use, e.g., "gs"
    host_to_connect: An optional host endpoint to connect to.

  Raises:
    LowAvailabilityError: when the storage provider has failed a high number of
        our RW requests that exceeds a threshold (>5%), we raise this error
        instead of collecting performance numbers from this run.
  """

  # The maximum amount of seconds we are willing to wait for list results to be
  # consistent in this benchmark
  LIST_CONSISTENCY_WAIT_TIME_LIMIT = 300

  # Total number of objects we will provision before we do the list
  LIST_CONSISTENCY_OBJECT_COUNT = 5000

  # Provisioning is done in parallel threads to reduce test iteration time!
  LIST_CONSISTENCY_THREAD_COUNT = 100

  # Provision the test with tons of one byte objects. Write them all at once.
  object_prefix = 'pkb_list_consistency_%f' % time.time()
  final_objects_written = []

  per_thread_objects_written = [[] for i in
                                 range(LIST_CONSISTENCY_THREAD_COUNT)]  # noqa

  threads = []

  for i in range(LIST_CONSISTENCY_THREAD_COUNT):
    my_prefix = '%s_%d' % (object_prefix, i)
    thread = Thread(target=WriteOneByteObjects,
                    args=(storage_schema, FLAGS.bucket, my_prefix,
                          LIST_CONSISTENCY_OBJECT_COUNT /
                          LIST_CONSISTENCY_THREAD_COUNT,
                          per_thread_objects_written[i], None, host_to_connect))
    thread.daemon = True
    thread.start()
    threads.append(thread)

  logging.info('All threads started, waiting for them to end...')

  for i in range(LIST_CONSISTENCY_THREAD_COUNT):
    try:
      threads[i].join()
      final_objects_written += per_thread_objects_written[i]
    except:
      logging.exception('Caught exception waiting for the %dth thread.' % i)
  logging.info('All threads ended...')

  write_finish_time = time.time()

  final_count = len(final_objects_written)
  if final_count < LIST_CONSISTENCY_OBJECT_COUNT * (1 - FAILURE_TOLERANCE):
    raise LowAvailabilityError('Failed to provision required number of '
                               'objects, exiting.')

  logging.info(('Done provisioning the objects, objects written %d. Now start '
                'doing the lists...') % final_count)

  # Now list this bucket under this prefix, compare the list results with
  # objects_written. If they are not the same, keep doing it until they
  # are the same.

  total_wait_time = 0
  final_objects_written_set = set(final_objects_written)
  list_count = 0
  result_consistent = False
  list_latency = 0
  while total_wait_time < LIST_CONSISTENCY_WAIT_TIME_LIMIT:

    list_start_time = time.time()
    list_result = _ListObjects(storage_schema, FLAGS.bucket, object_prefix,
                               host_to_connect)
    list_count += 1
    list_latency = time.time() - list_start_time

    if final_objects_written_set.difference(set(list_result)):
      total_wait_time = time.time() - write_finish_time
      continue
    else:
      result_consistent = True
      break

  if result_consistent:
    logging.info('Listed %d times until results are consistent.' % list_count)
    if list_count == 1:
      logging.info('List latency is: %f' % list_latency)
    else:
      logging.info('List-after-write inconsistency window is: %f' %
                   total_wait_time)
  else:
    logging.info('Results are still inconsistent after waiting for the max '
                 'limit!')

def Main(argv=sys.argv):

  logging.basicConfig(level=logging.INFO)

  try:
    argv = FLAGS(argv)  # parse flags
  except flags.FlagsError as e:
    logging.error(
        '%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
    sys.exit(1)

  if FLAGS.bucket is None:
    raise ValueError('Must specify a valid bucket for this test.')

  logging.info('Storage is %s, bucket is %s, scenario is %s' % (FLAGS.storage,
                                                                FLAGS.bucket,
                                                                FLAGS.scenario))
  host_to_connect = None
  if FLAGS.host is not None:
    logging.info('Will use user-specified host endpoint: %s' % FLAGS.host)
    host_to_connect = FLAGS.host

  if FLAGS.storage == 'AZURE':
    raise NotImplementedError('Storage type Azure is not implemented yet.')

  storage_schema = STORAGE_TO_SCHEMA_DICT[FLAGS.storage]

  if FLAGS.scenario == 'OneByteRW':
    return OneByteRWBenchmark(storage_schema, host_to_connect)
  elif FLAGS.scenario == 'ListConsistency':
    for _ in range(FLAGS.iterations):
      ListConsistencyBenchmark(storage_schema, host_to_connect)
    return 0
  elif FLAGS.scenario == 'SingleStreamThroughput':
    raise NotImplementedError('Single Stream Throughput is not implemented.')

if __name__ == '__main__':
  sys.exit(Main())
<|endoftext|>


<commit_before>#!/usr/local/bin/python
# Code Fights Add Border Problem

def minesweeper(matrix):
    num_mines = []
    rows = len(matrix)
    cols = len(matrix[0])
    adj = [-1, 0, 1]
    for r in range(rows):
        curr_row = []
        for c in range(cols):
            curr_row.append(sum([matrix[r + i][c + j] for i in adj if (r + i >= 0 and r + i < rows) for j in adj if (c + j >= 0 and c + j < cols and not(i == 0 and j == 0))]))
        num_mines.append(curr_row)

    return num_mines

def main():
    tests = [
        [
            [
                [True, False, False],
                [False, True, False],
                [False, False, False]
            ],
            [
                [1, 2, 1],
                [2, 1, 1],
                [1, 1, 1]
            ]
        ],
        [
            [
                [False, False, False],
                [False, False, False]
            ],
            [
                [0, 0, 0],
                [0, 0, 0]
            ]
        ],
        [
            [
                [True, False, False, True],
                [False, False, True, False],
                [True, True, False, True]
            ],
            [
                [0, 2, 2, 1],
                [3, 4, 3, 3],
                [1, 2, 3, 1]
            ]
        ]
    ]

    for t in tests:
        res = minesweeper(t[0])
        if t[1] == res:
            print("PASSED: minesweeper({}) returned {}"
                  .format(t[0], res))
        else:
            print("FAILED: minesweeper({}) returned {}, should have returned {}"
                  .format(t[0], res, t[1]))

if __name__ == '__main__':
    main()
<commit_msg>Add docstring and fix PEP errors<commit_after>#!/usr/local/bin/python
# Code Fights Add Border Problem

def minesweeper(matrix):
    '''
    Convert minesweeper matrix showing True if mine in that cell, False
        otherwise into a matrix with counts of how many neighboring mines.
        Neighbor is a cell that shares at least a corner.
    '''
    num_mines = []
    rows = len(matrix)
    cols = len(matrix[0])
    adj = [-1, 0, 1]
    for r in range(rows):
        curr_row = []
        for c in range(cols):
            # Add neighboring cells if not out of bounds or cell itself
            curr_row.append(sum([matrix[r + i][c + j] for i in adj
                            if (r + i >= 0 and r + i < rows) for j in adj
                            if (c + j >= 0 and c + j < cols and
                                not(i == 0 and j == 0))]))
        num_mines.append(curr_row)

    return num_mines

def main():
    tests = [
        [
            [
                [True, False, False],
                [False, True, False],
                [False, False, False]
            ],
            [
                [1, 2, 1],
                [2, 1, 1],
                [1, 1, 1]
            ]
        ],
        [
            [
                [False, False, False],
                [False, False, False]
            ],
            [
                [0, 0, 0],
                [0, 0, 0]
            ]
        ],
        [
            [
                [True, False, False, True],
                [False, False, True, False],
                [True, True, False, True]
            ],
            [
                [0, 2, 2, 1],
                [3, 4, 3, 3],
                [1, 2, 3, 1]
            ]
        ]
    ]

    for t in tests:
        res = minesweeper(t[0])
        if t[1] == res:
            print("PASSED: minesweeper({}) returned {}"
                  .format(t[0], res))
        else:
            print("FAILED: minesweeper({}) returned {}, should have returned {}"
                  .format(t[0], res, t[1]))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf-8
import os
import sys

# just some slight modifications to support sum and iter again
from subzero.sandbox import restore_builtins

module = sys.modules['__main__']
restore_builtins(module, {})

globals = getattr(module, "__builtins__")["globals"]
for key, value in getattr(module, "__builtins__").iteritems():
    if key != "globals":
        globals()[key] = value

import logger
sys.modules["logger"] = logger

import subliminal
import subliminal_patch
import support

import interface
sys.modules["interface"] = interface

from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
from subzero import intent
from interface.menu import *
from support.subtitlehelpers import getSubtitlesFromMetadata
from support.storage import storeSubtitleInfo
from support.config import config

def Start():
    HTTP.CacheTime = 0
    HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT

    # configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
    subliminal.region.configure('dogpile.cache.memory')

    # init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
    ValidatePrefs()
    Log.Debug(config.full_version)

    if not config.plex_api_working:
        Log.Error(lib_unaccessible_error)
        return

    scheduler.run()

def initSubliminalPatches():
    # configure custom subtitle destination folders for scanning pre-existing subs
    dest_folder = config.subtitleDestinationFolder
    subliminal_patch.patch_video.CUSTOM_PATHS = [dest_folder] if dest_folder else []
    subliminal_patch.patch_provider_pool.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
    subliminal_patch.patch_providers.addic7ed.USE_BOOST = bool(Prefs['provider.addic7ed.boost'])

def flattenToParts(media, kind="series"):
    """
    iterates through media and returns the associated parts (videos)
    :param media:
    :param kind:
    :return:
    """
    parts = []
    if kind == "series":
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ep = media.seasons[season].episodes[episode]
                for item in media.seasons[season].episodes[episode].items:
                    for part in item.parts:
                        parts.append({"video": part, "type": "episode", "title": ep.title, "series": media.title, "id": ep.id})
    else:
        for item in media.items:
            for part in item.parts:
                parts.append({"video": part, "type": "movie", "title": media.title, "id": media.id})
    return parts

IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")

def parseMediaToParts(media, kind="series"):
    """
    returns a list of parts to be used later on; ignores folders with an existing "subzero.ignore" file
    :param media:
    :param kind:
    :return:
    """
    parts = flattenToParts(media, kind=kind)
    if not Prefs["subtitles.ignore_fs"]:
        return parts

    use_parts = []
    check_ignore_paths = [".", "../"]
    if kind == "series":
        check_ignore_paths.append("../../")

    for part in parts:
        base_folder, fn = os.path.split(part["video"].file)

        ignore = False
        for rel_path in check_ignore_paths:
            fld = os.path.abspath(os.path.join(base_folder, rel_path))
            for ifn in IGNORE_FN:
                if os.path.isfile(os.path.join(fld, ifn)):
                    Log.Info(u'Ignoring "%s" because "%s" exists in "%s"', fn, ifn, fld)
                    ignore = True
                    break
            if ignore:
                break

        if not ignore:
            use_parts.append(part)
    return use_parts

def scanParts(parts, kind="series"):
    """
    receives a list of parts containing dictionaries returned by flattenToParts
    :param parts:
    :param kind: series or movies
    :return: dictionary of scanned videos of subliminal.video.scan_video
    """
    ret = {}
    for part in parts:
        force_refresh = intent.get("force", part["id"])
        hints = {"expected_title": [part["title"]]}
        hints.update({"type": "episode", "expected_series": [part["series"]]} if kind == "series" else {"type": "movie"})
        scanned_video = scanVideo(part["video"], ignore_all=force_refresh, hints=hints)
        if not scanned_video:
            continue

        scanned_video.id = part["id"]
        ret[scanned_video] = part["video"]
    return ret

def getItemIDs(media, kind="series"):
    ids = []
    if kind == "movies":
        ids.append(media.id)
    else:
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ids.append(media.seasons[season].episodes[episode].id)

    return ids

def scanVideo(part, ignore_all=False, hints=None):
    embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
    external_subtitles = not ignore_all and Prefs['subtitles.scan.external']

    if ignore_all:
        Log.Debug("Force refresh intended.")

    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (part.file, external_subtitles, embedded_subtitles))

    try:
        return subliminal.video.scan_video(part.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles, hints=hints or {})

    except ValueError:
        Log.Warn("File could not be guessed by subliminal")

def downloadBestSubtitles(video_part_map, min_score=0):
    hearing_impaired = Prefs['subtitles.search.hearingImpaired']
    languages = config.langList
    if not languages:
        return

    missing_languages = False
    for video, part in video_part_map.iteritems():
        if not Prefs['subtitles.save.filesystem']:
            # scan for existing metadata subtitles
            meta_subs = getSubtitlesFromMetadata(part)
            for language, subList in meta_subs.iteritems():
                if subList:
                    video.subtitle_languages.add(language)
                    Log.Debug("Found metadata subtitle %s for %s", language, video)

        if not (languages - video.subtitle_languages):
            Log.Debug('All languages %r exist for %s', languages, video)
            continue
        missing_languages = True
        break

    if missing_languages:
        Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" % (min_score, hearing_impaired))

        return subliminal.api.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
                                                      provider_configs=config.providerSettings)
    Log.Debug("All languages for all requested videos exist. Doing nothing.")

def saveSubtitles(videos, subtitles):
    if Prefs['subtitles.save.filesystem']:
        Log.Debug("Using filesystem as subtitle storage")
        saveSubtitlesToFile(subtitles)
        storage = "filesystem"
    else:
        Log.Debug("Using metadata as subtitle storage")
        saveSubtitlesToMetadata(videos, subtitles)
        storage = "metadata"

    storeSubtitleInfo(videos, subtitles, storage)

def saveSubtitlesToFile(subtitles):
    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None

    for video, video_subtitles in subtitles.items():
        if not video_subtitles:
            continue

        fld = None
        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
            # specific subFolder requested, create it if it doesn't exist
            fld_base = os.path.split(video.name)[0]
            if fld_custom:
                if fld_custom.startswith("/"):
                    # absolute folder
                    fld = fld_custom
                else:
                    fld = os.path.join(fld_base, fld_custom)
            else:
                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
            if not os.path.exists(fld):
                os.makedirs(fld)
        subliminal.api.save_subtitles(video, video_subtitles, directory=fld)

def saveSubtitlesToMetadata(videos, subtitles):
    for video, video_subtitles in subtitles.items():
        mediaPart = videos[video]
        for subtitle in video_subtitles:
            mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.page_link] = Proxy.Media(subtitle.content, ext="srt")

def updateLocalMedia(metadata, media, media_type="movies"):
    # Look for subtitles
    if media_type == "movies":
        for item in media.items:
            for part in item.parts:
                support.localmedia.findSubtitles(part)
        return

    # Look for subtitles for each episode.
    for s in media.seasons:
        # If we've got a date based season, ignore it for now, otherwise it'll collide with S/E folders/XML and PMS
        # prefers date-based (why?)
        if int(s) < 1900 or metadata.guid.startswith(PERSONAL_MEDIA_IDENTIFIER):
            for e in media.seasons[s].episodes:
                for i in media.seasons[s].episodes[e].items:

                    # Look for subtitles.
                    for part in i.parts:
                        support.localmedia.findSubtitles(part)
        else:
            pass

class SubZeroAgent(object):
    agent_type = None
    agent_type_short = None
    languages = [Locale.Language.English]
    primary_provider = False
    score_prefs_key = None

    def __init__(self, *args, **kwargs):
        super(SubZeroAgent, self).__init__(*args, **kwargs)
        self.agent_type = "movies" if isinstance(self, Agent.Movies) else "series"
        self.name = "Sub-Zero Subtitles (%s, %s)" % (self.agent_type_short, config.getVersion())

    def search(self, results, media, lang):
        Log.Debug("Sub-Zero %s, %s search" % (config.version, self.agent_type))
        results.Append(MetadataSearchResult(id='null', score=100))

    def update(self, metadata, media, lang):
        Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))

        if not media:
            Log.Error("Called with empty media, something is really wrong with your setup!")
            return

        set_refresh_menu_state(media, media_type=self.agent_type)

        item_ids = []
        try:
            initSubliminalPatches()
            parts = parseMediaToParts(media, kind=self.agent_type)
            use_score = Prefs[self.score_prefs_key]
            scanned_parts = scanParts(parts, kind=self.agent_type)
            subtitles = downloadBestSubtitles(scanned_parts, min_score=int(use_score))
            item_ids = getItemIDs(media, kind=self.agent_type)

            if subtitles:
                saveSubtitles(scanned_parts, subtitles)

            updateLocalMedia(metadata, media, media_type=self.agent_type)

        finally:
            # update the menu state
            set_refresh_menu_state(None)

            # notify any running tasks about our finished update
            for item_id in item_ids:
                scheduler.signal("updated_metadata", item_id)

                # resolve existing intent for that id
                intent.resolve("force", item_id)

class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies):
    contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', 'com.plexapp.agents.hama']
    score_prefs_key = "subtitles.search.minimumMovieScore"
    agent_type_short = "Movies"

class SubZeroSubtitlesAgentTvShows(SubZeroAgent, Agent.TV_Shows):
    contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv', 'com.plexapp.agents.hama']
    score_prefs_key = "subtitles.search.minimumTVScore"
    agent_type_short = "TV"
<commit_msg>rename SubZeroAgent.agent_type_short to agent_type_verbose<commit_after># coding=utf-8
import os
import sys

# just some slight modifications to support sum and iter again
from subzero.sandbox import restore_builtins

module = sys.modules['__main__']
restore_builtins(module, {})

globals = getattr(module, "__builtins__")["globals"]
for key, value in getattr(module, "__builtins__").iteritems():
    if key != "globals":
        globals()[key] = value

import logger
sys.modules["logger"] = logger

import subliminal
import subliminal_patch
import support

import interface
sys.modules["interface"] = interface

from subzero.constants import OS_PLEX_USERAGENT, PERSONAL_MEDIA_IDENTIFIER
from subzero import intent
from interface.menu import *
from support.subtitlehelpers import getSubtitlesFromMetadata
from support.storage import storeSubtitleInfo
from support.config import config

def Start():
    HTTP.CacheTime = 0
    HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT

    # configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
    subliminal.region.configure('dogpile.cache.memory')

    # init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
    ValidatePrefs()
    Log.Debug(config.full_version)

    if not config.plex_api_working:
        Log.Error(lib_unaccessible_error)
        return

    scheduler.run()

def initSubliminalPatches():
    # configure custom subtitle destination folders for scanning pre-existing subs
    dest_folder = config.subtitleDestinationFolder
    subliminal_patch.patch_video.CUSTOM_PATHS = [dest_folder] if dest_folder else []
    subliminal_patch.patch_provider_pool.DOWNLOAD_TRIES = int(Prefs['subtitles.try_downloads'])
    subliminal_patch.patch_providers.addic7ed.USE_BOOST = bool(Prefs['provider.addic7ed.boost'])

def flattenToParts(media, kind="series"):
    """
    iterates through media and returns the associated parts (videos)
    :param media:
    :param kind:
    :return:
    """
    parts = []
    if kind == "series":
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ep = media.seasons[season].episodes[episode]
                for item in media.seasons[season].episodes[episode].items:
                    for part in item.parts:
                        parts.append({"video": part, "type": "episode", "title": ep.title, "series": media.title, "id": ep.id})
    else:
        for item in media.items:
            for part in item.parts:
                parts.append({"video": part, "type": "movie", "title": media.title, "id": media.id})
    return parts

IGNORE_FN = ("subzero.ignore", ".subzero.ignore", ".nosz")

def parseMediaToParts(media, kind="series"):
    """
    returns a list of parts to be used later on; ignores folders with an existing "subzero.ignore" file
    :param media:
    :param kind:
    :return:
    """
    parts = flattenToParts(media, kind=kind)
    if not Prefs["subtitles.ignore_fs"]:
        return parts

    use_parts = []
    check_ignore_paths = [".", "../"]
    if kind == "series":
        check_ignore_paths.append("../../")

    for part in parts:
        base_folder, fn = os.path.split(part["video"].file)

        ignore = False
        for rel_path in check_ignore_paths:
            fld = os.path.abspath(os.path.join(base_folder, rel_path))
            for ifn in IGNORE_FN:
                if os.path.isfile(os.path.join(fld, ifn)):
                    Log.Info(u'Ignoring "%s" because "%s" exists in "%s"', fn, ifn, fld)
                    ignore = True
                    break
            if ignore:
                break

        if not ignore:
            use_parts.append(part)
    return use_parts

def scanParts(parts, kind="series"):
    """
    receives a list of parts containing dictionaries returned by flattenToParts
    :param parts:
    :param kind: series or movies
    :return: dictionary of scanned videos of subliminal.video.scan_video
    """
    ret = {}
    for part in parts:
        force_refresh = intent.get("force", part["id"])
        hints = {"expected_title": [part["title"]]}
        hints.update({"type": "episode", "expected_series": [part["series"]]} if kind == "series" else {"type": "movie"})
        scanned_video = scanVideo(part["video"], ignore_all=force_refresh, hints=hints)
        if not scanned_video:
            continue

        scanned_video.id = part["id"]
        ret[scanned_video] = part["video"]
    return ret

def getItemIDs(media, kind="series"):
    ids = []
    if kind == "movies":
        ids.append(media.id)
    else:
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ids.append(media.seasons[season].episodes[episode].id)

    return ids

def scanVideo(part, ignore_all=False, hints=None):
    embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
    external_subtitles = not ignore_all and Prefs['subtitles.scan.external']

    if ignore_all:
        Log.Debug("Force refresh intended.")

    Log.Debug("Scanning video: %s, subtitles=%s, embedded_subtitles=%s" % (part.file, external_subtitles, embedded_subtitles))

    try:
        return subliminal.video.scan_video(part.file, subtitles=external_subtitles, embedded_subtitles=embedded_subtitles, hints=hints or {})

    except ValueError:
        Log.Warn("File could not be guessed by subliminal")

def downloadBestSubtitles(video_part_map, min_score=0):
    hearing_impaired = Prefs['subtitles.search.hearingImpaired']
    languages = config.langList
    if not languages:
        return

    missing_languages = False
    for video, part in video_part_map.iteritems():
        if not Prefs['subtitles.save.filesystem']:
            # scan for existing metadata subtitles
            meta_subs = getSubtitlesFromMetadata(part)
            for language, subList in meta_subs.iteritems():
                if subList:
                    video.subtitle_languages.add(language)
                    Log.Debug("Found metadata subtitle %s for %s", language, video)

        if not (languages - video.subtitle_languages):
            Log.Debug('All languages %r exist for %s', languages, video)
            continue
        missing_languages = True
        break

    if missing_languages:
        Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s" % (min_score, hearing_impaired))

        return subliminal.api.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
                                                      provider_configs=config.providerSettings)
    Log.Debug("All languages for all requested videos exist. Doing nothing.")

def saveSubtitles(videos, subtitles):
    if Prefs['subtitles.save.filesystem']:
        Log.Debug("Using filesystem as subtitle storage")
        saveSubtitlesToFile(subtitles)
        storage = "filesystem"
    else:
        Log.Debug("Using metadata as subtitle storage")
        saveSubtitlesToMetadata(videos, subtitles)
        storage = "metadata"

    storeSubtitleInfo(videos, subtitles, storage)

def saveSubtitlesToFile(subtitles):
    fld_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None

    for video, video_subtitles in subtitles.items():
        if not video_subtitles:
            continue

        fld = None
        if fld_custom or Prefs["subtitles.save.subFolder"] != "current folder":
            # specific subFolder requested, create it if it doesn't exist
            fld_base = os.path.split(video.name)[0]
            if fld_custom:
                if fld_custom.startswith("/"):
                    # absolute folder
                    fld = fld_custom
                else:
                    fld = os.path.join(fld_base, fld_custom)
            else:
                fld = os.path.join(fld_base, Prefs["subtitles.save.subFolder"])
            if not os.path.exists(fld):
                os.makedirs(fld)
        subliminal.api.save_subtitles(video, video_subtitles, directory=fld)

def saveSubtitlesToMetadata(videos, subtitles):
    for video, video_subtitles in subtitles.items():
        mediaPart = videos[video]
        for subtitle in video_subtitles:
            mediaPart.subtitles[Locale.Language.Match(subtitle.language.alpha2)][subtitle.page_link] = Proxy.Media(subtitle.content, ext="srt")

def updateLocalMedia(metadata, media, media_type="movies"):
    # Look for subtitles
    if media_type == "movies":
        for item in media.items:
            for part in item.parts:
                support.localmedia.findSubtitles(part)
        return

    # Look for subtitles for each episode.
    for s in media.seasons:
        # If we've got a date based season, ignore it for now, otherwise it'll collide with S/E folders/XML and PMS
        # prefers date-based (why?)
        if int(s) < 1900 or metadata.guid.startswith(PERSONAL_MEDIA_IDENTIFIER):
            for e in media.seasons[s].episodes:
                for i in media.seasons[s].episodes[e].items:

                    # Look for subtitles.
                    for part in i.parts:
                        support.localmedia.findSubtitles(part)
        else:
            pass

class SubZeroAgent(object):
    agent_type = None
    agent_type_verbose = None
    languages = [Locale.Language.English]
    primary_provider = False
    score_prefs_key = None

    def __init__(self, *args, **kwargs):
        super(SubZeroAgent, self).__init__(*args, **kwargs)
        self.agent_type = "movies" if isinstance(self, Agent.Movies) else "series"
        self.name = "Sub-Zero Subtitles (%s, %s)" % (self.agent_type_verbose, config.getVersion())

    def search(self, results, media, lang):
        Log.Debug("Sub-Zero %s, %s search" % (config.version, self.agent_type))
        results.Append(MetadataSearchResult(id='null', score=100))

    def update(self, metadata, media, lang):
        Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))

        if not media:
            Log.Error("Called with empty media, something is really wrong with your setup!")
            return

        set_refresh_menu_state(media, media_type=self.agent_type)

        item_ids = []
        try:
            initSubliminalPatches()
            parts = parseMediaToParts(media, kind=self.agent_type)
            use_score = Prefs[self.score_prefs_key]
            scanned_parts = scanParts(parts, kind=self.agent_type)
            subtitles = downloadBestSubtitles(scanned_parts, min_score=int(use_score))
            item_ids = getItemIDs(media, kind=self.agent_type)

            if subtitles:
                saveSubtitles(scanned_parts, subtitles)

            updateLocalMedia(metadata, media, media_type=self.agent_type)

        finally:
            # update the menu state
            set_refresh_menu_state(None)

            # notify any running tasks about our finished update
            for item_id in item_ids:
                scheduler.signal("updated_metadata", item_id)

                # resolve existing intent for that id
                intent.resolve("force", item_id)

class SubZeroSubtitlesAgentMovies(SubZeroAgent, Agent.Movies):
    contributes_to = ['com.plexapp.agents.imdb', 'com.plexapp.agents.xbmcnfo', 'com.plexapp.agents.themoviedb', 'com.plexapp.agents.hama']
    score_prefs_key = "subtitles.search.minimumMovieScore"
    agent_type_verbose = "Movies"

class SubZeroSubtitlesAgentTvShows(SubZeroAgent, Agent.TV_Shows):
    contributes_to = ['com.plexapp.agents.thetvdb', 'com.plexapp.agents.thetvdbdvdorder', 'com.plexapp.agents.xbmcnfotv', 'com.plexapp.agents.hama']
    score_prefs_key = "subtitles.search.minimumTVScore"
    agent_type_verbose = "TV"
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Provides a `bind` function to plugins so they can simply bind a function to a queue.
"""

import logging
from json import loads, dumps, decoder
from os import environ
from os.path import join
import urllib
import time

import pika
from pythonjsonlogger import jsonlogger
from minio import Minio
from minio.error import AccessDenied

LOG = logging.getLogger()
HND = logging.StreamHandler()
HND.setFormatter(jsonlogger.JsonFormatter())
LOG.addHandler(HND)

class ProcessingError(Exception):
    """Exception raised for errors during processing.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message: str, job_id: str=None, task_id: str=None, extra: dict=None):
        super(ProcessingError, self).__init__(message)
        self.message = message
        self.extra = extra
        self.meta = {"job_id": job_id, "task_id": task_id}

def bind(function: callable, name: str, version="1.0.0"):
    """binds a function to the input message queue"""

    parameters = pika.ConnectionParameters(
        host=environ["AMQP_HOST"],
        port=int(environ["AMQP_PORT"]),
        credentials=pika.PlainCredentials(environ["AMQP_USER"], environ["AMQP_PASS"]))

    connection = pika.BlockingConnection(parameters)
    input_channel = connection.channel()
    output_channel = connection.channel()

    minio_client = Minio(
        environ["MINIO_HOST"] + ":" + environ["MINIO_PORT"],
        access_key=environ["MINIO_ACCESS"],
        secret_key=environ["MINIO_SECRET"],
        secure=True if environ["MINIO_SCHEME"] == "https" else False)
    minio_client.set_app_info(name, version)

    minio_client.list_buckets()

    monitor_url = environ["MONITOR_URL"]

    logging.info("initialised sld lib")

    def send(chan, method, properties, body: str):
        """unwraps a message and calls the user function"""

        logging.debug("consumed message", extra={
            "chan": chan,
            "method": method,
            "properties": properties})

        raw = body.decode("utf-8")
        payload = loads(raw)
        validate_payload(payload, name)

        next_plugin = get_next_plugin(name, payload["nmo"]["job"]["workflow"])
        if next_plugin is None:
            logging.debug("this is the final plugin", extra={
                "job_id": payload["nmo"]["job"]["job_id"],
                "task_id": payload["nmo"]["task"]["task_id"]})

        path = join(
            payload["nmo"]["task"]["task_id"],
            "input",
            "source",
            payload["nmo"]["source"]["name"])

        if not minio_client.bucket_exists(payload["nmo"]["job"]["job_id"]):
            raise ProcessingError(
                "job_id does not have a bucket",
                job_id: payload["nmo"]["job"]["job_id"],
                task_id: payload["nmo"]["task"]["task_id"])

        url = minio_client.presigned_get_object(payload["nmo"]["job"]["job_id"], path)

        # calls the user function to mutate the JSON-LD data

        result = function(payload["nmo"], payload["jsonld"], url)

        # if there are issues, just use the input and carry on the pipeline

        if result is None:
            logging.error("return value is None")
            result = payload["jsonld"]

        if not isinstance(result, dict):
            logging.error("return value must be of type dict, not %s", type(result))
            result = payload["jsonld"]

        if "jsonld" in result:
            result = result["jsonld"]
        else:
            result = result

        payload["jsonld"] = result

        logging.debug("finished running user code", extra={
            "job_id": payload["nmo"]["job"]["job_id"],
            "task_id": payload["nmo"]["task"]["task_id"]})

        try:
            urllib.request.Request(
                join(monitor_url, "/v1/task/status/",
                     payload["nmo"]["job"]["job_id"], payload["nmo"]["task"]["task_id"]),
                data=dumps({
                    "t": "",
                    "p": int(time.time())
                }).encode(),
                headers={"Content-Type: application/json"})

        except Exception as exp:
            logging.error(exp)

        input_channel.basic_ack(method.delivery_tag)

        if next_plugin:
            output_channel.queue_declare(
                next_plugin,
                False,
                True,
                False,
                False,
            )
            output_channel.basic_publish(
                "",
                next_plugin,
                dumps(payload)
            )

    logging.debug("consuming from", extra={"queue": name})

    try:
        while True:
            input_channel.queue_declare(name, False, True)
            method_frame, header_frame, body = input_channel.basic_get(name)
            if (method_frame, header_frame, body) == (None, None, None):
                continue  # queue empty

            if body is None:
                logging.error("body received was empty")
                continue  # body empty

            try:
                send(input_channel, method_frame, header_frame, body)

            except ProcessingError as exp:
                input_channel.basic_reject(method_frame.delivery_tag, False)
                logging.error("Processing Error: " + exp.message, extra={**exp.meta, **exp.extra})

            except Exception as exp:
                input_channel.basic_reject(method_frame.delivery_tag, False)
                logging.error("Other Error: " + exp)

    except pika.exceptions.RecursionError as exp:
        input_channel.stop_consuming()
        connection.close()
        raise exp

def validate_payload(payload: dict, name: str) -> bool:
    """ensures payload includes the required metadata and this plugin is in there"""

    if "nmo" not in payload:
        raise ProcessingError("no job in nmo")

    if "job" not in payload["nmo"]:
        raise ProcessingError("no job in nmo")

    if "task" not in payload["nmo"]:
        raise ProcessingError("no task in nmo")

    if not ensure_this_plugin(name, payload["nmo"]["job"]["workflow"]):
        raise ProcessingError(
            "declared plugin name does not match workflow",
            job_id=payload["nmo"]["job"]["job_id"],
            task_id=payload["nmo"]["task"]["task_id"])

def ensure_this_plugin(this_plugin: str, workflow: list)->bool:
    """ensures the current plugin is present in the workflow"""
    for workpipe in workflow:
        if workpipe["config"]["name"] == this_plugin:
            return True
    return False

def get_next_plugin(this_plugin: str, workflow: list) -> str:
    """returns the next plugin in the sequence"""
    found = False
    for workpipe in workflow:
        if not found:
            if workpipe["config"]["name"] == this_plugin:
                found = True
        else:
            return workpipe["config"]["name"]

    return None
<commit_msg>fixed bug with error print<commit_after>#!/usr/bin/env python3
"""
Provides a `bind` function to plugins so they can simply bind a function to a queue.
"""

import logging
from json import loads, dumps, decoder
from os import environ
from os.path import join
import urllib
import time

import pika
from pythonjsonlogger import jsonlogger
from minio import Minio
from minio.error import AccessDenied

LOG = logging.getLogger()
HND = logging.StreamHandler()
HND.setFormatter(jsonlogger.JsonFormatter())
LOG.addHandler(HND)

class ProcessingError(Exception):
    """Exception raised for errors during processing.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message: str, job_id: str=None, task_id: str=None, extra: dict=None):
        super(ProcessingError, self).__init__(message)
        self.message = message
        self.extra = extra or {}
        self.meta = {"job_id": job_id, "task_id": task_id}

def bind(function: callable, name: str, version="1.0.0"):
    """binds a function to the input message queue"""

    parameters = pika.ConnectionParameters(
        host=environ["AMQP_HOST"],
        port=int(environ["AMQP_PORT"]),
        credentials=pika.PlainCredentials(environ["AMQP_USER"], environ["AMQP_PASS"]))

    connection = pika.BlockingConnection(parameters)
    input_channel = connection.channel()
    output_channel = connection.channel()

    minio_client = Minio(
        environ["MINIO_HOST"] + ":" + environ["MINIO_PORT"],
        access_key=environ["MINIO_ACCESS"],
        secret_key=environ["MINIO_SECRET"],
        secure=True if environ["MINIO_SCHEME"] == "https" else False)
    minio_client.set_app_info(name, version)

    minio_client.list_buckets()

    monitor_url = environ["MONITOR_URL"]

    logging.info("initialised sld lib")

    def send(chan, method, properties, body: str):
        """unwraps a message and calls the user function"""

        logging.debug("consumed message", extra={
            "chan": chan,
            "method": method,
            "properties": properties})

        raw = body.decode("utf-8")
        payload = loads(raw)
        validate_payload(payload, name)

        next_plugin = get_next_plugin(name, payload["nmo"]["job"]["workflow"])
        if next_plugin is None:
            logging.debug("this is the final plugin", extra={
                "job_id": payload["nmo"]["job"]["job_id"],
                "task_id": payload["nmo"]["task"]["task_id"]})

        path = join(
            payload["nmo"]["task"]["task_id"],
            "input",
            "source",
            payload["nmo"]["source"]["name"])

        if not minio_client.bucket_exists(payload["nmo"]["job"]["job_id"]):
            raise ProcessingError(
                "job_id does not have a bucket",
                job_id=payload["nmo"]["job"]["job_id"],
                task_id=payload["nmo"]["task"]["task_id"])

        url = minio_client.presigned_get_object(payload["nmo"]["job"]["job_id"], path)

        # calls the user function to mutate the JSON-LD data

        result = function(payload["nmo"], payload["jsonld"], url)

        # if there are issues, just use the input and carry on the pipeline

        if result is None:
            logging.error("return value is None")
            result = payload["jsonld"]

        if not isinstance(result, dict):
            logging.error("return value must be of type dict, not %s", type(result))
            result = payload["jsonld"]

        if "jsonld" in result:
            result = result["jsonld"]
        else:
            result = result

        payload["jsonld"] = result

        logging.debug("finished running user code", extra={
            "job_id": payload["nmo"]["job"]["job_id"],
            "task_id": payload["nmo"]["task"]["task_id"]})

        try:
            urllib.request.Request(
                join(monitor_url, "/v1/task/status/",
                     payload["nmo"]["job"]["job_id"], payload["nmo"]["task"]["task_id"]),
                data=dumps({
                    "t": "",
                    "p": int(time.time())
                }).encode(),
                headers={"Content-Type: application/json"})

        except Exception as exp:
            logging.error(exp)

        input_channel.basic_ack(method.delivery_tag)

        if next_plugin:
            output_channel.queue_declare(
                next_plugin,
                False,
                True,
                False,
                False,
            )
            output_channel.basic_publish(
                "",
                next_plugin,
                dumps(payload)
            )

    logging.debug("consuming from", extra={"queue": name})

    try:
        while True:
            input_channel.queue_declare(name, False, True)
            method_frame, header_frame, body = input_channel.basic_get(name)
            if (method_frame, header_frame, body) == (None, None, None):
                continue  # queue empty

            if body is None:
                logging.error("body received was empty")
                continue  # body empty

            try:
                send(input_channel, method_frame, header_frame, body)

            except ProcessingError as exp:
                input_channel.basic_reject(method_frame.delivery_tag, False)
                logging.error("Processing Error: " + exp.message, extra={**exp.meta, **exp.extra})

            except Exception as exp:
                input_channel.basic_reject(method_frame.delivery_tag, False)
                logging.error("Other Error: " + exp)

    except pika.exceptions.RecursionError as exp:
        input_channel.stop_consuming()
        connection.close()
        raise exp

def validate_payload(payload: dict, name: str) -> bool:
    """ensures payload includes the required metadata and this plugin is in there"""

    if "nmo" not in payload:
        raise ProcessingError("no job in nmo")

    if "job" not in payload["nmo"]:
        raise ProcessingError("no job in nmo")

    if "task" not in payload["nmo"]:
        raise ProcessingError("no task in nmo")

    if not ensure_this_plugin(name, payload["nmo"]["job"]["workflow"]):
        raise ProcessingError(
            "declared plugin name does not match workflow",
            job_id=payload["nmo"]["job"]["job_id"],
            task_id=payload["nmo"]["task"]["task_id"])

def ensure_this_plugin(this_plugin: str, workflow: list)->bool:
    """ensures the current plugin is present in the workflow"""
    for workpipe in workflow:
        if workpipe["config"]["name"] == this_plugin:
            return True
    return False

def get_next_plugin(this_plugin: str, workflow: list) -> str:
    """returns the next plugin in the sequence"""
    found = False
    for workpipe in workflow:
        if not found:
            if workpipe["config"]["name"] == this_plugin:
                found = True
        else:
            return workpipe["config"]["name"]

    return None
<|endoftext|>


<commit_before>import cast

VIDEO_ENDPOINT = "http://www.giantbomb.com/api/video/%d/?api_key=%s&format=json"

def Start():
    HTTP.CacheTime = CACHE_1DAY

def getJSON(video_id):
    video_id = int(video_id)
    url = VIDEO_ENDPOINT % (video_id, Prefs['api_key'])

    return JSON.ObjectFromURL(url)['results']

class GiantBombAgent(Agent.Movies):

    name = 'Giant Bomb'
    languages = [Locale.Language.English]
    primary_provider = True
    accepts_from = ['com.plexapp.agents.localmedia']
    contributes_to = ['com.plexapp.agents.localmedia']

    def search(self, results, media, lang, manual):
        video_id = int(media.name.split()[0])
        obj = getJSON(video_id)
        year = obj['publish_date'].split("-")[0]

        results.Append(MetadataSearchResult(
            id=str(video_id),
            name=obj['name'],
            year=year,
            score=100,
            lang=lang
        ))

    def update(self, metadata, media, lang):
        obj = getJSON(metadata.id)

        metadata.collections = obj['video_type'].split(', ')
        metadata.duration    = obj['length_seconds'] * 1000
        metadata.studio      = 'Giant Bomb'
        metadata.summary     = obj['deck']
        metadata.title       = obj['name']
        metadata.year        = int(obj['publish_date'].split("-")[0])

        members = cast.cast_from_deck(obj['deck'])
        for member in members:
            role = metadata.roles.new()
            role.role = 'Self'
            role.actor = member

        image = obj['image']['super_url']
        metadata.posters[image] = Proxy.Media(HTTP.Request(image).content)
<commit_msg>Use HTTPS<commit_after>import cast

VIDEO_ENDPOINT = "https://www.giantbomb.com/api/video/%d/?api_key=%s&format=json"

def Start():
    HTTP.CacheTime = CACHE_1DAY

def getJSON(video_id):
    video_id = int(video_id)
    url = VIDEO_ENDPOINT % (video_id, Prefs['api_key'])

    return JSON.ObjectFromURL(url)['results']

class GiantBombAgent(Agent.Movies):

    name = 'Giant Bomb'
    languages = [Locale.Language.English]
    primary_provider = True
    accepts_from = ['com.plexapp.agents.localmedia']
    contributes_to = ['com.plexapp.agents.localmedia']

    def search(self, results, media, lang, manual):
        video_id = int(media.name.split()[0])
        obj = getJSON(video_id)
        year = obj['publish_date'].split("-")[0]

        results.Append(MetadataSearchResult(
            id=str(video_id),
            name=obj['name'],
            year=year,
            score=100,
            lang=lang
        ))

    def update(self, metadata, media, lang):
        obj = getJSON(metadata.id)

        metadata.collections = obj['video_type'].split(', ')
        metadata.duration    = obj['length_seconds'] * 1000
        metadata.studio      = 'Giant Bomb'
        metadata.summary     = obj['deck']
        metadata.title       = obj['name']
        metadata.year        = int(obj['publish_date'].split("-")[0])

        members = cast.cast_from_deck(obj['deck'])
        for member in members:
            role = metadata.roles.new()
            role.role = 'Self'
            role.actor = member

        image = obj['image']['super_url']
        metadata.posters[image] = Proxy.Media(HTTP.Request(image).content)
<|endoftext|>


<commit_before>import re
import urllib.parse

import web

regex = '\?([\w=&])'

class QueryMixIn:
    group = 0

    def respond(self):
        self.request.params = dict(urllib.parse.parse_qsl(self.groups[self.group], True))

def new(base, handler):
    class GenQueryHandler(handler, QueryMixIn):
        pass

    GenQueryHandler.group = re.compile(base).groups + 1

    return {base + regex: handler}
<commit_msg>change query member variable<commit_after>import re
import urllib.parse

import web

regex = '\?([\w=&])'

class QueryMixIn:
    group = 0

    def respond(self):
        self.request.query = dict(urllib.parse.parse_qsl(self.groups[self.group], True))

def new(base, handler):
    class GenQueryHandler(handler, QueryMixIn):
        pass

    GenQueryHandler.group = re.compile(base).groups + 1

    return {base + regex: handler}
<|endoftext|>


<commit_before>"""
Copyright 2015 Paul T. Grogan, Massachusetts Institute of Technology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
import math

from .game import Game
from .simulator import Simulator

class OFS(object):
    def __init__(self, elements=None, numPlayers=1,
                 initialCash=1200, numTurns=12,
                 seed=0, ops='d', fops=''):
        """
        @param elements: elements
        @type elements: L{str}
        @param numPlayers: number of players
        @type numPlayers: L{int}
        @param initialCash: initial cash
        @type initialCash: L{float}
        @param numTurns: number of turns
        @type numTurns: L{int}
        @param seed: random number seed
        @type seed: L{int}
        @param ops: operations specification
        @type ops: L{str}
        @param fops: federation operations specification
        @type fops: L{str}
        """
        self.game = Game(numPlayers=numPlayers,
                         initialCash=initialCash)
        self.context = self.game.generateContext(
            seed=seed, ops=ops, fops=fops)
        self.sim = Simulator(entities=[self.context],
                             initTime=0,  timeStep=1,
                             maxTime=numTurns)
        if elements is None:
            elements = []

        def initializeGame(time):
            federates = [federate for federation in self.context.federations
                         for federate in federation.federates]
            elementSets = []
            for federate in federates:
                elementSets.append([])

            for eId, element in enumerate(elements):
                specs = element.split(',')
                if len(specs) > 0 and len(specs[0].split('@')) == 2:
                    # parse player ownership and element type
                    if len(specs[0].split('@')[0].split('.')) == 2:
                        pId = int(specs[0].split('@')[0].split('.')[0])-1
                        eType = specs[0].split('@')[0].split('.')[1]
                    else:
                        pId = 0
                        eType = specs[0].split('@')[0]

                    # parse location
                    location = next((l for l in self.context.locations
                                     if l.name == specs[0].split('@')[1]), None)

                    # parse modules
                    if pId < len(federates) and location is not None:
                        # generate elements
                        element = self.game.generateElement(eType, pId, eId, mTypes=specs[1:])
                        if element is not None:
                            elementSets[pId].append({'element':element,
                                                    'location':location})

            for i, elementSet in enumerate(elementSets):
                federate = federates[i]
                if federate.initialCash == 0:
                    # special case if 0 initial cash: grant enough for initial element
                    for element in elementSet:
                        federate.initialCash += element['element'].getDesignCost()
                        federate.initialCash += element['element'].getCommissionCost(
                            element['location'], self.context)
                    federate.cash = federate.initialCash
                for element in elementSet:
                    federate.design(element['element'])
                    federate.commission(element['element'], element['location'], self.context)

        def finalizeGame(time):
            for federate in [federate for federation in self.context.federations
                             for federate in federation.federates]:
                federate.liquidate(self.context)
                logging.info('{} final cash: {}'
                            .format(federate.name, federate.cash))
                logging.info('{} ROI: {:.2f}'
                            .format(federate.name, (float(federate.cash)/federate.initialCash)
                                    if federate.initialCash != 0 else 0))
                logging.info('{} %ROI/turn: {:.2%}'
                            .format(federate.name, (math.pow(float(federate.cash)
                                                             /federate.initialCash,
                                                             1./self.sim.maxTime)-1)
                                    if federate.initialCash != 0 else 0))
        self.sim.on('init', initializeGame)
        self.sim.on('complete', finalizeGame)

    def execute(self):
        """
        Executes an OFS.
        @return: L{list}
        """
        self.sim.execute()
        results = []
        for federate in [federate for federation in self.context.federations
                         for federate in federation.federates]:
            results.append((federate.initialCash, federate.cash))
        return results<commit_msg>remove finalize computations<commit_after>"""
Copyright 2015 Paul T. Grogan, Massachusetts Institute of Technology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
import math

from .game import Game
from .simulator import Simulator

class OFS(object):
    def __init__(self, elements=None, numPlayers=1,
                 initialCash=1200, numTurns=12,
                 seed=0, ops='d', fops=''):
        """
        @param elements: elements
        @type elements: L{str}
        @param numPlayers: number of players
        @type numPlayers: L{int}
        @param initialCash: initial cash
        @type initialCash: L{float}
        @param numTurns: number of turns
        @type numTurns: L{int}
        @param seed: random number seed
        @type seed: L{int}
        @param ops: operations specification
        @type ops: L{str}
        @param fops: federation operations specification
        @type fops: L{str}
        """
        self.game = Game(numPlayers=numPlayers,
                         initialCash=initialCash)
        self.context = self.game.generateContext(
            seed=seed, ops=ops, fops=fops)
        self.sim = Simulator(entities=[self.context],
                             initTime=0,  timeStep=1,
                             maxTime=numTurns)
        if elements is None:
            elements = []

        def initializeGame(time):
            federates = [federate for federation in self.context.federations
                         for federate in federation.federates]
            elementSets = []
            for federate in federates:
                elementSets.append([])

            for eId, element in enumerate(elements):
                specs = element.split(',')
                if len(specs) > 0 and len(specs[0].split('@')) == 2:
                    # parse player ownership and element type
                    if len(specs[0].split('@')[0].split('.')) == 2:
                        pId = int(specs[0].split('@')[0].split('.')[0])-1
                        eType = specs[0].split('@')[0].split('.')[1]
                    else:
                        pId = 0
                        eType = specs[0].split('@')[0]

                    # parse location
                    location = next((l for l in self.context.locations
                                     if l.name == specs[0].split('@')[1]), None)

                    # parse modules
                    if pId < len(federates) and location is not None:
                        # generate elements
                        element = self.game.generateElement(eType, pId, eId, mTypes=specs[1:])
                        if element is not None:
                            elementSets[pId].append({'element':element,
                                                    'location':location})

            for i, elementSet in enumerate(elementSets):
                federate = federates[i]
                if federate.initialCash == 0:
                    # special case if 0 initial cash: grant enough for initial element
                    for element in elementSet:
                        federate.initialCash += element['element'].getDesignCost()
                        federate.initialCash += element['element'].getCommissionCost(
                            element['location'], self.context)
                    federate.cash = federate.initialCash
                for element in elementSet:
                    federate.design(element['element'])
                    federate.commission(element['element'], element['location'], self.context)

        def finalizeGame(time):
            for federate in [federate for federation in self.context.federations
                             for federate in federation.federates]:
                federate.liquidate(self.context)
                logging.info('{} final cash: {}'
                            .format(federate.name, federate.cash))
        self.sim.on('init', initializeGame)
        self.sim.on('complete', finalizeGame)

    def execute(self):
        """
        Executes an OFS.
        @return: L{list}
        """
        self.sim.execute()
        results = []
        for federate in [federate for federation in self.context.federations
                         for federate in federation.federates]:
            results.append((federate.initialCash, federate.cash))
        return results<|endoftext|>