<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Methods to read data in the graph."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.framework import ops
from tensorflow.python.ops import constant_op
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import gfile
from tensorflow.python.training import input as input_ops

def read_batch_examples(file_pattern, batch_size, reader,
                        randomize_input=True, num_epochs=None,
                        queue_capacity=10000, num_threads=1,
                        name=None):
  """Adds operations to read, queue, batch `Example` protos.

  Given file pattern (or list of files), will setup a queue for file names,
  read `Example` proto using provided `reader`, use batch queue to create
  batches of examples of size `batch_size`.

  All queue runners are added to the queue runners collection, and may be
  started via `start_queue_runners`.

  All ops are added to the default graph.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    reader: A function or class that returns an object with
      `read` method, (filename tensor) -> (example tensor).
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If `None`, cycles through the dataset forever.
      NOTE - If specified, creates a variable that must be initialized, so call
      `tf.initialize_all_variables()` as shown in the tests.
    queue_capacity: Capacity for input queue.
    num_threads: The number of threads enqueuing examples.
    name: Name of resulting op.

  Returns:
    String `Tensor` of batched `Example` proto.

  Raises:
    ValueError: for invalid inputs.
  """
  # Retrive files to read.
  if isinstance(file_pattern, list):
    file_names = file_pattern
    if not file_names:
      raise ValueError('No files given to dequeue_examples.')
  else:
    file_names = list(gfile.Glob(file_pattern))
    if not file_names:
      raise ValueError('No files match %s.' % file_pattern)

  # Sort files so it will be deterministic for unit tests. They'll be shuffled
  # in `string_input_producer` if `randomize_input` is enabled.
  if not randomize_input:
    file_names = sorted(file_names)

  # Check input parameters are given and reasonable.
  if (not queue_capacity) or (queue_capacity <= 0):
    raise ValueError('Invalid queue_capacity %s.' % queue_capacity)
  if (batch_size is None) or (
      (not isinstance(batch_size, ops.Tensor)) and
      (batch_size <= 0 or batch_size > queue_capacity)):
    raise ValueError(
        'Invalid batch_size %s, with queue_capacity %s.' %
        (batch_size, queue_capacity))
  if (not num_threads) or (num_threads <= 0):
    raise ValueError('Invalid num_threads %s.' % num_threads)
  if (num_epochs is not None) and (num_epochs <= 0):
    raise ValueError('Invalid num_epochs %s.' % num_epochs)

  with ops.op_scope([file_pattern], name, 'read_batch_examples') as scope:
    # Setup filename queue with shuffling.
    with ops.name_scope('file_name_queue') as file_name_queue_scope:
      file_name_queue = input_ops.string_input_producer(
          constant_op.constant(file_names, name='input'),
          shuffle=randomize_input, num_epochs=num_epochs,
          name=file_name_queue_scope)

    # Create readers, one per thread and set them to read from filename queue.
    with ops.name_scope('read'):
      example_list = []
      for _ in range(num_threads):
        _, example_proto = reader().read(file_name_queue)
        example_list.append([example_proto])

    # Setup batching queue given list of read example tensors.
    if randomize_input:
      if isinstance(batch_size, ops.Tensor):
        min_after_dequeue = int(queue_capacity * 0.4)
      else:
        min_after_dequeue = max(queue_capacity - (3 * batch_size), batch_size)
      examples = input_ops.shuffle_batch_join(
          example_list, batch_size, capacity=queue_capacity,
          min_after_dequeue=min_after_dequeue,
          name=scope)
    else:
      examples = input_ops.batch_join(
          example_list, batch_size, capacity=queue_capacity,
          name=scope)

    return examples

def read_batch_features(file_pattern, batch_size, features, reader,
                        randomize_input=True, num_epochs=None,
                        queue_capacity=10000, reader_num_threads=1,
                        parser_num_threads=1,
                        name=None):
  """Adds operations to read, queue, batch and parse `Example` protos.

  Given file pattern (or list of files), will setup a queue for file names,
  read `Example` proto using provided `reader`, use batch queue to create
  batches of examples of size `batch_size` and parse example given `features`
  specification.

  All queue runners are added to the queue runners collection, and may be
  started via `start_queue_runners`.

  All ops are added to the default graph.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    features: A `dict` mapping feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    reader: A function or class that returns an object with
      `read` method, (filename tensor) -> (example tensor).
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If None, cycles through the dataset forever. NOTE - If specified,
      creates a variable that must be initialized, so call
      tf.initialize_all_variables() as shown in the tests.
    queue_capacity: Capacity for input queue.
    reader_num_threads: The number of threads to read examples.
    parser_num_threads: The number of threads to parse examples.
    name: Name of resulting op.

  Returns:
    A dict of `Tensor` or `SparseTensor` objects for each in `features`.

  Raises:
    ValueError: for invalid inputs.
  """
  with ops.op_scope([file_pattern], name, 'read_batch_features') as scope:
    examples = read_batch_examples(
        file_pattern, batch_size, reader, randomize_input=randomize_input,
        num_epochs=num_epochs, queue_capacity=queue_capacity,
        num_threads=reader_num_threads, name=scope)

    # Parse features into tensors in many threads and put on the queue.
    features_list = []
    for _ in range(parser_num_threads):
      features_list.append(parsing_ops.parse_example(examples, features))
    return input_ops.batch_join(
        features_list,
        batch_size=batch_size,
        capacity=queue_capacity,
        enqueue_many=True,
        name='parse_example_batch_join')

def read_batch_record_features(file_pattern, batch_size, features,
                               randomize_input=True, num_epochs=None,
                               queue_capacity=10000, reader_num_threads=1,
                               parser_num_threads=1,
                               name='dequeue_record_examples'):
  """Reads TFRecord, queues, batches and parses `Example` proto.

  See more detailed description in `read_examples`.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    features: A `dict` mapping feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If None, cycles through the dataset forever. NOTE - If specified,
      creates a variable that must be initialized, so call
      tf.initialize_all_variables() as shown in the tests.
    queue_capacity: Capacity for input queue.
    reader_num_threads: The number of threads to read examples.
    parser_num_threads: The number of threads to parse examples.
    name: Name of resulting op.

  Returns:
    A dict of `Tensor` or `SparseTensor` objects for each in `features`.

  Raises:
    ValueError: for invalid inputs.
  """
  return read_batch_features(
      file_pattern=file_pattern, batch_size=batch_size, features=features,
      reader=io_ops.TFRecordReader,
      randomize_input=randomize_input, num_epochs=num_epochs,
      queue_capacity=queue_capacity, reader_num_threads=reader_num_threads,
      parser_num_threads=parser_num_threads, name=name)
<commit_msg>Speed up read_batch_features by skipping the parsing queue when there is a single parsing thread. Change: 123921139<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Methods to read data in the graph."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.framework import ops
from tensorflow.python.ops import constant_op
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import parsing_ops
from tensorflow.python.platform import gfile
from tensorflow.python.training import input as input_ops

def read_batch_examples(file_pattern, batch_size, reader,
                        randomize_input=True, num_epochs=None,
                        queue_capacity=10000, num_threads=1,
                        name=None):
  """Adds operations to read, queue, batch `Example` protos.

  Given file pattern (or list of files), will setup a queue for file names,
  read `Example` proto using provided `reader`, use batch queue to create
  batches of examples of size `batch_size`.

  All queue runners are added to the queue runners collection, and may be
  started via `start_queue_runners`.

  All ops are added to the default graph.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    reader: A function or class that returns an object with
      `read` method, (filename tensor) -> (example tensor).
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If `None`, cycles through the dataset forever.
      NOTE - If specified, creates a variable that must be initialized, so call
      `tf.initialize_all_variables()` as shown in the tests.
    queue_capacity: Capacity for input queue.
    num_threads: The number of threads enqueuing examples.
    name: Name of resulting op.

  Returns:
    String `Tensor` of batched `Example` proto.

  Raises:
    ValueError: for invalid inputs.
  """
  # Retrive files to read.
  if isinstance(file_pattern, list):
    file_names = file_pattern
    if not file_names:
      raise ValueError('No files given to dequeue_examples.')
  else:
    file_names = list(gfile.Glob(file_pattern))
    if not file_names:
      raise ValueError('No files match %s.' % file_pattern)

  # Sort files so it will be deterministic for unit tests. They'll be shuffled
  # in `string_input_producer` if `randomize_input` is enabled.
  if not randomize_input:
    file_names = sorted(file_names)

  # Check input parameters are given and reasonable.
  if (not queue_capacity) or (queue_capacity <= 0):
    raise ValueError('Invalid queue_capacity %s.' % queue_capacity)
  if (batch_size is None) or (
      (not isinstance(batch_size, ops.Tensor)) and
      (batch_size <= 0 or batch_size > queue_capacity)):
    raise ValueError(
        'Invalid batch_size %s, with queue_capacity %s.' %
        (batch_size, queue_capacity))
  if (not num_threads) or (num_threads <= 0):
    raise ValueError('Invalid num_threads %s.' % num_threads)
  if (num_epochs is not None) and (num_epochs <= 0):
    raise ValueError('Invalid num_epochs %s.' % num_epochs)

  with ops.op_scope([file_pattern], name, 'read_batch_examples') as scope:
    # Setup filename queue with shuffling.
    with ops.name_scope('file_name_queue') as file_name_queue_scope:
      file_name_queue = input_ops.string_input_producer(
          constant_op.constant(file_names, name='input'),
          shuffle=randomize_input, num_epochs=num_epochs,
          name=file_name_queue_scope)

    # Create readers, one per thread and set them to read from filename queue.
    with ops.name_scope('read'):
      example_list = []
      for _ in range(num_threads):
        _, example_proto = reader().read(file_name_queue)
        example_list.append([example_proto])

    # Setup batching queue given list of read example tensors.
    if randomize_input:
      if isinstance(batch_size, ops.Tensor):
        min_after_dequeue = int(queue_capacity * 0.4)
      else:
        min_after_dequeue = max(queue_capacity - (3 * batch_size), batch_size)
      examples = input_ops.shuffle_batch_join(
          example_list, batch_size, capacity=queue_capacity,
          min_after_dequeue=min_after_dequeue,
          name=scope)
    else:
      examples = input_ops.batch_join(
          example_list, batch_size, capacity=queue_capacity,
          name=scope)

    return examples

def read_batch_features(file_pattern, batch_size, features, reader,
                        randomize_input=True, num_epochs=None,
                        queue_capacity=10000, reader_num_threads=1,
                        parser_num_threads=1,
                        name=None):
  """Adds operations to read, queue, batch and parse `Example` protos.

  Given file pattern (or list of files), will setup a queue for file names,
  read `Example` proto using provided `reader`, use batch queue to create
  batches of examples of size `batch_size` and parse example given `features`
  specification.

  All queue runners are added to the queue runners collection, and may be
  started via `start_queue_runners`.

  All ops are added to the default graph.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    features: A `dict` mapping feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    reader: A function or class that returns an object with
      `read` method, (filename tensor) -> (example tensor).
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If None, cycles through the dataset forever. NOTE - If specified,
      creates a variable that must be initialized, so call
      tf.initialize_all_variables() as shown in the tests.
    queue_capacity: Capacity for input queue.
    reader_num_threads: The number of threads to read examples.
    parser_num_threads: The number of threads to parse examples.
    name: Name of resulting op.

  Returns:
    A dict of `Tensor` or `SparseTensor` objects for each in `features`.

  Raises:
    ValueError: for invalid inputs.
  """
  with ops.op_scope([file_pattern], name, 'read_batch_features') as scope:
    examples = read_batch_examples(
        file_pattern, batch_size, reader, randomize_input=randomize_input,
        num_epochs=num_epochs, queue_capacity=queue_capacity,
        num_threads=reader_num_threads, name=scope)

    if parser_num_threads == 1:
      # Avoid queue overhead for single thread
      return parsing_ops.parse_example(examples, features)
    else:
      # Parse features into tensors in many threads and put on the queue.
      features_list = []
      for _ in range(parser_num_threads):
        features_list.append(parsing_ops.parse_example(examples, features))
      return input_ops.batch_join(
          features_list,
          batch_size=batch_size,
          capacity=queue_capacity,
          enqueue_many=True,
          name='parse_example_batch_join')

def read_batch_record_features(file_pattern, batch_size, features,
                               randomize_input=True, num_epochs=None,
                               queue_capacity=10000, reader_num_threads=1,
                               parser_num_threads=1,
                               name='dequeue_record_examples'):
  """Reads TFRecord, queues, batches and parses `Example` proto.

  See more detailed description in `read_examples`.

  Args:
    file_pattern: List of files or pattern of file paths containing
        `Example` records. See `tf.gfile.Glob` for pattern rules.
    batch_size: An int or scalar `Tensor` specifying the batch size to use.
    features: A `dict` mapping feature keys to `FixedLenFeature` or
      `VarLenFeature` values.
    randomize_input: Whether the input should be randomized.
    num_epochs: Integer specifying the number of times to read through the
      dataset. If None, cycles through the dataset forever. NOTE - If specified,
      creates a variable that must be initialized, so call
      tf.initialize_all_variables() as shown in the tests.
    queue_capacity: Capacity for input queue.
    reader_num_threads: The number of threads to read examples.
    parser_num_threads: The number of threads to parse examples.
    name: Name of resulting op.

  Returns:
    A dict of `Tensor` or `SparseTensor` objects for each in `features`.

  Raises:
    ValueError: for invalid inputs.
  """
  return read_batch_features(
      file_pattern=file_pattern, batch_size=batch_size, features=features,
      reader=io_ops.TFRecordReader,
      randomize_input=randomize_input, num_epochs=num_epochs,
      queue_capacity=queue_capacity, reader_num_threads=reader_num_threads,
      parser_num_threads=parser_num_threads, name=name)
<|endoftext|>


<commit_before># Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License..
# ==============================================================================
"""TensorFlow is an open source machine learning framework for everyone.

[![Python](https://img.shields.io/pypi/pyversions/tensorflow.svg?style=plastic)](https://badge.fury.io/py/tensorflow)
[![PyPI](https://badge.fury.io/py/tensorflow.svg)](https://badge.fury.io/py/tensorflow)

TensorFlow is an open source software library for high performance numerical
computation. Its flexible architecture allows easy deployment of computation
across a variety of platforms (CPUs, GPUs, TPUs), and from desktops to clusters
of servers to mobile and edge devices.

Originally developed by researchers and engineers from the Google Brain team
within Google's AI organization, it comes with strong support for machine
learning and deep learning and the flexible numerical computation core is used
across many other scientific domains.
"""
# We use this to build installer wheels whose only job would be to install the
# third-party TensorFlow packages from Google's official partners.
# Note: This is experimental for now and is used internally for testing.
import sys

from setuptools import setup

# This version string is semver compatible, but incompatible with pip.
# For pip, we will remove all '-' characters from this string, and use the
# result for pip.
# Also update tensorflow/tensorflow.bzl and
# tensorflow/core/public/version.h
_VERSION = '2.9.1'

# We use the same setup.py for all tensorflow_* packages and for the nightly
# equivalents (tf_nightly_*). The package is controlled from the argument line
# when building the pip package.
project_name = 'tensorflow'
if '--project_name' in sys.argv:
  project_name_idx = sys.argv.index('--project_name')
  project_name = sys.argv[project_name_idx + 1]
  sys.argv.remove('--project_name')
  sys.argv.pop(project_name_idx)

# Returns standard if a tensorflow-* package is being built, and nightly if a
# tf_nightly-* package is being built.
def standard_or_nightly(standard, nightly):
  return nightly if 'tf_nightly' in project_name else standard

# Install the trusted partner packages with the same version as the installer
# wheel
REQUIRED_PACKAGES = [
    # Install the TensorFlow package built by AWS if the user is running
    # Linux on an Aarch64 machine.
    standard_or_nightly('tensorflow-cpu-aws', 'tf-nightly-cpu-aws ') + '==' +
    _VERSION +
    ';platform_system=="Linux" and (platform_machine=="arm64" or '
    'platform_machine=="aarch64")',
]
REQUIRED_PACKAGES = [p for p in REQUIRED_PACKAGES if p is not None]

DOCLINES = __doc__.split('\n')
if project_name.endswith('-gpu'):
  project_name_no_gpu = project_name[:-len('-gpu')]
  _GPU_PACKAGE_NOTE = 'Note that %s package by default supports both CPU and '\
      'GPU. %s has the same content and exists solely for backward '\
      'compatibility. Please migrate to %s for GPU support.'\
      % (project_name_no_gpu, project_name, project_name_no_gpu)
  DOCLINES.append(_GPU_PACKAGE_NOTE)

setup(
    name=project_name,
    version=_VERSION.replace('-', ''),
    description=DOCLINES[0],
    long_description='\n'.join(DOCLINES[2:]),
    long_description_content_type='text/markdown',
    url='https://www.tensorflow.org/',
    download_url='https://github.com/tensorflow/tensorflow/tags',
    author='Google Inc.',
    author_email='packages@tensorflow.org',
    install_requires=REQUIRED_PACKAGES,
    zip_safe=False,
    # Supported Python versions
    python_requires='>=3.7',
    # PyPI package information.
    classifiers=sorted([
        'Development Status :: 5 - Production/Stable',
        # TODO(angerson) Add IFTTT when possible
        'Environment :: GPU :: NVIDIA CUDA :: 11.2',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Scientific/Engineering',
        'Topic :: Scientific/Engineering :: Mathematics',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Software Development',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ]),
    license='Apache 2.0',
    keywords='tensorflow tensor machine learning',
)

<commit_msg>Add Intel built TensorFlow packages to the partner builds script<commit_after># Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License..
# ==============================================================================
"""TensorFlow is an open source machine learning framework for everyone.

[![Python](https://img.shields.io/pypi/pyversions/tensorflow.svg?style=plastic)](https://badge.fury.io/py/tensorflow)
[![PyPI](https://badge.fury.io/py/tensorflow.svg)](https://badge.fury.io/py/tensorflow)

TensorFlow is an open source software library for high performance numerical
computation. Its flexible architecture allows easy deployment of computation
across a variety of platforms (CPUs, GPUs, TPUs), and from desktops to clusters
of servers to mobile and edge devices.

Originally developed by researchers and engineers from the Google Brain team
within Google's AI organization, it comes with strong support for machine
learning and deep learning and the flexible numerical computation core is used
across many other scientific domains.
"""
# We use this to build installer wheels whose only job would be to install the
# third-party TensorFlow packages from Google's official partners.
# Note: This is experimental for now and is used internally for testing.
import sys

from setuptools import setup

# This version string is semver compatible, but incompatible with pip.
# For pip, we will remove all '-' characters from this string, and use the
# result for pip.
# Also update tensorflow/tensorflow.bzl and
# tensorflow/core/public/version.h
_VERSION = '2.10.0'

# We use the same setup.py for all tensorflow_* packages and for the nightly
# equivalents (tf_nightly_*). The package is controlled from the argument line
# when building the pip package.
project_name = 'tensorflow'
if '--project_name' in sys.argv:
  project_name_idx = sys.argv.index('--project_name')
  project_name = sys.argv[project_name_idx + 1]
  sys.argv.remove('--project_name')
  sys.argv.pop(project_name_idx)

# Returns standard if a tensorflow-* package is being built, and nightly if a
# tf_nightly-* package is being built.
def standard_or_nightly(standard, nightly):
  return nightly if 'tf_nightly' in project_name else standard

# Install the trusted partner packages with the same version as the installer
# wheel
REQUIRED_PACKAGES = [
    # Install the TensorFlow package built by AWS if the user is running
    # Linux on an Aarch64 machine.
    standard_or_nightly('tensorflow-cpu-aws', 'tf-nightly-cpu-aws ') + '==' +
    _VERSION +
    ';platform_system=="Linux" and (platform_machine=="arm64" or '
    'platform_machine=="aarch64")',
    # Install the TensorFlow package built by Intel if the user is on a Windows
    # machine.
    standard_or_nightly('tensorflow-intel', 'tf-nightly-intel') +
    '==' + _VERSION + ';platform_system=="Windows"',
]
REQUIRED_PACKAGES = [p for p in REQUIRED_PACKAGES if p is not None]

DOCLINES = __doc__.split('\n')
if project_name.endswith('-gpu'):
  project_name_no_gpu = project_name[:-len('-gpu')]
  _GPU_PACKAGE_NOTE = 'Note that %s package by default supports both CPU and '\
      'GPU. %s has the same content and exists solely for backward '\
      'compatibility. Please migrate to %s for GPU support.'\
      % (project_name_no_gpu, project_name, project_name_no_gpu)
  DOCLINES.append(_GPU_PACKAGE_NOTE)

setup(
    name=project_name,
    version=_VERSION.replace('-', ''),
    description=DOCLINES[0],
    long_description='\n'.join(DOCLINES[2:]),
    long_description_content_type='text/markdown',
    url='https://www.tensorflow.org/',
    download_url='https://github.com/tensorflow/tensorflow/tags',
    author='Google Inc.',
    author_email='packages@tensorflow.org',
    install_requires=REQUIRED_PACKAGES,
    zip_safe=False,
    # Supported Python versions
    python_requires='>=3.7',
    # PyPI package information.
    classifiers=sorted([
        'Development Status :: 5 - Production/Stable',
        # TODO(angerson) Add IFTTT when possible
        'Environment :: GPU :: NVIDIA CUDA :: 11.2',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Scientific/Engineering',
        'Topic :: Scientific/Engineering :: Mathematics',
        'Topic :: Scientific/Engineering :: Artificial Intelligence',
        'Topic :: Software Development',
        'Topic :: Software Development :: Libraries',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ]),
    license='Apache 2.0',
    keywords='tensorflow tensor machine learning',
)

<|endoftext|>


<commit_before>#!/usr/bin/env python

from matplotlib import rc
import matplotlib
from daft import PGM, Node, Plate
rc("font", family="serif", size=8)
rc("text", usetex=True)

pgm = PGM([9.5, 8.5], origin=[0., 0.2], observed_style='inner')

#pgm.add_node(Node('dispersion',r"\center{$\sigma_{Ia}$ \newline $\sigma_{!Ia}$}", 1,6,scale=1.2,aspect=1.8))
pgm.add_node(Node('rate',r"{\center Relative \newline Rates}", 2,8,scale=1.6,aspect=1.2))
#pgm.add_node(Node('theta_T',r"\center{$\alpha_{Ia}$, $\alpha_{!Ia}$ \newline $\beta_{Ia}$, $\beta_{!Ia}$}", 4,6,scale=1.4,aspect=1.8))
pgm.add_node(Node('theta_T',r"\center{SNe~Ia, Non-Ia \newline Populations}", 5,8,scale=1.8,aspect=1.2))
pgm.add_node(Node('Global Transmission',r"\centering{Global \newline Throughput}", 8, 8,scale=1.6,aspect=1.2))
pgm.add_node(Node('Transmission',r"Throughput", 8, 4,scale=1.6,aspect=1.2))
#pgm.add_node(Node('theta_T2',r"{Non-Ia}", 4,8,scale=1.6,aspect=1.2))
pgm.add_node(Node('mu',r"{\center Cosmology}", 7,8, scale=1.6,aspect=1.2))

#pgm.add_node(Node('G_Ni',r"$g_{Ni}$", 8,5))

#pgm.add_node(Node('Flux_g',r"$f_{gi}(\lambda)$", 8, 3))

#pgm.add_node(Node('z',r"$z_i$", 3, 3, fixed=True,offset=(-10,-5)))

#pgm.add_node(Node('Counts',r"$\overline{n}_i$", 7, 2,scale=1.2,fixed=True,offset=(-15,0)))
#pgm.add_node(Node('Counts_g',r"$\overline{\mathit{ADU}}_{gi}$", 8, 2,scale=1.2,fixed=True,offset=(20,0)))
#pgm.add_node(Node("Summary",r"$S({\mathit{ADU}_i})$", 9, 1, fixed=True, offset=(0,-25)))

pgm.add_node(Node('G_i',r"{\centering Host Galaxy \newline (Redshift)}", 3,7, scale=1.6,aspect=1.2))

pgm.add_node(Node('Type',r"\centering{Type \newline Subtype}", 2, 6, scale=1.6,aspect=1.2))

#pgm.add_node(Node('theta_Ti',r"\centering{Transient \newline Parameters}", 1,5,scale=1.5,aspect=1.2))
pgm.add_node(Node('HD',r"Distance", 7,6,fixed=True,offset=(-25,-15)))

pgm.add_node(Node('Luminosity',r"Luminosity", 5, 4,scale=1.6,aspect=1.2))

pgm.add_node(Node('Flux',r"Flux", 7, 3, scale=1.2,fixed=True,offset=(-20,-20)))

#pgm.add_node(Node('Stheta',r"Indicators", 1, 1,  observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('ST',r"\centering{Type \newline Subtype$_o$}", 2, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('Sz',r"\centering{Redshift$_o$\newline Phot \& Spec}", 3, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('^Counts',r"Photometry$_o$", 8, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('Gals',r"\centering{Host Properties$_o$\newline Phot \& Spec}", 4,1, observed=True,scale=1.6,aspect=1.2))

#pgm.add_node(Node('Soectroscopy',r"Transient Spectroscpy", 1, 1,  observed=True))

#pgm.add_edge("^Counts","Summary")
pgm.add_edge("G_i","Gals")
#pgm.add_edge("G_Ni","Gals")
pgm.add_edge("rate","Type")
#pgm.add_edge("Type", "theta_Ti")
pgm.add_edge("mu","HD")
#pgm.add_edge("theta_T", "theta_Ti")
pgm.add_edge("G_i","HD")
pgm.add_edge("G_i","Sz")
pgm.add_edge("G_i","Luminosity")
pgm.add_edge("G_i","Flux")
pgm.add_edge("HD","Flux")
#pgm.add_edge("z","Flux")
#pgm.add_edge("G_Ni","Flux_g")
#pgm.add_edge("HD","Flux_g")
#pgm.add_edge("z","Flux_g")
pgm.add_edge("G_i","Type")
#pgm.add_edge("G_i","theta_Ti")
pgm.add_edge("Type","Luminosity")
pgm.add_edge("Luminosity","Flux")
#pgm.add_edge("Type","Stheta")
pgm.add_edge("theta_T","Luminosity")
#pgm.add_edge("theta_Ti","Luminosity")
pgm.add_edge("Flux","^Counts")
#pgm.add_edge("Flux_g","Counts_g")
#pgm.add_edge("Counts","^Counts")
#pgm.add_edge("Counts_g","^Counts")
pgm.add_edge("Global Transmission","Transmission")
pgm.add_edge("Transmission","^Counts")
#pgm.add_edge("Transmission","Counts_g")
pgm.add_edge("Type","ST")
#pgm.add_edge("theta_Ti","Stheta")
#pgm.add_edge("Counts_g","^Counts")

# Big Plate: Galaxy
pgm.add_plate(Plate([1.4, 0.5, 7.2, 7.],
                    label=r"SNe $i = 1, \cdots, N_{SN}$",
                    shift=-0.2,label_offset=[20,2]))

pgm.add_plate(Plate([4.5, 0.6, 4, 4.],
                    label=r"{\centering LC Point $j = 1, \cdots, N_{band}$, $k = 1, \cdots, N_{date}$}",
                    shift=-0.2,label_offset=[2,2]))

# Render and save.
pgm.render()
pgm.figure.text(0.01,0.9,r'\underline{UNIVERSAL}',size='large')
pgm.figure.text(0.01,0.55,r'{\centering \underline{INDIVIDUAL} \newline \underline{SN}}',size='large')
pgm.figure.text(0.01,0.2,r'\underline{OBSERVATORY}',size='large')
pgm.figure.text(0.01,0.1,r'\underline{DATA}',size='large')

pgm.figure.savefig("../results/hdpgm.pdf")
<commit_msg>more general version of the pgm<commit_after>#!/usr/bin/env python

from matplotlib import rc
import matplotlib
from daft import PGM, Node, Plate
rc("font", family="serif", size=8)
rc("text", usetex=True)

pgm = PGM([9.5, 8.5], origin=[0., 0.2], observed_style='inner')

#pgm.add_node(Node('dispersion',r"\center{$\sigma_{Ia}$ \newline $\sigma_{!Ia}$}", 1,6,scale=1.2,aspect=1.8))
#pgm.add_node(Node('rate',r"{\center Relative \newline Rates}", 2,8,scale=1.6,aspect=1.2))
#pgm.add_node(Node('theta_T',r"\center{$\alpha_{Ia}$, $\alpha_{!Ia}$ \newline $\beta_{Ia}$, $\beta_{!Ia}$}", 4,6,scale=1.4,aspect=1.8))
pgm.add_node(Node('theta_T',r"\center{SNe~Ia, Non-Ia Populations \newline Rates}", 3,8,scale=1.8,aspect=3))
pgm.add_node(Node('Global Transmission',r"\centering{Global \newline Throughput}", 8, 8,scale=1.6,aspect=1.2))
pgm.add_node(Node('Transmission',r"Throughput", 8, 4,scale=1.6,aspect=1.2))
#pgm.add_node(Node('theta_T2',r"{Non-Ia}", 4,8,scale=1.6,aspect=1.2))
pgm.add_node(Node('mu',r"{\center Cosmology}", 7,8, scale=1.6,aspect=1.2))

#pgm.add_node(Node('G_Ni',r"$g_{Ni}$", 8,5))

#pgm.add_node(Node('Flux_g',r"$f_{gi}(\lambda)$", 8, 3))

#pgm.add_node(Node('z',r"$z_i$", 3, 3, fixed=True,offset=(-10,-5)))

#pgm.add_node(Node('Counts',r"$\overline{n}_i$", 7, 2,scale=1.2,fixed=True,offset=(-15,0)))
#pgm.add_node(Node('Counts_g',r"$\overline{\mathit{ADU}}_{gi}$", 8, 2,scale=1.2,fixed=True,offset=(20,0)))
#pgm.add_node(Node("Summary",r"$S({\mathit{ADU}_i})$", 9, 1, fixed=True, offset=(0,-25)))

pgm.add_node(Node('G_i',r"{\centering Host Galaxy \newline (Redshift)}", 3,7, scale=1.6,aspect=1.2))

pgm.add_node(Node('Type',r"\centering{Type \newline Subtype}", 2, 6, scale=1.6,aspect=1.2))

#pgm.add_node(Node('theta_Ti',r"\centering{Transient \newline Parameters}", 1,5,scale=1.5,aspect=1.2))
pgm.add_node(Node('HD',r"Distance", 7,6,fixed=True,offset=(-25,-15)))

pgm.add_node(Node('Luminosity',r"Luminosity", 5, 4,scale=1.6,aspect=1.2))

pgm.add_node(Node('Flux',r"Flux", 7, 3, scale=1.2,fixed=True,offset=(-20,-20)))

#pgm.add_node(Node('Stheta',r"Indicators", 1, 1,  observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('ST',r"\centering{Type \newline Subtype$_o$}", 2, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('Sz',r"\centering{Redshift$_o$\newline Phot \& Spec}", 3, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('^Counts',r"Photometry$_o$", 8, 1, observed=True,scale=1.6,aspect=1.2))
pgm.add_node(Node('Gals',r"\centering{Host Properties$_o$\newline Phot \& Spec}", 4,1, observed=True,scale=1.6,aspect=1.2))

#pgm.add_node(Node('Soectroscopy',r"Transient Spectroscpy", 1, 1,  observed=True))

#pgm.add_edge("^Counts","Summary")
pgm.add_edge("G_i","Gals")
#pgm.add_edge("G_Ni","Gals")
pgm.add_edge("theta_T","Type")
#pgm.add_edge("Type", "theta_Ti")
pgm.add_edge("mu","HD")
#pgm.add_edge("theta_T", "theta_Ti")
pgm.add_edge("G_i","HD")
pgm.add_edge("G_i","Sz")
pgm.add_edge("G_i","Luminosity")
pgm.add_edge("G_i","Flux")
pgm.add_edge("HD","Flux")
#pgm.add_edge("z","Flux")
#pgm.add_edge("G_Ni","Flux_g")
#pgm.add_edge("HD","Flux_g")
#pgm.add_edge("z","Flux_g")
pgm.add_edge("G_i","Type")
#pgm.add_edge("G_i","theta_Ti")
pgm.add_edge("Type","Luminosity")
pgm.add_edge("Luminosity","Flux")
#pgm.add_edge("Type","Stheta")
pgm.add_edge("theta_T","Luminosity")
#pgm.add_edge("theta_Ti","Luminosity")
pgm.add_edge("Flux","^Counts")
#pgm.add_edge("Flux_g","Counts_g")
#pgm.add_edge("Counts","^Counts")
#pgm.add_edge("Counts_g","^Counts")
pgm.add_edge("Global Transmission","Transmission")
pgm.add_edge("Transmission","^Counts")
#pgm.add_edge("Transmission","Counts_g")
pgm.add_edge("Type","ST")
#pgm.add_edge("theta_Ti","Stheta")
#pgm.add_edge("Counts_g","^Counts")

# Big Plate: Galaxy
pgm.add_plate(Plate([1.4, 0.5, 7.2, 7.],
                    label=r"SNe $i = 1, \cdots, N_{SN}$",
                    shift=-0.2,label_offset=[20,2]))

pgm.add_plate(Plate([4.5, 0.6, 4, 4.],
                    label=r"{\centering LC Point $j = 1, \cdots, N_{band}$, $k = 1, \cdots, N_{date}$}",
                    shift=-0.2,label_offset=[2,2]))

# Render and save.
pgm.render()
pgm.figure.text(0.01,0.9,r'\underline{UNIVERSAL}',size='large')
pgm.figure.text(0.01,0.55,r'{\centering \underline{INDIVIDUAL} \newline \underline{SN}}',size='large')
pgm.figure.text(0.01,0.2,r'\underline{OBSERVATORY}',size='large')
pgm.figure.text(0.01,0.1,r'\underline{DATA}',size='large')

pgm.figure.savefig("../results/hdpgm.pdf")
<|endoftext|>


<commit_before>from PIL import Image;
from os import listdir;
import numpy;

def main():
	# Create an array to hold all of our images
	images = [];

	# For every file inside our working directory...
	for fileName in (listdir("images\\")):
		# Open the image and create an array from it, then append that image array into our image list
		images.append(numpy.array(Image.open("images\\" + fileName)));

	# Use the built-in median function inside numpy to find the median image, and convert all values to uint8 (0 - 255)
	stacked = numpy.uint8(numpy.median(images, 0));

	# Use Pillow to create an image from an array of data
	output = Image.fromarray(stacked);

	# Save it as a PNG for max quality
	output.save("stacked.png", "PNG");

main();<commit_msg>Removed semicolons<commit_after>from PIL import Image
from os import listdir
import numpy

def main():
	# Create an array to hold all of our images
	images = []

	# For every file inside our working directory...
	for fileName in (listdir("images\\")):
		# Open the image and create an array from it, then append that image array into our image list
		images.append(numpy.array(Image.open("images\\" + fileName)))

	# Use the built-in median function inside numpy to find the median image, and convert all values to uint8 (0 - 255)
	stacked = numpy.uint8(numpy.median(images, 0))

	# Use Pillow to create an image from an array of data
	output = Image.fromarray(stacked)

	# Save it as a PNG for max quality
	output.save("stacked.png", "PNG")

main()
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
import asyncio
import re
import sys
import os, random

#scriptpath = "../modules/commandmodule.py" #relative path dont DO a n y  Funny business
#sys.path.append(os.path.abspath(scriptpath))

description = '''A star bot.'''
bot = commands.Bot(command_prefix='!', description=description)
@bot.event
async def on_ready():
	print('Logged in as')
	print(bot.user.name, bot.user.id)
	print('Servers: ', end='')
	for server in bot.servers:
		print(str('{0}, ').format(server), end='')
	print()
	print('----------------')
	#global starnumber
	#starnumber = 4
	isrunning = False

#fn = os.path.join(os.path.dirname(__file__), 'starbot.py')
#dir = os.path.dirnam(__file__)

#owner
owner = "141695444995670017"

#channels
#best_of = 332172659863453697
#bot-bot = 332229780797652992

#roles
#shut up = 329705402021183488

def getRandomFile(path):
  """
  Returns a random filename, chosen among the files of the given path.
  """
  files = os.listdir(path)
  index = random.randrange(0, len(files))
  return files[index]

###starboard code###

#@bot.event
#async def on_member_join(member):
#	if member.id == ("289588477492723712"):
#		muterole = discord.utils.get(message.server.roles, id='329705402021183488')
#		await bot.add_roles(member, muterole)

@bot.event
async def on_reaction_add(reaction, member):
	print ("detected")
	if reaction.emoji == ("⭐"):
		if reaction.count == 5:
			nsfw = bot.get_channel("nsfw-garbage-bin")
			if reaction.message.channel != nsfw:
				best_of = discord.utils.get(reaction.message.server.channels, name= "best_of")

			#embed message itself
				em = discord.Embed(title='Starred post', description=reaction.message.content, colour=0xFFD700)
				em.set_author(name=reaction.message.author, icon_url=reaction.message.author.avatar_url)

			#embed url images
				if reaction.message.content.endswith(('.png', '.jpg', '.gif', '.gifv', '.webm')):
					em.set_image(url = reaction.message.content)

			#modifying reaction.message.attachments
				attach = reaction.message.attachments
				detector = str(attach)
				if detector.startswith('[{'):
					modify2 = str(attach)
					junk, junk2 = modify2.split("'url': '")
					websitelink, junk3 = junk2.split("',", 1)
					em.set_image(url = websitelink)
			#sending actual embed
				await bot.send_message(best_of, embed=em)
				await bot.clear_reactions(reaction.message)

		else:
			pass

isrunning = False

'''commands'''
@bot.event
async def on_message(message):
#check the message
	global isrunning
	#global commanduser
	#commanduser = ("null")

	#terobot v2

	terobotchance=random.randrange(200)
	if terobotchance == 1:
		if message.channel.is_private == True:
			return
		if message.channel.server.id == "351885259199086593" or message.channel.server.id == "308265490877382656": #consider refactoring to check a list
			return
		else:
			this_dir = os.path.dirname(__file__)
			pastaraw = os.path.realpath("{0}/copypastas".format(this_dir))
			choosepasta = getRandomFile(pastaraw)
			copypastaaa = open(pastaraw+"\\"+choosepasta, "r",encoding="utf8")
			selectedcopypasta = copypastaaa.read()
			await bot.send_message(message.channel, "<@{}> {}".format(message.author.id, selectedcopypasta))

#dablist command
	if message.content.startswith('!listdabs'):
		#dablist = os.listdir("starbot\\dabs")

		this_dir = os.path.dirname(__file__)
		dabpath = os.path.realpath("{0}/dabs".format(this_dir))
		dablist = os.listdir(dabpath)
		liststr = '\n'.join(dablist)
		em = discord.Embed(title='a list of dabs for people with terrible memories', description=liststr, colour=0xFFD700)
		await bot.send_message(message.channel, embed=em)
#dablist command
	if message.content.startswith('!dab'):
	#checks for command !story

		if not isrunning:
			isrunning = True
			#commanduser = message.author
			try:
				storycommand = message.content.split("!dab ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "wtf it's empty what do you want me to do")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				isrunning = False
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "was unable to type properly")
				return
			try:
				this_dir = os.path.dirname(__file__)
				storypath = os.path.realpath("{0}/dabs".format(this_dir))
				story = open(storypath+"\\"+storycommand+".txt", "r",encoding="utf8")
			except IOError:
				isrunning = False
				kill2=await bot.send_message(message.channel, "Choose a real file you nigger")
				await asyncio.sleep(2)
				await bot.delete_message(kill2)
				await bot.delete_message(message)
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "was unable to find a file.")
				return
			msg = await bot.send_message(message.channel, story.readline())
			this_dir = os.path.dirname(__file__)
			pathstory2 = os.path.realpath("{0}/dabs/".format(this_dir))
			story = open(pathstory2+"\\"+storycommand+".txt", "r",encoding="utf8")
			await asyncio.sleep(.1)
			#waits 3 second (I think 3 seconds, not sure)
			dabstop = False
			for chunk in iter(lambda: story.readline(), ''):
			#loops until end of file
				await bot.edit_message(msg, chunk)
				#edits message to add next line
				#if dabstop == True:
					#return
				await asyncio.sleep(1)
				#waits 3 seconds
			await bot.delete_message(msg)
			await bot.delete_message(message)
			#deletes the message
			story.close()
			#closes the file
			isrunning = False
			if message.author == ("☭Liar☭#0946"):
					return
			print(message.author, "used !dab successfully.")

		else:
			kill = await bot.send_message(message.channel,"REEEEEEEEEEEEEEE WAIT YOUR TURN")
			await asyncio.sleep(2)
			await bot.delete_message(kill)
			await bot.delete_message(message)
			if message.author == ("☭Liar☭#0946"):
				return
			print (message.author, "was unable to wait his turn.")
#help command
	if message.content.startswith("!help"):
		this_dir = os.path.dirname(__file__)
		helppath = os.path.realpath("{0}".format(this_dir))
		helptext=open(helppath+"\\"+"help.txt", "r", encoding="utf8")
		helptextactual=helptext.read()
		em = discord.Embed(title='Help', description=helptextactual, colour=0xFFD700)
		await bot.send_message(message.channel, embed=em)
		print(message.author, "asked for help!")
		helptext.close()

#echo command
	if message.content.startswith("!echo"):
		if message.author != bot.user:
			try:
				echoecho = message.content.split("!echo ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "I can't echo silence")
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "is unable to speak properly")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				return
			await bot.send_message(message.channel, echoecho)
			try:
				print(message.author, "echoed: ", echoecho)
			except UnicodeEncodeError:
				print ("invalid characters for the console")
			await bot.delete_message(message)

#superecho command
	'''if message.content.startswith("!superecho"):
		if message.author != bot.user:
			try:
				echoecho = message.content.split("!superecho ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "I can't echo silence")
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "is unable to speak properly")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				return
			echonumber = 100
			while echonumber != 0:
				await bot.send_message(message.channel, "@everyone "+echoecho)
				echonumber = echonumber - 1
			try:
				print(message.author, "echoed: ", echoecho)
			except UnicodeEncodeError:
				print ("invalid characters for the console")
			await bot.delete_message(message)'''

#status command
	if message.content.startswith("!status"):
		oldstatus = message.content
		newstatus = oldstatus.replace("!status", "")
		await bot.change_presence(game=discord.Game(name=newstatus))
		print(message.author,"changed status.")

	if message.content.startswith("!wormhole"):
		rawwormholemsg = message.content
		refinedwrmmsg= rawwormholemsg.replace("!wormhole", "")
		await bot.send_message(message.channel, "<@127296623779774464> wormhole send"+refinedwrmmsg)

#hotdog command
	if message.content.startswith("!hotdog"):
		await bot.send_message(message.channel, "https://cdn.discordapp.com/attachments/322415222004514827/333425912932728832/image.jpg")

#4chan
	if message.channel.is_private == True:
		chan = bot.get_channel("347242341632835584")
		await bot.send_message(chan, message.content)
		print(message.author)
#rename command

#joke command - not really working
	if message.content.startswith("!joke"):
		this_dir = os.path.dirname(__file__)
		jokepath = os.path.realpath("{0}".format(this_dir))
		with open(jokepath+"\\"+"jokes.txt", 'r') as infile:
			randomlinecount = random.randrange(2,6,2)
			print (randomlinecount)
			lines = [line for line in [infile.readline() for _ in range(randomlinecount)] if len(line) ]
			await  bot.send_message(message.channel, '\n'.join(lines))

#trap
	#if message.content.startswith("!delet"):
		deletcount=0
		deletcount+=1
		deletmessage=logs_from(message.channel, limit = 1)
		if deletcount == 2:
			await bot.delete_message(deletmessage)

bot.run('get your own token')
<commit_msg>Fixed windows-only peasant stuff<commit_after>import discord
from discord.ext import commands
import asyncio
import re
import sys
import os, random

#scriptpath = "../modules/commandmodule.py" #relative path dont DO a n y  Funny business
#sys.path.append(os.path.abspath(scriptpath))

description = '''A star bot.'''
bot = commands.Bot(command_prefix='!', description=description)
@bot.event
async def on_ready():
	print('Logged in as')
	print(bot.user.name, bot.user.id)
	print('Servers: ', end='')
	for server in bot.servers:
		print(str('{0}, ').format(server), end='')
	print()
	print('----------------')
	#global starnumber
	#starnumber = 4
	isrunning = False

#fn = os.path.join(os.path.dirname(__file__), 'starbot.py')
#dir = os.path.dirnam(__file__)

#owner
owner = "141695444995670017"

#channels
#best_of = 332172659863453697
#bot-bot = 332229780797652992

#roles
#shut up = 329705402021183488

def getRandomFile(path):
  """
  Returns a random filename, chosen among the files of the given path.
  """
  files = os.listdir(path)
  index = random.randrange(0, len(files))
  return files[index]

###starboard code###

#@bot.event
#async def on_member_join(member):
#	if member.id == ("289588477492723712"):
#		muterole = discord.utils.get(message.server.roles, id='329705402021183488')
#		await bot.add_roles(member, muterole)

@bot.event
async def on_reaction_add(reaction, member):
	print ("detected")
	if reaction.emoji == ("⭐"):
		if reaction.count == 5:
			nsfw = bot.get_channel("nsfw-garbage-bin")
			if reaction.message.channel != nsfw:
				best_of = discord.utils.get(reaction.message.server.channels, name= "best_of")

			#embed message itself
				em = discord.Embed(title='Starred post', description=reaction.message.content, colour=0xFFD700)
				em.set_author(name=reaction.message.author, icon_url=reaction.message.author.avatar_url)

			#embed url images
				if reaction.message.content.endswith(('.png', '.jpg', '.gif', '.gifv', '.webm')):
					em.set_image(url = reaction.message.content)

			#modifying reaction.message.attachments
				attach = reaction.message.attachments
				detector = str(attach)
				if detector.startswith('[{'):
					modify2 = str(attach)
					junk, junk2 = modify2.split("'url': '")
					websitelink, junk3 = junk2.split("',", 1)
					em.set_image(url = websitelink)
			#sending actual embed
				await bot.send_message(best_of, embed=em)
				await bot.clear_reactions(reaction.message)

		else:
			pass

isrunning = False

'''commands'''
@bot.event
async def on_message(message):
#check the message
	global isrunning
	#global commanduser
	#commanduser = ("null")

	#terobot v2

	print(isrunning)

	terobotchance=random.randrange(200)
	if terobotchance == 1:
		if message.channel.is_private == True:
			return
		if message.channel.server.id == "351885259199086593" or message.channel.server.id == "308265490877382656": #consider refactoring to check a list
			return
		else:
			this_dir = os.path.dirname('.')
			pastaraw = os.path.realpath("{0}/copypastas".format(this_dir))
			choosepasta = getRandomFile(pastaraw)
			copypastaaa = open(pastaraw+"/"+choosepasta, "r",encoding="utf8")
			selectedcopypasta = copypastaaa.read()
			await bot.send_message(message.channel, "<@{}> {}".format(message.author.id, selectedcopypasta))

#dablist command
	if message.content.startswith('!listdabs'):
		#dablist = os.listdir("starbot/dabs")

		this_dir = os.path.realpath(".")

		dabpath = os.path.realpath("{0}/dabs".format(this_dir))
		dablist = os.listdir(dabpath)
		liststr = '\n'.join(dablist)
		em = discord.Embed(title='a list of dabs for people with terrible memories', description=liststr, colour=0xFFD700)
		await bot.send_message(message.channel, embed=em)
#dablist command
	if message.content.startswith('!dab'):
	#checks for command !story

		if not isrunning:
			isrunning = True
			#commanduser = message.author
			try:
				storycommand = message.content.split("!dab ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "wtf it's empty what do you want me to do")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				isrunning = False
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "was unable to type properly")
				return
			try:
				this_dir = os.path.dirname('.')
				storypath = os.path.realpath("{0}/dabs".format(this_dir))
				print(storypath+"/"+storycommand+".txt")
				story = open("./"+storypath+"/"+storycommand+".txt", "r",encoding="utf8")
			except IOError:
				isrunning = False
				kill2=await bot.send_message(message.channel, "Choose a real file you nigger")
				await asyncio.sleep(2)
				await bot.delete_message(kill2)
				await bot.delete_message(message)
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "was unable to find a file.")
				return
			msg = await bot.send_message(message.channel, story.readline())
			this_dir = os.path.dirname('.')
			pathstory2 = os.path.realpath("{0}/dabs/".format(this_dir))
			story = open("./" + pathstory2+"/"+storycommand+".txt", "r",encoding="utf8")

			await asyncio.sleep(.1)
			#waits 3 second (I think 3 seconds, not sure)
			dabstop = False
			for chunk in iter(lambda: story.readline(), ''):

				await bot.edit_message(msg, chunk)
				#edits message to add next line
				#if dabstop == True:
					#return
				await asyncio.sleep(1)
				#waits 3 seconds
			await bot.delete_message(msg)
			await bot.delete_message(message)
			#deletes the message
			story.close()
			#closes the file
			isrunning = False
			if message.author == ("☭Liar☭#0946"):
					return
			print(message.author, "used !dab successfully.")

		else:
			kill = await bot.send_message(message.channel,"REEEEEEEEEEEEEEE WAIT YOUR TURN")
			await asyncio.sleep(2)
			await bot.delete_message(kill)
			await bot.delete_message(message)
			if message.author == ("☭Liar☭#0946"):
				return
			print (message.author, "was unable to wait his turn.")
#help command
	if message.content.startswith("!help"):
		this_dir = os.path.dirname('.')
		helppath = os.path.realpath("{0}".format(this_dir))
		helptext=open(helppath+"/"+"help.txt", "r", encoding="utf8")
		helptextactual=helptext.read()
		em = discord.Embed(title='Help', description=helptextactual, colour=0xFFD700)
		await bot.send_message(message.channel, embed=em)
		print(message.author, "asked for help!")
		helptext.close()

#echo command
	if message.content.startswith("!echo"):
		if message.author != bot.user:
			try:
				echoecho = message.content.split("!echo ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "I can't echo silence")
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "is unable to speak properly")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				return
			await bot.send_message(message.channel, echoecho)
			try:
				print(message.author, "echoed: ", echoecho)
			except UnicodeEncodeError:
				print ("invalid characters for the console")
			await bot.delete_message(message)

#superecho command
	'''if message.content.startswith("!superecho"):
		if message.author != bot.user:
			try:
				echoecho = message.content.split("!superecho ",1)[1]
			except IndexError:
				kill3 = await bot.send_message(message.channel, "I can't echo silence")
				if message.author == ("☭Liar☭#0946"):
					return
				print(message.author, "is unable to speak properly")
				await asyncio.sleep(1)
				await bot.delete_message(kill3)
				await bot.delete_message(message)
				return
			echonumber = 100
			while echonumber != 0:
				await bot.send_message(message.channel, "@everyone "+echoecho)
				echonumber = echonumber - 1
			try:
				print(message.author, "echoed: ", echoecho)
			except UnicodeEncodeError:
				print ("invalid characters for the console")
			await bot.delete_message(message)'''

#status command
	if message.content.startswith("!status"):
		oldstatus = message.content
		newstatus = oldstatus.replace("!status", "")
		await bot.change_presence(game=discord.Game(name=newstatus))
		print(message.author,"changed status.")

	if message.content.startswith("!wormhole"):
		rawwormholemsg = message.content
		refinedwrmmsg= rawwormholemsg.replace("!wormhole", "")
		await bot.send_message(message.channel, "<@127296623779774464> wormhole send"+refinedwrmmsg)

#hotdog command
	if message.content.startswith("!hotdog"):
		await bot.send_message(message.channel, "https://cdn.discordapp.com/attachments/322415222004514827/333425912932728832/image.jpg")

#4chan
	if message.channel.is_private == True:
		chan = bot.get_channel("347242341632835584")
		await bot.send_message(chan, message.content)
		print(message.author)
#rename command

#joke command - not really working
	if message.content.startswith("!joke"):
		this_dir = os.path.dirname('.')
		jokepath = os.path.realpath("{0}".format(this_dir))
		with open(jokepath+"/"+"jokes.txt", 'r') as infile:
			randomlinecount = random.randrange(2,6,2)
			print (randomlinecount)
			lines = [line for line in [infile.readline() for _ in range(randomlinecount)] if len(line) ]
			await  bot.send_message(message.channel, '\n'.join(lines))

#trap
	#if message.content.startswith("!delet"):
		deletcount=0
		deletcount+=1
		deletmessage=logs_from(message.channel, limit = 1)
		if deletcount == 2:
			await bot.delete_message(deletmessage)

bot.run('MzI2NDQ0NzgyMTg2NjU5ODQw.DDDcqw.FLFwP6gCi68dVkjxRVJV5d44am0')
<|endoftext|>


<commit_before>""" Startup script for the server."""
import argparse
import sys
import tornado.web
import socket
import tornado.options
from metadatastore.mds import MDS, MDSRO
from metadataservice.server.engine import (RunStartHandler, RunStopHandler,
                                           EventDescriptorHandler,
                                           EventHandler, loop)

from metadataservice.server.conf import load_configuration

if __name__ == "__main__":

    config = {k: v for k, v in load_configuration('metadataservice', 'MDS',
                                                  ['host', 'port', 'timezone',
                                                   'database'],
                                                  allow_missing=True).items()
              if v is not None}

    parser = argparse.ArgumentParser()
    parser.add_argument('--database', dest='database', type=str,
                        help='name of database to use')
    parser.add_argument('--mongo-host', dest='mongohost', type=str,
                        help='mongodb host to connect to')
    parser.add_argument('--timezone', dest='timezone', type=str,
                        help='Local timezone')
    parser.add_argument('--mongo-port', dest='mongoport', type=int,
                        help='mongodb port to connect')
    parser.add_argument('--service-port', dest='serviceport', type=int,
                        help='port to broadcast from')

    args = parser.parse_args()
    if args.database is not None:
        config['database'] = args.database
    if args.mongohost is not None:
        config['mongohost'] = args.mongohost
    if args.timezone is not None:
        config['timezone'] = args.timezone
    if args.mongoport is not None:
        config['mongoport'] = args.mongoport
    if args.serviceport is not None:
        config['serviceport'] = args.serviceport
    if args.timezone is not None:
        config['timezone'] = args.timezone

    libconfig = dict(host=config['mongohost'], port=config['mongoport'],
                     timezone=config['timezone'], database=config['database'])
    mdsro = MDSRO(version=1, config=libconfig)
    mdsrw = MDS(version=1, config=libconfig)

    print('Connecting to mongodb...{}:{}/{}'.format(config['mongohost'],
                                                    config['mongoport'],
                                                    config['database']))
    args = sys.argv
    # args.append("--log_file_prefix=/tmp/metadataservice.log")
    # tornado.options.parse_command_line(args)
    application = tornado.web.Application([
        (r'/run_start', RunStartHandler), (r'/run_stop', RunStopHandler),
        (r'/event_descriptor', EventDescriptorHandler),
        (r'/event', EventHandler)
         ], mdsro=mdsro, mdsrw=mdsrw)
    application.listen(config['serviceport'])
    print('Service live on address {}:{}'.format(socket.gethostname(),
                                                 config['serviceport']))
    loop.start()
<commit_msg>ENH:command line args raise meaningful errors in case not filled<commit_after>""" Startup script for the server."""
import argparse
import sys
import tornado.web
import socket
import tornado.options
from metadatastore.mds import MDS, MDSRO
from metadataservice.server.engine import (RunStartHandler, RunStopHandler,
                                           EventDescriptorHandler,
                                           EventHandler, loop)

from metadataservice.server.conf import load_configuration

if __name__ == "__main__":

    # Command line args have oriority over built in config file. Server should be
    # explicitly stated at startup. Leaving this here in case server needs to use
    # a config.
    # config = {k: v for k, v in load_configuration('metadataservice', 'MDS',
    #                                               ['host', 'port', 'timezone',
    #                                               'database'],
    #                                               allow_missing=True).items()
    #           if v is not None}

    config = {}
    parser = argparse.ArgumentParser()
    parser.add_argument('--database', dest='database', type=str,
                        help='name of database to use')
    parser.add_argument('--mongo-host', dest='mongohost', type=str,
                        help='mongodb host to connect to')
    parser.add_argument('--timezone', dest='timezone', type=str,
                        help='Local timezone')
    parser.add_argument('--mongo-port', dest='mongoport', type=int,
                        help='mongodb port to connect')
    parser.add_argument('--service-port', dest='serviceport', type=int,
                        help='port to broadcast from')

    args = parser.parse_args()
    # name of the database server will talk to.
    # If db does not exist, creates one
    if args.database is not None:
        config['database'] = args.database
    else:
        raise KeyError('--database is a required arg')
    # name/ip address of the machine hosting mongodb
    if args.mongohost is not None:
        config['mongohost'] = args.mongohost
    else:
        raise KeyError('--mongo-host is a required arg')
    # US/Eastern for BNL
    if args.timezone is not None:
        config['timezone'] = args.timezone
    else:
        raise KeyError('--timezone is a required arg')
    # port mongo uses on the mongo-host machine, 27017 by default
    if args.mongoport is not None:
        config['mongoport'] = args.mongoport
    else:
        raise KeyError('--mongo-port is a required arg')
    # Port that this server will use to communicate
    if args.serviceport is not None:
        config['serviceport'] = args.serviceport
    else:
        raise KeyError('--service-port is a required arg')
    libconfig = dict(host=config['mongohost'], port=config['mongoport'],
                     timezone=config['timezone'], database=config['database'])
    mdsro = MDSRO(version=1, config=libconfig)
    mdsrw = MDS(version=1, config=libconfig)

    print('Connecting to mongodb...{}:{}/{}'.format(config['mongohost'],
                                                    config['mongoport'],
                                                    config['database']))
    args = sys.argv
    # args.append("--log_file_prefix=/tmp/metadataservice.log")
    # tornado.options.parse_command_line(args)
    application = tornado.web.Application([
        (r'/run_start', RunStartHandler), (r'/run_stop', RunStopHandler),
        (r'/event_descriptor', EventDescriptorHandler),
        (r'/event', EventHandler)
         ], mdsro=mdsro, mdsrw=mdsrw)
    application.listen(config['serviceport'])
    print('Service live on address {}:{}'.format(socket.gethostname(),
                                                 config['serviceport']))
    loop.start()
<|endoftext|>


<commit_before>import mimetypes
import os

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.files.uploadedfile import UploadedFile
from django.core.files.uploadhandler import FileUploadHandler, \
    StopFutureHandlers
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.utils.encoding import smart_str, force_unicode

from google.appengine.api import files
from google.appengine.api.images import get_serving_url, NotImageError
from google.appengine.ext.blobstore import BlobInfo, BlobKey, delete, \
    create_upload_url, BLOB_KEY_HEADER, BLOB_RANGE_HEADER, BlobReader

def prepare_upload(request, url, **kwargs):
    return create_upload_url(url), {}

def serve_file(request, file, save_as, content_type, **kwargs):
    if hasattr(file, 'file') and hasattr(file.file, 'blobstore_info'):
        blobkey = file.file.blobstore_info.key()
    elif hasattr(file, 'blobstore_info'):
        blobkey = file.blobstore_info.key()
    else:
        raise ValueError("The provided file can't be served via the "
                         "Google App Engine Blobstore.")
    response = HttpResponse(content_type=content_type)
    response[BLOB_KEY_HEADER] = str(blobkey)
    response['Accept-Ranges'] = 'bytes'
    http_range = request.META.get('HTTP_RANGE')
    if http_range is not None:
        response[BLOB_RANGE_HEADER] = http_range
    if save_as:
        response['Content-Disposition'] = smart_str(
            u'attachment; filename=%s' % save_as)
    if file.size is not None:
        response['Content-Length'] = file.size
    return response

class BlobstoreStorage(Storage):
    """Google App Engine Blobstore storage backend."""

    def _open(self, name, mode='rb'):
        return BlobstoreFile(name, mode, self)

    def _save(self, name, content):
        name = name.replace('\\', '/')
        if hasattr(content, 'file') and \
           hasattr(content.file, 'blobstore_info'):
            data = content.file.blobstore_info
        elif hasattr(content, 'blobstore_info'):
            data = content.blobstore_info
        elif isinstance(content, File):
            guessed_type = mimetypes.guess_type(name)[0]
            file_name = files.blobstore.create(mime_type=guessed_type or 'application/octet-stream',
                                               _blobinfo_uploaded_filename=name)

            with files.open(file_name, 'a') as f:
                for chunk in content.chunks():
                    f.write(chunk)

            files.finalize(file_name)

            data = files.blobstore.get_blob_key(file_name)
        else:
            raise ValueError("The App Engine storage backend only supports "
                             "BlobstoreFile instances or File instances.")

        if isinstance(data, (BlobInfo, BlobKey)):
            # We change the file name to the BlobKey's str() value.
            if isinstance(data, BlobInfo):
                data = data.key()
            return '%s/%s' % (data, name.lstrip('/'))
        else:
            raise ValueError("The App Engine Blobstore only supports "
                             "BlobInfo values. Data can't be uploaded "
                             "directly. You have to use the file upload "
                             "handler.")

    def delete(self, name):
        delete(self._get_key(name))

    def exists(self, name):
        return self._get_blobinfo(name) is not None

    def size(self, name):
        return self._get_blobinfo(name).size

    def url(self, name):
        try:
            #return a protocol-less URL, because django can't/won't pass
            #down an argument saying whether it should be secure or not
            url = get_serving_url(self._get_blobinfo(name))
            return re.sub("http://", "//", url)
        except NotImageError:
            return None

    def created_time(self, name):
        return self._get_blobinfo(name).creation

    def get_valid_name(self, name):
        return force_unicode(name).strip().replace('\\', '/')

    def get_available_name(self, name):
        return name.replace('\\', '/')

    def _get_key(self, name):
        return BlobKey(name.split('/', 1)[0])

    def _get_blobinfo(self, name):
        return BlobInfo.get(self._get_key(name))

class BlobstoreFile(File):

    def __init__(self, name, mode, storage):
        self.name = name
        self._storage = storage
        self._mode = mode
        self.blobstore_info = storage._get_blobinfo(name)

    @property
    def size(self):
        return self.blobstore_info.size

    def write(self, content):
        raise NotImplementedError()

    @property
    def file(self):
        if not hasattr(self, '_file'):
            self._file = BlobReader(self.blobstore_info.key())
        return self._file

class BlobstoreFileUploadHandler(FileUploadHandler):
    """
    File upload handler for the Google App Engine Blobstore.
    """

    def new_file(self, *args, **kwargs):
        super(BlobstoreFileUploadHandler, self).new_file(*args, **kwargs)
        blobkey = self.content_type_extra.get('blob-key')
        self.active = blobkey is not None
        if self.active:
            self.blobkey = BlobKey(blobkey)
            raise StopFutureHandlers()

    def receive_data_chunk(self, raw_data, start):
        """
        Add the data to the StringIO file.
        """
        if not self.active:
            return raw_data

    def file_complete(self, file_size):
        """
        Return a file object if we're activated.
        """
        if not self.active:
            return

        return BlobstoreUploadedFile(
            blobinfo=BlobInfo(self.blobkey),
            charset=self.charset)

class BlobstoreUploadedFile(UploadedFile):
    """
    A file uploaded into memory (i.e. stream-to-memory).
    """

    def __init__(self, blobinfo, charset):
        super(BlobstoreUploadedFile, self).__init__(
            BlobReader(blobinfo.key()), blobinfo.filename,
            blobinfo.content_type, blobinfo.size, charset)
        self.blobstore_info = blobinfo

    def open(self, mode=None):
        pass

    def chunks(self, chunk_size=1024 * 128):
        self.file.seek(0)
        while True:
            content = self.read(chunk_size)
            if not content:
                break
            yield content

    def multiple_chunks(self, chunk_size=1024 * 128):
        return True
<commit_msg>Allow passing a blobkey to serve_file. Fix a bug where filenames with spaces wouldn't be quoted properly<commit_after>import mimetypes
import os

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.files.uploadedfile import UploadedFile
from django.core.files.uploadhandler import FileUploadHandler, \
    StopFutureHandlers
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.utils.encoding import smart_str, force_unicode

from google.appengine.api import files
from google.appengine.api.images import get_serving_url, NotImageError
from google.appengine.ext.blobstore import BlobInfo, BlobKey, delete, \
    create_upload_url, BLOB_KEY_HEADER, BLOB_RANGE_HEADER, BlobReader

def prepare_upload(request, url, **kwargs):
    return create_upload_url(url), {}

def serve_file(request, file, save_as, content_type, **kwargs):
    if isinstance(file, BlobKey):
        blobkey = file
    elif hasattr(file, 'file') and hasattr(file.file, 'blobstore_info'):
        blobkey = file.file.blobstore_info.key()
    elif hasattr(file, 'blobstore_info'):
        blobkey = file.blobstore_info.key()
    else:
        raise ValueError("The provided file can't be served via the "
                         "Google App Engine Blobstore.")
    response = HttpResponse(content_type=content_type)
    response[BLOB_KEY_HEADER] = str(blobkey)
    response['Accept-Ranges'] = 'bytes'
    http_range = request.META.get('HTTP_RANGE')
    if http_range is not None:
        response[BLOB_RANGE_HEADER] = http_range
    if save_as:
        response['Content-Disposition'] = smart_str(
            u'attachment; filename="%s"' % save_as)

    info = BlobInfo.get(blobkey)
    if info.size is not None:
        response['Content-Length'] = info.size
    return response

class BlobstoreStorage(Storage):
    """Google App Engine Blobstore storage backend."""

    def _open(self, name, mode='rb'):
        return BlobstoreFile(name, mode, self)

    def _save(self, name, content):
        name = name.replace('\\', '/')
        if hasattr(content, 'file') and \
           hasattr(content.file, 'blobstore_info'):
            data = content.file.blobstore_info
        elif hasattr(content, 'blobstore_info'):
            data = content.blobstore_info
        elif isinstance(content, File):
            guessed_type = mimetypes.guess_type(name)[0]
            file_name = files.blobstore.create(mime_type=guessed_type or 'application/octet-stream',
                                               _blobinfo_uploaded_filename=name)

            with files.open(file_name, 'a') as f:
                for chunk in content.chunks():
                    f.write(chunk)

            files.finalize(file_name)

            data = files.blobstore.get_blob_key(file_name)
        else:
            raise ValueError("The App Engine storage backend only supports "
                             "BlobstoreFile instances or File instances.")

        if isinstance(data, (BlobInfo, BlobKey)):
            # We change the file name to the BlobKey's str() value.
            if isinstance(data, BlobInfo):
                data = data.key()
            return '%s/%s' % (data, name.lstrip('/'))
        else:
            raise ValueError("The App Engine Blobstore only supports "
                             "BlobInfo values. Data can't be uploaded "
                             "directly. You have to use the file upload "
                             "handler.")

    def delete(self, name):
        delete(self._get_key(name))

    def exists(self, name):
        return self._get_blobinfo(name) is not None

    def size(self, name):
        return self._get_blobinfo(name).size

    def url(self, name):
        try:
            #return a protocol-less URL, because django can't/won't pass
            #down an argument saying whether it should be secure or not
            url = get_serving_url(self._get_blobinfo(name))
            return re.sub("http://", "//", url)
        except NotImageError:
            return None

    def created_time(self, name):
        return self._get_blobinfo(name).creation

    def get_valid_name(self, name):
        return force_unicode(name).strip().replace('\\', '/')

    def get_available_name(self, name):
        return name.replace('\\', '/')

    def _get_key(self, name):
        return BlobKey(name.split('/', 1)[0])

    def _get_blobinfo(self, name):
        return BlobInfo.get(self._get_key(name))

class BlobstoreFile(File):

    def __init__(self, name, mode, storage):
        self.name = name
        self._storage = storage
        self._mode = mode
        self.blobstore_info = storage._get_blobinfo(name)

    @property
    def size(self):
        return self.blobstore_info.size

    def write(self, content):
        raise NotImplementedError()

    @property
    def file(self):
        if not hasattr(self, '_file'):
            self._file = BlobReader(self.blobstore_info.key())
        return self._file

class BlobstoreFileUploadHandler(FileUploadHandler):
    """
    File upload handler for the Google App Engine Blobstore.
    """

    def new_file(self, *args, **kwargs):
        super(BlobstoreFileUploadHandler, self).new_file(*args, **kwargs)
        blobkey = self.content_type_extra.get('blob-key')
        self.active = blobkey is not None
        if self.active:
            self.blobkey = BlobKey(blobkey)
            raise StopFutureHandlers()

    def receive_data_chunk(self, raw_data, start):
        """
        Add the data to the StringIO file.
        """
        if not self.active:
            return raw_data

    def file_complete(self, file_size):
        """
        Return a file object if we're activated.
        """
        if not self.active:
            return

        return BlobstoreUploadedFile(
            blobinfo=BlobInfo(self.blobkey),
            charset=self.charset)

class BlobstoreUploadedFile(UploadedFile):
    """
    A file uploaded into memory (i.e. stream-to-memory).
    """

    def __init__(self, blobinfo, charset):
        super(BlobstoreUploadedFile, self).__init__(
            BlobReader(blobinfo.key()), blobinfo.filename,
            blobinfo.content_type, blobinfo.size, charset)
        self.blobstore_info = blobinfo

    def open(self, mode=None):
        pass

    def chunks(self, chunk_size=1024 * 128):
        self.file.seek(0)
        while True:
            content = self.read(chunk_size)
            if not content:
                break
            yield content

    def multiple_chunks(self, chunk_size=1024 * 128):
        return True
<|endoftext|>


<commit_before>#Start Database

from gameheart.entities import models
from gameheart.entities import forms

user = User.objects.get(id=1)
has_profile = UserProfile.objects.get(user=user).count()
if user:
    if has_profile:
        form = UserProfileForm()
        model_instance = form.save(commit=False)
        model_instance.user = user
        model_instance.save()
<commit_msg>Made startdb.py actually work for a blank db<commit_after>#Start Database

from gameheart.entities.models import *
from gameheart.entities.forms import *

user = User.objects.get(id=1)

try:
	profile = UserProfile.objects.get(user=user)

except UserProfile.DoesNotExist:
	form = UserProfileForm()
	model_instance = form.save(commit=False)
	model_instance.user = user
	model_instance.save()

<|endoftext|>


<commit_before>#!/usr/bin/env/python

"""
    sv_orgs.py: Simple VIVO for Organizations

    Read a spreadsheet and follow the directions to add, update or remove organizations and/or
    organization attributes from VIVO.

    Produce a spreadsheet from VIVO that has the fields ready for editing and updating

    Exceptions are thrown, caught and logged for missing required elements that are missing

    See CHANGELOG.md for history

"""

# TODO: test file with test cases and notes for each
# TODO: Write do_update_orgs using rdflib
# TODO: Support lookup by name or uri

__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015, University of Florida"
__license__ = "New BSD License"
__version__ = "0.21"

from vivofoundation import read_csv
import argparse
import codecs

# Helper functions start here.  Reusable code for orgs will be moved to vivoorgs

def do_get_orgs(filename):
    from vivofoundation import vivo_sparql_query
    """
    Organization data is queried from VIVO and returned as a tab delimited text file suitable for
    editing using an editor or spreadsheet, and suitable for use by sv_orgs update.
    The following columns are returned:

    uri           VIVO uri of organization
    name          name of organization
    type          type of organization
    url           url of organization web site
    within        uri or name of parent organization(s).  Semi-colon delimited
    overview      text describing organization
    photo         filename of photo
    abbreviation  Abbreviated name of organization
    successor     uri or name of successor if any
    address       semi-colon delimited address lines
    phone         primary phone number
    email         primary email address
    isni          ISNI of org if any

    :param filename: Tab delimited file of org data
    :return:  None.  File is written
    """

    query = """
    SELECT ?uri ?name ?type ?url ?within ?overview ?photo ?abbreviation ?successor ?address1 ?address2 ?city ?state ?zip ?phone ?email ?isni
    WHERE {

        ?uri a foaf:Organization .
        ?uri a ufVivo:UFEntity .
        ?uri rdf:type ?type .
        FILTER (?type != foaf:Organization && ?type != foaf:Agent && ?type != owl:Thing && ?type != ufVivo:UFEntity)
        ?uri rdfs:label ?name .
        OPTIONAL { ?uri vivo:webpage ?weburi .  ?weburi vivo:linkURI ?url .}
        OPTIONAL { ?uri vivo:subOrganizationWithin ?within . }
        OPTIONAL { ?uri vivo:overview ?overview . }
        OPTIONAL { ?uri vitro-public:mainImage ?photouri . ?photouri vitro-public:filename ?photo .}
        OPTIONAL { ?uri vivo:abbreviation ?abbreviation . }
        OPTIONAL { ?uri vivo:hasSuccessorOrganization ?successor . }
        OPTIONAL { ?uri vivo:mailingAddress ?address .
            OPTIONAL{ ?address vivo:address1 ?address1 . }
            OPTIONAL{ ?address vivo:address2 ?address2 . }
            OPTIONAL{ ?address vivo:addressCity ?city . }
            OPTIONAL{ ?address vivo:addressPostalCode ?zip . }
            OPTIONAL{ ?address vivo:addressState ?state . }
            }
       OPTIONAL { ?uri vivo:phoneNumber ?phone . }
       OPTIONAL { ?uri vivo:email ?email . }
       OPTIONAL { ?uri ufVivo:isni ?isni . }

    }
    ORDER BY ?name
    """

    org_result_set = vivo_sparql_query(query)

    orgs = {}

    for binding in org_result_set['results']['bindings']:

        uri = binding['uri']['value']
        if uri not in orgs:
            orgs[uri] = {}

        #  Each property is either single-valued, multi-valued, and/or de-referenced.  We're not
        #  not ready for de-referenced yet.  That will require more complex SPARQL

        #  Single valued attributes.  If data has more than one value, use the last value found

        for name in ('uri', 'name', 'url', 'overview', 'photo', 'abbreviation', 'address1', 'address2', 'city', 'state',
                     'zip', 'phone', 'email', 'isni'):
            if name in binding:
                orgs[uri][name] = binding[name]

        # multi-valued attributes.  Collect all values into lists

        for name in ('successor', 'within', 'type'):
            if name in binding:
                if name in orgs[uri]:
                    orgs[uri][name].append(binding[name])
                else:
                    orgs[uri][name] = [binding[name]]

    # Write out the file

    outfile = codecs.open(filename, mode='w', encoding='ascii', errors='xmlcharrefreplace')

    columns = ('uri', 'name', 'type', 'within', 'url', 'phone', 'email', 'address1', 'address2', 'city', 'state',
               'zip', 'photo',
               'abbreviation', 'isni', 'successor', 'overview')
    outfile.write('\t'.join(columns))
    outfile.write('\n')

    for uri in sorted(orgs.keys()):
        for name in columns:
            if name in orgs[uri]:
                if type(orgs[uri][name]) is list:
                    if name == 'type':
                        val = ';'.join(set(org_types.get(x['value'], ' ') for x in orgs[uri][name]))
                    else:
                        val = ';'.join(set(x['value'] for x in orgs[uri][name]))
                else:
                    val = orgs[uri][name]['value'].replace('\n', ' ').replace('\r', ' ')
                outfile.write(val)
            if name != columns[len(columns)-1]:
                outfile.write('\t')
        outfile.write('\n')

    outfile.close()

    return len(orgs)

def get_org_triples():
    org_query = """
    SELECT ?s ?p ?o
    WHERE {
        {?s a foaf:Organization . ?s ?p ?o . }
        UNION
        {?o a foaf:Organization . ?s ?p ?o . }
    }
    """
    from vivofoundation import vivo_sparql_query
    triples = vivo_sparql_query(org_query)
    print len(triples["results"]["bindings"]), "org triples"
    return triples

def do_update_orgs(filename):
    """
    read updates from a spreadsheet filename.  Compare to orgs in VIVO.  generate add and sub
    rdf as necessary to process requested changes
    """
    norgs = 0
    triples = get_org_triples()
    return norgs

# Driver program starts here

org_type_data = read_csv("org_types.txt", delimiter=' ')
org_types = {}
org_uris = {}
for row in org_type_data.values():
    org_uris[row['type']] = row['uri']
    org_types[row['uri']] = row['type']
print org_types

parser = argparse.ArgumentParser()
parser.add_argument("action", help="desired action.  get = get org data from VIVO.  update = update VIVO organ"
                                   "izational data from a spreadsheet", default="update", nargs='?')
parser.add_argument("filename", help="name of spreadsheet containing org data to be updated in VIVO",
                    default="sv_orgs.txt", nargs='?')
args = parser.parse_args()

if args.action == 'get':
    print "will get org data from VIVO"
    norgs = do_get_orgs(args.filename)
    print norgs, "Organizations in", args.filename
elif args.action == "update":
    orgs = read_csv(args.filename, delimiter="\t")
    print orgs
    print len(orgs), "Organizations in", args.filename
    for row in sorted(orgs.keys()):
        print row, orgs[row]['uri']
    norgs = do_update_orgs(args.filename)
else:
    print "Unknown action.  Try sv_orgs -h for help"

<commit_msg>version 0.22 writes triples to nt file.  reads nt into rdflib Graph structure<commit_after>#!/usr/bin/env/python

"""
    sv_orgs.py: Simple VIVO for Organizations

    Read a spreadsheet and follow the directions to add, update or remove organizations and/or
    organization attributes from VIVO.

    Produce a spreadsheet from VIVO that has the fields ready for editing and updating

    Exceptions are thrown, caught and logged for missing required elements that are missing

    See CHANGELOG.md for history

"""

# TODO: test file with test cases and notes for each
# TODO: Write do_update_orgs using rdflib
# TODO: Support lookup by name or uri

__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015, University of Florida"
__license__ = "New BSD License"
__version__ = "0.22"

from vivofoundation import read_csv
import argparse
import codecs

# Helper functions start here.  Reusable code for orgs will be moved to vivoorgs

def do_get_orgs(filename):
    from vivofoundation import vivo_sparql_query
    """
    Organization data is queried from VIVO and returned as a tab delimited text file suitable for
    editing using an editor or spreadsheet, and suitable for use by sv_orgs update.
    The following columns are returned:

    uri           VIVO uri of organization
    name          name of organization
    type          type of organization
    url           url of organization web site
    within        uri or name of parent organization(s).  Semi-colon delimited
    overview      text describing organization
    photo         filename of photo
    abbreviation  Abbreviated name of organization
    successor     uri or name of successor if any
    address       semi-colon delimited address lines
    phone         primary phone number
    email         primary email address
    isni          ISNI of org if any

    :param filename: Tab delimited file of org data
    :return:  None.  File is written
    """

    query = """
    SELECT ?uri ?name ?type ?url ?within ?overview ?photo ?abbreviation ?successor ?address1 ?address2 ?city ?state ?zip ?phone ?email ?isni
    WHERE {

        ?uri a foaf:Organization .
        ?uri a ufVivo:UFEntity .
        ?uri rdf:type ?type .
        FILTER (?type != foaf:Organization && ?type != foaf:Agent && ?type != owl:Thing && ?type != ufVivo:UFEntity)
        ?uri rdfs:label ?name .
        OPTIONAL { ?uri vivo:webpage ?weburi .  ?weburi vivo:linkURI ?url .}
        OPTIONAL { ?uri vivo:subOrganizationWithin ?within . }
        OPTIONAL { ?uri vivo:overview ?overview . }
        OPTIONAL { ?uri vitro-public:mainImage ?photouri . ?photouri vitro-public:filename ?photo .}
        OPTIONAL { ?uri vivo:abbreviation ?abbreviation . }
        OPTIONAL { ?uri vivo:hasSuccessorOrganization ?successor . }
        OPTIONAL { ?uri vivo:mailingAddress ?address .
            OPTIONAL{ ?address vivo:address1 ?address1 . }
            OPTIONAL{ ?address vivo:address2 ?address2 . }
            OPTIONAL{ ?address vivo:addressCity ?city . }
            OPTIONAL{ ?address vivo:addressPostalCode ?zip . }
            OPTIONAL{ ?address vivo:addressState ?state . }
            }
       OPTIONAL { ?uri vivo:phoneNumber ?phone . }
       OPTIONAL { ?uri vivo:email ?email . }
       OPTIONAL { ?uri ufVivo:isni ?isni . }

    }
    ORDER BY ?name
    """

    org_result_set = vivo_sparql_query(query)

    orgs = {}

    for binding in org_result_set['results']['bindings']:

        uri = binding['uri']['value']
        if uri not in orgs:
            orgs[uri] = {}

        #  Each property is either single-valued, multi-valued, and/or de-referenced.  We're not
        #  not ready for de-referenced yet.  That will require more complex SPARQL

        #  Single valued attributes.  If data has more than one value, use the last value found

        for name in ('uri', 'name', 'url', 'overview', 'photo', 'abbreviation', 'address1', 'address2', 'city', 'state',
                     'zip', 'phone', 'email', 'isni'):
            if name in binding:
                orgs[uri][name] = binding[name]

        # multi-valued attributes.  Collect all values into lists

        for name in ('successor', 'within', 'type'):
            if name in binding:
                if name in orgs[uri]:
                    orgs[uri][name].append(binding[name])
                else:
                    orgs[uri][name] = [binding[name]]

    # Write out the file

    outfile = codecs.open(filename, mode='w', encoding='ascii', errors='xmlcharrefreplace')

    columns = ('uri', 'name', 'type', 'within', 'url', 'phone', 'email', 'address1', 'address2', 'city', 'state',
               'zip', 'photo',
               'abbreviation', 'isni', 'successor', 'overview')
    outfile.write('\t'.join(columns))
    outfile.write('\n')

    for uri in sorted(orgs.keys()):
        for name in columns:
            if name in orgs[uri]:
                if type(orgs[uri][name]) is list:
                    if name == 'type':
                        val = ';'.join(set(org_types.get(x['value'], ' ') for x in orgs[uri][name]))
                    else:
                        val = ';'.join(set(x['value'] for x in orgs[uri][name]))
                else:
                    val = orgs[uri][name]['value'].replace('\n', ' ').replace('\r', ' ')
                outfile.write(val)
            if name != columns[len(columns)-1]:
                outfile.write('\t')
        outfile.write('\n')

    outfile.close()

    return len(orgs)

def get_org_triples():
    org_query = """
    SELECT ?s ?p ?o
    WHERE {
        {?s a foaf:Organization . ?s ?p ?o . }
        UNION
        {?o a foaf:Organization . ?s ?p ?o . }
    }
    """
    from vivofoundation import vivo_sparql_query
    triples = vivo_sparql_query(org_query)
    print len(triples["results"]["bindings"]), "org triples"
    return triples

def iri_string(d):
    """
    Given a dict representing the value of an element returned by a fuseki query, generate
    the string version suitable for inclusion in an NT file.
    """
    return '<' + d['value'] + '>'

def iri_predicate(d):
    """
    Give a dict representing the value of an element returned by a fuseki query corresponding
    to the predicate of a triple, return the string version suitable for inclusion in an NT file.
    :param d: dictionary of a value returned by fuseki
    :return: string

    See http://www.w3.org/TR/n-triples/ for standards for n-triples
    """
    if d['type'] == 'literal' or d['type'] == 'typed-literal':
        s = '"' + d['value'].replace('"', '\\"').replace('\n', '\\n').replace('\r', '\\r') + '"'
        if 'xml:lang' in d:
            s = s + '@' + d['xml:lang']
        if 'datatype' in d:
            s = s + '^^<' + d['datatype'] + '>'
        return s
    else:
        return iri_string(d)

def write_triples(triples, filename):
    """
    Given a structure from fuseki with the triples, write the triples to a file in nt format

    :param triples: structure from fuseki query with s, p, o
    :param filename: filename to write triples to
    :return:
    """
    import codecs
    outfile = codecs.open('__'+filename, mode='w', encoding='ascii', errors='xmlcharrefreplace')
    for row in triples['results']['bindings']:
        outfile.write(iri_string(row['s']))
        outfile.write(' ')
        outfile.write(iri_string(row['p']))
        outfile.write(' ')
        outfile.write(iri_predicate(row['o']))
        outfile.write(' .\n')
    outfile.close()
    return

def do_update_orgs(filename):
    """
    read updates from a spreadsheet filename.  Compare to orgs in VIVO.  generate add and sub
    rdf as necessary to process requested changes
    """
    from rdflib import Graph
    triples = get_org_triples()
    write_triples(triples, filename)
    og = Graph()
    og.parse('__'+filename, format='nt')
    ug = og
    print 'Graphs ready for processing. Original has ', len(og), '. Update graph has', len(ug)
    return len(og)

# Driver program starts here

org_type_data = read_csv("org_types.txt", delimiter=' ')
org_types = {}
org_uris = {}
for row in org_type_data.values():
    org_uris[row['type']] = row['uri']
    org_types[row['uri']] = row['type']
print org_types

parser = argparse.ArgumentParser()
parser.add_argument("action", help="desired action.  get = get org data from VIVO.  update = update VIVO organ"
                                   "izational data from a spreadsheet", default="update", nargs='?')
parser.add_argument("filename", help="name of spreadsheet containing org data to be updated in VIVO",
                    default="sv_orgs.txt", nargs='?')
args = parser.parse_args()

if args.action == 'get':
    print "will get org data from VIVO"
    norgs = do_get_orgs(args.filename)
    print norgs, "Organizations in", args.filename
elif args.action == "update":
    orgs = read_csv(args.filename, delimiter="\t")
    print len(orgs), "Organizations in", args.filename
    norgs = do_update_orgs(args.filename)
else:
    print "Unknown action.  Try sv_orgs -h for help"

<|endoftext|>