<commit_before>#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""DNNRegressor with custom estimator for abalone dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys
import tempfile

from six.moves import urllib

import numpy as np
import tensorflow as tf
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib

FLAGS = None

tf.logging.set_verbosity(tf.logging.INFO)

# Learning rate for the model
LEARNING_RATE = 0.001

def maybe_download(train_data, test_data, predict_data):
  """Maybe downloads training data and returns train and test file names."""
  if train_data:
    train_file_name = train_data
  else:
    train_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_train.csv",
        train_file.name)
    train_file_name = train_file.name
    train_file.close()
    print("Training data is downloaded to %s" % train_file_name)

  if test_data:
    test_file_name = test_data
  else:
    test_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_test.csv", test_file.name)
    test_file_name = test_file.name
    test_file.close()
    print("Test data is downloaded to %s" % test_file_name)

  if predict_data:
    predict_file_name = predict_data
  else:
    predict_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_predict.csv",
        predict_file.name)
    predict_file_name = predict_file.name
    predict_file.close()
    print("Prediction data is downloaded to %s" % predict_file_name)

  return train_file_name, test_file_name, predict_file_name

def model_fn(features, targets, mode, params):
  """Model function for Estimator."""

  # Connect the first hidden layer to input layer
  # (features) with relu activation
  first_hidden_layer = tf.contrib.layers.relu(features, 10)

  # Connect the second hidden layer to first hidden layer with relu
  second_hidden_layer = tf.contrib.layers.relu(first_hidden_layer, 10)

  # Connect the output layer to second hidden layer (no activation fn)
  output_layer = tf.contrib.layers.linear(second_hidden_layer, 1)

  # Reshape output layer to 1-dim Tensor to return predictions
  predictions = tf.reshape(output_layer, [-1])
  predictions_dict = {"ages": predictions}

  # Calculate loss using mean squared error
  loss = tf.losses.mean_squared_error(targets, predictions)

  # Calculate root mean squared error as additional eval metric
  eval_metric_ops = {
      "rmse": tf.metrics.root_mean_squared_error(
          tf.cast(targets, tf.float64), predictions)
  }

  train_op = tf.contrib.layers.optimize_loss(
      loss=loss,
      global_step=tf.contrib.framework.get_global_step(),
      learning_rate=params["learning_rate"],
      optimizer="SGD")

  return model_fn_lib.ModelFnOps(
      mode=mode,
      predictions=predictions_dict,
      loss=loss,
      train_op=train_op,
      eval_metric_ops=eval_metric_ops)

def main(unused_argv):
  # Load datasets
  abalone_train, abalone_test, abalone_predict = maybe_download(
      FLAGS.train_data, FLAGS.test_data, FLAGS.predict_data)

  # Training examples
  training_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_train, target_dtype=np.int, features_dtype=np.float64)

  # Test examples
  test_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_test, target_dtype=np.int, features_dtype=np.float64)

  # Set of 7 examples for which to predict abalone ages
  prediction_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_predict, target_dtype=np.int, features_dtype=np.float64)

  # Set model params
  model_params = {"learning_rate": LEARNING_RATE}

  # Instantiate Estimator
  nn = tf.contrib.learn.Estimator(model_fn=model_fn, params=model_params)

  # Fit
  nn.fit(x=training_set.data, y=training_set.target, steps=5000)

  # Score accuracy
  ev = nn.evaluate(x=test_set.data, y=test_set.target, steps=1)
  print("Loss: %s" % ev["loss"])
  print("Root Mean Squared Error: %s" % ev["rmse"])

  # Print out predictions
  predictions = nn.predict(x=prediction_set.data, as_iterable=True)
  for i, p in enumerate(predictions):
    print("Prediction %s: %s" % (i + 1, p["ages"]))

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.register("type", "bool", lambda v: v.lower() == "true")
  parser.add_argument(
      "--train_data", type=str, default="", help="Path to the training data.")
  parser.add_argument(
      "--test_data", type=str, default="", help="Path to the test data.")
  parser.add_argument(
      "--predict_data",
      type=str,
      default="",
      help="Path to the prediction data.")
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<commit_msg>Add input function for training and testing (#9617) (#9650)<commit_after>#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""DNNRegressor with custom estimator for abalone dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import sys
import tempfile

from six.moves import urllib

import numpy as np
import tensorflow as tf
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib

FLAGS = None

tf.logging.set_verbosity(tf.logging.INFO)

# Learning rate for the model
LEARNING_RATE = 0.001

def maybe_download(train_data, test_data, predict_data):
  """Maybe downloads training data and returns train and test file names."""
  if train_data:
    train_file_name = train_data
  else:
    train_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_train.csv",
        train_file.name)
    train_file_name = train_file.name
    train_file.close()
    print("Training data is downloaded to %s" % train_file_name)

  if test_data:
    test_file_name = test_data
  else:
    test_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_test.csv", test_file.name)
    test_file_name = test_file.name
    test_file.close()
    print("Test data is downloaded to %s" % test_file_name)

  if predict_data:
    predict_file_name = predict_data
  else:
    predict_file = tempfile.NamedTemporaryFile(delete=False)
    urllib.request.urlretrieve(
        "http://download.tensorflow.org/data/abalone_predict.csv",
        predict_file.name)
    predict_file_name = predict_file.name
    predict_file.close()
    print("Prediction data is downloaded to %s" % predict_file_name)

  return train_file_name, test_file_name, predict_file_name

def model_fn(features, targets, mode, params):
  """Model function for Estimator."""

  # Connect the first hidden layer to input layer
  # (features) with relu activation
  first_hidden_layer = tf.contrib.layers.relu(features, 10)

  # Connect the second hidden layer to first hidden layer with relu
  second_hidden_layer = tf.contrib.layers.relu(first_hidden_layer, 10)

  # Connect the output layer to second hidden layer (no activation fn)
  output_layer = tf.contrib.layers.linear(second_hidden_layer, 1)

  # Reshape output layer to 1-dim Tensor to return predictions
  predictions = tf.reshape(output_layer, [-1])
  predictions_dict = {"ages": predictions}

  # Calculate loss using mean squared error
  loss = tf.losses.mean_squared_error(targets, predictions)

  # Calculate root mean squared error as additional eval metric
  eval_metric_ops = {
      "rmse": tf.metrics.root_mean_squared_error(
          tf.cast(targets, tf.float64), predictions)
  }

  train_op = tf.contrib.layers.optimize_loss(
      loss=loss,
      global_step=tf.contrib.framework.get_global_step(),
      learning_rate=params["learning_rate"],
      optimizer="SGD")

  return model_fn_lib.ModelFnOps(
      mode=mode,
      predictions=predictions_dict,
      loss=loss,
      train_op=train_op,
      eval_metric_ops=eval_metric_ops)

def main(unused_argv):
  # Load datasets
  abalone_train, abalone_test, abalone_predict = maybe_download(
      FLAGS.train_data, FLAGS.test_data, FLAGS.predict_data)

  # Training examples
  training_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_train, target_dtype=np.int, features_dtype=np.float64)

  # Test examples
  test_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_test, target_dtype=np.int, features_dtype=np.float64)

  # Set of 7 examples for which to predict abalone ages
  prediction_set = tf.contrib.learn.datasets.base.load_csv_without_header(
      filename=abalone_predict, target_dtype=np.int, features_dtype=np.float64)

  # Set model params
  model_params = {"learning_rate": LEARNING_RATE}

  # Instantiate Estimator
  nn = tf.contrib.learn.Estimator(model_fn=model_fn, params=model_params)

  def get_train_inputs():
    x = tf.constant(training_set.data)
    y = tf.constant(training_set.target)
    return x, y

  # Fit
  nn.fit(input_fn=get_train_inputs, steps=5000)

  # Score accuracy
  def get_test_inputs():
    x = tf.constant(test_set.data)
    y = tf.constant(test_set.target)
    return x, y

  ev = nn.evaluate(input_fn=get_test_inputs, steps=1)
  print("Loss: %s" % ev["loss"])
  print("Root Mean Squared Error: %s" % ev["rmse"])

  # Print out predictions
  predictions = nn.predict(x=prediction_set.data, as_iterable=True)
  for i, p in enumerate(predictions):
    print("Prediction %s: %s" % (i + 1, p["ages"]))

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.register("type", "bool", lambda v: v.lower() == "true")
  parser.add_argument(
      "--train_data", type=str, default="", help="Path to the training data.")
  parser.add_argument(
      "--test_data", type=str, default="", help="Path to the test data.")
  parser.add_argument(
      "--predict_data",
      type=str,
      default="",
      help="Path to the prediction data.")
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<|endoftext|>


<commit_before>import pysam
from collections import Counter
import argparse
from os.path import isfile
from Bio import SeqIO

bam_data = []
score = Counter()
endResult = []
merge_data = []
fastq_data = {}

def file_check():
    flag = True
    if not isfile(args.fastq_file):
        print("ERROR: Fastq file: '{}' not found.".format(args.fastq_file))
        flag = False
    if not isfile(args.bam_file):
        print("ERROR: bam file: '{}' not found.".format(args.bam_file))
        flag = False
    if flag == False:
        exit()

def bam_reader(bam_file, end, tag):
    bam = pysam.AlignmentFile(bam_file, "rb")
    chromosomes = bam.references
    for chrom_all in chromosomes:
        bam_filter = []
        data = bam.fetch(multiple_iterators=True, until_eof=True)
        if end == True:
            for line in data:
                try:
                    chrom_bam = bam.get_reference_name(line.reference_id)
                except:
                    continue
                if tag == False:
                    if (line.is_unmapped == False and line.has_tag("XS") == False and chrom_bam == chrom_all):
                        if line.is_reverse:
                            strand = '-'
                        else:
                            strand = '+'
                        if 'N' not in fastq_data[line.query_name]:
                            bam_filter.append((line.reference_start,
                                               line.reference_start + line.reference_length,
                                               line.query_name, strand,
                                               chrom_bam, fastq_data[line.query_name]))
                else:
                    if (line.is_unmapped == False and chrom_bam == chrom_all):
                        if line.is_reverse:
                            strand = '-'
                        else:
                            strand = '+'
                        if 'N' not in fastq_data[line.query_name]:
                            bam_filter.append((line.reference_start,
                                               line.reference_start + line.reference_length,
                                               line.query_name, strand,
                                               chrom_bam, fastq_data[line.query_name]))
        else:
            for line in data:
                try:
                    chrom_bam = bam.get_reference_name(line.reference_id)
                except:
                    continue
                if tag == False:
                    if (line.is_unmapped == False and line.has_tag("XS") == False and chrom_bam == chrom_all):
                        if line.is_reverse:
                            strand = '-'
                        else:
                            strand = '+'
                        if 'N' not in fastq_data[line.query_name]:
                            bam_filter.append((line.reference_start,
                                               line.query_name, strand,
                                               chrom_bam, fastq_data[line.query_name]))
                else:
                    if (line.is_unmapped == False and chrom_bam == chrom_all):
                        if line.is_reverse:
                            strand = '-'
                        else:
                            strand = '+'
                        if 'N' not in fastq_data[line.query_name]:
                            bam_filter.append((line.reference_start,
                                               line.query_name, strand,
                                               chrom_bam, fastq_data[line.query_name]))

        bam_data = chromosome_info(bam_filter, end)
        printing(bam_data, end)
    bam.close()

def fastq_reader(fastq_file):
    fastq_dt = {}
    update_fastq = fastq_dt.update
    for fastq_data in SeqIO.parse(fastq_file, "fastq"):
        update_fastq({str(fastq_data.id): str(fastq_data.seq)})
    return fastq_dt

def chromosome_counter(i, end, bam_data):
    # merge_checks contains reference chromosome, ref. start, ref. end and strand.
    if end == True:
        merge_checks = (bam_data[i][4], bam_data[i][0], bam_data[i][1], bam_data[i][5], bam_data[i][3])
        score.update([merge_checks])
        append_merge = merge_data.append
        if score[merge_checks] == 1:
            append_merge((bam_data[i][4], bam_data[i][0], bam_data[i][1], bam_data[i][2], bam_data[i][3]))
    else:
        merge_checks = (bam_data[i][3], bam_data[i][0], bam_data[i][4], bam_data[i][2])
        score.update([merge_checks])
        append_merge = merge_data.append
        if score[merge_checks] == 1:
            append_merge((bam_data[i][3], bam_data[i][0], bam_data[i][1], bam_data[i][2]))
    return merge_data

def chromosome_info(bam_data, end):
    chr_info = []
    if end == True:
        for i in range(len(bam_data)):
            rec_id = bam_data[i][2]
            if rec_id in fastq_data:
                chr_info = chromosome_counter(i, end, bam_data)
            else:
                print(rec_id + " ID not found in fastq file")
    else:
        for i in range(len(bam_data)):
            rec_id = bam_data[i][1]
            if rec_id in fastq_data:
                chr_info = chromosome_counter(i, end, bam_data)
            else:
                print(rec_id + " ID not found in fastq file")
    return chr_info

def printing(endResult, end):
    if endResult != [] :
        with open(args.output_file, "w") as f:
            if end == True:
                for entry in endResult:
                    wr = ("%s\t%s\t%s\t%s\t%s\t%s" % (entry[0], entry[1], entry[2], entry[3],
                                                      score[(entry[0], entry[1], entry[2], fastq_data[entry[3]],
                                                             entry[4])], entry[4]))
                    f.write(wr + '\n')
            elif end == False:
                for entry in endResult:
                    wr = ("%s\t%s\t%s\t%s\t%s" % (entry[0], entry[1], entry[2],
                                                      score[(entry[0], entry[1], fastq_data[entry[2]],
                                                             entry[3])], entry[3]))
                    f.write(wr + '\n')

tool_description = """
Merge PCR duplicates according to random barcode library.
Barcodes containing uncalled base 'N' are removed.
Input:
* bam file containing fastq read-id and other details
* fastq library of random barcodes
Output:
* bed6 file with random barcode in name field and number of PCR duplicates as
  score, sorted by fields chrom, start, stop, strand, name
Example usage:
python3 pysam_test.py example1.bam example2.fa
"""

epilog = """
Author: Fayyaz Hussain
Status: Testing
"""

# parse command line arguments
parser = argparse.ArgumentParser(prog="Chromosomes Information", description=tool_description,
                                 epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("bam_file", help="Path to bam file containing alignments.", metavar='BAM_File')
parser.add_argument("fastq_file", help="Path to fastq barcode library.", metavar='FASTQ_File')
parser.add_argument("-o", "--output_file", required=True, help="Write results to this file.",
                    metavar='Output_File')
parser.add_argument("-t", "--tag", action='store_true', help="If XS tag is to be excluded.")
parser.add_argument("-e", "--end", action='store_false', help="If sequence end needs to be considered.")
args = parser.parse_args()
file_check()
tag = args.tag
end = args.end
fastq_data = fastq_reader(args.fastq_file)
bam_reader(args.bam_file, end, tag)
<commit_msg>XS tag inclusion/exclusion<commit_after>import pysam
from collections import Counter
import argparse
from os.path import isfile
from Bio import SeqIO

bam_data = []
score = Counter()
endResult = []
merge_data = []
fastq_data = {}
bam_filter = []

def file_check():
    flag = True
    if not isfile(args.fastq_file):
        print("ERROR: Fastq file: '{}' not found.".format(args.fastq_file))
        flag = False
    if not isfile(args.bam_file):
        print("ERROR: bam file: '{}' not found.".format(args.bam_file))
        flag = False
    if flag == False:
        exit()

def get_bam_filter(end, line, chrom_bam):

    if end == True:
        if line.is_reverse:
            strand = '-'
        else:
            strand = '+'
        if 'N' not in fastq_data[line.query_name]:
            bam_filter.append((line.reference_start,
                               line.reference_start + line.reference_length,
                               line.query_name, strand,
                               chrom_bam, fastq_data[line.query_name]))

    else:
        if line.is_reverse:
            strand = '-'
        else:
            strand = '+'
        if 'N' not in fastq_data[line.query_name]:
            bam_filter.append((line.reference_start,
                               line.query_name, strand,
                               chrom_bam, fastq_data[line.query_name]))
    return bam_filter

def bam_reader(bam_file, end, tag):
    bam = pysam.AlignmentFile(bam_file, "rb")
    filtered_data = []
    chromosomes = bam.references
    for chrom_all in chromosomes:

        data = bam.fetch(multiple_iterators=True, until_eof=True)
        if end == True:
            for line in data:
                try:
                    chrom_bam = bam.get_reference_name(line.reference_id)
                except:
                    continue
                if tag == False:
                    if (line.is_unmapped == False and line.has_tag("XS") == False and chrom_bam == chrom_all):
                        filtered_data = get_bam_filter(end, line, chrom_bam)

                else:
                    if (line.is_unmapped == False and chrom_bam == chrom_all):
                        filtered_data = get_bam_filter(end, line, chrom_bam)

        else:
            for line in data:
                try:
                    chrom_bam = bam.get_reference_name(line.reference_id)
                except:
                    continue
                if tag == False:
                    if (line.is_unmapped == False and line.has_tag("XS") == False and chrom_bam == chrom_all):
                        filtered_data = get_bam_filter(end, line, chrom_bam)
                else:
                    if (line.is_unmapped == False and chrom_bam == chrom_all):
                        filtered_data = get_bam_filter(end, line, chrom_bam)

        bam_data = chromosome_info(filtered_data, end)
        printing(bam_data, end)
    bam.close()

def fastq_reader(fastq_file):
    fastq_dt = {}
    update_fastq = fastq_dt.update
    for fastq_data in SeqIO.parse(fastq_file, "fastq"):
        update_fastq({str(fastq_data.id): str(fastq_data.seq)})
    return fastq_dt

def chromosome_counter(i, end, bam_data):
    # merge_checks contains reference chromosome, ref. start, ref. end and strand.
    if end == True:
        merge_checks = (bam_data[i][4], bam_data[i][0], bam_data[i][1], bam_data[i][5], bam_data[i][3])
        score.update([merge_checks])
        append_merge = merge_data.append
        if score[merge_checks] == 1:
            append_merge((bam_data[i][4], bam_data[i][0], bam_data[i][1], bam_data[i][2], bam_data[i][3]))
    else:
        merge_checks = (bam_data[i][3], bam_data[i][0], bam_data[i][4], bam_data[i][2])
        score.update([merge_checks])
        append_merge = merge_data.append
        if score[merge_checks] == 1:
            append_merge((bam_data[i][3], bam_data[i][0], bam_data[i][1], bam_data[i][2]))
    return merge_data

def chromosome_info(bam_data, end):
    chr_info = []
    if end == True:
        for i in range(len(bam_data)):
            rec_id = bam_data[i][2]
            if rec_id in fastq_data:
                chr_info = chromosome_counter(i, end, bam_data)
            else:
                print(rec_id + " ID not found in fastq file")
    else:
        for i in range(len(bam_data)):
            rec_id = bam_data[i][1]
            if rec_id in fastq_data:
                chr_info = chromosome_counter(i, end, bam_data)
            else:
                print(rec_id + " ID not found in fastq file")
    return chr_info

def printing(endResult, end):
    if endResult != [] :
        with open(args.output_file, "w") as f:
            if end == True:
                for entry in endResult:
                    wr = ("%s\t%s\t%s\t%s\t%s\t%s" % (entry[0], entry[1], entry[2], entry[3],
                                                      score[(entry[0], entry[1], entry[2], fastq_data[entry[3]],
                                                             entry[4])], entry[4]))
                    f.write(wr + '\n')
            elif end == False:
                for entry in endResult:
                    wr = ("%s\t%s\t%s\t%s\t%s" % (entry[0], entry[1], entry[2],
                                                      score[(entry[0], entry[1], fastq_data[entry[2]],
                                                             entry[3])], entry[3]))
                    f.write(wr + '\n')

tool_description = """
Merge PCR duplicates according to random barcode library.
Barcodes containing uncalled base 'N' are removed.
Input:
* bam file containing fastq read-id and other details
* fastq library of random barcodes
Output:
* bed6 file with random barcode in name field and number of PCR duplicates as
  score, sorted by fields chrom, start, stop, strand, name
Example usage:
python3 pysam_test.py example1.bam example2.fa
"""

epilog = """
Author: Fayyaz Hussain
Status: Testing
"""

# parse command line arguments
parser = argparse.ArgumentParser(prog="Chromosomes Information", description=tool_description,
                                 epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("bam_file", help="Path to bam file containing alignments.", metavar='BAM_File')
parser.add_argument("fastq_file", help="Path to fastq barcode library.", metavar='FASTQ_File')
parser.add_argument("-o", "--output_file", required=True, help="Write results to this file.",
                    metavar='Output_File')
parser.add_argument("-t", "--tag", action='store_true', help="If XS tag is to be excluded.")
parser.add_argument("-e", "--end", action='store_false', help="If sequence end needs to be considered.")
args = parser.parse_args()
file_check()
tag = args.tag
end = args.end
fastq_data = fastq_reader(args.fastq_file)
bam_reader(args.bam_file, end, tag)

<|endoftext|>


<commit_before>
from .api import API, Identity

class ComputeAPI(API):
	baseURI = 'https://compute.tyo1.conoha.io/v2/'

class VMPlan(ComputeAPI):
	planId = None
	name = None
	disk = None
	ram = None
	vcpus = None

	def __init__(self, data):
		self.planId = data['id']
		self.name = data['name']
		self.disk = data['disk']
		self.ram = data['ram']
		self.vcpus = data['vcpus']

class VMPlanList(ComputeAPI):
	_flavors = None

	def __init__(self, identity):
		path = identity.getTenantId() + '/flavors/detail'
		self.identity = identity
		res = self._GET(path)
		self._flavors = res['flavors']

	def __iter__(self):
		for f in self._flavors:
			yield VMPlan(f)

class VMImage(ComputeAPI):
	imageId = None
	name = None
	minDisk = None
	minRam = None
	progress = None
	status = None
	created = None
	updated = None

	def __init__(self, data):
		self.imageId = data['id']
		self.name = data['name']
		self.minDisk = data['minDisk']
		self.minRam = data['minRam']
		self.progress = data['progress']
		self.status = data['status']
		self.created = data['created']
		self.updated = data['updated']

class VMImageList(ComputeAPI):
	_images = None

	def __init__(self, identity):
		path = identity.getTenantId() + '/images/detail'
		self.identity = identity
		res = self._GET(path)
		self._images = res['images']

	def __iter__(self):
		for i in self._images:
			yield VMImage(i)

class VMList(ComputeAPI):
	_servers = None

	def __init__(self, identity):
		self.baseURI += identity.getTenantId() + '/'
		self.identity = identity
		res = self._GET('servers/detail')
		self._servers = res['servers']

	def __iter__(self):
		for v in self._servers:
			yield VM(self.identity, v)

	def getServer(self, vmid):
		res = self._GET('servers/'+vmid)
		return VM(self.identity, res['server'])

	def add(self, image, flavor):
		data = {'server' : {
				'imageRef' : image,
				'flavorRef' : flavor,
				'adminPass' : '(Rasdfjklweiojfdsakl'
				}}
		res = self._POST('servers', data)
		return res['server']['id']

	def delete(self, vmid):
		self._DELETE('servers/'+vmid, isDeserialize=False)

class VM(ComputeAPI):
	vmid = None
	flavorId = None
	hostId = None
	imageId = None
	tenantId = None
	name = None
	status = None
	created = None
	updated = None
	addressList = None
	securityGroupList = None

	def __init__(self, identity, info):
		self.vmid = info['id']
		self.flavorId = info['flavor']['id']
		self.hostId = info['hostId']
		self.imageId = info['image']['id']
		self.tenantId = info['tenant_id']
		self.name = info['name']
		self.status = info['status']
		self.created = info['created']
		self.updated = info['updated']
		self.status = info['status']
		self.created = info['created']
		self.updated = info['updated']
		self.addressList = info['addresses']
		self.securityGroupList = info['security_groups']

		self.identity = identity
		self.baseURI += self.tenantId + '/servers/' + self.vmid + '/'

	def _action(self, actionName, actionValue=None):
		action = {actionName: actionValue}
		self._POST('action', action, isDeserialize=False)
	def run(self):
		self._action('os-start')
	def stop(self, force=False):
		if force:
			self._action('os-stop', {'force_shutdown': True})
		else:
			self._action('os-stop')
	def restart(self):
		self._action('reboot', {'type': 'SOFT'})
	def resize(self, flavorId):
		self._action('resize', {'flavorRef': flavorId})
	def confirmResize(self):
		self._action('confirmResize')
	def revertResize(self):
		self._action('revertResize')
	def getStatus(self):
		res = self._GET('')
		return res['server']['status']

class KeyList(ComputeAPI):
	keys = None

	def __init__(self, identity):
		self.baseURI += identity.getTenantId() + '/'
		self.identity = identity
		res = self._GET('os-keypairs')
		self.keys = (keypair['keypair'] for keypair in res['keypairs'])

	def __iter__(self):
		for key in self.keys:
			yield Key(key)

	def add(self, name, publicKey=None, publicKeyFile=None):
		"""
		publicKey が文字列なら、そのキーを使用
		publicKeyFile が文字列なら、そのキーを使用
		publicKey or publicKeyFile がfile likeオブジェクトなら、そのキーを使用
		publicKey is None ならば、新しいキーを作成。必ず返されるKey objectからprivateKeyを取得し、保存すること
		"""
		data = {'keypair':{'name': name}}
		keyString = None

		if type(publicKey) is str:
			keyString = publicKey
		elif type(publicKeyFile) is str:
			with open(publicKeyFile) as f:
				keyString = f.read()
		elif (publicKey or publicKeyFile) is not None:
			keyString = (publicKey or publicKeyFile).read()
		else:
			res = self._POST('os-keypairs', data=data)
			return Key(res['keypair'])

		data['keypair']['public_key'] = keyString
		res = self._POST('os-keypairs', data=data)
		return Key(res['keypair'])

	def delete(self, keyName):
		self._DELETE('os-keypairs/'+keyName, isDeserialize=False)

class Key(ComputeAPI):
	name = None
	fingerprint = None
	publicKey = None
	privateKey = None

	def __init__(self, info):
		self.name = info['name']
		self.fingerprint = info['fingerprint']
		self.publicKey = info['public_key']
		self.privateKey = info.get('private_key')

<commit_msg>KeyListのkeyフィールドをプライベートにした<commit_after>
from .api import API, Identity

class ComputeAPI(API):
	baseURI = 'https://compute.tyo1.conoha.io/v2/'

class VMPlan(ComputeAPI):
	planId = None
	name = None
	disk = None
	ram = None
	vcpus = None

	def __init__(self, data):
		self.planId = data['id']
		self.name = data['name']
		self.disk = data['disk']
		self.ram = data['ram']
		self.vcpus = data['vcpus']

class VMPlanList(ComputeAPI):
	_flavors = None

	def __init__(self, identity):
		path = identity.getTenantId() + '/flavors/detail'
		self.identity = identity
		res = self._GET(path)
		self._flavors = res['flavors']

	def __iter__(self):
		for f in self._flavors:
			yield VMPlan(f)

class VMImage(ComputeAPI):
	imageId = None
	name = None
	minDisk = None
	minRam = None
	progress = None
	status = None
	created = None
	updated = None

	def __init__(self, data):
		self.imageId = data['id']
		self.name = data['name']
		self.minDisk = data['minDisk']
		self.minRam = data['minRam']
		self.progress = data['progress']
		self.status = data['status']
		self.created = data['created']
		self.updated = data['updated']

class VMImageList(ComputeAPI):
	_images = None

	def __init__(self, identity):
		path = identity.getTenantId() + '/images/detail'
		self.identity = identity
		res = self._GET(path)
		self._images = res['images']

	def __iter__(self):
		for i in self._images:
			yield VMImage(i)

class VMList(ComputeAPI):
	_servers = None

	def __init__(self, identity):
		self.baseURI += identity.getTenantId() + '/'
		self.identity = identity
		res = self._GET('servers/detail')
		self._servers = res['servers']

	def __iter__(self):
		for v in self._servers:
			yield VM(self.identity, v)

	def getServer(self, vmid):
		res = self._GET('servers/'+vmid)
		return VM(self.identity, res['server'])

	def add(self, image, flavor):
		data = {'server' : {
				'imageRef' : image,
				'flavorRef' : flavor,
				'adminPass' : '(Rasdfjklweiojfdsakl'
				}}
		res = self._POST('servers', data)
		return res['server']['id']

	def delete(self, vmid):
		self._DELETE('servers/'+vmid, isDeserialize=False)

class VM(ComputeAPI):
	vmid = None
	flavorId = None
	hostId = None
	imageId = None
	tenantId = None
	name = None
	status = None
	created = None
	updated = None
	addressList = None
	securityGroupList = None

	def __init__(self, identity, info):
		self.vmid = info['id']
		self.flavorId = info['flavor']['id']
		self.hostId = info['hostId']
		self.imageId = info['image']['id']
		self.tenantId = info['tenant_id']
		self.name = info['name']
		self.status = info['status']
		self.created = info['created']
		self.updated = info['updated']
		self.status = info['status']
		self.created = info['created']
		self.updated = info['updated']
		self.addressList = info['addresses']
		self.securityGroupList = info['security_groups']

		self.identity = identity
		self.baseURI += self.tenantId + '/servers/' + self.vmid + '/'

	def _action(self, actionName, actionValue=None):
		action = {actionName: actionValue}
		self._POST('action', action, isDeserialize=False)
	def run(self):
		self._action('os-start')
	def stop(self, force=False):
		if force:
			self._action('os-stop', {'force_shutdown': True})
		else:
			self._action('os-stop')
	def restart(self):
		self._action('reboot', {'type': 'SOFT'})
	def resize(self, flavorId):
		self._action('resize', {'flavorRef': flavorId})
	def confirmResize(self):
		self._action('confirmResize')
	def revertResize(self):
		self._action('revertResize')
	def getStatus(self):
		res = self._GET('')
		return res['server']['status']

class KeyList(ComputeAPI):
	_keys = None

	def __init__(self, identity):
		self.baseURI += identity.getTenantId() + '/'
		self.identity = identity
		res = self._GET('os-keypairs')
		self._keys = (keypair['keypair'] for keypair in res['keypairs'])

	def __iter__(self):
		for key in self._keys:
			yield Key(key)

	def add(self, name, publicKey=None, publicKeyFile=None):
		"""
		publicKey が文字列なら、そのキーを使用
		publicKeyFile が文字列なら、そのキーを使用
		publicKey or publicKeyFile がfile likeオブジェクトなら、そのキーを使用
		publicKey is None ならば、新しいキーを作成。必ず返されるKey objectからprivateKeyを取得し、保存すること
		"""
		data = {'keypair':{'name': name}}
		keyString = None

		if type(publicKey) is str:
			keyString = publicKey
		elif type(publicKeyFile) is str:
			with open(publicKeyFile) as f:
				keyString = f.read()
		elif (publicKey or publicKeyFile) is not None:
			keyString = (publicKey or publicKeyFile).read()
		else:
			res = self._POST('os-keypairs', data=data)
			return Key(res['keypair'])

		data['keypair']['public_key'] = keyString
		res = self._POST('os-keypairs', data=data)
		return Key(res['keypair'])

	def delete(self, keyName):
		self._DELETE('os-keypairs/'+keyName, isDeserialize=False)

class Key(ComputeAPI):
	name = None
	fingerprint = None
	publicKey = None
	privateKey = None

	def __init__(self, info):
		self.name = info['name']
		self.fingerprint = info['fingerprint']
		self.publicKey = info['public_key']
		self.privateKey = info.get('private_key')

<|endoftext|>


<commit_before>import psycopg2
import psycopg2.extras
from ..models.user import User
from twatter import config

def get_connection():
    return psycopg2.connect(config.db['db_string'])

#GET STUFF

def get_one_with_id(table, id):
    assert table
    assert id
    query = "SELECT * FROM {} WHERE id = %s".format(table)
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (id,))
    result = cur.fetchone()
    cur.close()
    conn.close()
    return result

def get_fields_from_table_with_id(fields, table, id_name, id):
    assert fields
    assert table
    assert id_name
    assert id
    query = '''select %s from {} where {} = %s'''.format(table, id_name)
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (fields, id))
    results = cur.fetchall()
    cur.close()
    conn.close()
    return results

def get_all_from_table(table):
    assert table
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute("SELECT * FROM %s", (table,))
    result = cur.fetchone()
    cur.close()
    conn.close()
    return results

def get_custom_query(query, vars=None):
    assert query
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, vars)
    results = cur.fetchall()
    cur.close()
    conn.close()
    return results

def get_favorited_twaats_for_user(id):
    assert id
    query = """
    SELECT * FROM favorited_twaats WHERE who_id = %s
    """
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (id,))
    twaats = [dict(record) for record in cur.fetchall()] # it calls .fecthone() in loop
    cur.close()
    conn.close()
    return twaats

def get_search_results(type, term):
    term = str(term.lower())
    query = '';
    if type.lower() in 'users':
        query = """SELECT * FROM users WHERE lower(full_name) like %s """
    if type.lower() == 'tags':
        return ['not implemented yet']

    try:
        conn = psycopg2.connect(config.db['db_string'])
        cur = conn.cursor()
        cur.execute(query, (['%'+term+'%']))
        search_results = cur.fetchall()
        #some problems with psycon, doesnt accept loop here..
        cur.close()
        conn.close()
        return [User(x[0], x[1], x[2], x[3], x[4], x[5], x[6]) for x in search_results]
    except Exception as e:
        print e
        return e

# INSERT stuff

def insert_new_favorite_twaat_for_id(twaat_id, user_id):
    try:
        conn = psycopg2.connect(config.db['db_string'])
        cur = conn.cursor()
        cur.execute("""
        INSERT INTO favorited_twaats (who_id, twaat_id) VALUES (%s, %s)
        """, (user_id, twaat_id))
        conn.commit()
        cur.close()
        conn.close()
    except Exception as e:
        if('duplicate' in e.pgerror):
            return 'Twaat already favorited'
        print e
        return e

def insert_new_twaat(user_id, text, parent_id):
    try:
        conn = get_connection()
        cur = conn.cursor()
        cur.execute("""
        INSERT INTO twaat (user_id, text, parent_id) VALUES (%s, %s, %s)
        """, (user_id, text, parent_id))
        conn.commit()
        cur.close()
        conn.close()
        # session['username'] = request.form['username']
    except Exception as e:
        print e
        return e

# UPDATE STUFF

def update_custom_query(query, vars=None):
    try:
        conn = get_connection()
        cur = conn.cursor()
        cur.execute(query, vars)
        conn.commit()
        cur.close()
        conn.close()
    except Exception as e:
        if('duplicate' in e.pgerror):
            return 'Twaat already favorited'
        print e
        return e
<commit_msg>added update_user_avatar<commit_after>import psycopg2
import psycopg2.extras
from ..models.user import User
from twatter import config

def get_connection():
    return psycopg2.connect(config.db['db_string'])

#GET STUFF

def get_one_with_id(table, id):
    assert table
    assert id
    query = "SELECT * FROM {} WHERE id = %s".format(table)
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (id,))
    result = cur.fetchone()
    cur.close()
    conn.close()
    return result

def get_fields_from_table_with_id(fields, table, id_name, id):
    assert fields
    assert table
    assert id_name
    assert id
    query = '''select %s from {} where {} = %s'''.format(table, id_name)
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (fields, id))
    results = cur.fetchall()
    cur.close()
    conn.close()
    return results

def get_all_from_table(table):
    assert table
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute("SELECT * FROM %s", (table,))
    result = cur.fetchone()
    cur.close()
    conn.close()
    return results

def get_custom_query(query, vars=None):
    assert query
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, vars)
    results = cur.fetchall()
    cur.close()
    conn.close()
    return results

def get_favorited_twaats_for_user(id):
    assert id
    query = """
    SELECT * FROM favorited_twaats WHERE who_id = %s
    """
    conn = psycopg2.connect(config.db['db_string'])
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, (id,))
    twaats = [dict(record) for record in cur.fetchall()] # it calls .fecthone() in loop
    cur.close()
    conn.close()
    return twaats

def get_search_results(type, term):
    term = str(term.lower())
    query = '';
    if type.lower() in 'users':
        query = """SELECT * FROM users WHERE lower(full_name) like %s """
    if type.lower() == 'tags':
        return ['not implemented yet']

    try:
        conn = psycopg2.connect(config.db['db_string'])
        cur = conn.cursor()
        cur.execute(query, (['%'+term+'%']))
        search_results = cur.fetchall()
        #some problems with psycon, doesnt accept loop here..
        cur.close()
        conn.close()
        return [User(x[0], x[1], x[2], x[3], x[4], x[5], x[6]) for x in search_results]
    except Exception as e:
        print e
        return e

# INSERT stuff

def insert_new_favorite_twaat_for_id(twaat_id, user_id):
    try:
        conn = psycopg2.connect(config.db['db_string'])
        cur = conn.cursor()
        cur.execute("""
        INSERT INTO favorited_twaats (who_id, twaat_id) VALUES (%s, %s)
        """, (user_id, twaat_id))
        conn.commit()
        cur.close()
        conn.close()
    except Exception as e:
        if('duplicate' in e.pgerror):
            return 'Twaat already favorited'
        print e
        return e

def insert_new_twaat(user_id, text, parent_id):
    try:
        conn = get_connection()
        cur = conn.cursor()
        cur.execute("""
        INSERT INTO twaat (user_id, text, parent_id) VALUES (%s, %s, %s)
        """, (user_id, text, parent_id))
        conn.commit()
        cur.close()
        conn.close()
        # session['username'] = request.form['username']
    except Exception as e:
        print e
        return e

# UPDATE STUFF

def update_custom_query(query, vars=None):
    try:
        conn = get_connection()
        cur = conn.cursor()
        cur.execute(query, vars)
        conn.commit()
        cur.close()
        conn.close()
    except Exception as e:
        if('duplicate' in e.pgerror):
            return 'Twaat already favorited'
        print e
        return e

def update_user_avatar(id, path):
    try:
        conn = get_connection()
        cur = conn.cursor()
        cur.execute("""UPDATE users SET avatar = %s  WHERE id = %s""", (path, id))
        conn.commit()
        cur.close()
        conn.close()
    except Exception as e:
        if('duplicate' in e.pgerror):
            return 'Twaat already favorited'
        print e
        return e<|endoftext|>


<commit_before><commit_msg>UploadArquivoForm tinha sido movida da app<commit_after><|endoftext|>


<commit_before>import unittest

import six

import numpy as np

import chainer
from chainer import optimizers
from chainer import testing

@testing.parameterize(*testing.product({
    'impl': [
        optimizers.AdaDelta,
        optimizers.AdaGrad,
        optimizers.Adam,
        optimizers.CorrectedMomentumSGD,
        optimizers.MomentumSGD,
        optimizers.MSVAG,
        optimizers.NesterovAG,
        optimizers.RMSprop,
        optimizers.RMSpropGraves,
        optimizers.SGD,
        optimizers.SMORMS3,
    ]
}))
class TestOptimizerHyperparameter(unittest.TestCase):

    def setUp(self):
        self.target = chainer.Link()
        with self.target.init_scope():
            self.target.w = chainer.Parameter()

    def create(self, *args, **kwargs):
        self.optimizer = self.impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def get_hyperparam(self, name):
        return getattr(self.target.w.update_rule.hyperparam, name)

    def test_hyperparams(self):
        self.create()
        default = self.optimizer.hyperparam.get_dict()
        for name, default_value in six.iteritems(default):
            self.create()
            self.assertEqual(self.get_hyperparam(name), default_value)
            new_value = default_value + 0.1
            self.create(**{name: new_value})
            self.assertEqual(self.get_hyperparam(name), new_value)

class WeightSaveHook(object):
    name = 'WeightSaveHook'
    call_for_each_param = True

    def __init__(self):
        self.value = None

    def __call__(self, rule, param):
        p, g = param.data, param.grad
        if p is None or g is None:
            return
        self.value = np.copy(p)

class SimpleChain(chainer.Chain):

    def __init__(self):
        super(SimpleChain, self).__init__()
        with self.init_scope():
            self.w = chainer.Parameter(42, (), 'w')

    def __call__(self, x):
        return (x - self.w) ** 2

@testing.parameterize(*testing.product({
    'impl': [
        optimizers.AdaDelta,
        optimizers.AdaGrad,
        optimizers.Adam,
        optimizers.CorrectedMomentumSGD,
        optimizers.MomentumSGD,
        optimizers.MSVAG,
        optimizers.NesterovAG,
        optimizers.RMSprop,
        optimizers.RMSpropGraves,
        optimizers.SGD,
        optimizers.SMORMS3,
    ]
}))
class TestOptimizerHooks(unittest.TestCase):

    def setUp(self):
        self.target = SimpleChain()

    def create(self, *args, **kwargs):
        self.optimizer = self.impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def get_hyperparam(self, name):
        return getattr(self.target.w.update_rule.hyperparam, name)

    def test_hooks(self):
        w_pre = np.copy(self.target.w.data)
        h_pre = WeightSaveHook()
        h_post = WeightSaveHook()
        self.create()
        self.optimizer.add_hook(h_pre, timing='pre')
        self.optimizer.add_hook(h_post, name='WeightSaveHookPost',
                                timing='post')

        x = chainer.Variable(np.array(5., dtype=np.float32))
        self.optimizer.update(self.target, x)
        w_post = np.copy(self.target.w.data)

        self.assertEqual(w_pre, h_pre.value)
        self.assertEqual(w_post, h_post.value)
        self.assertNotEqual(h_pre.value, h_post.value)

    def test_hooks_auto(self):
        w_pre = np.copy(self.target.w.data)
        h_pre = WeightSaveHook()
        h_pre.timing = 'pre'
        h_post = WeightSaveHook()
        h_post.timing = 'post'
        self.create()
        self.optimizer.add_hook(h_pre, timing='auto')
        self.optimizer.add_hook(h_post, name='WeightSaveHookPost',
                                timing='auto')

        x = chainer.Variable(np.array(5., dtype=np.float32))
        self.optimizer.update(self.target, x)
        w_post = np.copy(self.target.w.data)

        self.assertEqual(w_pre, h_pre.value)
        self.assertEqual(w_post, h_post.value)
        self.assertNotEqual(h_pre.value, h_post.value)

@testing.parameterize(*testing.product({
    'impl': [
        optimizers.AdaDelta,
        optimizers.AdaGrad,
        optimizers.Adam,
        optimizers.CorrectedMomentumSGD,
        optimizers.MomentumSGD,
        optimizers.MSVAG,
        optimizers.NesterovAG,
        optimizers.RMSprop,
        optimizers.RMSpropGraves,
        optimizers.SGD,
        optimizers.SMORMS3,
    ]
}))
class TestOptimizerLossScaling(unittest.TestCase):

    def setUp(self):
        self.target = SimpleChain()

    def create(self, *args, **kwargs):
        self.optimizer = self.impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def test_invalid_configs(self):
        self.create()
        with self.assertRaises(ValueError):
            self.optimizer.loss_scaling(interval=0)
        with self.assertRaises(ValueError):
            self.optimizer.loss_scaling(scale=-1)

testing.run_module(__name__, __file__)
<commit_msg>DRY optimizer test parameterizations<commit_after>import unittest

import six

import numpy as np

import chainer
from chainer import optimizers
from chainer import testing

_parameterize_optimizers = testing.parameterize(*testing.product({
    'optimizer_impl': [
        optimizers.AdaDelta,
        optimizers.AdaGrad,
        optimizers.Adam,
        optimizers.CorrectedMomentumSGD,
        optimizers.MomentumSGD,
        optimizers.MSVAG,
        optimizers.NesterovAG,
        optimizers.RMSprop,
        optimizers.RMSpropGraves,
        optimizers.SGD,
        optimizers.SMORMS3,
    ]
}))

@_parameterize_optimizers
class TestOptimizerHyperparameter(unittest.TestCase):

    def setUp(self):
        self.target = chainer.Link()
        with self.target.init_scope():
            self.target.w = chainer.Parameter()

    def create(self, *args, **kwargs):
        self.optimizer = self.optimizer_impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def get_hyperparam(self, name):
        return getattr(self.target.w.update_rule.hyperparam, name)

    def test_hyperparams(self):
        self.create()
        default = self.optimizer.hyperparam.get_dict()
        for name, default_value in six.iteritems(default):
            self.create()
            self.assertEqual(self.get_hyperparam(name), default_value)
            new_value = default_value + 0.1
            self.create(**{name: new_value})
            self.assertEqual(self.get_hyperparam(name), new_value)

class WeightSaveHook(object):
    name = 'WeightSaveHook'
    call_for_each_param = True

    def __init__(self):
        self.value = None

    def __call__(self, rule, param):
        p, g = param.data, param.grad
        if p is None or g is None:
            return
        self.value = np.copy(p)

class SimpleChain(chainer.Chain):

    def __init__(self):
        super(SimpleChain, self).__init__()
        with self.init_scope():
            self.w = chainer.Parameter(42, (), 'w')

    def __call__(self, x):
        return (x - self.w) ** 2

@_parameterize_optimizers
class TestOptimizerHooks(unittest.TestCase):

    def setUp(self):
        self.target = SimpleChain()

    def create(self, *args, **kwargs):
        self.optimizer = self.optimizer_impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def get_hyperparam(self, name):
        return getattr(self.target.w.update_rule.hyperparam, name)

    def test_hooks(self):
        w_pre = np.copy(self.target.w.data)
        h_pre = WeightSaveHook()
        h_post = WeightSaveHook()
        self.create()
        self.optimizer.add_hook(h_pre, timing='pre')
        self.optimizer.add_hook(h_post, name='WeightSaveHookPost',
                                timing='post')

        x = chainer.Variable(np.array(5., dtype=np.float32))
        self.optimizer.update(self.target, x)
        w_post = np.copy(self.target.w.data)

        self.assertEqual(w_pre, h_pre.value)
        self.assertEqual(w_post, h_post.value)
        self.assertNotEqual(h_pre.value, h_post.value)

    def test_hooks_auto(self):
        w_pre = np.copy(self.target.w.data)
        h_pre = WeightSaveHook()
        h_pre.timing = 'pre'
        h_post = WeightSaveHook()
        h_post.timing = 'post'
        self.create()
        self.optimizer.add_hook(h_pre, timing='auto')
        self.optimizer.add_hook(h_post, name='WeightSaveHookPost',
                                timing='auto')

        x = chainer.Variable(np.array(5., dtype=np.float32))
        self.optimizer.update(self.target, x)
        w_post = np.copy(self.target.w.data)

        self.assertEqual(w_pre, h_pre.value)
        self.assertEqual(w_post, h_post.value)
        self.assertNotEqual(h_pre.value, h_post.value)

@_parameterize_optimizers
class TestOptimizerLossScaling(unittest.TestCase):

    def setUp(self):
        self.target = SimpleChain()

    def create(self, *args, **kwargs):
        self.optimizer = self.optimizer_impl(*args, **kwargs)
        self.optimizer.setup(self.target)

    def test_invalid_configs(self):
        self.create()
        with self.assertRaises(ValueError):
            self.optimizer.loss_scaling(interval=0)
        with self.assertRaises(ValueError):
            self.optimizer.loss_scaling(scale=-1)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before># Copyright 2015 Planet Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

'''manage a queue of datalake files

This allows users to enqueue files to be uploaded to the datalake. An uploader
process runs that actually does the uploader work.

Under the hood, the queue is a directory which the Uploader watches. The
Enqueuer enqueues files by tarring them up with their metadata and writing them
to the queue directory. This ensures that the enqueuer fails in the user's face
instead of silently behind the user's back if the metadata is not right. The
uploader uses inotify to monitor the queue directory. When a file arrives, it
gets uploaded. On success, it gets deleted. If the upload fails for some
reason, the file remains in the queue.
'''
from os import environ
import os
from datalake_common.errors import InsufficientConfiguration
from logging import getLogger
import time
from threading import Thread
from six.moves._thread import interrupt_main
from six.moves.queue import Queue

from datalake import File, InvalidDatalakeBundle

'''whether or not queue feature is available

Users may wish to check if s3 features are available before invoking them. If
they are unavailable, the affected functions will raise
InsufficientConfiguration.'''
has_queue = True
try:
    import pyinotify
except ImportError:
    has_queue = False

    class FakePyinotify(object):

        class ProcessEvent(object):
            pass

    pyinotify = FakePyinotify

def requires_queue(f):
    def wrapped(*args, **kwargs):
        if not has_queue:
            msg = 'This feature requires the queuable deps.  '
            msg += '`pip install datalake[queuable]` to turn this feature on.'
            raise InsufficientConfiguration(msg)
        return f(*args, **kwargs)
    return wrapped

log = getLogger('datalake-queue')

class DatalakeQueueBase(object):

    @requires_queue
    def __init__(self, queue_dir=None):
        self.queue_dir = queue_dir or environ.get('DATALAKE_QUEUE_DIR')
        self._validate_queue_dir()

    def _validate_queue_dir(self):
        if self.queue_dir is None:
            raise InsufficientConfiguration('Please set DATALAKE_QUEUE_DIR')
        self.queue_dir = os.path.abspath(self.queue_dir)

class Enqueuer(DatalakeQueueBase):

    def enqueue(self, filename, compress=False, **metadata_fields):
        '''enqueue a file with the specified metadata to be pushed

        Args:
            filename: the file to enqueue

            compress: whether or not to compress the file before enqueueing

        Returns the File with complete metadata that will be pushed.

        '''
        log.info('Enqueing ' + filename)
        if compress:
            f = File.from_filename_compressed(filename, **metadata_fields)
        else:
            f = File.from_filename(filename, **metadata_fields)
        fname = f.metadata['id'] + '.tar'
        dest = os.path.join(self.queue_dir, fname)
        f.to_bundle(dest)
        return f

class Uploader(DatalakeQueueBase):

    def __init__(self, archive, queue_dir, callback=None):
        '''create an uploader that listens to queue_dir and pushes to archive

        The callback (if any) gets called with the filename after each
        successful upload. Note that it may be called from a thread. So be safe
        out there.
        '''
        super(Uploader, self).__init__(queue_dir)
        self._archive = archive
        self._callback = callback

    class EventHandler(pyinotify.ProcessEvent):

        def __init__(self, callback):
            super(Uploader.EventHandler, self).__init__()
            self.callback = callback

        def process_IN_CLOSE_WRITE(self, event):
            self.callback(event.pathname)

        def process_IN_MOVED_TO(self, event):
            self.callback(event.pathname)

    def _setup_watch_manager(self, timeout):
        if timeout is not None:
            timeout = int(timeout * 1000)
        self._wm = pyinotify.WatchManager()
        self._handler = Uploader.EventHandler(self._push)
        self._notifier = pyinotify.Notifier(self._wm, self._handler,
                                            timeout=timeout)
        self._wm.add_watch(self.queue_dir,
                           pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO)

    def _push(self, filename):
        if os.path.basename(filename).startswith('.'):
            return
        if self._workers == []:
            self._synchronous_push(filename)
        else:
            self._threaded_push(filename)

    def _synchronous_push(self, filename):
        try:
            f = File.from_bundle(filename)
        except InvalidDatalakeBundle as e:
            msg = '{}. Skipping upload.'.format(e.args[0])
            log.exception(msg)
            return
        url = self._archive.push(f)
        msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url)
        log.info(msg)
        os.unlink(filename)
        if self._callback is not None:
            self._callback(filename)

    def _threaded_push(self, filename):
        self._queue.put(filename)

    def _threaded_worker(self, worker_number):
        log.info('upload worker {} starting.'.format(worker_number))
        try:
            while True:
                filename = self._queue.get(block=True)
                msg = 'upload worker {} handling {}'
                msg = msg.format(worker_number, filename)
                log.info(msg)
                self._synchronous_push(filename)
                self._queue.task_done()
        except Exception as e:
            log.exception(e)
            # when a worker fails, we fail the entire process.
            interrupt_main()

    def listen(self, timeout=None, workers=1):
        try:
            self._listen(timeout=timeout, workers=workers)
        except Exception as e:
            log.exception(e)
            raise

    def _listen(self, timeout=None, workers=1):
        '''listen for files in the queue directory and push them'''
        from . import __version__

        log.info('------------------------------')
        log.info('datalake ' + __version__)

        self._workers = []
        if workers <= 0:
            msg = 'number of upload workers cannot be zero or negative'
            raise InsufficientConfiguration(msg)
        if workers > 1:
            # when multipe workers are requested, the main thread monitors the
            # queue directory and puts the files in a Queue that is serviced by
            # the worker threads. So the word queue is a bit overloaded in this
            # module.
            self._queue = Queue()
            self._workers = [self._create_worker(i) for i in range(workers)]

        for f in os.listdir(self.queue_dir):
            path = os.path.join(self.queue_dir, f)
            self._push(path)

        self._run(timeout)

    def _create_worker(self, worker_number):
        w = Thread(target=self._threaded_worker, args=(worker_number,))
        w.setDaemon(True)
        w.start()
        return w

    INFINITY = None

    def _run(self, timeout):

        self._prepare_to_track_run_time(timeout)
        self._notifier.process_events()
        while self._notifier.check_events():
            self._notifier.read_events()
            self._notifier.process_events()
            if self._update_time_remaining() == 0:
                break

    def _update_time_remaining(self):
        if self._run_time_remaining is self.INFINITY:
            return self.INFINITY
        now = time.time()
        duration = now - self._run_start
        self._run_time_remaining -= duration
        self._run_time_remaining = max(self._run_time_remaining, 0)
        self._run_start = now
        return self._run_time_remaining

    def _prepare_to_track_run_time(self, timeout):
        self._setup_watch_manager(timeout)
        self._run_start = time.time()
        self._run_time_remaining = timeout
<commit_msg>fallback to uncompressed uploads when compression fails<commit_after># Copyright 2015 Planet Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

'''manage a queue of datalake files

This allows users to enqueue files to be uploaded to the datalake. An uploader
process runs that actually does the uploader work.

Under the hood, the queue is a directory which the Uploader watches. The
Enqueuer enqueues files by tarring them up with their metadata and writing them
to the queue directory. This ensures that the enqueuer fails in the user's face
instead of silently behind the user's back if the metadata is not right. The
uploader uses inotify to monitor the queue directory. When a file arrives, it
gets uploaded. On success, it gets deleted. If the upload fails for some
reason, the file remains in the queue.
'''
from os import environ
import os
from datalake_common.errors import InsufficientConfiguration
from logging import getLogger
import time
from threading import Thread
from six.moves._thread import interrupt_main
from six.moves.queue import Queue

from datalake import File, InvalidDatalakeBundle

'''whether or not queue feature is available

Users may wish to check if s3 features are available before invoking them. If
they are unavailable, the affected functions will raise
InsufficientConfiguration.'''
has_queue = True
try:
    import pyinotify
except ImportError:
    has_queue = False

    class FakePyinotify(object):

        class ProcessEvent(object):
            pass

    pyinotify = FakePyinotify

def requires_queue(f):
    def wrapped(*args, **kwargs):
        if not has_queue:
            msg = 'This feature requires the queuable deps.  '
            msg += '`pip install datalake[queuable]` to turn this feature on.'
            raise InsufficientConfiguration(msg)
        return f(*args, **kwargs)
    return wrapped

log = getLogger('datalake-queue')

class DatalakeQueueBase(object):

    @requires_queue
    def __init__(self, queue_dir=None):
        self.queue_dir = queue_dir or environ.get('DATALAKE_QUEUE_DIR')
        self._validate_queue_dir()

    def _validate_queue_dir(self):
        if self.queue_dir is None:
            raise InsufficientConfiguration('Please set DATALAKE_QUEUE_DIR')
        self.queue_dir = os.path.abspath(self.queue_dir)

class Enqueuer(DatalakeQueueBase):

    def enqueue(self, filename, compress=False, **metadata_fields):
        '''enqueue a file with the specified metadata to be pushed

        Args:
            filename: the file to enqueue

            compress: whether or not to compress the file before enqueueing

        Returns the File with complete metadata that will be pushed.

        '''
        log.info('Enqueing ' + filename)
        if compress:
            try:
                f = File.from_filename_compressed(filename, **metadata_fields)
            except OverflowError:
                f = File.from_filename(filename, **metadata_fields)
        else:
            f = File.from_filename(filename, **metadata_fields)
        fname = f.metadata['id'] + '.tar'
        dest = os.path.join(self.queue_dir, fname)
        f.to_bundle(dest)
        return f

class Uploader(DatalakeQueueBase):

    def __init__(self, archive, queue_dir, callback=None):
        '''create an uploader that listens to queue_dir and pushes to archive

        The callback (if any) gets called with the filename after each
        successful upload. Note that it may be called from a thread. So be safe
        out there.
        '''
        super(Uploader, self).__init__(queue_dir)
        self._archive = archive
        self._callback = callback

    class EventHandler(pyinotify.ProcessEvent):

        def __init__(self, callback):
            super(Uploader.EventHandler, self).__init__()
            self.callback = callback

        def process_IN_CLOSE_WRITE(self, event):
            self.callback(event.pathname)

        def process_IN_MOVED_TO(self, event):
            self.callback(event.pathname)

    def _setup_watch_manager(self, timeout):
        if timeout is not None:
            timeout = int(timeout * 1000)
        self._wm = pyinotify.WatchManager()
        self._handler = Uploader.EventHandler(self._push)
        self._notifier = pyinotify.Notifier(self._wm, self._handler,
                                            timeout=timeout)
        self._wm.add_watch(self.queue_dir,
                           pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO)

    def _push(self, filename):
        if os.path.basename(filename).startswith('.'):
            return
        if self._workers == []:
            self._synchronous_push(filename)
        else:
            self._threaded_push(filename)

    def _synchronous_push(self, filename):
        try:
            f = File.from_bundle(filename)
        except InvalidDatalakeBundle as e:
            msg = '{}. Skipping upload.'.format(e.args[0])
            log.exception(msg)
            return
        url = self._archive.push(f)
        msg = 'Pushed {}({}) to {}'.format(filename, f.metadata['path'], url)
        log.info(msg)
        os.unlink(filename)
        if self._callback is not None:
            self._callback(filename)

    def _threaded_push(self, filename):
        self._queue.put(filename)

    def _threaded_worker(self, worker_number):
        log.info('upload worker {} starting.'.format(worker_number))
        try:
            while True:
                filename = self._queue.get(block=True)
                msg = 'upload worker {} handling {}'
                msg = msg.format(worker_number, filename)
                log.info(msg)
                self._synchronous_push(filename)
                self._queue.task_done()
        except Exception as e:
            log.exception(e)
            # when a worker fails, we fail the entire process.
            interrupt_main()

    def listen(self, timeout=None, workers=1):
        try:
            self._listen(timeout=timeout, workers=workers)
        except Exception as e:
            log.exception(e)
            raise

    def _listen(self, timeout=None, workers=1):
        '''listen for files in the queue directory and push them'''
        from . import __version__

        log.info('------------------------------')
        log.info('datalake ' + __version__)

        self._workers = []
        if workers <= 0:
            msg = 'number of upload workers cannot be zero or negative'
            raise InsufficientConfiguration(msg)
        if workers > 1:
            # when multipe workers are requested, the main thread monitors the
            # queue directory and puts the files in a Queue that is serviced by
            # the worker threads. So the word queue is a bit overloaded in this
            # module.
            self._queue = Queue()
            self._workers = [self._create_worker(i) for i in range(workers)]

        for f in os.listdir(self.queue_dir):
            path = os.path.join(self.queue_dir, f)
            self._push(path)

        self._run(timeout)

    def _create_worker(self, worker_number):
        w = Thread(target=self._threaded_worker, args=(worker_number,))
        w.setDaemon(True)
        w.start()
        return w

    INFINITY = None

    def _run(self, timeout):

        self._prepare_to_track_run_time(timeout)
        self._notifier.process_events()
        while self._notifier.check_events():
            self._notifier.read_events()
            self._notifier.process_events()
            if self._update_time_remaining() == 0:
                break

    def _update_time_remaining(self):
        if self._run_time_remaining is self.INFINITY:
            return self.INFINITY
        now = time.time()
        duration = now - self._run_start
        self._run_time_remaining -= duration
        self._run_time_remaining = max(self._run_time_remaining, 0)
        self._run_start = now
        return self._run_time_remaining

    def _prepare_to_track_run_time(self, timeout):
        self._setup_watch_manager(timeout)
        self._run_start = time.time()
        self._run_time_remaining = timeout
<|endoftext|>