<commit_before>import sys, os
import commands
import time
import threading
import multiprocessing
import random

# thread class to run the Bayenv test phase in multiple threads
class RunInThread(threading.Thread):

    def __init__(self, cmd, thread_id, testdata):
        threading.Thread.__init__(self)
        self.cmd = cmd
        self.thread_id = thread_id
        self.testdata = testdata
        #self.queue = queue

    def run(self):
        # execute the command, queue the result
        #(status, output) = commands.getstatusoutput(self.cmd)
        #elf.queue.put((self.cmd, output, status))
        #result_file = "bf_environ." + env_file
        errors = open("errors.out", 'w')
        outfile = "bf_results_t" + str(self.thread_id) + ".txt"
        k = 0
        dataset = open(self.testdata, 'r')
        thread_id = str(self.thread_id)
        cmd = self.cmd
        while dataset:
            k += 1
            line1 = dataset.readline()
            if line1 == '':
                break
            l = line1.split("\t")
            marker_file = l.pop(0) + "-" + str(self.thread_id)
            FILE = open(marker_file, 'w')
            line1 = "\t".join(l)
            line2 = dataset.readline()
            FILE.write(line1 + line2)
            FILE.close()
            print "BAYENV: thread " + thread_id + " is processing " + marker_file + " (" + str(k) + ")...",
            start_test = time.time()
            sys.stdout.flush()
            failure, output = commands.getstatusoutput(cmd % (marker_file, outfile))
            elapsed = (time.time() - start_test)
            print "done. %f sec to complete" % elapsed
            #total = ((elapsed * 33000)/60)/60
            #print "Total: " + str(total)

            if failure:
                print output
                errors.write(output)
                #sys.exit(1)
                error =  "Could not test locus: " + marker_file +"\n"\
                      + line1 + line2
                errors.write(error)
                os.remove(marker_file)
                os.remove(marker_file + ".freqs")
                continue
            os.remove(marker_file)
            os.remove(marker_file + ".freqs")
        errors.close()
        dataset.close()
        #new_result_file = "iter" + str(i) + "_" + result_file
        #os.rename(result_file, new_result_file) #Rename result file.

# Process class to run the Bayenv test phase in paralell
class RunInProcess(multiprocessing.Process):

    def __init__(self, cmd, thread_id, testdata, testsize):
        multiprocessing.Process.__init__(self)
        self.cmd = cmd
        self.thread_id = thread_id
        self.testdata = testdata
        self.testsize = testsize

    def run(self):
        errors = open("errors.out", 'w')
        outfile = "results/bf_results_t" + str(self.thread_id)
        k = 0
        dataset = open(self.testdata, 'r')
        thread_id = str(self.thread_id)
        cmd = self.cmd
        data_size = self.testsize
        while dataset:
            k += 1
            line1 = dataset.readline()
            if line1 == '':
                break
            l = line1.split("\t")
            marker_file = l.pop(0) + "-" + str(self.thread_id)
            FILE = open(marker_file, 'w')
            line1 = "\t".join(l)
            line2 = dataset.readline()
            FILE.write(line1 + line2)
            FILE.close()
            print "BAYENV: process " + thread_id + " is processing " + marker_file + " (" + str(k) + ")...",
            start_test = time.time()
            sys.stdout.flush()
            failure, output = commands.getstatusoutput(cmd % (marker_file, outfile))
            elapsed = (time.time() - start_test)
            remaining = ((data_size-k)*elapsed)/60
            print "done. %f sec to complete. Estimated time remaining: %f minutes" % (elapsed, remaining)

            if failure:
                print output
                errors.write(output)
                error =  "Could not test locus: " + marker_file +"\n"\
                      + line1 + line2
                errors.write(error)
                os.remove(marker_file)
                os.remove(marker_file + ".freqs")
                continue
            os.remove(marker_file)
            os.remove(marker_file + ".freqs")
        errors.close()
        dataset.close()

#Running BAYENV sequentially
def test_run_seq(cmd, thread_id, testdata):

    thread_id = str(thread_id)
    errors = open("errors.out", 'w')
    outfile = "bf_results_t" + thread_id + ".txt"
    k = 0
    dataset = open(testdata, 'r')
    while dataset:
        k += 1
        line1 = dataset.readline()
        if line1 == '':
            break
        l = line1.split("\t")
        marker_file = l.pop(0) + "-" + thread_id
        FILE = open(marker_file, 'w')
        line1 = "\t".join(l)
        line2 = dataset.readline()
        FILE.write(line1 + line2)
        FILE.close()
        print "BAYENV: sequence " + thread_id + " is processing " + marker_file + " (" + str(k) + ")...",
        start_test = time.time()
        sys.stdout.flush()
        failure, output = commands.getstatusoutput(cmd % (marker_file, outfile))
        elapsed = (time.time() - start_test)
        print "done. %f sec to complete" % elapsed
            #total = ((elapsed * 33000)/60)/60
            #print "Total: " + str(total)

        if failure:
            print output
            errors.write(output)
                #sys.exit(1)
            error =  "Could not test locus: " + marker_file +"\n"\
                + line1 + line2
            errors.write(error)
            os.remove(marker_file)
            os.remove(marker_file + ".freqs")
            continue
        os.remove(marker_file)
        os.remove(marker_file + ".freqs")
    errors.close()
    dataset.close()
        #new_result_file = "iter" + str(i) + "_" + result_file
        #os.rename(result_file, new_result_file) #Rename result file.

############# Running BAYENV sequentially ###############
def test_all_snps_seq(testdata, env_file, no_env_var):

    cmds = []
    num_runs = 4
    rand_seeds = [int(random.uniform(1,99999)) for i in range(1, num_runs+1)]

    for i, rand_seed in enumerate(rand_seeds):
        cmds.append("bayenv2 -i %s -m " + "covar_iter" + str(i+1) + ".txt" + \
                    " -e " + env_file + \
                    " -p 3 " + \
                    "-k 20000 " + \
                    " -n " + str(no_env_var) + \
                    " -t -c -r " + str(rand_seed) + " -o %s")

    start = time.time()
    for i in range(len(cmds)):
        test_run_seq(cmds[i], i, testdata)
    print "Elapsed time: %s" % (time.time()-start)
    exit(0)

############# Running BAYENV with threads ###############
def test_all_snps_multit(testdata, env_file, no_env_var):

    threads = []
    cmds = []
    num_runs = 4
    rand_seeds = [int(random.uniform(1,99999)) for i in range(1, num_runs+1)]

    for i, rand_seed in enumerate(rand_seeds):
        cmds.append("bayenv2 -i %s -m " + "covar_iter" + str(i+1) + ".txt" + \
                    " -e " + env_file + \
                    " -p 3 " + \
                    "-k 10000 " + \
                    " -n " + str(no_env_var) + \
                    " -t -c -r " + str(rand_seed) + " -o %s")
    start = time.time()
    for i in range(len(cmds)):
        thread = RunInThread(cmds[i], i, testdata)
        threads.append(thread)
        thread.start()

    for t in threads:
        t.join()
    print "Elapsed time: %s" % (time.time()-start)

    #Call the second difference method
    #compute_results(num_runs, 1, 1.5)

    #exit(0)

############# Running BAYENV with multiprocessing ###############
def test_all_snps_multip(testdata, cmds, testsize):

    procs = []

    start = time.time()
    for i in range(len(cmds)):
        proc = RunInProcess(cmds[i], i, testdata, testsize)
        procs.append(proc)
        proc.start()

    for p in procs:
        p.join()
    print "Elapsed time: %s" % (time.time()-start)
    #Call the second difference method
    #compute_results(len(cmds), 1, 0.5)
    #exit(0)

############# Running Bayenv in test modus ##############
def test_all_snps(testdata, env_file, n_iter, no_env_var):

    result_file = "bf_environ." + env_file
    errors = open("errors.out", 'w')

    for i in range(1, int(n_iter)+1):

        dataset = open(testdata, 'r')
        covar_file = "covar_iter" + str(i) + ".txt"
        #cmd = "bayenv 1 3 -47123 30000 %s " + env_file + " " + \
        cmd = "bayenv 1 3 -74123 50000 %s " + env_file + " " + \
              str(no_env_var) + " " + covar_file

        cmd = "bayenv2 -i %s -m " + covar_file + " -e " + env_file + " -p 3 " + \
              "-k 50000 -n " + str(no_env_var) + " -t -c -r 44332"

        k = 0
        while dataset:
            k += 1
            line1 = dataset.readline()
            if line1 == '':
                break
            l = line1.split("\t")
            marker_file = l.pop(0)
            FILE = open(marker_file, 'w')
            line1 = "\t".join(l)
            line2 = dataset.readline()
            FILE.write(line1 + line2)
            FILE.close()
            print "BAYENV processing " + marker_file + " (" + str(k) + ")...",
            start_test = time.time()
            sys.stdout.flush()
            failure, output = commands.getstatusoutput(cmd % marker_file)
            elapsed = (time.time() - start_test)
            print "done. %f sec to complete" % elapsed
            #total = ((elapsed * 33000)/60)/60
            #print "Total: " + str(total)

            if failure:
                print output
                errors.write(output)
                #sys.exit(1)
                error =  "Could not test locus: " + marker_file +"\n"\
                      + line1 + line2
                errors.write(error)
                os.remove(marker_file)
                continue
            os.remove(marker_file)
        errors.close()
        dataset.close()
        new_result_file = "iter" + str(i) + "_" + result_file
        os.rename(result_file, new_result_file) #Rename result file.
        #run_r_cmd(new_result_file, i)

        #make_pdf_report(i)

#Calculating covariance matrices every 5000 iterations - Bayenv 1.0
def compute_null_hyp(num_pops, rand_seed, iterations, snpfile):

    time_usage = open("t_usage.out", 'w')
    cmd = "bayenv 0 " + num_pops +" "+ rand_seed +" "+ iterations \
          +" "+ snpfile + " > covars.txt"

    #print cmd

    print "BAYENV calculating the covariance matrices...",
    start_test = time.time()
    sys.stdout.flush()
    failure, output = commands.getstatusoutput(cmd)
    elapsed = (time.time() - start_test)
    usage = "done. %f sec to complete" % elapsed
    print usage
    time_usage.write(usage)
    time_usage.close()

#Calculating covariance matrices every 500 iterations - Bayenv 2.0
def compute_null_model_bayenv2(num_pops, iterations, snpfile):

    print "#############################################"

    time_usage = open("t_usage.out", 'w')

    rand_seed = str(int(random.uniform(1,99999)))
    print "Random seed = " + rand_seed

    cmd_str = open("covar-cmd.txt", "wb")

    cmd = "bayenv2 -i " + snpfile + " -p " + str(num_pops) + " -k " + str(iterations) \
        + " -r " + str(rand_seed) + " > covars.txt"

    print cmd
    cmd_str.write(cmd)
    cmd_str.close()

    print "BAYENV calculating the covariance matrices...",
    start_test = time.time()
    sys.stdout.flush()
    failure, output = commands.getstatusoutput(cmd)
    elapsed = (time.time() - start_test)
    usage = "done. %f sec to complete" % elapsed
    print usage
    time_usage.write(usage)
    time_usage.close()
<commit_msg>Updating run_bayenv.py<commit_after>import sys, os
import commands
import time
import multiprocessing
import random

# Process class to run the Bayenv test phase in paralell
class RunInProcess(multiprocessing.Process):

    def __init__(self, cmd, thread_id, testdata, testsize):
        multiprocessing.Process.__init__(self)
        self.cmd = cmd
        self.thread_id = thread_id
        self.testdata = testdata
        self.testsize = testsize

    def run(self):
        errors = open("errors.out", 'w')
        outfile = "results/bf_results_t" + str(self.thread_id)
        k = 0
        dataset = open(self.testdata, 'r')
        thread_id = str(self.thread_id)
        cmd = self.cmd
        data_size = self.testsize
        while dataset:
            k += 1
            line1 = dataset.readline()
            if line1 == '':
                break
            l = line1.split("\t")
            marker_file = l.pop(0) + "-" + str(self.thread_id)
            FILE = open(marker_file, 'w')
            line1 = "\t".join(l)
            line2 = dataset.readline()
            FILE.write(line1 + line2)
            FILE.close()
            print "BAYENV: process " + thread_id + " is processing " + marker_file + " (" + str(k) + ")...",
            start_test = time.time()
            sys.stdout.flush()
            failure, output = commands.getstatusoutput(cmd % (marker_file, outfile))
            elapsed = (time.time() - start_test)
            remaining = ((data_size-k)*elapsed)/60
            print "done. %f sec to complete. Estimated time remaining: %f minutes" % (elapsed, remaining)

            if failure:
                print output
                errors.write(output)
                error =  "Could not test locus: " + marker_file +"\n"\
                      + line1 + line2
                errors.write(error)
                os.remove(marker_file)
                os.remove(marker_file + ".freqs")
                continue
            os.remove(marker_file)
            os.remove(marker_file + ".freqs")
        errors.close()
        dataset.close()

############# Running BAYENV with multiprocessing ###############
def test_all_snps_multip(testdata, cmds, testsize):

    procs = []

    start = time.time()
    for i in range(len(cmds)):
        proc = RunInProcess(cmds[i], i, testdata, testsize)
        procs.append(proc)
        proc.start()

    for p in procs:
        p.join()
    print "Elapsed time: %s" % (time.time()-start)

#Calculating covariance matrices every 500 iterations - Bayenv 2.0
def compute_null_model_bayenv2(num_pops, iterations, snpfile):

    print "#############################################"

    time_usage = open("t_usage.out", 'w')

    rand_seed = str(int(random.uniform(1,99999)))
    print "Random seed = " + rand_seed

    cmd_str = open("covar-cmd.txt", "wb")

    cmd = "bayenv2 -i " + snpfile + " -p " + str(num_pops) + " -k " + str(iterations) \
        + " -r " + str(rand_seed) + " > covars.txt"

    print cmd
    cmd_str.write(cmd)
    cmd_str.close()

    print "BAYENV calculating the covariance matrices...",
    start_test = time.time()
    sys.stdout.flush()
    failure, output = commands.getstatusoutput(cmd)
    elapsed = (time.time() - start_test)
    usage = "done. %f sec to complete" % elapsed
    print usage
    time_usage.write(usage)
    time_usage.close()
<|endoftext|>


<commit_before># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Common
PROJECT_ID_PROP = "project.id"

# Data
CSV_HEADER = "header"
CSV_INFER_SCHEMA = "inferSchema"
FORMAT_JSON = "json"
FORMAT_CSV = "csv"
FORMAT_AVRO = "avro"
FORMAT_PRQT = "parquet"
FORMAT_AVRO_EXTD = "com.databricks.spark.avro"
FORMAT_BIGQUERY = "com.google.cloud.spark.bigquery"
FORMAT_JDBC = "jdbc"
JDBC_URL = "url"
JDBC_TABLE = "dbtable"
JDBC_DRIVER = "driver"
JDBC_BATCH_SIZE = "batchsize"
JDBC_PARTITIONCOLUMN = "partitionColumn"
JDBC_LOWERBOUND = "lowerBound"
JDBC_UPPERBOUND = "upperBound"
JDBC_NUMPARTITIONS = "numPartitions"
JDBC_CREATE_TABLE_OPTIONS = "createTableOptions"
FORMAT_HBASE = "org.apache.hadoop.hbase.spark"
TABLE = "table"
TEMP_GCS_BUCKET="temporaryGcsBucket"

# Output mode
OUTPUT_MODE_OVERWRITE = "overwrite"
OUTPUT_MODE_APPEND = "append"
OUTPUT_MODE_IGNORE = "ignore"
OUTPUT_MODE_ERRORIFEXISTS = "errorifexists"

# GCS to BigQuery
GCS_BQ_INPUT_LOCATION = "gcs.bigquery.input.location"
GCS_BQ_INPUT_FORMAT = "gcs.bigquery.input.format"
GCS_BQ_OUTPUT_DATASET = "gcs.bigquery.output.dataset"
GCS_BQ_OUTPUT_TABLE = "gcs.bigquery.output.table"
GCS_BQ_OUTPUT_MODE = "gcs.bigquery.output.mode"
GCS_BQ_TEMP_BUCKET = "temporaryGcsBucket"
GCS_BQ_LD_TEMP_BUCKET_NAME = "gcs.bigquery.temp.bucket.name"

# GCS to JDBC
GCS_JDBC_INPUT_LOCATION = "gcs.jdbc.input.location"
GCS_JDBC_INPUT_FORMAT = "gcs.jdbc.input.format"
GCS_JDBC_OUTPUT_TABLE = "gcs.jdbc.output.table"
GCS_JDBC_OUTPUT_MODE = "gcs.jdbc.output.mode"
GCS_JDBC_OUTPUT_URL = "gcs.jdbc.output.url"
GCS_JDBC_OUTPUT_DRIVER = "gcs.jdbc.output.driver"
GCS_JDBC_BATCH_SIZE = "gcs.jdbc.batch.size"

# GCS to BigTable
GCS_BT_INPUT_LOCATION = "gcs.bigtable.input.location"
GCS_BT_INPUT_FORMAT = "gcs.bigtable.input.format"
GCS_BT_HBASE_CATALOG_JSON = "gcs.bigtable.hbase.catalog.json"

# BigQuery to GCS
BQ_GCS_INPUT_TABLE = "bigquery.gcs.input.table"
BQ_GCS_OUTPUT_FORMAT = "bigquery.gcs.output.format"
BQ_GCS_OUTPUT_MODE = "bigquery.gcs.output.mode"
BQ_GCS_OUTPUT_LOCATION = "bigquery.gcs.output.location"

# Hive to BigQuery
HIVE_BQ_OUTPUT_MODE = "hive.bigquery.output.mode"
HIVE_BQ_LD_TEMP_BUCKET_NAME = "hive.bigquery.temp.bucket.name"
HIVE_BQ_OUTPUT_DATASET = "hive.bigquery.output.dataset"
HIVE_BQ_OUTPUT_TABLE = "hive.bigquery.output.table"
HIVE_BQ_INPUT_DATABASE = "hive.bigquery.input.database"
HIVE_BQ_INPUT_TABLE = "hive.bigquery.input.table"

# Hive to GCS
HIVE_GCS_INPUT_DATABASE="hive.gcs.input.database"
HIVE_GCS_INPUT_TABLE = "hive.gcs.input.table"
HIVE_GCS_OUTPUT_LOCATION = "hive.gcs.output.location"
HIVE_GCS_OUTPUT_FORMAT = "hive.gcs.output.format"
HIVE_GCS_OUTPUT_MODE = "hive.gcs.output.mode"

#Hbase to GCS
HBASE_GCS_OUTPUT_LOCATION = "hbase.gcs.output.location"
HBASE_GCS_OUTPUT_FORMAT = "hbase.gcs.output.format"
HBASE_GCS_OUTPUT_MODE = "hbase.gcs.output.mode"
HBASE_GCS_CATALOG_JSON = "hbase.gcs.catalog.json"

# GCS to JDBC
JDBCTOJDBC_INPUT_URL = "jdbctojdbc.input.url"
JDBCTOJDBC_INPUT_DRIVER = "jdbctojdbc.input.driver"
JDBCTOJDBC_INPUT_TABLE = "jdbctojdbc.input.table"
JDBCTOJDBC_INPUT_PARTITIONCOLUMN = "jdbctojdbc.input.partitioncolumn"
JDBCTOJDBC_INPUT_LOWERBOUND = "jdbctojdbc.input.lowerbound"
JDBCTOJDBC_INPUT_UPPERBOUND = "jdbctojdbc.input.upperbound"
JDBCTOJDBC_NUMPARTITIONS = "jdbctojdbc.input.numpartitions"
JDBCTOJDBC_OUTPUT_URL = "jdbctojdbc.output.url"
JDBCTOJDBC_OUTPUT_DRIVER = "jdbctojdbc.output.driver"
JDBCTOJDBC_OUTPUT_TABLE = "jdbctojdbc.output.table"
JDBCTOJDBC_OUTPUT_CREATE_TABLE_OPTION = "jdbctojdbc.output.create_table.option"
JDBCTOJDBC_OUTPUT_MODE = "jdbctojdbc.output.mode"
JDBCTOJDBC_OUTPUT_BATCH_SIZE = "jdbctojdbc.output.batch.size"<commit_msg>jdbc to jdbc<commit_after># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Common
PROJECT_ID_PROP = "project.id"

# Data
CSV_HEADER = "header"
CSV_INFER_SCHEMA = "inferSchema"
FORMAT_JSON = "json"
FORMAT_CSV = "csv"
FORMAT_AVRO = "avro"
FORMAT_PRQT = "parquet"
FORMAT_AVRO_EXTD = "com.databricks.spark.avro"
FORMAT_BIGQUERY = "com.google.cloud.spark.bigquery"
FORMAT_JDBC = "jdbc"
JDBC_URL = "url"
JDBC_TABLE = "dbtable"
JDBC_DRIVER = "driver"
JDBC_BATCH_SIZE = "batchsize"
JDBC_PARTITIONCOLUMN = "partitionColumn"
JDBC_LOWERBOUND = "lowerBound"
JDBC_UPPERBOUND = "upperBound"
JDBC_NUMPARTITIONS = "numPartitions"
JDBC_CREATE_TABLE_OPTIONS = "createTableOptions"
FORMAT_HBASE = "org.apache.hadoop.hbase.spark"
TABLE = "table"
TEMP_GCS_BUCKET="temporaryGcsBucket"

# Output mode
OUTPUT_MODE_OVERWRITE = "overwrite"
OUTPUT_MODE_APPEND = "append"
OUTPUT_MODE_IGNORE = "ignore"
OUTPUT_MODE_ERRORIFEXISTS = "errorifexists"

# GCS to BigQuery
GCS_BQ_INPUT_LOCATION = "gcs.bigquery.input.location"
GCS_BQ_INPUT_FORMAT = "gcs.bigquery.input.format"
GCS_BQ_OUTPUT_DATASET = "gcs.bigquery.output.dataset"
GCS_BQ_OUTPUT_TABLE = "gcs.bigquery.output.table"
GCS_BQ_OUTPUT_MODE = "gcs.bigquery.output.mode"
GCS_BQ_TEMP_BUCKET = "temporaryGcsBucket"
GCS_BQ_LD_TEMP_BUCKET_NAME = "gcs.bigquery.temp.bucket.name"

# GCS to JDBC
GCS_JDBC_INPUT_LOCATION = "gcs.jdbc.input.location"
GCS_JDBC_INPUT_FORMAT = "gcs.jdbc.input.format"
GCS_JDBC_OUTPUT_TABLE = "gcs.jdbc.output.table"
GCS_JDBC_OUTPUT_MODE = "gcs.jdbc.output.mode"
GCS_JDBC_OUTPUT_URL = "gcs.jdbc.output.url"
GCS_JDBC_OUTPUT_DRIVER = "gcs.jdbc.output.driver"
GCS_JDBC_BATCH_SIZE = "gcs.jdbc.batch.size"

# GCS to BigTable
GCS_BT_INPUT_LOCATION = "gcs.bigtable.input.location"
GCS_BT_INPUT_FORMAT = "gcs.bigtable.input.format"
GCS_BT_HBASE_CATALOG_JSON = "gcs.bigtable.hbase.catalog.json"

# BigQuery to GCS
BQ_GCS_INPUT_TABLE = "bigquery.gcs.input.table"
BQ_GCS_OUTPUT_FORMAT = "bigquery.gcs.output.format"
BQ_GCS_OUTPUT_MODE = "bigquery.gcs.output.mode"
BQ_GCS_OUTPUT_LOCATION = "bigquery.gcs.output.location"

# Hive to BigQuery
HIVE_BQ_OUTPUT_MODE = "hive.bigquery.output.mode"
HIVE_BQ_LD_TEMP_BUCKET_NAME = "hive.bigquery.temp.bucket.name"
HIVE_BQ_OUTPUT_DATASET = "hive.bigquery.output.dataset"
HIVE_BQ_OUTPUT_TABLE = "hive.bigquery.output.table"
HIVE_BQ_INPUT_DATABASE = "hive.bigquery.input.database"
HIVE_BQ_INPUT_TABLE = "hive.bigquery.input.table"

# Hive to GCS
HIVE_GCS_INPUT_DATABASE="hive.gcs.input.database"
HIVE_GCS_INPUT_TABLE = "hive.gcs.input.table"
HIVE_GCS_OUTPUT_LOCATION = "hive.gcs.output.location"
HIVE_GCS_OUTPUT_FORMAT = "hive.gcs.output.format"
HIVE_GCS_OUTPUT_MODE = "hive.gcs.output.mode"

#Hbase to GCS
HBASE_GCS_OUTPUT_LOCATION = "hbase.gcs.output.location"
HBASE_GCS_OUTPUT_FORMAT = "hbase.gcs.output.format"
HBASE_GCS_OUTPUT_MODE = "hbase.gcs.output.mode"
HBASE_GCS_CATALOG_JSON = "hbase.gcs.catalog.json"

# JDBC to JDBC
JDBCTOJDBC_INPUT_URL = "jdbctojdbc.input.url"
JDBCTOJDBC_INPUT_DRIVER = "jdbctojdbc.input.driver"
JDBCTOJDBC_INPUT_TABLE = "jdbctojdbc.input.table"
JDBCTOJDBC_INPUT_PARTITIONCOLUMN = "jdbctojdbc.input.partitioncolumn"
JDBCTOJDBC_INPUT_LOWERBOUND = "jdbctojdbc.input.lowerbound"
JDBCTOJDBC_INPUT_UPPERBOUND = "jdbctojdbc.input.upperbound"
JDBCTOJDBC_NUMPARTITIONS = "jdbctojdbc.input.numpartitions"
JDBCTOJDBC_OUTPUT_URL = "jdbctojdbc.output.url"
JDBCTOJDBC_OUTPUT_DRIVER = "jdbctojdbc.output.driver"
JDBCTOJDBC_OUTPUT_TABLE = "jdbctojdbc.output.table"
JDBCTOJDBC_OUTPUT_CREATE_TABLE_OPTION = "jdbctojdbc.output.create_table.option"
JDBCTOJDBC_OUTPUT_MODE = "jdbctojdbc.output.mode"
JDBCTOJDBC_OUTPUT_BATCH_SIZE = "jdbctojdbc.output.batch.size"<|endoftext|>


<commit_before>import django

DEBUG = True
TEMPLATE_DEBUG = DEBUG

import os

PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'testdb.sqlite',
    }
}

LANGUAGE_CODE = 'en-us'
SITE_ID = 1

STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

SECRET_KEY = 'qd@j3*it@3j2cgc#7t@m)^r1bnc53uam7o6u_+x$f5w3$b@3ix'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pybb.contrib.profiles.middleware.PybbMiddleware',
    'pybb.contrib.ban.middleware.PybbBanMiddleware',
)

ROOT_URLCONF = 'pybb.tests.urls'

TEMPLATE_DIRS = (
    os.path.join(PROJECT_ROOT, 'templates'),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'pybb',
    'pybb.contrib.profiles',
    'pybb.contrib.mentions',
    'pybb.contrib.quotes',
    'pybb.contrib.smilies',
    'pybb.contrib.reports',
    'pybb.contrib.ban',
    'sorl.thumbnail',
    'pytils',
    'registration',
    'guardian',
    'django.contrib.humanize',
)

STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.static',
    'pybb.context_processors.processor',
)

AUTH_PROFILE_MODULE = 'pybb.Profile'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

PYBB_ATTACHMENT_ENABLE = True

PYBB_SMILIES_USE_CACHE = False

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.eggs.Loader',
    'django.template.loaders.app_directories.Loader',
)

ANONYMOUS_USER_ID = -1

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'guardian.backends.ObjectPermissionBackend',
)

PYBB_BBCODE_MARKUP_FORMATTERS = (
    ('quote', ('pybb.contrib.quotes.formatters.quote', None)),
    ('mention', ('pybb.contrib.mentions.formatters.mention', None)),
)

PYBB_POST_CREATE_VIEW = 'pybb.contrib.quotes.views.PostCreateView'

PYBB_MARKUP_PREPROCESSORS = (
    'pybb.contrib.mentions.processors.MentionProcessor',
    'pybb.contrib.smilies.processors.SmileyProcessor',
)

PYBB_NOTIFICATION_ENABLE = True

PYBB_DUPLICATE_TOPIC_SLUG_ALLOWED = False

PYBB_BAN_CHECK_TIMEOUT = 0

PYBB_BODY_CLEANERS = [
    'pybb.util.rstrip_str',
    'pybb.util.filter_blanks',
]

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': 'http://127.0.0.1:9200/',
        'INDEX_NAME': 'public_forums',
    },
}

if django.VERSION <= (1, 6):
    TEST_RUNNER = 'discover_runner.DiscoverRunner'

AUTH_PROFILE_MODULE = 'pybb.Profile'
<commit_msg>Add filters to remove deprecation warning<commit_after>import django

DEBUG = True
TEMPLATE_DEBUG = DEBUG

import os

PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'testdb.sqlite',
    }
}

LANGUAGE_CODE = 'en-us'
SITE_ID = 1

STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

SECRET_KEY = 'qd@j3*it@3j2cgc#7t@m)^r1bnc53uam7o6u_+x$f5w3$b@3ix'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pybb.contrib.profiles.middleware.PybbMiddleware',
    'pybb.contrib.ban.middleware.PybbBanMiddleware',
)

ROOT_URLCONF = 'pybb.tests.urls'

TEMPLATE_DIRS = (
    os.path.join(PROJECT_ROOT, 'templates'),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'pybb',
    'pybb.contrib.profiles',
    'pybb.contrib.mentions',
    'pybb.contrib.quotes',
    'pybb.contrib.smilies',
    'pybb.contrib.reports',
    'pybb.contrib.ban',
    'sorl.thumbnail',
    'pytils',
    'registration',
    'guardian',
    'django.contrib.humanize',
)

STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.static',
    'pybb.context_processors.processor',
)

AUTH_PROFILE_MODULE = 'pybb.Profile'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

PYBB_ATTACHMENT_ENABLE = True

PYBB_SMILIES_USE_CACHE = False

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.eggs.Loader',
    'django.template.loaders.app_directories.Loader',
)

ANONYMOUS_USER_ID = -1

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'guardian.backends.ObjectPermissionBackend',
)

PYBB_BBCODE_MARKUP_FORMATTERS = (
    ('quote', ('pybb.contrib.quotes.formatters.quote', None)),
    ('mention', ('pybb.contrib.mentions.formatters.mention', None)),
)

PYBB_POST_CREATE_VIEW = 'pybb.contrib.quotes.views.PostCreateView'

PYBB_MARKUP_PREPROCESSORS = (
    'pybb.contrib.mentions.processors.MentionProcessor',
    'pybb.contrib.smilies.processors.SmileyProcessor',
)

PYBB_NOTIFICATION_ENABLE = True

PYBB_DUPLICATE_TOPIC_SLUG_ALLOWED = False

PYBB_BAN_CHECK_TIMEOUT = 0

PYBB_BODY_CLEANERS = [
    'pybb.util.rstrip_str',
    'pybb.util.filter_blanks',
]

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': 'http://127.0.0.1:9200/',
        'INDEX_NAME': 'public_forums',
    },
}

if django.VERSION <= (1, 6):
    TEST_RUNNER = 'discover_runner.DiscoverRunner'

AUTH_PROFILE_MODULE = 'pybb.Profile'
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2015 SciFabric LTD.
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
"""Flickr view for PyBossa."""
from flask import Blueprint, request, url_for, flash, redirect, session, current_app
from pybossa.core import flickr
from flask_oauthlib.client import OAuthException

blueprint = Blueprint('flickr', __name__)

@blueprint.route('/')
def login():
    """Login using Flickr view."""
    callback_url = url_for('.oauth_authorized', next=request.args.get('next'))
    return flickr.authorize(callback=callback_url, perms='read')

@blueprint.route('/revoke-access')
def logout():
    """Log out."""
    next_url = request.args.get('next') or url_for('home.home')
    flickr.remove_credentials(session)
    return redirect(next_url)

@blueprint.route('/oauth-authorized')
def oauth_authorized():
    """Authorize Flickr login."""
    next_url = request.args.get('next')
    resp = flickr.authorized_response()
    if resp is None:
        flash(u'You denied the request to sign in.')
        return redirect(next_url)
    if isinstance(resp, OAuthException):
        flash('Access denied: %s' % resp.message)
        current_app.logger.error(resp)
        return redirect(next_url)
    flickr_token = dict(oauth_token=resp['oauth_token'],
                        oauth_token_secret=resp['oauth_token_secret'])
    flickr_user = dict(username=resp['username'], user_nsid=resp['user_nsid'])
    flickr.save_credentials(session, flickr_token, flickr_user)
    return redirect(next_url)
<commit_msg>Add flickr albums endpoint<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2015 SciFabric LTD.
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.
"""Flickr view for PyBossa."""
import json
from flask import (Blueprint, request, url_for, flash, redirect, session,
    current_app, Response)
from pybossa.core import flickr
from flask_oauthlib.client import OAuthException

blueprint = Blueprint('flickr', __name__)

@blueprint.route('/')
def login():
    """Login using Flickr view."""
    callback_url = url_for('.oauth_authorized', next=request.args.get('next'))
    return flickr.authorize(callback=callback_url, perms='read')

@blueprint.route('/revoke-access')
def logout():
    """Log out."""
    next_url = request.args.get('next') or url_for('home.home')
    flickr.remove_credentials(session)
    return redirect(next_url)

@blueprint.route('/oauth-authorized')
def oauth_authorized():
    """Authorize Flickr login."""
    next_url = request.args.get('next')
    resp = flickr.authorized_response()
    if resp is None:
        flash(u'You denied the request to sign in.')
        return redirect(next_url)
    if isinstance(resp, OAuthException):
        flash('Access denied: %s' % resp.message)
        current_app.logger.error(resp)
        return redirect(next_url)
    flickr_token = dict(oauth_token=resp['oauth_token'],
                        oauth_token_secret=resp['oauth_token_secret'])
    flickr_user = dict(username=resp['username'], user_nsid=resp['user_nsid'])
    flickr.save_credentials(session, flickr_token, flickr_user)
    return redirect(next_url)

@blueprint.route('/albums')
def user_albums():
    albums = flickr.get_user_albums(session)
    return Response(json.dumps(albums), mimetype='application/json')
<|endoftext|>


<commit_before>import logging
import os.path

import numpy

from smqtk.algorithms.nn_index.lsh.functors import LshFunctor
from smqtk.representation.descriptor_element import elements_to_matrix

__author__ = "paul.tunison@kitware.com"

class ItqFunctor (LshFunctor):
    """
    Nearest neighbor implementation using Iterative Quantization (ITQ), a method
    to convert a descriptor (e.g. 4000-dim vector) to few bits (e.g. 64 bits).

    The method first appeared in
    ```
    Yunchao Gong and Svetlana Lazebnik. Iterative Quantization: A Procrustes
    Approach to Learning Binary Codes. In CVPR 2011.
    ```
    It was originally implemented in Matlab by Yunchao Gong
    (yunchao@cs.unc.edu).

    IMPORTANT:
        For consistency, we treat bit vectors such that the bit at index 0 is
        considered the most significant bit (big-endian).

    """

    @classmethod
    def is_usable(cls):
        return True

    def __init__(self, mean_vec_filepath=None, rotation_filepath=None,
                 bit_length=8, itq_iterations=50, random_seed=None):
        """
        Initialize IQR functor.

        File Path Note
        --------------
        Model filepaths should point to numpy files and include the ".npy" file
        suffix. If not, then when files are saves, numpy will automatically
        append the ".npy" suffix (e.g. "example" -> "example.npy"). This can
        lead to not being able to use the same configuration for fitting and
        application, as the file name's as given will not be found.

        :param mean_vec_filepath: Optional file location to load/store the mean
            vector when initialized and/or built. When None, this will only be
            stored in memory. This will use numpy to save/load, so this should
            have a ``.npy`` suffix, or one will be added at save time.
        :type mean_vec_filepath: str

        :param rotation_filepath: Optional file location to load/store the
            rotation matrix when initialize and/or built. When None, this will
            only be stored in memory. This will use numpy to save/load, so this
            should have a ``.npy`` suffix, or one will be added at save time.
        :type rotation_filepath: str

        :param bit_length: Number of bits used to represent descriptors (hash
            code). This must be greater than 0. If given an existing
        :type bit_length: int

        :param itq_iterations: Number of iterations for the ITQ algorithm to
            perform. This must be greater than 0.
        :type itq_iterations: int

        :param random_seed: Integer to use as the random number generator seed.
        :type random_seed: int

        """
        super(ItqFunctor, self).__init__()

        self.mean_vec_filepath = mean_vec_filepath
        self.rotation_filepath = rotation_filepath
        self.bit_length = bit_length
        self.itq_iterations = itq_iterations
        self.random_seed = random_seed

        # Model components
        self.mean_vec = None
        self.rotation = None

        self.load_model()

    def get_config(self):
        return {
            "mean_vec_filepath": self.mean_vec_filepath,
            "rotation_filepath": self.rotation_filepath,
            "bit_length": self.bit_length,
            "itq_iterations": self.itq_iterations,
            "random_seed": self.random_seed,
        }

    def has_model(self):
        return (self.mean_vec is not None) and (self.rotation is not None)

    def load_model(self):
        if (self.mean_vec_filepath and
                os.path.isfile(self.mean_vec_filepath) and
                self.rotation_filepath and
                os.path.isfile(self.rotation_filepath)):
            self.mean_vec = numpy.load(self.mean_vec_filepath)
            self.rotation = numpy.load(self.rotation_filepath)

    def save_model(self):
        if (self.mean_vec_filepath and self.rotation_filepath and
                self.mean_vec is not None and self.rotation is not None):
            numpy.save(self.mean_vec_filepath, self.mean_vec)
            numpy.save(self.rotation_filepath, self.rotation)

    def _find_itq_rotation(self, v, n_iter):
        """
        Finds a rotation of the PCA embedded data. Number of iterations must be
        greater than 0.

        This is equivalent to the ITQ function from UNC-CH's implementation.

        ``self`` is used only for logging. Otherwise this has no side effects
        on this instance.

        :param v: 2D numpy array, n*c PCA embedded data, n is the number of
            data elements and c is the code length.
        :type v: numpy.core.multiarray.ndarray

        :param n_iter: max number of iterations, 50 is usually enough
        :type n_iter: int

        :return: [b, r]
           b: 2D numpy array, n*c binary matrix,
           r: 2D numpy array, the c*c rotation matrix found by ITQ
        :rtype: numpy.core.multiarray.ndarray[bool],
            numpy.core.multiarray.ndarray[float]

        """
        # initialize with an orthogonal random rotation
        bit = v.shape[1]
        if self.random_seed is not None:
            numpy.random.seed(self.random_seed)
        r = numpy.random.randn(bit, bit)
        u11, s2, v2 = numpy.linalg.svd(r)
        r = u11[:, :bit]

        # ITQ to find optimal rotation
        self._log.debug("ITQ iterations to determine optimal rotation: %d",
                        n_iter)
        for i in range(n_iter):
            self._log.debug("ITQ iter %d", i + 1)
            # TODO: @numba.jit decorate
            z = numpy.dot(v, r)
            ux = numpy.ones(z.shape) * (-1)
            ux[z >= 0] = 1
            c = numpy.dot(ux.transpose(), v)
            ub, sigma, ua = numpy.linalg.svd(c)
            r = numpy.dot(ua, ub.transpose())

        # Make B binary matrix using final rotation matrix
        #   - original code returned B transformed by second to last rotation
        #       matrix, there by returning, de-synchronized matrices
        #   - Recomputing Z here so as to generate up-to-date B for the final
        #       rotation matrix computed.
        # TODO: Could move this step up one level and just return rotation mat?
        z = numpy.dot(v, r)
        b = numpy.zeros(z.shape, dtype=numpy.bool)
        b[z >= 0] = True

        return b, r

    def fit(self, descriptors):
        """
        Fit the ITQ model given the input set of descriptors

        :param descriptors: Iterable of ``DescriptorElement`` vectors to fit
            the model to.
        :type descriptors:
            collections.Iterable[smqtk.representation.DescriptorElement]

        :raises RuntimeError: There is already a model loaded

        :return: Matrix hash codes for provided descriptors in order.
        :rtype: numpy.ndarray[bool]

        """
        if self.has_model():
            raise RuntimeError("Model components have already been loaded.")

        self._log.info("Creating matrix of descriptors for fitting")
        dbg_report_interval = None
        if self.logger().getEffectiveLevel() <= logging.DEBUG:
            dbg_report_interval = 1.0  # seconds
        x = elements_to_matrix(list(descriptors),
                               report_interval=dbg_report_interval)
        self._log.debug("descriptor matrix shape: %s", x.shape)

        self._log.info("Centering data")
        self.mean_vec = numpy.mean(x, axis=0)
        x -= self.mean_vec

        self._log.info("Computing PCA transformation")
        # numpy and matlab observation format is flipped, thus the added
        # transpose.
        self._log.debug("-- computing covariance")
        c = numpy.cov(x.transpose())

        # Direct translation from UNC matlab code
        # - eigen vectors are the columns of ``pc``
        self._log.debug('-- computing linalg.eig')
        l, pc = numpy.linalg.eig(c)
        # ordered by greatest eigenvalue magnitude, keeping top ``bit_len``
        self._log.debug('-- computing top pairs')
        top_pairs = sorted(zip(l, pc.transpose()),
                           key=lambda p: p[0],
                           reverse=1
                           )[:self.bit_length]

        # # Harry translation -- Uses singular values / vectors, not eigen
        # # - singular vectors are the rows of pc
        # pc, l, _ = numpy.linalg.svd(c)
        # top_pairs = sorted(zip(l, pc),
        #                    key=lambda p: p[0],
        #                    reverse=1
        #                    )[:self.bit_length]

        # Eigen-vectors of top ``bit_len`` magnitude eigenvalues
        self._log.debug("-- top vector extraction")
        pc_top = numpy.array([p[1] for p in top_pairs]).transpose()
        self._log.debug("-- transform centered data by PC matrix")
        xx = numpy.dot(x, pc_top)

        self._log.info("Performing ITQ to find optimal rotation")
        c, self.rotation = self._find_itq_rotation(xx, self.itq_iterations)
        # De-adjust rotation with PC vector
        self.rotation = numpy.dot(pc_top, self.rotation)

        self.save_model()

        return c

    def get_hash(self, descriptor):
        """
        Get the locality-sensitive hash code for the input descriptor.

        :param descriptor: Descriptor vector we should generate the hash of.
        :type descriptor: numpy.ndarray[float]

        :return: Generated bit-vector as a numpy array of booleans.
        :rtype: numpy.ndarray[bool]

        """
        z = numpy.dot(descriptor - self.mean_vec, self.rotation)
        b = numpy.zeros(z.shape, dtype=bool)
        b[z >= 0] = True
        return b
<commit_msg>Added optional descriptor normalization to ITQ functor<commit_after>import logging
import os.path

import numpy

from smqtk.algorithms.nn_index.lsh.functors import LshFunctor
from smqtk.representation.descriptor_element import elements_to_matrix

__author__ = "paul.tunison@kitware.com"

class ItqFunctor (LshFunctor):
    """
    Nearest neighbor implementation using Iterative Quantization (ITQ), a method
    to convert a descriptor (e.g. 4000-dim vector) to few bits (e.g. 64 bits).

    The method first appeared in
    ```
    Yunchao Gong and Svetlana Lazebnik. Iterative Quantization: A Procrustes
    Approach to Learning Binary Codes. In CVPR 2011.
    ```
    It was originally implemented in Matlab by Yunchao Gong
    (yunchao@cs.unc.edu).

    IMPORTANT:
        For consistency, we treat bit vectors such that the bit at index 0 is
        considered the most significant bit (big-endian).

    """

    @classmethod
    def is_usable(cls):
        return True

    def __init__(self, mean_vec_filepath=None, rotation_filepath=None,
                 bit_length=8, itq_iterations=50, normalize=None,
                 random_seed=None):
        """
        Initialize IQR functor.

        File Path Note
        --------------
        Model filepaths should point to numpy files and include the ".npy" file
        suffix. If not, then when files are saves, numpy will automatically
        append the ".npy" suffix (e.g. "example" -> "example.npy"). This can
        lead to not being able to use the same configuration for fitting and
        application, as the file name's as given will not be found.

        :param mean_vec_filepath: Optional file location to load/store the mean
            vector when initialized and/or built. When None, this will only be
            stored in memory. This will use numpy to save/load, so this should
            have a ``.npy`` suffix, or one will be added at save time.
        :type mean_vec_filepath: str

        :param rotation_filepath: Optional file location to load/store the
            rotation matrix when initialize and/or built. When None, this will
            only be stored in memory. This will use numpy to save/load, so this
            should have a ``.npy`` suffix, or one will be added at save time.
        :type rotation_filepath: str

        :param bit_length: Number of bits used to represent descriptors (hash
            code). This must be greater than 0. If given an existing
        :type bit_length: int

        :param itq_iterations: Number of iterations for the ITQ algorithm to
            perform. This must be greater than 0.
        :type itq_iterations: int

        :param normalize: Normalize input vectors when fitting and generation
            hash vectors using ``numpy.linalg.norm``. This may either
            be  ``None``, disabling normalization, or any valid value that
            could be passed to the ``ord`` parameter in ``numpy.linalg.norm``
            for 1D arrays. This is ``None`` by default (no normalization).
        :type normalize: None | int | float | str

        :param random_seed: Integer to use as the random number generator seed.
        :type random_seed: int

        """
        super(ItqFunctor, self).__init__()

        self.mean_vec_filepath = mean_vec_filepath
        self.rotation_filepath = rotation_filepath
        self.bit_length = bit_length
        self.itq_iterations = itq_iterations
        self.normalize = normalize
        self.random_seed = random_seed

        # Validate normalization parameter by trying it on a random vector
        if normalize is not None:
            self._norm_vector(numpy.random.rand(8))

        # Model components
        self.mean_vec = None
        self.rotation = None

        self.load_model()

    def _norm_vector(self, v):
        """
        Class standard array normalization. Normalized along max dimension (a=0
        for a 1D array, a=1 for a 2D array, etc.).

        :param v: Vector to normalize
        :type v: numpy.ndarray

        :return: Returns the normalized version of input array ``v``.
        :rtype: numpy.ndarray

        """
        if self.normalize is not None:
            n = numpy.linalg.norm(v, self.normalize, v.ndim - 1,
                                  keepdims=True)
            # replace 0's with 1's, preventing div-by-zero
            n[n == 0.] = 1.
            return v / n

        # Normalization off
        return v

    def get_config(self):
        return {
            "mean_vec_filepath": self.mean_vec_filepath,
            "rotation_filepath": self.rotation_filepath,
            "bit_length": self.bit_length,
            "itq_iterations": self.itq_iterations,
            "random_seed": self.random_seed,
        }

    def has_model(self):
        return (self.mean_vec is not None) and (self.rotation is not None)

    def load_model(self):
        if (self.mean_vec_filepath and
                os.path.isfile(self.mean_vec_filepath) and
                self.rotation_filepath and
                os.path.isfile(self.rotation_filepath)):
            self.mean_vec = numpy.load(self.mean_vec_filepath)
            self.rotation = numpy.load(self.rotation_filepath)

    def save_model(self):
        if (self.mean_vec_filepath and self.rotation_filepath and
                self.mean_vec is not None and self.rotation is not None):
            numpy.save(self.mean_vec_filepath, self.mean_vec)
            numpy.save(self.rotation_filepath, self.rotation)

    def _find_itq_rotation(self, v, n_iter):
        """
        Finds a rotation of the PCA embedded data. Number of iterations must be
        greater than 0.

        This is equivalent to the ITQ function from UNC-CH's implementation.

        ``self`` is used only for logging. Otherwise this has no side effects
        on this instance.

        :param v: 2D numpy array, n*c PCA embedded data, n is the number of
            data elements and c is the code length.
        :type v: numpy.core.multiarray.ndarray

        :param n_iter: max number of iterations, 50 is usually enough
        :type n_iter: int

        :return: [b, r]
           b: 2D numpy array, n*c binary matrix,
           r: 2D numpy array, the c*c rotation matrix found by ITQ
        :rtype: numpy.core.multiarray.ndarray[bool],
            numpy.core.multiarray.ndarray[float]

        """
        # initialize with an orthogonal random rotation
        bit = v.shape[1]
        if self.random_seed is not None:
            numpy.random.seed(self.random_seed)
        r = numpy.random.randn(bit, bit)
        u11, s2, v2 = numpy.linalg.svd(r)
        r = u11[:, :bit]

        # ITQ to find optimal rotation
        self._log.debug("ITQ iterations to determine optimal rotation: %d",
                        n_iter)
        for i in range(n_iter):
            self._log.debug("ITQ iter %d", i + 1)
            # TODO: @numba.jit decorate
            z = numpy.dot(v, r)
            ux = numpy.ones(z.shape) * (-1)
            ux[z >= 0] = 1
            c = numpy.dot(ux.transpose(), v)
            ub, sigma, ua = numpy.linalg.svd(c)
            r = numpy.dot(ua, ub.transpose())

        # Make B binary matrix using final rotation matrix
        #   - original code returned B transformed by second to last rotation
        #       matrix, there by returning, de-synchronized matrices
        #   - Recomputing Z here so as to generate up-to-date B for the final
        #       rotation matrix computed.
        # TODO: Could move this step up one level and just return rotation mat?
        z = numpy.dot(v, r)
        b = numpy.zeros(z.shape, dtype=numpy.bool)
        b[z >= 0] = True

        return b, r

    def fit(self, descriptors):
        """
        Fit the ITQ model given the input set of descriptors

        :param descriptors: Iterable of ``DescriptorElement`` vectors to fit
            the model to.
        :type descriptors:
            collections.Iterable[smqtk.representation.DescriptorElement]

        :raises RuntimeError: There is already a model loaded

        :return: Matrix hash codes for provided descriptors in order.
        :rtype: numpy.ndarray[bool]

        """
        if self.has_model():
            raise RuntimeError("Model components have already been loaded.")

        self._log.info("Creating matrix of descriptors for fitting")
        dbg_report_interval = None
        if self.logger().getEffectiveLevel() <= logging.DEBUG:
            dbg_report_interval = 1.0  # seconds
        x = elements_to_matrix(list(descriptors),
                               report_interval=dbg_report_interval)
        self._log.debug("descriptor matrix shape: %s", x.shape)

        self._log.debug("Info normalizing descriptors by factor: %s",
                        self.normalize)
        x = self._norm_vector(x)

        self._log.info("Centering data")
        self.mean_vec = numpy.mean(x, axis=0)
        x -= self.mean_vec

        self._log.info("Computing PCA transformation")
        # numpy and matlab observation format is flipped, thus the added
        # transpose.
        self._log.debug("-- computing covariance")
        c = numpy.cov(x.transpose())

        # Direct translation from UNC matlab code
        # - eigen vectors are the columns of ``pc``
        self._log.debug('-- computing linalg.eig')
        l, pc = numpy.linalg.eig(c)
        # ordered by greatest eigenvalue magnitude, keeping top ``bit_len``
        self._log.debug('-- computing top pairs')
        top_pairs = sorted(zip(l, pc.transpose()),
                           key=lambda p: p[0],
                           reverse=1
                           )[:self.bit_length]

        # # Harry translation -- Uses singular values / vectors, not eigen
        # # - singular vectors are the rows of pc
        # pc, l, _ = numpy.linalg.svd(c)
        # top_pairs = sorted(zip(l, pc),
        #                    key=lambda p: p[0],
        #                    reverse=1
        #                    )[:self.bit_length]

        # Eigen-vectors of top ``bit_len`` magnitude eigenvalues
        self._log.debug("-- top vector extraction")
        pc_top = numpy.array([p[1] for p in top_pairs]).transpose()
        self._log.debug("-- transform centered data by PC matrix")
        xx = numpy.dot(x, pc_top)

        self._log.info("Performing ITQ to find optimal rotation")
        c, self.rotation = self._find_itq_rotation(xx, self.itq_iterations)
        # De-adjust rotation with PC vector
        self.rotation = numpy.dot(pc_top, self.rotation)

        self.save_model()

        return c

    def get_hash(self, descriptor):
        """
        Get the locality-sensitive hash code for the input descriptor.

        :param descriptor: Descriptor vector we should generate the hash of.
        :type descriptor: numpy.ndarray[float]

        :return: Generated bit-vector as a numpy array of booleans.
        :rtype: numpy.ndarray[bool]

        """
        z = numpy.dot(self._norm_vector(descriptor) - self.mean_vec,
                      self.rotation)
        b = numpy.zeros(z.shape, dtype=bool)
        b[z >= 0] = True
        return b
<|endoftext|>


<commit_before>from __future__ import print_function
import sys, traceback
from gi.repository import GLib, Gio
from . import generic
from .exitable import ExitableWithAliases

class ObjectWrapper(ExitableWithAliases("unwrap")):
	__slots__ = ["object", "outargs", "property_types"]

	def __init__(self, object, interfaces):
		self.object = object

		self.outargs = {}
		for iface in interfaces:
			for method in iface.methods:
				self.outargs[iface.name + "." + method.name] = [arg.signature for arg in method.out_args]

		self.property_types = {}
		for iface in interfaces:
			for prop in iface.properties:
				self.property_types[iface.name + "." + prop.name] = prop.signature

		for iface in interfaces:
			for signal in iface.signals:
				s_name = signal.name
				def EmitSignal(iface, signal):
					return lambda *args: self.SignalEmitted(iface.name, signal.name, GLib.Variant("(" + "".join(s.signature for s in signal.args) + ")", args))
				self._at_exit(getattr(object, signal.name).connect(EmitSignal(iface, signal)).__exit__)

		if "org.freedesktop.DBus.Properties" not in (iface.name for iface in interfaces):
			try:
				def onPropertiesChanged(iface, changed, invalidated):
					changed = {key: GLib.Variant(self.property_types[iface + "." + key], val) for key, val in changed.items()}
					args = GLib.Variant("(sa{sv}as)", (iface, changed, invalidated))
					self.SignalEmitted("org.freedesktop.DBus.Properties", "PropertiesChanged", args)
				self._at_exit(object.PropertiesChanged.connect(onPropertiesChanged).__exit__)
			except AttributeError:
				pass

	SignalEmitted = generic.signal()

	def call_method(self, connection, sender, object_path, interface_name, method_name, parameters, invocation):
		try:
			outargs = self.outargs[interface_name + "." + method_name]
			soutargs = "(" + "".join(outargs) + ")"

			method = getattr(self.object, method_name)

			result = method(*parameters)

			#if len(outargs) == 1:
			#	result = (result,)

			if len(outargs) == 0:
				invocation.return_value(None)
			else:
				invocation.return_value(GLib.Variant(soutargs, result))

		except Exception as e:
			#TODO Think of a better way to translate Python exception types to DBus error types.
			e_type = type(e).__name__
			if not "." in e_type:
				e_type = "unknown." + e_type
			invocation.return_dbus_error(e_type, str(e))

	def get_property(self, connection, sender, object_path, interface_name, property_name):
		# Note: It's impossible to correctly return an exception, as
		# g_dbus_connection_register_object_with_closures does not support it
		try:
			raise KeyError("aaa")
			type = self.property_types[interface_name + "." + property_name]
			result = getattr(self.object, property_name)
			return GLib.Variant(type, result)
		except Exception:
			print(traceback.format_exc(), file=sys.stderr)
			return None

	def set_property(self, connection, sender, object_path, interface_name, property_name, value):
		# Note: It's impossible to correctly return an exception, as
		# g_dbus_connection_register_object_with_closures does not support it
		try:
			type = self.property_types[interface_name + "." + property_name]
			assert(value.is_signature(type))
			setattr(self.object, property_name, value.unpack())
		except Exception:
			print(traceback.format_exc(), file=sys.stderr)
			return None

class ObjectRegistration(ExitableWithAliases("unregister")):
	__slots__ = ()

	def __init__(self, con, path, interfaces, wrapper, own_wrapper=False):
		if own_wrapper:
			self._at_exit(wrapper.__exit__)

		def func(interface_name, signal_name, parameters):
			con.emit_signal(None, path, interface_name, signal_name, parameters)

		self._at_exit(wrapper.SignalEmitted.connect(func).__exit__)

		ids = [con.register_object(path, interface, wrapper.call_method, wrapper.get_property, wrapper.set_property) for interface in interfaces]
		self._at_exit(lambda: [con.unregister_object(id) for id in ids])

class RegistrationMixin:
	__slots__ = ()

	def register_object(self, path, object, node_info):
		if node_info is None:
			try:
				node_info = type(object).dbus
			except AttributeError:
				node_info = type(object).__doc__

		if type(node_info) != list and type(node_info) != tuple:
			node_info = [node_info]

		node_info = [Gio.DBusNodeInfo.new_for_xml(ni) for ni in node_info]
		interfaces = sum((ni.interfaces for ni in node_info), [])

		wrapper = ObjectWrapper(object, interfaces)
		return ObjectRegistration(self.con, path, interfaces, wrapper, own_wrapper=True)
<commit_msg>Remove debugging leftover...<commit_after>from __future__ import print_function
import sys, traceback
from gi.repository import GLib, Gio
from . import generic
from .exitable import ExitableWithAliases

class ObjectWrapper(ExitableWithAliases("unwrap")):
	__slots__ = ["object", "outargs", "property_types"]

	def __init__(self, object, interfaces):
		self.object = object

		self.outargs = {}
		for iface in interfaces:
			for method in iface.methods:
				self.outargs[iface.name + "." + method.name] = [arg.signature for arg in method.out_args]

		self.property_types = {}
		for iface in interfaces:
			for prop in iface.properties:
				self.property_types[iface.name + "." + prop.name] = prop.signature

		for iface in interfaces:
			for signal in iface.signals:
				s_name = signal.name
				def EmitSignal(iface, signal):
					return lambda *args: self.SignalEmitted(iface.name, signal.name, GLib.Variant("(" + "".join(s.signature for s in signal.args) + ")", args))
				self._at_exit(getattr(object, signal.name).connect(EmitSignal(iface, signal)).__exit__)

		if "org.freedesktop.DBus.Properties" not in (iface.name for iface in interfaces):
			try:
				def onPropertiesChanged(iface, changed, invalidated):
					changed = {key: GLib.Variant(self.property_types[iface + "." + key], val) for key, val in changed.items()}
					args = GLib.Variant("(sa{sv}as)", (iface, changed, invalidated))
					self.SignalEmitted("org.freedesktop.DBus.Properties", "PropertiesChanged", args)
				self._at_exit(object.PropertiesChanged.connect(onPropertiesChanged).__exit__)
			except AttributeError:
				pass

	SignalEmitted = generic.signal()

	def call_method(self, connection, sender, object_path, interface_name, method_name, parameters, invocation):
		try:
			outargs = self.outargs[interface_name + "." + method_name]
			soutargs = "(" + "".join(outargs) + ")"

			method = getattr(self.object, method_name)

			result = method(*parameters)

			#if len(outargs) == 1:
			#	result = (result,)

			if len(outargs) == 0:
				invocation.return_value(None)
			else:
				invocation.return_value(GLib.Variant(soutargs, result))

		except Exception as e:
			#TODO Think of a better way to translate Python exception types to DBus error types.
			e_type = type(e).__name__
			if not "." in e_type:
				e_type = "unknown." + e_type
			invocation.return_dbus_error(e_type, str(e))

	def get_property(self, connection, sender, object_path, interface_name, property_name):
		# Note: It's impossible to correctly return an exception, as
		# g_dbus_connection_register_object_with_closures does not support it
		try:
			type = self.property_types[interface_name + "." + property_name]
			result = getattr(self.object, property_name)
			return GLib.Variant(type, result)
		except Exception:
			print(traceback.format_exc(), file=sys.stderr)
			return None

	def set_property(self, connection, sender, object_path, interface_name, property_name, value):
		# Note: It's impossible to correctly return an exception, as
		# g_dbus_connection_register_object_with_closures does not support it
		try:
			type = self.property_types[interface_name + "." + property_name]
			assert(value.is_signature(type))
			setattr(self.object, property_name, value.unpack())
		except Exception:
			print(traceback.format_exc(), file=sys.stderr)
			return None

class ObjectRegistration(ExitableWithAliases("unregister")):
	__slots__ = ()

	def __init__(self, con, path, interfaces, wrapper, own_wrapper=False):
		if own_wrapper:
			self._at_exit(wrapper.__exit__)

		def func(interface_name, signal_name, parameters):
			con.emit_signal(None, path, interface_name, signal_name, parameters)

		self._at_exit(wrapper.SignalEmitted.connect(func).__exit__)

		ids = [con.register_object(path, interface, wrapper.call_method, wrapper.get_property, wrapper.set_property) for interface in interfaces]
		self._at_exit(lambda: [con.unregister_object(id) for id in ids])

class RegistrationMixin:
	__slots__ = ()

	def register_object(self, path, object, node_info):
		if node_info is None:
			try:
				node_info = type(object).dbus
			except AttributeError:
				node_info = type(object).__doc__

		if type(node_info) != list and type(node_info) != tuple:
			node_info = [node_info]

		node_info = [Gio.DBusNodeInfo.new_for_xml(ni) for ni in node_info]
		interfaces = sum((ni.interfaces for ni in node_info), [])

		wrapper = ObjectWrapper(object, interfaces)
		return ObjectRegistration(self.con, path, interfaces, wrapper, own_wrapper=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import mutagen
import os
import copy
from collections     import namedtuple
from mutagen.easymp4 import EasyMP4KeyError
from mutagen.easyid3 import EasyID3KeyError

from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
from media.monitor.log        import Loggable
from media.monitor.pure       import format_length, truncate_to_length
import media.monitor.pure as mmp

# emf related stuff
from media.metadata.process import global_reader
import media.metadata.definitions as defs
defs.load_definitions()

"""
list of supported easy tags in mutagen version 1.20
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm',
'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak',
'organization']
"""

airtime2mutagen = {
    "MDATA_KEY_TITLE"       : "title",
    "MDATA_KEY_CREATOR"     : "artist",
    "MDATA_KEY_SOURCE"      : "album",
    "MDATA_KEY_GENRE"       : "genre",
    "MDATA_KEY_MOOD"        : "mood",
    "MDATA_KEY_TRACKNUMBER" : "tracknumber",
    "MDATA_KEY_BPM"         : "bpm",
    "MDATA_KEY_LABEL"       : "label",
    "MDATA_KEY_COMPOSER"    : "composer",
    "MDATA_KEY_ENCODER"     : "encodedby",
    "MDATA_KEY_CONDUCTOR"   : "conductor",
    "MDATA_KEY_YEAR"        : "date",
    "MDATA_KEY_URL"         : "website",
    "MDATA_KEY_ISRC"        : "isrc",
    "MDATA_KEY_COPYRIGHT"   : "copyright",
}

class FakeMutagen(dict):
    """
    Need this fake mutagen object so that airtime_special functions
    return a proper default value instead of throwing an exceptions for
    files that mutagen doesn't recognize
    """
    FakeInfo = namedtuple('FakeInfo','length bitrate')
    def __init__(self,path):
        self.path = path
        self.mime = ['audio/wav']
        self.info = FakeMutagen.FakeInfo(0.0, '')
        dict.__init__(self)
    def set_length(self,l):
        old_bitrate = self.info.bitrate
        self.info = FakeMutagen.FakeInfo(l, old_bitrate)

# Some airtime attributes are special because they must use the mutagen object
# itself to calculate the value that they need. The lambda associated with each
# key should attempt to extract the corresponding value from the mutagen object
# itself pass as 'm'. In the case when nothing can be extracted the lambda
# should return some default value to be assigned anyway or None so that the
# airtime metadata object will skip the attribute outright.

airtime_special = {
    "MDATA_KEY_DURATION" :
        lambda m: format_length(getattr(m.info, u'length', 0.0)),
    "MDATA_KEY_BITRATE" :
        lambda m: getattr(m.info, "bitrate", ''),
    "MDATA_KEY_SAMPLERATE" :
        lambda m: getattr(m.info, u'sample_rate', 0),
    "MDATA_KEY_MIME" :
        lambda m: m.mime[0] if len(m.mime) > 0 else u'',
}
mutagen2airtime = dict( (v,k) for k,v in airtime2mutagen.iteritems()
        if isinstance(v, str) )

truncate_table = {
        'MDATA_KEY_GENRE'     : 64,
        'MDATA_KEY_TITLE'     : 512,
        'MDATA_KEY_CREATOR'   : 512,
        'MDATA_KEY_SOURCE'    : 512,
        'MDATA_KEY_MOOD'      : 64,
        'MDATA_KEY_LABEL'     : 512,
        'MDATA_KEY_COMPOSER'  : 512,
        'MDATA_KEY_ENCODER'   : 255,
        'MDATA_KEY_CONDUCTOR' : 512,
        'MDATA_KEY_YEAR'      : 16,
        'MDATA_KEY_URL'       : 512,
        'MDATA_KEY_ISRC'      : 512,
        'MDATA_KEY_COPYRIGHT' : 512,
}

class Metadata(Loggable):
    # TODO : refactor the way metadata is being handled. Right now things are a
    # little bit messy. Some of the handling is in m.m.pure while the rest is
    # here. Also interface is not very consistent

    @staticmethod
    def fix_title(path):
        # If we have no title in path we will format it
        # TODO : this is very hacky so make sure to fix it
        m = mutagen.File(path, easy=True)
        if u'title' not in m:
            new_title = unicode( mmp.no_extension_basename(path) )
            m[u'title'] = new_title
            m.save()

    @staticmethod
    def airtime_dict(d):
        """
        Converts mutagen dictionary 'd' into airtime dictionary
        """
        temp_dict = {}
        for m_key, m_val in d.iteritems():
            # TODO : some files have multiple fields for the same metadata.
            # genre is one example. In that case mutagen will return a list
            # of values

            if isinstance(m_val, list):
                # TODO : does it make more sense to just skip the element in
                # this case?
                if len(m_val) == 0: assign_val = ''
                else: assign_val = m_val[0]
            else: assign_val = m_val

            temp_dict[ m_key ] = assign_val
        airtime_dictionary = {}
        for muta_k, muta_v in temp_dict.iteritems():
            # We must check if we can actually translate the mutagen key into
            # an airtime key before doing the conversion
            if muta_k in mutagen2airtime:
                airtime_key = mutagen2airtime[muta_k]
                # Apply truncation in the case where airtime_key is in our
                # truncation table
                muta_v =  \
                        truncate_to_length(muta_v, truncate_table[airtime_key])\
                        if airtime_key in truncate_table else muta_v
                airtime_dictionary[ airtime_key ] = muta_v
        return airtime_dictionary

    @staticmethod
    def write_unsafe(path,md):
        """
        Writes 'md' metadata into 'path' through mutagen. Converts all
        dictionary values to strings because mutagen will not write anything
        else
        """
        if not os.path.exists(path): raise BadSongFile(path)
        song_file = mutagen.File(path, easy=True)
        exceptions = [] # for bad keys
        for airtime_k, airtime_v in md.iteritems():
            if airtime_k in airtime2mutagen:
                # The unicode cast here is mostly for integers that need to be
                # strings
                try:
                    song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v)
                except (EasyMP4KeyError, EasyID3KeyError) as e:
                    exceptions.append(InvalidMetadataElement(e, airtime_k,
                        path))
        song_file.save()
        # bubble dem up so that user knows that something is wrong
        for e in exceptions: raise e

    def __init__(self, fpath):
        self.__metadata = global_reader.read_mutagen(fpath)

    def __init__2(self, fpath):
        # Forcing the unicode through
        try    : fpath = fpath.decode("utf-8")
        except : pass

        if not mmp.file_playable(fpath): raise BadSongFile(fpath)

        try              : full_mutagen  = mutagen.File(fpath, easy=True)
        except Exception : raise BadSongFile(fpath)

        self.path = fpath
        if not os.path.exists(self.path):
            self.logger.info("Attempting to read metadata of file \
                    that does not exist. Setting metadata to {}")
            self.__metadata = {}
            return
        # TODO : Simplify the way all of these rules are handled right now it's
        # extremely unclear and needs to be refactored.
        #if full_mutagen is None: raise BadSongFile(fpath)

        if full_mutagen is None: full_mutagen = FakeMutagen(fpath)
        self.__metadata = Metadata.airtime_dict(full_mutagen)
        # Now we extra the special values that are calculated from the mutagen
        # object itself:

        if mmp.extension(fpath) == 'wav':
            full_mutagen.set_length(mmp.read_wave_duration(fpath))

        for special_key,f in airtime_special.iteritems():
            try:
                new_val = f(full_mutagen)
                if new_val is not None:
                    self.__metadata[special_key] = new_val
            except Exception as e:
                self.logger.info("Could not get special key %s for %s" %
                        (special_key, fpath))
                self.logger.info(str(e))
        # Finally, we "normalize" all the metadata here:
        self.__metadata = mmp.normalized_metadata(self.__metadata, fpath)
        # Now we must load the md5:
        # TODO : perhaps we shouldn't hard code how many bytes we're reading
        # from the file?
        self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100)

        #try: # emf stuff for testing:
            #if full_mutagen:
                #normalized = global_reader.read_mutagen(fpath)
                #self.logger.info("EMF--------------------")
                #self.logger.info(pformat(normalized))
                #self.logger.info("OLD--------------------")
                #self.logger.info(pformat(self.__metadata))
                #self.logger.info("-----------------------")

        #except Exception as e:
            #self.unexpected_exception(e)

    def is_recorded(self):
        """
        returns true if the file has been created by airtime through recording
        """
        return mmp.is_airtime_recorded( self.__metadata )

    def extract(self):
        """
        returns a copy of the metadata that was loaded when object was
        constructed
        """
        return copy.deepcopy(self.__metadata)

    def utf8(self):
        """
        Returns a unicode aware representation of the data that is compatible
        with what is spent to airtime
        """
        return mmp.convert_dict_value_to_utf8(self.extract())
<commit_msg>Removed old commented code<commit_after># -*- coding: utf-8 -*-
import mutagen
import os
import copy
from collections     import namedtuple
from mutagen.easymp4 import EasyMP4KeyError
from mutagen.easyid3 import EasyID3KeyError

from media.monitor.exceptions import BadSongFile, InvalidMetadataElement
from media.monitor.log        import Loggable
from media.monitor.pure       import format_length, truncate_to_length
import media.monitor.pure as mmp

# emf related stuff
from media.metadata.process import global_reader
import media.metadata.definitions as defs
defs.load_definitions()

"""
list of supported easy tags in mutagen version 1.20
['albumartistsort', 'musicbrainz_albumstatus', 'lyricist', 'releasecountry',
'date', 'performer', 'musicbrainz_albumartistid', 'composer', 'encodedby',
'tracknumber', 'musicbrainz_albumid', 'album', 'asin', 'musicbrainz_artistid',
'mood', 'copyright', 'author', 'media', 'length', 'version', 'artistsort',
'titlesort', 'discsubtitle', 'website', 'musicip_fingerprint', 'conductor',
'compilation', 'barcode', 'performer:*', 'composersort', 'musicbrainz_discid',
'musicbrainz_albumtype', 'genre', 'isrc', 'discnumber', 'musicbrainz_trmid',
'replaygain_*_gain', 'musicip_puid', 'artist', 'title', 'bpm',
'musicbrainz_trackid', 'arranger', 'albumsort', 'replaygain_*_peak',
'organization']
"""

airtime2mutagen = {
    "MDATA_KEY_TITLE"       : "title",
    "MDATA_KEY_CREATOR"     : "artist",
    "MDATA_KEY_SOURCE"      : "album",
    "MDATA_KEY_GENRE"       : "genre",
    "MDATA_KEY_MOOD"        : "mood",
    "MDATA_KEY_TRACKNUMBER" : "tracknumber",
    "MDATA_KEY_BPM"         : "bpm",
    "MDATA_KEY_LABEL"       : "label",
    "MDATA_KEY_COMPOSER"    : "composer",
    "MDATA_KEY_ENCODER"     : "encodedby",
    "MDATA_KEY_CONDUCTOR"   : "conductor",
    "MDATA_KEY_YEAR"        : "date",
    "MDATA_KEY_URL"         : "website",
    "MDATA_KEY_ISRC"        : "isrc",
    "MDATA_KEY_COPYRIGHT"   : "copyright",
}

class FakeMutagen(dict):
    """
    Need this fake mutagen object so that airtime_special functions
    return a proper default value instead of throwing an exceptions for
    files that mutagen doesn't recognize
    """
    FakeInfo = namedtuple('FakeInfo','length bitrate')
    def __init__(self,path):
        self.path = path
        self.mime = ['audio/wav']
        self.info = FakeMutagen.FakeInfo(0.0, '')
        dict.__init__(self)
    def set_length(self,l):
        old_bitrate = self.info.bitrate
        self.info = FakeMutagen.FakeInfo(l, old_bitrate)

# Some airtime attributes are special because they must use the mutagen object
# itself to calculate the value that they need. The lambda associated with each
# key should attempt to extract the corresponding value from the mutagen object
# itself pass as 'm'. In the case when nothing can be extracted the lambda
# should return some default value to be assigned anyway or None so that the
# airtime metadata object will skip the attribute outright.

airtime_special = {
    "MDATA_KEY_DURATION" :
        lambda m: format_length(getattr(m.info, u'length', 0.0)),
    "MDATA_KEY_BITRATE" :
        lambda m: getattr(m.info, "bitrate", ''),
    "MDATA_KEY_SAMPLERATE" :
        lambda m: getattr(m.info, u'sample_rate', 0),
    "MDATA_KEY_MIME" :
        lambda m: m.mime[0] if len(m.mime) > 0 else u'',
}
mutagen2airtime = dict( (v,k) for k,v in airtime2mutagen.iteritems()
        if isinstance(v, str) )

truncate_table = {
        'MDATA_KEY_GENRE'     : 64,
        'MDATA_KEY_TITLE'     : 512,
        'MDATA_KEY_CREATOR'   : 512,
        'MDATA_KEY_SOURCE'    : 512,
        'MDATA_KEY_MOOD'      : 64,
        'MDATA_KEY_LABEL'     : 512,
        'MDATA_KEY_COMPOSER'  : 512,
        'MDATA_KEY_ENCODER'   : 255,
        'MDATA_KEY_CONDUCTOR' : 512,
        'MDATA_KEY_YEAR'      : 16,
        'MDATA_KEY_URL'       : 512,
        'MDATA_KEY_ISRC'      : 512,
        'MDATA_KEY_COPYRIGHT' : 512,
}

class Metadata(Loggable):
    # TODO : refactor the way metadata is being handled. Right now things are a
    # little bit messy. Some of the handling is in m.m.pure while the rest is
    # here. Also interface is not very consistent

    @staticmethod
    def fix_title(path):
        # If we have no title in path we will format it
        # TODO : this is very hacky so make sure to fix it
        m = mutagen.File(path, easy=True)
        if u'title' not in m:
            new_title = unicode( mmp.no_extension_basename(path) )
            m[u'title'] = new_title
            m.save()

    @staticmethod
    def airtime_dict(d):
        """
        Converts mutagen dictionary 'd' into airtime dictionary
        """
        temp_dict = {}
        for m_key, m_val in d.iteritems():
            # TODO : some files have multiple fields for the same metadata.
            # genre is one example. In that case mutagen will return a list
            # of values

            if isinstance(m_val, list):
                # TODO : does it make more sense to just skip the element in
                # this case?
                if len(m_val) == 0: assign_val = ''
                else: assign_val = m_val[0]
            else: assign_val = m_val

            temp_dict[ m_key ] = assign_val
        airtime_dictionary = {}
        for muta_k, muta_v in temp_dict.iteritems():
            # We must check if we can actually translate the mutagen key into
            # an airtime key before doing the conversion
            if muta_k in mutagen2airtime:
                airtime_key = mutagen2airtime[muta_k]
                # Apply truncation in the case where airtime_key is in our
                # truncation table
                muta_v =  \
                        truncate_to_length(muta_v, truncate_table[airtime_key])\
                        if airtime_key in truncate_table else muta_v
                airtime_dictionary[ airtime_key ] = muta_v
        return airtime_dictionary

    @staticmethod
    def write_unsafe(path,md):
        """
        Writes 'md' metadata into 'path' through mutagen. Converts all
        dictionary values to strings because mutagen will not write anything
        else
        """
        if not os.path.exists(path): raise BadSongFile(path)
        song_file = mutagen.File(path, easy=True)
        exceptions = [] # for bad keys
        for airtime_k, airtime_v in md.iteritems():
            if airtime_k in airtime2mutagen:
                # The unicode cast here is mostly for integers that need to be
                # strings
                try:
                    song_file[ airtime2mutagen[airtime_k] ] = unicode(airtime_v)
                except (EasyMP4KeyError, EasyID3KeyError) as e:
                    exceptions.append(InvalidMetadataElement(e, airtime_k,
                        path))
        song_file.save()
        # bubble dem up so that user knows that something is wrong
        for e in exceptions: raise e

    def __init__(self, fpath):
        self.__metadata = global_reader.read_mutagen(fpath)

    def __init__2(self, fpath):
        # Forcing the unicode through
        try    : fpath = fpath.decode("utf-8")
        except : pass

        if not mmp.file_playable(fpath): raise BadSongFile(fpath)

        try              : full_mutagen  = mutagen.File(fpath, easy=True)
        except Exception : raise BadSongFile(fpath)

        self.path = fpath
        if not os.path.exists(self.path):
            self.logger.info("Attempting to read metadata of file \
                    that does not exist. Setting metadata to {}")
            self.__metadata = {}
            return
        # TODO : Simplify the way all of these rules are handled right now it's
        # extremely unclear and needs to be refactored.
        #if full_mutagen is None: raise BadSongFile(fpath)

        if full_mutagen is None: full_mutagen = FakeMutagen(fpath)
        self.__metadata = Metadata.airtime_dict(full_mutagen)
        # Now we extra the special values that are calculated from the mutagen
        # object itself:

        if mmp.extension(fpath) == 'wav':
            full_mutagen.set_length(mmp.read_wave_duration(fpath))

        for special_key,f in airtime_special.iteritems():
            try:
                new_val = f(full_mutagen)
                if new_val is not None:
                    self.__metadata[special_key] = new_val
            except Exception as e:
                self.logger.info("Could not get special key %s for %s" %
                        (special_key, fpath))
                self.logger.info(str(e))
        # Finally, we "normalize" all the metadata here:
        self.__metadata = mmp.normalized_metadata(self.__metadata, fpath)
        # Now we must load the md5:
        # TODO : perhaps we shouldn't hard code how many bytes we're reading
        # from the file?
        self.__metadata['MDATA_KEY_MD5'] = mmp.file_md5(fpath,max_length=100)

    def is_recorded(self):
        """
        returns true if the file has been created by airtime through recording
        """
        return mmp.is_airtime_recorded( self.__metadata )

    def extract(self):
        """
        returns a copy of the metadata that was loaded when object was
        constructed
        """
        return copy.deepcopy(self.__metadata)

    def utf8(self):
        """
        Returns a unicode aware representation of the data that is compatible
        with what is spent to airtime
        """
        return mmp.convert_dict_value_to_utf8(self.extract())
<|endoftext|>