<commit_before>import sqlalchemy
import sqlalchemy.orm as orm
import sqlalchemy.ext.declarative as declarative
import geoalchemy_util
import itertools
import os
import datetime

import alchemy
alchemy_config = alchemy.get_config()
alchemy_session = alchemy.fetch_session()
base = declarative.declarative_base()

class RawGoogle(base):
    __tablename__ = 'raw_google'
    id = sqlalchemy.Column("rowid", sqlalchemy.Integer, primary_key=True)
    input_address = sqlalchemy.Column(sqlalchemy.String)
    output_address = sqlalchemy.Column(sqlalchemy.String)
    hierarchy = sqlalchemy.Column(sqlalchemy.String)
    latitude = sqlalchemy.Column(sqlalchemy.REAL)
    longitude = sqlalchemy.Column(sqlalchemy.REAL)

    def __init__(self, input_address, output_address, hierarchy, latitude, longitude):
        self.input_address = input_address
        self.output_address = output_address
        self.hierarchy = hierarchy
        self.latitude = latitude
        self.longitude = longitude

    def __repr__(self):
        return "<RawGoogle('%s','%s','%s','%s','%s')>" % (self.input_address, self.output_address, self.hierarchy, self.latitude, self.longitude)

raw_google_dbpath = os.path.join(
    alchemy_config.get("location").get('path'),
    alchemy_config.get("location").get('database'))
raw_google_engine = sqlalchemy.create_engine('sqlite:///%s' % raw_google_dbpath)
raw_google_session_class = orm.sessionmaker(bind=raw_google_engine)
raw_google_session = raw_google_session_class()

def run_geo_match(key, default, match_group, counter, runtime):
    most_freq = 0
    if key != "nolocationfound":
        if len(match_group) > 1:
            # if key exists, look at the frequency of each item
            # to determine the default summarization
            clean = alchemy_session.query(alchemy.Location).filter(alchemy.Location.id == key).first()
            if clean:
                for loc in clean.rawlocations:
                    freq = len(loc.rawassignees) + len(loc.rawinventors)
                    if freq > most_freq:
                        default.update(loc.summarize)
                        most_freq = freq

            # took a look at the frequency of the items in the match_group
            for loc in match_group:
                freq = len(loc.rawassignees) + len(loc.rawinventors)
                if freq > most_freq:
                    default.update(loc.summarize)
                    most_freq = freq

        alchemy.match(match_group, alchemy_session, default, commit=False)
    if (counter + 1) % alchemy_config.get("location").get("commit_frequency") == 0:
        print " *", (counter + 1), datetime.datetime.now() - runtime
        alchemy_session.commit()

def main(limit=10000, offset=0):
    t = datetime.datetime.now()
    print "geocoding started", t
    #Get all of the raw locations from the XML parsing
    raw_parsed_locations = alchemy_session.query(alchemy.RawLocation).limit(limit).offset(offset)
    if raw_parsed_locations.count() == 0:
        return False
    # raw_parsed_locations = alchemy_session.query(alchemy.RawLocation).filter(alchemy.RawLocation.location_id == None)
    #raw_google_locations = raw_google_session.query(RawGoogle)
    grouped_locations = []
    for instance in raw_parsed_locations:
        #Convert the location into a string that matches the Google format
        parsed_raw_location = geoalchemy_util.concatenate_location(instance.city, instance.state, instance.country)
        cleaned_location = geoalchemy_util.clean_raw_location(parsed_raw_location)
        #Find the location from the raw_google database that matches this input
        matching_location = raw_google_session.query(RawGoogle).filter_by(input_address=cleaned_location).first()
        #alchemy.match(instance)
        if matching_location:
            if(matching_location.latitude != ''):
                grouping_id = u"%s|%s" % (matching_location.latitude, matching_location.longitude)
            else:
                grouping_id = u"nolocationfound"
            grouped_locations.append({"raw_location": instance,
                                      "matching_location": matching_location,

                                      "grouping_id": grouping_id})
    print "grouped_locations created", datetime.datetime.now() - t
    t = datetime.datetime.now()
    #We now have a list of all locations in the file, along with their
    #matching locations and the id used to group them
    #Sort the list by the grouping_id
    keyfunc = lambda x: x['grouping_id']
    grouped_locations.sort(key=keyfunc)
    print "grouped_locations sorted", datetime.datetime.now() - t
    t = datetime.datetime.now()
    #Group by the grouping_id
    for i, item in enumerate(itertools.groupby(grouped_locations, keyfunc)):
        key, grouping = item
        #match_group is the list of RawLocation objects which we call match on
        #We need to get only the RawLocation objects back from the dict
        match_group = []
        #Get the latitude and longitude for the group
        splitkey = key.split('|')

        #The length should be 2
        default = {"id": key}
        if len(splitkey) == 2:
            default.update({"latitude": splitkey[0], "longitude": splitkey[1]})
        for grouped_location in grouping:
            match_group.append(grouped_location["raw_location"])

        # determine most frequent if list of match_group
        run_geo_match(key, default, match_group, i, t)
    alchemy_session.commit()

    print "Matches made!", datetime.datetime.now() - t
    print "%s groups formed from %s locations" % (len(grouped_locations), raw_parsed_locations.count())
<commit_msg>just find top query (no iteration needed)<commit_after>import sqlalchemy
import sqlalchemy.orm as orm
import sqlalchemy.ext.declarative as declarative
import geoalchemy_util
import itertools
import os
import datetime

import alchemy
alchemy_config = alchemy.get_config()
alchemy_session = alchemy.fetch_session()
base = declarative.declarative_base()

class RawGoogle(base):
    __tablename__ = 'raw_google'
    id = sqlalchemy.Column("rowid", sqlalchemy.Integer, primary_key=True)
    input_address = sqlalchemy.Column(sqlalchemy.String)
    output_address = sqlalchemy.Column(sqlalchemy.String)
    hierarchy = sqlalchemy.Column(sqlalchemy.String)
    latitude = sqlalchemy.Column(sqlalchemy.REAL)
    longitude = sqlalchemy.Column(sqlalchemy.REAL)

    def __init__(self, input_address, output_address, hierarchy, latitude, longitude):
        self.input_address = input_address
        self.output_address = output_address
        self.hierarchy = hierarchy
        self.latitude = latitude
        self.longitude = longitude

    def __repr__(self):
        return "<RawGoogle('%s','%s','%s','%s','%s')>" % (self.input_address, self.output_address, self.hierarchy, self.latitude, self.longitude)

raw_google_dbpath = os.path.join(
    alchemy_config.get("location").get('path'),
    alchemy_config.get("location").get('database'))
raw_google_engine = sqlalchemy.create_engine('sqlite:///%s' % raw_google_dbpath)
raw_google_session_class = orm.sessionmaker(bind=raw_google_engine)
raw_google_session = raw_google_session_class()

def run_geo_match(key, default, match_group, counter, runtime):
    most_freq = 0
    if key != "nolocationfound":
        if len(match_group) > 1:
            # if key exists, look at the frequency
            # to determine the default summarization
            clean = alchemy_session.query(alchemy.Location).filter(alchemy.Location.id == key).first()
            if clean:
                param = clean.summarize
                param.pop("id")
                param.pop("latitude")
                param.pop("longitude")
                loc = alchemy_session.query(alchemy.RawLocation)\
                    .filter(alchemy.RawLocation.city == param["city"])\
                    .filter(alchemy.RawLocation.state == param["state"])\
                    .filter(alchemy.RawLocation.country == param["country"])\
                    .first()
                most_freq = len(loc.rawassignees) + len(loc.rawinventors)
                default.update(param)

            # took a look at the frequency of the items in the match_group
            for loc in match_group:
                freq = len(loc.rawassignees) + len(loc.rawinventors)
                if freq > most_freq:
                    default.update(loc.summarize)
                    most_freq = freq

        alchemy.match(match_group, alchemy_session, default, commit=False)
    if (counter + 1) % alchemy_config.get("location").get("commit_frequency") == 0:
        print " *", (counter + 1), datetime.datetime.now() - runtime
        alchemy_session.commit()

def main(limit=10000, offset=0):
    t = datetime.datetime.now()
    print "geocoding started", t
    #Get all of the raw locations from the XML parsing
    raw_parsed_locations = alchemy_session.query(alchemy.RawLocation).limit(limit).offset(offset)
    if raw_parsed_locations.count() == 0:
        return False
    # raw_parsed_locations = alchemy_session.query(alchemy.RawLocation).filter(alchemy.RawLocation.location_id == None)
    #raw_google_locations = raw_google_session.query(RawGoogle)
    grouped_locations = []
    for instance in raw_parsed_locations:
        #Convert the location into a string that matches the Google format
        parsed_raw_location = geoalchemy_util.concatenate_location(instance.city, instance.state, instance.country)
        cleaned_location = geoalchemy_util.clean_raw_location(parsed_raw_location)
        #Find the location from the raw_google database that matches this input
        matching_location = raw_google_session.query(RawGoogle).filter_by(input_address=cleaned_location).first()
        #alchemy.match(instance)
        if matching_location:
            if(matching_location.latitude != ''):
                grouping_id = u"%s|%s" % (matching_location.latitude, matching_location.longitude)
            else:
                grouping_id = u"nolocationfound"
            grouped_locations.append({"raw_location": instance,
                                      "matching_location": matching_location,

                                      "grouping_id": grouping_id})
    print "grouped_locations created", datetime.datetime.now() - t
    t = datetime.datetime.now()
    #We now have a list of all locations in the file, along with their
    #matching locations and the id used to group them
    #Sort the list by the grouping_id
    keyfunc = lambda x: x['grouping_id']
    grouped_locations.sort(key=keyfunc)
    print "grouped_locations sorted", datetime.datetime.now() - t
    t = datetime.datetime.now()
    #Group by the grouping_id
    for i, item in enumerate(itertools.groupby(grouped_locations, keyfunc)):
        key, grouping = item
        #match_group is the list of RawLocation objects which we call match on
        #We need to get only the RawLocation objects back from the dict
        match_group = []
        #Get the latitude and longitude for the group
        splitkey = key.split('|')

        #The length should be 2
        default = {"id": key}
        if len(splitkey) == 2:
            default.update({"latitude": splitkey[0], "longitude": splitkey[1]})
        for grouped_location in grouping:
            match_group.append(grouped_location["raw_location"])

        # determine most frequent if list of match_group
        run_geo_match(key, default, match_group, i, t)
    alchemy_session.commit()

    print "Matches made!", datetime.datetime.now() - t
    print "%s groups formed from %s locations" % (len(grouped_locations), raw_parsed_locations.count())
<|endoftext|>


<commit_before>load("//tools/bzl:maven_jar.bzl", "MAVEN_CENTRAL", "maven_jar")

_JGIT_VERS = "4.11.8.201904181247-r"

_DOC_VERS = _JGIT_VERS  # Set to _JGIT_VERS unless using a snapshot

JGIT_DOC_URL = "https://download.eclipse.org/jgit/site/" + _DOC_VERS + "/apidocs"

_JGIT_REPO = MAVEN_CENTRAL  # Leave here even if set to MAVEN_CENTRAL.

# set this to use a local version.
# "/home/<user>/projects/jgit"
LOCAL_JGIT_REPO = ""

def jgit_repos():
    if LOCAL_JGIT_REPO:
        native.local_repository(
            name = "jgit",
            path = LOCAL_JGIT_REPO,
        )
        jgit_maven_repos_dev()
    else:
        jgit_maven_repos()

def jgit_maven_repos_dev():
    # Transitive dependencies from JGit's WORKSPACE.
    maven_jar(
        name = "hamcrest-library",
        artifact = "org.hamcrest:hamcrest-library:1.3",
        sha1 = "4785a3c21320980282f9f33d0d1264a69040538f",
    )

def jgit_maven_repos():
    maven_jar(
        name = "jgit-lib",
        artifact = "org.eclipse.jgit:org.eclipse.jgit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "d7bc31008c827e63d33b14c68eb149dd2087e41b",
        src_sha1 = "111ac9eb58c42673f560fa242ea31aec3d0eb7d3",
        unsign = True,
    )
    maven_jar(
        name = "jgit-servlet",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.http.server:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "ceefadfc63548bce8d3eedf8656759cc1c41b1ee",
        unsign = True,
    )
    maven_jar(
        name = "jgit-archive",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.archive:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "10565873a029aad92b18908786a0e097df975175",
    )
    maven_jar(
        name = "jgit-junit",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.junit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "b8c33396d6083f357cc5ac820814a55aac17f862",
        unsign = True,
    )

def jgit_dep(name):
    mapping = {
        "@jgit-archive//jar": "@jgit//org.eclipse.jgit.archive:jgit-archive",
        "@jgit-junit//jar": "@jgit//org.eclipse.jgit.junit:junit",
        "@jgit-lib//jar": "@jgit//org.eclipse.jgit:jgit",
        "@jgit-lib//jar:src": "@jgit//org.eclipse.jgit:libjgit-src.jar",
        "@jgit-servlet//jar": "@jgit//org.eclipse.jgit.http.server:jgit-servlet",
    }

    if LOCAL_JGIT_REPO:
        return mapping[name]
    else:
        return name
<commit_msg>Upgrade JGit to 4.11.9.201909030838-r<commit_after>load("//tools/bzl:maven_jar.bzl", "MAVEN_CENTRAL", "maven_jar")

_JGIT_VERS = "4.11.9.201909030838-r"

_DOC_VERS = _JGIT_VERS  # Set to _JGIT_VERS unless using a snapshot

JGIT_DOC_URL = "https://download.eclipse.org/jgit/site/" + _DOC_VERS + "/apidocs"

_JGIT_REPO = MAVEN_CENTRAL  # Leave here even if set to MAVEN_CENTRAL.

# set this to use a local version.
# "/home/<user>/projects/jgit"
LOCAL_JGIT_REPO = ""

def jgit_repos():
    if LOCAL_JGIT_REPO:
        native.local_repository(
            name = "jgit",
            path = LOCAL_JGIT_REPO,
        )
        jgit_maven_repos_dev()
    else:
        jgit_maven_repos()

def jgit_maven_repos_dev():
    # Transitive dependencies from JGit's WORKSPACE.
    maven_jar(
        name = "hamcrest-library",
        artifact = "org.hamcrest:hamcrest-library:1.3",
        sha1 = "4785a3c21320980282f9f33d0d1264a69040538f",
    )

def jgit_maven_repos():
    maven_jar(
        name = "jgit-lib",
        artifact = "org.eclipse.jgit:org.eclipse.jgit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "3bc74ffed6186bf2fc37404216e5ef16f904d0b0",
        src_sha1 = "ddfdec70e78d145e7d99c8d72286cb714b0239ae",
        unsign = True,
    )
    maven_jar(
        name = "jgit-servlet",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.http.server:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "a870e53f414992a548264758bdb17c74f6b79e19",
        unsign = True,
    )
    maven_jar(
        name = "jgit-archive",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.archive:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "65bcf563c3155f9555735992f6493f3fe35b13a2",
    )
    maven_jar(
        name = "jgit-junit",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.junit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "c7a8883d5431149d1d3b9282eda3d79efffd319e",
        unsign = True,
    )

def jgit_dep(name):
    mapping = {
        "@jgit-archive//jar": "@jgit//org.eclipse.jgit.archive:jgit-archive",
        "@jgit-junit//jar": "@jgit//org.eclipse.jgit.junit:junit",
        "@jgit-lib//jar": "@jgit//org.eclipse.jgit:jgit",
        "@jgit-lib//jar:src": "@jgit//org.eclipse.jgit:libjgit-src.jar",
        "@jgit-servlet//jar": "@jgit//org.eclipse.jgit.http.server:jgit-servlet",
    }

    if LOCAL_JGIT_REPO:
        return mapping[name]
    else:
        return name
<|endoftext|>


<commit_before>load("//tools/bzl:maven_jar.bzl", "GERRIT", "MAVEN_LOCAL", "MAVEN_CENTRAL", "maven_jar")

_JGIT_VERS = "4.9.2.201712150930-r.171-gfdbaa25db"

_DOC_VERS = "4.9.2.201712150930-r"  # Set to _JGIT_VERS unless using a snapshot

JGIT_DOC_URL = "http://download.eclipse.org/jgit/site/" + _DOC_VERS + "/apidocs"

_JGIT_REPO = GERRIT  # Leave here even if set to MAVEN_CENTRAL.

# set this to use a local version.
# "/home/<user>/projects/jgit"
LOCAL_JGIT_REPO = ""

def jgit_repos():
  if LOCAL_JGIT_REPO:
    native.local_repository(
        name = "jgit",
        path = LOCAL_JGIT_REPO,
    )
  else:
    jgit_maven_repos()

def jgit_maven_repos():
    maven_jar(
        name = "jgit_lib",
        artifact = "org.eclipse.jgit:org.eclipse.jgit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "29b822410b29286a09df728f8379e5cb8b1a486e",
        src_sha1 = "5106b81910a057470cfd2584d9cb3502bcbebbc2",
        unsign = True,
    )
    maven_jar(
        name = "jgit_servlet",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.http.server:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "01f6718f6b629e28caad38e00190811b38574e74",
        unsign = True,
    )
    maven_jar(
        name = "jgit_archive",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.archive:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "9c9e9332e7dc724dbe1837e21feccd98bc25e6b4",
    )
    maven_jar(
        name = "jgit_junit",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.junit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "4154c70b78b62035dad446332b24f7816b7a2a1b",
        unsign = True,
    )

def jgit_dep(name):
  mapping = {
      "@jgit_junit//jar": "@jgit//org.eclipse.jgit.junit:junit",
      "@jgit_lib//jar:src": "@jgit//org.eclipse.jgit:libjgit-src.jar",
      "@jgit_lib//jar": "@jgit//org.eclipse.jgit:jgit",
      "@jgit_servlet//jar":"@jgit//org.eclipse.jgit.http.server:jgit-servlet",
      "@jgit_archive//jar": "@jgit//org.eclipse.jgit.archive:jgit-archive",
  }

  if LOCAL_JGIT_REPO:
    return mapping[name]
  else:
    return name
<commit_msg>Update JGit to 4.9.2.201712150930-r.175-gd8a24ac1c<commit_after>load("//tools/bzl:maven_jar.bzl", "GERRIT", "MAVEN_LOCAL", "MAVEN_CENTRAL", "maven_jar")

_JGIT_VERS = "4.9.2.201712150930-r.175-gd8a24ac1c"

_DOC_VERS = "4.9.2.201712150930-r"  # Set to _JGIT_VERS unless using a snapshot

JGIT_DOC_URL = "http://download.eclipse.org/jgit/site/" + _DOC_VERS + "/apidocs"

_JGIT_REPO = GERRIT  # Leave here even if set to MAVEN_CENTRAL.

# set this to use a local version.
# "/home/<user>/projects/jgit"
LOCAL_JGIT_REPO = ""

def jgit_repos():
  if LOCAL_JGIT_REPO:
    native.local_repository(
        name = "jgit",
        path = LOCAL_JGIT_REPO,
    )
  else:
    jgit_maven_repos()

def jgit_maven_repos():
    maven_jar(
        name = "jgit_lib",
        artifact = "org.eclipse.jgit:org.eclipse.jgit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "4286555f5851fbfcf0ff89ec884f7f806b0c7e37",
        src_sha1 = "5e38b7e7936ebbd778914dc4f9d76d245a5a4518",
        unsign = True,
    )
    maven_jar(
        name = "jgit_servlet",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.http.server:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "da0d2c7a048cc213274cd06a5baf277c85ea152e",
        unsign = True,
    )
    maven_jar(
        name = "jgit_archive",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.archive:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "1cd91bedf8b591626d341c2d896181ddba5f9aa9",
    )
    maven_jar(
        name = "jgit_junit",
        artifact = "org.eclipse.jgit:org.eclipse.jgit.junit:" + _JGIT_VERS,
        repository = _JGIT_REPO,
        sha1 = "5b7cc1aa0ba062ad587b6daa64743b704b997f74",
        unsign = True,
    )

def jgit_dep(name):
  mapping = {
      "@jgit_junit//jar": "@jgit//org.eclipse.jgit.junit:junit",
      "@jgit_lib//jar:src": "@jgit//org.eclipse.jgit:libjgit-src.jar",
      "@jgit_lib//jar": "@jgit//org.eclipse.jgit:jgit",
      "@jgit_servlet//jar":"@jgit//org.eclipse.jgit.http.server:jgit-servlet",
      "@jgit_archive//jar": "@jgit//org.eclipse.jgit.archive:jgit-archive",
  }

  if LOCAL_JGIT_REPO:
    return mapping[name]
  else:
    return name
<|endoftext|>


<commit_before>from lib.commands import *

def apply_pre_function(pre_function, value):
    result = value

    if pre_function is not None:
        try:
            if type(value) is list:
                result = [apply_safe(pre_function, x) if type(x) is not list else x for x in value]
            else:
                result = apply_safe(pre_function, value)
        except:
            pass

    return result

def vectorized_evaluation(a, b, function, pre_function=None):
    """
    Uses the given function in the form of a lambda and produces a vectorized result if
    at least one of the items is an iterable value. Otherwise, it evaluates it as normal.
    :param a: The first value for the function
    :param b: The second value for the function
    :param function: A 2-arity lambda used for the function
    :param pre_function: A function that is called before vectorization (optional)
    :return: A value or array of values depending on a and b
    """

    try:
        a = apply_pre_function(pre_function, a)
        b = apply_pre_function(pre_function, b)

        # When both are lists
        if type(a) is list and type(b) is list:

            # Get the minimum and maximum of both lists in order to prevent index errors
            vector_range = min(len(a), len(b))
            max_range = max(len(a), len(b))
            vectorized_result = []

            # Compute the function for all in range elements
            for index in range(0, vector_range):
                vectorized_result.append(apply_safe(function, a[index], b[index]))

            # Append all out of range elements without being processed
            for index in range(vector_range, max_range):
                if len(a) == max_range:
                    vectorized_result.append(a[index])
                else:
                    vectorized_result.append(b[index])

            return vectorized_result

        # When only type a is a list
        elif type(a) is list and type(b) is not list:

            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    vectorized_evaluation(element, b, function, pre_function)
                )

            return vectorized_result

        # When only type b is a list
        elif type(a) is not list and type(b) is list:

            vectorized_result = []
            for element in b:
                vectorized_result.append(
                    vectorized_evaluation(a, element, function, pre_function)
                )

            return vectorized_result

        return apply_safe(function, a, b)

    except:
        if type(a) is list and type(b) is not list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    vectorized_evaluation(element, b, function, pre_function)
                )
            return vectorized_result
        elif type(a) is not list and type(b) is list:
            vectorized_result = []
            for element in b:
                vectorized_result.append(
                    vectorized_evaluation(a, element, function, pre_function)
                )
            return vectorized_result

        raise Exception

def single_vectorized_evaluation(a, function, pre_function=None):
    try:
        a = apply_pre_function(pre_function, a)

        if type(a) is list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    single_vectorized_evaluation(element, function, pre_function)
                )

            return vectorized_result

        return apply_safe(function, a)
    except:
        if type(a) is list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    single_vectorized_evaluation(element, function, pre_function)
                )
            return vectorized_result

        raise Exception

# Vectorized aggregation
# The "function" argument takes a function(accumulator, value), where:
#   * accumulator is the result of previously aggregated values
#   * value is the current value to aggregate
#
# The "start" argument is the starting value of the aggregation
# If not provided, the starting point will be the first value to pass the
# pre_function check without an exception been raised
#
# Result will keep the same order of values/sublists than input
# Aggregated value will be at the first non-list valid value
# encountered in the input
def vectorized_aggregator(a, function, pre_function=None, start=None):
    index = -1
    subresults = []
    values = []

    result = list(start) if start is list else start

    if type(a) is not list:
        a = str(a)

    for i in range(len(a)):
        if type(a[i]) is list:
            subresults.append(vectorized_aggregator(a[i], function, pre_function, start))
        else:
            try:
                if pre_function is not None:
                    a[i] = pre_function(a[i])
                if index == -1:
                    index = i
                if result is None:
                    result = a[i]
                else:
                    values.append(a[i])
            except:
                pass

    for i in values:
        result = apply_safe(function, result, i)

    if subresults:
        if index > -1:
            subresults.insert(index, result)
        result = subresults

    return result if result is not None else []

# Vectorized filter
# Keep only elements of a where function(a) is truthy
def vectorized_filter(a, function, pre_function=None):
    a = apply_pre_function(pre_function, a)

    vectorized_result = []
    for element in a:
        if type(element) is list:
            vectorized_result.append(vectorized_filter(element, function, pre_function))
        elif function(element):
            vectorized_result.append(element)

    return vectorized_result if type(a) is list else ''.join(vectorized_result)
<commit_msg>codeclimate: reduce complexity<commit_after>from lib.commands import *

def apply_pre_function(pre_function, value):
    if pre_function is not None:
        if type(value) is list:
            value = [apply_safe(pre_function, x) if type(x) is not list else x for x in value]
        else:
            value = apply_safe(pre_function, value)

    return value

def vectorized_evaluation(a, b, function, pre_function=None):
    """
    Uses the given function in the form of a lambda and produces a vectorized result if
    at least one of the items is an iterable value. Otherwise, it evaluates it as normal.
    :param a: The first value for the function
    :param b: The second value for the function
    :param function: A 2-arity lambda used for the function
    :param pre_function: A function that is called before vectorization (optional)
    :return: A value or array of values depending on a and b
    """

    try:
        a = apply_pre_function(pre_function, a)
        b = apply_pre_function(pre_function, b)

        # When both are lists
        if type(a) is list and type(b) is list:

            # Get the minimum and maximum of both lists in order to prevent index errors
            vector_range = min(len(a), len(b))
            max_range = max(len(a), len(b))
            vectorized_result = []

            # Compute the function for all in range elements
            for index in range(0, vector_range):
                vectorized_result.append(apply_safe(function, a[index], b[index]))

            # Append all out of range elements without being processed
            for index in range(vector_range, max_range):
                if len(a) == max_range:
                    vectorized_result.append(a[index])
                else:
                    vectorized_result.append(b[index])

            return vectorized_result

        # When only type a is a list
        elif type(a) is list and type(b) is not list:

            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    vectorized_evaluation(element, b, function, pre_function)
                )

            return vectorized_result

        # When only type b is a list
        elif type(a) is not list and type(b) is list:

            vectorized_result = []
            for element in b:
                vectorized_result.append(
                    vectorized_evaluation(a, element, function, pre_function)
                )

            return vectorized_result

        return apply_safe(function, a, b)

    except:
        if type(a) is list and type(b) is not list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    vectorized_evaluation(element, b, function, pre_function)
                )
            return vectorized_result
        elif type(a) is not list and type(b) is list:
            vectorized_result = []
            for element in b:
                vectorized_result.append(
                    vectorized_evaluation(a, element, function, pre_function)
                )
            return vectorized_result

        raise Exception

def single_vectorized_evaluation(a, function, pre_function=None):
    try:
        a = apply_pre_function(pre_function, a)

        if type(a) is list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    single_vectorized_evaluation(element, function, pre_function)
                )

            return vectorized_result

        return apply_safe(function, a)
    except:
        if type(a) is list:
            vectorized_result = []
            for element in a:
                vectorized_result.append(
                    single_vectorized_evaluation(element, function, pre_function)
                )
            return vectorized_result

        raise Exception

# Vectorized aggregation
# The "function" argument takes a function(accumulator, value), where:
#   * accumulator is the result of previously aggregated values
#   * value is the current value to aggregate
#
# The "start" argument is the starting value of the aggregation
# If not provided, the starting point will be the first value to pass the
# pre_function check without an exception been raised
#
# Result will keep the same order of values/sublists than input
# Aggregated value will be at the first non-list valid value
# encountered in the input
def vectorized_aggregator(a, function, pre_function=None, start=None):
    index = -1
    subresults = []
    values = []

    result = list(start) if start is list else start

    if type(a) is not list:
        a = str(a)

    for i in range(len(a)):
        if type(a[i]) is list:
            subresults.append(vectorized_aggregator(a[i], function, pre_function, start))
        else:
            try:
                if pre_function is not None:
                    a[i] = pre_function(a[i])
                if index == -1:
                    index = i
                if result is None:
                    result = a[i]
                else:
                    values.append(a[i])
            except:
                pass

    for i in values:
        result = apply_safe(function, result, i)

    if subresults:
        if index > -1:
            subresults.insert(index, result)
        result = subresults

    return result if result is not None else []

# Vectorized filter
# Keep only elements of a where function(a) is truthy
def vectorized_filter(a, function, pre_function=None):
    a = apply_pre_function(pre_function, a)

    vectorized_result = []
    for element in a:
        if type(element) is list:
            vectorized_result.append(vectorized_filter(element, function, pre_function))
        elif function(element):
            vectorized_result.append(element)

    return vectorized_result if type(a) is list else ''.join(vectorized_result)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding=utf-8

import logging
import time

def get_logger():
    logger = logging.getLogger("threading_example")
    logger.setLevel(logging.DEBUG)
    date_info = time.strftime('%Y-%m-%d')
    fh = logging.FileHandler("/data/www/logs/zhuanlan.log.%s" % date_info)
    # fh = logging.FileHandler("/Users/youqingkui/PycharmProjects/zhihufav/zhuanlan.log.%s" % date_info)
    fmt = '%(asctime)s|%(threadName)s|%(levelname)s| %(message)s'
    formatter = logging.Formatter(fmt)
    fh.setFormatter(formatter)

    logger.addHandler(fh)
    return logger

logger = get_logger()<commit_msg>add logger print<commit_after>#!/usr/bin/env python
#coding=utf-8

import logging
import time

def get_logger():
    logger = logging.getLogger("threading_example")
    logger.setLevel(logging.INFO)
    date_info = time.strftime('%Y-%m-%d')
    fh = logging.FileHandler("/data/www/logs/zhuanlan.log.%s" % date_info)
    # fh = logging.FileHandler("/Users/youqingkui/PycharmProjects/zhihufav/zhuanlan.log.%s" % date_info)
    fmt = '%(asctime)s|%(threadName)s|%(levelname)s| %(message)s'
    formatter = logging.Formatter(fmt)
    fh.setFormatter(formatter)

    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    logger.addHandler(fh)
    logger.addHandler(ch)
    return logger

logger = get_logger()

if __name__ == '__main__':
    logger.debug("hello debug")
    logger.warning("hello warning")
    logger.info("hello info")
    logger.error("hello error")<|endoftext|>


<commit_before># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for all nameserver related activity. Health checks. requests."""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

import datetime
import random

import dns.name
import dns.rdataclass
import dns.rdatatype
import dns.resolver
import dns.reversename

import util

GOOGLE_CLASS_B = ('74.125',)
WWW_GOOGLE_RESPONSE = ('CNAME www.l.google.com',)
WWW_PAYPAL_RESPONSE = ('66.211.169.', '64.4.241.')
OPENDNS_NS = '208.67.220.220'
WILDCARD_DOMAIN = 'blogspot.com.'
MIN_SHARING_DELTA_MS = 2
MAX_SHARING_DELTA_MS = 240

class NameServer(object):
  """Hold information about a particular nameserver."""

  def __init__(self, ip, name=None, internal=False, primary=False):
    self.name = name
    self.ip = ip
    self.is_internal = internal
    self.is_primary = primary
    self.timeout = 4
    self.health_timeout = 2.5

    self.warnings = []
    self.shared_with = []
    self.is_healthy = True
    self.checks = []
    self.cache_check = None
    self.is_slower_replica = False

  def _get_check_duration(self):
    return sum([x[3] for x in self.checks])
  check_duration = property(_get_check_duration, None)

  def __str__(self):
    return '%s [%s]' % (self.name, self.ip)

  def __repr__(self):
    return self.__str__()

  def Query(self, request, timeout):
    return dns.query.udp(request, self.ip, timeout, 53)

  def TimedRequest(self, type_string, record_string, timeout=None):
    """Make a DNS request, returning the reply and duration it took.

    Args:
      type_string: DNS record type to query (string)
      record_string: DNS record name to query (string)
      timeout: optional timeout (float)

    Returns:
      A tuple of (response, duration [float], exception)

    In the case of a DNS response timeout, the response object will be None.
    """
    request_type = dns.rdatatype.from_text(type_string)
    record = dns.name.from_text(record_string, None)
    return_type = dns.rdataclass.IN
    request = dns.message.make_query(record, request_type, return_type)
    if not timeout:
      timeout = self.timeout

    start_time = datetime.datetime.now()
    exc = None
    try:
      response = self.Query(request, timeout)
    except (dns.exception.Timeout), exc:
      response = None
    except (dns.query.BadResponse, dns.message.TrailingJunk,
            dns.query.UnexpectedSource), exc:
      response = None
    duration = util.TimeDeltaToMilliseconds(datetime.datetime.now() -
                                            start_time)
    return (response, duration, exc)

  def TestAnswers(self, record_type, record, expected):
    """Test to see that an answer returns correct IP's.

    Args:
      record_type: text record type for NS query (A, CNAME, etc)
      record: string to query for
      expected: tuple of strings expected in all answers

    Returns:
      (is_broken, warning, duration)
    """
    is_broken = False
    warning = None
    (response, duration, exc) = self.TimedRequest(record_type, record,
                                                  timeout=self.health_timeout)
    failures = []
    if not response:
      is_broken = True
      warning = exc.__class__
    elif not response.answer:
      is_broken = True
      warning = 'No answer'
    else:
      for a in response.answer:
        failed = True
        for string in expected:
          if string in str(a):
#            print "%s is in %s, marking not failed: %s" % (string, a, self.name)
            failed=False
            break
        if failed:
          failures.append(a)
    if failures:
      answers = [' + '.join(map(str, x.items)) for x in response.answer]
      answer_text = ' -> '.join(answers)
      warning = '%s may be hijacked (%s)' % (record, answer_text)
    return (is_broken, warning, duration)

  def ResponseToAscii(self, response):
    answers = [' + '.join(map(str, x.items)) for x in response.answer]
    return ' -> '.join(answers)

  def TestGoogleComResponse(self):
    return self.TestAnswers('A', 'google.com.', GOOGLE_CLASS_B)

  def TestWwwGoogleComResponse(self):
    return self.TestAnswers('CNAME', 'www.google.com.', WWW_GOOGLE_RESPONSE)

  def TestWwwPaypalComResponse(self):
    return self.TestAnswers('A', 'www.paypal.com.', WWW_PAYPAL_RESPONSE)

  def TestNegativeResponse(self):
    """Test for NXDOMAIN hijaaking."""
    is_broken = False
    warning = None
    poison_test = 'nb.%s.google.com.' % random.random()
    (response, duration, exc) = self.TimedRequest('A', poison_test,
                                                  timeout=self.health_timeout)
    if not response:
      is_broken = True
      warning = exc.__class__
    elif response.answer:
      warning = 'NXDOMAIN Hijacking (%s)' % self.ResponseToAscii(response)
    return (is_broken, warning, duration)

  def QueryWildcardCache(self, hostname=None, save=True):
    """Make a cache to a random wildcard DNS host, storing the record."""
    is_broken = False
    warning = None
    if not hostname:
      hostname = 'www%s.%s' % (random.random(), WILDCARD_DOMAIN)

    (response, duration, exc) = self.TimedRequest('A', hostname,
                                                  timeout=self.health_timeout)
    ttl = None
    if not response:
      is_broken = True
      warning = '%s (%sms)' % (exc.__class__, duration)
    elif not response.answer:
      is_broken = True
      warning = 'No response'
    else:
      ttl = response.answer[0].ttl

    if save:
      self.cache_check = (hostname, ttl)

    return (response, is_broken, warning, duration)

  def TestWildcardCaching(self):
    return self.QueryWildcardCache(save=True)[1:]

  def TestSharedCache(self, other_ns):
    """Is this nameserver sharing a cache with another nameserver?

    Args:
      other_ns: A nameserver to compare it to.

    Returns:
      A tuple containing:
        - Boolean of whether or not this host has a shared cache
        - The faster NameServer object
        - The slower NameServer object
    """
    if other_ns.cache_check:
      (cache_id, other_ttl) = other_ns.cache_check
    else:
      print "* cache check for %s is missing (skipping)" % other_ns
      return (False, None, None)
    (response, is_broken) = self.QueryWildcardCache(cache_id, save=False)[0:2]
    if is_broken:
      self.is_healthy = False
    else:
      delta = abs(other_ttl - response.answer[0].ttl)

      if delta > 0:
        if other_ns.check_duration > self.check_duration:
          slower = other_ns
          faster = self
        else:
          slower = self
          faster = other_ns

        if delta > MIN_SHARING_DELTA_MS and delta < MAX_SHARING_DELTA_MS:
          return (True, slower, faster)

    return (False, None, None)

  def CheckHealth(self):
    """Qualify a nameserver to see if it is any good."""
    tests = [self.TestWwwGoogleComResponse,
             self.TestGoogleComResponse,
             self.TestNegativeResponse,
             self.TestWildcardCaching,
             self.TestWwwPaypalComResponse]
    self.checks = []
    self.warnings = []

    for test in tests:
      (is_broken, warning, duration) = test()
      if is_broken:
        self.is_healthy = False
        if self.is_primary:
          print '  * %s is unhealthy: %s %s' % (self, test.__name__, warning)
        break

      if warning:
        self.warnings.append(warning)
      self.checks.append((test.__name__, is_broken, warning, duration))

    if self.warnings:
      print '  o %s: %s' % (self.name, ', '.join(self.warnings))

    return self.is_healthy

<commit_msg>Remove debug line<commit_after># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for all nameserver related activity. Health checks. requests."""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

import datetime
import random

import dns.name
import dns.rdataclass
import dns.rdatatype
import dns.resolver
import dns.reversename

import util

GOOGLE_CLASS_B = ('74.125',)
WWW_GOOGLE_RESPONSE = ('CNAME www.l.google.com',)
WWW_PAYPAL_RESPONSE = ('66.211.169.', '64.4.241.')
OPENDNS_NS = '208.67.220.220'
WILDCARD_DOMAIN = 'blogspot.com.'
MIN_SHARING_DELTA_MS = 2
MAX_SHARING_DELTA_MS = 240

class NameServer(object):
  """Hold information about a particular nameserver."""

  def __init__(self, ip, name=None, internal=False, primary=False):
    self.name = name
    self.ip = ip
    self.is_internal = internal
    self.is_primary = primary
    self.timeout = 4
    self.health_timeout = 2.5

    self.warnings = []
    self.shared_with = []
    self.is_healthy = True
    self.checks = []
    self.cache_check = None
    self.is_slower_replica = False

  def _get_check_duration(self):
    return sum([x[3] for x in self.checks])
  check_duration = property(_get_check_duration, None)

  def __str__(self):
    return '%s [%s]' % (self.name, self.ip)

  def __repr__(self):
    return self.__str__()

  def Query(self, request, timeout):
    return dns.query.udp(request, self.ip, timeout, 53)

  def TimedRequest(self, type_string, record_string, timeout=None):
    """Make a DNS request, returning the reply and duration it took.

    Args:
      type_string: DNS record type to query (string)
      record_string: DNS record name to query (string)
      timeout: optional timeout (float)

    Returns:
      A tuple of (response, duration [float], exception)

    In the case of a DNS response timeout, the response object will be None.
    """
#    print '%s:%s -> %s [%s]' % (type_string, record_string, self.ip, self.name)
    request_type = dns.rdatatype.from_text(type_string)
    record = dns.name.from_text(record_string, None)
    return_type = dns.rdataclass.IN
    request = dns.message.make_query(record, request_type, return_type)
    if not timeout:
      timeout = self.timeout

    start_time = datetime.datetime.now()
    exc = None
    try:
      response = self.Query(request, timeout)
    except (dns.exception.Timeout), exc:
      response = None
    except (dns.query.BadResponse, dns.message.TrailingJunk,
            dns.query.UnexpectedSource), exc:
      response = None
    duration = util.TimeDeltaToMilliseconds(datetime.datetime.now() -
                                            start_time)
    return (response, duration, exc)

  def TestAnswers(self, record_type, record, expected):
    """Test to see that an answer returns correct IP's.

    Args:
      record_type: text record type for NS query (A, CNAME, etc)
      record: string to query for
      expected: tuple of strings expected in all answers

    Returns:
      (is_broken, warning, duration)
    """
    is_broken = False
    warning = None
    (response, duration, exc) = self.TimedRequest(record_type, record,
                                                  timeout=self.health_timeout)
    failures = []
    if not response:
      is_broken = True
      warning = exc.__class__
    elif not response.answer:
      is_broken = True
      warning = 'No answer'
    else:
      for a in response.answer:
        failed = True
        for string in expected:
          if string in str(a):
#            print "%s is in %s, marking not failed: %s" % (string, a, self.name)
            failed=False
            break
        if failed:
          failures.append(a)
    if failures:
      answers = [' + '.join(map(str, x.items)) for x in response.answer]
      answer_text = ' -> '.join(answers)
      warning = '%s may be hijacked (%s)' % (record, answer_text)
    return (is_broken, warning, duration)

  def ResponseToAscii(self, response):
    answers = [' + '.join(map(str, x.items)) for x in response.answer]
    return ' -> '.join(answers)

  def TestGoogleComResponse(self):
    return self.TestAnswers('A', 'google.com.', GOOGLE_CLASS_B)

  def TestWwwGoogleComResponse(self):
    return self.TestAnswers('CNAME', 'www.google.com.', WWW_GOOGLE_RESPONSE)

  def TestWwwPaypalComResponse(self):
    return self.TestAnswers('A', 'www.paypal.com.', WWW_PAYPAL_RESPONSE)

  def TestNegativeResponse(self):
    """Test for NXDOMAIN hijaaking."""
    is_broken = False
    warning = None
    poison_test = 'nb.%s.google.com.' % random.random()
    (response, duration, exc) = self.TimedRequest('A', poison_test,
                                                  timeout=self.health_timeout)
    if not response:
      is_broken = True
      warning = exc.__class__
    elif response.answer:
      warning = 'NXDOMAIN Hijacking (%s)' % self.ResponseToAscii(response)
    return (is_broken, warning, duration)

  def QueryWildcardCache(self, hostname=None, save=True):
    """Make a cache to a random wildcard DNS host, storing the record."""
    is_broken = False
    warning = None
    if not hostname:
      hostname = 'www%s.%s' % (random.random(), WILDCARD_DOMAIN)

    (response, duration, exc) = self.TimedRequest('A', hostname,
                                                  timeout=self.health_timeout)
    ttl = None
    if not response:
      is_broken = True
      warning = '%s (%sms)' % (exc.__class__, duration)
    elif not response.answer:
      is_broken = True
      warning = 'No response'
    else:
      ttl = response.answer[0].ttl

    if save:
      self.cache_check = (hostname, ttl)

    return (response, is_broken, warning, duration)

  def TestWildcardCaching(self):
    return self.QueryWildcardCache(save=True)[1:]

  def TestSharedCache(self, other_ns):
    """Is this nameserver sharing a cache with another nameserver?

    Args:
      other_ns: A nameserver to compare it to.

    Returns:
      A tuple containing:
        - Boolean of whether or not this host has a shared cache
        - The faster NameServer object
        - The slower NameServer object
    """
    if other_ns.cache_check:
      (cache_id, other_ttl) = other_ns.cache_check
    else:
      print "* cache check for %s is missing (skipping)" % other_ns
      return (False, None, None)
    (response, is_broken) = self.QueryWildcardCache(cache_id, save=False)[0:2]
    if is_broken:
      self.is_healthy = False
    else:
      delta = abs(other_ttl - response.answer[0].ttl)

      if delta > 0:
        if other_ns.check_duration > self.check_duration:
          slower = other_ns
          faster = self
        else:
          slower = self
          faster = other_ns

        if delta > MIN_SHARING_DELTA_MS and delta < MAX_SHARING_DELTA_MS:
          return (True, slower, faster)

    return (False, None, None)

  def CheckHealth(self):
    """Qualify a nameserver to see if it is any good."""
    tests = [self.TestWwwGoogleComResponse,
             self.TestGoogleComResponse,
             self.TestNegativeResponse,
             self.TestWildcardCaching,
             self.TestWwwPaypalComResponse]
    self.checks = []
    self.warnings = []

    for test in tests:
      (is_broken, warning, duration) = test()
      if is_broken:
        self.is_healthy = False
        if self.is_primary:
          print '  * %s is unhealthy: %s %s' % (self, test.__name__, warning)
        break

      if warning:
        self.warnings.append(warning)
      self.checks.append((test.__name__, is_broken, warning, duration))

    if self.warnings:
      print '  o %s: %s' % (self.name, ', '.join(self.warnings))

    return self.is_healthy

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Output routines for audio and analysis"""

import csv

import numpy as np
import scipy
import scipy.io.wavfile

import librosa.core

def annotation(path, time_start, time_end, annotations=None, delimiter=',', fmt='%0.3f'):
    r'''Save annotations in a 3-column format::

        time_start[0],time_end[0],annotations[0]\n
        time_start[1],time_end[1],annotations[1]\n
        time_start[2],time_end[2],annotations[2]\n
        ...

    This can be used for segment or chord annotations.

    :usage:
        >>> # Detect segment boundaries
        >>> boundaries = librosa.segment.agglomerative(data, k=10)
        >>> # Convert to time
        >>> boundary_times = librosa.frames_to_time(boundaries, sr=sr,
                                                    hop_length=hop_length)
        >>> # Convert boundaries to start-ends
        >>> time_start, time_end = boundaries[:-1], boundaries[1:]
        >>> # Make some fake annotations
        >>> labels = ['Segment #%03d' % i for i in range(len(time_start))]
        >>> # Save the output
        >>> librosa.output.annotation('segments.csv', time_start, time_end,
                                      annotations=annotations)

    :parameters:
      - path : str
          path to save the output CSV file

      - time_start : list-like
          array of starting times for annotations

      - time_end : list-like
          array of ending times for annotations

      - annotations : None or list-like
          optional list of annotation strings. ``annotations[i]`` applies to the time
          range ``time_start[i]`` to ``time_end[i]``

      - delimiter : str
          character to separate fields

      - fmt : str
          format-string for rendering time data

    :raises:
      - ValueError
          if ``time_start`` and ``time_end`` have different length
      - ValueError
          if ``annotations`` is not ``None`` and length does not match ``time_start``
    '''

    if len(time_start) != len(time_end):
        raise ValueError('len(time_start) != len(time_end)')

    if annotations is not None and len(annotations) != len(time_start):
        raise ValueError('len(annotations) != len(time_start)')

    with open(path, 'w') as output_file:
        writer = csv.writer(output_file, delimiter=delimiter)

        if annotations is None:
            for t_s, t_e in zip(time_start, time_end):
                writer.writerow([fmt % t_s, fmt % t_e])
        else:
            for t_s, t_e, lab in zip(time_start, time_end, annotations):
                writer.writerow([fmt % t_s, fmt % t_e, lab])

def frames_csv(path, frames, sr=22050, hop_length=512, **kwargs):
    """Convert frames to time and store the output in CSV format.

    :usage:
        >>> tempo, beats = librosa.beat.beat_track(y, sr=sr, hop_length=64)
        >>> librosa.output.frames_csv('beat_times.csv', frames, sr=sr, hop_length=64)

    :parameters:
      - path : string
          path to save the output CSV file

      - frames : list-like of ints
          list of frame numbers for beat events

      - sr : int > 0
          audio sampling rate

      - hop_length : int > 0
          number of samples between success frames

      - *kwargs*
          additional keyword arguments.  See ``librosa.output.times_csv``
    """

    times = librosa.frames_to_time(frames, sr=sr, hop_length=hop_length)

    times_csv(path, times, **kwargs)

def times_csv(path, times, annotations=None, delimiter=',', fmt='%0.3f'):
    r"""Save time steps as in CSV format.  This can be used to store the output
    of a beat-tracker or segmentation algorihtm.

    If only ``times`` are provided, the file will contain each value of ``times`` on a row::

        times[0]\n
        times[1]\n
        times[2]\n
        ...

    If ``annotations`` are also provided, the file will contain delimiter-separated values::

        times[0],annotations[0]\n
        times[1],annotations[1]\n
        times[2],annotations[2]\n
        ...

    :usage:
        >>> tempo, beats = librosa.beat.beat_track(y, sr=sr, hop_length=64)
        >>> times = librosa.frames_to_time(beats, sr=sr, hop_length=64)
        >>> librosa.output.times_csv('beat_times.csv', times)

    :parameters:
      - path : string
          path to save the output CSV file

      - times : list-like of floats
          list of frame numbers for beat events

      - annotations : None or list-like
          optional annotations for each time step

      - delimiter : str
          character to separate fields

      - fmt : str
          format-string for rendering time

    :raises:
      - ValueError
          if ``annotations`` is not ``None`` and length does not match ``times``
    """

    if annotations is not None and len(annotations) != len(times):
        raise ValueError('len(annotations) != len(times)')

    with open(path, 'w') as output_file:
        writer = csv.writer(output_file, delimiter=delimiter)

        if annotations is None:
            for t in times:
                writer.writerow([fmt % t])
        else:
            for t, lab in zip(times, annotations):
                writer.writerow([(fmt % t), lab])

def write_wav(path, y, sr, normalize=True):
    """Output a time series as a .wav file

    :usage:
        >>> # Trim a signal to 5 seconds and save it back
        >>> y, sr = librosa.load('file.wav', duration=5)
        >>> librosa.output.write_wav('file_trim_5s.wav', y, sr)

    :parameters:
      - path : str
          path to save the output wav file

      - y : np.ndarray
          audio time series

      - sr : int > 0
          sampling rate of ``y``

      - normalize : boolean
          enable amplitude normalization
    """

    # normalize
    if normalize:
        wav = y / np.max(np.abs(y))
    else:
        wav = y

    # Scale up to pcm range
    #wav = (wav - wav.min()) * (1<<15) - (1<<15)
    wav = wav * 32767

    # Convert to 16bit int
    wav = wav.astype('<i2')

    # Save
    scipy.io.wavfile.write(path, sr, wav)

<commit_msg>standardized annotation output to interval format<commit_after>#!/usr/bin/env python
"""Output routines for audio and analysis"""

import csv

import numpy as np
import scipy
import scipy.io.wavfile

import librosa.core

def annotation(path, intervals, annotations=None, delimiter=',', fmt='%0.3f'):
    r'''Save annotations in a 3-column format::

        intervals[0, 0],intervals[0, 1],annotations[0]\n
        intervals[1, 0],intervals[1, 1],annotations[1]\n
        intervals[2, 0],intervals[2, 1],annotations[2]\n
        ...

    This can be used for segment or chord annotations.

    :usage:
        >>> # Detect segment boundaries
        >>> boundaries = librosa.segment.agglomerative(data, k=10)
        >>> # Convert to time
        >>> boundary_times = librosa.frames_to_time(boundaries, sr=sr,
                                                    hop_length=hop_length)
        >>> # Make some fake annotations
        >>> labels = ['Segment #%03d' % i for i in range(len(time_start))]
        >>> # Save the output
        >>> librosa.output.annotation('segments.csv', boundary_times,
                                      annotations=annotations)

    :parameters:
      - path : str
          path to save the output CSV file

      - intervals : np.ndarray, shape=(n, 2)
          array of interval start and end-times.

          - ``intervals[i, 0]`` marks the start time of interval ``i``

          - ``intervals[i, 1]`` marks the endtime of interval ``i``

      - annotations : None or list-like
          optional list of annotation strings. ``annotations[i]`` applies to the time
          range ``intervals[i, 0]`` to ``intervals[i, 1]``

      - delimiter : str
          character to separate fields

      - fmt : str
          format-string for rendering time data

    :raises:
      - ValueError
          if ``annotations`` is not ``None`` and length does not match ``intervals``
    '''

    if annotations is not None and len(annotations) != len(intervals):
        raise ValueError('len(annotations) != len(intervals)')

    with open(path, 'w') as output_file:
        writer = csv.writer(output_file, delimiter=delimiter)

        if annotations is None:
            for t_int in intervals:
                writer.writerow([fmt % t_int[0], fmt % t_int[1]])
        else:
            for t_int, lab in zip(intervals, annotations):
                writer.writerow([fmt % t_int[0], fmt % t_int[1], lab])

def frames_csv(path, frames, sr=22050, hop_length=512, **kwargs):
    """Convert frames to time and store the output in CSV format.

    :usage:
        >>> tempo, beats = librosa.beat.beat_track(y, sr=sr, hop_length=64)
        >>> librosa.output.frames_csv('beat_times.csv', frames, sr=sr, hop_length=64)

    :parameters:
      - path : string
          path to save the output CSV file

      - frames : list-like of ints
          list of frame numbers for beat events

      - sr : int > 0
          audio sampling rate

      - hop_length : int > 0
          number of samples between success frames

      - *kwargs*
          additional keyword arguments.  See ``librosa.output.times_csv``
    """

    times = librosa.frames_to_time(frames, sr=sr, hop_length=hop_length)

    times_csv(path, times, **kwargs)

def times_csv(path, times, annotations=None, delimiter=',', fmt='%0.3f'):
    r"""Save time steps as in CSV format.  This can be used to store the output
    of a beat-tracker or segmentation algorihtm.

    If only ``times`` are provided, the file will contain each value of ``times`` on a row::

        times[0]\n
        times[1]\n
        times[2]\n
        ...

    If ``annotations`` are also provided, the file will contain delimiter-separated values::

        times[0],annotations[0]\n
        times[1],annotations[1]\n
        times[2],annotations[2]\n
        ...

    :usage:
        >>> tempo, beats = librosa.beat.beat_track(y, sr=sr, hop_length=64)
        >>> times = librosa.frames_to_time(beats, sr=sr, hop_length=64)
        >>> librosa.output.times_csv('beat_times.csv', times)

    :parameters:
      - path : string
          path to save the output CSV file

      - times : list-like of floats
          list of frame numbers for beat events

      - annotations : None or list-like
          optional annotations for each time step

      - delimiter : str
          character to separate fields

      - fmt : str
          format-string for rendering time

    :raises:
      - ValueError
          if ``annotations`` is not ``None`` and length does not match ``times``
    """

    if annotations is not None and len(annotations) != len(times):
        raise ValueError('len(annotations) != len(times)')

    with open(path, 'w') as output_file:
        writer = csv.writer(output_file, delimiter=delimiter)

        if annotations is None:
            for t in times:
                writer.writerow([fmt % t])
        else:
            for t, lab in zip(times, annotations):
                writer.writerow([(fmt % t), lab])

def write_wav(path, y, sr, normalize=True):
    """Output a time series as a .wav file

    :usage:
        >>> # Trim a signal to 5 seconds and save it back
        >>> y, sr = librosa.load('file.wav', duration=5)
        >>> librosa.output.write_wav('file_trim_5s.wav', y, sr)

    :parameters:
      - path : str
          path to save the output wav file

      - y : np.ndarray
          audio time series

      - sr : int > 0
          sampling rate of ``y``

      - normalize : boolean
          enable amplitude normalization
    """

    # normalize
    if normalize:
        wav = y / np.max(np.abs(y))
    else:
        wav = y

    # Scale up to pcm range
    #wav = (wav - wav.min()) * (1<<15) - (1<<15)
    wav = wav * 32767

    # Convert to 16bit int
    wav = wav.astype('<i2')

    # Save
    scipy.io.wavfile.write(path, sr, wav)

<|endoftext|>


<commit_before>from __future__ import division

try:
  from glob import glob
  import libtbx.load_env
  import os
  dials_path = libtbx.env.dist_path('dials')
  filenames = glob(os.path.join(dials_path, "extensions", "*.pyc"))
  if len(filenames) > 0:
    print "Cleaning up 'dials/extensions':"
    for filename in filenames:
      print " Deleting %s" % filename
      os.remove(filename)
except Exception:
  pass

try:
  from dials.framework import env
  env.cache.wipe()
except Exception:
  pass

def _install_dials_autocompletion():
  '''generate init.sh and SConscript file in /build/dials/autocomplete'''
  import libtbx.load_env
  import os

  # Find the dials source directory
  dist_path = libtbx.env.dist_path('dials')

  # Set the location of the output directory
  output_directory = libtbx.env.under_build(os.path.join('dials', 'autocomplete'))
  try:
    os.makedirs(output_directory)
  except OSError:
    pass

  commands_dir = os.path.join(dist_path, 'command_line')
  command_list = []
  print 'Identifying autocompletable commands:',
  for file in sorted(os.listdir(commands_dir)):
    if not file.startswith('_') and file.endswith('.py'):
      if 'DIALS_ENABLE_COMMAND_LINE_COMPLETION' in open(os.path.join(commands_dir, file)).read():
        command_name = 'dials.%s' % file[:-3]
        print command_name,
        command_list.append(command_name)
  print

  # Generate the autocompletion SConscript.
  with open(os.path.join(output_directory, 'SConscript'), 'w') as builder:
    builder.write('''Import("env")
import os.path
import libtbx.load_env
def dispatcher_outer(name):
  return os.path.join(libtbx.env.under_build('bin'), name)
def dispatcher_inner(name):
  return os.path.join(libtbx.env.dist_path('dials'), 'command_line', '%%s.py' %% name.partition('.')[2])
env.Append( BUILDERS={'AutoComplete': Builder(action='$SOURCE --export-autocomplete-hints > $TARGET')} )
for cmd in [%s]:
  ac = env.AutoComplete(cmd, [dispatcher_outer(cmd), dispatcher_inner(cmd)])
  Requires(ac, Dir(libtbx.env.under_build('lib')))
''' % ( ', '.join(["'%s'" % cmd for cmd in command_list]) ))

  # Generate a bash script activating command line completion for each relevant command
  with open(os.path.join(output_directory, 'bash.sh'), 'w') as script:
    for cmd in command_list:
      script.write("complete -F _dials_autocomplete %s\n" % cmd)

  # Find the dials build directory
  build_path = abs(libtbx.env.build_path)

  # Permanently install the autocompletion script into setpaths-scripts.
  print "Installing autocompletion script into:",
  for file in os.listdir(build_path):
    if file.startswith('setpath') and file.endswith('.sh'):
      original_file = open(os.path.join(build_path, file)).read()
      if not 'DIALS_ENABLE_COMMAND_LINE_COMPLETION' in original_file:
        marker = "\nexport PATH\n"
        insert_position = original_file.find(marker) + len(marker)
        if insert_position >= 0:
          print file,
          added_script = \
            '# DIALS_ENABLE_COMMAND_LINE_COMPLETION\n' \
            '[ -z "$BASH_VERSIONINFO" ] && {\n' \
            ' source %s\n' \
            ' source %s\n' \
            '}\n' % (
              os.path.join('$LIBTBX_BUILD', '..', 'modules', 'dials', 'util', 'autocomplete.sh'),
              os.path.join('$LIBTBX_BUILD', 'dials', 'autocomplete', 'bash.sh'))
          with open(os.path.join(build_path, file), 'w') as script:
            script.write(original_file[:insert_position] +
                         added_script +
                         original_file[insert_position:])
  print

_install_dials_autocompletion()
<commit_msg>putative fix for David's path issues<commit_after>from __future__ import division

try:
  from glob import glob
  import libtbx.load_env
  import os
  dials_path = libtbx.env.dist_path('dials')
  filenames = glob(os.path.join(dials_path, "extensions", "*.pyc"))
  if len(filenames) > 0:
    print "Cleaning up 'dials/extensions':"
    for filename in filenames:
      print " Deleting %s" % filename
      os.remove(filename)
except Exception:
  pass

try:
  from dials.framework import env
  env.cache.wipe()
except Exception:
  pass

def _install_dials_autocompletion():
  '''generate init.sh and SConscript file in /build/dials/autocomplete'''
  import libtbx.load_env
  import os

  # Find the dials source directory
  dist_path = libtbx.env.dist_path('dials')

  # Set the location of the output directory
  output_directory = libtbx.env.under_build(os.path.join('dials', 'autocomplete'))
  try:
    os.makedirs(output_directory)
  except OSError:
    pass

  commands_dir = os.path.join(dist_path, 'command_line')
  command_list = []
  print 'Identifying autocompletable commands:',
  for file in sorted(os.listdir(commands_dir)):
    if not file.startswith('_') and file.endswith('.py'):
      if 'DIALS_ENABLE_COMMAND_LINE_COMPLETION' in open(os.path.join(commands_dir, file)).read():
        command_name = 'dials.%s' % file[:-3]
        print command_name,
        command_list.append(command_name)
  print

  # Generate the autocompletion SConscript.
  with open(os.path.join(output_directory, 'SConscript'), 'w') as builder:
    builder.write('''Import("env")
import os.path
import libtbx.load_env
def dispatcher_outer(name):
  return os.path.join(libtbx.env.under_build('bin'), name)
def dispatcher_inner(name):
  return os.path.join(libtbx.env.dist_path('dials'), 'command_line', '%%s.py' %% name.partition('.')[2])
env.Append( BUILDERS={'AutoComplete': Builder(action='$SOURCE --export-autocomplete-hints > $TARGET')} )
for cmd in [%s]:
  ac = env.AutoComplete(cmd, [dispatcher_outer(cmd), dispatcher_inner(cmd)])
  Requires(ac, Dir(libtbx.env.under_build('lib')))
''' % ( ', '.join(["'%s'" % cmd for cmd in command_list]) ))

  # Generate a bash script activating command line completion for each relevant command
  with open(os.path.join(output_directory, 'bash.sh'), 'w') as script:
    for cmd in command_list:
      script.write("complete -F _dials_autocomplete %s\n" % cmd)

  # Find the dials build directory
  build_path = abs(libtbx.env.build_path)

  # Permanently install the autocompletion script into setpaths-scripts.
  print "Installing autocompletion script into:",
  for file in os.listdir(build_path):
    if file.startswith('setpath') and file.endswith('.sh'):
      original_file = open(os.path.join(build_path, file)).read()
      if not 'DIALS_ENABLE_COMMAND_LINE_COMPLETION' in original_file:
        marker = "\nexport PATH\n"
        insert_position = original_file.find(marker) + len(marker)
        if insert_position >= 0:
          print file,
          added_script = \
            '# DIALS_ENABLE_COMMAND_LINE_COMPLETION\n' \
            '[ -z "$BASH_VERSIONINFO" ] && {\n' \
            ' source $(libtbx.find_in_repositories dials/util/autocomplete.sh)\n' \
            ' source %s\n' \
            '}\n' % (
              os.path.join('$LIBTBX_BUILD', 'dials', 'autocomplete', 'bash.sh'))
          with open(os.path.join(build_path, file), 'w') as script:
            script.write(original_file[:insert_position] +
                         added_script +
                         original_file[insert_position:])
  print

_install_dials_autocompletion()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class CachedTranslation(object):

    def __init__(self, **kwargs):
        from .models import Translation

        self.instances = ['instance', 'translation']

        self.fields = Translation._meta.get_all_field_names()
        self.fields.remove('id')

        attrs = self.fields + self.instances

        for attr in attrs:
            setattr(self, attr, None)

        self.__dict__.update(**kwargs)

        self.is_new = True
        self.has_changed = False

        if self.instance is not None:
            self.identifier = self.instance.linguist_identifier
            self.object_id = self.instance.pk

        if self.translation is not None:
            self.is_new = bool(self.translation.pk is None)
            for attr in ('language', 'field_name', 'field_value'):
                setattr(self, attr, getattr(self.translation, attr))

    @property
    def attrs(self):
        """
        Returns Translation attributes to pass as kwargs for creating or updating objects.
        """
        return dict((k, getattr(self, k)) for k in self.fields)

    @property
    def lookup(self):
        """
        Returns lookup for get() and filter() methods.
        """
        lookup = dict((k, getattr(self, k)) for k in self.fields)
        lookup.pop('field_value')
        return lookup

    @classmethod
    def from_object(cls, obj):
        """
        Updates values from the given object.
        """
        from .models import Translation

        fields = Translation._meta.get_all_field_names()
        fields.remove('id')

        return cls(**dict((field, getattr(obj, field)) for field in fields))

    def __str__(self):
        return '%s:%s:%s:%s' % (
            self.identifier,
            self.object_id,
            self.field_name,
            self.language)
<commit_msg>Remove useless attribute<commit_after># -*- coding: utf-8 -*-
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class CachedTranslation(object):

    def __init__(self, **kwargs):
        from .models import Translation

        self.fields = Translation._meta.get_all_field_names()
        self.fields.remove('id')

        attrs = self.fields + ['instance', 'translation']

        for attr in attrs:
            setattr(self, attr, None)

        self.__dict__.update(**kwargs)

        self.is_new = True
        self.has_changed = False

        if self.instance is not None:
            self.identifier = self.instance.linguist_identifier
            self.object_id = self.instance.pk

        if self.translation is not None:
            self.is_new = bool(self.translation.pk is None)
            for attr in ('language', 'field_name', 'field_value'):
                setattr(self, attr, getattr(self.translation, attr))

    @property
    def attrs(self):
        """
        Returns Translation attributes to pass as kwargs for creating or updating objects.
        """
        return dict((k, getattr(self, k)) for k in self.fields)

    @property
    def lookup(self):
        """
        Returns lookup for get() and filter() methods.
        """
        lookup = dict((k, getattr(self, k)) for k in self.fields)
        lookup.pop('field_value')
        return lookup

    @classmethod
    def from_object(cls, obj):
        """
        Updates values from the given object.
        """
        from .models import Translation

        fields = Translation._meta.get_all_field_names()
        fields.remove('id')

        return cls(**dict((field, getattr(obj, field)) for field in fields))

    def __str__(self):
        return '%s:%s:%s:%s' % (
            self.identifier,
            self.object_id,
            self.field_name,
            self.language)
<|endoftext|>


<commit_before>from django.contrib import admin

from .models import Region, City, GatheringCenter, Resource

admin.site.register(Region)
admin.site.register(City)

class PublishMixin(object):
    actions = ('publish', 'unpublish')

    def publish(self, request, queryset):
        queryset.update(published=True)

    def unpublish(self, request, queryset):
        queryset.update(published=False)

class GatheringCenterAdmin(PublishMixin, admin.ModelAdmin):
    list_filter = ('published', 'city', 'created')
    list_editable = ('published', )
    list_display = ('location_name', 'created', 'published', 'author', 'city')
    raw_id_fields = ('author', )

admin.site.register(GatheringCenter, GatheringCenterAdmin)

class ResourceAdmin(PublishMixin, admin.ModelAdmin):
    list_filter = ('published', 'created')
    list_editable = ('published', )
    list_display = ('name', 'created', 'published', 'author', 'url', 'country')
    raw_id_fields = ('author', )

admin.site.register(Resource, ResourceAdmin)

<commit_msg>Order by create, don't filter<commit_after>from django.contrib import admin

from .models import Region, City, GatheringCenter, Resource

admin.site.register(Region)
admin.site.register(City)

class PublishMixin(object):
    actions = ('publish', 'unpublish')

    def publish(self, request, queryset):
        queryset.update(published=True)

    def unpublish(self, request, queryset):
        queryset.update(published=False)

class GatheringCenterAdmin(PublishMixin, admin.ModelAdmin):
    date_hierarchy = 'created'
    list_filter = ('published', 'city', )
    list_editable = ('published', )
    list_display = ('location_name', 'created', 'published', 'author', 'city')
    raw_id_fields = ('author', )

admin.site.register(GatheringCenter, GatheringCenterAdmin)

class ResourceAdmin(PublishMixin, admin.ModelAdmin):
    date_hierarchy = 'created'
    list_filter = ('published', )
    list_editable = ('published', )
    list_display = ('name', 'created', 'published', 'author', 'url', 'country')
    raw_id_fields = ('author', )

admin.site.register(Resource, ResourceAdmin)

<|endoftext|>


<commit_before>
import os.path
import sys
import hashlib
from cStringIO import StringIO

default_encoding = sys.getfilesystemencoding()
if default_encoding is None or default_encoding.lower() == 'ascii':
	default_encoding = 'utf-8'

class decoder:
	def __init__(self, bytes):
		self.bytes = bytes
		self.i = 0
	def decode_value(self):
		x = self.bytes[self.i]
		if x.isdigit():
			return self.decode_string()
		self.i += 1
		if x == 'd':
			v = {}
			while self.peek() != 'e':
				k = self.decode_string()
				v[k] = self.decode_value()
			self.i += 1
			return v
		elif x == 'l':
			v = []
			while self.peek() != 'e':
				v.append(self.decode_value())
			self.i += 1
			return v
		elif x == 'i':
			return self.decode_int()
		else:
			raise NotImplementedError(x)
	def decode_string(self):
		i = self.bytes.index(':', self.i)
		n = int(self.bytes[self.i:i])
		s = self.bytes[i+1:i+1+n]
		self.i = i + 1 + n
		return s
	def decode_int(self):
		e = self.bytes.index('e', self.i)
		n = int(self.bytes[self.i:e])
		self.i = e + 1
		return n
	def peek(self):
		return self.bytes[self.i]

class encoder:
	def __init__(self, stream):
		self.stream = stream
	def encode(self, v):
		if type(v) == str:
			self.stream.write(str(len(v)))
			self.stream.write(':')
			self.stream.write(v)
		elif type(v) == dict:
			self.stream.write('d')
			for k in sorted(v):
				self.encode(k)
				self.encode(v[k])
			self.stream.write('e')
		elif type(v) == list:
			self.stream.write('l')
			for x in v:
				self.encode(x)
			self.stream.write('e')
		elif type(v) == int:
			self.stream.write('i')
			self.stream.write(str(v))
			self.stream.write('e')
		else:
			raise NotImplementedError(type(v))

def bdecode(bytes):
	return decoder(bytes).decode_value()

def bencode(v):
	from cStringIO import StringIO
	stream = StringIO()
	encoder(stream).encode(v)
	return stream.getvalue()

def info_hash(path):
	with open(path, 'rb') as stream:
		return hashlib.sha1(bencode(bdecode(stream.read())['info'])).hexdigest()

def encode_path(path):
	return path.decode('utf-8').encode(default_encoding)

class sha1_reader:
	def __init__(self, pieces, progress_callback=None):
		assert pieces
		assert len(pieces) % 20 == 0
		self.total = len(pieces)/20
		self.processed = 0
		self.stream = StringIO(pieces)
		self.progress_callback = progress_callback
	def next_sha1(self):
		self.processed += 1
		if self.progress_callback:
			self.progress_callback(float(self.processed)/self.total)
		return self.stream.read(20)

def verify_bt_single_file(path, info, progress_callback=None):
	# TODO: check md5sum if available
	if os.path.getsize(path) != info['length']:
		return False
	piece_length = info['piece length']
	assert piece_length <= 1024*1024
	sha1_stream = sha1_reader(info['pieces'], progress_callback=progress_callback)
	with open(path, 'rb') as stream:
		while True:
			bytes = stream.read(piece_length)
			sha1 = sha1_stream.next_sha1()
			if bytes:
				assert len(sha1) == 20
				if hashlib.sha1(bytes).digest() != sha1:
					return False
			else:
				assert len(sha1) == 0
				break
	assert sha1_stream.next_sha1() == ''
	return True

def verify_bt_multiple(folder, info, progress_callback=None):
	# TODO: check md5sum if available
	piece_length = info['piece length']
	assert piece_length <= 1024*1024
	files = [{'path':os.path.join(folder, apply(os.path.join, x['path'])), 'length':x['length']} for x in info['files']]

	sha1_stream = sha1_reader(info['pieces'], progress_callback=progress_callback)
	sha1sum = hashlib.sha1()

	piece_left = piece_length
	complete_piece = True

	while files:
		f = files.pop(0)
		path = f['path']
		size = f['length']
		if os.path.exists(path):
			if os.path.getsize(path) != size:
				return False
			if size <= piece_left:
				with open(path, 'rb') as stream:
					bytes = stream.read()
				assert len(bytes) == size
				sha1sum.update(bytes)
				piece_left -= size
				if not piece_left:
					if complete_piece and sha1sum.digest() != sha1_stream.next_sha1():
						return False
					complete_piece = True
					sha1sum = hashlib.sha1()
					piece_left = piece_length
			else:
				with open(path, 'rb') as stream:
					while size >= piece_left:
						bytes = stream.read(piece_left)
						assert len(bytes) == piece_left
						size -= piece_left
						sha1sum.update(bytes)
						if sha1sum.digest() != sha1_stream.next_sha1():
							return False
						sha1sum = hashlib.sha1()
						piece_left = piece_length
					if size:
						bytes = stream.read(size)
						assert len(bytes) == size
						sha1sum.update(bytes)
		else:
			while size >= piece_left:
				size -= piece_left
				sha1_stream.next_sha1()
				sha1sum = hashlib.sha1()
				piece_left = piece_length
			if size:
				complete_piece = False
				piece_left -= size
			else:
				complete_piece = True

	if piece_left < piece_length:
		if complete_piece:
			if sha1sum.digest() != sha1_stream.next_sha1():
				return False
		else:
			sha1_stream.next_sha1()
	assert sha1_stream.next_sha1() == ''

	return True

def verify_bt(path, info, progress_callback=None):
	if 'files' not in info:
		if os.path.isfile(path):
			return verify_bt_single_file(path, info, progress_callback=progress_callback)
		else:
			path = os.path.join(path, encode_path(info['name']))
			return verify_bt_single_file(path, info, progress_callback=progress_callback)
	else:
		return verify_bt_multiple(path, info, progress_callback=progress_callback)

def verify_bt_file(path, torrent_path, progress_callback=None):
	with open(torrent_path, 'rb') as x:
		return verify_bt(path, bdecode(x.read())['info'], progress_callback)

<commit_msg>check if the bt folder exists before hash checking<commit_after>
import os.path
import sys
import hashlib
from cStringIO import StringIO

default_encoding = sys.getfilesystemencoding()
if default_encoding is None or default_encoding.lower() == 'ascii':
	default_encoding = 'utf-8'

class decoder:
	def __init__(self, bytes):
		self.bytes = bytes
		self.i = 0
	def decode_value(self):
		x = self.bytes[self.i]
		if x.isdigit():
			return self.decode_string()
		self.i += 1
		if x == 'd':
			v = {}
			while self.peek() != 'e':
				k = self.decode_string()
				v[k] = self.decode_value()
			self.i += 1
			return v
		elif x == 'l':
			v = []
			while self.peek() != 'e':
				v.append(self.decode_value())
			self.i += 1
			return v
		elif x == 'i':
			return self.decode_int()
		else:
			raise NotImplementedError(x)
	def decode_string(self):
		i = self.bytes.index(':', self.i)
		n = int(self.bytes[self.i:i])
		s = self.bytes[i+1:i+1+n]
		self.i = i + 1 + n
		return s
	def decode_int(self):
		e = self.bytes.index('e', self.i)
		n = int(self.bytes[self.i:e])
		self.i = e + 1
		return n
	def peek(self):
		return self.bytes[self.i]

class encoder:
	def __init__(self, stream):
		self.stream = stream
	def encode(self, v):
		if type(v) == str:
			self.stream.write(str(len(v)))
			self.stream.write(':')
			self.stream.write(v)
		elif type(v) == dict:
			self.stream.write('d')
			for k in sorted(v):
				self.encode(k)
				self.encode(v[k])
			self.stream.write('e')
		elif type(v) == list:
			self.stream.write('l')
			for x in v:
				self.encode(x)
			self.stream.write('e')
		elif type(v) == int:
			self.stream.write('i')
			self.stream.write(str(v))
			self.stream.write('e')
		else:
			raise NotImplementedError(type(v))

def bdecode(bytes):
	return decoder(bytes).decode_value()

def bencode(v):
	from cStringIO import StringIO
	stream = StringIO()
	encoder(stream).encode(v)
	return stream.getvalue()

def info_hash(path):
	with open(path, 'rb') as stream:
		return hashlib.sha1(bencode(bdecode(stream.read())['info'])).hexdigest()

def encode_path(path):
	return path.decode('utf-8').encode(default_encoding)

class sha1_reader:
	def __init__(self, pieces, progress_callback=None):
		assert pieces
		assert len(pieces) % 20 == 0
		self.total = len(pieces)/20
		self.processed = 0
		self.stream = StringIO(pieces)
		self.progress_callback = progress_callback
	def next_sha1(self):
		self.processed += 1
		if self.progress_callback:
			self.progress_callback(float(self.processed)/self.total)
		return self.stream.read(20)

def verify_bt_single_file(path, info, progress_callback=None):
	# TODO: check md5sum if available
	if os.path.getsize(path) != info['length']:
		return False
	piece_length = info['piece length']
	assert piece_length <= 1024*1024
	sha1_stream = sha1_reader(info['pieces'], progress_callback=progress_callback)
	with open(path, 'rb') as stream:
		while True:
			bytes = stream.read(piece_length)
			sha1 = sha1_stream.next_sha1()
			if bytes:
				assert len(sha1) == 20
				if hashlib.sha1(bytes).digest() != sha1:
					return False
			else:
				assert len(sha1) == 0
				break
	assert sha1_stream.next_sha1() == ''
	return True

def verify_bt_multiple(folder, info, progress_callback=None):
	# TODO: check md5sum if available
	piece_length = info['piece length']
	assert piece_length <= 1024*1024
	files = [{'path':os.path.join(folder, apply(os.path.join, x['path'])), 'length':x['length']} for x in info['files']]

	sha1_stream = sha1_reader(info['pieces'], progress_callback=progress_callback)
	sha1sum = hashlib.sha1()

	piece_left = piece_length
	complete_piece = True

	while files:
		f = files.pop(0)
		path = f['path']
		size = f['length']
		if os.path.exists(path):
			if os.path.getsize(path) != size:
				return False
			if size <= piece_left:
				with open(path, 'rb') as stream:
					bytes = stream.read()
				assert len(bytes) == size
				sha1sum.update(bytes)
				piece_left -= size
				if not piece_left:
					if complete_piece and sha1sum.digest() != sha1_stream.next_sha1():
						return False
					complete_piece = True
					sha1sum = hashlib.sha1()
					piece_left = piece_length
			else:
				with open(path, 'rb') as stream:
					while size >= piece_left:
						bytes = stream.read(piece_left)
						assert len(bytes) == piece_left
						size -= piece_left
						sha1sum.update(bytes)
						if sha1sum.digest() != sha1_stream.next_sha1():
							return False
						sha1sum = hashlib.sha1()
						piece_left = piece_length
					if size:
						bytes = stream.read(size)
						assert len(bytes) == size
						sha1sum.update(bytes)
		else:
			while size >= piece_left:
				size -= piece_left
				sha1_stream.next_sha1()
				sha1sum = hashlib.sha1()
				piece_left = piece_length
			if size:
				complete_piece = False
				piece_left -= size
			else:
				complete_piece = True

	if piece_left < piece_length:
		if complete_piece:
			if sha1sum.digest() != sha1_stream.next_sha1():
				return False
		else:
			sha1_stream.next_sha1()
	assert sha1_stream.next_sha1() == ''

	return True

def verify_bt(path, info, progress_callback=None):
	if not os.path.exists(path):
		raise Exception("File doesn't exist: %s" % path)
	if 'files' not in info:
		if os.path.isfile(path):
			return verify_bt_single_file(path, info, progress_callback=progress_callback)
		else:
			path = os.path.join(path, encode_path(info['name']))
			return verify_bt_single_file(path, info, progress_callback=progress_callback)
	else:
		return verify_bt_multiple(path, info, progress_callback=progress_callback)

def verify_bt_file(path, torrent_path, progress_callback=None):
	with open(torrent_path, 'rb') as x:
		return verify_bt(path, bdecode(x.read())['info'], progress_callback)

<|endoftext|>