<commit_before># -*- coding: utf-8 -*-

"""The idea for this script originates from
https://github.com/BergWerkGIS/convert-bev-address-data/blob/master/README.md

The input are the files STRASSE.csv, GEMEINDE.csv and ADRESSE.csv from the
publicly available dataset of addresses of Austria, available for download at
http://www.bev.gv.at/portal/page?_pageid=713,1604469&_dad=portal&_schema=PORTAL
"""

import csv
import argparse
from osgeo import osr
from osgeo import ogr

# command line arguments are evaluated
parser = argparse.ArgumentParser(prog='python3 convert-addresses.py')
parser.add_argument('-epsg',
                    help='Specify the EPSG code of the coordinate  system used'
                         ' for the results. If none is given, this value'
                         ' defaults to WGS84',
                    type=int, default=4326, dest='epsg')
parser.add_argument('-gkz',
                    help='Specify if GKZ should be included or not.',
                    action='store_true', dest='gkz')
args = parser.parse_args()
# the target EPSG is set according to the argument
targetRef = osr.SpatialReference()
targetRef.ImportFromEPSG(args.epsg)

westRef = osr.SpatialReference()
westRef.ImportFromEPSG(31254)
centerRef = osr.SpatialReference()
centerRef.ImportFromEPSG(31255)
eastRef = osr.SpatialReference()
eastRef.ImportFromEPSG(31256)

westTransform = osr.CoordinateTransformation(westRef, targetRef)
centralTransform = osr.CoordinateTransformation(centerRef, targetRef)
eastTransfrom = osr.CoordinateTransformation(eastRef, targetRef)

def reproject(sourceCRS, points):
    """This function reprojects an array of coordinates (a point) to EPSG:31287
    depending on their original CRS given by the parameter sourceCRS"""

    #point = ogr.CreateGeometryFromWkt("POINT (" + str(points[0]) + " " + str(points[1]) + ")")
    point = ogr.CreateGeometryFromWkt("POINT ({} {})".format(points[0],
                                                             points[1]))
    if sourceCRS == '31254':
        point.Transform(westTransform)
    elif sourceCRS == '31255':
        point.Transform(centralTransform)
    elif sourceCRS == '31256':
        point.Transform(eastTransfrom)
    else:
        print("unkown CRS: {}".format(sourceCRS))
        return([0, 0])
    wktPoint = point.ExportToWkt()
    transformedPoint = wktPoint.split("(")[1][:-1].split(" ")
    del(point)
    return [round(float(p), 6) for p in transformedPoint]

def buildHausNumber(hausnrtext, hausnrzahl1, hausnrbuchstabe1, hausnrverbindung1, hausnrzahl2, hausnrbuchstabe2, hausnrbereich):
    """This function takes all the different single parts of the input file
    that belong to the house number and combines them into one single string"""

    hausnr1 = hausnrzahl1
    hausnr2 = hausnrzahl2
    compiledHausNr = ""
    if hausnrbuchstabe1 != "": hausnr1 += hausnrbuchstabe1
    if hausnrbuchstabe2 != "": hausnr2 += hausnrbuchstabe2
    if hausnrverbindung1 != "":
        compiledHausNr = hausnr1 + hausnrverbindung1 + hausnr2
    elif hausnr2 != "":
        compiledHausNr = hausnr1 + " " + hausnr2
    else:
        compiledHausNr = hausnr1
    if hausnrtext != "": compiledHausNr += " ,{}".format(hausnrtext)
    if hausnrbereich != "keine Angabe": compiledHausNr += " ,{}".format(hausnrbereich)
    return compiledHausNr

if __name__ == '__main__':
    print("buffering streets ...")
    streetReader = csv.reader(open('STRASSE.csv', 'r'), delimiter=';', quotechar='"')

    streets = {}
    headerstreets = next(streetReader, None)
    for streetrow in streetReader:
        streets[streetrow[0]] = streetrow[1] + " " + streetrow[2]

    print("buffering districts ...")
    districtReader = csv.reader(open('GEMEINDE.csv', 'r'), delimiter=';', quotechar='"')
    districts = {}
    headerdistricts = next(districtReader, None)
    for districtrow in districtReader:
        districts[districtrow[0]] = districtrow[1]

    print("processing addresses ...")
    addressReader = csv.reader(open('ADRESSE.csv', 'r'), delimiter=';', quotechar='"')
    headeraddresses = next(addressReader, None)

    outputFilename = "bev_addressesEPSG{}.csv".format(args.epsg)
    addressWriter = csv.writer(open(outputFilename, 'w'), delimiter=";", quotechar='"')
    row = ['Gemeinde', 'plz', 'strasse', 'nummer', 'hausname', 'x', 'y']
    if args.gkz:
        row.append('gkz')
    addressWriter.writerow(row)

    # get the total file size for status output
    total_addresses = sum(1 for row in csv.reader(open('ADRESSE.csv', 'r'), delimiter=';', quotechar='"'))
    previous_percentage = 0
    # the main loop is this: each line in the ADRESSE.csv is parsed one by one
    for i, addressrow in enumerate(addressReader):
        current_percentage = round(float(i) / total_addresses * 100)
        if current_percentage != previous_percentage:
            print("{} %".format(current_percentage))
            previous_percentage = current_percentage
        streetname = streets[addressrow[4]]
        streetname = streetname.strip()  # remove the trailing whitespace after each street name
        districtname = districts[addressrow[1]]
        gkz = addressrow[1]
        plzname = addressrow[3]
        hausnr = buildHausNumber(addressrow[6], addressrow[7], addressrow[8], addressrow[9], addressrow[10], addressrow[11], addressrow[12])
        hausname = addressrow[14]
        x = addressrow[15]
        y = addressrow[16]
        # some entries don't have coordinates: ignore these entries
        if x == '' or y == '':
            continue
        usedprojection = addressrow[17]
        coords = reproject(usedprojection, [x, y])
        # if the reprojection returned [0,0], this indicates an error: ignore these entries
        if coords[0] == '0' or coords[1] == '0':
            continue
        row = [districtname, plzname, streetname, hausnr, hausname, coords[0], coords[1]]
        if args.gkz:
            row.append(gkz)
        addressWriter.writerow(row)
    print("finished")
<commit_msg>added check if STRASSE.csv is existing and added a nicer progress bar<commit_after># -*- coding: utf-8 -*-

"""The idea for this script originates from
https://github.com/BergWerkGIS/convert-bev-address-data/blob/master/README.md

The input are the files STRASSE.csv, GEMEINDE.csv and ADRESSE.csv from the
publicly available dataset of addresses of Austria, available for download at
http://www.bev.gv.at/portal/page?_pageid=713,1604469&_dad=portal&_schema=PORTAL
"""

import sys
import csv
import argparse
from osgeo import osr
from osgeo import ogr

# command line arguments are evaluated
parser = argparse.ArgumentParser(prog='python3 convert-addresses.py')
parser.add_argument('-epsg',
                    help='Specify the EPSG code of the coordinate  system used'
                         ' for the results. If none is given, this value'
                         ' defaults to WGS84',
                    type=int, default=4326, dest='epsg')
parser.add_argument('-gkz',
                    help='Specify if GKZ should be included or not.',
                    action='store_true', dest='gkz')
args = parser.parse_args()
# the target EPSG is set according to the argument
targetRef = osr.SpatialReference()
targetRef.ImportFromEPSG(args.epsg)

westRef = osr.SpatialReference()
westRef.ImportFromEPSG(31254)
centerRef = osr.SpatialReference()
centerRef.ImportFromEPSG(31255)
eastRef = osr.SpatialReference()
eastRef.ImportFromEPSG(31256)

westTransform = osr.CoordinateTransformation(westRef, targetRef)
centralTransform = osr.CoordinateTransformation(centerRef, targetRef)
eastTransfrom = osr.CoordinateTransformation(eastRef, targetRef)

def reproject(sourceCRS, points):
    """This function reprojects an array of coordinates (a point) to EPSG:31287
    depending on their original CRS given by the parameter sourceCRS"""

    #point = ogr.CreateGeometryFromWkt("POINT (" + str(points[0]) + " " + str(points[1]) + ")")
    point = ogr.CreateGeometryFromWkt("POINT ({} {})".format(points[0],
                                                             points[1]))
    if sourceCRS == '31254':
        point.Transform(westTransform)
    elif sourceCRS == '31255':
        point.Transform(centralTransform)
    elif sourceCRS == '31256':
        point.Transform(eastTransfrom)
    else:
        print("unkown CRS: {}".format(sourceCRS))
        return([0, 0])
    wktPoint = point.ExportToWkt()
    transformedPoint = wktPoint.split("(")[1][:-1].split(" ")
    del(point)
    return [round(float(p), 6) for p in transformedPoint]

def buildHausNumber(hausnrtext, hausnrzahl1, hausnrbuchstabe1, hausnrverbindung1, hausnrzahl2, hausnrbuchstabe2, hausnrbereich):
    """This function takes all the different single parts of the input file
    that belong to the house number and combines them into one single string"""

    hausnr1 = hausnrzahl1
    hausnr2 = hausnrzahl2
    compiledHausNr = ""
    if hausnrbuchstabe1 != "": hausnr1 += hausnrbuchstabe1
    if hausnrbuchstabe2 != "": hausnr2 += hausnrbuchstabe2
    if hausnrverbindung1 != "":
        compiledHausNr = hausnr1 + hausnrverbindung1 + hausnr2
    elif hausnr2 != "":
        compiledHausNr = hausnr1 + " " + hausnr2
    else:
        compiledHausNr = hausnr1
    if hausnrtext != "": compiledHausNr += " ,{}".format(hausnrtext)
    if hausnrbereich != "keine Angabe": compiledHausNr += " ,{}".format(hausnrbereich)
    return compiledHausNr

if __name__ == '__main__':
    print("buffering streets ...")
    try:
        streetReader = csv.reader(open('STRASSE.csv', 'r'), delimiter=';', quotechar='"')
    except IOError:
        print("The file 'STRASSE.csv' was not found. Please download and unpack the BEV Address data from http://www.bev.gv.at/portal/page?_pageid=713,1604469&_dad=portal&_schema=PORTAL")

    streets = {}
    headerstreets = next(streetReader, None)
    for streetrow in streetReader:
        streets[streetrow[0]] = streetrow[1] + " " + streetrow[2]

    print("buffering districts ...")
    districtReader = csv.reader(open('GEMEINDE.csv', 'r'), delimiter=';', quotechar='"')
    districts = {}
    headerdistricts = next(districtReader, None)
    for districtrow in districtReader:
        districts[districtrow[0]] = districtrow[1]

    print("processing addresses ...")
    addressReader = csv.reader(open('ADRESSE.csv', 'r'), delimiter=';', quotechar='"')
    headeraddresses = next(addressReader, None)

    outputFilename = "bev_addressesEPSG{}.csv".format(args.epsg)
    addressWriter = csv.writer(open(outputFilename, 'w'), delimiter=";", quotechar='"')
    row = ['Gemeinde', 'plz', 'strasse', 'nummer', 'hausname', 'x', 'y']
    if args.gkz:
        row.append('gkz')
    addressWriter.writerow(row)

    # get the total file size for status output
    total_addresses = sum(1 for row in csv.reader(open('ADRESSE.csv', 'r'), delimiter=';', quotechar='"'))
    previous_percentage = 0
    # the main loop is this: each line in the ADRESSE.csv is parsed one by one
    for i, addressrow in enumerate(addressReader):
        current_percentage = round(float(i) / total_addresses * 100,1)
        if current_percentage != previous_percentage:
            # we draw a nice progess bar
            sys.stdout.write("\r{} %   ".format(current_percentage))
            sys.stdout.write('[{}]'.format(('#' * (int(current_percentage) / 2)).ljust(50)))
            sys.stdout.flush()
            previous_percentage = current_percentage
        streetname = streets[addressrow[4]]
        streetname = streetname.strip()  # remove the trailing whitespace after each street name
        districtname = districts[addressrow[1]]
        gkz = addressrow[1]
        plzname = addressrow[3]
        hausnr = buildHausNumber(addressrow[6], addressrow[7], addressrow[8], addressrow[9], addressrow[10], addressrow[11], addressrow[12])
        hausname = addressrow[14]
        x = addressrow[15]
        y = addressrow[16]
        # some entries don't have coordinates: ignore these entries
        if x == '' or y == '':
            continue
        usedprojection = addressrow[17]
        coords = reproject(usedprojection, [x, y])
        # if the reprojection returned [0,0], this indicates an error: ignore these entries
        if coords[0] == '0' or coords[1] == '0':
            continue
        row = [districtname, plzname, streetname, hausnr, hausname, coords[0], coords[1]]
        if args.gkz:
            row.append(gkz)
        addressWriter.writerow(row)
    print("finished")
<|endoftext|>


<commit_before>"""
"""
import os
import commands

class handbrake(object):

    """ Function:   _cleanUp
            Removes the log file and the input movie because these files are
                no longer needed by this script

        Inputs:
            log         (Str): File path of the log to remove
            oldMovie    (Str): File path of the movie to remove

        Outputs:
            None
    """
    def _cleanUp(log, oldMovie):
        os.remove(log)
        os.remove(oldMovie)

    """ Function:   _updateQueue
            Removes the recently processed movie from the queue so it's not
                processed again

        Inputs:
            None

        Outputs:
            None
    """
    def _updateQueue(self):
        f = open("%s/.makemkvautoripper/queue" % self.home, "w")
        for line in self.lines:
            if (self.movie not in line):
                f.write(line)

        f.write("\n")
        f.close()

    """ Function:   _cleanQueue

        Inputs:
            None

        Outputs:
            None
    """
    def _cleanQueue(self):
        try:
            f = open("%s/.makemkvautoripper/queue" % self.home, "r")
            lines = f.readlines()
            f.close()
        except:
            print "Could not read queue file"

        f = open("%s/.makemkvautoripper/queue" % self.home, "w")

        for line in lines:
            if (line != "\n"):
                f.write(line)

        f.write("\n")
        f.close()

    """ Function:   findProcess

        Inputs:
            None

        Outputs:
            None
    """
    def findProcess(self):
        processname = 'HandBrakeCLI'
        for line in os.popen("ps xa"):
            fields = line.split()
            process = fields[4]
            if process.find(processname) >= 0:
                return True
                break

        return False

    """ Function:   loadMovie

        Inputs:
            None

        Outputs:
            None
    """
    def loadMovie(self):
        try:
            self.home = os.path.expanduser("~")
            self._cleanQueue()

            if os.path.isfile("%s/.makemkvautoripper/queue" % self.home):
                f = open("%s/.makemkvautoripper/queue" % self.home, "r")
                self.lines = f.readlines()
                f.close()

                movie = self.lines[0].replace("\n", "")
                movie = movie.split('|')

                self.path = movie[0]
                self.movie = movie[1]
                self.inputMovie = movie[2]
                self.outputMovie = movie[3]

                return True
            else:
                return False
        except:
            return False

    """ Function:   convert
            Passes the nessesary parameters to HandBrake to start an encoding
            Assigns a nice value to allow give normal system tasks priority

            Upon successful encode, clean up the output logs and remove the
                input movie as they are no longer needed

        Inputs:
            nice    (Int): Priority to assign to task (nice value)
            args    (Str): All of the handbrake arguments taken from the
                            settings file
            output  (Str): File to log to. Used to see if the job completed
                            successfully

        Outputs:
            None
    """
    def convert(self, nice, args, output):
        path = "%s/%s" % (self.path, self.movie)
        inMovie = "%s/%s" % (path, self.inputMovie)
        outMovie = "%s/%s" % (path, self.outputMovie)

        print "Converting..."
        commands.getstatusoutput(
            'nice -n %d HandBrakeCLI --verbose 1 -i "%s" -o "%s" %s 2> %s'
            %
            (nice, inMovie, outMovie, args, output))

        checks = 0
        try:
            tempFile = open(output, 'r')
            for line in tempFile.readlines():
                if "average encoding speed for job" in line:
                    checks += 1
                if "Encode done!" in line:
                    checks += 1
        except:
            print "Could not read output file, no cleanup will be done"

        if checks == 0:
            self._updateQueue()
            self._cleanUp(log=output, oldMovie=inMovie)
<commit_msg>Added more comments<commit_after>"""
"""
import os
import commands

class handbrake(object):

    """ Function:   _cleanUp
            Removes the log file and the input movie because these files are
                no longer needed by this script

        Inputs:
            log         (Str): File path of the log to remove
            oldMovie    (Str): File path of the movie to remove

        Outputs:
            None
    """
    def _cleanUp(log, oldMovie):
        os.remove(log)
        os.remove(oldMovie)

    """ Function:   _updateQueue
            Removes the recently processed movie from the queue so it's not
                processed again

        Inputs:
            None

        Outputs:
            None
    """
    def _updateQueue(self):
        f = open("%s/.makemkvautoripper/queue" % self.home, "w")
        for line in self.lines:
            if (self.movie not in line):
                f.write(line)

        f.write("\n")
        f.close()

    """ Function:   _cleanQueue
            Removes blank lines and excess new lines

        Inputs:
            None

        Outputs:
            None
    """
    def _cleanQueue(self):
        try:
            f = open("%s/.makemkvautoripper/queue" % self.home, "r")
            lines = f.readlines()
            f.close()
        except:
            print "Could not read queue file"

        f = open("%s/.makemkvautoripper/queue" % self.home, "w")

        for line in lines:
            if (line != "\n"):
                f.write(line)

        f.write("\n")
        f.close()

    """ Function:   findProcess
            Goes through all of the running proccess and tries to find the
                HandBrake proccess.
            If it is running return true

        Inputs:
            None

        Outputs:
            Boolean (True/False)
    """
    def findProcess(self):
        processname = 'HandBrakeCLI'
        for line in os.popen("ps xa"):
            fields = line.split()
            process = fields[4]
            if process.find(processname) >= 0:
                return True
                break

        return False

    """ Function:   loadMovie
            Check to see if the queue file exists, if it does load the first
                line and proccess it for the rest of the script to use

        Inputs:
            None

        Outputs:
            None
    """
    def loadMovie(self):
        try:
            self.home = os.path.expanduser("~")
            self._cleanQueue()

            if os.path.isfile("%s/.makemkvautoripper/queue" % self.home):
                f = open("%s/.makemkvautoripper/queue" % self.home, "r")
                self.lines = f.readlines()
                f.close()

                movie = self.lines[0].replace("\n", "")
                movie = movie.split('|')

                self.path = movie[0]
                self.movie = movie[1]
                self.inputMovie = movie[2]
                self.outputMovie = movie[3]

                return True
            else:
                return False
        except:
            return False

    """ Function:   convert
            Passes the nessesary parameters to HandBrake to start an encoding
            Assigns a nice value to allow give normal system tasks priority

            Upon successful encode, clean up the output logs and remove the
                input movie as they are no longer needed

        Inputs:
            nice    (Int): Priority to assign to task (nice value)
            args    (Str): All of the handbrake arguments taken from the
                            settings file
            output  (Str): File to log to. Used to see if the job completed
                            successfully

        Outputs:
            None
    """
    def convert(self, nice, args, output):
        path = "%s/%s" % (self.path, self.movie)
        inMovie = "%s/%s" % (path, self.inputMovie)
        outMovie = "%s/%s" % (path, self.outputMovie)

        print "Converting..."
        commands.getstatusoutput(
            'nice -n %d HandBrakeCLI --verbose 1 -i "%s" -o "%s" %s 2> %s'
            %
            (nice, inMovie, outMovie, args, output))

        checks = 0
        try:
            tempFile = open(output, 'r')
            for line in tempFile.readlines():
                if "average encoding speed for job" in line:
                    checks += 1
                if "Encode done!" in line:
                    checks += 1
        except:
            print "Could not read output file, no cleanup will be done"

        if checks == 0:
            self._updateQueue()
            self._cleanUp(log=output, oldMovie=inMovie)
<|endoftext|>


<commit_before>import os

class Config(object):
    DEBUG = False
    ROOT_DIR = os.environ['HOME']

class Prod(Config):
    DEBUG = False
    MONGO_URL = os.environ['MONGO_URL']

class Dev(Config):
    DEBUG = True
    MONGO_URL = {
        'host': 'mongodb://USERNAME:PASSWORD@HOST',
    }<commit_msg>Rename MONGO_URL to MONGO_URI<commit_after>import os

class Config(object):
    DEBUG = False
    ROOT_DIR = os.environ['HOME']

class Prod(Config):
    DEBUG = False
    MONGO_URI = os.environ['MONGO_URI']

class Dev(Config):
    DEBUG = True
    MONGO_URI = {
        'host': 'mongodb://USERNAME:PASSWORD@HOST',
    }<|endoftext|>


<commit_before>import sublime
from sublime_plugin import WindowCommand

from ..git_command import GitCommand
from ...common import util

class GsLogCommand(WindowCommand, GitCommand):

    def run(self, filename=None, limit=6000, author=None):
        self._pagination = 0
        self._filename = filename
        self._limit = limit
        self._author = author
        sublime.set_timeout_async(lambda: self.run_async(), 1)

    def run_async(self):
        log_output = self.git(
            "log",
            "-{}".format(self._limit) if self._limit else None,
            "--skip={}".format(self._pagination) if self._pagination else None,
            "--author={}".format(self._author) if self._author else None,
            '--format=%h%n%H%n%s%n%an%n%at%x00',
            "--" if self._filename else None,
            self._filename
        ).strip("\x00")

        self._entries = []
        self._hashes = []
        for entry in log_output.split("\x00"):
            try:
                short_hash, long_hash, summary, author, datetime = entry.strip("\n").split("\n")
                self._entries.append([
                    short_hash + " " + summary,
                    author + ", " + util.dates.fuzzy(datetime)
                ])
                self._hashes.append(long_hash)

            except ValueError:
                # Empty line - less expensive to catch the exception once than
                # to check truthiness of entry.strip() each time.
                pass

        if not len(self._entries) < self._limit:
            self._entries.append([
                ">>> NEXT {} COMMITS >>>".format(self._limit),
                "Skip this set of commits and choose from the next-oldest batch."
            ])

        self.window.show_quick_panel(
            self._entries,
            self.on_selection,
            flags=sublime.MONOSPACE_FONT
        )

    def on_selection(self, index):
        if index == -1:
            return
        if index == self._limit:
            self._pagination += self._limit
            sublime.set_timeout_async(lambda: self.run_async(), 1)
            return

        selected_hash = self._hashes[index]
        self.window.run_command("gs_show_commit", {"commit_hash": selected_hash})

class GsLogCurrentFileCommand(WindowCommand, GitCommand):

    def run(self):
        self.window.run_command("gs_log", {"filename": self.file_path})

class GsLogByAuthorCommand(WindowCommand, GitCommand):

    """
    Prompt the user for author pattern, pre-filled with local user's
    git name and email.  Once provided, display a quick panel with all
    commits made by the specified author.
    """

    def run(self):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        name = self.git("config", "user.name").strip()
        email = self.git("config", "user.email").strip()
        default_author = "{} <{}>".format(name, email)
        print(default_author)
        self.window.show_input_panel(
            "Enter author pattern:",
            default_author,
            self.on_entered,
            None,
            None
            )

    def on_entered(self, author_text):
        self.window.run_command("gs_log", {"author": author_text})
<commit_msg>Fix: Display `git: log` quick panel before keypress in Windows.<commit_after>import sublime
from sublime_plugin import WindowCommand

from ..git_command import GitCommand
from ...common import util

class GsLogCommand(WindowCommand, GitCommand):

    def run(self, filename=None, limit=6000, author=None):
        self._pagination = 0
        self._filename = filename
        self._limit = limit
        self._author = author
        sublime.set_timeout_async(self.run_async)

    def run_async(self):
        log_output = self.git(
            "log",
            "-{}".format(self._limit) if self._limit else None,
            "--skip={}".format(self._pagination) if self._pagination else None,
            "--author={}".format(self._author) if self._author else None,
            '--format=%h%n%H%n%s%n%an%n%at%x00',
            "--" if self._filename else None,
            self._filename
        ).strip("\x00")

        self._entries = []
        self._hashes = []
        for entry in log_output.split("\x00"):
            try:
                short_hash, long_hash, summary, author, datetime = entry.strip("\n").split("\n")
                self._entries.append([
                    short_hash + " " + summary,
                    author + ", " + util.dates.fuzzy(datetime)
                ])
                self._hashes.append(long_hash)

            except ValueError:
                # Empty line - less expensive to catch the exception once than
                # to check truthiness of entry.strip() each time.
                pass

        if not len(self._entries) < self._limit:
            self._entries.append([
                ">>> NEXT {} COMMITS >>>".format(self._limit),
                "Skip this set of commits and choose from the next-oldest batch."
            ])

        self.window.show_quick_panel(
            self._entries,
            self.on_selection,
            flags=sublime.MONOSPACE_FONT
        )

    def on_selection(self, index):
        if index == -1:
            return
        if index == self._limit:
            self._pagination += self._limit
            sublime.set_timeout_async(lambda: self.run_async(), 1)
            return

        selected_hash = self._hashes[index]
        self.window.run_command("gs_show_commit", {"commit_hash": selected_hash})

class GsLogCurrentFileCommand(WindowCommand, GitCommand):

    def run(self):
        self.window.run_command("gs_log", {"filename": self.file_path})

class GsLogByAuthorCommand(WindowCommand, GitCommand):

    """
    Prompt the user for author pattern, pre-filled with local user's
    git name and email.  Once provided, display a quick panel with all
    commits made by the specified author.
    """

    def run(self):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        name = self.git("config", "user.name").strip()
        email = self.git("config", "user.email").strip()
        default_author = "{} <{}>".format(name, email)
        print(default_author)
        self.window.show_input_panel(
            "Enter author pattern:",
            default_author,
            self.on_entered,
            None,
            None
            )

    def on_entered(self, author_text):
        self.window.run_command("gs_log", {"author": author_text})
<|endoftext|>


<commit_before>from django.utils.timesince import timesince
from django.core.urlresolvers import reverse
from restlib import http, resources
from avocado.fields import logictree
from avocado.store.forms import ScopeForm, SessionScopeForm

__all__ = ('ScopeResource', 'SessionScopeResource', 'ScopeResourceCollection')

class ScopeResource(resources.ModelResource):
    model = 'avocado.Scope'

    default_for_related = False

    fields = (':pk', 'name', 'description', 'store', 'count',
        'conditions', 'has_changed', 'timesince')

    middleware = (
        'serrano.api.middleware.NeverCache',
    ) + resources.Resource.middleware

    @classmethod
    def timesince(self, obj):
        if obj.modified:
            return '%s ago' % timesince(obj.modified)

    @classmethod
    def queryset(self, request):
        return self.model._default_manager.filter(user=request.user)

    def DELETE(self, request, pk):
        scope = request.session['scope']

        # the requested object is the currently referenced by the session,
        # now we delete and deference
        if scope.references(pk):
            scope.reference.delete()
            scope.reference = None
            scope.save()
        else:
            target = self.queryset(request).filter(pk=pk)
            target.delete()

        # nothing to see..
        return http.NO_CONTENT

    def GET(self, request, pk):
        "Fetches a scope and sets the session scope to be a proxy."
        scope = request.session['scope']

        # if this object is already referenced by the session, simple return
        if scope.references(pk):
            return scope

        # attempt to fetch the requested object
        target = self.get(request, pk=pk)
        if not target:
            return http.NOT_FOUND

        # setup the reference on the session object,
        target.reset(scope, exclude=('pk', 'reference', 'session'))
        scope.reference = target
        scope.commit()
        return scope

    def PUT(self, request, pk):
        "Explicitly updates an existing object given the request data."
        scope = request.session['scope']

        if scope.references(pk):
            target = scope.reference
        else:
            target = self.get(request, pk=pk)
            if not target:
                return http.NOT_FOUND

        form = ScopeForm(request.data, instance=target)

        if form.is_valid():
            target = form.save()
            # update the session to reflect the new changes
            target.reset(scope, exclude=('pk', 'reference', 'session'))
            scope.reference = target
            scope.commit()
            return target

        return form.errors

class SessionScopeResource(ScopeResource):
    default_for_related = True

    fields = (':pk', 'name', 'description', 'store', 'count',
        'condition_groups->conditions', 'has_changed', 'timesince', 'reference')

    @classmethod
    def _condition(self, json):
        text = logictree.transform(json).text

        j = ''
        if text.has_key('type'):
            j = ' %s ' % text['type']

        return {
            'concept_id': json['concept_id'],
            'condition': j.join(text['conditions'])
        }

    @classmethod
    def condition_groups(self, obj):
        if obj.store:
            if obj.store.has_key('children'):
                return map(self._condition, obj.store['children'])
            return [self._condition(obj.store)]

    @classmethod
    def reference(self, obj):
        if obj.reference:
            return {
                'id': obj.reference.pk,
                'url': reverse('api:scope:read', args=[obj.reference.pk]),
            }

    def GET(self, request):
        "Return this session's current perspective."
        return request.session['scope']

    def POST(self, request):
        json = request.data

        if not any(x in json for x in ('type', 'operator')):
            return http.BAD_REQUEST

        return self._condition(json)

    def PUT(self, request):
        "Explicitly updates an existing object given the request data."
        scope = request.session['scope']

        form = SessionScopeForm(request.data, instance=scope)

        if form.is_valid():
            form.save()
            return scope
        return form.errors

    def PATCH(self, request):
        """Adds support for incrementally updating the Scope store. It handles
        adding/removing a condition(s) for a single Concept.
        """
        scope = request.session['scope']
        operation = request.data.get('operation', None)
        condition = request.data.get('condition', None)

        # both must exist to be processed
        if not operation or not condition:
            return http.UNPROCESSABLE_ENTITY

        concept_id = condition['concept_id']

        # TODO this logic assumes concept conditions are only first-level
        # children. when concept conditions can be nested, this will need
        # to be more robust at checking
        if operation == 'remove':
            if not scope.store:
                return http.CONFLICT

            # denotes a logical operator node e.g. AND | OR.
            if scope.store.has_key('children'):
                for i, x in enumerate(iter(scope.store['children'])):
                    # TODO this logic assumes one condition per concept, update
                    # this once this is not the case
                    if x['concept_id'] == concept_id:
                        scope.store['children'].pop(i)
                        # move up the condition to the top node if it is the
                        # last one
                        if len(scope.store['children']) == 1:
                            scope.store = scope.store['children'][0]
                        break
                else:
                    return http.CONFLICT

            # standalone condition
            elif scope.store.get('concept_id') == concept_id:
                scope.store = None

            # a conflict in state between the client and the server
            else:
                return http.CONFLICT

        # add operations must check the validity and permission of the
        # requested content
        elif operation == 'add':
            # ensure the object is valid or fail
            if not scope.is_valid(condition):
                return http.UNPROCESSABLE_ENTITY

            # ensure the user has permission to make use of corresponding fields
            if not scope.has_permission(condition, request.user):
                return http.UNAUTHORIZED

            if not scope.store:
                scope.store = condition
            elif scope.store.has_key('children'):
                scope.store['children'].append(condition)
            else:
                scope.store = {'type': 'and', 'children': [scope.store, condition]}

        # any other operations are not supported
        else:
            return http.UNPROCESSABLE_ENTITY

        scope.save()
        return self._condition(condition)

class ScopeResourceCollection(resources.ModelResourceCollection):
    resource = ScopeResource
<commit_msg>Update PATCH request body format More terse format where the key is the operation and the value is the condition, e.g. {"add": {…}}<commit_after>from django.utils.timesince import timesince
from django.core.urlresolvers import reverse
from restlib import http, resources
from avocado.fields import logictree
from avocado.store.forms import ScopeForm, SessionScopeForm

__all__ = ('ScopeResource', 'SessionScopeResource', 'ScopeResourceCollection')

PATCH_OPERATIONS = ('add', 'remote')

class ScopeResource(resources.ModelResource):
    model = 'avocado.Scope'

    default_for_related = False

    fields = (':pk', 'name', 'description', 'store', 'count',
        'conditions', 'has_changed', 'timesince')

    middleware = (
        'serrano.api.middleware.NeverCache',
    ) + resources.Resource.middleware

    @classmethod
    def timesince(self, obj):
        if obj.modified:
            return '%s ago' % timesince(obj.modified)

    @classmethod
    def queryset(self, request):
        return self.model._default_manager.filter(user=request.user)

    def DELETE(self, request, pk):
        scope = request.session['scope']

        # the requested object is the currently referenced by the session,
        # now we delete and deference
        if scope.references(pk):
            scope.reference.delete()
            scope.reference = None
            scope.save()
        else:
            target = self.queryset(request).filter(pk=pk)
            target.delete()

        # nothing to see..
        return http.NO_CONTENT

    def GET(self, request, pk):
        "Fetches a scope and sets the session scope to be a proxy."
        scope = request.session['scope']

        # if this object is already referenced by the session, simple return
        if scope.references(pk):
            return scope

        # attempt to fetch the requested object
        target = self.get(request, pk=pk)
        if not target:
            return http.NOT_FOUND

        # setup the reference on the session object,
        target.reset(scope, exclude=('pk', 'reference', 'session'))
        scope.reference = target
        scope.commit()
        return scope

    def PUT(self, request, pk):
        "Explicitly updates an existing object given the request data."
        scope = request.session['scope']

        if scope.references(pk):
            target = scope.reference
        else:
            target = self.get(request, pk=pk)
            if not target:
                return http.NOT_FOUND

        form = ScopeForm(request.data, instance=target)

        if form.is_valid():
            target = form.save()
            # update the session to reflect the new changes
            target.reset(scope, exclude=('pk', 'reference', 'session'))
            scope.reference = target
            scope.commit()
            return target

        return form.errors

class SessionScopeResource(ScopeResource):
    default_for_related = True

    fields = (':pk', 'name', 'description', 'store', 'count',
        'condition_groups->conditions', 'has_changed', 'timesince', 'reference')

    @classmethod
    def _condition(self, json):
        text = logictree.transform(json).text

        j = ''
        if text.has_key('type'):
            j = ' %s ' % text['type']

        return {
            'concept_id': json['concept_id'],
            'condition': j.join(text['conditions'])
        }

    @classmethod
    def condition_groups(self, obj):
        if obj.store:
            if obj.store.has_key('children'):
                return map(self._condition, obj.store['children'])
            return [self._condition(obj.store)]

    @classmethod
    def reference(self, obj):
        if obj.reference:
            return {
                'id': obj.reference.pk,
                'url': reverse('api:scope:read', args=[obj.reference.pk]),
            }

    def GET(self, request):
        "Return this session's current perspective."
        return request.session['scope']

    def POST(self, request):
        json = request.data

        if not any(x in json for x in ('type', 'operator')):
            return http.BAD_REQUEST

        return self._condition(json)

    def PUT(self, request):
        "Explicitly updates an existing object given the request data."
        scope = request.session['scope']

        form = SessionScopeForm(request.data, instance=scope)

        if form.is_valid():
            form.save()
            return scope
        return form.errors

    def PATCH(self, request):
        """Adds support for incrementally updating the Scope store. It handles
        adding/removing a condition(s) for a single Concept.
        """
        scope = request.session['scope']

        if len(request.data) != 1:
            return http.UNPROCESSABLE_ENTITY

        operation, condition = request.data.items()[0]

        # both must exist to be processed
        if operation not in PATCH_OPERATIONS or not condition:
            return http.UNPROCESSABLE_ENTITY

        concept_id = condition['concept_id']

        # TODO this logic assumes concept conditions are only first-level
        # children. when concept conditions can be nested, this will need
        # to be more robust at checking
        if operation == 'remove':
            if not scope.store:
                return http.CONFLICT

            # denotes a logical operator node e.g. AND | OR.
            if scope.store.has_key('children'):
                for i, x in enumerate(iter(scope.store['children'])):
                    # TODO this logic assumes one condition per concept, update
                    # this once this is not the case
                    if x['concept_id'] == concept_id:
                        scope.store['children'].pop(i)
                        # move up the condition to the top node if it is the
                        # last one
                        if len(scope.store['children']) == 1:
                            scope.store = scope.store['children'][0]
                        break
                else:
                    return http.CONFLICT

            # standalone condition
            elif scope.store.get('concept_id') == concept_id:
                scope.store = None

            # a conflict in state between the client and the server
            else:
                return http.CONFLICT

        # add operations must check the validity and permission of the
        # requested content
        elif operation == 'add':
            # ensure the object is valid or fail
            if not scope.is_valid(condition):
                return http.UNPROCESSABLE_ENTITY

            # ensure the user has permission to make use of corresponding fields
            if not scope.has_permission(condition, request.user):
                return http.UNAUTHORIZED

            if not scope.store:
                scope.store = condition
            elif scope.store.has_key('children'):
                scope.store['children'].append(condition)
            else:
                scope.store = {'type': 'and', 'children': [scope.store, condition]}

        # any other operations are not supported
        else:
            return http.UNPROCESSABLE_ENTITY

        scope.save()
        return self._condition(condition)

class ScopeResourceCollection(resources.ModelResourceCollection):
    resource = ScopeResource
<|endoftext|>


<commit_before>import os
import logging
import pymongo
from bson.objectid import ObjectId
from flask import current_app as app
from superdesk import get_resource_service
from superdesk.notification import push_notification

from .app_settings import BLOGSLIST_ASSETS_DIR
from .exceptions import MediaStorageUnsupportedForBlogPublishing

logger = logging.getLogger('superdesk')

def is_s3_storage_enabled():
    return type(app.media).__name__ is 'AmazonMediaStorage'

def check_media_storage():
    if not is_s3_storage_enabled():
        raise MediaStorageUnsupportedForBlogPublishing()

def get_blog_path(blog_id, theme=None, output_id=None):
    return 'blogs/{}/{}{}index.html'.format(blog_id,
                                            '{}/'.format(theme) if theme else '',
                                            '{}/'.format(output_id) if output_id else '')

def get_bloglist_path():
    return os.path.join(BLOGSLIST_ASSETS_DIR, 'index.html')

def is_relative_to_current_folder(url):
    return not (url.startswith('/') or url.startswith('http://') or url.startswith('https://'))

def get_blog(blog_or_id):
    blogs = get_resource_service('client_blogs')
    if isinstance(blog_or_id, (str, ObjectId)):
        blog_id = blog_or_id
        blog = blogs.find_one(req=None, _id=blog_or_id)
    elif isinstance(blog_or_id, dict):
        blog = blog_or_id
        blog_id = blog['_id']
    else:
        raise ValueError(blog_or_id)

    return blog_id, blog

def is_seo_enabled(blog_or_id):
    """
    Return True if blog has seo output enabled in theme.
    """
    blog_id, blog = get_blog(blog_or_id)
    if not blog:
        return

    themes = get_resource_service('themes')
    blog_preferences = blog.get('blog_preferences')
    if not blog_preferences:
        return

    theme_name = blog_preferences.get('theme')
    if not theme_name:
        return

    theme = themes.find_one(req=None, name=theme_name)
    if not theme:
        return
    if theme.get('seoTheme'):
        return True

    outputs_service = get_resource_service('outputs')
    for output in outputs_service.get(req=None, lookup=dict(blog=blog_id)):
        theme_name = output.get('theme')
        if not theme_name:
            continue

        theme = themes.find_one(req=None, name=theme_name)
        if not theme:
            continue

        if theme.get('seoTheme'):
            return True

def check_limit_and_delete_oldest(blog_id):
    """
    Simple method that checks if a blog has reached the limit of posts and
    deletes ({deleted: true}) the oldest post that belongs to the given blog
    """

    UNLIMITED = 0

    blog = get_resource_service('blogs').find_one(req=None, _id=blog_id)
    if not blog:
        return

    post_service = get_resource_service('posts')
    query_params = {'blog': ObjectId(blog_id), 'particular_type': 'post', 'deleted': False}

    if blog['posts_limit'] != UNLIMITED and blog['total_posts'] >= blog['posts_limit']:
        oldest_post = post_service.find(query_params).sort('_created', pymongo.ASCENDING).limit(1)

        for doc in oldest_post:
            post_service.update(doc['_id'], {'deleted': True}, doc)
            logger.warning(
                'Deleted oldest post `%s` because posts_limit (%s) in blog `%s` has been reached'
                % (doc['_id'], blog['posts_limit'], blog['_id']))

        stats = get_blog_stats(blog_id)
        if stats:
            push_notification('blog:limits', blog_id=blog_id, stats=stats)

def get_blog_stats(blog_or_id):
    """
    This find some specific information about the given blog
    like posts_limit, total_posts and returns it in a dictionary
    """
    posts = get_resource_service('posts')

    blog_id, blog = get_blog(blog_or_id)
    if not blog:
        return

    total_posts = posts.find({'$and': [
        {'blog': blog_id},
        {'post_status': 'open'},
        {'deleted': False}
    ]}).count()

    return {'total_posts': total_posts, 'posts_limit': blog['posts_limit']}
<commit_msg>LBSD-2278 Small final adjustments<commit_after>import os
import logging
import pymongo
from bson.objectid import ObjectId
from flask import current_app as app
from superdesk import get_resource_service
from superdesk.notification import push_notification

from .app_settings import BLOGSLIST_ASSETS_DIR
from .exceptions import MediaStorageUnsupportedForBlogPublishing

logger = logging.getLogger('superdesk')

def is_s3_storage_enabled():
    return type(app.media).__name__ is 'AmazonMediaStorage'

def check_media_storage():
    if not is_s3_storage_enabled():
        raise MediaStorageUnsupportedForBlogPublishing()

def get_blog_path(blog_id, theme=None, output_id=None):
    return 'blogs/{}/{}{}index.html'.format(blog_id,
                                            '{}/'.format(theme) if theme else '',
                                            '{}/'.format(output_id) if output_id else '')

def get_bloglist_path():
    return os.path.join(BLOGSLIST_ASSETS_DIR, 'index.html')

def is_relative_to_current_folder(url):
    return not (url.startswith('/') or url.startswith('http://') or url.startswith('https://'))

def get_blog(blog_or_id):
    blogs = get_resource_service('client_blogs')
    if isinstance(blog_or_id, (str, ObjectId)):
        blog_id = blog_or_id
        blog = blogs.find_one(req=None, _id=blog_or_id)
    elif isinstance(blog_or_id, dict):
        blog = blog_or_id
        blog_id = blog['_id']
    else:
        raise ValueError(blog_or_id)

    return blog_id, blog

def is_seo_enabled(blog_or_id):
    """
    Return True if blog has seo output enabled in theme.
    """
    blog_id, blog = get_blog(blog_or_id)
    if not blog:
        return

    themes = get_resource_service('themes')
    blog_preferences = blog.get('blog_preferences')
    if not blog_preferences:
        return

    theme_name = blog_preferences.get('theme')
    if not theme_name:
        return

    theme = themes.find_one(req=None, name=theme_name)
    if not theme:
        return
    if theme.get('seoTheme'):
        return True

    outputs_service = get_resource_service('outputs')
    for output in outputs_service.get(req=None, lookup=dict(blog=blog_id)):
        theme_name = output.get('theme')
        if not theme_name:
            continue

        theme = themes.find_one(req=None, name=theme_name)
        if not theme:
            continue

        if theme.get('seoTheme'):
            return True

def check_limit_and_delete_oldest(blog_id):
    """
    Simple method that checks if a blog has reached the limit of posts and
    deletes ({deleted: true}) the oldest post that belongs to the given blog
    """

    UNLIMITED = 0

    blog = get_resource_service('blogs').find_one(req=None, _id=blog_id)
    if not blog:
        return

    post_service = get_resource_service('posts')
    query_params = {'blog': ObjectId(blog_id), 'particular_type': 'post', 'deleted': False}

    if blog['posts_limit'] != UNLIMITED and blog['total_posts'] > blog['posts_limit']:
        oldest_post = post_service.find(query_params).sort('_created', pymongo.ASCENDING).limit(1)

        for doc in oldest_post:
            post_service.update(doc['_id'], {'deleted': True}, doc)
            logger.warning(
                'Deleted oldest post `%s` because posts_limit (%s) in blog `%s` has been reached'
                % (doc['_id'], blog['posts_limit'], blog['_id']))

        stats = get_blog_stats(blog_id)
        if stats:
            push_notification('blog:limits', blog_id=blog_id, stats=stats)

def get_blog_stats(blog_or_id):
    """
    This find some specific information about the given blog
    like posts_limit, total_posts and returns it in a dictionary
    """
    posts = get_resource_service('posts')

    blog_id, blog = get_blog(blog_or_id)
    if not blog:
        return

    total_posts = posts.find({'$and': [
        {'blog': blog_id},
        {'post_status': 'open'},
        {'deleted': False}
    ]}).count()

    return {'total_posts': total_posts, 'posts_limit': blog['posts_limit']}
<|endoftext|>


<commit_before>from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

import sys, time, urllib
from concurrent.futures import ThreadPoolExecutor
from functools import partial, wraps

import tangoREST
from config import Config

tangoREST = tangoREST.TangoREST()
EXECUTOR = ThreadPoolExecutor(max_workers=4)

# Regex for the resources
SHA1_KEY = ".+" # So that we can have better error messages
COURSELAB = ".+"
OUTPUTFILE = ".+"
IMAGE = ".+"
NUM = "[0-9]+"
JOBID = "[0-9]+"
DEADJOBS=".+"

@app.route('/')
def MainHandler():
    """ get - Default route to check if RESTful Tango is up."""
    return ("Hello, world! RESTful Tango here!\n")

@app.route('/open/<key>/<courselab>/')
def OpenHandler(key, courselab):
    """ get - Handles the get request to open."""
    return jsonify(tangoREST.open(key, courselab))

@app.route('/upload/<key>/<courselab>/', methods=['POST'])
def UploadHandler(key, courselab):
    """ post - Handles the post request to upload."""
    return jsonify(tangoREST.upload(key, courselab, self.request.headers['Filename'], self.request.body))

@app.route('/addJob/<key>/<courselab>/', methods=['POST'])
def AddJobHandler(key, courselab):
    """ post - Handles the post request to add a job."""
    return jsonify(tangoREST.addJob(key, courselab, self.request.body))

@app.route('/poll/<key>/<courselab>/<outputFile>/')
def PollHandler(key, courselab, outputFile):
    """ get - Handles the get request to poll."""
    self.set_header('Content-Type', 'application/octet-stream')
    return jsonify(tangoREST.poll(key, courselab, urllib.unquote(outputFile)))

@app.route('/info/<key>/')
def InfoHandler(key):
    """ get - Handles the get request to info."""
    return jsonify(tangoREST.info(key))

@app.route('/jobs/<key>/<int:deadJobs>/')
def JobsHandler(key, deadJobs):
    """ get - Handles the get request to jobs."""
    return jsonify(tangoREST.jobs(key, deadJobs))

@app.route('/pool/<key>/<image>/')
def PoolHandler(key, image):
    """ get - Handles the get request to pool."""
    return jsonify(tangoREST.pool(key, image))

@app.route('/prealloc/<key>/<image>/<num>/', methods=['POST'])
def PreallocHandler(key, image, num):
    """ post - Handles the post request to prealloc."""
    return jsonify(tangoREST.prealloc(key, image, num, self.request.body))

if __name__ == "__main__":

    port = Config.PORT
    if len(sys.argv) > 0:
        port = int(sys.argv[1])

    tangoREST.resetTango()
    app.run(port=port, debug=True)

    print("Starting the RESTful Tango server on port %d..." % (port))
<commit_msg>Fix requests<commit_after>from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

import sys, time, urllib
from concurrent.futures import ThreadPoolExecutor
from functools import partial, wraps

import tangoREST
from config import Config

tangoREST = tangoREST.TangoREST()
EXECUTOR = ThreadPoolExecutor(max_workers=4)

# Regex for the resources
SHA1_KEY = ".+" # So that we can have better error messages
COURSELAB = ".+"
OUTPUTFILE = ".+"
IMAGE = ".+"
NUM = "[0-9]+"
JOBID = "[0-9]+"
DEADJOBS=".+"

@app.route('/')
def MainHandler():
    """ get - Default route to check if RESTful Tango is up."""
    return ("Hello, world! RESTful Tango here!\n")

@app.route('/open/<key>/<courselab>/')
def OpenHandler(key, courselab):
    """ get - Handles the get request to open."""
    return jsonify(tangoREST.open(key, courselab))

@app.route('/upload/<key>/<courselab>/', methods=['POST'])
def UploadHandler(key, courselab):
    """ post - Handles the post request to upload."""
    return jsonify(tangoREST.upload(key, courselab, request.headers['Filename'], request.body))

@app.route('/addJob/<key>/<courselab>/', methods=['POST'])
def AddJobHandler(key, courselab):
    """ post - Handles the post request to add a job."""
    return jsonify(tangoREST.addJob(key, courselab, request.body))

@app.route('/poll/<key>/<courselab>/<outputFile>/')
def PollHandler(key, courselab, outputFile):
    """ get - Handles the get request to poll."""
    self.set_header('Content-Type', 'application/octet-stream')
    return jsonify(tangoREST.poll(key, courselab, urllib.unquote(outputFile)))

@app.route('/info/<key>/')
def InfoHandler(key):
    """ get - Handles the get request to info."""
    return jsonify(tangoREST.info(key))

@app.route('/jobs/<key>/<int:deadJobs>/')
def JobsHandler(key, deadJobs):
    """ get - Handles the get request to jobs."""
    return jsonify(tangoREST.jobs(key, deadJobs))

@app.route('/pool/<key>/<image>/')
def PoolHandler(key, image):
    """ get - Handles the get request to pool."""
    return jsonify(tangoREST.pool(key, image))

@app.route('/prealloc/<key>/<image>/<num>/', methods=['POST'])
def PreallocHandler(key, image, num):
    """ post - Handles the post request to prealloc."""
    return jsonify(tangoREST.prealloc(key, image, num, request.body))

if __name__ == "__main__":

    port = Config.PORT
    if len(sys.argv) > 0:
        port = int(sys.argv[1])

    tangoREST.resetTango()
    app.run(port=port, debug=True)

    print("Starting the RESTful Tango server on port %d..." % (port))
<|endoftext|>


<commit_before>import json
from urllib import urlencode
from urllib2 import Request, urlopen

from configurations.env_configs import RAILS_TOKEN, API_LIMIT

class APIService(object):
	limit = API_LIMIT

	def __init__(self):
		super(APIService, self).__init__()

	def request_filter(self, uri):
		print ("Sending request to:", uri)
		request = Request(uri)
		request.add_header('HTTP_X_IVY_SESSION_TOKEN', RAILS_TOKEN)
		return json.loads(urlopen(request).read())

	def get_request(self, uri):
		data = []
		offset = 0
		while True:
			params = urlencode({'limit':self.limit, 'offset':offset})
			url = uri + '?' + params
			print ("Sending request to:", url)
			request = Request(url)
			request.add_header('HTTP_X_IVY_SESSION_TOKEN', RAILS_TOKEN)
			response = json.loads(urlopen(request).read())
			size = len(response)
			print ("Response size: ", size)
			if size < 1:
				break
			else:
				data += response
				offset += self.limit

			print("Total results: ", len(data))
			return data<commit_msg>fix api service bug<commit_after>import json
from urllib import urlencode
from urllib2 import Request, urlopen

from configurations.env_configs import RAILS_TOKEN, API_LIMIT

class APIService(object):
	limit = API_LIMIT

	def __init__(self):
		super(APIService, self).__init__()

	def request_filter(self, uri):
		print ("Sending request to:", uri)
		request = Request(uri)
		request.add_header('HTTP_X_IVY_SESSION_TOKEN', RAILS_TOKEN)
		return json.loads(urlopen(request).read())

	def get_request(self, uri):
		data = []
		offset = 0
		while True:
			params = urlencode({'limit':self.limit, 'offset':offset})
			url = uri + '?' + params
			print ("Sending request to:", url)
			request = Request(url)
			request.add_header('HTTP_X_IVY_SESSION_TOKEN', RAILS_TOKEN)
			response = json.loads(urlopen(request).read())
			size = len(response)
			print ("Response size: ", size)
			if size < 1:
				break
			else:
				data += response
				offset += self.limit

		print("Total results: ", len(data))
		return data<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<i@binux.me>
#         http://binux.me
# Created on 2014-10-19 15:37:46

import Queue
import logging
logger = logging.getLogger("result")

class ResultWorker(object):
    """
    do with result
    override this if needed.
    """

    def __init__(self, resultdb, inqueue):
        self.resultdb = resultdb
        self.inqueue = inqueue
        self._quit = False

    def on_result(self, task, result):
        if not result:
            return
        assert 'taskid' in task, 'need taskid in task'
        assert 'project' in task, 'need project in task'
        assert 'url' in task, 'need url in task'
        return self.resultdb.save(project=task['project'], taskid=task['taskid'], url=task['url'], result=result)

    def run(self):
        while not self._quit:
            try:
                task, result = self.inqueue.get()
                if 'taskid' in task and 'project' in task and 'url' in task:
                    logger.info('result %s:%s %s -> %.30r' % (
                        task['project'], task['taskid'], task['url'], result))
                else:
                    logger.warning('result UNKNOW -> %.30r' % result)
                ret = self.on_result(task, result)
            except Queue.Empty as e:
                time.sleep(1)
                continue
            except KeyboardInterrupt:
                break
            except AssertionError as e:
                logger.error(e)
                continue
            except Exception as e:
                logger.exception(e)
                continue
<commit_msg>result work not block<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<i@binux.me>
#         http://binux.me
# Created on 2014-10-19 15:37:46

import Queue
import logging
logger = logging.getLogger("result")

class ResultWorker(object):
    """
    do with result
    override this if needed.
    """

    def __init__(self, resultdb, inqueue):
        self.resultdb = resultdb
        self.inqueue = inqueue
        self._quit = False

    def on_result(self, task, result):
        if not result:
            return
        assert 'taskid' in task, 'need taskid in task'
        assert 'project' in task, 'need project in task'
        assert 'url' in task, 'need url in task'
        return self.resultdb.save(project=task['project'], taskid=task['taskid'], url=task['url'], result=result)

    def quit(self):
        self._quit = True

    def run(self):
        while not self._quit:
            try:
                task, result = self.inqueue.get(timeout=1)
                if 'taskid' in task and 'project' in task and 'url' in task:
                    logger.info('result %s:%s %s -> %.30r' % (
                        task['project'], task['taskid'], task['url'], result))
                else:
                    logger.warning('result UNKNOW -> %.30r' % result)
                ret = self.on_result(task, result)
            except Queue.Empty as e:
                continue
            except KeyboardInterrupt:
                break
            except AssertionError as e:
                logger.error(e)
                continue
            except Exception as e:
                logger.exception(e)
                continue
<|endoftext|>


<commit_before>#
# Copyright (c) 2017 Luis F. Simoes (github: @lfsimoes)
#
# Licensed under the MIT License. See the LICENSE file for details.

from math import sqrt, sin, cos, acos, pi

import numpy as np
import PyKEP as pk

from .constants import MU_SUN, AU, EARTH_VELOCITY, DAY2SEC, asteroids

# ==================================== ## ==================================== #

def edelbaum_dv(ast1, ast2, t):
	(a1, _, i1, W1, _, _) = ast1.osculating_elements(t)
	(a2, _, i2, W2, _, _) = ast2.osculating_elements(t)
	vc1 = sqrt(MU_SUN / a1)
	vc2 = sqrt(MU_SUN / a2)
	cos_i_rel = cos(i1) * cos(i2) + sin(i1) * sin(i2) * cos(W1) * cos(W2) + \
	            sin(i1) * sin(i2) * sin(W1) * sin(W2)
	if cos_i_rel > 1 or cos_i_rel < -1:
		cos_i_rel = 1
	i_rel = acos(cos_i_rel)
	return sqrt(vc1 * vc1 - 2. * vc1 * vc2 * cos(pi / 2. * i_rel) + vc2 * vc2)

def rate__edelbaum(dep_ast, dep_t, **kwargs):
	dep_ast = asteroids[dep_ast]
	dep_t = pk.epoch(dep_t, 'mjd')
	return [edelbaum_dv(dep_ast, arr_ast, dep_t)
	        for arr_ast in asteroids]

# ==================================== ## ==================================== #

def eph_normalize(eph, refn_r=AU, refn_v=EARTH_VELOCITY):
	"""
	Normalize a body's ephemeris with respect to the given reference values:
	    (r / refn_r, v / refn_v)
	If unspecified, reference defaults to refn_r = AU, refn_v = EARTH_VELOCITY.
	Accepts either a single ephemeris, or a matrix with one per row.

	Example:
	>>> eph_normalize(pk.planet.gtoc5(1).eph(0))
	array([-3.41381819, -0.57375407, -0.28305754,
	        0.13551015, -0.43057761, -0.02779567])
	"""
	if type(eph) is tuple:
		eph = np.hstack(eph)
	e = eph.reshape(-1, 6)

	# normalize r
	e[:, :3] /= refn_r
	# normalize v
	e[:,-3:] /= refn_v

	return e.reshape(eph.shape)

def eph_reference(ref_ast, t):
	"""
	Calculate reference r and v magnitudes for use in the normalization
	of asteroid ephemeris.

	Returns:
		(|r|, |v|)
	where (r,v) is the ephemeris of the given asteroid `ref_ast` (id, or object)
	at epoch `t` (int/float, or epoch object).
	"""
	if type(ref_ast) is int:
		ref_ast = asteroids[ref_ast]
	if type(t) in [float, int]:
		t = pk.epoch(t, 'mjd')

	r, v = ref_ast.eph(t)
	return np.linalg.norm(r), np.linalg.norm(v)

def eph_matrix(asts, t, ref_ast=None):
	"""
	Given a list `asts` of asteroids (either by IDs, or as PyKEP.planet objects)
	and an epoch `t` (either as an mjd int/float, or as a PyKEP.epoch object),
	produce the matrix with their normalized ephemerides.

	If a reference asteroid `ref_ast` is given, its ephemeris will be used
	as reference for the normalization. Otherwise, AU, EARTH_VELOCITY will
	be used instead.
	"""
	if type(asts[0]) is int:
		asts = [asteroids[ast] for a in asts]
	if type(t) in [float, int]:
		t = pk.epoch(t, 'mjd')

	if ref_ast is None:
		norm_ref = AU, EARTH_VELOCITY
	else:
		norm_ref = eph_reference(ref_ast, t)

	# obtain all asteroids' ephemerides
	eph = np.array([a.eph(t) for a in asts])
	# reshape matrix, so each asteroid's ephemeris will be represented by
	# a single 6 dimensional vector
	eph = eph.reshape((-1, 6))

	# normalize the full matrix
	return eph_normalize(eph, *norm_ref)

def rate__euclidean(curr_ast, t, **kwargs):
	"""
	Euclidean Phasing Indicator.
	See: http://arxiv.org/pdf/1511.00821.pdf#page=12

	Estimates the cost of a transfer from departure asteroid `dep_ast` at epoch
	`dep_t`, towards each of the available asteroids.
	"""
	eph = eph_matrix(asteroids, t, asteroids[curr_ast])
	return np.linalg.norm(eph - eph[curr_ast], axis=1)

# ==================================== ## ==================================== #

def orbital_indicator(body, t, dT, neg_v=False, **kwargs):
	# reimplemented from:
	# https://github.com/esa/pykep/blob/master/PyKEP/phasing/_knn.py#L46
	(r1, r2, r3), (v1, v2, v3) = body.eph(t)
	r1 /= dT
	r2 /= dT
	r3 /= dT
	if neg_v:
		return (r1 - v1, r2 - v2, r3 - v3,
		        r1, r2, r3)
	else:
		return (r1 + v1, r2 + v2, r3 + v3,
		        r1, r2, r3)

def rate__orbital(dep_ast, dep_t, leg_dT, **kwargs):
	"""
	Orbital Phasing Indicator.
	See: http://arxiv.org/pdf/1511.00821.pdf#page=12

	Estimates the cost of a `leg_dT` days transfer from departure asteroid
	`dep_ast` at epoch `dep_t`, towards each of the available asteroids.
	"""
	dep_t = pk.epoch(dep_t, 'mjd')
	leg_dT *= DAY2SEC
	orbi = [orbital_indicator(ast, dep_t, leg_dT, **kwargs)
	        for ast in asteroids]
	ast_orbi = np.array(orbi[dep_ast])
	return np.linalg.norm(ast_orbi - orbi, axis=1)

def rate__orbital_2(dep_ast, dep_t, leg_dT, **kwargs):
	"""
	Refinement over the Orbital indicator.
	Attempts to correct bad decisions made by the linear model by checking
	which asteroids are close at arrival, and not only which are close at the
	beginning.
	"""
	r1 = rate__orbital(dep_ast, dep_t, leg_dT)
	r2 = rate__orbital(dep_ast, dep_t + leg_dT, leg_dT, neg_v=True)
#	return np.minimum(r1, r2)
	return np.mean([r1, r2], axis=0)

<commit_msg>no message<commit_after>#
# Copyright (c) 2017 Luis F. Simoes (github: @lfsimoes)
#
# Licensed under the MIT License. See the LICENSE file for details.

from math import sqrt, sin, cos, acos, pi

import numpy as np
import PyKEP as pk

from .constants import MU_SUN, AU, EARTH_VELOCITY, DAY2SEC, asteroids

# ==================================== ## ==================================== #

def edelbaum_dv(ast1, ast2, t):
	"""
	Edelbaum's equation for constant acceleration, circle to inclined circle
	orbit transfer.

	Implements Eq. 45 from:
	Theodore N. Edelbaum. "Propulsion Requirements for Controllable Satellites",
	ARS Journal, Vol. 31, No. 8 (1961), pp. 1079-1089.
	http://dx.doi.org/10.2514/8.5723
	"""
	(a1, _, i1, W1, _, _) = ast1.osculating_elements(t)
	(a2, _, i2, W2, _, _) = ast2.osculating_elements(t)

	vc1 = sqrt(MU_SUN / a1)
	vc2 = sqrt(MU_SUN / a2)

	cos_i_rel = cos(i1) * cos(i2) + sin(i1) * sin(i2) * cos(W1) * cos(W2) + \
	            sin(i1) * sin(i2) * sin(W1) * sin(W2)
	if cos_i_rel > 1 or cos_i_rel < -1:
		cos_i_rel = 1
	i_rel = acos(cos_i_rel)

	dV = sqrt(vc1 * vc1 - 2. * vc1 * vc2 * cos(pi / 2. * i_rel) + vc2 * vc2)
	return dV

def rate__edelbaum(dep_ast, dep_t, **kwargs):
	dep_ast = asteroids[dep_ast]
	dep_t = pk.epoch(dep_t, 'mjd')
	return [edelbaum_dv(dep_ast, arr_ast, dep_t)
	        for arr_ast in asteroids]

# ==================================== ## ==================================== #

def eph_normalize(eph, refn_r=AU, refn_v=EARTH_VELOCITY):
	"""
	Normalize a body's ephemeris with respect to the given reference values:
	    (r / refn_r, v / refn_v)
	If unspecified, reference defaults to refn_r = AU, refn_v = EARTH_VELOCITY.
	Accepts either a single ephemeris, or a matrix with one per row.

	Example:
	>>> eph_normalize(pk.planet.gtoc5(1).eph(0))
	array([-3.41381819, -0.57375407, -0.28305754,
	        0.13551015, -0.43057761, -0.02779567])
	"""
	if type(eph) is tuple:
		eph = np.hstack(eph)
	e = eph.reshape(-1, 6)

	# normalize r
	e[:, :3] /= refn_r
	# normalize v
	e[:,-3:] /= refn_v

	return e.reshape(eph.shape)

def eph_reference(ref_ast, t):
	"""
	Calculate reference r and v magnitudes for use in the normalization
	of asteroid ephemeris.

	Returns:
		(|r|, |v|)
	where (r,v) is the ephemeris of the given asteroid `ref_ast` (id, or object)
	at epoch `t` (int/float, or epoch object).
	"""
	if type(ref_ast) is int:
		ref_ast = asteroids[ref_ast]
	if type(t) in [float, int]:
		t = pk.epoch(t, 'mjd')

	r, v = ref_ast.eph(t)
	return np.linalg.norm(r), np.linalg.norm(v)

def eph_matrix(asts, t, ref_ast=None):
	"""
	Given a list `asts` of asteroids (either by IDs, or as PyKEP.planet objects)
	and an epoch `t` (either as an mjd int/float, or as a PyKEP.epoch object),
	produce the matrix with their normalized ephemerides.

	If a reference asteroid `ref_ast` is given, its ephemeris will be used
	as reference for the normalization. Otherwise, AU, EARTH_VELOCITY will
	be used instead.
	"""
	if type(asts[0]) is int:
		asts = [asteroids[ast] for a in asts]
	if type(t) in [float, int]:
		t = pk.epoch(t, 'mjd')

	if ref_ast is None:
		norm_ref = AU, EARTH_VELOCITY
	else:
		norm_ref = eph_reference(ref_ast, t)

	# obtain all asteroids' ephemerides
	eph = np.array([a.eph(t) for a in asts])
	# reshape matrix, so each asteroid's ephemeris will be represented by
	# a single 6 dimensional vector
	eph = eph.reshape((-1, 6))

	# normalize the full matrix
	return eph_normalize(eph, *norm_ref)

def rate__euclidean(curr_ast, t, **kwargs):
	"""
	Euclidean Phasing Indicator.
	See: http://arxiv.org/pdf/1511.00821.pdf#page=12

	Estimates the cost of a transfer from departure asteroid `dep_ast` at epoch
	`dep_t`, towards each of the available asteroids.
	"""
	eph = eph_matrix(asteroids, t, asteroids[curr_ast])
	return np.linalg.norm(eph - eph[curr_ast], axis=1)

# ==================================== ## ==================================== #

def orbital_indicator(body, t, dT, neg_v=False, **kwargs):
	# reimplemented from:
	# https://github.com/esa/pykep/blob/master/PyKEP/phasing/_knn.py#L46
	(r1, r2, r3), (v1, v2, v3) = body.eph(t)
	r1 /= dT
	r2 /= dT
	r3 /= dT
	if neg_v:
		return (r1 - v1, r2 - v2, r3 - v3,
		        r1, r2, r3)
	else:
		return (r1 + v1, r2 + v2, r3 + v3,
		        r1, r2, r3)

def rate__orbital(dep_ast, dep_t, leg_dT, **kwargs):
	"""
	Orbital Phasing Indicator.
	See: http://arxiv.org/pdf/1511.00821.pdf#page=12

	Estimates the cost of a `leg_dT` days transfer from departure asteroid
	`dep_ast` at epoch `dep_t`, towards each of the available asteroids.
	"""
	dep_t = pk.epoch(dep_t, 'mjd')
	leg_dT *= DAY2SEC
	orbi = [orbital_indicator(ast, dep_t, leg_dT, **kwargs)
	        for ast in asteroids]
	ast_orbi = np.array(orbi[dep_ast])
	return np.linalg.norm(ast_orbi - orbi, axis=1)

def rate__orbital_2(dep_ast, dep_t, leg_dT, **kwargs):
	"""
	Refinement over the Orbital indicator.
	Attempts to correct bad decisions made by the linear model by checking
	which asteroids are close at arrival, and not only which are close at the
	beginning.
	"""
	r1 = rate__orbital(dep_ast, dep_t, leg_dT)
	r2 = rate__orbital(dep_ast, dep_t + leg_dT, leg_dT, neg_v=True)
#	return np.minimum(r1, r2)
	return np.mean([r1, r2], axis=0)

<|endoftext|>


<commit_before>import gettext

class _(str):

    observers = set()
    lang = None

    def __new__(cls, s, *args, **kwargs):
        if _.lang is None:
            _.switch_lang('en')
        t = _.translate(s, *args, **kwargs)
        o = super(_, cls).__new__(cls, t)
        o.source_text = s
        return o

    @staticmethod
    def translate(s, *args, **kwargs):
        return _.lang(s).format(args, kwargs)

    @staticmethod
    def bind(label):
        try:
            _.observers.add(label)
        except:
            pass
        # garbage collection
        new = set()
        for label in _.observers:
            try:
                new.add(label)
            except:
                pass
        _.observers = new

    @staticmethod
    def switch_lang(lang):
        # get the right locales directory, and instanciate a gettext
        from electrum.i18n import LOCALE_DIR
        locales = gettext.translation('electrum', LOCALE_DIR, languages=[lang], fallback=True)
        _.lang = locales.gettext
        for label in _.observers:
            try:
                label.text = _(label.text.source_text)
            except:
                pass
<commit_msg>kivy: fix i18n with unicode<commit_after>import gettext

class _(unicode):

    observers = set()
    lang = None

    def __new__(cls, s, *args, **kwargs):
        if _.lang is None:
            _.switch_lang('en')
        t = _.translate(s, *args, **kwargs)
        o = super(_, cls).__new__(cls, t)
        o.source_text = s
        return o

    @staticmethod
    def translate(s, *args, **kwargs):
        tr = _.lang(s).format(args, kwargs)
        tr = tr.decode('utf8')
        return tr

    @staticmethod
    def bind(label):
        try:
            _.observers.add(label)
        except:
            pass
        # garbage collection
        new = set()
        for label in _.observers:
            try:
                new.add(label)
            except:
                pass
        _.observers = new

    @staticmethod
    def switch_lang(lang):
        # get the right locales directory, and instanciate a gettext
        from electrum.i18n import LOCALE_DIR
        locales = gettext.translation('electrum', LOCALE_DIR, languages=[lang], fallback=True)
        _.lang = locales.gettext
        for label in _.observers:
            try:
                label.text = _(label.text.source_text)
            except:
                pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -
#
# This file is part of gunicorn released under the MIT license.
# See the NOTICE for more information.

import errno
import logging
import os
import socket
import sys
import time

from gunicorn import util

log = logging.getLogger(__name__)

class BaseSocket(object):

    def __init__(self, conf, fd=None):
        self.conf = conf
        self.address = conf.address
        if fd is None:
            sock = socket.socket(self.FAMILY, socket.SOCK_STREAM)
        else:
            sock = socket.fromfd(fd, self.FAMILY, socket.SOCK_STREAM)
        self.sock = self.set_options(sock, bound=(fd is not None))

    def __str__(self, name):
        return "<socket %d>" % self.sock.fileno()

    def __getattr__(self, name):
        return getattr(self.sock, name)

    def set_options(self, sock, bound=False):
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        if not bound:
            self.bind(sock)
        sock.setblocking(0)
        sock.listen(self.conf.backlog)

        return sock

    def bind(self, sock):
        sock.bind(self.address)

class TCPSocket(BaseSocket):

    FAMILY = socket.AF_INET

    def __str__(self):
        return "http://%s:%d" % self.sock.getsockname()

    def set_options(self, sock, bound=False):
        if hasattr(socket, "TCP_CORK"):
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1)
        elif hasattr(socket, "TCP_NOPUSH"):
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NOPUSH, 1)
        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        return super(TCPSocket, self).set_options(sock, bound=bound)

class UnixSocket(BaseSocket):

    FAMILY = socket.AF_UNIX

    def __init__(self, conf, fd=None):
        if fd is None:
            try:
                os.remove(conf.address)
            except OSError:
                pass
        super(UnixSocket, self).__init__(conf, fd=fd)

    def __str__(self):
        return "unix:%s" % self.address

    def bind(self, sock):
        old_umask = os.umask(self.conf['umask'])
        sock.bind(self.address)
        util.chown(self.address, self.conf.uid, self.conf.gid)
        os.umask(old_umask)

def create_socket(conf):
    """
    Create a new socket for the given address. If the
    address is a tuple, a TCP socket is created. If it
    is a string, a Unix socket is created. Otherwise
    a TypeError is raised.
    """
    # get it only once
    addr = conf.address

    if isinstance(addr, tuple):
        sock_type = TCPSocket
    elif isinstance(addr, basestring):
        sock_type = UnixSocket
    else:
        raise TypeError("Unable to create socket from: %r" % addr)

    if 'GUNICORN_FD' in os.environ:
        fd = int(os.environ.pop('GUNICORN_FD'))
        try:
            return sock_type(conf, fd=fd)
        except socket.error, e:
            if e[0] == errno.ENOTCONN:
                log.error("GUNICORN_FD should refer to an open socket.")
            else:
                raise

    # If we fail to create a socket from GUNICORN_FD
    # we fall through and try and open the socket
    # normally.

    for i in range(5):
        try:
            return sock_type(conf)
        except socket.error, e:
            if e[0] == errno.EADDRINUSE:
                log.error("Connection in use: %s" % str(addr))
            if e[0] == errno.EADDRNOTAVAIL:
                log.error("Invalid address: %s" % str(addr))
                sys.exit(1)
            if i < 5:
                log.error("Retrying in 1 second.")
                time.sleep(1)

    log.error("Can't connect to %s" % str(addr))
    sys.exit(1)
<commit_msg>fix issue #59.<commit_after># -*- coding: utf-8 -
#
# This file is part of gunicorn released under the MIT license.
# See the NOTICE for more information.

import errno
import logging
import os
import socket
import sys
import time

from gunicorn import util

log = logging.getLogger(__name__)

class BaseSocket(object):

    def __init__(self, conf, fd=None):
        self.conf = conf
        self.address = conf.address
        if fd is None:
            sock = socket.socket(self.FAMILY, socket.SOCK_STREAM)
        else:
            sock = socket.fromfd(fd, self.FAMILY, socket.SOCK_STREAM)
        self.sock = self.set_options(sock, bound=(fd is not None))

    def __str__(self, name):
        return "<socket %d>" % self.sock.fileno()

    def __getattr__(self, name):
        return getattr(self.sock, name)

    def set_options(self, sock, bound=False):
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        if not bound:
            self.bind(sock)
        sock.setblocking(0)
        sock.listen(self.conf.backlog)

        return sock

    def bind(self, sock):
        sock.bind(self.address)

class TCPSocket(BaseSocket):

    FAMILY = socket.AF_INET

    def __str__(self):
        return "http://%s:%d" % self.sock.getsockname()

    def set_options(self, sock, bound=False):
        if hasattr(socket, "TCP_CORK"):
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1)
        elif hasattr(socket, "TCP_NOPUSH"):
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NOPUSH, 1)
        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        return super(TCPSocket, self).set_options(sock, bound=bound)

class UnixSocket(BaseSocket):

    FAMILY = socket.AF_UNIX

    def __init__(self, conf, fd=None):
        if fd is None:
            try:
                os.remove(conf.address)
            except OSError:
                pass
        super(UnixSocket, self).__init__(conf, fd=fd)

    def __str__(self):
        return "unix:%s" % self.address

    def bind(self, sock):
        old_umask = os.umask(self.conf.umask)
        sock.bind(self.address)
        util.chown(self.address, self.conf.uid, self.conf.gid)
        os.umask(old_umask)

def create_socket(conf):
    """
    Create a new socket for the given address. If the
    address is a tuple, a TCP socket is created. If it
    is a string, a Unix socket is created. Otherwise
    a TypeError is raised.
    """
    # get it only once
    addr = conf.address

    if isinstance(addr, tuple):
        sock_type = TCPSocket
    elif isinstance(addr, basestring):
        sock_type = UnixSocket
    else:
        raise TypeError("Unable to create socket from: %r" % addr)

    if 'GUNICORN_FD' in os.environ:
        fd = int(os.environ.pop('GUNICORN_FD'))
        try:
            return sock_type(conf, fd=fd)
        except socket.error, e:
            if e[0] == errno.ENOTCONN:
                log.error("GUNICORN_FD should refer to an open socket.")
            else:
                raise

    # If we fail to create a socket from GUNICORN_FD
    # we fall through and try and open the socket
    # normally.

    for i in range(5):
        try:
            return sock_type(conf)
        except socket.error, e:
            if e[0] == errno.EADDRINUSE:
                log.error("Connection in use: %s" % str(addr))
            if e[0] == errno.EADDRNOTAVAIL:
                log.error("Invalid address: %s" % str(addr))
                sys.exit(1)
            if i < 5:
                log.error("Retrying in 1 second.")
                time.sleep(1)

    log.error("Can't connect to %s" % str(addr))
    sys.exit(1)
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import urllib
#csv is for the csv writer
import csv

#initiates the dictionary to hold the output

holder = []

#***********************************#
#***********************************#
#***CHANGE THE TARGET URL***********#
#***********************************#
#***AND NAME OF OUTPUT CSV FILE*****#
#***********************************#
#***********************************#

#this is the target URL
target_url = "http://sousuo.gov.cn/s.htm?q=&n=80&p=&t=paper&advance=true&title=&content=&puborg=&pcodeJiguan=%E5%9B%BD%E5%8F%91&pcodeYear=2016&pcodeNum=&childtype=&subchildtype=&filetype=&timetype=timeqb&mintime=&maxtime=&sort=pubtime&nocorrect=&sortType=1"

csv_name = "guofa-scraper-phase-three-output.csv"

def scraper(url):

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url, 'lxml')

    #***********************************#
    #******TITLE SECTION****************#
    #***********************************#

    #finds all of the titles
    doc_title = soup.find_all('h3')

    #creates a list called title mini holder
    title_mini_holder = []

    #for each of the titles
    for element in doc_title:

        #print element

        #this gets just the text of the element
        text = element.text.encode('utf-8')
        #print text

        #this gets just the url of the element
        #by finding the thing tagged with 'a' and then pulling the 'href'
        doc_url = element.find('a').attrs['href']
        #print doc_url

        #bundles the output of each round as a list
        title_mini_holder = [text, doc_url]
        #adds that list to the larger list
        holder.append(title_mini_holder)

    #***********************************#
    #******EVERYTHING ELSE SECTION****************#
    #***********************************#

    #finds the doc numbers, promilgation date, and interet release date
    #which are in
    #this will end up with all three values over and over
    #that means that you will need to chunk them into threes
    other_elements = soup.find_all('span', {'class':'sp'})

    #creates a list called number mini holder
    values_mini_holder = []

    #for each of the numbers
    for element in other_elements:

        #pulls the text of each element
        text = element.text.encode('utf-8')

        #adds teh element to values_mini_holder
        values_mini_holder.append(text)

    #these variables allow matching between the title/URL and the other data

    #this keeps track of the document (we have the title and URL already)
    holder_spot_counter = 0
    #this keeps track of three new elements (doc num, prom date, release date)
    #since these are in chunks of three it needs to have its own variable
    other_three_counter = 0

    #this adds the three relevant values to the existing title/url
    while holder_spot_counter < len(holder):

        #the goal here is to add doc num, prom date, and release date
        #so for each existing spot you need to move three steps through values_mini_holder

        #for example the first doc (holder[0]) needs
        #the values_mini_holder[0], values_mini_holder[1], & values_mini_holder[2]

        #then holder[1] needs
        #values_mini_holder[3], values_mini_holder[4], values_mini_holder[5]

        holder[holder_spot_counter].append(values_mini_holder[other_three_counter])
        holder[holder_spot_counter].append(values_mini_holder[other_three_counter+1])
        holder[holder_spot_counter].append(values_mini_holder[other_three_counter+2])

        holder_spot_counter += 1
        other_three_counter += 3

#fills in the category (you need to go to the child page)
def scraper2(row):

    #variable to identify the row in holder
    g = holder[row]
    #the url is in spot [1] of the row
    url = g[1]

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url, 'lxml')

    data = []

    #finds the table
    table = soup.find('tbody')
    #zeros in on the element that contains the data that matters
    #two things match this criteria, you only want the second one
    table_data = table.find_all('td', {'width':'360'})

    #this section is because some of the pages have a
    #width of 260 instead of 360

    if len(table_data) > 0:

        #let's just keep the second thing becusae  that's all that matters
        for element in table_data:
            #pulls just the text
            element_cleaned = element.text.encode('utf-8')
            #adds it to data
            data.append(element_cleaned)

    elif len(table_data) == 0:
        table_data_short = table.find_all('td', {'width':'260'})
        for element in table_data_short:
            #pulls just the text
            element_cleaned = element.text.encode('utf-8')
            #adds it to data
            data.append(element_cleaned)

    else:
        print "error"

    #the 2 has to be a variable
    holder[row].append(data[1])

    print "added category for row %s of %s" % ((row+1), len(holder))

#scrapes the body of the child page and outputs a html doc based on the content
def bodyscraper(row):

    #variable to identify the row in holder
    g = holder[row]
    #the url is in spot [1] of the row
    url = g[1]
    #the name of the doc is in spot [2] of the row
    docname = g[2]
    #this removes "发文字号：" from the start of every file name
    #it turns out that those are represented by the first 15
    #characters in the string. I have no idea why and got to 15
    #by trial and error
    docname_fixed = docname[15:]

    data = []

    #names the file based on the name of the document
    filename = docname_fixed+".html"
    target = open(filename, 'w')

    #writest the header for the html file so that the browser decodes chinese
    target.write('<meta charset="utf-8"/>')

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url.decode("utf-8"), 'lxml')
    #finds the body text
    body = soup.find('td', {'class':'b12c'})
    #write the whole decoded body to html directly
    target.write("%s\n" % body)

    #appends the body to the data list
    data.append(body)
    #appends the data list to the holder list
    holder[row].append(data)

    print "created html for row %s of %s" % ((row+1), len(holder))

scraper(target_url)

#run scraper2 for each row in holder
scraper2_row_counter = 0
while scraper2_row_counter < len(holder):
    scraper2(scraper2_row_counter)
    scraper2_row_counter += 1

#outputs holder to a csv
#this is before the bodyscraper because the CSV should not have the HTML field
#if you want the csv to have the HTML field at some point just move it
#to the end

with open(csv_name, 'wb') as f:
	writer = csv.writer(f)
	writer.writerows(holder)

#run bodyscraper for each row in holder
bodyscraper_row_counter = 0
while bodyscraper_row_counter < len(holder):
    bodyscraper(bodyscraper_row_counter)
    bodyscraper_row_counter += 1
<commit_msg>removed the chinese characters in comments <commit_after>from bs4 import BeautifulSoup
import urllib
#csv is for the csv writer
import csv

#initiates the dictionary to hold the output

holder = []

#***********************************#
#***********************************#
#***CHANGE THE TARGET URL***********#
#***********************************#
#***AND NAME OF OUTPUT CSV FILE*****#
#***********************************#
#***********************************#

#this is the target URL
target_url = "http://sousuo.gov.cn/s.htm?q=&n=80&p=&t=paper&advance=true&title=&content=&puborg=&pcodeJiguan=%E5%9B%BD%E5%8F%91&pcodeYear=2016&pcodeNum=&childtype=&subchildtype=&filetype=&timetype=timeqb&mintime=&maxtime=&sort=pubtime&nocorrect=&sortType=1"

csv_name = "guofa-scraper-phase-three-output.csv"

def scraper(url):

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url, 'lxml')

    #***********************************#
    #******TITLE SECTION****************#
    #***********************************#

    #finds all of the titles
    doc_title = soup.find_all('h3')

    #creates a list called title mini holder
    title_mini_holder = []

    #for each of the titles
    for element in doc_title:

        #print element

        #this gets just the text of the element
        text = element.text.encode('utf-8')
        #print text

        #this gets just the url of the element
        #by finding the thing tagged with 'a' and then pulling the 'href'
        doc_url = element.find('a').attrs['href']
        #print doc_url

        #bundles the output of each round as a list
        title_mini_holder = [text, doc_url]
        #adds that list to the larger list
        holder.append(title_mini_holder)

    #***********************************#
    #******EVERYTHING ELSE SECTION****************#
    #***********************************#

    #finds the doc numbers, promilgation date, and interet release date
    #which are in
    #this will end up with all three values over and over
    #that means that you will need to chunk them into threes
    other_elements = soup.find_all('span', {'class':'sp'})

    #creates a list called number mini holder
    values_mini_holder = []

    #for each of the numbers
    for element in other_elements:

        #pulls the text of each element
        text = element.text.encode('utf-8')

        #adds teh element to values_mini_holder
        values_mini_holder.append(text)

    #these variables allow matching between the title/URL and the other data

    #this keeps track of the document (we have the title and URL already)
    holder_spot_counter = 0
    #this keeps track of three new elements (doc num, prom date, release date)
    #since these are in chunks of three it needs to have its own variable
    other_three_counter = 0

    #this adds the three relevant values to the existing title/url
    while holder_spot_counter < len(holder):

        #the goal here is to add doc num, prom date, and release date
        #so for each existing spot you need to move three steps through values_mini_holder

        #for example the first doc (holder[0]) needs
        #the values_mini_holder[0], values_mini_holder[1], & values_mini_holder[2]

        #then holder[1] needs
        #values_mini_holder[3], values_mini_holder[4], values_mini_holder[5]

        holder[holder_spot_counter].append(values_mini_holder[other_three_counter])
        holder[holder_spot_counter].append(values_mini_holder[other_three_counter+1])
        holder[holder_spot_counter].append(values_mini_holder[other_three_counter+2])

        holder_spot_counter += 1
        other_three_counter += 3

#fills in the category (you need to go to the child page)
def scraper2(row):

    #variable to identify the row in holder
    g = holder[row]
    #the url is in spot [1] of the row
    url = g[1]

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url, 'lxml')

    data = []

    #finds the table
    table = soup.find('tbody')
    #zeros in on the element that contains the data that matters
    #two things match this criteria, you only want the second one
    table_data = table.find_all('td', {'width':'360'})

    #this section is because some of the pages have a
    #width of 260 instead of 360

    if len(table_data) > 0:

        #let's just keep the second thing becusae  that's all that matters
        for element in table_data:
            #pulls just the text
            element_cleaned = element.text.encode('utf-8')
            #adds it to data
            data.append(element_cleaned)

    elif len(table_data) == 0:
        table_data_short = table.find_all('td', {'width':'260'})
        for element in table_data_short:
            #pulls just the text
            element_cleaned = element.text.encode('utf-8')
            #adds it to data
            data.append(element_cleaned)

    else:
        print "error"

    #the 2 has to be a variable
    holder[row].append(data[1])

    print "added category for row %s of %s" % ((row+1), len(holder))

#scrapes the body of the child page and outputs a html doc based on the content
def bodyscraper(row):

    #variable to identify the row in holder
    g = holder[row]
    #the url is in spot [1] of the row
    url = g[1]
    #the name of the doc is in spot [2] of the row
    docname = g[2]
    #this removes the header from the start of every file name
    #it turns out that those are represented by the first 15
    #characters in the string. I have no idea why and got to 15
    #by trial and error
    docname_fixed = docname[15:]

    data = []

    #names the file based on the name of the document
    filename = docname_fixed+".html"
    target = open(filename, 'w')

    #writest the header for the html file so that the browser decodes chinese
    target.write('<meta charset="utf-8"/>')

    #opens the url for read access
    this_url = urllib.urlopen(url).read()
    #creates a new BS holder based on the URL
    soup = BeautifulSoup(this_url.decode("utf-8"), 'lxml')
    #finds the body text
    body = soup.find('td', {'class':'b12c'})
    #write the whole decoded body to html directly
    target.write("%s\n" % body)

    #appends the body to the data list
    data.append(body)
    #appends the data list to the holder list
    holder[row].append(data)

    print "created html for row %s of %s" % ((row+1), len(holder))

scraper(target_url)

#run scraper2 for each row in holder
scraper2_row_counter = 0
while scraper2_row_counter < len(holder):
    scraper2(scraper2_row_counter)
    scraper2_row_counter += 1

#outputs holder to a csv
#this is before the bodyscraper because the CSV should not have the HTML field
#if you want the csv to have the HTML field at some point just move it
#to the end

with open(csv_name, 'wb') as f:
	writer = csv.writer(f)
	writer.writerows(holder)

#run bodyscraper for each row in holder
bodyscraper_row_counter = 0
while bodyscraper_row_counter < len(holder):
    bodyscraper(bodyscraper_row_counter)
    bodyscraper_row_counter += 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

from matplotlib import pyplot as PLT
import numpy as NP
import collections
import sys

if len(sys.argv) != 2:
  print "Usage: verify.py <filename>"
  exit(1)

with open(sys.argv[1]) as f:
  v = NP.loadtxt(f, dtype='int', comments='#', usecols=[1])

v_hist = NP.ravel(v)   # 'flatten' v

vertex_degrees = collections.Counter(v_hist)
degrees = collections.Counter(vertex_degrees.values())

fig = PLT.figure()
ax1 = fig.add_subplot(111)

ax1.scatter(degrees.keys(), degrees.values())

ax1.set_xscale('log')
ax1.set_yscale('log')

PLT.show()
<commit_msg>modify the verify.py to save the graph as figure.pdf<commit_after>#!/usr/bin/env python

from matplotlib import pyplot as PLT
from matplotlib.backends.backend_pdf import PdfPages
import numpy as NP
import collections
import sys

if len(sys.argv) != 2:
  print "Usage: verify.py <filename>"
  exit(1)

with open(sys.argv[1]) as f:
  v = NP.loadtxt(f, dtype='int', comments='#', usecols=[1])

v_hist = NP.ravel(v)   # 'flatten' v

vertex_degrees = collections.Counter(v_hist)
degrees = collections.Counter(vertex_degrees.values())

with PdfPages('figure.pdf') as pdf:
	fig = PLT.figure()
	ax1 = fig.add_subplot(111)

	ax1.scatter(degrees.keys(), degrees.values())

	ax1.set_xscale('log')
	ax1.set_yscale('log')

	PLT.plot()
	pdf.savefig()
<|endoftext|>