<commit_before>#!/usr/bin/python

import os
import json
import urllib
import sys
import collections
import dateutil.parser
import requests
import subprocess
import httplib
import sqlite3

from secrets import *

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permenanet tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

# XXX We're going to need to check for exceptions like this and delete the auth files and reauth
# You can get these exceptions either just above, or in the dance too - basically in the whole file...
#twitter.api.TwitterHTTPError: Twitter sent status 401 for URL: 1.1/followers/list.json using parameters: (oauth_consumer_key=3CejKAAW7OGqni9lxuU09g&oauth_nonce=14791547903118891158&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1360055733&oauth_token=PFcsB0z7nf7kNDVq030T6VZSK1PwTMLjuLxLi6U7PU&oauth_version=1.0&screen_name=spikingneural&oauth_signature=szYU8AYsfSp3m5Kzo%2FYGnKHZyP8%3D)
#details: {"errors":[{"message":"Invalid or expired token","code":89}]}

#########################################################################
# Helper functions

# Stores one Twitter user in the ScraperWiki database
def save_user(batch, user, table_name):
    data = collections.OrderedDict()

    data['id'] = user['id']
    data['name'] = user['name']
    data['screen_name'] = user['screen_name']
    data['created_at'] = dateutil.parser.parse(user['created_at'])

    data['description'] = user['description']
    data['url'] = user['url']
    data['profile_image_url_https'] = user['profile_image_url_https']

    data['statuses_count'] = user['statuses_count']
    data['followers_count'] = user['followers_count']
    data['following_count'] = user['friends_count']

    data['location'] = user['location']

    data['batch'] = batch

    scraperwiki.sqlite.save(['id'], data, table_name=table_name)

# After detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global current_status

    extra['status'] = status
    print json.dumps(extra)

    requests.post("https://x.scraperwiki.com/api/status", data={'type':typ, 'message':message})

    current_status = status
    save_status()

    sys.exit()

# Store all our progress variables
def save_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    # Update progress indicators...

    # For number of users got, we count the total of:
    # 1) all followers in the last full batch
    # 2) all followers transferred into the new batch so far
    # i.e. all those for whom batch >= (current_batch - 1)
    try:
        batch_got = scraperwiki.sqlite.select("count(*) as c from twitter_followers where batch >= %d" % (current_batch - 1))[0]['c']
    except:
        batch_got = 0

    data = {
        'id': 'followers',
        'current_batch': current_batch,
        'next_cursor': next_cursor,
        'batch_got': batch_got,
        'batch_expected': batch_expected,
        'current_status': current_status
    }
    scraperwiki.sqlite.save(['id'], data, table_name='status')

# Load in all our progress variables
current_batch = 1
next_cursor = -1
batch_got = 0
batch_expected = 0
current_status = 'clean-slate'
def get_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    try:
        data = scraperwiki.sqlite.select("* from status where id='followers'")
    except sqlite3.OperationalError, e:
        if str(e) == "no such table: status":
            return
        raise
    if len(data) == 0:
        return
    assert(len(data) == 1)
    data = data[0]

    current_batch = data['current_batch']
    next_cursor = data['next_cursor']
    batch_got = data['batch_got']
    batch_expected = data['batch_expected']
    current_status = data['current_status']

#########################################################################
# Main code

pages_got = 0
try:
    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        os.system("rm -fr scraperwiki.sqlite; crontab -r >/dev/null 2>&1")
        import scraperwiki
        set_status_and_exit('clean-slate', 'error', 'No user set')
        sys.exit()

    import scraperwiki

    # Get user we're working on from file we store it in
    screen_name = open("user.txt").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()

    # A batch is one scan through the list of followers - we have to scan as we only
    # get 20 per API call, and have 15 API calls / 15 minutes (as of Feb 2013).
    # The cursor is Twitter's identifier of where in the current batch we are.
    get_status()
    # Note that each user is only in the most recent batch they've been found in
    # (we don't keep all the history)

    # Look up latest followers count
    profile = tw.users.lookup(screen_name=screen_name)
    batch_expected = profile[0]['followers_count']

    # Things basically working, so make sure we run again
    os.system("crontab tool/crontab")

    # Get as many pages in the batch as we can (most likely 15!)
    while True:
        #raise httplib.IncompleteRead('hi') # for testing

        if next_cursor == -1:
            result = tw.followers.list(screen_name=screen_name)
        else:
            result = tw.followers.list(screen_name=screen_name, cursor=next_cursor)
        pages_got += 1
        for user in result['users']:
            save_user(current_batch, user, "twitter_followers")
        next_cursor = result['next_cursor']
        save_status()

        # While debugging, only do one page to avoid rate limits by uncommenting this:
        # break

        if next_cursor == 0:
            # We've finished a batch
            next_cursor = -1
            current_batch += 1
            save_status()
            break

except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # rate limit exceeded
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter is rate limiting you')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the conncetion')

# Save progress message
if batch_got == batch_expected:
    set_status_and_exit("ok-updating", 'ok', "Fully up to date")
else:
    set_status_and_exit("ok-updating", 'error', "Running... %d/%d" % (batch_got, batch_expected))

<commit_msg>Change order of columns to be more useful<commit_after>#!/usr/bin/python

import os
import json
import urllib
import sys
import collections
import dateutil.parser
import requests
import subprocess
import httplib
import sqlite3

from secrets import *

# Make sure you install this version of "twitter":
# http://pypi.python.org/pypi/twitter
# http://mike.verdone.ca/twitter/
# https://github.com/sixohsix/twitter
import twitter

#########################################################################
# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which
# currently only has a function for PIN authentication, not redirect)
from twitter.api import Twitter
from twitter.oauth import OAuth, write_token_file, read_token_file
from twitter.oauth_dance import parse_oauth_tokens
def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):
    # Verification happens in two stages...

    # 1) If we haven't done a pre-verification yet... Then we get credentials from Twitter
    # that will be used to sign our redirect to them, find the redirect, and instruct the Javascript
    # that called us to do the redirect.
    if not os.path.exists(CREDS_PRE_VERIFIY):
        twitter = Twitter(auth=OAuth('', '', consumer_key, consumer_secret), format='', api_version=None)
        oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.request_token(oauth_callback = callback_url))
        write_token_file(pre_verify_token_filename, oauth_token, oauth_token_secret)

        oauth_url = 'https://api.twitter.com/oauth/authorize?' + urllib.urlencode({ 'oauth_token': oauth_token })
        return oauth_url

    # 2) We've done pre-verification, hopefully the user has authed us in Twitter
    # and we've been redirected to. Check we are and ask for the permenanet tokens.
    oauth_token, oauth_token_secret = read_token_file(CREDS_PRE_VERIFIY)
    twitter = Twitter(auth=OAuth( oauth_token, oauth_token_secret, consumer_key, consumer_secret), format='', api_version=None)
    oauth_token, oauth_token_secret = parse_oauth_tokens(twitter.oauth.access_token(oauth_verifier=oauth_verifier))
    write_token_file(verified_token_filename, oauth_token, oauth_token_secret)
    return oauth_token, oauth_token_secret

def do_tool_oauth():
    if not os.path.exists(CREDS_VERIFIED):
        if len(sys.argv) < 3:
            result = "need-oauth"
        else:
            (callback_url, oauth_verifier) = (sys.argv[1], sys.argv[2])
            result = oauth_url_dance(CONSUMER_KEY, CONSUMER_SECRET, callback_url, oauth_verifier, CREDS_PRE_VERIFIY, CREDS_VERIFIED)
        # a string means a URL for a redirect (otherwise we get a tuple back with auth tokens in)
        if type(result) == str:
            set_status_and_exit('auth-redirect', 'error', 'Permission needed from Twitter', { 'url': result } )

    oauth_token, oauth_token_secret = read_token_file(CREDS_VERIFIED)
    tw = twitter.Twitter(auth=twitter.OAuth( oauth_token, oauth_token_secret, CONSUMER_KEY, CONSUMER_SECRET))
    return tw

# XXX We're going to need to check for exceptions like this and delete the auth files and reauth
# You can get these exceptions either just above, or in the dance too - basically in the whole file...
#twitter.api.TwitterHTTPError: Twitter sent status 401 for URL: 1.1/followers/list.json using parameters: (oauth_consumer_key=3CejKAAW7OGqni9lxuU09g&oauth_nonce=14791547903118891158&oauth_signature_method=HMAC-SHA1&oauth_timestamp=1360055733&oauth_token=PFcsB0z7nf7kNDVq030T6VZSK1PwTMLjuLxLi6U7PU&oauth_version=1.0&screen_name=spikingneural&oauth_signature=szYU8AYsfSp3m5Kzo%2FYGnKHZyP8%3D)
#details: {"errors":[{"message":"Invalid or expired token","code":89}]}

#########################################################################
# Helper functions

# Stores one Twitter user in the ScraperWiki database
def save_user(batch, user, table_name):
    data = collections.OrderedDict()

    data['id'] = user['id']
    data['name'] = user['name']
    data['screen_name'] = user['screen_name']
    data['profile_image'] = user['profile_image_url_https'] # shorten to avoid wasting horizontal space
    data['created_at'] = dateutil.parser.parse(user['created_at'])

    data['description'] = user['description']
    data['location'] = user['location']
    data['url'] = user['url']

    data['statuses_count'] = user['statuses_count']
    data['followers_count'] = user['followers_count']
    data['following_count'] = user['friends_count'] # rename as "friends" is confusing to end users

    data['batch'] = batch

    scraperwiki.sqlite.save(['id'], data, table_name=table_name)

# After detecting an auth failed error mid work, call this
def clear_auth_and_restart():
    # remove auth files and respawn
    try:
        os.remove(CREDS_PRE_VERIFIY)
        os.remove(CREDS_VERIFIED)
    except OSError:
        # don't worry if the files aren't there
        pass
    subprocess.call(sys.argv)
    sys.exit()

# Signal back to the calling Javascript, to the database, and custard's status API, our status
def set_status_and_exit(status, typ, message, extra = {}):
    global current_status

    extra['status'] = status
    print json.dumps(extra)

    requests.post("https://x.scraperwiki.com/api/status", data={'type':typ, 'message':message})

    current_status = status
    save_status()

    sys.exit()

# Store all our progress variables
def save_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    # Update progress indicators...

    # For number of users got, we count the total of:
    # 1) all followers in the last full batch
    # 2) all followers transferred into the new batch so far
    # i.e. all those for whom batch >= (current_batch - 1)
    try:
        batch_got = scraperwiki.sqlite.select("count(*) as c from twitter_followers where batch >= %d" % (current_batch - 1))[0]['c']
    except:
        batch_got = 0

    data = {
        'id': 'followers',
        'current_batch': current_batch,
        'next_cursor': next_cursor,
        'batch_got': batch_got,
        'batch_expected': batch_expected,
        'current_status': current_status
    }
    scraperwiki.sqlite.save(['id'], data, table_name='status')

# Load in all our progress variables
current_batch = 1
next_cursor = -1
batch_got = 0
batch_expected = 0
current_status = 'clean-slate'
def get_status():
    global current_batch, next_cursor, batch_got, batch_expected, current_status

    try:
        data = scraperwiki.sqlite.select("* from status where id='followers'")
    except sqlite3.OperationalError, e:
        if str(e) == "no such table: status":
            return
        raise
    if len(data) == 0:
        return
    assert(len(data) == 1)
    data = data[0]

    current_batch = data['current_batch']
    next_cursor = data['next_cursor']
    batch_got = data['batch_got']
    batch_expected = data['batch_expected']
    current_status = data['current_status']

#########################################################################
# Main code

pages_got = 0
try:
    # Parameters to this command vary:
    #   a. None: try and scrape Twitter followers
    #   b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens
    #   c. "clean-slate": wipe database and start again
    if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':
        os.system("rm -fr scraperwiki.sqlite; crontab -r >/dev/null 2>&1")
        import scraperwiki
        set_status_and_exit('clean-slate', 'error', 'No user set')
        sys.exit()

    import scraperwiki

    # Get user we're working on from file we store it in
    screen_name = open("user.txt").read().strip()

    # Connect to Twitter
    tw = do_tool_oauth()

    # A batch is one scan through the list of followers - we have to scan as we only
    # get 20 per API call, and have 15 API calls / 15 minutes (as of Feb 2013).
    # The cursor is Twitter's identifier of where in the current batch we are.
    get_status()
    # Note that each user is only in the most recent batch they've been found in
    # (we don't keep all the history)

    # Look up latest followers count
    profile = tw.users.lookup(screen_name=screen_name)
    batch_expected = profile[0]['followers_count']

    # Things basically working, so make sure we run again
    os.system("crontab tool/crontab")

    # Get as many pages in the batch as we can (most likely 15!)
    while True:
        #raise httplib.IncompleteRead('hi') # for testing

        if next_cursor == -1:
            result = tw.followers.list(screen_name=screen_name)
        else:
            result = tw.followers.list(screen_name=screen_name, cursor=next_cursor)
        pages_got += 1
        for user in result['users']:
            save_user(current_batch, user, "twitter_followers")
        next_cursor = result['next_cursor']
        save_status()

        # While debugging, only do one page to avoid rate limits by uncommenting this:
        # break

        if next_cursor == 0:
            # We've finished a batch
            next_cursor = -1
            current_batch += 1
            save_status()
            break

except twitter.api.TwitterHTTPError, e:
    if "Twitter sent status 401 for URL" in str(e):
        clear_auth_and_restart()

    # https://dev.twitter.com/docs/error-codes-responses
    obj = json.loads(e.response_data)
    code = obj['errors'][0]['code']
    # authentication failure
    if (code in [32, 89]):
        clear_auth_and_restart()
    # rate limit exceeded
    if code == 34:
        set_status_and_exit('not-there', 'error', 'User not on Twitter')
    if code == 88:
        # provided we got at least one page, rate limit isn't an error but expected
        if pages_got == 0:
            set_status_and_exit('rate-limit', 'error', 'Twitter is rate limiting you')
    else:
        # anything else is an unexpected error - if ones occur a lot, add the above instead
        raise
except httplib.IncompleteRead, e:
    # I think this is effectively a rate limit error - so only count if it was first error
    if pages_got == 0:
        set_status_and_exit('rate-limit', 'error', 'Twitter broke the conncetion')

# Save progress message
if batch_got == batch_expected:
    set_status_and_exit("ok-updating", 'ok', "Fully up to date")
else:
    set_status_and_exit("ok-updating", 'error', "Running... %d/%d" % (batch_got, batch_expected))

<|endoftext|>


<commit_before># Copyright 2012 Jeff Trawick, http://emptyhammock.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re
import subprocess
import sys

import collect
import httpd
import process_model

class pstack:

    def __init__(self, **kwargs):
        self.corefile = kwargs.get('corefile')
        self.exe = kwargs.get('exe')
        self.hdr = None
        self.pstackout = kwargs.get('debuglog')
        self.proc = kwargs.get('proc')
        if not self.proc:
            self.proc = process_model.process()
        self.pid = self.proc.get_pid()

    def parse(self):
        if not self.pstackout:
            self.get_output()
        thr = None
        if collect.is_hdr(self.pstackout[0]):
            self.hdr = self.pstackout[0]
            self.pstackout = self.pstackout[1:]
        if 'REM /usr/bin/pstack' in self.pstackout[0]:
            self.pstackout = self.pstackout[1:]
        m = re.search('^(\d+):[ \t]+(/[^ ]+)', self.pstackout[0])
        if m:
            self.pid = m.group(1)
            self.exe = m.group(2)
        else:
            m = re.search('^core +.* of (\d+):[ \t]+([^ ]+)', self.pstackout[0])
            if not m:
                raise Exception('could not parse >%s<' % self.pstackout[0])
            self.pid = m.group(1)
            self.exe = m.group(2)
        for l in self.pstackout[1:]:
            if 'REM ' in l and l[0:3] == 'REM ':
                break
            m = re.search('^-+ +lwp# +\d+ +/ +thread# +(\d+) +', l)
            if m:
                thr = process_model.thread(m.group(1))
                self.proc.add_thread(thr)
                frameno = 0
                continue
            m = re.search('^ +([\da-f]+) +([^ ]+) +(\([^)]*\))', l)
            if m:
                addr = m.group(1)
                fn = m.group(2)
                fnargs = m.group(3)
                frameno += 1
                fr = process_model.frame(frameno, fn)
                thr.add_frame(fr)
                continue
            m = re.search('^ +([\da-f]+) +([^ ]+) *(\([^)]*\)), exit value =', l)
            if m:
                thr.set_exited()
        if self.pid and not self.proc.pid:
            self.proc.pid = self.pid
        if self.exe and not self.proc.exe:
            self.proc.exe = self.exe

    def get_output(self):
        self.pstackout = collect.pstack_collect(None, self.pid, self.corefile)

if __name__ == "__main__":
    print >> sys.stderr, "Don't run this directly."
<commit_msg>add missing fnargs parameter to frame() constructor<commit_after># Copyright 2012 Jeff Trawick, http://emptyhammock.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import re
import subprocess
import sys

import collect
import httpd
import process_model

class pstack:

    def __init__(self, **kwargs):
        self.corefile = kwargs.get('corefile')
        self.exe = kwargs.get('exe')
        self.hdr = None
        self.pstackout = kwargs.get('debuglog')
        self.proc = kwargs.get('proc')
        if not self.proc:
            self.proc = process_model.process()
        self.pid = self.proc.get_pid()

    def parse(self):
        if not self.pstackout:
            self.get_output()
        thr = None
        if collect.is_hdr(self.pstackout[0]):
            self.hdr = self.pstackout[0]
            self.pstackout = self.pstackout[1:]
        if 'REM /usr/bin/pstack' in self.pstackout[0]:
            self.pstackout = self.pstackout[1:]
        m = re.search('^(\d+):[ \t]+(/[^ ]+)', self.pstackout[0])
        if m:
            self.pid = m.group(1)
            self.exe = m.group(2)
        else:
            m = re.search('^core +.* of (\d+):[ \t]+([^ ]+)', self.pstackout[0])
            if not m:
                raise Exception('could not parse >%s<' % self.pstackout[0])
            self.pid = m.group(1)
            self.exe = m.group(2)
        for l in self.pstackout[1:]:
            if 'REM ' in l and l[0:3] == 'REM ':
                break
            m = re.search('^-+ +lwp# +\d+ +/ +thread# +(\d+) +', l)
            if m:
                thr = process_model.thread(m.group(1))
                self.proc.add_thread(thr)
                frameno = 0
                continue
            m = re.search('^ +([\da-f]+) +([^ ]+) +(\([^)]*\))', l)
            if m:
                addr = m.group(1)
                fn = m.group(2)
                fnargs = m.group(3)
                frameno += 1
                fr = process_model.frame(frameno, fn, fnargs)
                thr.add_frame(fr)
                continue
            m = re.search('^ +([\da-f]+) +([^ ]+) *(\([^)]*\)), exit value =', l)
            if m:
                thr.set_exited()
        if self.pid and not self.proc.pid:
            self.proc.pid = self.pid
        if self.exe and not self.proc.exe:
            self.proc.exe = self.exe

    def get_output(self):
        self.pstackout = collect.pstack_collect(None, self.pid, self.corefile)

if __name__ == "__main__":
    print >> sys.stderr, "Don't run this directly."
<|endoftext|>


<commit_before>#!/usr/bin/env python

# author: Alp Sayin
# mail: alpsayin[at]gmail
# sources:
# http://www.movable-type.co.uk/scripts/latlong.html
# http://gis.stackexchange.com/questions/29239/calculate-bearing-between-two-decimal-gps-coordinates
# 31.07.2013

import math
from math import sin, cos, acos, tan, atan2, sqrt, radians, degrees, asin, floor, log, pi

R = 6371 #km

def convertHMStoDecimal( hms):
	h = hms[0]
	m = hms[1]
	s = hms[2]
	if h < 0:
		sign = -1
	else:
		sign = 1
	dec = (h + (m/60.0) + (s/3600.0) ) * sign#/1000000.0;
	return round(dec, 6)

def convertDecimaltoHMS( dec):
	if dec < 0:
		sign = -1
	else:
		sign= 1

	h = floor(dec ) * sign
	m = floor(  ((dec) - floor(dec)) * 60)
	s = round( floor(((((dec) - floor(dec)) * 60) - floor(((dec) - floor(dec)) * 60)) * 100000) * 60/100000, 2)
	return (h, m, s)

class Gps(object):
	"""a simple gps class for holding lat and long and calculating distances and locations"""
	def __init__(self, lat=0.0, lon=0.0, alt=0.0):
		super(Gps, self).__init__()
		self.lat = round(lat, 6)
		self.lon = round(lon, 6)
		self.alt = round(alt, 6)

	def get_lattitude(self):
		return self.lat

	def get_longtitude(self):
		return self.lon

	def get_altitude(self):
		return self.alt

	def distanceTo(self, otherGps):
		'''result in meters'''
		deltaLatRad = radians(self.lat-otherGps.lat)
		deltaLonRad = radians(self.lon-otherGps.lon)
		lat1Rad = radians(self.lat)
		lat2Rad = radians(otherGps.lat)
		lon1Rad = radians(self.lon)
		lon2Rad = radians(otherGps.lon)

		# Spherical law of cosines
		d = acos( (sin(lat1Rad)*sin(lat2Rad)) + (cos(lat1Rad)*cos(lat2Rad)*cos(lon2Rad-lon1Rad))) * R;

		# Haversine formula
		# a = (sin(deltaLatRad/2) * sin(deltaLatRad/2)) + (sin(deltaLonRad/2) * sin(deltaLonRad/2) * cos(lat1Rad) * cos(lat2Rad))
		# c = 2 * atan2(sqrt(a), sqrt(1-a))
		# d = R * c
		return d*1000

	def bearingTo(self, otherGps):
		deltaLon = radians(otherGps.lon) - radians(self.lon)
		deltaLat = radians(otherGps.lat) - radians(self.lat)
		lat1 = radians(self.lat)
		lat2 = radians(otherGps.lat)
		# http://gis.stackexchange.com/questions/29239/calculate-bearing-between-two-decimal-gps-coordinates
		dPhi = log(tan(lat2/2.0+pi/4.0)/tan(lat1/2.0+pi/4.0))
		if abs(deltaLon) > pi:
			if deltaLon > 0.0:
				deltaLon = -(2.0 * pi - dLong)
			else:
				deltaLon = (2.0 * pi + dLong)

		bearing = (degrees(atan2(deltaLon, dPhi)) + 360.0) % 360.0
		return round(bearing, 6)

	def locationOf(self, bearing, distance):
		'''bearing in degrees, distance in meters'''
		distance_km = distance/1000.0
		bearing_r = radians(bearing)
		lat1 = radians(self.lat)
		lon1 = radians(self.lon)
		lat2 = asin( (sin(lat1)*cos(distance_km/R)) + (cos(lat1)*sin(distance_km/R)*cos(bearing_r)) )
		lon2 = lon1 + atan2( sin(bearing_r) * sin(distance_km/R) * cos(lat1), cos(distance_km/R) - (sin(lat1)*sin(lat2)) );
		return Gps(lat=degrees(lat2), lon=degrees(lon2))

	def isInBoundingCircle(self, otherGps, radius):
		'''radius in meters'''
		return self.distanceTo(otherGps) <= radius

	def generateWaypointFile(self, altitude, commit=False):
		template = '''QGC WPL 110
0	0	3	16	0.000000	0.000000	0.000000	0.000000	lat	lon	0.000000	1
1	0	3	16	0.000000	0.000000	0.000000	0.000000	lat	lon	alt.000000	1
2	1	3	17	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1
3	0	3	20	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1'''
		template = template.replace('lat', str(self.get_lattitude()))
		template = template.replace('lon', str(self.get_longtitude()))
		template = template.replace('alt', str(int(altitude)))
		if commit:
			wpfile = open('wp.txt', 'w')
			wpfile.write(template)
			wpfile.close()

	def __unicode__(self):
		return self.__str__()

	def __str__(self):
		lathms = convertDecimaltoHMS(self.lat)
		lonhms = convertDecimaltoHMS(self.lon)
		return 'D('+str(self.lat)+', '+str(self.lon)+') : HMS'+str(lathms)+', '+ str(lonhms)

if __name__ == '__main__':
	coordHms1 = ( (39,52,4.35), (32,45,5.27))
	coordHms2 = ( (39,52,7.79), (32,45,11.68))
	print 'HMS 1 -> ' + str(coordHms1)
	print 'HMS 2 -> ' + str(coordHms2)
	coordDec1 = ( convertHMStoDecimal(coordHms1[0]), convertHMStoDecimal(coordHms1[1]))
	coordDec2 = ( convertHMStoDecimal(coordHms2[0]), convertHMStoDecimal(coordHms2[1]))
	print 'Dec 1 ->' + str(coordDec1)
	print 'Dec 2 ->' + str(coordDec2)
	backHms1 = ( convertDecimaltoHMS(coordDec1[0]), convertDecimaltoHMS(coordDec1[1]))
	backHms2 = ( convertDecimaltoHMS(coordDec2[0]), convertDecimaltoHMS(coordDec2[1]))
	print 'HMS 1 -> ' + str(backHms1)
	print 'HMS 2 -> ' + str(backHms2)
	loc1 = Gps( coordDec1[0], coordDec1[1] )
	loc2 = Gps( coordDec2[0], coordDec2[1] )
	distance = loc1.distanceTo(loc2)
	bearing1 = loc1.bearingTo(loc2)
	bearing2 = loc2.bearingTo(loc1)
	print 'distance: ' + str(distance)
	print 'bearing12: ' + str(bearing1)
	print 'bearing12: ' + str(convertDecimaltoHMS(bearing1))
	print 'bearing21: ' + str(bearing2)
	print 'bearing21: ' + str(convertDecimaltoHMS(bearing2))
	loc2fromDistance = loc1.locationOf(bearing1, distance)
	loc1fromDistance = loc2.locationOf(bearing2, distance)
	print loc1
	print loc1fromDistance
	print loc2
	print loc2fromDistance

	balciLatHms = (39, 53, 23.23)
	balciLonHms = (32, 46, 30.70)
	balciLatDec = convertHMStoDecimal(balciLatHms)
	balciLonDec = convertHMStoDecimal(balciLonHms)
	balci = Gps(balciLatDec, balciLonDec)
	bodrumLatHms = (37, 01, 49.38)
	bodrumLonHms = (27, 25, 01.41)
	bodrumLatDec = convertHMStoDecimal(bodrumLatHms)
	bodrumLonDec = convertHMStoDecimal(bodrumLonHms)
	bodrum = Gps(bodrumLatDec, bodrumLonDec)
	bodrum = Gps(bodrumLatDec, bodrumLonDec)
	print 'balci : '+str(balci)
	print 'bodrum : '+str(bodrum)
	distance = balci.distanceTo(bodrum)/1000
	bearing = balci.bearingTo(bodrum)
	print 'distance: '+str(distance)
	print 'bearing: '+str(bearing)
	balci.generateWaypointFile(100, True)
	<commit_msg>add filename option to generateWaypointsFile function<commit_after>#!/usr/bin/env python

# author: Alp Sayin
# mail: alpsayin[at]gmail
# sources:
# http://www.movable-type.co.uk/scripts/latlong.html
# http://gis.stackexchange.com/questions/29239/calculate-bearing-between-two-decimal-gps-coordinates
# 31.07.2013

import math
from math import sin, cos, acos, tan, atan2, sqrt, radians, degrees, asin, floor, log, pi

R = 6371 #km

def convertHMStoDecimal( hms):
	h = hms[0]
	m = hms[1]
	s = hms[2]
	if h < 0:
		sign = -1
	else:
		sign = 1
	dec = (h + (m/60.0) + (s/3600.0) ) * sign#/1000000.0;
	return round(dec, 6)

def convertDecimaltoHMS( dec):
	if dec < 0:
		sign = -1
	else:
		sign= 1

	h = floor(dec ) * sign
	m = floor(  ((dec) - floor(dec)) * 60)
	s = round( floor(((((dec) - floor(dec)) * 60) - floor(((dec) - floor(dec)) * 60)) * 100000) * 60/100000, 2)
	return (h, m, s)

class Gps(object):
	"""a simple gps class for holding lat and long and calculating distances and locations"""
	def __init__(self, lat=0.0, lon=0.0, alt=0.0):
		super(Gps, self).__init__()
		self.lat = round(lat, 6)
		self.lon = round(lon, 6)
		self.alt = round(alt, 6)

	def get_lattitude(self):
		return self.lat

	def get_longtitude(self):
		return self.lon

	def get_altitude(self):
		return self.alt

	def distanceTo(self, otherGps):
		'''result in meters'''
		deltaLatRad = radians(self.lat-otherGps.lat)
		deltaLonRad = radians(self.lon-otherGps.lon)
		lat1Rad = radians(self.lat)
		lat2Rad = radians(otherGps.lat)
		lon1Rad = radians(self.lon)
		lon2Rad = radians(otherGps.lon)

		# Spherical law of cosines
		d = acos( (sin(lat1Rad)*sin(lat2Rad)) + (cos(lat1Rad)*cos(lat2Rad)*cos(lon2Rad-lon1Rad))) * R;

		# Haversine formula
		# a = (sin(deltaLatRad/2) * sin(deltaLatRad/2)) + (sin(deltaLonRad/2) * sin(deltaLonRad/2) * cos(lat1Rad) * cos(lat2Rad))
		# c = 2 * atan2(sqrt(a), sqrt(1-a))
		# d = R * c
		return d*1000

	def bearingTo(self, otherGps):
		deltaLon = radians(otherGps.lon) - radians(self.lon)
		deltaLat = radians(otherGps.lat) - radians(self.lat)
		lat1 = radians(self.lat)
		lat2 = radians(otherGps.lat)
		# http://gis.stackexchange.com/questions/29239/calculate-bearing-between-two-decimal-gps-coordinates
		dPhi = log(tan(lat2/2.0+pi/4.0)/tan(lat1/2.0+pi/4.0))
		if abs(deltaLon) > pi:
			if deltaLon > 0.0:
				deltaLon = -(2.0 * pi - dLong)
			else:
				deltaLon = (2.0 * pi + dLong)

		bearing = (degrees(atan2(deltaLon, dPhi)) + 360.0) % 360.0
		return round(bearing, 6)

	def locationOf(self, bearing, distance):
		'''bearing in degrees, distance in meters'''
		distance_km = distance/1000.0
		bearing_r = radians(bearing)
		lat1 = radians(self.lat)
		lon1 = radians(self.lon)
		lat2 = asin( (sin(lat1)*cos(distance_km/R)) + (cos(lat1)*sin(distance_km/R)*cos(bearing_r)) )
		lon2 = lon1 + atan2( sin(bearing_r) * sin(distance_km/R) * cos(lat1), cos(distance_km/R) - (sin(lat1)*sin(lat2)) );
		return Gps(lat=degrees(lat2), lon=degrees(lon2))

	def isInBoundingCircle(self, otherGps, radius):
		'''radius in meters'''
		return self.distanceTo(otherGps) <= radius

	def generateWaypointFile(self, altitude, commit=False, filename='wp.txt'):
		template = '''QGC WPL 110
0	0	3	16	0.000000	0.000000	0.000000	0.000000	lat	lon	0.000000	1
1	0	3	16	0.000000	0.000000	0.000000	0.000000	lat	lon	alt.000000	1
2	1	3	17	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1
3	0	3	20	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	1'''
		template = template.replace('lat', str(self.get_lattitude()))
		template = template.replace('lon', str(self.get_longtitude()))
		template = template.replace('alt', str(int(altitude)))
		if commit:
			wpfile = open( filename, 'w')
			wpfile.write(template)
			wpfile.close()

	def __unicode__(self):
		return self.__str__()

	def __str__(self):
		lathms = convertDecimaltoHMS(self.lat)
		lonhms = convertDecimaltoHMS(self.lon)
		return 'D('+str(self.lat)+', '+str(self.lon)+') : HMS'+str(lathms)+', '+ str(lonhms)

if __name__ == '__main__':
	coordHms1 = ( (39,52,4.35), (32,45,5.27))
	coordHms2 = ( (39,52,7.79), (32,45,11.68))
	print 'HMS 1 -> ' + str(coordHms1)
	print 'HMS 2 -> ' + str(coordHms2)
	coordDec1 = ( convertHMStoDecimal(coordHms1[0]), convertHMStoDecimal(coordHms1[1]))
	coordDec2 = ( convertHMStoDecimal(coordHms2[0]), convertHMStoDecimal(coordHms2[1]))
	print 'Dec 1 ->' + str(coordDec1)
	print 'Dec 2 ->' + str(coordDec2)
	backHms1 = ( convertDecimaltoHMS(coordDec1[0]), convertDecimaltoHMS(coordDec1[1]))
	backHms2 = ( convertDecimaltoHMS(coordDec2[0]), convertDecimaltoHMS(coordDec2[1]))
	print 'HMS 1 -> ' + str(backHms1)
	print 'HMS 2 -> ' + str(backHms2)
	loc1 = Gps( coordDec1[0], coordDec1[1] )
	loc2 = Gps( coordDec2[0], coordDec2[1] )
	distance = loc1.distanceTo(loc2)
	bearing1 = loc1.bearingTo(loc2)
	bearing2 = loc2.bearingTo(loc1)
	print 'distance: ' + str(distance)
	print 'bearing12: ' + str(bearing1)
	print 'bearing12: ' + str(convertDecimaltoHMS(bearing1))
	print 'bearing21: ' + str(bearing2)
	print 'bearing21: ' + str(convertDecimaltoHMS(bearing2))
	loc2fromDistance = loc1.locationOf(bearing1, distance)
	loc1fromDistance = loc2.locationOf(bearing2, distance)
	print loc1
	print loc1fromDistance
	print loc2
	print loc2fromDistance

	balciLatHms = (39, 53, 23.23)
	balciLonHms = (32, 46, 30.70)
	balciLatDec = convertHMStoDecimal(balciLatHms)
	balciLonDec = convertHMStoDecimal(balciLonHms)
	balci = Gps(balciLatDec, balciLonDec)
	bodrumLatHms = (37, 01, 49.38)
	bodrumLonHms = (27, 25, 01.41)
	bodrumLatDec = convertHMStoDecimal(bodrumLatHms)
	bodrumLonDec = convertHMStoDecimal(bodrumLonHms)
	bodrum = Gps(bodrumLatDec, bodrumLonDec)
	bodrum = Gps(bodrumLatDec, bodrumLonDec)
	print 'balci : '+str(balci)
	print 'bodrum : '+str(bodrum)
	distance = balci.distanceTo(bodrum)/1000
	bearing = balci.bearingTo(bodrum)
	print 'distance: '+str(distance)
	print 'bearing: '+str(bearing)
	balci.generateWaypointFile(100, True)
	<|endoftext|>


<commit_before>from addict import Dict

__version__ = '2.4.3'

config = Dict()

config.video_domains = []
config.audio_domains = []
config.image_domains = []

config.video_formatting = ''
config.audio_formatting = ''
config.image_formatting = ''
config.header = ''

config.subreddits_to_check = []

config.tor_mods = []

config.perform_header_check = True
config.debug_mode = False

# section for gifs
config.no_gifs = []

# global flag to enable / disable placing the triggers
# for the OCR bot
config.OCR = True
<commit_msg>forgot to update version number for config change<commit_after>from addict import Dict

__version__ = '2.5.1'

config = Dict()

config.video_domains = []
config.audio_domains = []
config.image_domains = []

config.video_formatting = ''
config.audio_formatting = ''
config.image_formatting = ''
config.header = ''

config.subreddits_to_check = []

config.tor_mods = []

config.perform_header_check = True
config.debug_mode = False

# section for gifs
config.no_gifs = []

# global flag to enable / disable placing the triggers
# for the OCR bot
config.OCR = True
<|endoftext|>


<commit_before>mydict = {1:'foo',2:'bar',3:'Lumberjack',4:'I work all day'}
for item in mydict:
  print(mydict[item])
<commit_msg>Added use of Python shelve<commit_after>import shelve
s = shelve.open('drows.db')
try:
  s['drows'] = {1:'foo',2:'bar',3:'Lumberjack',4:'I work all day'}
finally:
  s.close()

s = shelve.open('drows.db')
drow = s['drows']

for item in drow:
  print(drow[item])
<|endoftext|>


<commit_before>from toydist.core import \
        PackageDescription
from toydist.core.utils import \
        InvalidPackage, validate_package as _validate_package
from toydist.core.pkg_objects import \
        Executable

_PKG_TO_DIST = {
        "ext_modules": lambda pkg: [v for v  in \
                                    pkg.extensions.values()],
        "platforms": lambda pkg: [v for v  in pkg.platforms],
        "packages": lambda pkg: [v for v  in pkg.packages],
        "py_modules": lambda pkg: [v for v  in pkg.py_modules],
}

for k in ["name", "version", "summary", "url", "author",
          "author_email", "maintainer", "maintainer_email",
          "license", "description", "download_url"]:
    _PKG_TO_DIST[k] = lambda pkg: getattr(pkg, k)

def pkg_to_distutils(pkg):
    """Convert PackageDescription instance to a dict which may be used
    as argument to distutils/setuptools setup function."""
    d = {}

    for k, v in _PKG_TO_DIST.items():
        d[k] = v(pkg)

    return d

def validate_packages(pkgs):
    ret_pkgs = []
    for pkg in pkgs:
        try:
            _validate_package(pkg, ".")
        except InvalidPackage:
            # FIXME: add the package as data here
            pass
        else:
            ret_pkgs.append(pkg)
    return ret_pkgs

def distutils_to_package_description(dist):
    data = {}

    data['name'] = dist.get_name()
    data['version'] = dist.get_version()
    data['author'] = dist.get_author()
    data['author_email'] = dist.get_author_email()
    data['maintainer'] = dist.get_contact()
    data['maintainer_email'] = dist.get_contact_email()
    data['summary'] = dist.get_description()
    data['description'] = dist.get_long_description()
    data['license'] = dist.get_license()
    data['platforms'] = dist.get_platforms()

    data['download_url'] = dist.get_download_url()
    data['url'] = dist.get_url()

    # XXX: reliable way to detect whether Distribution was monkey-patched by
    # setuptools
    try:
        reqs = getattr(dist, "install_requires")
        # FIXME: how to detect this correctly
        if issubclass(type(reqs), (str, unicode)):
            reqs = [reqs]
        data['install_requires'] = reqs
    except AttributeError:
        pass

    data['py_modules'] = dist.py_modules
    data['packages'] = validate_packages(dist.packages)
    data['extensions'] = dist.ext_modules
    data['classifiers'] = dist.get_classifiers()

    entry_points = entry_points_from_dist(dist)

    data["executables"] = {}
    try:
        console_scripts = entry_points["console_scripts"]
    except KeyError:
        console_scripts = []
    for entry in console_scripts:
        exe = Executable.from_representation(entry)
        data["executables"][exe.name] = exe

    return PackageDescription(**data)

_DIST_CONV_DICT = {
    "long_description": lambda meta: meta.description,
    "description": lambda meta: meta.summary,
    # TODO: keywords not implemented yet
    "keywords": lambda meta: [],
    "fullname": lambda meta: "%s-%s" % (meta.name, meta.version),
    "contact": lambda meta: (meta.maintainer or
                             meta.author or
                             "UNKNOWN"),
    "contact_email": lambda meta: (meta.maintainer_email or
                                   meta.author_email or
                                   "UNKNOWN"),
    "requires": lambda meta: meta.install_requires
}

def to_distutils_meta(meta):
    from distutils.dist import DistributionMetadata
    ret = DistributionMetadata()
    for m in ret._METHOD_BASENAMES:
        try:
            val = _DIST_CONV_DICT[m](meta)
        except KeyError:
            val = getattr(meta, m)
        setattr(ret, m, val)

    return ret

def write_pkg_info(pkg, file):
    dist_meta = to_distutils_meta(pkg)
    dist_meta.write_pkg_file(file)

def entry_points_from_dist(dist):
    if hasattr(dist, "entry_points"):
        from pkg_resources import split_sections
        if isinstance(dist.entry_points, basestring):
            entry_points = {}
            sections = split_sections(dist.entry_points)
            for group, lines in sections:
                group = group.strip()
                entry_points[group] = lines
        else:
            entry_points = dist.entry_points
    else:
        entry_points = {}
    return entry_points
<commit_msg>BUG: fix entry_points handling in conversion.<commit_after>from toydist.core import \
        PackageDescription
from toydist.core.utils import \
        InvalidPackage, validate_package as _validate_package
from toydist.core.pkg_objects import \
        Executable

_PKG_TO_DIST = {
        "ext_modules": lambda pkg: [v for v  in \
                                    pkg.extensions.values()],
        "platforms": lambda pkg: [v for v  in pkg.platforms],
        "packages": lambda pkg: [v for v  in pkg.packages],
        "py_modules": lambda pkg: [v for v  in pkg.py_modules],
}

for k in ["name", "version", "summary", "url", "author",
          "author_email", "maintainer", "maintainer_email",
          "license", "description", "download_url"]:
    _PKG_TO_DIST[k] = lambda pkg: getattr(pkg, k)

def pkg_to_distutils(pkg):
    """Convert PackageDescription instance to a dict which may be used
    as argument to distutils/setuptools setup function."""
    d = {}

    for k, v in _PKG_TO_DIST.items():
        d[k] = v(pkg)

    return d

def validate_packages(pkgs):
    ret_pkgs = []
    for pkg in pkgs:
        try:
            _validate_package(pkg, ".")
        except InvalidPackage:
            # FIXME: add the package as data here
            pass
        else:
            ret_pkgs.append(pkg)
    return ret_pkgs

def distutils_to_package_description(dist):
    data = {}

    data['name'] = dist.get_name()
    data['version'] = dist.get_version()
    data['author'] = dist.get_author()
    data['author_email'] = dist.get_author_email()
    data['maintainer'] = dist.get_contact()
    data['maintainer_email'] = dist.get_contact_email()
    data['summary'] = dist.get_description()
    data['description'] = dist.get_long_description()
    data['license'] = dist.get_license()
    data['platforms'] = dist.get_platforms()

    data['download_url'] = dist.get_download_url()
    data['url'] = dist.get_url()

    # XXX: reliable way to detect whether Distribution was monkey-patched by
    # setuptools
    try:
        reqs = getattr(dist, "install_requires")
        # FIXME: how to detect this correctly
        if issubclass(type(reqs), (str, unicode)):
            reqs = [reqs]
        data['install_requires'] = reqs
    except AttributeError:
        pass

    data['py_modules'] = dist.py_modules
    data['packages'] = validate_packages(dist.packages)
    data['extensions'] = dist.ext_modules
    data['classifiers'] = dist.get_classifiers()

    data["executables"] = {}

    entry_points = entry_points_from_dist(dist)
    if entry_points:
        console_scripts = entry_points.get("console_scripts", [])
        for entry in console_scripts:
            exe = Executable.from_representation(entry)
            data["executables"][exe.name] = exe

    return PackageDescription(**data)

_DIST_CONV_DICT = {
    "long_description": lambda meta: meta.description,
    "description": lambda meta: meta.summary,
    # TODO: keywords not implemented yet
    "keywords": lambda meta: [],
    "fullname": lambda meta: "%s-%s" % (meta.name, meta.version),
    "contact": lambda meta: (meta.maintainer or
                             meta.author or
                             "UNKNOWN"),
    "contact_email": lambda meta: (meta.maintainer_email or
                                   meta.author_email or
                                   "UNKNOWN"),
    "requires": lambda meta: meta.install_requires
}

def to_distutils_meta(meta):
    from distutils.dist import DistributionMetadata
    ret = DistributionMetadata()
    for m in ret._METHOD_BASENAMES:
        try:
            val = _DIST_CONV_DICT[m](meta)
        except KeyError:
            val = getattr(meta, m)
        setattr(ret, m, val)

    return ret

def write_pkg_info(pkg, file):
    dist_meta = to_distutils_meta(pkg)
    dist_meta.write_pkg_file(file)

def entry_points_from_dist(dist):
    if hasattr(dist, "entry_points"):
        from pkg_resources import split_sections
        if isinstance(dist.entry_points, basestring):
            entry_points = {}
            sections = split_sections(dist.entry_points)
            for group, lines in sections:
                group = group.strip()
                entry_points[group] = lines
        else:
            entry_points = dist.entry_points
    else:
        entry_points = {}
    return entry_points
<|endoftext|>


<commit_before># coding:utf-8
import time
from logger.log import crawler
from tasks.workers import app
from page_parse.user import public
from page_get.basic import get_page
from db.seed_ids import get_home_ids
from config.conf import get_max_home_page
from page_parse.weibo import get_weibo_list
# new import for wb_pic
from db.wb_pic import insert_weibo_pics
from db.seed_ids import set_seed_home_crawled
from db.db_proxy import set_proxy_score, get_a_random_proxy

# import for exception test
# from page_parse.weibo import check_no_bottom, check_no_bottom, parse_json_to_dict
# end
ori_wb_temp_url = 'http://m.weibo.cn/api/container/getIndex?containerid={}_-_WEIBO_SECOND_PROFILE_WEIBO_ORI&luicode={}&lfid={}&featurecode={}&type=uid&value={}&page_type={}&page={}'

# @app.task(ignore_result=True)
def crawl_weibo(uid):

    limit = get_max_home_page()
    cur_page = 1

    pic_count = 0
    max_pic_count = 150

    max_retry_cnt = 2
    cur_retry_cnt = 0

    containerid = '230413' + uid
    luicode = '10000011'
    lfid = '230283' + uid
    featurecode = '20000180'
    value = uid
    page_type = '03'
    page = cur_page

    proxy = get_a_random_proxy()

    url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
    html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

    # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
    # 如果重试还是返回空html，那么两个proxy均不扣分，记录uid异常后直接return，如果返回非空但无效的html，则在后面流程进行扣分
    if html == '':
        if cur_retry_cnt < max_retry_cnt:
            proxy = get_a_random_proxy()
            html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
            if html == '':
                crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, 5))
                set_seed_home_crawled(uid, 3)
                return
        else:
            exception_uid_handler(uid, 1, proxy)
            return
    # end

    weibo_pics = get_weibo_list(html)

    if weibo_pics == None:
        exception_uid_handler(uid, 2, proxy, html)
        return
    elif weibo_pics == False:
        finish_uid_handler(uid, proxy)
        return
    elif weibo_pics:
        insert_weibo_pics(weibo_pics)

    pic_count = pic_count + len(weibo_pics)

    cur_page += 1

    while cur_page <= limit and pic_count < max_pic_count:

        page = cur_page
        url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
        html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

        # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
        if html == '':
            if cur_retry_cnt < max_retry_cnt:
                proxy = get_a_random_proxy()
                html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
                if html == '':
                    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, 6))
                    set_seed_home_crawled(uid, 3)
                    return
            else:
                exception_uid_handler(uid, 3, proxy)
                return
        # end

        weibo_pics = get_weibo_list(html)

        if weibo_pics == None:
            exception_uid_handler(uid, 4, proxy, html)
            return
        elif weibo_pics == False:
            finish_uid_handler(uid, proxy)
            return
        elif weibo_pics:
            insert_weibo_pics(weibo_pics)

        pic_count = pic_count + len(weibo_pics)

        cur_page += 1

    finish_uid_handler(uid, proxy)
    return

# @app.task
def excute_weibo_task():
    id_objs = get_home_ids(0, 50)

    for id_obj in id_objs:
        # app.send_task('tasks.weibo.crawl_weibo', args=(id_obj.uid,), queue='weibo_crawler',
        #               routing_key='weibo_info')
        crawl_weibo(id_obj.uid)

def finish_uid_handler(uid, proxy):
    crawler.warning('用户id为{}的相册采集完成'.format(uid))
    set_seed_home_crawled(uid, 4)
    set_proxy_score(proxy, 1)

def exception_uid_handler(uid, err_code, proxy ={}, html=''):
    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, err_code))
    set_seed_home_crawled(uid, 3)
    if proxy:
        set_proxy_score(proxy, -1)

<commit_msg>test for proxy<commit_after># coding:utf-8
import time
from logger.log import crawler
from tasks.workers import app
from page_parse.user import public
from page_get.basic import get_page
from db.seed_ids import get_home_ids
from config.conf import get_max_home_page
from page_parse.weibo import get_weibo_list
# new import for wb_pic
from db.wb_pic import insert_weibo_pics
from db.seed_ids import set_seed_home_crawled
from db.db_proxy import set_proxy_score, get_a_random_proxy

# import for exception test
# from page_parse.weibo import check_no_bottom, check_no_bottom, parse_json_to_dict
# end
ori_wb_temp_url = 'http://m.weibo.cn/api/container/getIndex?containerid={}_-_WEIBO_SECOND_PROFILE_WEIBO_ORI&luicode={}&lfid={}&featurecode={}&type=uid&value={}&page_type={}&page={}'

# @app.task(ignore_result=True)
def crawl_weibo(uid):

    limit = get_max_home_page()
    cur_page = 1

    pic_count = 0
    max_pic_count = 150

    max_retry_cnt = 2
    cur_retry_cnt = 0

    containerid = '230413' + uid
    luicode = '10000011'
    lfid = '230283' + uid
    featurecode = '20000180'
    value = uid
    page_type = '03'
    page = cur_page

    proxy = get_a_random_proxy()

    # test for proxy
    print(proxy)
    # end

    url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
    html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

    # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
    # 如果重试还是返回空html，那么两个proxy均不扣分，记录uid异常后直接return，如果返回非空但无效的html，则在后面流程进行扣分
    if html == '':
        if cur_retry_cnt < max_retry_cnt:
            proxy = get_a_random_proxy()
            html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
            if html == '':
                crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, 5))
                set_seed_home_crawled(uid, 3)
                return
        else:
            exception_uid_handler(uid, 1, proxy)
            return
    # end

    weibo_pics = get_weibo_list(html)

    if weibo_pics == None:
        exception_uid_handler(uid, 2, proxy, html)
        return
    elif weibo_pics == False:
        finish_uid_handler(uid, proxy)
        return
    elif weibo_pics:
        insert_weibo_pics(weibo_pics)

    pic_count = pic_count + len(weibo_pics)

    cur_page += 1

    while cur_page <= limit and pic_count < max_pic_count:

        page = cur_page
        url = ori_wb_temp_url.format(containerid, luicode, lfid, featurecode, value, page_type, page)
        html = get_page(url, user_verify=False, need_login=False, proxys=proxy)

        # html为空也有可能是其他原因，但是代理问题应该是大概率，因此对代理进行扣分。
        if html == '':
            if cur_retry_cnt < max_retry_cnt:
                proxy = get_a_random_proxy()
                html = get_page(url, user_verify=False, need_login=False, proxys=proxy)
                if html == '':
                    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, 6))
                    set_seed_home_crawled(uid, 3)
                    return
            else:
                exception_uid_handler(uid, 3, proxy)
                return
        # end

        weibo_pics = get_weibo_list(html)

        if weibo_pics == None:
            exception_uid_handler(uid, 4, proxy, html)
            return
        elif weibo_pics == False:
            finish_uid_handler(uid, proxy)
            return
        elif weibo_pics:
            insert_weibo_pics(weibo_pics)

        pic_count = pic_count + len(weibo_pics)

        cur_page += 1

    finish_uid_handler(uid, proxy)
    return

# @app.task
def excute_weibo_task():
    id_objs = get_home_ids(0, 50)

    for id_obj in id_objs:
        # app.send_task('tasks.weibo.crawl_weibo', args=(id_obj.uid,), queue='weibo_crawler',
        #               routing_key='weibo_info')
        crawl_weibo(id_obj.uid)

def finish_uid_handler(uid, proxy):
    crawler.warning('用户id为{}的相册采集完成'.format(uid))
    set_seed_home_crawled(uid, 4)
    set_proxy_score(proxy, 1)

def exception_uid_handler(uid, err_code, proxy ={}, html=''):
    crawler.warning('用户id为{}的相册采集出错，这一请求接收到的内容为{}，状态码{}'.format(uid, html, err_code))
    set_seed_home_crawled(uid, 3)
    if proxy:
        set_proxy_score(proxy, -1)

<|endoftext|>


<commit_before>"""Training script for the DeepLab-ResNet network on the PASCAL VOC dataset
   for semantic image segmentation.

This script trains the model using augmented PASCAL VOC,
which contains approximately 10000 images for training and 1500 images for validation.
"""

from __future__ import print_function

import argparse
from datetime import datetime
import os
import sys
import time

import tensorflow as tf
import numpy as np

from deeplab_resnet import DeepLabResNetModel, decode_labels, inv_preprocess, prepare_label

from deeplab_resnet.LUNA16image_reader import ImageReader_LUNA16

IMG_MEAN = np.array((0,0,0), dtype=np.float32)

BATCH_SIZE = 10
DATA_DIRECTORY = '/home/zack/Data/LUNA16/'
MASK_DIRECTORY = '/home/zack/Data/LUNA16/seg-lungs-LUNA16raw'
IGNORE_LABEL = 255
INPUT_SIZE = '312,312'
LEARNING_RATE = 2.5e-4
MOMENTUM = 0.9
NUM_CLASSES = 4
NUM_STEPS = 20001
POWER = 0.9
RANDOM_SEED = 1234
# RESTORE_FROM = './deeplab_resnet.ckpt'
RESTORE_FROM = None

SAVE_NUM_IMAGES = 2
SAVE_PRED_EVERY = 1000
SNAPSHOT_DIR = './snapshots/'
WEIGHT_DECAY = 0.0005

def get_arguments():
    """Parse all the arguments provided from the CLI.

    Returns:
      A list of parsed arguments.
    """
    parser = argparse.ArgumentParser(description="DeepLab-ResNet Network")
    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
                        help="Number of images sent to the network in one step.")
    parser.add_argument("--data-dir", type=str, default=DATA_DIRECTORY,
                        help="Path to the directory containing the PASCAL VOC dataset.")
    parser.add_argument("--mask-dir", type=str, default=MASK_DIRECTORY,
                        help="Path to the file listing the images in the dataset.")
    parser.add_argument("--ignore-label", type=int, default=IGNORE_LABEL,
                        help="The index of the label to ignore during the training.")
    parser.add_argument("--input-size", type=str, default=INPUT_SIZE,
                        help="Comma-separated string with height and width of images.")
    parser.add_argument("--is-training", action="store_true",
                        help="Whether to updates the running means and variances during the training.")
    parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                        help="Base learning rate for training with polynomial decay.")
    parser.add_argument("--momentum", type=float, default=MOMENTUM,
                        help="Momentum component of the optimiser.")
    parser.add_argument("--not-restore-last", action="store_true",
                        help="Whether to not restore last (FC) layers.")
    parser.add_argument("--num-classes", type=int, default=NUM_CLASSES,
                        help="Number of classes to predict (including background).")
    parser.add_argument("--num-steps", type=int, default=NUM_STEPS,
                        help="Number of training steps.")
    parser.add_argument("--power", type=float, default=POWER,
                        help="Decay parameter to compute the learning rate.")
    parser.add_argument("--random-mirror", action="store_true",
                        help="Whether to randomly mirror the inputs during the training.")
    parser.add_argument("--random-scale", action="store_true",
                        help="Whether to randomly scale the inputs during the training.")
    parser.add_argument("--random-seed", type=int, default=RANDOM_SEED,
                        help="Random seed to have reproducible results.")
    parser.add_argument("--restore-from", type=str, default=RESTORE_FROM,
                        help="Where restore model parameters from.")
    parser.add_argument("--save-num-images", type=int, default=SAVE_NUM_IMAGES,
                        help="How many images to save.")
    parser.add_argument("--save-pred-every", type=int, default=SAVE_PRED_EVERY,
                        help="Save summaries and checkpoint every often.")
    parser.add_argument("--snapshot-dir", type=str, default=SNAPSHOT_DIR,
                        help="Where to save snapshots of the model.")
    parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                        help="Regularisation parameter for L2-loss.")
    return parser.parse_args()

def save(saver, sess, logdir, step):
   '''Save weights.

   Args:
     saver: TensorFlow Saver object.
     sess: TensorFlow session.
     logdir: path to the snapshots directory.
     step: current training step.
   '''
   model_name = 'model.ckpt'
   checkpoint_path = os.path.join(logdir, model_name)

   if not os.path.exists(logdir):
      os.makedirs(logdir)
   saver.save(sess, checkpoint_path, global_step=step)
   print('The checkpoint has been created.')

def load(saver, sess, ckpt_path):
    '''Load trained weights.

    Args:
      saver: TensorFlow Saver object.
      sess: TensorFlow session.
      ckpt_path: path to checkpoint file with parameters.
    '''
    saver.restore(sess, ckpt_path)
    print("Restored model parameters from {}".format(ckpt_path))

def main():
    """Create the model and start the training."""
    args = get_arguments()
    print (args)
    print (args.mask_dir)

    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)

    tf.set_random_seed(args.random_seed)

    # Create queue coordinator.
    coord = tf.train.Coordinator()

    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader_LUNA16(
            args.data_dir,
            args.mask_dir,
            input_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            IMG_MEAN,
            coord)

        image_batch, label_batch = reader.dequeue(args.batch_size)

    # Create network.
    net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
    # For a small batch size, it is better to keep
    # the statistics of the BN layers (running means and variances)
    # frozen, and to not update the values provided by the pre-trained model.
    # If is_training=True, the statistics will be updated during the training.
    # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
    # if they are presented in var_list of the optimiser definition.

    # Predictions.
    raw_output = net.layers['fc1_voc12']
    # Which variables to load. Running means and variances are not trainable,
    # thus all_variables() should be restored.
    restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
    all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
    fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
    fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
    assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable))
    assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))

    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
    label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w]
    raw_gt = tf.reshape(label_proc, [-1,])
    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    prediction = tf.gather(raw_prediction, indices)

    # Pixel-wise softmax loss.
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
    reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)

    # Processed predictions: for visualisation.
    raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
    raw_output_up = tf.argmax(raw_output_up, dimension=3)
    pred = tf.expand_dims(raw_output_up, dim=3)

    # Image summary.
    images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
    labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
    preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)

    total_summary = tf.summary.image('images',
                                     tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
                                     max_outputs=args.save_num_images) # Concatenate row-wise.
    summary_writer = tf.summary.FileWriter(args.snapshot_dir,
                                           graph=tf.get_default_graph())

    # Define loss and optimisation parameters.
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))

    opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)

    grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
    grads_conv = grads[:len(conv_trainable)]
    grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))]
    grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]

    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)

    # Set up tf session and initialize variables.
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()

    sess.run(init)

    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)

    # Load variables if the checkpoint is provided.
    if args.restore_from is not None:
        loader = tf.train.Saver(var_list=restore_var)
        load(loader, sess, args.restore_from)

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        feed_dict = { step_ph : step }

        if step % args.save_pred_every == 0:
            loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, train_op], feed_dict=feed_dict)
            summary_writer.add_summary(summary, step)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
    coord.request_stop()
    coord.join(threads)

if __name__ == '__main__':
    main()
<commit_msg>Test successful for LUNA16 on hawk<commit_after>"""Training script for the DeepLab-ResNet network on the PASCAL VOC dataset
   for semantic image segmentation.

This script trains the model using augmented PASCAL VOC,
which contains approximately 10000 images for training and 1500 images for validation.
"""

from __future__ import print_function

import argparse
from datetime import datetime
import os
import sys
import time

import tensorflow as tf
import numpy as np

from deeplab_resnet import DeepLabResNetModel, decode_labels, inv_preprocess, prepare_label
from deeplab_resnet.LUNA16image_reader import ImageReader_LUNA16

IMG_MEAN = np.array((0,0,0), dtype=np.float32)

BATCH_SIZE = 10
DATA_DIRECTORY = '/home/zack/Data/LUNA16/'
MASK_DIRECTORY = '/home/zack/Data/LUNA16/seg-lungs-LUNA16braw'
IGNORE_LABEL = 0
INPUT_SIZE = '312,312'
LEARNING_RATE = 2.5e-4
MOMENTUM = 0.9
NUM_CLASSES = 4
NUM_STEPS = 20001
POWER = 0.9
RANDOM_SEED = 1234
RESTORE_FROM = './deeplab_resnet.ckpt'

SAVE_NUM_IMAGES = 2
SAVE_PRED_EVERY = 1000
SNAPSHOT_DIR = './snapshots/'
WEIGHT_DECAY = 0.0005

def get_arguments():
    """Parse all the arguments provided from the CLI.

    Returns:
      A list of parsed arguments.
    """
    parser = argparse.ArgumentParser(description="DeepLab-ResNet Network")
    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
                        help="Number of images sent to the network in one step.")
    parser.add_argument("--data-dir", type=str, default=DATA_DIRECTORY,
                        help="Path to the directory containing the PASCAL VOC dataset.")
    parser.add_argument("--mask-dir", type=str, default=MASK_DIRECTORY,
                        help="Path to the file listing the images in the dataset.")
    parser.add_argument("--ignore-label", type=int, default=IGNORE_LABEL,
                        help="The index of the label to ignore during the training.")
    parser.add_argument("--input-size", type=str, default=INPUT_SIZE,
                        help="Comma-separated string with height and width of images.")
    parser.add_argument("--is-training", action="store_true",
                        help="Whether to updates the running means and variances during the training.")
    parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                        help="Base learning rate for training with polynomial decay.")
    parser.add_argument("--momentum", type=float, default=MOMENTUM,
                        help="Momentum component of the optimiser.")
    parser.add_argument("--not-restore-last", action="store_true",
                        help="Whether to not restore last (FC) layers.")
    parser.add_argument("--num-classes", type=int, default=NUM_CLASSES,
                        help="Number of classes to predict (including background).")
    parser.add_argument("--num-steps", type=int, default=NUM_STEPS,
                        help="Number of training steps.")
    parser.add_argument("--power", type=float, default=POWER,
                        help="Decay parameter to compute the learning rate.")
    parser.add_argument("--random-mirror", action="store_true",
                        help="Whether to randomly mirror the inputs during the training.")
    parser.add_argument("--random-scale", action="store_true",
                        help="Whether to randomly scale the inputs during the training.")
    parser.add_argument("--random-seed", type=int, default=RANDOM_SEED,
                        help="Random seed to have reproducible results.")
    parser.add_argument("--restore-from", type=str, default=RESTORE_FROM,
                        help="Where restore model parameters from.")
    parser.add_argument("--save-num-images", type=int, default=SAVE_NUM_IMAGES,
                        help="How many images to save.")
    parser.add_argument("--save-pred-every", type=int, default=SAVE_PRED_EVERY,
                        help="Save summaries and checkpoint every often.")
    parser.add_argument("--snapshot-dir", type=str, default=SNAPSHOT_DIR,
                        help="Where to save snapshots of the model.")
    parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                        help="Regularisation parameter for L2-loss.")
    return parser.parse_args()

def save(saver, sess, logdir, step):
   '''Save weights.

   Args:
     saver: TensorFlow Saver object.
     sess: TensorFlow session.
     logdir: path to the snapshots directory.
     step: current training step.
   '''
   model_name = 'model.ckpt'
   checkpoint_path = os.path.join(logdir, model_name)

   if not os.path.exists(logdir):
      os.makedirs(logdir)
   saver.save(sess, checkpoint_path, global_step=step)
   print('The checkpoint has been created.')

def load(saver, sess, ckpt_path):
    '''Load trained weights.

    Args:
      saver: TensorFlow Saver object.
      sess: TensorFlow session.
      ckpt_path: path to checkpoint file with parameters.
    '''
    saver.restore(sess, ckpt_path)
    print("Restored model parameters from {}".format(ckpt_path))

def main():
    """Create the model and start the training."""
    args = get_arguments()
    print (args)
    print (args.mask_dir)

    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)

    tf.set_random_seed(args.random_seed)

    # Create queue coordinator.
    coord = tf.train.Coordinator()

    # Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReader_LUNA16(
            args.data_dir,
            args.mask_dir,
            input_size,
            args.random_scale,
            args.random_mirror,
            args.ignore_label,
            IMG_MEAN,
            coord)

        image_batch, label_batch = reader.dequeue(args.batch_size)

    # Create network.
    net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
    # For a small batch size, it is better to keep
    # the statistics of the BN layers (running means and variances)
    # frozen, and to not update the values provided by the pre-trained model.
    # If is_training=True, the statistics will be updated during the training.
    # Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
    # if they are presented in var_list of the optimiser definition.

    # Predictions.
    raw_output = net.layers['fc1_voc12']
    # Which variables to load. Running means and variances are not trainable,
    # thus all_variables() should be restored.
    restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
    all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
    fc_trainable = [v for v in all_trainable if 'fc' in v.name]
    conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
    fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
    fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
    assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable))
    assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))

    # Predictions: ignoring all predictions with labels greater or equal than n_classes
    raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
    label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w]
    raw_gt = tf.reshape(label_proc, [-1,])
    indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
    gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
    prediction = tf.gather(raw_prediction, indices)

    # Pixel-wise softmax loss.
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
    l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
    reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)

    # Processed predictions: for visualisation.
    raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
    raw_output_up = tf.argmax(raw_output_up, dimension=3)
    pred = tf.expand_dims(raw_output_up, dim=3)

    # Image summary.
    images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
    labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
    preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)

    total_summary = tf.summary.image('images',
                                     tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
                                     max_outputs=args.save_num_images) # Concatenate row-wise.
    summary_writer = tf.summary.FileWriter(args.snapshot_dir,
                                           graph=tf.get_default_graph())

    # Define loss and optimisation parameters.
    base_lr = tf.constant(args.learning_rate)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))

    opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
    opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
    opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)

    grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
    grads_conv = grads[:len(conv_trainable)]
    grads_fc_w = grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))]
    grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]

    train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
    train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
    train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))

    train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)

    # Set up tf session and initialize variables.
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()

    sess.run(init)

    # Saver for storing checkpoints of the model.
    saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)

    # Load variables if the checkpoint is provided.
    if args.restore_from is not None:
        loader = tf.train.Saver(var_list=restore_var)
        load(loader, sess, args.restore_from)

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(args.num_steps):
        start_time = time.time()
        feed_dict = { step_ph : step }

        if step % args.save_pred_every == 0:
            loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, train_op], feed_dict=feed_dict)
            summary_writer.add_summary(summary, step)
            save(saver, sess, args.snapshot_dir, step)
        else:
            loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
    coord.request_stop()
    coord.join(threads)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals
import xgboost as xg
import numpy as np
import pandas as pd
from os import listdir

def aggregate_features(X):
    return np.mean(X, axis=0)

def get_features():
    biz_id_list = []
    result_array = np.zeros((2000, 1024))
    path = 'features/inception-21k-global/'
    feature_files = listdir(path)
    for i, f in enumerate(feature_files):
        biz_id = int(f[:-4])
        feature_bag = np.load('features/inception-21k-global/' + f)
        out_feature = aggregate_features(feature_bag)
        biz_id_list.append(biz_id)
        result_array[i, :] = out_feature
    col_names = ['incpt21k-glp-avg-' + str(i) for i in range(result_array.shape[1])]
    feature_frame = pd.DataFrame(data=result_array, index=biz_id_list, columns=col_names)
    return feature_frame

def get_response():
    biz2label = pd.read_csv("rawdata/train.csv")
    result_array = np.zeros((2000, 9))
    for class_no in range(9):
        response = [
            1 if str(class_no) in str(l).split(" ") else 0 for l in biz2label["labels"]]
        result_array[:, class_no] = response
    response_frame = pd.DataFrame(
        data=result_array,
        index=biz2label["business_id"],
        columns=['class' + str(i) for i in range(9)],
        dtype=int)
    return response_frame

X = get_features()
Y = get_response()
dataframe = pd.merge(X, Y, left_index=True, right_index=True)

<commit_msg>prototype for model training<commit_after>from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals
import xgboost as xgb
import numpy as np
import pandas as pd
from os import listdir
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.metrics import f1_score

def aggregate_features(X):
    return np.mean(X, axis=0)

def get_features():
    biz_id_list = []
    result_array = np.zeros((2000, 1024))
    path = 'features/inception-21k-global/'
    feature_files = listdir(path)
    for i, f in enumerate(feature_files):
        biz_id = int(f[:-4])
        feature_bag = np.load('features/inception-21k-global/' + f)
        out_feature = aggregate_features(feature_bag)
        biz_id_list.append(biz_id)
        result_array[i, :] = out_feature
    col_names = ['incpt21k-glp-avg-' + str(i) for i in range(result_array.shape[1])]
    feature_frame = pd.DataFrame(data=result_array, index=biz_id_list, columns=col_names)
    return feature_frame

def get_response():
    biz2label = pd.read_csv("rawdata/train.csv")
    result_array = np.zeros((2000, 9))
    for class_no in range(9):
        response = [
            1 if str(class_no) in str(l).split(" ") else 0 for l in biz2label["labels"]]
        result_array[:, class_no] = response
    response_frame = pd.DataFrame(
        data=result_array,
        index=biz2label["business_id"],
        columns=['class' + str(i) for i in range(9)],
        dtype=int)
    return response_frame

def get_data():
    X = get_features()
    Y = get_response()
    dataframe = pd.merge(X, Y, left_index=True, right_index=True)
    return dataframe

dataframe = get_data().as_matrix()
X = dataframe[:, 0:1024]
y = dataframe[:, -1].astype(int)
print(np.sum(y == 1) / y.shape[0])
dtrain = xgb.DMatrix(X, label=y)

param = {'max_depth':1, 'eta':1, 'silent':1, 'objective':'binary:logitraw', 'lambda':1}
param['nthread'] = 4

# watchlist  = [(dtrain,'train'), (dtest,'eval')]
num_round = 2000

def evalerror(preds, dtrain):
    pred_labels = [1 if p > 0 else 0 for p in preds]
    labels = dtrain.get_label()
    return 'f1-score', f1_score(labels, pred_labels)

def fpreproc(dtrain, dtest, param):
    label = dtrain.get_label()
    ratio = float(np.sum(label == 0)) / np.sum(label==1)
    param['scale_pos_weight'] = ratio
    return (dtrain, dtest, param)

xgb.cv(param, dtrain, num_round, feval=evalerror, early_stopping_rounds=30, maximize=True, verbose_eval=True, fpreproc=fpreproc)

# sss = StratifiedShuffleSplit(dataframe[:, -3], 1, test_size=0.2)
# for train_index, test_index in sss:
#     X_train, X_test = X[train_index], X[test_index]
#     y_train, y_test = y[train_index], y[test_index]

# dtrain = xgb.DMatrix(X_train, label=y_train)
# dtest = xgb.DMatrix(X_test, label=y_test)
# param = {'max_depth':3, 'eta':0.1, 'silent':1, 'objective':'binary:logitraw', 'lambda':1}
# param['nthread'] = 4

# watchlist  = [(dtrain,'train'), (dtest,'eval')]
# num_round = 2000

# def evalerror(preds, dtrain):
#     pred_labels = [1 if p > 0 else 0 for p in preds]
#     labels = dtrain.get_label()
#     return 'f1-score', f1_score(labels, pred_labels)

# bst = xgb.train(param, dtrain, num_round, watchlist, feval=evalerror, early_stopping_rounds=30, maximize=True)
<|endoftext|>


<commit_before>from embeddingCalc import *
from rnn_reader import *
import sys

EMBEDDINGS = loadFromFile("savedEmbeddings/embeddings.pkl")
WORDS, WORD_TO_NUM = getAllWords()

# convert a character to ascii code (8 bits)
def prepareChar(ch):
    if len(ch) != 1:
        raise ValueError("Please supply a single character for the ascii lookup")
    binStr = "{0:08b}".format(ord(ch))
    binList = []
    for d in binStr:
        binList.append(int(d))

    return binList

# converts a word into an ascii matrix and ignores non alphabets
def prepareWord(word):
    asciiList = []
    for ch in word:
        if not ch.isalpha():
            continue
        asciiList.append(prepareChar(ch))

    batchSize = len(asciiList)
    return [batchSize, asciiList, toEmbedding(word)]

def trainModel(epochs):
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        # loadModel()
        for ep in range(epochs):
            loss_list = []
            for i in range(VOCAB_SIZE):
                data = prepareWord(WORDS[i])
                _current_state = np.zeros([NUM_LAYERS, 2, ASCII_VEC_DIM, STATE_SIZE])
                # print(_current_state.shape)
                n = 0
                embed_series = [data[2]]*TRUNC_BACKPROP_LENGTH
                while n < data[0]:
                    ascii_series = None
                    if n + TRUNC_BACKPROP_LENGTH < data[0]:
                        ascii_series = data[1][n:n+TRUNC_BACKPROP_LENGTH]
                    else:
                        batch_data = np.array(data[1][n:])
                        padding = np.zeros([n+TRUNC_BACKPROP_LENGTH- data[0], ASCII_VEC_DIM])
                        # print(batch_data.shape, padding.shape)
                        ascii_series = np.concatenate([batch_data, padding], axis=0)

                    _, _total_loss, _current_state, _embed_predict = sess.run(
                        [train_step, total_loss, current_state, embed_predict_series],
                        feed_dict={
                            ascii_placeholder: np.transpose(ascii_series),
                            embedding_placeholder: np.transpose(embed_series),
                            init_state: _current_state
                        }
                    )

                    n += TRUNC_BACKPROP_LENGTH

                embed_result = _embed_predict[-1]
                loss_list.append(_total_loss)
            print("Epoch %s - Avg.Loss: %.3f"%(ep, sum(loss_list)/VOCAB_SIZE))
            saveModel()
                # print(_embed_predict[-1][-1] - _embed_predict[-1][-2])

if __name__ == "__main__":
    print("training started...")
    trainModel(500)<commit_msg>now training and saving the models periodically<commit_after>from embeddingCalc import *
from rnn_reader import *
import sys

EMBEDDINGS = loadFromFile("savedEmbeddings/embeddings.pkl")
WORDS, WORD_TO_NUM = getAllWords()

# convert a character to ascii code (8 bits)
def prepareChar(ch):
    if len(ch) != 1:
        raise ValueError("Please supply a single character for the ascii lookup")
    binStr = "{0:08b}".format(ord(ch))
    binList = []
    for d in binStr:
        binList.append(int(d))

    return binList

# converts a word into an ascii matrix and ignores non alphabets
def prepareWord(word):
    asciiList = []
    for ch in word:
        if not ch.isalpha():
            continue
        asciiList.append(prepareChar(ch))

    batchSize = len(asciiList)
    return [batchSize, asciiList, toEmbedding(word)]

def trainModel(epochs):
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        # loadModel()
        for ep in range(epochs):
            loss_list = []
            for i in range(VOCAB_SIZE):
                data = prepareWord(WORDS[i])
                _current_state = np.zeros([NUM_LAYERS, 2, ASCII_VEC_DIM, STATE_SIZE])
                # print(_current_state.shape)
                n = 0
                embed_series = [data[2]]*TRUNC_BACKPROP_LENGTH
                while n < data[0]:
                    ascii_series = None
                    if n + TRUNC_BACKPROP_LENGTH < data[0]:
                        ascii_series = data[1][n:n+TRUNC_BACKPROP_LENGTH]
                    else:
                        batch_data = np.array(data[1][n:])
                        padding = np.zeros([n+TRUNC_BACKPROP_LENGTH- data[0], ASCII_VEC_DIM])
                        # print(batch_data.shape, padding.shape)
                        ascii_series = np.concatenate([batch_data, padding], axis=0)

                    _, _total_loss, _current_state, _embed_predict = sess.run(
                        [train_step, total_loss, current_state, embed_predict_series],
                        feed_dict={
                            ascii_placeholder: np.transpose(ascii_series),
                            embedding_placeholder: np.transpose(embed_series),
                            init_state: _current_state
                        }
                    )

                    n += TRUNC_BACKPROP_LENGTH

                embed_result = _embed_predict[-1]
                loss_list.append(_total_loss)
            print("Epoch %s - Avg.Loss: %.3f"%(ep, sum(loss_list)/VOCAB_SIZE))
            saveModel()
                # print(_embed_predict[-1][-1] - _embed_predict[-1][-2])

if __name__ == "__main__":
    print("training started...")
    trainModel(100)<|endoftext|>


<commit_before>from smashbox.utilities import *
from smashbox.utilities.hash_files import *
from smashbox.protocol import *

import smashbox.curl

"""
Observations:

eos attr set sys.tmp.etag= -- changes mtime (and DAV:getlastmodified) of the file, no effect on the directory, this is NOT a regression (same behaviour on eosuser)

NOTE: I need to handle ETAG quotes in my script (set_tmp_etags.sh)!

"""

FILTER_OUT_ATTRS=True
EOS_BUG_2732 = True
IGNORE_FOLDER_SIZE=True

@add_worker
def main(step):

    user=config.oc_account_name
    user_letter=user[0]
    server=config.oc_server
    extra_path=''

    assert(user!='.')
    assert(config.oc_server in ['eoshome','eosuser'])

    if config.oc_server == 'eoshome':
       server = 'eoshome-'+user_letter
       extra_path='/.ongoing'

    URL = 'http://'+server+':8000/cernbox/desktop/remote.php/webdav/home/'

    smashbox.curl.Client.extra_headers={'Remote-user':user, 'CBOX-CLIENT-MAPPING':'cernbox/desktop/remote.php/webdav/home', 'CBOX-SERVER-MAPPING':'eos/user%s/%s/%s'%(extra_path,user_letter,user),'X-Real-IP':'127.0.0.1'}

    propfind = ls_prop_desktop20
    #propfind = all_prop_android

    BASEURL=URL.replace('/cernbox/desktop/remote.php/webdav/home','')

    NSDAV="{DAV:}"
    NSOC="{http://owncloud.org/ns}"

    ids_files=[]
    ids_directories=[]
    etags_directories=[]

    fout = file('report.propfind.%s.%s.txt'%(user,config.oc_server),'w',0)

    propfind_list=[]

    def scan_dir(URL):
        r0=propfind(URL,depth=0).propfind_response
        r1=propfind(URL,depth=1).propfind_response

        # the parent dir (.) reported by Depth1 must be identical as the parent dir reported by Depth0
        r10=[x for x in r1 if x[0] == r0[0][0]]

        if r0 != r10 and not EOS_BUG_2732:
            print "ERROR: r0 != r10"
            print "URL",URL
            print "r0",r0
            print "r10",r10
            sys.exit(-1)

        path=r0[0][0]
        attrs=r0[0][1]['HTTP/1.1 200 OK']

        ids_directories.append(attrs[NSOC+'id'])
        etags_directories.append(attrs[NSDAV+'getetag'])

        if FILTER_OUT_ATTRS:
            attrs[NSOC+'id'] = 'DDD' # this attribute does not matter for comparison of directories
            attrs[NSDAV+'getlastmodified'] = 'DDD' # this attribute does not matter for comparison of directories
            attrs[NSDAV+'getetag'] = 'DDD' # this attribute does not matter for comparison of directories
            if IGNORE_FOLDER_SIZE:
                attrs[NSOC+'size'] = 'DDD'

        print >> fout, repr(path),repr(attrs)

        for x in sorted(r1):
            if x[0] == r0[0][0]:
                continue
            path=x[0]
            attrs=x[1]['HTTP/1.1 200 OK']

            if attrs['{DAV:}resourcetype'] == '{DAV:}collection':
                # recurse into the directory
                scan_dir(BASEURL+path)
            else:

                ids_files.append(attrs[NSOC+'id'])
                if FILTER_OUT_ATTRS:
                    attrs[NSOC+'id'] = 'FFF' # this attribute does not matter for comparison of files
                    if EOS_BUG_2732:
                        attrs[NSDAV+'getlastmodified'] = 'FFF' # buggy mtime should not be compared

                propfind_list.append((repr(path),repr(attrs)))

                print >> fout, repr(path),repr(attrs)

    scan_dir(URL)

    fout.close()

    fout1 = file('report.f-ids.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(ids_files):
        print >> fout1, x

    fout2 = file('report.d-ids.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(ids_directories):
        print >> fout2, x

    fout3 = file('report.d-etags.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(etags_directories):
        print >> fout3, x

<commit_msg>migration: do not ignore folder size, print URL in case of propfind failure<commit_after>from smashbox.utilities import *
from smashbox.utilities.hash_files import *
from smashbox.protocol import *

import smashbox.curl

"""
Observations:

eos attr set sys.tmp.etag= -- changes mtime (and DAV:getlastmodified) of the file, no effect on the directory, this is NOT a regression (same behaviour on eosuser)

NOTE: I need to handle ETAG quotes in my script (set_tmp_etags.sh)!

"""

FILTER_OUT_ATTRS=True
EOS_BUG_2732 = True
IGNORE_FOLDER_SIZE=False

@add_worker
def main(step):

    user=config.oc_account_name
    user_letter=user[0]
    server=config.oc_server
    extra_path=''

    assert(user!='.')
    assert(config.oc_server in ['eoshome','eosuser'])

    if config.oc_server == 'eoshome':
       server = 'eoshome-'+user_letter
       extra_path='/.ongoing'

    URL = 'http://'+server+':8000/cernbox/desktop/remote.php/webdav/home/'

    smashbox.curl.Client.extra_headers={'Remote-user':user, 'CBOX-CLIENT-MAPPING':'cernbox/desktop/remote.php/webdav/home', 'CBOX-SERVER-MAPPING':'eos/user%s/%s/%s'%(extra_path,user_letter,user),'X-Real-IP':'127.0.0.1'}

    propfind = ls_prop_desktop20
    #propfind = all_prop_android

    BASEURL=URL.replace('/cernbox/desktop/remote.php/webdav/home','')

    NSDAV="{DAV:}"
    NSOC="{http://owncloud.org/ns}"

    ids_files=[]
    ids_directories=[]
    etags_directories=[]

    fout = file('report.propfind.%s.%s.txt'%(user,config.oc_server),'w',0)

    propfind_list=[]

    def scan_dir(URL):
        try:
            r0=propfind(URL,depth=0).propfind_response
            r1=propfind(URL,depth=1).propfind_response
        except Exception,x:
            print >> sys.stderr, "ERROR: Failed to propfind",URL
            raise

        # the parent dir (.) reported by Depth1 must be identical as the parent dir reported by Depth0
        r10=[x for x in r1 if x[0] == r0[0][0]]

        if r0 != r10 and not EOS_BUG_2732:
            print "ERROR: r0 != r10"
            print "URL",URL
            print "r0",r0
            print "r10",r10
            sys.exit(-1)

        path=r0[0][0]
        attrs=r0[0][1]['HTTP/1.1 200 OK']

        ids_directories.append(attrs[NSOC+'id'])
        etags_directories.append(attrs[NSDAV+'getetag'])

        if FILTER_OUT_ATTRS:
            attrs[NSOC+'id'] = 'DDD' # this attribute does not matter for comparison of directories
            attrs[NSDAV+'getlastmodified'] = 'DDD' # this attribute does not matter for comparison of directories
            attrs[NSDAV+'getetag'] = 'DDD' # this attribute does not matter for comparison of directories
            if IGNORE_FOLDER_SIZE:
                attrs[NSOC+'size'] = 'DDD'

        print >> fout, repr(path),repr(attrs)

        for x in sorted(r1):
            if x[0] == r0[0][0]:
                continue
            path=x[0]
            attrs=x[1]['HTTP/1.1 200 OK']

            if attrs['{DAV:}resourcetype'] == '{DAV:}collection':
                # recurse into the directory
                scan_dir(BASEURL+path)
            else:

                ids_files.append(attrs[NSOC+'id'])
                if FILTER_OUT_ATTRS:
                    attrs[NSOC+'id'] = 'FFF' # this attribute does not matter for comparison of files
                    if EOS_BUG_2732:
                        attrs[NSDAV+'getlastmodified'] = 'FFF' # buggy mtime should not be compared

                propfind_list.append((repr(path),repr(attrs)))

                print >> fout, repr(path),repr(attrs)

    scan_dir(URL)

    fout.close()

    fout1 = file('report.f-ids.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(ids_files):
        print >> fout1, x

    fout2 = file('report.d-ids.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(ids_directories):
        print >> fout2, x

    fout3 = file('report.d-etags.%s.%s.txt'%(user,config.oc_server),'w',0)
    for x in sorted(etags_directories):
        print >> fout3, x

<|endoftext|>