<commit_before>#! /usr/bin/env python3
#
# 4chan thread live shitposter IRC bot

from __future__ import unicode_literals, absolute_import, print_function, division

import sys
import time

import re
import threading
import configparser
from io import StringIO
import textwrap
from urllib.request import urlopen
from bs4 import BeautifulSoup

import basc_py4chan
from basc_py4chan.util import clean_comment_body
import irc.client

cfg = configparser.ConfigParser()
cfg.read('config.cfg')

# configuration here
irc_server = cfg['IRC'].get('server', fallback='irc.rizon.net')
irc_port = cfg['IRC'].getint('port', fallback=6697)
irc_nick = cfg['IRC'].get('nick', fallback='pyemugenbot')
irc_channel = cfg['IRC'].get('channel', fallback='#emugentest')
admins = cfg['IRC'].get('admins', fallback='').split(',') # comma separated list of users who can restart and shutdown the bot

board_name = cfg['4chan'].get('board', fallback='vg')
general = cfg['4chan'].get('general', fallback='emugen|emulation') # matching pattern to find thread
archive = cfg['4chan'].get('archive', fallback='boards.fireden.net')
https = cfg['4chan'].getboolean('https', fallback=True)

board = basc_py4chan.Board(board_name, https)
thread = board.get_thread(0)
bumplimit_warning = True

def archive_url():
    return 'https://' + archive + '/' + board_name + '/thread/' + str(thread.topic.post_id)

def print_new_posts():
    update = thread.update()
    if (update > 0):
        new_posts = thread.posts[-update:]
        for post in new_posts:
            output = StringIO(newline='')
            #print("New post", post.post_id, "::")
            print('[{}] '.format(post.post_id),end="",file=output)
            if post.name != 'Anonymous':
                print('[name: ',end="",file=output)
                if post.name != 'None':
                    print('{}'.format(post.name),end="",file=output)
                if not post.tripcode is None:
                    print(post.tripcode,end="",file=output)
                print('] ',end="",file=output)
            if post.has_file:
                print('[img: {}]'.format(post.file_url),file=output)
            comment = post.text_comment
            lines = comment.split('\n')
            for line in lines:
                if not re.match(r'^\s*$', line):
                    print(line,file=output)
            print(output.getvalue(),end="")
            output.close()

def set_thread(board, thread_id):
    global thread
    thread = board.get_thread(thread_id)

def thread_alive(board, thread):
    #return (board.thread_exists(thread_id))
    return board.thread_exists(thread.id) and thread.archived == False

def find_current_thread(board, general):
    for thread in board.get_all_threads():
        if thread.topic.subject is not None:
            if re.search(general, thread.topic.subject, re.I):
                print('Found current thread:', thread.url)
                return thread.topic.post_id
    print("No thread up at the moment")
    return -1

def youtube_match(string):
    youtube = r'(youtu(?<=(v|V)/)|(?<=be/)|(?<=(\?|\&)v=)|(?<=embed/))([\w-]+)'
    return re.search(youtube, string)

def youtube_video_title_lookup(string, include_url=False):
    yt_match = youtube_match(string)
    if yt_match: #youtube handling
        splitline = string.split(' ')
        for i, word in enumerate(splitline):
            if youtube_match(word):
                video_id = yt_match.group(0)
                bs = BeautifulSoup(urlopen('https://www.youtube.com/watch?v=' + video_id), 'html.parser') # html.parser is 7% slower than lxml
                video_title = bs.title.string[0:-10]
                word= '[You\x0301,05Tube\x0f] \x0304' + video_title + '\x0f'
                if include_url:
                    word = word + ' [https://youtu.be/' + video_id + ']'
                splitline[i] = ('').join(word)
        string = (' ').join(splitline)
    return string

def chat_all_new_posts(c, target):
    update = thread.update()
    if (thread_alive(board, thread)):
        if (update > 0):
            new_posts = thread.posts[-update:]
            for post in new_posts:
                output = StringIO(newline='')
                print('[\x02\x0310{}\x0f] '.format(post.post_id),end="",file=output)
                if post.name != 'Anonymous':
                    print('[\x0314name:\x0f ',end="",file=output)
                    if not post.name is None:
                        print(post.name,end="",file=output)
                    if not post.tripcode is None:
                        print('\x0313{}\x0f'.format(post.tripcode),end="",file=output)
                    print('] ',end="",file=output)
                if post.has_file:
                    # File.filename_original attribute has been merged upstream but is not yet in release
                    # https://github.com/bibanon/BASC-py4chan/commit/205d001
                    print('[\x0314file:\x0f {} (\x0319{}\x0f)]'.format(post.file_url, post.file.filename_original),file=output)
                comment = post.comment
                if re.search('<s>', comment):
                    comment = comment.replace('<s>', '\x0301,01')
                    comment = comment.replace('</s>', '\x0f')
                    comment = clean_comment_body(comment)
                else:
                    comment = post.text_comment
                lines = comment.split('\n')
                for line in lines:
                    if not re.match(r'^\s*$', line): # no checking of blank lines
                        quote = r'>>((\d+)|>((((/\w+)*)*)/?))'
                        if re.search(quote, line):
                            splitline = line.split(' ')
                            for i, word in enumerate(splitline):
                                if re.match(quote, word): # quote handling
                                    word = '\x0304', word, '\x0f'
                                    splitline[i] = ('').join(word)
                            line = (' ').join(splitline)

                        if youtube_match(line):
                            line = youtube_video_title_lookup(line, True)
                        greentext = '^>[^>\n]+$'
                        if re.match(greentext, line):
                            print('\x0303{}\x0f'.format(line),file=output)
                        else:
                            print(line,file=output)

                print(output.getvalue(),end="")
                for line in output.getvalue().split('\n'):
                    for wrapped in textwrap.wrap(line, 425): # IRC messages must be under 512 total bytes
                        try:
                            c.privmsg(target, wrapped)
                        except:
                            print(sys.exc_info()[0])

                output.close()
            return True
        else:
            print('No new posts')
            global bumplimit_warning
            if thread.bumplimit:
                print('Bump limit reached ({} posts), looking for new thread'.format(len(thread.posts)))
                old_thread = thread.id
                set_thread(board, wait_for_new_thread())
                if thread.id != old_thread:
                    discovered = '[\x0308ATTENTION!\x0f] Discovered next thread: ' + thread.url
                    print(discovered)
                    c.privmsg(target, discovered)
                    bumplimit_warning = True
                    return True
                else:
                    if bumplimit_warning:
                        warning = '[\x0305WARNING!\x0f] Current thread has now reached the \x0307bump limit\x0f!'
                        c.privmsg(target, warning)
                        bumplimit_warning = False
            return False
    else:
        print('Thread is dead ' + str(thread.topic.post_id))
        c.privmsg(target, '[\x0305WARNING!\x0f] THREAD IS \x0305DEAD+\x0f! Archive URL: ' + archive_url())
        set_thread(board, wait_for_new_thread())
        discovered = '[\x0308ATTENTION!\x0f] Discovered new thread: ' + thread.url
        print(discovered)
        c.privmsg(target, discovered)
        return True

def wait_for_new_thread():
    new_id = find_current_thread(board, general)

    check_interval = 10
    while (new_id < 1):
        print('Waiting for thread - {}s refresh'.format(check_interval))
        time.sleep(check_interval)
        new_id = find_current_thread(board, general)
        if check_interval < 120:
            check_interval += 5

    return new_id

def feed_loop(c, target):
    print('Bot started up, looking for thread')
    set_thread(board, wait_for_new_thread())
    check_interval = 5
    while (1):
        time.sleep(check_interval)
        if (chat_all_new_posts(c, target)):
            check_interval = 5
        else:
            if (check_interval < 30):
                check_interval += 5
            print("Waiting {} seconds".format(check_interval))

def on_pubmsg(connection, event):
    yt_match = youtube_match(event.arguments[0])
    if re.search('^' + connection.get_nickname() + ':', event.arguments[0]):
        print('Detected my own nick mentioned')
        cmd = event.arguments[0].split(' ')[1]
        if (cmd == 'thread'):
            connection.privmsg(irc_channel, thread.url)
        elif (cmd == 'posts'):
            connection.privmsg(irc_channel, 'Thread is currently at ' + str(len(thread.posts)) + ' posts')
        elif (cmd == 'ppm' or cmd == 'speed'):
            time_since = int(time.time()) - thread.topic.timestamp
            ppm = len(thread.posts) / (time_since / 60)
            connection.privmsg(irc_channel, "{0:.2f} posts per minute".format(ppm))
        elif (cmd == 'commands'):
            commands = ['thread: returns URL to current thread',
                        'posts: returns post count of current thread',
                        'ppm|speed: returns the average posts per minute for this thread'
                       ]
            for msg in commands:
                connection.privmsg(event.source.nick, msg)
        elif (cmd == 'restart' or cmd == 'die'):
            if (event.source.nick in admins):
                connection.disconnect('As you wish my lord...')
                if (cmd == 'die'):
                    sys.exit()
                import os
                os.execv(__file__, sys.argv)
    elif yt_match:
        output = '↑↑ ' + youtube_video_title_lookup(event.arguments[0]) + ' ↑↑'
        connection.privmsg(irc_channel, output)

def on_disconnect(connection, event):
    time.sleep(3)
    connection.reconnect()

def main():
    reactor = irc.client.Reactor()
    try:
        c = reactor.server()
        c.buffer_class = irc.buffer.LenientDecodingLineBuffer
        c.connect(irc_server, irc_port, irc_nick)

        time.sleep(2)
        c.join(irc_channel)
        c.set_keepalive(60)

        c.add_global_handler('pubmsg', on_pubmsg)
        c.add_global_handler('disconnect', on_disconnect)

        t = threading.Thread(target=feed_loop, args=(c,irc_channel,))
        t.start()

        print('Bot runloop started')

    except irc.client.ServerConnectionError:
        print(sys.exc_info()[0])
        raise SystemExit(1)

    print('Main process moving into connection maintainance')
    reactor.process_forever()

if __name__ == '__main__':
    main()
<commit_msg>Cleanups<commit_after>#! /usr/bin/env python3
#
# 4chan thread live shitposter IRC bot

from __future__ import unicode_literals, absolute_import, print_function, division

import sys
import time

import re
import threading
import configparser
from io import StringIO
import textwrap
from urllib.request import urlopen
from bs4 import BeautifulSoup

import basc_py4chan
from basc_py4chan.util import clean_comment_body
import irc.client

cfg = configparser.ConfigParser()
cfg.read('config.cfg')

# configuration here
irc_server = cfg['IRC'].get('server', fallback='irc.rizon.net')
irc_port = cfg['IRC'].getint('port', fallback=6697)
irc_nick = cfg['IRC'].get('nick', fallback='pyemugenbot')
irc_channel = cfg['IRC'].get('channel', fallback='#emugentest')
admins = cfg['IRC'].get('admins', fallback='').split(',') # comma separated list of users who can restart and shutdown the bot

board_name = cfg['4chan'].get('board', fallback='vg')
general = cfg['4chan'].get('general', fallback='emugen|emulation') # matching pattern to find thread
archive = cfg['4chan'].get('archive', fallback='boards.fireden.net')
https = cfg['4chan'].getboolean('https', fallback=True)

board = basc_py4chan.Board(board_name, https)
thread = board.get_thread(0)
bumplimit_warning = True

def archive_url():
    return 'https://' + archive + '/' + board_name + '/thread/' + str(thread.topic.post_id)

def set_thread(board, thread_id):
    global thread
    thread = board.get_thread(thread_id)

def thread_alive(board, thread):
    #return (board.thread_exists(thread_id))
    return board.thread_exists(thread.id) and thread.archived == False

def find_current_thread(board, general):
    for thread in board.get_all_threads():
        if thread.topic.subject is not None:
            if re.search(general, thread.topic.subject, re.I):
                print('Found current thread:', thread.url)
                return thread.topic.post_id
    print("No thread up at the moment")
    return -1

def youtube_match(string):
    youtube = r'(youtu(?<=(v|V)/)|(?<=be/)|(?<=(\?|\&)v=)|(?<=embed/))([\w-]+)'
    return re.search(youtube, string)

def youtube_video_title_lookup(string, include_url=False):
    yt_match = youtube_match(string)
    if yt_match: #youtube handling
        splitline = string.split(' ')
        for i, word in enumerate(splitline):
            if youtube_match(word):
                video_id = yt_match.group(0)
                bs = BeautifulSoup(urlopen('https://www.youtube.com/watch?v=' + video_id), 'html.parser') # html.parser is 7% slower than lxml
                video_title = bs.title.string[0:-10]
                word= '[You\x0301,05Tube\x0f] \x0304' + video_title + '\x0f'
                if include_url:
                    word = word + ' [https://youtu.be/' + video_id + ']'
                splitline[i] = ('').join(word)
        string = (' ').join(splitline)
    return string

def chat_new_posts(c, target):
    update = thread.update()
    if (thread_alive(board, thread)):
        if (update > 0):
            new_posts = thread.posts[-update:]
            for post in new_posts:
                output = StringIO(newline='')
                print('[\x02\x0310{}\x0f] '.format(post.post_id),end="",file=output)
                if post.name != 'Anonymous':
                    print('[\x0314name:\x0f ',end="",file=output)
                    if not post.name is None:
                        print(post.name,end="",file=output)
                    if not post.tripcode is None:
                        print('\x0313{}\x0f'.format(post.tripcode),end="",file=output)
                    print('] ',end="",file=output)
                if post.has_file:
                    # File.filename_original attribute has been merged upstream but is not yet in release
                    # https://github.com/bibanon/BASC-py4chan/commit/205d001
                    print('[\x0314file:\x0f {} (\x0319{}\x0f)]'.format(post.file_url, post.file.filename_original),file=output)
                comment = post.comment
                if re.search('<s>', comment):
                    comment = comment.replace('<s>', '\x0301,01')
                    comment = comment.replace('</s>', '\x0f')
                    comment = clean_comment_body(comment)
                else:
                    comment = post.text_comment
                lines = comment.split('\n')
                for line in lines:
                    if not re.match(r'^\s*$', line): # no checking of blank lines
                        quote = r'>>((\d+)|>((((/\w+)*)*)/?))'
                        if re.search(quote, line):
                            splitline = line.split(' ')
                            for i, word in enumerate(splitline):
                                if re.match(quote, word): # quote handling
                                    word = '\x0304', word, '\x0f'
                                    splitline[i] = ('').join(word)
                            line = (' ').join(splitline)

                        if youtube_match(line):
                            line = youtube_video_title_lookup(line, True)
                        greentext = '^>[^>\n]+$'
                        if re.match(greentext, line):
                            print('\x0303{}\x0f'.format(line),file=output)
                        else:
                            print(line,file=output)

                print(output.getvalue(),end="")
                for line in output.getvalue().split('\n'):
                    for wrapped in textwrap.wrap(line, 425): # IRC messages must be under 512 total bytes
                        try:
                            c.privmsg(target, wrapped)
                        except:
                            print(sys.exc_info()[0])

                output.close()
            return True
        else:
            print('No new posts')
            global bumplimit_warning
            if thread.bumplimit:
                print('Bump limit reached ({} posts), looking for new thread'.format(len(thread.posts)))
                old_thread = thread.id
                set_thread(board, wait_for_new_thread())
                if thread.id != old_thread:
                    discovered = '[\x0308ATTENTION!\x0f] Discovered next thread: ' + thread.url
                    print(discovered)
                    c.privmsg(target, discovered)
                    bumplimit_warning = True
                    return True
                else:
                    if bumplimit_warning:
                        warning = '[\x0305WARNING!\x0f] Current thread has now reached the \x0307bump limit\x0f!'
                        c.privmsg(target, warning)
                        bumplimit_warning = False
            return False
    else:
        print('Thread is dead ' + str(thread.topic.post_id))
        c.privmsg(target, '[\x0305WARNING!\x0f] THREAD IS \x0305DEAD+\x0f! Archive URL: ' + archive_url())
        set_thread(board, wait_for_new_thread())
        discovered = '[\x0308ATTENTION!\x0f] Discovered new thread: ' + thread.url
        print(discovered)
        c.privmsg(target, discovered)
        return True

def wait_for_new_thread():
    new_id = find_current_thread(board, general)

    check_interval = 10
    while (new_id < 1):
        print('Waiting for thread - {}s refresh'.format(check_interval))
        time.sleep(check_interval)
        new_id = find_current_thread(board, general)
        if check_interval < 120:
            check_interval += 5

    return new_id

def feed_loop(c, target):
    print('Bot started up, looking for thread')
    set_thread(board, wait_for_new_thread())
    check_interval = 5
    while (1):
        time.sleep(check_interval)
        if (chat_new_posts(c, target)):
            check_interval = 5
        else:
            if (check_interval < 30):
                check_interval += 5
            print("Waiting {} seconds".format(check_interval))

def on_pubmsg(connection, event):
    yt_match = youtube_match(event.arguments[0])
    if re.search('^' + connection.get_nickname() + ':', event.arguments[0]):
        print('Detected my own nick mentioned')
        cmd = event.arguments[0].split(' ')[1]
        if (cmd == 'thread'):
            connection.privmsg(irc_channel, thread.url)
        elif (cmd == 'posts'):
            connection.privmsg(irc_channel, 'Thread is currently at ' + str(len(thread.posts)) + ' posts')
        elif (cmd == 'ppm' or cmd == 'speed'):
            time_since = int(time.time()) - thread.topic.timestamp
            ppm = len(thread.posts) / (time_since / 60)
            connection.privmsg(irc_channel, "{0:.2f} posts per minute".format(ppm))
        elif (cmd == 'commands'):
            commands = ['thread: returns URL to current thread',
                        'posts: returns post count of current thread',
                        'ppm|speed: returns the average posts per minute for this thread'
                       ]
            for msg in commands:
                connection.privmsg(event.source.nick, msg)
        elif (cmd == 'restart' or cmd == 'die'):
            if (event.source.nick in admins):
                connection.disconnect('As you wish my lord...')
                if (cmd == 'die'):
                    sys.exit()
                import os
                os.execv(__file__, sys.argv)
    elif yt_match:
        output = '↑↑ ' + youtube_video_title_lookup(event.arguments[0]) + ' ↑↑'
        connection.privmsg(irc_channel, output)

def on_disconnect(connection, event):
    time.sleep(3)
    connection.reconnect()

def main():
    reactor = irc.client.Reactor()
    try:
        c = reactor.server()
        c.buffer_class = irc.buffer.LenientDecodingLineBuffer
        c.connect(irc_server, irc_port, irc_nick)

        time.sleep(2)
        c.join(irc_channel)
        c.set_keepalive(60)

        c.add_global_handler('pubmsg', on_pubmsg)
        c.add_global_handler('disconnect', on_disconnect)

        t = threading.Thread(target=feed_loop, args=(c,irc_channel,))
        t.start()

        print('Bot runloop started')

    except irc.client.ServerConnectionError:
        print(sys.exc_info()[0])
        raise SystemExit(1)

    print('Main process moving into connection maintainance')
    reactor.process_forever()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.admin.views.main import ChangeList
from django import forms
from django.db.models import Count
from django.contrib.auth.models import Permission
from django.utils.translation import ugettext, ugettext_lazy as _
from django.template.response import TemplateResponse
from django.http import HttpResponse, HttpResponseRedirect

from guardian.models import UserObjectPermission, GroupObjectPermission
from suit.admin import SortableTabularInline, SortableModelAdmin

from tickle.models import Person, Event, Product, Holding, TicketType, Delivery, Purchase, SpecialNutrition, \
    TickleUser, StudentUnionDiscount, ProductDiscount, Discount, HoldingDiscount, PersonalDiscount, DiscountEligibility
from tickle.views.admin import AddProductToShoppingCartAdminView

class PurchaseInline(admin.StackedInline):
    model = Purchase
    extra = 0
    # filter_horizontal = ('holdings',)

@admin.register(Event)
class EventAdmin(admin.ModelAdmin):
    pass

class ProductDiscountInline(SortableTabularInline):
    model = ProductDiscount
    extra = 0
    sortable = 'order'

@admin.register(Product)
class ProductAdmin(SortableModelAdmin):
    list_display = ('name', '_public_name', 'price', 'published', 'quantitative', 'purchased_quantity', 'total_quantity')
    list_editable = ('published',)
    inlines = (ProductDiscountInline,)
    sortable = 'order'

    def purchased_quantity(self, obj):
        return obj.holdings.purchased().quantity()

    purchased_quantity.short_description = _('purchased quantity')

    def total_quantity(self, obj):
        return obj.holdings.quantity()

    total_quantity.short_description = _('total quantity')

class HoldingDiscountInline(SortableTabularInline):
    model = HoldingDiscount
    sortable = 'order'

class HoldingPurchasedListFilter(admin.SimpleListFilter):
    title = _('purchased')

    parameter_name = 'purchased'

    def lookups(self, request, model_admin):
        return (
            ('y', _('Yes')),
            ('n', _('No'))
        )

    def queryset(self, request, queryset):
        if self.value() == 'y':
            return queryset.filter(purchase__isnull=False)
        elif self.value() == 'n':
            return queryset.filter(purchase__isnull=True)
        return queryset

@admin.register(Holding)
class HoldingAdmin(admin.ModelAdmin):
    list_display = ('person', 'product',)
    list_filter = ('product', HoldingPurchasedListFilter)

    exclude = ('transferee',)

    raw_id_fields = ('person', 'purchase', 'shopping_cart',)
    inlines = (HoldingDiscountInline,)

    search_fields = ('person__first_name', 'person__last_name', 'person__email', 'person__liu_id', 'person__pid_code')
    actions = ('csv_export_action',)

    def csv_export_action(self, request, queryset):
        import unicodecsv

        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="holdings.csv"'

        writer = unicodecsv.writer(response, delimiter=b';')

        # We can't use lazy translation here.
        writer.writerow(
            (ugettext('PID'), ugettext('Name'), ugettext('Product')))
        for h in queryset.order_by('person__birth_date', 'person__pid_code'):
            writer.writerow([h.person.pid, h.person.full_name, h.product])

        return response

@admin.register(TicketType)
class TicketTypeAdmin(ProductAdmin):
    filter_horizontal = ('events',)

@admin.register(Delivery)
class DeliveryAdmin(admin.ModelAdmin):
    pass

class HoldingInline(admin.TabularInline):
    model = Holding
    raw_id_fields = ('person', 'product', 'purchase', 'shopping_cart')
    readonly_fields = ('shopping_cart', 'transferee')
    extra = 0

class PersonHoldingInline(HoldingInline):
    fk_name = 'person'

@admin.register(Purchase)
class PurchaseAdmin(admin.ModelAdmin):
    inlines = (HoldingInline,)
    list_display = ('person', 'purchased')
    date_hierarchy = 'purchased'

    search_fields = ('person__first_name', 'person__last_name', 'person__email', 'person__liu_id', 'person__pid_code')

@admin.register(SpecialNutrition)
class SpecialNutritionAdmin(admin.ModelAdmin):
    pass

class TickleUserAdminForm(forms.ModelForm):
    class Meta:
        model = TickleUser
        fields = ('person', 'is_active', 'is_staff')

@admin.register(TickleUser)
class TickleUserAdmin(UserAdmin):
    form = TickleUserAdminForm
    add_form = TickleUserAdminForm
    fieldsets = (
        (None, {'fields': ('person', )}),
        ('Permissions', {'fields': ('groups', 'user_permissions')}),
        ('Stuff', {'fields': ('is_active', 'is_staff', 'is_superuser')}),
    )
    add_fieldsets = (
        (None, {'fields': ('person', )}),
        ('Permissions', {'fields': ('groups', 'user_permissions')}),
        ('Stuff', {'fields': ('is_active', 'is_staff', 'is_superuser')}),
    )

    list_display = ('person', 'is_active', 'is_staff')
    list_filter = ('is_staff', 'is_superuser', 'is_active', 'groups')
    raw_id_fields = ('person',)
    ordering = None  # Parent class uses 'username'

class AlwaysChangedModelForm(forms.ModelForm):
    def has_changed(self):
        return True

class TickleUserInline(admin.StackedInline):
    model = TickleUser
    form = AlwaysChangedModelForm  # This way we can just press Add, not change anything and still get an account
    # created
    filter_horizontal = ('groups', 'user_permissions')
    fieldsets = (
        (None, {'fields': ('is_active', 'is_staff')}),
        (_('Permissions'), {'fields': ('groups', 'user_permissions', 'is_superuser')})
    )
    extra = 0
    max_num = 1

    exclude = ('password',)

class DiscountEligibilityInline(admin.TabularInline):
    model = DiscountEligibility
    extra = 0
    can_delete = 0
    readonly_fields = ('discount',)

    def has_add_permission(self, request):
        return False

class EventVisitorListFilter(admin.SimpleListFilter):
    title = _('event')

    parameter_name = 'event'

    def lookups(self, request, model_admin):
        return [(i.pk, i.name) for i in Event.objects.all()]

    def queryset(self, request, queryset):
        if self.value():
            event = Event.objects.get(pk=self.value())
            return queryset.filter(holdings__product__ticket_type__events=event)
        else:
            return queryset

class PersonChangeList(ChangeList):
    def get_results(self, *args, **kwargs):
        super(PersonChangeList, self).get_results(*args, **kwargs)

        self.special_nutrition_totals = self.queryset.order_by().values(
            'special_nutrition__name').annotate(Count('pk'))

@admin.register(Person)
class PersonAdmin(admin.ModelAdmin):
    inlines = (DiscountEligibilityInline, PurchaseInline, PersonHoldingInline, TickleUserInline,)

    list_display = ('first_name', 'last_name', 'pid', 'email', 'phone', 'liu_id', 'special_nutrition_render', 'notes')
    list_display_links = ('first_name', 'last_name', 'pid')
    list_filter = ('special_nutrition', EventVisitorListFilter)

    filter_horizontal = ('special_nutrition',)

    search_fields = ('first_name', 'last_name', 'email', 'liu_id', 'notes', 'pid_code')

    actions = ['generate_email_recipient_list_action',
               'fill_kobra_data_action',
               'create_user_action',
               'reset_password_action',
               'add_product_to_shopping_cart_action']

    list_max_show_all = 2000

    def get_urls(self):
        from django.conf.urls import url
        urls = super(PersonAdmin, self).get_urls()

        info = self.model._meta.app_label, self.model._meta.model_name

        custom_urls = [
            url(r'^add-product/$', AddProductToShoppingCartAdminView.as_view(), name='%s_%s_add_product' % info),
        ]

        return custom_urls + urls  # Add our own first so the pk filter won't catch them

    def get_changelist(self, request, **kwargs):
        return PersonChangeList

    def special_nutrition_render(self, obj):
        items = obj.special_nutrition.values_list('name', flat=True)
        if not items:
            return ''

        output = '
'
        for i in items:
            output += '
{0}
'.format(i)
        output += '
'
        return output

    special_nutrition_render.short_description = _('special nutrition')
    special_nutrition_render.allow_tags = True
    special_nutrition_render.admin_order_field = 'special_nutrition'

    def create_user_action(self, request, queryset):
        for person in queryset:
            person.get_or_create_user()
        self.message_user(request, _('Created user accounts.'))

    create_user_action.short_description = _('Create user accounts (if not existing)')

    def reset_password_action(self, request, queryset):
        for person in queryset:
            if hasattr(person, 'user') and not getattr(person, 'liu_id'):
                person.user.generate_and_send_password()
        self.message_user(request, _('Reset and sent passwords.'))

    reset_password_action.short_description = _('Reset passwords')

    def fill_kobra_data_action(self, request, queryset):
        queryset.fill_kobra_data()
        self.message_user(request, _('Fetched KOBRA data.'))

    fill_kobra_data_action.short_description = _('Fetch KOBRA data (use with caution)')

    def generate_email_recipient_list_action(self, request, queryset):
        context = {
            'recipients': queryset.pretty_emails_string()
        }
        return TemplateResponse(request, 'tickle/admin/person/email_recipient_list.html', context)

    generate_email_recipient_list_action.short_description = _('Generate email recipient list')

    def add_product_to_shopping_cart_action(self, request, queryset):
        selected = queryset.values_list('pk', flat=True)
        # selected = request.POST.getlist(admin.ACTION_CHECKBOX_NAME)
        return HttpResponseRedirect('add-product/?ids=%s' % ",".join(map(str, selected)))

    add_product_to_shopping_cart_action.short_description = _('Add product to shopping cart...')

@admin.register(Permission)
class PermissionAdmin(admin.ModelAdmin):
    list_filter = ('content_type',)

@admin.register(UserObjectPermission)
class UserObjectPermissionAdmin(admin.ModelAdmin):
    search_fields = (
        'user__person__first_name',
        'user__person__last_name',
        'user__person__email',
        'permission__name',
        'permission__codename'
    )
    list_filter = ('content_type',)

@admin.register(GroupObjectPermission)
class GroupObjectPermissionAdmin(admin.ModelAdmin):
    search_fields = (
        'group__name',
        'permission__name',
        'permission__codename'
    )
    list_filter = ('content_type',)

@admin.register(Discount)
class DiscountAdmin(admin.ModelAdmin):
    pass

@admin.register(StudentUnionDiscount)
class StudentUnionDiscountAdmin(admin.ModelAdmin):
    pass

@admin.register(PersonalDiscount)
class PersonalDiscountAdmin(admin.ModelAdmin):
    filter_horizontal = ('people',)
<commit_msg>Ugly-Isn solution for finding deliveries<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.admin.views.main import ChangeList
from django import forms
from django.db.models import Count
from django.contrib.auth.models import Permission
from django.utils.translation import ugettext, ugettext_lazy as _
from django.template.response import TemplateResponse
from django.http import HttpResponse, HttpResponseRedirect

from guardian.models import UserObjectPermission, GroupObjectPermission
from suit.admin import SortableTabularInline, SortableModelAdmin

from tickle.models import Person, Event, Product, Holding, TicketType, Delivery, Purchase, SpecialNutrition, \
    TickleUser, StudentUnionDiscount, ProductDiscount, Discount, HoldingDiscount, PersonalDiscount, DiscountEligibility
from tickle.views.admin import AddProductToShoppingCartAdminView

class PurchaseInline(admin.StackedInline):
    model = Purchase
    extra = 0
    # filter_horizontal = ('holdings',)

@admin.register(Event)
class EventAdmin(admin.ModelAdmin):
    pass

class ProductDiscountInline(SortableTabularInline):
    model = ProductDiscount
    extra = 0
    sortable = 'order'

@admin.register(Product)
class ProductAdmin(SortableModelAdmin):
    list_display = ('name', '_public_name', 'price', 'published', 'quantitative', 'purchased_quantity', 'total_quantity')
    list_editable = ('published',)
    inlines = (ProductDiscountInline,)
    sortable = 'order'

    def purchased_quantity(self, obj):
        return obj.holdings.purchased().quantity()

    purchased_quantity.short_description = _('purchased quantity')

    def total_quantity(self, obj):
        return obj.holdings.quantity()

    total_quantity.short_description = _('total quantity')

class HoldingDiscountInline(SortableTabularInline):
    model = HoldingDiscount
    sortable = 'order'

class HoldingPurchasedListFilter(admin.SimpleListFilter):
    title = _('purchased')

    parameter_name = 'purchased'

    def lookups(self, request, model_admin):
        return (
            ('y', _('Yes')),
            ('n', _('No'))
        )

    def queryset(self, request, queryset):
        if self.value() == 'y':
            return queryset.filter(purchase__isnull=False)
        elif self.value() == 'n':
            return queryset.filter(purchase__isnull=True)
        return queryset

@admin.register(Holding)
class HoldingAdmin(admin.ModelAdmin):
    list_display = ('person', 'product',)
    list_filter = ('product', HoldingPurchasedListFilter)

    exclude = ('transferee',)

    raw_id_fields = ('person', 'purchase', 'shopping_cart',)
    inlines = (HoldingDiscountInline,)

    search_fields = ('person__first_name', 'person__last_name', 'person__email', 'person__liu_id', 'person__pid_code')
    actions = ('csv_export_action',)

    def csv_export_action(self, request, queryset):
        import unicodecsv

        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename="holdings.csv"'

        writer = unicodecsv.writer(response, delimiter=b';')

        # We can't use lazy translation here.
        writer.writerow(
            (ugettext('PID'), ugettext('Name'), ugettext('Product')))
        for h in queryset.order_by('person__birth_date', 'person__pid_code'):
            writer.writerow([h.person.pid, h.person.full_name, h.product])

        return response

@admin.register(TicketType)
class TicketTypeAdmin(ProductAdmin):
    filter_horizontal = ('events',)

@admin.register(Delivery)
class DeliveryAdmin(admin.ModelAdmin):
    search_fields = ('holdings__person__first_name',
                     'holdings__person__last_name')

class HoldingInline(admin.TabularInline):
    model = Holding
    raw_id_fields = ('person', 'product', 'purchase', 'shopping_cart')
    readonly_fields = ('shopping_cart', 'transferee')
    extra = 0

class PersonHoldingInline(HoldingInline):
    fk_name = 'person'

@admin.register(Purchase)
class PurchaseAdmin(admin.ModelAdmin):
    inlines = (HoldingInline,)
    list_display = ('person', 'purchased')
    date_hierarchy = 'purchased'

    search_fields = ('person__first_name', 'person__last_name', 'person__email', 'person__liu_id', 'person__pid_code')

@admin.register(SpecialNutrition)
class SpecialNutritionAdmin(admin.ModelAdmin):
    pass

class TickleUserAdminForm(forms.ModelForm):
    class Meta:
        model = TickleUser
        fields = ('person', 'is_active', 'is_staff')

@admin.register(TickleUser)
class TickleUserAdmin(UserAdmin):
    form = TickleUserAdminForm
    add_form = TickleUserAdminForm
    fieldsets = (
        (None, {'fields': ('person', )}),
        ('Permissions', {'fields': ('groups', 'user_permissions')}),
        ('Stuff', {'fields': ('is_active', 'is_staff', 'is_superuser')}),
    )
    add_fieldsets = (
        (None, {'fields': ('person', )}),
        ('Permissions', {'fields': ('groups', 'user_permissions')}),
        ('Stuff', {'fields': ('is_active', 'is_staff', 'is_superuser')}),
    )

    list_display = ('person', 'is_active', 'is_staff')
    list_filter = ('is_staff', 'is_superuser', 'is_active', 'groups')
    raw_id_fields = ('person',)
    ordering = None  # Parent class uses 'username'

class AlwaysChangedModelForm(forms.ModelForm):
    def has_changed(self):
        return True

class TickleUserInline(admin.StackedInline):
    model = TickleUser
    form = AlwaysChangedModelForm  # This way we can just press Add, not change anything and still get an account
    # created
    filter_horizontal = ('groups', 'user_permissions')
    fieldsets = (
        (None, {'fields': ('is_active', 'is_staff')}),
        (_('Permissions'), {'fields': ('groups', 'user_permissions', 'is_superuser')})
    )
    extra = 0
    max_num = 1

    exclude = ('password',)

class DiscountEligibilityInline(admin.TabularInline):
    model = DiscountEligibility
    extra = 0
    can_delete = 0
    readonly_fields = ('discount',)

    def has_add_permission(self, request):
        return False

class EventVisitorListFilter(admin.SimpleListFilter):
    title = _('event')

    parameter_name = 'event'

    def lookups(self, request, model_admin):
        return [(i.pk, i.name) for i in Event.objects.all()]

    def queryset(self, request, queryset):
        if self.value():
            event = Event.objects.get(pk=self.value())
            return queryset.filter(holdings__product__ticket_type__events=event)
        else:
            return queryset

class PersonChangeList(ChangeList):
    def get_results(self, *args, **kwargs):
        super(PersonChangeList, self).get_results(*args, **kwargs)

        self.special_nutrition_totals = self.queryset.order_by().values(
            'special_nutrition__name').annotate(Count('pk'))

@admin.register(Person)
class PersonAdmin(admin.ModelAdmin):
    inlines = (DiscountEligibilityInline, PurchaseInline, PersonHoldingInline, TickleUserInline,)

    list_display = ('first_name', 'last_name', 'pid', 'email', 'phone', 'liu_id', 'special_nutrition_render', 'notes')
    list_display_links = ('first_name', 'last_name', 'pid')
    list_filter = ('special_nutrition', EventVisitorListFilter)

    filter_horizontal = ('special_nutrition',)

    search_fields = ('first_name', 'last_name', 'email', 'liu_id', 'notes', 'pid_code')

    actions = ['generate_email_recipient_list_action',
               'fill_kobra_data_action',
               'create_user_action',
               'reset_password_action',
               'add_product_to_shopping_cart_action']

    list_max_show_all = 2000

    def get_urls(self):
        from django.conf.urls import url
        urls = super(PersonAdmin, self).get_urls()

        info = self.model._meta.app_label, self.model._meta.model_name

        custom_urls = [
            url(r'^add-product/$', AddProductToShoppingCartAdminView.as_view(), name='%s_%s_add_product' % info),
        ]

        return custom_urls + urls  # Add our own first so the pk filter won't catch them

    def get_changelist(self, request, **kwargs):
        return PersonChangeList

    def special_nutrition_render(self, obj):
        items = obj.special_nutrition.values_list('name', flat=True)
        if not items:
            return ''

        output = '
'
        for i in items:
            output += '
{0}
'.format(i)
        output += '
'
        return output

    special_nutrition_render.short_description = _('special nutrition')
    special_nutrition_render.allow_tags = True
    special_nutrition_render.admin_order_field = 'special_nutrition'

    def create_user_action(self, request, queryset):
        for person in queryset:
            person.get_or_create_user()
        self.message_user(request, _('Created user accounts.'))

    create_user_action.short_description = _('Create user accounts (if not existing)')

    def reset_password_action(self, request, queryset):
        for person in queryset:
            if hasattr(person, 'user') and not getattr(person, 'liu_id'):
                person.user.generate_and_send_password()
        self.message_user(request, _('Reset and sent passwords.'))

    reset_password_action.short_description = _('Reset passwords')

    def fill_kobra_data_action(self, request, queryset):
        queryset.fill_kobra_data()
        self.message_user(request, _('Fetched KOBRA data.'))

    fill_kobra_data_action.short_description = _('Fetch KOBRA data (use with caution)')

    def generate_email_recipient_list_action(self, request, queryset):
        context = {
            'recipients': queryset.pretty_emails_string()
        }
        return TemplateResponse(request, 'tickle/admin/person/email_recipient_list.html', context)

    generate_email_recipient_list_action.short_description = _('Generate email recipient list')

    def add_product_to_shopping_cart_action(self, request, queryset):
        selected = queryset.values_list('pk', flat=True)
        # selected = request.POST.getlist(admin.ACTION_CHECKBOX_NAME)
        return HttpResponseRedirect('add-product/?ids=%s' % ",".join(map(str, selected)))

    add_product_to_shopping_cart_action.short_description = _('Add product to shopping cart...')

@admin.register(Permission)
class PermissionAdmin(admin.ModelAdmin):
    list_filter = ('content_type',)

@admin.register(UserObjectPermission)
class UserObjectPermissionAdmin(admin.ModelAdmin):
    search_fields = (
        'user__person__first_name',
        'user__person__last_name',
        'user__person__email',
        'permission__name',
        'permission__codename'
    )
    list_filter = ('content_type',)

@admin.register(GroupObjectPermission)
class GroupObjectPermissionAdmin(admin.ModelAdmin):
    search_fields = (
        'group__name',
        'permission__name',
        'permission__codename'
    )
    list_filter = ('content_type',)

@admin.register(Discount)
class DiscountAdmin(admin.ModelAdmin):
    pass

@admin.register(StudentUnionDiscount)
class StudentUnionDiscountAdmin(admin.ModelAdmin):
    pass

@admin.register(PersonalDiscount)
class PersonalDiscountAdmin(admin.ModelAdmin):
    filter_horizontal = ('people',)
<|endoftext|>


<commit_before>from datetime import timedelta
import datetime

import math
from operator import itemgetter

import re
import calendar
import pytz
from django.conf import settings
from django.utils import timezone
from django.utils.translation import ugettext

DATE_FORMAT = '%Y-%m-%d'
DATETIME_FORMAT = '%Y-%m-%d %H:%M'
FB_DATETIME_FORMAT  = '%Y-%m-%dT%H:%M:%S+0000'

def get_list_of_monday_thru_friday_dates(starting_date=None, include_weekend=False):
    """
        pass a week number and get a list of dates starting on monday of that week.
        if today is a saturday or sunday, return next week instead
    """
    current_week_number, current_weekday = datetime.datetime.now().isocalendar()[1:3]
    if starting_date is None:
        force = False
        starting_date = datetime.datetime.now()
    else:
        force = True
    selected_week_number = starting_date.isocalendar()[1]

    current_monday = (datetime.datetime.now()-datetime.timedelta(days=current_weekday-1))
    selected_monday = (starting_date-datetime.timedelta(days=starting_date.isocalendar()[2]-1))

    if not force and current_weekday>5 and to_date_object(current_monday)==to_date_object(selected_monday):
        # if today is already weekend, shift to next week
        # xxx too automagical perhaps...
        selected_monday += datetime.timedelta(days=7)
        selected_week_number+=1
    date_list = [to_date_object(selected_monday+datetime.timedelta(days=d)) for d in range(0,include_weekend and 7 or 5 )]
    return list(date_list)

def range_days(start_date, duration_days, epoch = False):
    time_start = timezone.datetime(start_date.year, start_date.month, start_date.day)
    time_stop = timedelta(hours=24*duration_days)+time_start

    if epoch:
        return {'time_start':to_epoch(time_start), 'time_stop': to_epoch(time_stop)}
    else:
        return {'time_start':time_start, 'time_stop': time_stop}

def total_seconds(time_delta):
    return (time_delta.days * 86400) + time_delta.seconds + (time_delta.microseconds*.000001)

def range_one_day(day = None, epoch = False):
    if day is None:
        day = timezone.now()
    return range_days(day, duration_days=1, epoch=epoch)

def extract_time_range(parameters):
    # time for
    time_start = remote_date_str_to_date(parameters.get('time_start','1970-01-01T00:00:00'))

    time_stop = timezone.now()
    if 'time_stop' in parameters:
        time_stop = remote_date_str_to_date(parameters.get('time_stop'))

    return {'time_start':time_start, 'time_stop':time_stop}

def utc_to_tz(dt_object, timezone):
    localized_dt = pytz.timezone("UTC").localize(dt_object)
    return localized_dt.astimezone(timezone)

def tz_fix_from_account_to_utc(dt_object, timezone):
    if to_epoch(dt_object) > 0:
        localized_dt = timezone.localize(dt_object)
        return localized_dt.astimezone(pytz.timezone("UTC"))
    else:
        return dt_object

def tz_fix_utc_epoch(utc_epoch, timezone):
    """Take a UTC epoch seconds value, and convert it into a date time string in the provided timezone"""
    return timezone.normalize(
                        pytz.utc.localize(
                                    from_epoch(utc_epoch)
                        )
            ).strftime(DATETIME_FORMAT)

def fb_request_timestamp_to_date(dt_object):
    """Convert a datetime object into a facebook stats request date object

    :param datetime.datetime dt_object: datetime object
    :return: dict
    """
    if dt_object.year == 1970:
        return None
    return {
        "month":dt_object.month,
        "day"  :dt_object.day,
        "year" :dt_object.year
    }

def get_month_list(start_date, end_date):
    """
    Return sorted year+month for all dates in the range.

    :param datetime.date start_date:
    :param datetime.date end_date:
    :return: a list of unique year,month tuples for all dates in the range.
    """
    year_months = set()
    if start_date==end_date:
        end_date+=datetime.timedelta(days=1)
    for d in daterange(start_date, end_date):
        year_months.add((d.year, d.month))
    return sorted(list(year_months), key=itemgetter(1))

def date_range(start_date, end_date, epoch=True):
    """
    see `daterange`
    """
    return daterange(start_date=start_date, end_date=end_date, epoch=epoch)

def daterange(start_date, end_date, epoch=False):
    """
    returns a generator that lists all dates starting with `start_date` up to and including `end_date`,
    if `start_date` is newer than `end_date`, the dates are returned in reverse order.

    The date most in the future is never included in the resulting list - the list is end exclusive.

    :param bool epoch: return in unix timestamp at midnight UTC
    :param datetime.date start_date: starting from...
    :param datetime.date end_date: up to and including...
    :return: generator that iterates dates
    """
    if end_date<start_date:
        days = (start_date-end_date).days
        multiplier = -1
        r = range(1,days+1)
    else:
        days = (end_date-start_date).days
        multiplier = 1
        r = range(0,days)

    for n in r:
        v = start_date + timedelta(days=n*multiplier)
        if epoch:
            v = to_epoch(get_midnight(v))
        yield v

def get_time_ranges(start_date, end_date, epoch=True):
    """
    Return {'time_start' : DATE, 'time_stop' : DATE}, {'time_start' : DATE+1, 'time_stop' : DATE+1}   ... etc

    :param datetime.date start_date:
    :param datetime.date end_date:
    :param bool epoch: dates become epoch seconds if true, otherwise date objects
    :return:
    """
    if end_date<start_date:
        days = (start_date-end_date).days
        multiplier = -1
    else:
        multiplier = 1
        days = (end_date-start_date).days

    ranges = []
    for add_days in range(0,days+1):
        ranges.append(
            range_days(start_date+timedelta(days=add_days*multiplier), 0, epoch=epoch)
        )
    return ranges

def to_epoch(dt, return_none=False):
    """Return the number of seconds since the epoch in UTC. Accepts strings in an the following datetime format (YYYY-MM-DD HH:MM) or a datetime object.

        :param datetime.datetime dt: Datetime object or string
        :param return_none:  if `dt` is invalid, return None if this is true, otherwise return 0
        :return: the epoch seconds as an `int`
    """
    if dt is None:
        if return_none:
            return None
        else:
            return 0

    if isinstance(dt, datetime.date):
        return int(calendar.timegm(dt.timetuple()))
    elif not isinstance(dt, datetime.datetime):
        try:
            dt = datetime.datetime.strptime(dt, DATETIME_FORMAT)
        except (TypeError, AttributeError, ValueError):
            return 0

    return int(calendar.timegm(dt.utctimetuple()))

def parse_fb_timestamp(timestamp):
    if re.search(r'^\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d\+0000$', timestamp):
        return timezone.datetime.strptime(timestamp, FB_DATETIME_FORMAT)
    else:
        raise ValueError("Invalid facebook timestamp [%s]" % timestamp)

def from_epoch(epo):
    """
    Because facebook insists on sending dates and times in a different format in the graph API, we are forced to
    parse this as well
    :param int epo: epoc seconds to turn into a `datetime.datetime` object
    :type epo: int
    :return: Datetime object
    """
    if epo is None:
        return timezone.make_aware(timezone.datetime(1970,1,1), timezone.utc)

    return timezone.make_aware(timezone.datetime.utcfromtimestamp(int(epo)), timezone.utc)

def chunks(l, chunk_size):
    """ Yield successive n-sized chunks from l.
    """
    for i in xrange(0, len(l), chunk_size):
        yield l[i:i+chunk_size]

def get_date_ranges_excluding_gaps(dates, max_days_per_range=30):
    """
    given a list of dates, the dates are returned as a list of ranges, with gaps being excluded and ranges being
    at most `max_days_per_range` long.

    date ranges are given in an exclusive way, meaning including the first date, and UP UNTIL but not including the last date
    TODO: make inclusive
    the date ranges are themselves arrays [[start_date, end_date], ....]
    """

    if len(dates)==0:
        return []

    previous_date = dates[0]
    current_range = [previous_date, previous_date]
    ranges = [current_range]

    for date in dates:

        date_diff = date - previous_date
        range_diff = date - current_range[0]

        if date_diff.days > 1 or range_diff.days >= max_days_per_range:

            current_range = [date, date + datetime.timedelta(days=1)]
            ranges.append(current_range)
        else:
            current_range[1] = date + datetime.timedelta(days=1)

        previous_date = date

    return ranges

def date_range_chunks(start_date, end_date, chunk_size, last_date_excluded=True):
    """
    Given a date range, split it into several chunks being at most
    `chunk_size` days long.

    """
    ranges = []
    date_range_list = list(daterange(start_date, end_date))
    range_list = chunks(date_range_list, chunk_size)
    for current_range in range_list:
        stop_ = current_range[-1]
        if last_date_excluded:
            stop_+=datetime.timedelta(days=1)
        ranges.append([current_range[0], stop_])

    return ranges

def get_gap_ranges_from_dates(dates, start_date, end_date, max_days_per_range=30):
    """
    given a list of dates, the gaps are returned as an array of date ranges.
    The ranges are never longer than `max_days_per_range`.

    date ranges are given in an exclusive way, meaning including the first date, and excluding the last date

    the date ranges are themselves arrays [[start_date, end_date], ....]
    """
    one_day = timedelta(days=1)
    start_date = to_date_object(start_date)
    end_date = to_date_object(end_date)
    corrected_start_date = start_date-one_day
    dates = [corrected_start_date, end_date] + list(dates)
    dates = sorted(set(dates))

    ranges = []

    dates = dates[dates.index(corrected_start_date):]
    previous_date = dates[0]
    for current_date in dates[1:]:

        date_diff = current_date - previous_date

        if date_diff.days > 1:
            current_range = [previous_date+one_day, current_date]
            split_ranges = date_range_chunks(current_range[0],
                                             current_range[1],
                                             chunk_size=max_days_per_range,
                                             last_date_excluded=True)
            ranges += split_ranges

        previous_date = current_date

    return ranges

def to_date_object(date_or_datetime_object):
    if isinstance(date_or_datetime_object, datetime.datetime):
        return date_or_datetime_object.date()
    elif isinstance(date_or_datetime_object, datetime.date):
        return date_or_datetime_object
    else:
        raise TypeError("Object passed is not a date or datetime.")

def get_midnight(dt_obj, add_days = 0):
    tz = timezone.get_current_timezone()
    midnight = timezone.datetime(dt_obj.year, dt_obj.month, dt_obj.day)+timedelta(days=add_days)
    return tz.localize(midnight)

def get_working_hours_in_month(year, month, until_date=None, work_hours_per_day=8):
    hours = 0
    for monthday, weekday in calendar.Calendar(0).itermonthdays2(year=year, month=month):
        if monthday==0 or weekday in (5,6):
            continue
        this_date = datetime.date(year=year, month=month, day=monthday)
        if until_date and this_date>=until_date:
            break
        hours += work_hours_per_day
    return hours

def convert_to_pst(date):
    pst = pytz.timezone("US/Pacific")
    return pst.normalize(date.astimezone(pst))

def midnight_pst(days=0):
    """
    Facebook likes midnights in PST, so we oblige.
    """
    import datetime
    pst = pytz.timezone("US/Pacific")
    pacific_time = convert_to_pst(pytz.utc.localize(datetime.datetime.utcnow()))
    n = pacific_time+timedelta(days=days)
    return pst.localize(datetime.datetime(n.year,n.month,n.day))

def midnight(days=0):
    n = timezone.datetime.utcnow()+timedelta(days=days)
    return timezone.datetime(n.year,n.month,n.day)

def remote_date_str_to_date(strdatetime):
    try:
        return timezone.datetime.strptime(strdatetime, DATETIME_FORMAT)
    except (TypeError, ValueError):
        try:
            return timezone.datetime.strptime(strdatetime, '%Y-%m-%dT%H:%M:%S')
        except (TypeError, ValueError):
            return timezone.datetime(1970,1,1)

def remote_stop_datetime_str_to_time(strdatetime):
    time = remote_datetime_str_to_time(strdatetime)

    if time == datetime.time(hour=0, minute=0):
        time = datetime.time(hour=23, minute=59)
    else:
        time = (timezone.datetime(year=1970, month=1, day=1, hour=time.hour, minute=time.minute) - timedelta(minutes=1)).time()

    return time

def remote_datetime_str_to_time(strdatetime):

    return timezone.datetime.strptime(strdatetime, DATETIME_FORMAT + ':%S').time()

def round_off(date_obj, round_to = 15):
    """
    round the given datetime object to the nearest whole minute.

    :param date_obj: A datetime object.
    :param round_to: Nearest number of minutes to round to. Default is 15.
    :return: The resulting datetime object.
    """
    date_obj += timedelta(minutes= round_to/2)
    date_obj -= timedelta(minutes=date_obj.minute % round_to,
                             seconds=date_obj.second,
                             microseconds=date_obj.microsecond)
    return date_obj

def pretty_duration(sec):
    is_negative = sec<0
    sec = math.fabs(sec)

    minutes, seconds = divmod(sec, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)

    days, hours, minutes, seconds = [int(d) for d in [days, hours, minutes, seconds]]
    return {
        'is_negative':is_negative,
        'days':days,
        'hours':hours,
        'minutes':minutes,
        'seconds':seconds
    }
<commit_msg>daterange method now allows you to skip certain dates in the given range<commit_after>from datetime import timedelta
import datetime

import math
from operator import itemgetter

import re
import calendar
import pytz
from django.conf import settings
from django.utils import timezone
from django.utils.translation import ugettext

DATE_FORMAT = '%Y-%m-%d'
DATETIME_FORMAT = '%Y-%m-%d %H:%M'
FB_DATETIME_FORMAT  = '%Y-%m-%dT%H:%M:%S+0000'

def get_list_of_monday_thru_friday_dates(starting_date=None, include_weekend=False):
    """
        pass a week number and get a list of dates starting on monday of that week.
        if today is a saturday or sunday, return next week instead
    """
    current_week_number, current_weekday = datetime.datetime.now().isocalendar()[1:3]
    if starting_date is None:
        force = False
        starting_date = datetime.datetime.now()
    else:
        force = True
    selected_week_number = starting_date.isocalendar()[1]

    current_monday = (datetime.datetime.now()-datetime.timedelta(days=current_weekday-1))
    selected_monday = (starting_date-datetime.timedelta(days=starting_date.isocalendar()[2]-1))

    if not force and current_weekday>5 and to_date_object(current_monday)==to_date_object(selected_monday):
        # if today is already weekend, shift to next week
        # xxx too automagical perhaps...
        selected_monday += datetime.timedelta(days=7)
        selected_week_number+=1
    date_list = [to_date_object(selected_monday+datetime.timedelta(days=d)) for d in range(0,include_weekend and 7 or 5 )]
    return list(date_list)

def range_days(start_date, duration_days, epoch = False):
    time_start = timezone.datetime(start_date.year, start_date.month, start_date.day)
    time_stop = timedelta(hours=24*duration_days)+time_start

    if epoch:
        return {'time_start':to_epoch(time_start), 'time_stop': to_epoch(time_stop)}
    else:
        return {'time_start':time_start, 'time_stop': time_stop}

def total_seconds(time_delta):
    return (time_delta.days * 86400) + time_delta.seconds + (time_delta.microseconds*.000001)

def range_one_day(day = None, epoch = False):
    if day is None:
        day = timezone.now()
    return range_days(day, duration_days=1, epoch=epoch)

def extract_time_range(parameters):
    # time for
    time_start = remote_date_str_to_date(parameters.get('time_start','1970-01-01T00:00:00'))

    time_stop = timezone.now()
    if 'time_stop' in parameters:
        time_stop = remote_date_str_to_date(parameters.get('time_stop'))

    return {'time_start':time_start, 'time_stop':time_stop}

def utc_to_tz(dt_object, timezone):
    localized_dt = pytz.timezone("UTC").localize(dt_object)
    return localized_dt.astimezone(timezone)

def tz_fix_from_account_to_utc(dt_object, timezone):
    if to_epoch(dt_object) > 0:
        localized_dt = timezone.localize(dt_object)
        return localized_dt.astimezone(pytz.timezone("UTC"))
    else:
        return dt_object

def tz_fix_utc_epoch(utc_epoch, timezone):
    """Take a UTC epoch seconds value, and convert it into a date time string in the provided timezone"""
    return timezone.normalize(
                        pytz.utc.localize(
                                    from_epoch(utc_epoch)
                        )
            ).strftime(DATETIME_FORMAT)

def fb_request_timestamp_to_date(dt_object):
    """Convert a datetime object into a facebook stats request date object

    :param datetime.datetime dt_object: datetime object
    :return: dict
    """
    if dt_object.year == 1970:
        return None
    return {
        "month":dt_object.month,
        "day"  :dt_object.day,
        "year" :dt_object.year
    }

def get_month_list(start_date, end_date):
    """
    Return sorted year+month for all dates in the range.

    :param datetime.date start_date:
    :param datetime.date end_date:
    :return: a list of unique year,month tuples for all dates in the range.
    """
    year_months = set()
    if start_date==end_date:
        end_date+=datetime.timedelta(days=1)
    for d in daterange(start_date, end_date):
        year_months.add((d.year, d.month))
    return sorted(list(year_months), key=itemgetter(1))

def date_range(start_date, end_date, epoch=True):
    """
    see `daterange`
    """
    return daterange(start_date=start_date, end_date=end_date, epoch=epoch)

def daterange(start_date, end_date, epoch=False, skip_func=None):
    """
    returns a generator that lists all dates starting with `start_date` up to and including `end_date`,
    if `start_date` is newer than `end_date`, the dates are returned in reverse order.

    The date most in the future is never included in the resulting list - the list is end exclusive.

    :param bool epoch: return in unix timestamp at midnight UTC
    :param datetime.date start_date: starting from...
    :param datetime.date end_date: up to and including...
    :param func skip_func: a function that takes a date in the range and returns True or False, if False, do not include the given date.
    :return: generator that iterates dates
    """
    if end_date<start_date:
        days = (start_date-end_date).days
        multiplier = -1
        r = range(1,days+1)
    else:
        days = (end_date-start_date).days
        multiplier = 1
        r = range(0,days)

    for n in r:
        v = start_date + timedelta(days=n*multiplier)
        skip = False
        if skip_func:
            skip = skip_func(v)

        if not skip:
            if epoch:
                v = to_epoch(get_midnight(v))
            yield v

def get_time_ranges(start_date, end_date, epoch=True):
    """
    Return {'time_start' : DATE, 'time_stop' : DATE}, {'time_start' : DATE+1, 'time_stop' : DATE+1}   ... etc

    :param datetime.date start_date:
    :param datetime.date end_date:
    :param bool epoch: dates become epoch seconds if true, otherwise date objects
    :return:
    """
    if end_date<start_date:
        days = (start_date-end_date).days
        multiplier = -1
    else:
        multiplier = 1
        days = (end_date-start_date).days

    ranges = []
    for add_days in range(0,days+1):
        ranges.append(
            range_days(start_date+timedelta(days=add_days*multiplier), 0, epoch=epoch)
        )
    return ranges

def to_epoch(dt, return_none=False):
    """Return the number of seconds since the epoch in UTC. Accepts strings in an the following datetime format (YYYY-MM-DD HH:MM) or a datetime object.

        :param datetime.datetime dt: Datetime object or string
        :param return_none:  if `dt` is invalid, return None if this is true, otherwise return 0
        :return: the epoch seconds as an `int`
    """
    if dt is None:
        if return_none:
            return None
        else:
            return 0

    if isinstance(dt, datetime.date):
        return int(calendar.timegm(dt.timetuple()))
    elif not isinstance(dt, datetime.datetime):
        try:
            dt = datetime.datetime.strptime(dt, DATETIME_FORMAT)
        except (TypeError, AttributeError, ValueError):
            return 0

    return int(calendar.timegm(dt.utctimetuple()))

def parse_fb_timestamp(timestamp):
    if re.search(r'^\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d\+0000$', timestamp):
        return timezone.datetime.strptime(timestamp, FB_DATETIME_FORMAT)
    else:
        raise ValueError("Invalid facebook timestamp [%s]" % timestamp)

def from_epoch(epo):
    """
    Because facebook insists on sending dates and times in a different format in the graph API, we are forced to
    parse this as well
    :param int epo: epoc seconds to turn into a `datetime.datetime` object
    :type epo: int
    :return: Datetime object
    """
    if epo is None:
        return timezone.make_aware(timezone.datetime(1970,1,1), timezone.utc)

    return timezone.make_aware(timezone.datetime.utcfromtimestamp(int(epo)), timezone.utc)

def chunks(l, chunk_size):
    """ Yield successive n-sized chunks from l.
    """
    for i in xrange(0, len(l), chunk_size):
        yield l[i:i+chunk_size]

def get_date_ranges_excluding_gaps(dates, max_days_per_range=30):
    """
    given a list of dates, the dates are returned as a list of ranges, with gaps being excluded and ranges being
    at most `max_days_per_range` long.

    date ranges are given in an exclusive way, meaning including the first date, and UP UNTIL but not including the last date
    TODO: make inclusive
    the date ranges are themselves arrays [[start_date, end_date], ....]
    """

    if len(dates)==0:
        return []

    previous_date = dates[0]
    current_range = [previous_date, previous_date]
    ranges = [current_range]

    for date in dates:

        date_diff = date - previous_date
        range_diff = date - current_range[0]

        if date_diff.days > 1 or range_diff.days >= max_days_per_range:

            current_range = [date, date + datetime.timedelta(days=1)]
            ranges.append(current_range)
        else:
            current_range[1] = date + datetime.timedelta(days=1)

        previous_date = date

    return ranges

def date_range_chunks(start_date, end_date, chunk_size, last_date_excluded=True):
    """
    Given a date range, split it into several chunks being at most
    `chunk_size` days long.

    """
    ranges = []
    date_range_list = list(daterange(start_date, end_date))
    range_list = chunks(date_range_list, chunk_size)
    for current_range in range_list:
        stop_ = current_range[-1]
        if last_date_excluded:
            stop_+=datetime.timedelta(days=1)
        ranges.append([current_range[0], stop_])

    return ranges

def get_gap_ranges_from_dates(dates, start_date, end_date, max_days_per_range=30):
    """
    given a list of dates, the gaps are returned as an array of date ranges.
    The ranges are never longer than `max_days_per_range`.

    date ranges are given in an exclusive way, meaning including the first date, and excluding the last date

    the date ranges are themselves arrays [[start_date, end_date], ....]
    """
    one_day = timedelta(days=1)
    start_date = to_date_object(start_date)
    end_date = to_date_object(end_date)
    corrected_start_date = start_date-one_day
    dates = [corrected_start_date, end_date] + list(dates)
    dates = sorted(set(dates))

    ranges = []

    dates = dates[dates.index(corrected_start_date):]
    previous_date = dates[0]
    for current_date in dates[1:]:

        date_diff = current_date - previous_date

        if date_diff.days > 1:
            current_range = [previous_date+one_day, current_date]
            split_ranges = date_range_chunks(current_range[0],
                                             current_range[1],
                                             chunk_size=max_days_per_range,
                                             last_date_excluded=True)
            ranges += split_ranges

        previous_date = current_date

    return ranges

def to_date_object(date_or_datetime_object):
    if isinstance(date_or_datetime_object, datetime.datetime):
        return date_or_datetime_object.date()
    elif isinstance(date_or_datetime_object, datetime.date):
        return date_or_datetime_object
    else:
        raise TypeError("Object passed is not a date or datetime.")

def get_midnight(dt_obj, add_days = 0):
    tz = timezone.get_current_timezone()
    midnight = timezone.datetime(dt_obj.year, dt_obj.month, dt_obj.day)+timedelta(days=add_days)
    return tz.localize(midnight)

def get_working_hours_in_month(year, month, until_date=None, work_hours_per_day=8):
    hours = 0
    for monthday, weekday in calendar.Calendar(0).itermonthdays2(year=year, month=month):
        if monthday==0 or weekday in (5,6):
            continue
        this_date = datetime.date(year=year, month=month, day=monthday)
        if until_date and this_date>=until_date:
            break
        hours += work_hours_per_day
    return hours

def convert_to_pst(date):
    pst = pytz.timezone("US/Pacific")
    return pst.normalize(date.astimezone(pst))

def midnight_pst(days=0):
    """
    Facebook likes midnights in PST, so we oblige.
    """
    import datetime
    pst = pytz.timezone("US/Pacific")
    pacific_time = convert_to_pst(pytz.utc.localize(datetime.datetime.utcnow()))
    n = pacific_time+timedelta(days=days)
    return pst.localize(datetime.datetime(n.year,n.month,n.day))

def midnight(days=0):
    n = timezone.datetime.utcnow()+timedelta(days=days)
    return timezone.datetime(n.year,n.month,n.day)

def remote_date_str_to_date(strdatetime):
    try:
        return timezone.datetime.strptime(strdatetime, DATETIME_FORMAT)
    except (TypeError, ValueError):
        try:
            return timezone.datetime.strptime(strdatetime, '%Y-%m-%dT%H:%M:%S')
        except (TypeError, ValueError):
            return timezone.datetime(1970,1,1)

def remote_stop_datetime_str_to_time(strdatetime):
    time = remote_datetime_str_to_time(strdatetime)

    if time == datetime.time(hour=0, minute=0):
        time = datetime.time(hour=23, minute=59)
    else:
        time = (timezone.datetime(year=1970, month=1, day=1, hour=time.hour, minute=time.minute) - timedelta(minutes=1)).time()

    return time

def remote_datetime_str_to_time(strdatetime):

    return timezone.datetime.strptime(strdatetime, DATETIME_FORMAT + ':%S').time()

def round_off(date_obj, round_to = 15):
    """
    round the given datetime object to the nearest whole minute.

    :param date_obj: A datetime object.
    :param round_to: Nearest number of minutes to round to. Default is 15.
    :return: The resulting datetime object.
    """
    date_obj += timedelta(minutes= round_to/2)
    date_obj -= timedelta(minutes=date_obj.minute % round_to,
                             seconds=date_obj.second,
                             microseconds=date_obj.microsecond)
    return date_obj

def pretty_duration(sec):
    is_negative = sec<0
    sec = math.fabs(sec)

    minutes, seconds = divmod(sec, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)

    days, hours, minutes, seconds = [int(d) for d in [days, hours, minutes, seconds]]
    return {
        'is_negative':is_negative,
        'days':days,
        'hours':hours,
        'minutes':minutes,
        'seconds':seconds
    }
<|endoftext|>


<commit_before># -*- coding: utf8 - *-
"""
    tmuxp.config
    ~~~~~~~~~~~~

    tmuxp helps you manage tmux workspaces.

    :copyright: Copyright 2013 Tony Narlock.
    :license: BSD, see LICENSE for details
"""

def expand_config(config):
    '''Expand configuration into full form. Enables shorthand forms for tmuxp
    config.

    This is necessary to keep the code in the :class:`Builder` clean and also
    allow for neat, short-hand configurations.

    As a simple example, internally, tmuxp expects that config options
    like ``shell_command`` are a list (array)::

        'shell_command': ['htop']

    tmuxp configs allow for it to be simply a string::

        'shell_command': 'htop'

    Kaptan will load JSON/YAML/INI files into python dicts for you.
    :param config: the configuration for the session
    :type config: dict

    iterate through session, windows, and panes for ``shell_command``, if
    it is a string, turn to list.

    shell_command: 'string' => shell_command: list('string')

    :param config: the session configuration
    :type config: dict
    '''

    def _expand(c):
        '''any config section, session, window, pane that can
        contain the 'shell_command' value
        '''
        if ('shell_command' in c and
                isinstance(c['shell_command'], basestring)):
                c['shell_command'] = [c['shell_command']]

        return c

    def _expand_shell_command_before(c):
        '''any config section, session, window, pane that can
        contain the 'shell_command_before' value
        '''
        if ('shell_command_before' in c and
                isinstance(c['shell_command_before'], basestring)):
                c['shell_command_before'] = [c['shell_command_before']]

        return c

    config = _expand(config)
    for window in config['windows']:
        window = _expand(window)
        window['panes'] = [_expand(pane) for pane in window['panes']]
        window = _expand_shell_command_before(window)
        window['panes'] = [_expand_shell_command_before(pane) for pane in window['panes']]

    return config

def trickledown_config(config):
    '''Trickle down / inherit config values

    This will only work if config has been expand with ConfigExpand()

    tmuxp allows certain commands to be default at the session, window
    level. shell_command_before trickles down and prepends the
    ``shell_command`` for the pane.

    :param config: the session configuration
    :type config: dict
    '''

    '''
    prepends a pane's ``shell_command`` list with the window and sessions'
    ``shell_command_before``.
    '''

    if 'shell_command_before' in config:
        assert isinstance(config['shell_command_before'], list)
        session_shell_command_before = config['shell_command_before']
    else:
        session_shell_command_before = []

    for windowconfitem in config['windows']:
        window_shell_command_before = []
        if 'shell_command_before' in windowconfitem:
            window_shell_command_before = windowconfitem['shell_command_before']

        for paneconfitem in windowconfitem['panes']:
            pane_shell_command_before = []
            if 'shell_command_before' in paneconfitem:
                pane_shell_command_before += paneconfitem['shell_command_before']

            if 'shell_command' not in paneconfitem:
                paneconfitem['shell_command'] = list()

            paneconfitem['shell_command'] = session_shell_command_before + window_shell_command_before + pane_shell_command_before + paneconfitem['shell_command']

    return config
<commit_msg>Some mccabe<commit_after># -*- coding: utf8 - *-
"""
    tmuxp.config
    ~~~~~~~~~~~~

    tmuxp helps you manage tmux workspaces.

    :copyright: Copyright 2013 Tony Narlock.
    :license: BSD, see LICENSE for details
"""

def _expand_shell_command(c):
    '''any config section, session, window, pane that can
    contain the 'shell_command' value
    '''
    if ('shell_command' in c and
            isinstance(c['shell_command'], basestring)):
            c['shell_command'] = [c['shell_command']]

    if ('shell_command_before' in c and
            isinstance(c['shell_command_before'], basestring)):
            c['shell_command_before'] = [c['shell_command_before']]

    return c

def expand_config(config):
    '''Expand configuration into full form. Enables shorthand forms for tmuxp
    config.

    This is necessary to keep the code in the :class:`Builder` clean and also
    allow for neat, short-hand configurations.

    As a simple example, internally, tmuxp expects that config options
    like ``shell_command`` are a list (array)::

        'shell_command': ['htop']

    tmuxp configs allow for it to be simply a string::

        'shell_command': 'htop'

    Kaptan will load JSON/YAML/INI files into python dicts for you.
    :param config: the configuration for the session
    :type config: dict

    iterate through session, windows, and panes for ``shell_command``, if
    it is a string, turn to list.

    shell_command: 'string' => shell_command: list('string')

    :param config: the session configuration
    :type config: dict
    '''

    config = _expand_shell_command(config)
    for window in config['windows']:
        window = _expand_shell_command(window)
        window['panes'] = [_expand_shell_command(
            pane) for pane in window['panes']]

    return config

def trickledown_config(config):
    '''Trickle down / inherit config values

    This will only work if config has been expand with ConfigExpand()

    tmuxp allows certain commands to be default at the session, window
    level. shell_command_before trickles down and prepends the
    ``shell_command`` for the pane.

    :param config: the session configuration
    :type config: dict
    '''

    '''
    prepends a pane's ``shell_command`` list with the window and sessions'
    ``shell_command_before``.
    '''

    if 'shell_command_before' in config:
        assert isinstance(config['shell_command_before'], list)
        session_shell_command_before = config['shell_command_before']
    else:
        session_shell_command_before = []

    for windowconfig in config['windows']:

        if 'shell_command_before' in windowconfig:
            window_shell_command_before = windowconfig['shell_command_before']
        else:
            window_shell_command_before = []

        for paneconfig in windowconfig['panes']:

            if 'shell_command_before' in paneconfig:
                pane_shell_command_before += paneconfig['shell_command_before']
            else:
                pane_shell_command_before = []

            if 'shell_command' not in paneconfig:
                paneconfig['shell_command'] = list()

            paneconfig['shell_command'] = session_shell_command_before + \
                window_shell_command_before + \
                pane_shell_command_before + paneconfig['shell_command']

    return config
<|endoftext|>


<commit_before># Do not edit, pipeline versioning governed by git tags!
__version__ = '1.4.2b4-61-g1a5ca76a'
<commit_msg>[Issue356] updated version<commit_after># Do not edit, pipeline versioning governed by git tags!
__version__ = '1.4.2b4-62-g03b6311a'
<|endoftext|>


<commit_before># Do not edit, pipeline versioning governed by git tags!
__version__ = '1.4.2b12'
<commit_msg>new version<commit_after># Do not edit, pipeline versioning governed by git tags!
__version__ = '1.4.2b13'
<|endoftext|>


<commit_before>#
# gdb commands for debugging/inspecting beng-proxy.
#
# Author: Max Kellermann <mk@cm4all.com>
#

import gdb

def for_each_hashmap(h):
    if h.type.code != gdb.lookup_type('struct hashmap').code:
        print "not a hashmap"
        return

    capacity = h['capacity']
    slots = h['slots']
    for i in range(capacity):
        slot = slots[i]
        if not slot['pair']['key']:
            continue

        while slot:
            pair = slot['pair']
            yield i, pair['key'].string(), pair['value']
            slot = slot['next']

class DumpHashmapSlot(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap_slot", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        arg_list = gdb.string_to_argv(arg)
        if len(arg_list) != 2:
            print "usage: bp_dump_hashmap ptr i"
            return

        h = gdb.parse_and_eval(arg_list[0])
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        i = int(arg_list[1])
        slot = h['slots'][i]

        if not slot['pair']['key']:
            print "empty"
            return

        while slot:
            print slot['pair']['key'].string()
            slot = slot['next']

class DumpHashmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        n_slots = 0
        n_total = 0
        biggest_slot = 0
        i_biggest_slot = -1

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if slot['pair']['key']:
                s = slot['next']
                n = 1
                while s:
                    n += 1
                    s = s['next']
                if n > biggest_slot:
                    biggest_slot = n
                    i_biggest_slot = i
                if n > 1000:
                    print "big", i, n
                n_total += n
                n_slots += 1
                #print n, slot['pair']['key']
                if n_slots % 256 == 0:
                    print n_slots, n_total, biggest_slot, i_biggest_slot
        print n_slots, n_total, biggest_slot, i_biggest_slot

class DumpHashmap2(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap2", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        for i, key, value in for_each_hashmap(h.dereference()):
            print i, key, value

class DumpStrmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_strmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct strmap').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        string_type = gdb.lookup_type('char').pointer()
        for i, key, value in for_each_hashmap(h['hashmap'].dereference()):
            print key, '=', value.cast(string_type).string()

def pool_size(pool):
    return int(pool['size'])

def pool_sizes(pool):
    netto_size = pool_size(pool)
    if pool['type'] == 1:
        # linear pool
        brutto_size = 0
        area = pool['current_area']['linear']
        while area:
            brutto_size += int(area['size'])
            area = area['prev']
    else:
        brutto_size = netto_size
    return brutto_size, netto_size

def for_each_list_head(head):
    item = head['next']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['next']

def for_each_list_head_reverse(head):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['prev']

def for_each_list_item_reverse(head, cast):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item.cast(cast)
        item = item['prev']

def pool_recursive_sizes(pool):
    pool_pointer = gdb.lookup_type('struct pool').pointer()

    brutto_size, netto_size = pool_sizes(pool)
    for child in for_each_list_head(pool['children']):
        child_brutto_size, child_netto_size = pool_recursive_sizes(child.cast(pool_pointer))
        brutto_size += child_brutto_size
        netto_size += child_netto_size

    return brutto_size, netto_size

class DumpPoolStats(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_stats", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        print "pool '%s' type=%d" % (pool['name'].string(), pool['type'])
        print "size", pool_sizes(pool)
        print "recursive_size", pool_recursive_sizes(pool)

class DumpPoolRefs(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_refs", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def _dump_refs(self, pool, label, head):
        print "pool '%s' %s:" % (pool['name'].string(), label)

        ref_pointer = gdb.lookup_type('struct pool_ref').pointer()
        for r in for_each_list_item_reverse(head, ref_pointer):
            print '%4u %s:%u' % (r['count'], r['file'].string().replace('../', ''), r['line'])

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        for i in ('refs', 'unrefs'):
            self._dump_refs(pool, i, pool[i])

class DumpPoolAllocations(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_allocations", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        allocation_pointer = gdb.lookup_type('struct allocation_info').pointer()
        for a in for_each_list_item_reverse(pool['allocations'], allocation_pointer):
            print '%8u %s:%u' % (a['size'], a['file'].string().replace('../', ''), a['line'])

class FindChild(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_find_child", gdb.COMMAND_DATA, gdb.COMPLETE_NONE, True)

    def _find_child_by_pid(self, pid):
        lh = gdb.parse_and_eval('children')
        if lh.type.code != gdb.lookup_type('struct list_head').code:
            print "not a list_head"
            return None

        child_pointer = gdb.lookup_type('struct child').pointer()
        for li in for_each_list_head(lh):
            child = li.cast(child_pointer)
            if child['pid'] == pid:
                return child

        return None

    def invoke(self, arg, from_tty):
        pid = gdb.parse_and_eval(arg)
        child = self._find_child_by_pid(pid)
        if child is None:
            print "Not found"
        else:
            print child, child.dereference()

class FindChildStockClient(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_find_child_stock_client", gdb.COMMAND_DATA, gdb.COMPLETE_NONE, True)

    def _find_child_by_pid(self, pid):
        lh = gdb.parse_and_eval('children')
        if lh.type.code != gdb.lookup_type('struct list_head').code:
            print "not a list_head"
            return None

        child_pointer = gdb.lookup_type('struct child').pointer()
        for li in for_each_list_head(lh):
            child = li.cast(child_pointer)
            if child['pid'] == pid:
                return child

        return None

    def invoke(self, arg, from_tty):
        pid = gdb.parse_and_eval(arg)
        child = self._find_child_by_pid(pid)
        if child is None:
            print "Not found"
            return

        stock_type = gdb.lookup_type('struct stock').pointer()
        child_stock_item_type = gdb.lookup_type('struct child_stock_item').pointer()
        child_stock_item = child['callback_ctx'].cast(child_stock_item_type)

        string_type = gdb.lookup_type('char').pointer()
        fcgi_connection_type = gdb.lookup_type('struct fcgi_connection').pointer()

        fcgi_stock = gdb.parse_and_eval('global_fcgi_stock')
        h = fcgi_stock['hstock']['stocks']

        for i, key, value in for_each_hashmap(h.dereference()):
            stock = value.cast(stock_type)

            for x in ('idle', 'busy'):
                for lh in for_each_list_head(stock[x]):
                    c = lh.cast(fcgi_connection_type)

                    if c['child'] == child_stock_item:
                        print key, c, c.dereference()

DumpHashmapSlot()
DumpHashmap()
DumpHashmap2()
DumpStrmap()
DumpPoolStats()
DumpPoolRefs()
DumpPoolAllocations()
FindChild()
FindChildStockClient()
<commit_msg>gdb: add command "bp_find_pool"<commit_after>#
# gdb commands for debugging/inspecting beng-proxy.
#
# Author: Max Kellermann <mk@cm4all.com>
#

import gdb

def for_each_hashmap(h):
    if h.type.code != gdb.lookup_type('struct hashmap').code:
        print "not a hashmap"
        return

    capacity = h['capacity']
    slots = h['slots']
    for i in range(capacity):
        slot = slots[i]
        if not slot['pair']['key']:
            continue

        while slot:
            pair = slot['pair']
            yield i, pair['key'].string(), pair['value']
            slot = slot['next']

class DumpHashmapSlot(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap_slot", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        arg_list = gdb.string_to_argv(arg)
        if len(arg_list) != 2:
            print "usage: bp_dump_hashmap ptr i"
            return

        h = gdb.parse_and_eval(arg_list[0])
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        i = int(arg_list[1])
        slot = h['slots'][i]

        if not slot['pair']['key']:
            print "empty"
            return

        while slot:
            print slot['pair']['key'].string()
            slot = slot['next']

class DumpHashmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        n_slots = 0
        n_total = 0
        biggest_slot = 0
        i_biggest_slot = -1

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if slot['pair']['key']:
                s = slot['next']
                n = 1
                while s:
                    n += 1
                    s = s['next']
                if n > biggest_slot:
                    biggest_slot = n
                    i_biggest_slot = i
                if n > 1000:
                    print "big", i, n
                n_total += n
                n_slots += 1
                #print n, slot['pair']['key']
                if n_slots % 256 == 0:
                    print n_slots, n_total, biggest_slot, i_biggest_slot
        print n_slots, n_total, biggest_slot, i_biggest_slot

class DumpHashmap2(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap2", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        for i, key, value in for_each_hashmap(h.dereference()):
            print i, key, value

class DumpStrmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_strmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct strmap').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        string_type = gdb.lookup_type('char').pointer()
        for i, key, value in for_each_hashmap(h['hashmap'].dereference()):
            print key, '=', value.cast(string_type).string()

def pool_size(pool):
    return int(pool['size'])

def pool_sizes(pool):
    netto_size = pool_size(pool)
    if pool['type'] == 1:
        # linear pool
        brutto_size = 0
        area = pool['current_area']['linear']
        while area:
            brutto_size += int(area['size'])
            area = area['prev']
    else:
        brutto_size = netto_size
    return brutto_size, netto_size

def for_each_list_head(head):
    item = head['next']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['next']

def for_each_list_head_reverse(head):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['prev']

def for_each_list_item_reverse(head, cast):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item.cast(cast)
        item = item['prev']

def for_each_recursive_pool(pool):
    yield pool

    pool_pointer = gdb.lookup_type('struct pool').pointer()

    for child in for_each_list_head(pool['children']):
        child = child.cast(pool_pointer)
        for x in for_each_recursive_pool(child):
            yield x

def pool_recursive_sizes(pool):
    pool_pointer = gdb.lookup_type('struct pool').pointer()

    brutto_size, netto_size = pool_sizes(pool)
    for child in for_each_list_head(pool['children']):
        child_brutto_size, child_netto_size = pool_recursive_sizes(child.cast(pool_pointer))
        brutto_size += child_brutto_size
        netto_size += child_netto_size

    return brutto_size, netto_size

class DumpPoolStats(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_stats", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        print "pool '%s' type=%d" % (pool['name'].string(), pool['type'])
        print "size", pool_sizes(pool)
        print "recursive_size", pool_recursive_sizes(pool)

class DumpPoolRefs(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_refs", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def _dump_refs(self, pool, label, head):
        print "pool '%s' %s:" % (pool['name'].string(), label)

        ref_pointer = gdb.lookup_type('struct pool_ref').pointer()
        for r in for_each_list_item_reverse(head, ref_pointer):
            print '%4u %s:%u' % (r['count'], r['file'].string().replace('../', ''), r['line'])

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        for i in ('refs', 'unrefs'):
            self._dump_refs(pool, i, pool[i])

class DumpPoolAllocations(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_allocations", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        allocation_pointer = gdb.lookup_type('struct allocation_info').pointer()
        for a in for_each_list_item_reverse(pool['allocations'], allocation_pointer):
            print '%8u %s:%u' % (a['size'], a['file'].string().replace('../', ''), a['line'])

class FindPool(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_find_pool", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        arg_list = gdb.string_to_argv(arg)
        if len(arg_list) != 2:
            print "usage: bp_find_pool pool name"
            return

        pool = gdb.parse_and_eval(arg_list[0])
        name = arg_list[1]

        for x in for_each_recursive_pool(pool):
            if x['name'].string() == name:
                print x, x.dereference()

class FindChild(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_find_child", gdb.COMMAND_DATA, gdb.COMPLETE_NONE, True)

    def _find_child_by_pid(self, pid):
        lh = gdb.parse_and_eval('children')
        if lh.type.code != gdb.lookup_type('struct list_head').code:
            print "not a list_head"
            return None

        child_pointer = gdb.lookup_type('struct child').pointer()
        for li in for_each_list_head(lh):
            child = li.cast(child_pointer)
            if child['pid'] == pid:
                return child

        return None

    def invoke(self, arg, from_tty):
        pid = gdb.parse_and_eval(arg)
        child = self._find_child_by_pid(pid)
        if child is None:
            print "Not found"
        else:
            print child, child.dereference()

class FindChildStockClient(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_find_child_stock_client", gdb.COMMAND_DATA, gdb.COMPLETE_NONE, True)

    def _find_child_by_pid(self, pid):
        lh = gdb.parse_and_eval('children')
        if lh.type.code != gdb.lookup_type('struct list_head').code:
            print "not a list_head"
            return None

        child_pointer = gdb.lookup_type('struct child').pointer()
        for li in for_each_list_head(lh):
            child = li.cast(child_pointer)
            if child['pid'] == pid:
                return child

        return None

    def invoke(self, arg, from_tty):
        pid = gdb.parse_and_eval(arg)
        child = self._find_child_by_pid(pid)
        if child is None:
            print "Not found"
            return

        stock_type = gdb.lookup_type('struct stock').pointer()
        child_stock_item_type = gdb.lookup_type('struct child_stock_item').pointer()
        child_stock_item = child['callback_ctx'].cast(child_stock_item_type)

        string_type = gdb.lookup_type('char').pointer()
        fcgi_connection_type = gdb.lookup_type('struct fcgi_connection').pointer()

        fcgi_stock = gdb.parse_and_eval('global_fcgi_stock')
        h = fcgi_stock['hstock']['stocks']

        for i, key, value in for_each_hashmap(h.dereference()):
            stock = value.cast(stock_type)

            for x in ('idle', 'busy'):
                for lh in for_each_list_head(stock[x]):
                    c = lh.cast(fcgi_connection_type)

                    if c['child'] == child_stock_item:
                        print key, c, c.dereference()

DumpHashmapSlot()
DumpHashmap()
DumpHashmap2()
DumpStrmap()
DumpPoolStats()
DumpPoolRefs()
DumpPoolAllocations()
FindPool()
FindChild()
FindChildStockClient()
<|endoftext|>