<commit_before>from django.contrib import admin
from django.forms import ModelForm, ModelMultipleChoiceField

from server.models import *
from server.utils import reload_plugins_model

class BusinessUnitFilter(admin.SimpleListFilter):
    title = 'Business Unit'
    parameter_name = 'business_unit'

    def lookups(self, request, model_admin):
        return ((bu.name, bu.name) for bu in BusinessUnit.objects.all())

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(machine__machine_group__business_unit__name=self.value())
        else:
            return queryset

class MachineGroupFilter(admin.SimpleListFilter):
    title = 'Machine Group'
    parameter_name = 'machine_group'

    def lookups(self, request, model_admin):
        return ((mg.name, mg.name) for mg in MachineGroup.objects.all())

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(machine__machine_group__name=self.value())
        else:
            return queryset

class MachineGroupInline(admin.TabularInline):
    model = MachineGroup

class PluginScriptRowInline(admin.StackedInline):
    model = PluginScriptRow
    extra = 0

class UpdateHistoryItemInline(admin.TabularInline):
    model = UpdateHistoryItem
    extra = 0

class UserProfileInline(admin.StackedInline):
    model = UserProfile

class BusinessUnitForm(ModelForm):

    class Meta:
        model = BusinessUnit
        fields = ['name', 'users']

    users = ModelMultipleChoiceField(
        queryset=User.objects.exclude(userprofile__level='GA'),
        label=('Members'),
        required=False,
        widget=admin.widgets.FilteredSelectMultiple(verbose_name="Users", is_stacked=False))

def number_of_users(obj):
    return obj.users.count()

def number_of_machine_groups(obj):
    return obj.machinegroup_set.count()

def number_of_machines(obj):
    if isinstance(obj, MachineGroup):
        return Machine.objects.filter(machine_group=obj).count()
    else:
        return Machine.objects.filter(machine_group__in=obj.machinegroup_set.all()).count()

def business_unit(obj):
    return obj.machine_group.business_unit.name

class ApiKeyAdmin(admin.ModelAdmin):
    list_display = ('name', 'public_key', 'private_key')

class BusinessUnitAdmin(admin.ModelAdmin):
    inlines = [MachineGroupInline, ]
    form = BusinessUnitForm
    list_display = ('name', number_of_users, number_of_machine_groups, number_of_machines)
    fields = (('name', number_of_users, number_of_machine_groups, number_of_machines), 'users')
    readonly_fields = (number_of_users, number_of_machine_groups, number_of_machines)

class ConditionAdmin(admin.ModelAdmin):
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'condition_name')
    list_display = ('condition_name', 'condition_data', 'machine')
    search_fields = ('condition_name', 'condition_data', 'machine__hostname')

class ManagementSourceAdmin(admin.ModelAdmin):
    list_display = ('name', )

class ManagedItemAdmin(admin.ModelAdmin):
    list_display = ('name', 'management_source', 'machine', 'date_managed', 'status')
    list_filter = ('management_source', BusinessUnitFilter, MachineGroupFilter, 'date_managed', 'status')
    search_fields = ('name', 'data', 'machine')

class FactAdmin(admin.ModelAdmin):
    list_display = ('fact_name', 'fact_data', 'machine', 'management_source')
    list_filter = ('management_source', BusinessUnitFilter, MachineGroupFilter, 'fact_name')
    search_fields = ('fact_name', 'fact_data', 'machine__hostname')

class HistoricalFactAdmin(admin.ModelAdmin):
    list_display = ('fact_name', 'fact_data', 'machine', 'management_source', 'fact_recorded')
    list_filter = ('management_source', BusinessUnitFilter, MachineGroupFilter, 'fact_name')
    search_fields = ('fact_name', 'fact_data', 'machine__hostname')
    date_hierarchy = 'fact_recorded'

class InstalledUpdateAdmin(admin.ModelAdmin):
    list_display = ('update', 'display_name', 'machine', 'update_version', 'installed')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'update')
    search_fields = ('machine__hostname', 'display_name', 'update')

class MachineAdmin(admin.ModelAdmin):
    list_display = ('hostname', 'serial', 'machine_model', 'operating_system', 'deployed')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'operating_system', 'os_family',
                   'machine_model', 'last_checkin', 'errors', 'warnings', 'puppet_errors',
                   'deployed')
    fields = (
        (business_unit, 'machine_group'),
        ('hostname', 'serial', 'console_user'),
        ('machine_model', 'machine_model_friendly'),
        ('cpu_type', 'cpu_speed'), ('memory', 'memory_kb'), ('hd_space', 'hd_total', 'hd_percent'),
        ('operating_system', 'os_family'),
        ('munki_version', 'manifest', 'errors', 'warnings'),
        ('last_checkin', 'first_checkin'),
        ('puppet_version', 'last_puppet_run', 'puppet_errors'),
        ('sal_version', 'deployed', 'broken_client', 'activity'),
        'report'
    )
    readonly_fields = (business_unit, 'first_checkin', 'last_checkin', 'last_puppet_run')
    search_fields = ('hostname', 'console_user')

class MachineDetailPluginAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing MachineDetailPlugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(MachineDetailPluginAdmin, self).get_queryset(request)

class MachineGroupAdmin(admin.ModelAdmin):
    list_display = ('name', 'business_unit', number_of_machines)
    list_filter = (BusinessUnitFilter,)
    fields = ('name', 'business_unit', number_of_machines, 'key')
    readonly_fields = (number_of_machines, 'key')

class PendingUpdateAdmin(admin.ModelAdmin):
    list_filter = (BusinessUnitFilter, MachineGroupFilter)
    list_display = ('update', 'display_name', 'update_version', 'machine')
    search_fields = ('update', 'display_name', 'machine__hostname')

class PluginAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing Plugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(PluginAdmin, self).get_queryset(request)

class PluginScriptRowAdmin(admin.ModelAdmin):
    search_fields = ('pluginscript_name',)

class PluginScriptSubmissionAdmin(admin.ModelAdmin):
    inlines = [PluginScriptRowInline, ]
    list_display = ('plugin', 'machine', 'recorded', 'historical')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'plugin', 'historical', 'recorded')
    search_fields = ('plugin', 'machine__hostname')
    date_hierarchy = 'recorded'

class ReportAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing MachineDetailPlugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(ReportAdmin, self).get_queryset(request)

class SalSettingAdmin(admin.ModelAdmin):
    list_display = ('name', 'value')

class FriendlyNameCacheAdmin(admin.ModelAdmin):
    list_display = ('serial_stub', 'friendly_name')

class UpdateHistoryAdmin(admin.ModelAdmin):
    inlines = [UpdateHistoryItemInline, ]
    list_display = ('name', 'machine', 'update_type', 'version')
    list_filter = ('update_type', BusinessUnitFilter, MachineGroupFilter)
    search_fields = ('name', 'machine__hostname', 'version')

class UpdateHistoryItemAdmin(admin.ModelAdmin):
    search_fields = ('update_history__name', 'update_history__machine__hostname')
    list_display = ('update_history', 'recorded', 'status')
    date_hierarchy = 'recorded'

class CustomUserAdmin(admin.ModelAdmin):
    inlines = (UserProfileInline,)
    list_display = ('username', 'profile_level', 'last_login')
    list_filter = ('userprofile__level',)
    search_fields = ('username', )
    fields = (
        ('username', 'first_name', 'last_name', 'email'),
        ('is_staff', 'is_active', 'is_superuser'),
        ('last_login', 'date_joined'),
    )

    def profile_level(self, user):
        return user.userprofile.level

admin.site.register(ApiKey, ApiKeyAdmin)
admin.site.register(BusinessUnit, BusinessUnitAdmin)
admin.site.register(Condition, ConditionAdmin)
admin.site.register(Fact, FactAdmin)
admin.site.register(HistoricalFact, HistoricalFactAdmin)
admin.site.register(InstalledUpdate, InstalledUpdateAdmin)
admin.site.register(Machine, MachineAdmin)
admin.site.register(MachineDetailPlugin, MachineDetailPluginAdmin)
admin.site.register(MachineGroup, MachineGroupAdmin)
admin.site.register(PendingAppleUpdate, PendingUpdateAdmin)
admin.site.register(PendingUpdate, PendingUpdateAdmin)
admin.site.register(Plugin, PluginAdmin)
admin.site.register(PluginScriptRow, PluginScriptRowAdmin)
admin.site.register(PluginScriptSubmission, PluginScriptSubmissionAdmin)
admin.site.register(Report, ReportAdmin)
admin.site.register(SalSetting, SalSettingAdmin)
admin.site.register(UpdateHistory, UpdateHistoryAdmin)
admin.site.register(UpdateHistoryItem, UpdateHistoryItemAdmin)
admin.site.unregister(User)
admin.site.register(User, CustomUserAdmin)
admin.site.register(FriendlyNameCache, FriendlyNameCacheAdmin)
admin.site.register(ManagementSource, ManagementSourceAdmin)
admin.site.register(ManagedItem, ManagedItemAdmin)
<commit_msg>Fix admin search for ManagedItem.<commit_after>from django.contrib import admin
from django.forms import ModelForm, ModelMultipleChoiceField

from server.models import *
from server.utils import reload_plugins_model

class BusinessUnitFilter(admin.SimpleListFilter):
    title = 'Business Unit'
    parameter_name = 'business_unit'

    def lookups(self, request, model_admin):
        return ((bu.name, bu.name) for bu in BusinessUnit.objects.all())

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(machine__machine_group__business_unit__name=self.value())
        else:
            return queryset

class MachineGroupFilter(admin.SimpleListFilter):
    title = 'Machine Group'
    parameter_name = 'machine_group'

    def lookups(self, request, model_admin):
        return ((mg.name, mg.name) for mg in MachineGroup.objects.all())

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(machine__machine_group__name=self.value())
        else:
            return queryset

class MachineGroupInline(admin.TabularInline):
    model = MachineGroup

class PluginScriptRowInline(admin.StackedInline):
    model = PluginScriptRow
    extra = 0

class UpdateHistoryItemInline(admin.TabularInline):
    model = UpdateHistoryItem
    extra = 0

class UserProfileInline(admin.StackedInline):
    model = UserProfile

class BusinessUnitForm(ModelForm):

    class Meta:
        model = BusinessUnit
        fields = ['name', 'users']

    users = ModelMultipleChoiceField(
        queryset=User.objects.exclude(userprofile__level='GA'),
        label=('Members'),
        required=False,
        widget=admin.widgets.FilteredSelectMultiple(verbose_name="Users", is_stacked=False))

def number_of_users(obj):
    return obj.users.count()

def number_of_machine_groups(obj):
    return obj.machinegroup_set.count()

def number_of_machines(obj):
    if isinstance(obj, MachineGroup):
        return Machine.objects.filter(machine_group=obj).count()
    else:
        return Machine.objects.filter(machine_group__in=obj.machinegroup_set.all()).count()

def business_unit(obj):
    return obj.machine_group.business_unit.name

class ApiKeyAdmin(admin.ModelAdmin):
    list_display = ('name', 'public_key', 'private_key')

class BusinessUnitAdmin(admin.ModelAdmin):
    inlines = [MachineGroupInline, ]
    form = BusinessUnitForm
    list_display = ('name', number_of_users, number_of_machine_groups, number_of_machines)
    fields = (('name', number_of_users, number_of_machine_groups, number_of_machines), 'users')
    readonly_fields = (number_of_users, number_of_machine_groups, number_of_machines)

class ConditionAdmin(admin.ModelAdmin):
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'condition_name')
    list_display = ('condition_name', 'condition_data', 'machine')
    search_fields = ('condition_name', 'condition_data', 'machine__hostname')

class ManagementSourceAdmin(admin.ModelAdmin):
    list_display = ('name', )

class ManagedItemAdmin(admin.ModelAdmin):
    list_display = ('name', 'management_source', 'machine', 'date_managed', 'status')
    list_filter = (
        'management_source', BusinessUnitFilter, MachineGroupFilter, 'date_managed', 'status')
    search_fields = ('name', 'data', 'machine__hostname', 'machine__serial')

class FactAdmin(admin.ModelAdmin):
    list_display = ('fact_name', 'fact_data', 'machine', 'management_source')
    list_filter = ('management_source', BusinessUnitFilter, MachineGroupFilter, 'fact_name')
    search_fields = ('fact_name', 'fact_data', 'machine__hostname')

class HistoricalFactAdmin(admin.ModelAdmin):
    list_display = ('fact_name', 'fact_data', 'machine', 'management_source', 'fact_recorded')
    list_filter = ('management_source', BusinessUnitFilter, MachineGroupFilter, 'fact_name')
    search_fields = ('fact_name', 'fact_data', 'machine__hostname')
    date_hierarchy = 'fact_recorded'

class InstalledUpdateAdmin(admin.ModelAdmin):
    list_display = ('update', 'display_name', 'machine', 'update_version', 'installed')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'update')
    search_fields = ('machine__hostname', 'display_name', 'update')

class MachineAdmin(admin.ModelAdmin):
    list_display = ('hostname', 'serial', 'machine_model', 'operating_system', 'deployed')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'operating_system', 'os_family',
                   'machine_model', 'last_checkin', 'errors', 'warnings', 'puppet_errors',
                   'deployed')
    fields = (
        (business_unit, 'machine_group'),
        ('hostname', 'serial', 'console_user'),
        ('machine_model', 'machine_model_friendly'),
        ('cpu_type', 'cpu_speed'), ('memory', 'memory_kb'), ('hd_space', 'hd_total', 'hd_percent'),
        ('operating_system', 'os_family'),
        ('munki_version', 'manifest', 'errors', 'warnings'),
        ('last_checkin', 'first_checkin'),
        ('puppet_version', 'last_puppet_run', 'puppet_errors'),
        ('sal_version', 'deployed', 'broken_client', 'activity'),
        'report'
    )
    readonly_fields = (business_unit, 'first_checkin', 'last_checkin', 'last_puppet_run')
    search_fields = ('hostname', 'console_user')

class MachineDetailPluginAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing MachineDetailPlugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(MachineDetailPluginAdmin, self).get_queryset(request)

class MachineGroupAdmin(admin.ModelAdmin):
    list_display = ('name', 'business_unit', number_of_machines)
    list_filter = (BusinessUnitFilter,)
    fields = ('name', 'business_unit', number_of_machines, 'key')
    readonly_fields = (number_of_machines, 'key')

class PendingUpdateAdmin(admin.ModelAdmin):
    list_filter = (BusinessUnitFilter, MachineGroupFilter)
    list_display = ('update', 'display_name', 'update_version', 'machine')
    search_fields = ('update', 'display_name', 'machine__hostname')

class PluginAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing Plugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(PluginAdmin, self).get_queryset(request)

class PluginScriptRowAdmin(admin.ModelAdmin):
    search_fields = ('pluginscript_name',)

class PluginScriptSubmissionAdmin(admin.ModelAdmin):
    inlines = [PluginScriptRowInline, ]
    list_display = ('plugin', 'machine', 'recorded', 'historical')
    list_filter = (BusinessUnitFilter, MachineGroupFilter, 'plugin', 'historical', 'recorded')
    search_fields = ('plugin', 'machine__hostname')
    date_hierarchy = 'recorded'

class ReportAdmin(admin.ModelAdmin):
    list_display = ('name',)

    def get_queryset(self, request):
        """Update db prior to retrieving plugins.

        Views listing MachineDetailPlugins must first update the list of
        installed plugins.
        """
        reload_plugins_model()
        return super(ReportAdmin, self).get_queryset(request)

class SalSettingAdmin(admin.ModelAdmin):
    list_display = ('name', 'value')

class FriendlyNameCacheAdmin(admin.ModelAdmin):
    list_display = ('serial_stub', 'friendly_name')

class UpdateHistoryAdmin(admin.ModelAdmin):
    inlines = [UpdateHistoryItemInline, ]
    list_display = ('name', 'machine', 'update_type', 'version')
    list_filter = ('update_type', BusinessUnitFilter, MachineGroupFilter)
    search_fields = ('name', 'machine__hostname', 'version')

class UpdateHistoryItemAdmin(admin.ModelAdmin):
    search_fields = ('update_history__name', 'update_history__machine__hostname')
    list_display = ('update_history', 'recorded', 'status')
    date_hierarchy = 'recorded'

class CustomUserAdmin(admin.ModelAdmin):
    inlines = (UserProfileInline,)
    list_display = ('username', 'profile_level', 'last_login')
    list_filter = ('userprofile__level',)
    search_fields = ('username', )
    fields = (
        ('username', 'first_name', 'last_name', 'email'),
        ('is_staff', 'is_active', 'is_superuser'),
        ('last_login', 'date_joined'),
    )

    def profile_level(self, user):
        return user.userprofile.level

admin.site.register(ApiKey, ApiKeyAdmin)
admin.site.register(BusinessUnit, BusinessUnitAdmin)
admin.site.register(Condition, ConditionAdmin)
admin.site.register(Fact, FactAdmin)
admin.site.register(HistoricalFact, HistoricalFactAdmin)
admin.site.register(InstalledUpdate, InstalledUpdateAdmin)
admin.site.register(Machine, MachineAdmin)
admin.site.register(MachineDetailPlugin, MachineDetailPluginAdmin)
admin.site.register(MachineGroup, MachineGroupAdmin)
admin.site.register(PendingAppleUpdate, PendingUpdateAdmin)
admin.site.register(PendingUpdate, PendingUpdateAdmin)
admin.site.register(Plugin, PluginAdmin)
admin.site.register(PluginScriptRow, PluginScriptRowAdmin)
admin.site.register(PluginScriptSubmission, PluginScriptSubmissionAdmin)
admin.site.register(Report, ReportAdmin)
admin.site.register(SalSetting, SalSettingAdmin)
admin.site.register(UpdateHistory, UpdateHistoryAdmin)
admin.site.register(UpdateHistoryItem, UpdateHistoryItemAdmin)
admin.site.unregister(User)
admin.site.register(User, CustomUserAdmin)
admin.site.register(FriendlyNameCache, FriendlyNameCacheAdmin)
admin.site.register(ManagementSource, ManagementSourceAdmin)
admin.site.register(ManagedItem, ManagedItemAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import json
import datetime
import re

import click
import flask
import markdown2

app = flask.Flask(__name__)

class Message(object):
    def __init__(self, USER_DATA, message):
        self.__USER_DATA = USER_DATA
        self._message = message

    ##############
    # Properties #
    ##############

    @property
    def user_id(self):
        return self._message["user"]

    @property
    def username(self):
        try:
            return self.__USER_DATA[self._message["user"]]["name"]
        except KeyError:
            # In case this is a bot or something, we fallback to "username"
            if"username" in self._message:
                return self._message["username"]
            # If that fails to, it's probably USLACKBOT...
            else:
                return self._message["user"]

    @property
    def time(self):
        # Handle this: "ts": "1456427378.000002"
        tsepoch = float(self._message["ts"].split(".")[0])
        return str(datetime.datetime.fromtimestamp(tsepoch)).split('.')[0]

    @property
    def msg(self):
        message = self._message["text"]
        # Handle "<@U0BM1CGQY|calvinchanubc> has joined the channel"
        message = re.sub(r"<@U0\w+\|[A-Za-z0-9.-_]+>",
                         self._annotated_mention, message)
        # Handle "<@U0BM1CGQY>"
        message = re.sub(r"<@U0\w+>", self._mention, message)
        # Handle "<url>"
        message = re.sub(r"<http(s|)://.*>", self._url, message)
        # Handle hashtags (that are meant to be hashtags and not headings)
        message = re.sub(r"^#\S+", self._hashtag, message)

        message = markdown2.markdown(message, extras=["cuddled-lists"]).strip()
        # markdown2 likes to wrap everything in
 tags
        if message.startswith("
") and message.endswith("
"):
            message = message[3:-4]

        return message

    @property
    def img(self):
        try:
            return self.__USER_DATA[self._message["user"]]["profile"]["image_72"]
        except KeyError:
            return ""

    ###################
    # Private Methods #
    ###################

    def _mention(self, matchobj):
        return "@{}".format(
            self.__USER_DATA[matchobj.group(0)[2:-1]]["name"]
        )

    def _annotated_mention(self, matchobj):
        return "@{}".format((matchobj.group(0)[2:-1]).split("|")[1])

    def _url(self, matchobj):
        compound = matchobj.group(0)[1:-1]
        if len(compound.split("|")) == 2:
            url, title = compound.split("|")
        else:
            url, title = compound, compound
        return "[{title}]({url})".format(url=url, title=title)

    def _hashtag(self, matchobj):
        text = matchobj.group(0)
        return "_{}_".format(text)

def compile_channels(path, users):
    channels = [d for d in os.listdir(path)
                if os.path.isdir(os.path.join(path, d))]
    chats = {}
    for channel in channels:
        channel_dir_path = os.path.join(path, channel)
        messages = []
        for day in os.listdir(channel_dir_path):
            with open(os.path.join(channel_dir_path, day)) as f:
                day_messages = json.load(f)
                messages.extend([Message(users, d) for d in day_messages])
        chats[channel] = messages
    return chats

def get_users(path):
    with open(os.path.join(path, "users.json")) as f:
        return {u["id"]: u for u in json.load(f)}

@app.route("/channel/<name>")
def channel_name(name):
    messages = flask._app_ctx_stack.channels[name]
    return flask.render_template("archive.html", messages=messages,
                                 title="#{name}".format(name=name))

@click.command()
@click.option("-d", "--dir", required=True)
@click.option("-p", "--port", default=5000, type=click.INT)
def main(dir, port):
    users = get_users(dir)
    channels = compile_channels(dir, users=users)

    top = flask._app_ctx_stack
    top.channels = channels

    app.debug = True
    app.run(
        host='0.0.0.0',
        port=port
    )

if __name__ == '__main__':
    main()
<commit_msg>Handle channel references; resolves #7<commit_after>#!/usr/bin/env python3

import os
import json
import datetime
import re

import click
import flask
import markdown2

app = flask.Flask(__name__)

class Message(object):
    def __init__(self, USER_DATA, CHANNEL_DATA, message):
        self.__USER_DATA = USER_DATA
        self.__CHANNEL_DATA = CHANNEL_DATA
        self._message = message

    ##############
    # Properties #
    ##############

    @property
    def user_id(self):
        return self._message["user"]

    @property
    def username(self):
        try:
            return self.__USER_DATA[self._message["user"]]["name"]
        except KeyError:
            # In case this is a bot or something, we fallback to "username"
            if"username" in self._message:
                return self._message["username"]
            # If that fails to, it's probably USLACKBOT...
            else:
                return self._message["user"]

    @property
    def time(self):
        # Handle this: "ts": "1456427378.000002"
        tsepoch = float(self._message["ts"].split(".")[0])
        return str(datetime.datetime.fromtimestamp(tsepoch)).split('.')[0]

    @property
    def msg(self):
        message = self._message["text"]
        # Handle "<@U0BM1CGQY|calvinchanubc> has joined the channel"
        message = re.sub(r"<@U0\w+\|[A-Za-z0-9.-_]+>",
                         self._annotated_mention, message)
        # Handle "<@U0BM1CGQY>"
        message = re.sub(r"<@U0\w+>", self._mention, message)
        # Handle "<http(s|)://...>"
        message = re.sub(r"<http(s|)://.*>", self._url, message)
        # Handle hashtags (that are meant to be hashtags and not headings)
        message = re.sub(r"^#\S+", self._hashtag, message)
        # Handle channel references
        message = re.sub(r"<#C0\w+>", self._channel_ref, message)

        message = markdown2.markdown(message, extras=["cuddled-lists"]).strip()
        # markdown2 likes to wrap everything in
 tags
        if message.startswith("
") and message.endswith("
"):
            message = message[3:-4]

        return message

    @property
    def img(self):
        try:
            return self.__USER_DATA[self._message["user"]]["profile"]["image_72"]
        except KeyError:
            return ""

    ###################
    # Private Methods #
    ###################

    def _mention(self, matchobj):
        return "@{}".format(
            self.__USER_DATA[matchobj.group(0)[2:-1]]["name"]
        )

    def _annotated_mention(self, matchobj):
        return "@{}".format((matchobj.group(0)[2:-1]).split("|")[1])

    def _url(self, matchobj):
        compound = matchobj.group(0)[1:-1]
        if len(compound.split("|")) == 2:
            url, title = compound.split("|")
        else:
            url, title = compound, compound
        return "[{title}]({url})".format(url=url, title=title)

    def _hashtag(self, matchobj):
        text = matchobj.group(0)
        return "_{}_".format(text)

    def _channel_ref(self, matchobj):
        channel_id = matchobj.group(0)[2:-1]
        channel_name = self.__CHANNEL_DATA[channel_id]["name"]
        return "_#{}_".format(channel_name)

def compile_channels(path, user_data, channel_data):
    channels = [d for d in os.listdir(path)
                if os.path.isdir(os.path.join(path, d))]
    chats = {}
    for channel in channels:
        channel_dir_path = os.path.join(path, channel)
        messages = []
        for day in os.listdir(channel_dir_path):
            with open(os.path.join(channel_dir_path, day)) as f:
                day_messages = json.load(f)
                messages.extend([Message(user_data, channel_data, d) for d in
                                 day_messages])
        chats[channel] = messages
    return chats

def get_users(path):
    with open(os.path.join(path, "users.json")) as f:
        return {u["id"]: u for u in json.load(f)}

def get_channels(path):
    with open(os.path.join(path, "channels.json")) as f:
        return {u["id"]: u for u in json.load(f)}

@app.route("/channel/<name>")
def channel_name(name):
    messages = flask._app_ctx_stack.channels[name]
    return flask.render_template("archive.html", messages=messages,
                                 title="#{name}".format(name=name))

@click.command()
@click.option("-d", "--dir", required=True)
@click.option("-p", "--port", default=5000, type=click.INT)
def main(dir, port):
    user_data = get_users(dir)
    channel_data = get_channels(dir)
    channels = compile_channels(dir, user_data, channel_data)

    top = flask._app_ctx_stack
    top.channels = channels

    app.debug = True
    app.run(
        host='0.0.0.0',
        port=port
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from mythril.laser.ethereum.svm import LaserEVM
from mythril.laser.ethereum.plugins import LaserPlugin
from time import time
import matplotlib.pyplot as plt
import logging

log = logging.getLogger(__name__)

class BenchmarkPlugin(LaserPlugin):
    """Benchmark Plugin

    This plugin aggregates the following information:
    - duration
    - code coverage over time
    - final code coverage
    - total number of executed instructions

    """

    def __init__(self, name=None):
        """Creates BenchmarkPlugin

        :param name: name of this benchmark, used for storing the results
        """
        self.nr_of_executed_insns = 0
        self.begin = None
        self.end = None
        self.coverage = {}
        self.name = name

    def initialize(self, symbolic_vm: LaserEVM):
        """Initializes the BenchmarkPlugin

        Introduces hooks in symbolic_vm to track the desired values
        :param symbolic_vm: Symbolic virtual machine to analyze
        """
        self._reset()

        @symbolic_vm.laser_hook("execute_state")
        def execute_state_hook(_):
            current_time = time() - self.begin
            self.nr_of_executed_insns += 1

            for key, value in symbolic_vm.coverage.items():
                try:
                    self.coverage[key][current_time] = sum(value[1]) * 100 / value[0]
                except KeyError:
                    self.coverage[key] = {}
                    self.coverage[key][current_time] = sum(value[1]) * 100 / value[0]

        @symbolic_vm.laser_hook("start_sym_exec")
        def start_sym_exec_hook():
            self.begin = time()

        @symbolic_vm.laser_hook("stop_sym_exec")
        def stop_sym_exec_hook():
            self.end = time()

            self._write_to_graph()
            self._store_report()

    def _reset(self):
        """Reset this plugin"""
        self.nr_of_executed_insns = 0
        self.begin = None
        self.end = None
        self.coverage = {}

    def _store_report(self):
        """Store the results of this plugin"""
        pass

    def _write_to_graph(self):
        """Write the coverage results to a graph"""
        traces = []
        for byte_code, trace_data in self.coverage.items():
            traces += [list(trace_data.keys()), list(trace_data.values()), "r--"]

        plt.plot(*traces)
        plt.axis([0, self.end - self.begin, 0, 100])
        plt.xlabel("Duration (seconds)")
        plt.ylabel("Coverage (percentage)")

        plt.savefig("{}.png".format(self.name))
<commit_msg>fix broken import<commit_after>from mythril.laser.ethereum.svm import LaserEVM
from mythril.laser.ethereum.plugins.plugin import LaserPlugin
from time import time
import matplotlib.pyplot as plt
import logging

log = logging.getLogger(__name__)

class BenchmarkPlugin(LaserPlugin):
    """Benchmark Plugin

    This plugin aggregates the following information:
    - duration
    - code coverage over time
    - final code coverage
    - total number of executed instructions

    """

    def __init__(self, name=None):
        """Creates BenchmarkPlugin

        :param name: name of this benchmark, used for storing the results
        """
        self.nr_of_executed_insns = 0
        self.begin = None
        self.end = None
        self.coverage = {}
        self.name = name

    def initialize(self, symbolic_vm: LaserEVM):
        """Initializes the BenchmarkPlugin

        Introduces hooks in symbolic_vm to track the desired values
        :param symbolic_vm: Symbolic virtual machine to analyze
        """
        self._reset()

        @symbolic_vm.laser_hook("execute_state")
        def execute_state_hook(_):
            current_time = time() - self.begin
            self.nr_of_executed_insns += 1

            for key, value in symbolic_vm.coverage.items():
                try:
                    self.coverage[key][current_time] = sum(value[1]) * 100 / value[0]
                except KeyError:
                    self.coverage[key] = {}
                    self.coverage[key][current_time] = sum(value[1]) * 100 / value[0]

        @symbolic_vm.laser_hook("start_sym_exec")
        def start_sym_exec_hook():
            self.begin = time()

        @symbolic_vm.laser_hook("stop_sym_exec")
        def stop_sym_exec_hook():
            self.end = time()

            self._write_to_graph()
            self._store_report()

    def _reset(self):
        """Reset this plugin"""
        self.nr_of_executed_insns = 0
        self.begin = None
        self.end = None
        self.coverage = {}

    def _store_report(self):
        """Store the results of this plugin"""
        pass

    def _write_to_graph(self):
        """Write the coverage results to a graph"""
        traces = []
        for byte_code, trace_data in self.coverage.items():
            traces += [list(trace_data.keys()), list(trace_data.values()), "r--"]

        plt.plot(*traces)
        plt.axis([0, self.end - self.begin, 0, 100])
        plt.xlabel("Duration (seconds)")
        plt.ylabel("Coverage (percentage)")

        plt.savefig("{}.png".format(self.name))
<|endoftext|>


<commit_before>from __future__ import print_function
import pytest
import xarray as xr
import numpy as np

from xgcm.grid import Grid, Axis
from xgcm.test.datasets import datasets_grid_metric

@pytest.mark.parametrize("funcname", ["interp", "diff", "min", "max", "cumsum"])
@pytest.mark.parametrize("grid_type", ["B", "C"])
@pytest.mark.parametrize("variable", ["tracer", "u", "v"])
class TestParametrized:
    @pytest.mark.parametrize("axis", ["X", "Y"])
    @pytest.mark.parametrize("metric_weighted", ["X", ("Y",), ("X", "Y"), ["X", "Y"]])
    @pytest.mark.parametrize(
        "periodic", ["True", "False", {"X": True, "Y": False}, {"X": False, "Y": True}]
    )
    @pytest.mark.parametrize("boundary", ["fill", "extend"])
    def test_weighted_metric(
        self, funcname, grid_type, variable, axis, metric_weighted, periodic, boundary
    ):
        """tests the correct execution of weighted ops along a single axis"""
        # metric_weighted allows the interpolation of e.g. a surface flux to be conservative
        # It multiplies the values with a metric like the area, then performs interpolation
        # and divides by the same metric (area) for the new grid position
        ds, coords, metrics = datasets_grid_metric(grid_type)
        grid = Grid(ds, coords=coords, metrics=metrics, periodic=periodic)
        func = getattr(grid, funcname)

        metric = grid.get_metric(ds[variable], metric_weighted)
        expected_raw = func(ds[variable] * metric, axis, boundary=boundary)
        metric_new = grid.get_metric(expected_raw, metric_weighted)
        expected = expected_raw / metric_new
        new = func(
            ds[variable], axis, metric_weighted=metric_weighted, boundary=boundary
        )
        assert new.equals(expected)

        @pytest.mark.parametrize(
            "multi_axis", ["X", ["X"], ("Y"), ["X", "Y"], ("Y", "X")]
        )
        def test_weighted_metric_multi_axis(
            self, funcname, grid_type, variable, multi_axis, metric_weighted, boundary
        ):
            """tests if the output for multiple axis is the same as when
            executing the single axis ops in serial"""
            ds, coords, metrics = datasets_grid_metric(grid_type)
            grid = Grid(ds, coords=coords, metrics=metrics)

            func = getattr(grid, funcname)
            expected = ds[variable]
            for ax in multi_axis:
                if isinstance(metric_weighted, dict):
                    metric_weighted_axis = metric_weighted[ax]
                else:
                    metric_weighted_axis = metric_weighted
                expected = func(
                    expected,
                    ax,
                    metric_weighted=metric_weighted_axis,
                    boundary=boundary,
                )

            new = func(
                ds[variable],
                multi_axis,
                metric_weighted=metric_weighted,
                boundary=boundary,
            )
            assert new.equals(expected)

@pytest.mark.parametrize(
    "funcname",
    ["interp", "diff", "min", "max", "cumsum", "derivative", "cumint"],
)
@pytest.mark.parametrize("boundary", ["fill", "extend"])
@pytest.mark.parametrize("fill_value", [0, 10, None])
def test_boundary_global_input(funcname, boundary, fill_value):
    """Test that globally defined boundary values result in
    the same output as when the parameters are defined on either
    the grid or axis methods
    """
    ds, coords, metrics = datasets_grid_metric("C")
    axis = "X"

    # Test results by globally specifying boundary on grid object
    grid_global = Grid(
        ds,
        coords=coords,
        metrics=metrics,
        periodic=False,
        boundary=boundary,
        fill_value=fill_value,
    )
    func_global = getattr(grid_global, funcname)
    global_result = func_global(ds.tracer, axis)

    # Test results by manually specifying fill value/boundary on grid method
    grid_manual = Grid(
        ds, coords=coords, metrics=metrics, periodic=False, boundary=boundary
    )
    func_manual = getattr(grid_manual, funcname)
    manual_result = func_manual(
        ds.tracer, axis, boundary=boundary, fill_value=fill_value
    )
    xr.testing.assert_allclose(global_result, manual_result)

def test_derivative_uniform_grid():
    # this is a uniform grid
    # a non-uniform grid would provide a more rigorous test
    dx = 10.0
    dy = 10.0
    arr = [
        [1.0, 2.0, 4.0, 3.0],
        [4.0, 7.0, 1.0, 2.0],
        [3.0, 1.0, 0.0, 9.0],
        [8.0, 5.0, 2.0, 1.0],
    ]
    ds = xr.Dataset(
        {"foo": (("XC", "YC"), arr)},
        coords={
            "XC": (("XC",), [0.5, 1.5, 2.5, 3.5]),
            "XG": (("XG",), [0, 1.0, 2.0, 3.0]),
            "dXC": (("XC",), [dx, dx, dx, dx]),
            "dXG": (("XG",), [dx, dx, dx, dx]),
            "YC": (("YC",), [0.5, 1.5, 2.5, 3.5]),
            "YG": (("YG",), [0, 1.0, 2.0, 3.0]),
            "dYC": (("YC",), [dy, dy, dy, dy]),
            "dYG": (("YG",), [dy, dy, dy, dy]),
        },
    )

    grid = Grid(
        ds,
        coords={
            "X": {"center": "XC", "left": "XG"},
            "Y": {"center": "YC", "left": "YG"},
        },
        metrics={("X",): ["dXC", "dXG"], ("Y",): ["dYC", "dYG"]},
        periodic=True,
    )

    # Test x direction
    dfoo_dx = grid.derivative(ds.foo, "X")
    expected = grid.diff(ds.foo, "X") / dx
    assert dfoo_dx.equals(expected)

    # Test y direction
    dfoo_dy = grid.derivative(ds.foo, "Y")
    expected = grid.diff(ds.foo, "Y") / dy
    assert dfoo_dy.equals(expected)

def test_derivative_c_grid():
    # test derivatives with synthetic C grid data

    ds, coords, metrics = datasets_grid_metric("C")
    grid = Grid(ds, coords=coords, metrics=metrics)

    # tracer point
    var = "tracer"
    test_axes = ["X", "Y", "Z"]
    test_dx = ["dx_e", "dy_n", "dz_w"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # zonal velocity point
    var = "u"
    test_dx = ["dx_t", "dy_ne", "dz_w_e"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # meridional velocity point
    var = "v"
    test_dx = ["dx_ne", "dy_t", "dz_w_n"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # vertical velocity point
    var = "wt"
    test_dx = ["dx_e", "dy_n", "dz_t"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

def test_derivative_b_grid():
    # test derivatives with synthetic B grid data

    ds, coords, metrics = datasets_grid_metric("B")
    grid = Grid(ds, coords=coords, metrics=metrics)

    # tracer point
    var = "tracer"
    test_axes = ["X", "Y", "Z"]
    test_dx = ["dx_e", "dy_n", "dz_w"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # zonal velocity point
    var = "u"
    test_dx = ["dx_n", "dy_e", "dz_w_ne"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # meridional velocity point
    var = "v"
    test_dx = ["dx_n", "dy_e", "dz_w_ne"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # vertical velocity point
    var = "wt"
    test_dx = ["dx_e", "dy_n", "dz_t"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

# run this for each axis and each field in dataset
def _run_single_derivative_test(grid, axis, fld, dx):

    dvar_dx = grid.derivative(fld, axis)
    expected = grid.diff(fld, axis) / dx

    assert dvar_dx.equals(expected.reset_coords(drop=True))
<commit_msg>Update xgcm/test/test_metrics_ops_single_axis.py<commit_after>from __future__ import print_function
import pytest
import xarray as xr
import numpy as np

from xgcm.grid import Grid, Axis
from xgcm.test.datasets import datasets_grid_metric

@pytest.mark.parametrize("funcname", ["interp", "diff", "min", "max", "cumsum"])
@pytest.mark.parametrize("grid_type", ["B", "C"])
@pytest.mark.parametrize("variable", ["tracer", "u", "v"])
class TestParametrized:
    @pytest.mark.parametrize("axis", ["X", "Y"])
    @pytest.mark.parametrize("metric_weighted", ["X", ("Y",), ("X", "Y"), ["X", "Y"]])
    @pytest.mark.parametrize(
        "periodic", ["True", "False", {"X": True, "Y": False}, {"X": False, "Y": True}]
    )
    @pytest.mark.parametrize("boundary", ["fill", "extend"])
    def test_weighted_metric(
        self, funcname, grid_type, variable, axis, metric_weighted, periodic, boundary
    ):
        """tests the correct execution of weighted ops along a single axis"""
        # metric_weighted allows the interpolation of e.g. a surface flux to be conservative
        # It multiplies the values with a metric like the area, then performs interpolation
        # and divides by the same metric (area) for the new grid position
        ds, coords, metrics = datasets_grid_metric(grid_type)
        grid = Grid(ds, coords=coords, metrics=metrics, periodic=periodic)
        func = getattr(grid, funcname)

        metric = grid.get_metric(ds[variable], metric_weighted)
        expected_raw = func(ds[variable] * metric, axis, boundary=boundary)
        metric_new = grid.get_metric(expected_raw, metric_weighted)
        expected = expected_raw / metric_new
        new = func(
            ds[variable], axis, metric_weighted=metric_weighted, boundary=boundary
        )
        assert new.equals(expected)

        @pytest.mark.parametrize(
            "multi_axis", ["X", ["X"], ("Y"), ["X", "Y"], ("Y", "X")]
        )
        def test_weighted_metric_multi_axis(
            self, funcname, grid_type, variable, multi_axis, metric_weighted, boundary
        ):
            """tests if the output for multiple axis is the same as when
            executing the single axis ops in serial"""
            ds, coords, metrics = datasets_grid_metric(grid_type)
            grid = Grid(ds, coords=coords, metrics=metrics)

            func = getattr(grid, funcname)
            expected = ds[variable]
            for ax in multi_axis:
                if isinstance(metric_weighted, dict):
                    metric_weighted_axis = metric_weighted[ax]
                else:
                    metric_weighted_axis = metric_weighted
                expected = func(
                    expected,
                    ax,
                    metric_weighted=metric_weighted_axis,
                    boundary=boundary,
                )

            new = func(
                ds[variable],
                multi_axis,
                metric_weighted=metric_weighted,
                boundary=boundary,
            )
            assert new.equals(expected)

@pytest.mark.parametrize(
    "funcname",
    ["interp", "diff", "min", "max", "cumsum", "derivative", "cumint"],
)
@pytest.mark.parametrize("boundary", ["fill", "extend"])
@pytest.mark.parametrize("fill_value", [0, 10, None])
def test_boundary_global_input(funcname, boundary, fill_value):
    """Test that globally defined boundary values result in
    the same output as when the parameters are defined on either
    the grid or axis methods
    """
    ds, coords, metrics = datasets_grid_metric("C")
    axis = "X"

    # Test results by globally specifying fill value/boundary on grid object
    grid_global = Grid(
        ds,
        coords=coords,
        metrics=metrics,
        periodic=False,
        boundary=boundary,
        fill_value=fill_value,
    )
    func_global = getattr(grid_global, funcname)
    global_result = func_global(ds.tracer, axis)

    # Test results by manually specifying fill value/boundary on grid method
    grid_manual = Grid(
        ds, coords=coords, metrics=metrics, periodic=False, boundary=boundary
    )
    func_manual = getattr(grid_manual, funcname)
    manual_result = func_manual(
        ds.tracer, axis, boundary=boundary, fill_value=fill_value
    )
    xr.testing.assert_allclose(global_result, manual_result)

def test_derivative_uniform_grid():
    # this is a uniform grid
    # a non-uniform grid would provide a more rigorous test
    dx = 10.0
    dy = 10.0
    arr = [
        [1.0, 2.0, 4.0, 3.0],
        [4.0, 7.0, 1.0, 2.0],
        [3.0, 1.0, 0.0, 9.0],
        [8.0, 5.0, 2.0, 1.0],
    ]
    ds = xr.Dataset(
        {"foo": (("XC", "YC"), arr)},
        coords={
            "XC": (("XC",), [0.5, 1.5, 2.5, 3.5]),
            "XG": (("XG",), [0, 1.0, 2.0, 3.0]),
            "dXC": (("XC",), [dx, dx, dx, dx]),
            "dXG": (("XG",), [dx, dx, dx, dx]),
            "YC": (("YC",), [0.5, 1.5, 2.5, 3.5]),
            "YG": (("YG",), [0, 1.0, 2.0, 3.0]),
            "dYC": (("YC",), [dy, dy, dy, dy]),
            "dYG": (("YG",), [dy, dy, dy, dy]),
        },
    )

    grid = Grid(
        ds,
        coords={
            "X": {"center": "XC", "left": "XG"},
            "Y": {"center": "YC", "left": "YG"},
        },
        metrics={("X",): ["dXC", "dXG"], ("Y",): ["dYC", "dYG"]},
        periodic=True,
    )

    # Test x direction
    dfoo_dx = grid.derivative(ds.foo, "X")
    expected = grid.diff(ds.foo, "X") / dx
    assert dfoo_dx.equals(expected)

    # Test y direction
    dfoo_dy = grid.derivative(ds.foo, "Y")
    expected = grid.diff(ds.foo, "Y") / dy
    assert dfoo_dy.equals(expected)

def test_derivative_c_grid():
    # test derivatives with synthetic C grid data

    ds, coords, metrics = datasets_grid_metric("C")
    grid = Grid(ds, coords=coords, metrics=metrics)

    # tracer point
    var = "tracer"
    test_axes = ["X", "Y", "Z"]
    test_dx = ["dx_e", "dy_n", "dz_w"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # zonal velocity point
    var = "u"
    test_dx = ["dx_t", "dy_ne", "dz_w_e"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # meridional velocity point
    var = "v"
    test_dx = ["dx_ne", "dy_t", "dz_w_n"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # vertical velocity point
    var = "wt"
    test_dx = ["dx_e", "dy_n", "dz_t"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

def test_derivative_b_grid():
    # test derivatives with synthetic B grid data

    ds, coords, metrics = datasets_grid_metric("B")
    grid = Grid(ds, coords=coords, metrics=metrics)

    # tracer point
    var = "tracer"
    test_axes = ["X", "Y", "Z"]
    test_dx = ["dx_e", "dy_n", "dz_w"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # zonal velocity point
    var = "u"
    test_dx = ["dx_n", "dy_e", "dz_w_ne"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # meridional velocity point
    var = "v"
    test_dx = ["dx_n", "dy_e", "dz_w_ne"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

    # vertical velocity point
    var = "wt"
    test_dx = ["dx_e", "dy_n", "dz_t"]
    for ax, dx in zip(test_axes, test_dx):
        _run_single_derivative_test(grid, ax, ds[var], ds[dx])

# run this for each axis and each field in dataset
def _run_single_derivative_test(grid, axis, fld, dx):

    dvar_dx = grid.derivative(fld, axis)
    expected = grid.diff(fld, axis) / dx

    assert dvar_dx.equals(expected.reset_coords(drop=True))
<|endoftext|>


<commit_before>#!/usr/bin/python

import RPi.GPIO as GPIO
import nrf24
import sys
import time
import spidev
import argparse
import os
import sqlite3
import datetime
import errno
import socket

PIPES = ([0xe7, 0xe7, 0xe7, 0xe7, 0xe7], [0xc2, 0xc2, 0xc2, 0xc2, 0xc2])

class Relay:
    def __init__(self, pins):
        self.pins = pins
        for pin in self.pins.values():
            GPIO.setup(pin, GPIO.OUT, initial=GPIO.HIGH)

    def output(self, pin, state):
        # print "setting pin", pin, state and "on" or "off"
        GPIO.output(self.pins[pin], not state)  # These devices are active-low.

    def cleanup(self):
        pass  # this will be done later: GPIO.cleanup()

class Temperature:
    def __init__(self, major=0, minor=0):
        self.spi = spidev.SpiDev()
        self.spi.open(major, minor)

    def rawread(self):
        return self.spi.xfer2([0, 0])

    def read(self):
        return self.calc_temp(self.rawread())

    @staticmethod
    def calc_temp(buf):
        return (((buf[0] << 8) | buf[1]) >> 3) * 0.0625

    def cleanup(self):
        self.spi.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class Boiler:
    def __init__(self, major, minor, ce_pin, irq_pin, temperature, relay):
        self.relay = relay
        self.temperature = temperature
        self.radio = nrf24.NRF24()
        self.radio.begin(major, minor, ce_pin, irq_pin)
        self.radio.enableDynamicPayloads()
        self.radio.printDetails()
        self.radio.openWritingPipe(PIPES[0])
        self.radio.openReadingPipe(1, PIPES[1])
        self.radio.setAutoAck(1)

    def run(self):
        while True:
            try:
                self.radio.startListening()
                recv_buffer = self.recv(10)
                for byte in recv_buffer:
                    pin = byte >> 1
                    state = byte & 0x1
                    self.relay.output(pin, state)
                self.radio.stopListening()
                self.radio.write(self.temperature.rawread())
            except Exception:
                pass

    def recv(self, timeout=None):
        end = time.time() + timeout
        pipe = [0]
        while not self.radio.available(pipe) and (timeout is None or time.time() < end):
            time.sleep(10000 / 1e6)
        if self.radio.available(pipe):
            recv_buffer = []
            self.radio.read(recv_buffer)
            return recv_buffer
        return []

    def cleanup(self):
        self.radio.end()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class Controller:
    def __init__(self, major, minor, ce_pin, irq_pin, temperature, db, sock):
        self.temperature = temperature
        self.db = db
        self.sock = sock
        self.radio = nrf24.NRF24()
        self.radio.begin(major, minor, ce_pin, irq_pin)
        self.radio.enableDynamicPayloads()
        self.radio.printDetails()
        self.radio.openWritingPipe(PIPES[0])
        self.radio.openReadingPipe(1, PIPES[1])

    def run(self):
        try:
            while True:
                self.radio.startListening()
                recv_buffer = self.recv(10)
                self.radio.stopListening()
                if recv_buffer:
                    self.db.write(1, self.temperature.calc_temp(recv_buffer))
                self.db.write(0, self.temperature.read())
                try:
                    conn, addr = sock.accept()
                except socket.error as e:
                    if e.errno != errno.EAGAIN:
                        raise
                else:
                    print conn
                    conn.settimeout(10)
                    try:
                        recv_line = conn.recv(1024)
                        print repr(recv_line)
                        if recv_line[-1] == '\n':
                            state, pin = recv_line[:-1].split()
                            cmd = int(pin) << 1 | (state.lower() == 'on')
                            print "sending", cmd
                            self.radio.write(chr(cmd))
                    finally:
                        print "closing", conn
                        conn.close()
        except KeyboardInterrupt:
            pass

    def recv(self, timeout=None):
        end = time.time() + timeout
        pipe = [0]
        while not self.radio.available(pipe) and (timeout is None or time.time() < end):
            time.sleep(10000 / 1e6)
        if self.radio.available(pipe):
            recv_buffer = []
            self.radio.read(recv_buffer)
            return recv_buffer
        return []

    def cleanup(self):
        self.radio.end()
        self.db.close()
        self.temperature.cleanup()
        self.sock.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class DBWriter:
    def __init__(self):
        self.conn = sqlite3.connect('/var/lib/autoboiler/autoboiler.sqlite3')
        self.conn.isolation_level = None
        self.c = self.conn.cursor()
        self.c.execute('''CREATE TABLE IF NOT EXISTS temperature
                          (date datetime, sensor integer, temperature real)''')
        self.c.execute('''CREATE INDEX IF NOT EXISTS temperature_date
                          ON temperature(date)''')

    def write(self, idx, value):
        line = "%s %d %f" % (datetime.datetime.now(), idx, value)
        if idx > 0:
            print '\033[%dC' % len(line * idx),
        print line, '\r',
        sys.stdout.flush()
        try:
            self.c.execute('''insert into temperature values (?, ?, ?)''',
                           (datetime.datetime.now(), idx, value))
        except sqlite3.OperationalError:
            pass

    def close(self):
        self.conn.commit()
        self.c.close()
        self.conn.close()

if __name__ == '__main__':
    GPIO.setmode(GPIO.BCM)
    parser = argparse.ArgumentParser()
    parser.add_argument('--mode', required=True, choices=['boiler', 'controller'])
    parser.add_argument('--pidfile',  '-p')
    args = parser.parse_args()
    with open('/etc/default/templog') as f:
        i = int(f.read())
    if args.pidfile:
        with open(args.pidfile, 'w') as f:
            print >>f, os.getpid()
    try:
        if args.mode == 'boiler':
            with Boiler(0, 0, 25, 24, Temperature(0, 1), Relay({0: 17, 1: 18})) as radio:
                radio.run()
        elif args.mode == 'controller':
            sockname = '/var/lib/autoboiler/autoboiler.socket'
            try:
                os.unlink(sockname)
            except OSError as e:
                if e.errno != errno.ENOENT and os.path.exists(sockname):
                    raise
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.bind(sockname)
            os.chmod(sockname, 0777)
            sock.setblocking(0)
            sock.listen(1)
            with Controller(0, 1, 25, 24, Temperature(0, 0), DBWriter(), sock) as radio:
                radio.run()
    finally:
        GPIO.cleanup()
        if args.pidfile:
            os.unlink(args.pidfile)
    sys.exit(0)

# vim: set et sw=4 ts=4 sts=4 ai:
<commit_msg>Change channel to 96 to avoid Wifi interference.<commit_after>#!/usr/bin/python

import RPi.GPIO as GPIO
import nrf24
import sys
import time
import spidev
import argparse
import os
import sqlite3
import datetime
import errno
import socket

PIPES = ([0xe7, 0xe7, 0xe7, 0xe7, 0xe7], [0xc2, 0xc2, 0xc2, 0xc2, 0xc2])

class Relay:
    def __init__(self, pins):
        self.pins = pins
        for pin in self.pins.values():
            GPIO.setup(pin, GPIO.OUT, initial=GPIO.HIGH)

    def output(self, pin, state):
        # print "setting pin", pin, state and "on" or "off"
        GPIO.output(self.pins[pin], not state)  # These devices are active-low.

    def cleanup(self):
        pass  # this will be done later: GPIO.cleanup()

class Temperature:
    def __init__(self, major=0, minor=0):
        self.spi = spidev.SpiDev()
        self.spi.open(major, minor)

    def rawread(self):
        return self.spi.xfer2([0, 0])

    def read(self):
        return self.calc_temp(self.rawread())

    @staticmethod
    def calc_temp(buf):
        return (((buf[0] << 8) | buf[1]) >> 3) * 0.0625

    def cleanup(self):
        self.spi.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class Boiler:
    def __init__(self, major, minor, ce_pin, irq_pin, temperature, relay):
        self.relay = relay
        self.temperature = temperature
        self.radio = nrf24.NRF24()
        self.radio.begin(major, minor, ce_pin, irq_pin)
        self.radio.setChannel(96)
        self.radio.enableDynamicPayloads()
        self.radio.printDetails()
        self.radio.openWritingPipe(PIPES[0])
        self.radio.openReadingPipe(1, PIPES[1])
        self.radio.setAutoAck(1)

    def run(self):
        while True:
            try:
                self.radio.startListening()
                recv_buffer = self.recv(10)
                for byte in recv_buffer:
                    pin = byte >> 1
                    state = byte & 0x1
                    self.relay.output(pin, state)
                self.radio.stopListening()
                self.radio.write(self.temperature.rawread())
            except Exception:
                pass

    def recv(self, timeout=None):
        end = time.time() + timeout
        pipe = [0]
        while not self.radio.available(pipe) and (timeout is None or time.time() < end):
            time.sleep(10000 / 1e6)
        if self.radio.available(pipe):
            recv_buffer = []
            self.radio.read(recv_buffer)
            return recv_buffer
        return []

    def cleanup(self):
        self.radio.end()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class Controller:
    def __init__(self, major, minor, ce_pin, irq_pin, temperature, db, sock):
        self.temperature = temperature
        self.db = db
        self.sock = sock
        self.radio = nrf24.NRF24()
        self.radio.begin(major, minor, ce_pin, irq_pin)
        self.radio.setChannel(96)
        self.radio.enableDynamicPayloads()
        self.radio.printDetails()
        self.radio.openWritingPipe(PIPES[0])
        self.radio.openReadingPipe(1, PIPES[1])

    def run(self):
        try:
            while True:
                self.radio.startListening()
                recv_buffer = self.recv(10)
                self.radio.stopListening()
                if recv_buffer:
                    self.db.write(1, self.temperature.calc_temp(recv_buffer))
                self.db.write(0, self.temperature.read())
                try:
                    conn, addr = sock.accept()
                except socket.error as e:
                    if e.errno != errno.EAGAIN:
                        raise
                else:
                    print conn
                    conn.settimeout(10)
                    try:
                        recv_line = conn.recv(1024)
                        print repr(recv_line)
                        if recv_line[-1] == '\n':
                            state, pin = recv_line[:-1].split()
                            cmd = int(pin) << 1 | (state.lower() == 'on')
                            print "sending", cmd
                            self.radio.write(chr(cmd))
                    finally:
                        print "closing", conn
                        conn.close()
        except KeyboardInterrupt:
            pass

    def recv(self, timeout=None):
        end = time.time() + timeout
        pipe = [0]
        while not self.radio.available(pipe) and (timeout is None or time.time() < end):
            time.sleep(10000 / 1e6)
        if self.radio.available(pipe):
            recv_buffer = []
            self.radio.read(recv_buffer)
            return recv_buffer
        return []

    def cleanup(self):
        self.radio.end()
        self.db.close()
        self.temperature.cleanup()
        self.sock.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.cleanup()

class DBWriter:
    def __init__(self):
        self.conn = sqlite3.connect('/var/lib/autoboiler/autoboiler.sqlite3')
        self.conn.isolation_level = None
        self.c = self.conn.cursor()
        self.c.execute('''CREATE TABLE IF NOT EXISTS temperature
                          (date datetime, sensor integer, temperature real)''')
        self.c.execute('''CREATE INDEX IF NOT EXISTS temperature_date
                          ON temperature(date)''')

    def write(self, idx, value):
        line = "%s %d %f" % (datetime.datetime.now(), idx, value)
        if idx > 0:
            print '\033[%dC' % len(line * idx),
        print line, '\r',
        sys.stdout.flush()
        try:
            self.c.execute('''insert into temperature values (?, ?, ?)''',
                           (datetime.datetime.now(), idx, value))
        except sqlite3.OperationalError:
            pass

    def close(self):
        self.conn.commit()
        self.c.close()
        self.conn.close()

if __name__ == '__main__':
    GPIO.setmode(GPIO.BCM)
    parser = argparse.ArgumentParser()
    parser.add_argument('--mode', required=True, choices=['boiler', 'controller'])
    parser.add_argument('--pidfile',  '-p')
    args = parser.parse_args()
    with open('/etc/default/templog') as f:
        i = int(f.read())
    if args.pidfile:
        with open(args.pidfile, 'w') as f:
            print >>f, os.getpid()
    try:
        if args.mode == 'boiler':
            with Boiler(0, 0, 25, 24, Temperature(0, 1), Relay({0: 17, 1: 18})) as radio:
                radio.run()
        elif args.mode == 'controller':
            sockname = '/var/lib/autoboiler/autoboiler.socket'
            try:
                os.unlink(sockname)
            except OSError as e:
                if e.errno != errno.ENOENT and os.path.exists(sockname):
                    raise
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.bind(sockname)
            os.chmod(sockname, 0777)
            sock.setblocking(0)
            sock.listen(1)
            with Controller(0, 1, 25, 24, Temperature(0, 0), DBWriter(), sock) as radio:
                radio.run()
    finally:
        GPIO.cleanup()
        if args.pidfile:
            os.unlink(args.pidfile)
    sys.exit(0)

# vim: set et sw=4 ts=4 sts=4 ai:
<|endoftext|>


<commit_before>#!/usr/bin/python3
import json
import os
import re
import struct
import subprocess

def get_next_part_meta(img, imgsize):
    if img.tell() == imgsize:
        return None
    pathlen = struct.unpack('!H', img.read(2))[0]
    mountpoint = img.read(pathlen).decode('utf8')
    jsonlen = struct.unpack('!I', img.read(4))[0]
    metadata = json.loads(img.read(jsonlen).decode('utf8'))
    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip filesystem type
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip orig devname (redundant with json)
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip padding
    nextlen = struct.unpack('!Q', img.read(8))[0]
    img.seek(nextlen, 1)  # go to next section
    return metadata

def get_multipart_image_meta(img):
    img.seek(0, 2)
    imgsize = img.tell()
    img.seek(31)
    partinfo = get_next_part_meta(img, imgsize)
    while partinfo:
        yield partinfo
        partinfo = get_next_part_meta(img, imgsize)

def get_image_metadata(imgpath):
    with open(imgpath, 'rb') as img:
        header = img.read(16)
        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
            for md in get_multipart_image_meta(img):
                yield md
        else:
            raise Exception('Installation from single part image not supported')

class PartedRunner():
    def __init__(self, disk):
        self.disk = disk

    def run(self, command):
        command = command.split()
        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
        return subprocess.check_output(command).decode('utf8')

def fixup(rootdir, vols):
    fstabfile = os.path.join(rootdir, 'etc/fstab')
    with open(fstabfile) as tfile:
        fstab = tfile.read().split('\n')

def had_swap():
    with open('/etc/fstab') as tabfile:
        tabs = tabfile.read().split('\n')
        for tab in tabs:
            tab = tab.split()
            if len(tab) < 3:
                continue
            if tab[2] == 'swap':
                return True
    return False

def install_to_disk(imgpath):
    lvmvols = {}
    deftotsize = 0
    mintotsize = 0
    deflvmsize = 0
    minlvmsize = 0
    biggestsize = 0
    biggestfs = None
    plainvols = {}
    allvols = []
    swapsize = 0
    if had_swap():
        with open('/proc/meminfo') as meminfo:
            swapsize = meminfo.read().split('\n')[0]
        swapsize = int(swapsize.split()[1])
        if swapsize < 2097152:
            swapsize = swapsize * 2
        elif swapsize > 8388608 and swapsize < 67108864:
            swapsize = swapsize * 0.5
        elif swapsize >= 67108864:
            swapsize = 33554432
        swapsize = swapsize * 1024
    deftotsize = swapsize
    mintotsize = swapsize
    for fs in get_image_metadata('/run/imginst/sourceimage/rootimg.sfs'):
        allvols.append(fs)
        deftotsize += fs['initsize']
        mintotsize += fs['minsize']
        if fs['initsize'] > biggestsize:
            biggestfs = fs
            biggestsize = fs['initsize']
        if fs['device'].startswith('/dev/mapper'):
            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
            deflvmsize += fs['initsize']
            minlvmsize += fs['minsize']
        else:
            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
    with open('/tmp/installdisk') as diskin:
        instdisk = diskin.read()
    instdisk = '/dev/' + instdisk
    parted = PartedRunner(instdisk)
    dinfo = parted.run('unit s print')
    dinfo = dinfo.split('\n')
    sectors = 0
    sectorsize = 0
    for inf in dinfo:
        if inf.startswith('Disk {0}:'.format(instdisk)):
            _, sectors = inf.split(': ')
            sectors = int(sectors.replace('s', ''))
        if inf.startswith('Sector size (logical/physical):'):
            _, sectorsize = inf.split(':')
            sectorsize = sectorsize.split('/')[0]
            sectorsize = sectorsize.replace('B', '')
            sectorsize = int(sectorsize)
    # for now, only support resizing/growing the largest partition
    minexcsize = deftotsize - biggestfs['initsize']
    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
    minsectors = mintotsize // sectorsize
    if sectors < (minsectors + 65536):
        raise Exception('Disk too small to fit image')
    biggestsectors = sectors - (minexcsize // sectorsize)
    biggestsize = sectorsize * biggestsectors
    parted.run('mklabel gpt')
    curroffset = 2048
    for volidx in sorted(plainvols):
        vol = plainvols[volidx]
        if vol is not biggestfs:
            size = vol['initsize'] // sectorsize
        else:
            size = biggestsize // sectorsize
        size += 2047 - (size % 2048)
        end = curroffset + size
        if end > sectors:
            end = sectors
        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
        curroffset += size + 1
    if not lvmvols:
        if swapsize:
            swapsize = swapsize // sectorsize
            swapsize += 2047 - (size % 2048)
            end = curroffset + swapsize
            if end > sectors:
                end = sectors
            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
    else:
        parted.run('mkpart lvm {}s 100%'.format(curroffset))
        lvmpart = instdisk + '{}'.format(volidx + 1)
        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
        vginfo = vginfo.split('\n')
        pesize = 0
        pes = 0
        for infline in vginfo:
            infline = infline.split()
            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
                pesize = int(infline[2])
            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
                pes = int(infline[4])
        takeaway = swapsize // pesize
        for volidx in lvmvols:
            vol = lvmvols[volidx]
            if vol is biggestfs:
                continue
            takeaway += vol['initsize'] // pesize
            takeaway += 1
        biggestextents = pes - takeaway
        for volidx in lvmvols:
            vol = lvmvols[volidx]
            if vol is biggestfs:
                extents = biggestextents
            else:
                extents = vol['initsize'] // pesize
                extents += 1
            if vol['mount'] == '/':
                lvname = 'root'
            else:
                lvname = vol['mount'].replace('/', '_')
            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
        if swapsize:
            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
        os.makedirs('/run/imginst/targ')
        for vol in allvols:
            with open(vol['targetdisk'], 'wb') as partition:
                partition.write(b'\x00' * 1 * 1024 * 1024)
            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
            source = vol['mount'].replace('/', '_')
            source = '/run/imginst/sources/' + source
            subprocess.check_call(['cp', '-ax', source + '/.', '/run/imginst/targ'])
            subprocess.check_call(['umount', '/run/imginst/targ'])
        for vol in allvols:
            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
        fixup('/run/imginst/targ', allvols)

if __name__ == '__main__':
    install_to_disk('/run/imginst/sourceimage/rootimg.sfs')<commit_msg>Start fixup, first up is the fstab<commit_after>#!/usr/bin/python3
import json
import os
import re
import struct
import subprocess

def get_next_part_meta(img, imgsize):
    if img.tell() == imgsize:
        return None
    pathlen = struct.unpack('!H', img.read(2))[0]
    mountpoint = img.read(pathlen).decode('utf8')
    jsonlen = struct.unpack('!I', img.read(4))[0]
    metadata = json.loads(img.read(jsonlen).decode('utf8'))
    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip filesystem type
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip orig devname (redundant with json)
    nextlen = struct.unpack('!H', img.read(2))[0]
    img.seek(nextlen, 1) # skip padding
    nextlen = struct.unpack('!Q', img.read(8))[0]
    img.seek(nextlen, 1)  # go to next section
    return metadata

def get_multipart_image_meta(img):
    img.seek(0, 2)
    imgsize = img.tell()
    img.seek(31)
    partinfo = get_next_part_meta(img, imgsize)
    while partinfo:
        yield partinfo
        partinfo = get_next_part_meta(img, imgsize)

def get_image_metadata(imgpath):
    with open(imgpath, 'rb') as img:
        header = img.read(16)
        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
            for md in get_multipart_image_meta(img):
                yield md
        else:
            raise Exception('Installation from single part image not supported')

class PartedRunner():
    def __init__(self, disk):
        self.disk = disk

    def run(self, command):
        command = command.split()
        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
        return subprocess.check_output(command).decode('utf8')

def fixup(rootdir, vols):
    devbymount = {}
    for vol in vols:
        devbymount[vol['mount']] = vol['targetdisk']
    fstabfile = os.path.join(rootdir, 'etc/fstab')
    with open(fstabfile) as tfile:
        fstab = tfile.read().split('\n')
    while not fstab[0]:
        fstab = fstab[1:]
    with open(fstabfile, 'w') as tfile:
        for tab in fstab:
            entry = tab.split()
            if tab.startswith('#ORIGFSTAB#'):
                if entry[1] in devbymount:
                    targetdev = devbymount[entry[1]]
                    if targetdev.startswith('/dev/localstorage/'):
                        entry[0] = targetdev
                    else:
                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
                        uuid = uuid.strip()
                        entry[0] = 'UUID={}'.format(uuid)
                elif entry[2] == 'swap':
                    entry[0] = '/dev/mapper/localstorage-swap'
                entry[0] = entry[0].ljust(42)
                entry[1] = entry[1].ljust(16)
                entry[3] = entry[3].ljust(28)
                tab = '\t'.join(entry)
            tfile.write(tab + '\n')

def had_swap():
    with open('/etc/fstab') as tabfile:
        tabs = tabfile.read().split('\n')
        for tab in tabs:
            tab = tab.split()
            if len(tab) < 3:
                continue
            if tab[2] == 'swap':
                return True
    return False

def install_to_disk(imgpath):
    lvmvols = {}
    deftotsize = 0
    mintotsize = 0
    deflvmsize = 0
    minlvmsize = 0
    biggestsize = 0
    biggestfs = None
    plainvols = {}
    allvols = []
    swapsize = 0
    if had_swap():
        with open('/proc/meminfo') as meminfo:
            swapsize = meminfo.read().split('\n')[0]
        swapsize = int(swapsize.split()[1])
        if swapsize < 2097152:
            swapsize = swapsize * 2
        elif swapsize > 8388608 and swapsize < 67108864:
            swapsize = swapsize * 0.5
        elif swapsize >= 67108864:
            swapsize = 33554432
        swapsize = swapsize * 1024
    deftotsize = swapsize
    mintotsize = swapsize
    for fs in get_image_metadata('/run/imginst/sourceimage/rootimg.sfs'):
        allvols.append(fs)
        deftotsize += fs['initsize']
        mintotsize += fs['minsize']
        if fs['initsize'] > biggestsize:
            biggestfs = fs
            biggestsize = fs['initsize']
        if fs['device'].startswith('/dev/mapper'):
            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
            deflvmsize += fs['initsize']
            minlvmsize += fs['minsize']
        else:
            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
    with open('/tmp/installdisk') as diskin:
        instdisk = diskin.read()
    instdisk = '/dev/' + instdisk
    parted = PartedRunner(instdisk)
    dinfo = parted.run('unit s print')
    dinfo = dinfo.split('\n')
    sectors = 0
    sectorsize = 0
    for inf in dinfo:
        if inf.startswith('Disk {0}:'.format(instdisk)):
            _, sectors = inf.split(': ')
            sectors = int(sectors.replace('s', ''))
        if inf.startswith('Sector size (logical/physical):'):
            _, sectorsize = inf.split(':')
            sectorsize = sectorsize.split('/')[0]
            sectorsize = sectorsize.replace('B', '')
            sectorsize = int(sectorsize)
    # for now, only support resizing/growing the largest partition
    minexcsize = deftotsize - biggestfs['initsize']
    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
    minsectors = mintotsize // sectorsize
    if sectors < (minsectors + 65536):
        raise Exception('Disk too small to fit image')
    biggestsectors = sectors - (minexcsize // sectorsize)
    biggestsize = sectorsize * biggestsectors
    parted.run('mklabel gpt')
    curroffset = 2048
    for volidx in sorted(plainvols):
        vol = plainvols[volidx]
        if vol is not biggestfs:
            size = vol['initsize'] // sectorsize
        else:
            size = biggestsize // sectorsize
        size += 2047 - (size % 2048)
        end = curroffset + size
        if end > sectors:
            end = sectors
        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
        curroffset += size + 1
    if not lvmvols:
        if swapsize:
            swapsize = swapsize // sectorsize
            swapsize += 2047 - (size % 2048)
            end = curroffset + swapsize
            if end > sectors:
                end = sectors
            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
    else:
        parted.run('mkpart lvm {}s 100%'.format(curroffset))
        lvmpart = instdisk + '{}'.format(volidx + 1)
        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
        vginfo = vginfo.split('\n')
        pesize = 0
        pes = 0
        for infline in vginfo:
            infline = infline.split()
            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
                pesize = int(infline[2])
            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
                pes = int(infline[4])
        takeaway = swapsize // pesize
        for volidx in lvmvols:
            vol = lvmvols[volidx]
            if vol is biggestfs:
                continue
            takeaway += vol['initsize'] // pesize
            takeaway += 1
        biggestextents = pes - takeaway
        for volidx in lvmvols:
            vol = lvmvols[volidx]
            if vol is biggestfs:
                extents = biggestextents
            else:
                extents = vol['initsize'] // pesize
                extents += 1
            if vol['mount'] == '/':
                lvname = 'root'
            else:
                lvname = vol['mount'].replace('/', '_')
            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
        if swapsize:
            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
        os.makedirs('/run/imginst/targ')
        for vol in allvols:
            with open(vol['targetdisk'], 'wb') as partition:
                partition.write(b'\x00' * 1 * 1024 * 1024)
            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
            source = vol['mount'].replace('/', '_')
            source = '/run/imginst/sources/' + source
            subprocess.check_call(['cp', '-ax', source + '/.', '/run/imginst/targ'])
            subprocess.check_call(['umount', '/run/imginst/targ'])
        for vol in allvols:
            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
        fixup('/run/imginst/targ', allvols)

if __name__ == '__main__':
    install_to_disk('/run/imginst/sourceimage/rootimg.sfs')<|endoftext|>


<commit_before>"""
Django settings for {{ cookiecutter.repo_name }} project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

from os.path import abspath, basename, dirname, join, normpath
from sys import path

# Absolute filesystem path to the Django project directory:
DJANGO_ROOT = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
PROJECT_ROOT = dirname(DJANGO_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(DJANGO_ROOT)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# Do not set SECRET_KEY or LDAP password or any other sensitive data here.
# Instead, create a local.py file on the server.

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'compressor',
    'taggit',
    'modelcluster',

    'wagtail.wagtailcore',
    'wagtail.wagtailadmin',
    'wagtail.wagtaildocs',
    'wagtail.wagtailsnippets',
    'wagtail.wagtailusers',
    'wagtail.wagtailsites',
    'wagtail.wagtailimages',
    'wagtail.wagtailembeds',
    'wagtail.wagtailsearch',
    'wagtail.wagtailredirects',
    'wagtail.wagtailforms',

    'utils',
    'core',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',

    'wagtail.wagtailcore.middleware.SiteMiddleware',
    'wagtail.wagtailredirects.middleware.RedirectMiddleware',
)

ROOT_URLCONF = '{{ cookiecutter.repo_name }}.urls'
WSGI_APPLICATION = '{{ cookiecutter.repo_name }}.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': '{{ cookiecutter.repo_name }}',
        'USER': '',
        'HOST': '',  # Set to empty string for localhost.
        'PORT': '',  # Set to empty string for default.
        'CONN_MAX_AGE': 600,  # number of seconds database connections should persist for
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-gb'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_ROOT = join(PROJECT_ROOT, 'static')
STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
)

STATICFILES_DIRS = (
    join(DJANGO_ROOT, 'static'),
)

MEDIA_ROOT = join(PROJECT_ROOT, 'media')
MEDIA_URL = '/media/'

# Django compressor settings
# http://django-compressor.readthedocs.org/en/latest/settings/

COMPRESS_PRECOMPILERS = (
    ('text/x-scss', 'django_libsass.SassCompiler'),
)

# Template configuration

from django.conf import global_settings

TEMPLATE_CONTEXT_PROCESSORS = global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
    'django.core.context_processors.request',
)

TEMPLATE_DIRS = (
    normpath(join(DJANGO_ROOT, 'templates')),
)

# Use Redis as the cache backend for extra performance

CACHES = {
    'default': {
        'BACKEND': 'redis_cache.cache.RedisCache',
        'LOCATION': '127.0.0.1:6379',
        'KEY_PREFIX': '{{ cookiecutter.repo_name }}',
        'OPTIONS': {
            'CLIENT_CLASS': 'redis_cache.client.DefaultClient',
        }
    }
}

# Wagtail settings

LOGIN_URL = 'wagtailadmin_login'
LOGIN_REDIRECT_URL = 'wagtailadmin_home'

WAGTAIL_SITE_NAME = "{{ cookiecutter.project_name }}"

# Use Elasticsearch as the search backend for extra performance and better search results
WAGTAILSEARCH_BACKENDS = {
    'default': {
        'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
        'INDEX': '{{ cookiecutter.repo_name }}',
    },
}

# Celery settings
# When you have multiple sites using the same Redis server,
# specify a different Redis DB. e.g. redis://localhost/5

BROKER_URL = 'redis://'

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERYD_LOG_COLOR = False
<commit_msg>commented redis and elastic search<commit_after>"""
Django settings for {{ cookiecutter.repo_name }} project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

from os.path import abspath, basename, dirname, join, normpath
from sys import path

# Absolute filesystem path to the Django project directory:
DJANGO_ROOT = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
PROJECT_ROOT = dirname(DJANGO_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(DJANGO_ROOT)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# Do not set SECRET_KEY or LDAP password or any other sensitive data here.
# Instead, create a local.py file on the server.

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'compressor',
    'taggit',
    'modelcluster',

    'wagtail.wagtailcore',
    'wagtail.wagtailadmin',
    'wagtail.wagtaildocs',
    'wagtail.wagtailsnippets',
    'wagtail.wagtailusers',
    'wagtail.wagtailsites',
    'wagtail.wagtailimages',
    'wagtail.wagtailembeds',
    'wagtail.wagtailsearch',
    'wagtail.wagtailredirects',
    'wagtail.wagtailforms',

    'utils',
    'core',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',

    'wagtail.wagtailcore.middleware.SiteMiddleware',
    'wagtail.wagtailredirects.middleware.RedirectMiddleware',
)

ROOT_URLCONF = '{{ cookiecutter.repo_name }}.urls'
WSGI_APPLICATION = '{{ cookiecutter.repo_name }}.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': '{{ cookiecutter.repo_name }}',
        'USER': '',
        'HOST': '',  # Set to empty string for localhost.
        'PORT': '',  # Set to empty string for default.
        'CONN_MAX_AGE': 600,  # number of seconds database connections should persist for
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-gb'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

STATIC_ROOT = join(PROJECT_ROOT, 'static')
STATIC_URL = '/static/'

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'compressor.finders.CompressorFinder',
)

STATICFILES_DIRS = (
    join(DJANGO_ROOT, 'static'),
)

MEDIA_ROOT = join(PROJECT_ROOT, 'media')
MEDIA_URL = '/media/'

# Django compressor settings
# http://django-compressor.readthedocs.org/en/latest/settings/

COMPRESS_PRECOMPILERS = (
    ('text/x-scss', 'django_libsass.SassCompiler'),
)

# Template configuration

from django.conf import global_settings

TEMPLATE_CONTEXT_PROCESSORS = global_settings.TEMPLATE_CONTEXT_PROCESSORS + (
    'django.core.context_processors.request',
)

TEMPLATE_DIRS = (
    normpath(join(DJANGO_ROOT, 'templates')),
)

# Use Redis as the cache backend for extra performance

# CACHES = {
#     'default': {
#         'BACKEND': 'redis_cache.cache.RedisCache',
#         'LOCATION': '127.0.0.1:6379',
#         'KEY_PREFIX': '{{ cookiecutter.repo_name }}',
#         'OPTIONS': {
#             'CLIENT_CLASS': 'redis_cache.client.DefaultClient',
#         }
#     }
# }

# Wagtail settings

LOGIN_URL = 'wagtailadmin_login'
LOGIN_REDIRECT_URL = 'wagtailadmin_home'

WAGTAIL_SITE_NAME = "{{ cookiecutter.project_name }}"

# Use Elasticsearch as the search backend for extra performance and better search results
# WAGTAILSEARCH_BACKENDS = {
#     'default': {
#         'BACKEND': 'wagtail.wagtailsearch.backends.elasticsearch.ElasticSearch',
#         'INDEX': '{{ cookiecutter.repo_name }}',
#     },
# }

# Celery settings
# When you have multiple sites using the same Redis server,
# specify a different Redis DB. e.g. redis://localhost/5

BROKER_URL = 'redis://'

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERYD_LOG_COLOR = False
<|endoftext|>