<commit_before>julia_package_env = {
    'CFLAGS':None,
    'CPPFLAGS': None,
    'LLVM_CMAKE': util.Property('llvm_cmake', default=None),
}

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_mac,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s cleanall")],
        env=julia_package_env,
    ),

    # Windows has a nasty habit of leaving julia-<gitsha> folders lying around....
    steps.ShellCommand(
        name="Clean out temporary windows cruft",
        command=["/bin/bash", "-c", "rm -rf julia-*"],
        doStepIf=is_windows,
        flunkOnFailure=False,
    ),

    # Clear out old .cov and .mem files.  Note that this should be removed once we always start from scratch.
    steps.ShellCommand(
        name="Clear out usr",
        command=["/bin/bash", "-c", "find . \( -name *.jl.*.cov -o -name *.jl.mem \) -print -delete"],
        env=julia_package_env,
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    steps.ShellCommand(
        name="make release",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s %(prop:extra_make_flags)s release")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),
    steps.ShellCommand(
        name="make debug",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s %(prop:extra_make_flags)s debug")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Make win-extras on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env=julia_package_env,
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get build artifact filename",
        command=["make", "print-JULIA_BINARYDIST_FILENAME"],
        property="artifact_filename",
    ),
    steps.SetPropertyFromCommand(
        name="Munge artifact filename",
        command=munge_artifact_filename,
        property="dummy",
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # On OSX, deal with non-sf/consistent_distnames makefile nonsense by wrapping up all
    # the complexity into `render_make_app`.
    steps.ShellCommand(
        name="make .app",
        command=render_make_app,
        haltOnFailure = True,
        doStepIf=is_mac,
        env=julia_package_env,
    ),

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),

    steps.FileUpload(
        workersrc=util.Interpolate("%(prop:local_filename)s"),
        masterdest=util.Interpolate("/tmp/julia_package/%(prop:upload_filename)s")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=render_upload_command,
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=render_latest_upload_command,
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/%(prop:upload_filename)s")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another worker for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties={
            'url': render_download_url,
            'commitmessage': util.Property('commitmessage'),
            'commitname': util.Property('commitname'),
            'commitemail': util.Property('commitemail'),
            'authorname': util.Property('authorname'),
            'authoremail': util.Property('authoremail'),
            'shortcommit': util.Property('shortcommit'),
        },
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Build a builder-worker mapping based off of the parent mapping in inventory.py
packager_mapping = {("package_" + k): v for k, v in builder_mapping.iteritems()}

# Add a few builders that don't exist in the typical mapping
packager_mapping["build_ubuntu32"] = "ubuntu16_04-x86"
packager_mapping["build_ubuntu64"] = "ubuntu16_04-x64"
packager_mapping["build_centos64"] = "centos7_3-x64"

packager_scheduler = schedulers.AnyBranchScheduler(
    name="Julia binary packaging",
    change_filter=util.ChangeFilter(
        project=['JuliaLang/julia','staticfloat/julia'],
        # Only build `master` or `release-*`
        branch_fn=lambda b: b == "master" or b.startswith("release-")
    ),
    builderNames=packager_mapping.keys(),
    treeStableTimer=1
)
c['schedulers'].append(packager_scheduler)

for packager, worker in packager_mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[worker],
        collapseRequests=False,
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="package",
    label="Force Julia build/packaging",
    builderNames=packager_mapping.keys(),
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[
        util.StringParameter(name="extra_make_flags", label="Extra Make Flags", size=30, default=""),
    ]
)
c['schedulers'].append(force_build_scheduler)
<commit_msg>move win-extra step early (#66)<commit_after>julia_package_env = {
    'CFLAGS':None,
    'CPPFLAGS': None,
    'LLVM_CMAKE': util.Property('llvm_cmake', default=None),
}

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_mac,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s cleanall")],
        env=julia_package_env,
    ),

    # Windows has a nasty habit of leaving julia-<gitsha> folders lying around....
    steps.ShellCommand(
        name="Clean out temporary windows cruft",
        command=["/bin/bash", "-c", "rm -rf julia-*"],
        doStepIf=is_windows,
        flunkOnFailure=False,
    ),

    # Clear out old .cov and .mem files.  Note that this should be removed once we always start from scratch.
    steps.ShellCommand(
        name="Clear out usr",
        command=["/bin/bash", "-c", "find . \( -name *.jl.*.cov -o -name *.jl.mem \) -print -delete"],
        env=julia_package_env,
    ),

    # Get win-extras files ready on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env=julia_package_env,
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    steps.ShellCommand(
        name="make release",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s %(prop:extra_make_flags)s release")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),
    steps.ShellCommand(
        name="make debug",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s %(prop:extra_make_flags)s debug")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get build artifact filename",
        command=["make", "print-JULIA_BINARYDIST_FILENAME"],
        property="artifact_filename",
    ),
    steps.SetPropertyFromCommand(
        name="Munge artifact filename",
        command=munge_artifact_filename,
        property="dummy",
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s %(prop:extra_make_flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # On OSX, deal with non-sf/consistent_distnames makefile nonsense by wrapping up all
    # the complexity into `render_make_app`.
    steps.ShellCommand(
        name="make .app",
        command=render_make_app,
        haltOnFailure = True,
        doStepIf=is_mac,
        env=julia_package_env,
    ),

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),

    steps.FileUpload(
        workersrc=util.Interpolate("%(prop:local_filename)s"),
        masterdest=util.Interpolate("/tmp/julia_package/%(prop:upload_filename)s")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=render_upload_command,
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=render_latest_upload_command,
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/%(prop:upload_filename)s")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another worker for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties={
            'url': render_download_url,
            'commitmessage': util.Property('commitmessage'),
            'commitname': util.Property('commitname'),
            'commitemail': util.Property('commitemail'),
            'authorname': util.Property('authorname'),
            'authoremail': util.Property('authoremail'),
            'shortcommit': util.Property('shortcommit'),
        },
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Build a builder-worker mapping based off of the parent mapping in inventory.py
packager_mapping = {("package_" + k): v for k, v in builder_mapping.iteritems()}

# Add a few builders that don't exist in the typical mapping
packager_mapping["build_ubuntu32"] = "ubuntu16_04-x86"
packager_mapping["build_ubuntu64"] = "ubuntu16_04-x64"
packager_mapping["build_centos64"] = "centos7_3-x64"

packager_scheduler = schedulers.AnyBranchScheduler(
    name="Julia binary packaging",
    change_filter=util.ChangeFilter(
        project=['JuliaLang/julia','staticfloat/julia'],
        # Only build `master` or `release-*`
        branch_fn=lambda b: b == "master" or b.startswith("release-")
    ),
    builderNames=packager_mapping.keys(),
    treeStableTimer=1
)
c['schedulers'].append(packager_scheduler)

for packager, worker in packager_mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[worker],
        collapseRequests=False,
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="package",
    label="Force Julia build/packaging",
    builderNames=packager_mapping.keys(),
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[
        util.StringParameter(name="extra_make_flags", label="Extra Make Flags", size=30, default=""),
    ]
)
c['schedulers'].append(force_build_scheduler)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (C) 2013 Matthias Klumpp <mak@debian.org>
#
# Licensed under the GNU General Public License Version 3
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import gzip
import os.path
from ConfigParser import SafeConfigParser
from apt_pkg import TagFile, TagSection

class PackageInfo():
    def __init__(self, pkgname, pkgversion, dist, component, archs):
        self.pkgname = pkgname
        self.version = pkgversion
        self.dist = dist
        self.component = component
        self.archs = archs
        self.info = ""
        self.installedArchs = []

    def getVersionNoEpoch():
        v = self.version
        if ":" in v:
            return v[v.index(":")+1:]
        else:
            return v

    def __str__(self):
        return "PackageInfo_Obj: name: %s | version: %s | dist: %s | comp.: %s | archs: %s" % (self.pkgname, self.version, self.dist, self.component, self.archs)

class PackageInfoRetriever():
    def __init__(self):
        parser = SafeConfigParser()
        parser.read('jenkins-dak.conf')
        path = parser.get('Archive', 'path')
        self._archivePath = path
        self._archiveComponents = parser.get('Archive', 'components').split (" ")
        self._archiveDists = parser.get('Archive', 'dists').split (" ")
        self._supportedArchs = parser.get('Archive', 'archs').split (" ")
        self._supportedArchs += ["all"]

    def _setPkgInstalledForArch(self, dirname, pkg, binaryName):
        fileExt = "deb"
        if "-udeb" in binaryName:
            fileExt = "udeb"

        for arch in self._supportedArchs:
            if arch in pkg.installedArchs:
                continue
            binaryPkgName = "%s_%s_%s.%s" % (binaryName, pkg.getVersionNoEpoch, arch, fileExt)
            expectedPackagePath = self._archivePath + "/%s/%s" % (dirname, binaryPkgName)

            if os.path.isfile(expectedPackagePath):
                pkg.installedArchs += [arch]

    def _getPackagesFor(self, dist, component):
        source_path = self._archivePath + "/dists/%s/%s/source/Sources.gz" % (dist, component)
        f = gzip.open(source_path, 'rb')
        tagf = TagFile (f)
        packageList = []
        for section in tagf:
            # don't even try to build source-only packages
            if section.get('Extra-Source-Only', 'no') == 'yes':
                pass

            archs = section['Architecture']
            binaries = section['Binary']
            pkgversion = section['Version']
            pkgname = section['Package']
            pkg = PackageInfo(pkgname, pkgversion, dist, component, archs)

            pkg.info = ("Package: %s\nBinary Packages: %s\nMaintainer: %s\nCo-Maintainers: %s\nVCS-Browser: %s" %
                        (pkgname, binaries, section['Maintainer'], section.get('Uploaders', 'Nobody'), section.get('Vcs-Browser', 'None set')))

            # we check if one of the arch-binaries exists. if it does, we consider the package built for this architecture
            # FIXME: This does not work well for binNMUed packages! Implement a possible solution later.
            # (at time, a version-check prevents packages from being built twice)
            if "," in binaries:
                binaryPkgs = binaries.split(', ')
            else:
                binaryPkgs = [binaries]
            for binaryName in binaryPkgs:
                self._setPkgInstalledForArch(section["Directory"], pkg, binaryName)
                #if (pkg.installedArchs != ["all"]) or (len(binaryPkgs) <= 0:

            packageList += [pkg]

        return packageList

    def getAllPackages(self):
        packageList = []
        for dist in self._archiveDists:
            for comp in self._archiveComponents:
                packageList += self._getPackagesFor(dist, comp)
        return packageList

<commit_msg>trivial: Fix pkginfo code<commit_after>#!/usr/bin/python
# Copyright (C) 2013 Matthias Klumpp <mak@debian.org>
#
# Licensed under the GNU General Public License Version 3
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import gzip
import os.path
from ConfigParser import SafeConfigParser
from apt_pkg import TagFile, TagSection

class PackageInfo():
    def __init__(self, pkgname, pkgversion, dist, component, archs):
        self.pkgname = pkgname
        self.version = pkgversion
        self.dist = dist
        self.component = component
        self.archs = archs
        self.info = ""
        self.installedArchs = []

    def getVersionNoEpoch(self):
        v = self.version
        if ":" in v:
            return v[v.index(":")+1:]
        else:
            return v

    def __str__(self):
        return "PackageInfo_Obj: name: %s | version: %s | dist: %s | comp.: %s | archs: %s" % (self.pkgname, self.version, self.dist, self.component, self.archs)

class PackageInfoRetriever():
    def __init__(self):
        parser = SafeConfigParser()
        parser.read('jenkins-dak.conf')
        path = parser.get('Archive', 'path')
        self._archivePath = path
        self._archiveComponents = parser.get('Archive', 'components').split (" ")
        self._archiveDists = parser.get('Archive', 'dists').split (" ")
        self._supportedArchs = parser.get('Archive', 'archs').split (" ")
        self._supportedArchs += ["all"]

    def _setPkgInstalledForArch(self, dirname, pkg, binaryName):
        fileExt = "deb"
        if "-udeb" in binaryName:
            fileExt = "udeb"

        for arch in self._supportedArchs:
            if arch in pkg.installedArchs:
                continue
            binaryPkgName = "%s_%s_%s.%s" % (binaryName, pkg.getVersionNoEpoch(), arch, fileExt)
            expectedPackagePath = self._archivePath + "/%s/%s" % (dirname, binaryPkgName)

            if os.path.isfile(expectedPackagePath):
                pkg.installedArchs += [arch]

    def _getPackagesFor(self, dist, component):
        source_path = self._archivePath + "/dists/%s/%s/source/Sources.gz" % (dist, component)
        f = gzip.open(source_path, 'rb')
        tagf = TagFile (f)
        packageList = []
        for section in tagf:
            # don't even try to build source-only packages
            if section.get('Extra-Source-Only', 'no') == 'yes':
                pass

            archs = section['Architecture']
            binaries = section['Binary']
            pkgversion = section['Version']
            pkgname = section['Package']
            pkg = PackageInfo(pkgname, pkgversion, dist, component, archs)

            pkg.info = ("Package: %s\nBinary Packages: %s\nMaintainer: %s\nCo-Maintainers: %s\nVCS-Browser: %s" %
                        (pkgname, binaries, section['Maintainer'], section.get('Uploaders', 'Nobody'), section.get('Vcs-Browser', 'None set')))

            # we check if one of the arch-binaries exists. if it does, we consider the package built for this architecture
            # FIXME: This does not work well for binNMUed packages! Implement a possible solution later.
            # (at time, a version-check prevents packages from being built twice)
            if "," in binaries:
                binaryPkgs = binaries.split(', ')
            else:
                binaryPkgs = [binaries]
            for binaryName in binaryPkgs:
                self._setPkgInstalledForArch(section["Directory"], pkg, binaryName)
                #if (pkg.installedArchs != ["all"]) or (len(binaryPkgs) <= 0:

            packageList += [pkg]

        return packageList

    def getAllPackages(self):
        packageList = []
        for dist in self._archiveDists:
            for comp in self._archiveComponents:
                packageList += self._getPackagesFor(dist, comp)
        return packageList

<|endoftext|>


<commit_before>from aiohttp import web
import asyncio
import time
import subprocess
import sys
import os

PRIVOXY_LOGFILE = '/var/log/privoxy/logfile'
MAX_IDLE_TIME = 30 * 60  # 30 minutes
LIFETIME = 12 * 60 * 60  # 12 hours
kill_at = time.time() + LIFETIME

async def handle_deathclock(request):
    data = await request.post()
    new_time = data.get('time')
    set_deathclock(new_time)
    return web.Response(text='OK')

async def handle_delete(request):
    set_deathclock(5)
    return web.Response(text='OK')

async def handle_newnym(request):
    # throw it to the wind.  It'll work!  If it don't, I'll rewrite this
    cmd = """echo -e 'authenticate ""\nsignal newnym\nquit\n' | nc localhost 9051"""
    subprocess.run(cmd, shell=True)
    return web.Response(text='OK')

def set_deathclock(seconds_from_now):
    global kill_at
    print("Setting deathclock to {}s".format(seconds_from_now)
    seconds_from_now = float(seconds_from_now)
    kill_at = time.time() + seconds_from_now

async def deathclock_loop():
    while True:
        if time.time() > kill_at:
            print("killing process due deathclock expiration")
            sys.stdout.flush()
            sys.exit(0)
        await asyncio.sleep(1)

async def log_loop(logfile):
    last_filesize = -1
    last_change_time = time.time()
    while True:
        try:
            filesize = os.stat(logfile).st_size
            if filesize != last_filesize:
                last_filesize = filesize
                last_change_time = time.time()
        except OSError:
            pass
        if last_change_time + MAX_IDLE_TIME < time.time():
            print("killing process due to unchanging log filesize")
            sys.stdout.flush()
            sys.exit(0)
        await asyncio.sleep(10)

def run_services():
    subprocess.run('service tor start', shell=True)
    subprocess.run('service privoxy start', shell=True)

def main():
    loop = asyncio.get_event_loop()
    app = web.Application(loop=loop)
    app.router.add_post('/deathclock', handle_deathclock)
    app.router.add_delete('/', handle_delete)
    app.router.add_get('/newnym', handle_newnym)
    run_services()
    loop.create_task(deathclock_loop())
    loop.create_task(log_loop(PRIVOXY_LOGFILE))
    web.run_app(app)

if __name__ == '__main__':
    main()
<commit_msg>default shell doesnt work as expected, shell out to bash instead, fix up typos<commit_after>import asyncio
import time
import subprocess
import sys
import os

from aiohttp import web

PRIVOXY_LOGFILE = '/var/log/privoxy/logfile'
MAX_IDLE_TIME = 30 * 60  # 30 minutes
LIFETIME = 12 * 60 * 60  # 12 hours
kill_at = time.time() + LIFETIME

async def handle_deathclock(request):
    data = await request.post()
    new_time = data.get('time')
    set_deathclock(new_time)
    return web.Response(text='OK')

async def handle_delete(request):
    set_deathclock(5)
    return web.Response(text='OK')

async def handle_newnym(request):
    # throw it to the wind.  It'll work!  If it don't, I'll rewrite this
    cmd = """echo -e 'authenticate ""\nsignal newnym\nquit\n' | nc localhost 9051"""
    subprocess.Popen(['/bin/bash', '-c', cmd])
    return web.Response(text='OK')

def set_deathclock(seconds_from_now):
    global kill_at
    print("Setting deathclock to {}s".format(seconds_from_now))
    seconds_from_now = float(seconds_from_now)
    kill_at = time.time() + seconds_from_now

async def deathclock_loop():
    while True:
        if time.time() > kill_at:
            print("killing process due deathclock expiration")
            sys.stdout.flush()
            sys.exit(0)
        await asyncio.sleep(1)

async def log_loop(logfile):
    last_filesize = -1
    last_change_time = time.time()
    while True:
        try:
            filesize = os.stat(logfile).st_size
            if filesize != last_filesize:
                last_filesize = filesize
                last_change_time = time.time()
        except OSError:
            pass
        if last_change_time + MAX_IDLE_TIME < time.time():
            print("killing process due to unchanging log filesize")
            sys.stdout.flush()
            sys.exit(0)
        await asyncio.sleep(10)

def run_services():
    subprocess.run('service tor start', shell=True)
    subprocess.run('service privoxy start', shell=True)

def main():
    loop = asyncio.get_event_loop()
    app = web.Application(loop=loop)
    app.router.add_post('/deathclock', handle_deathclock)
    app.router.add_delete('/', handle_delete)
    app.router.add_get('/newnym', handle_newnym)
    run_services()
    loop.create_task(deathclock_loop())
    loop.create_task(log_loop(PRIVOXY_LOGFILE))
    web.run_app(app)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from django.db import models
from django.contrib.auth.models import User
from leagues.models import League

class Match(models.Model):
	COUNTRY_CHOICES = (
		('AL', "Albania"),
		('AT', "Austria"),
		('BE', "Belgium"),
		('HR', "Croatia"),
		('CZ', "Czech"),
		('EN', "England"),
		('FR', "France"),
		('DE', "Germany"),
		('HU', "Hungary"),
		('IS', "Iceland"),
		('IT', "Italy"),
		('NI', "Northern Ireland"),
		('PL', "Poland"),
		('PT', "Portugal"),
		('RI', "Republic of Ireland"),
		('RO', "Romania"),
		('RU', "Russia"),
		('SK', "Slovakia"),
		('SP', "Spain"),
		('SE', "Sweden"),
		('SW', "Switzerland"),
		('TK', "Turkey"),
		('UK', "Ukraine"),
		('WA', "Wales"),
	)
	home_team = models.CharField(max_length = 2, choices = COUNTRY_CHOICES)
	away_team = models.CharField(max_length = 2, choices = COUNTRY_CHOICES)
	home_score = models.IntegerField(default = 0)
	away_score = models.IntegerField(default = 0)
	start_date = models.DateTimeField()
	is_finished = models.BooleanField(default = False)

	def __str__(self):
		return self.get_home_team_display() + "-" + self.get_away_team_display() + ": " + str(self.home_score) + "-" + str(self.away_score)

class Tip(models.Model):
	user = models.ForeignKey(User, models.CASCADE)
	league = models.ForeignKey(League, models.CASCADE, null = True)
	match = models.ForeignKey(Match, models.CASCADE)
	home_score_tip = models.IntegerField()
	away_score_tip = models.IntegerField()

	def __str__(self):
		return self.user.username + " - " + self.league.league_name + ": " + self.match.home_team + " - " + self.match.away_team + ": " + str(self.home_score_tip) + " - " + str(self.away_score_tip)

<commit_msg>Change match toString<commit_after>from __future__ import unicode_literals
from django.db import models
from django.contrib.auth.models import User
from leagues.models import League

class Match(models.Model):
	COUNTRY_CHOICES = (
		('AL', "Albania"),
		('AT', "Austria"),
		('BE', "Belgium"),
		('HR', "Croatia"),
		('CZ', "Czech"),
		('EN', "England"),
		('FR', "France"),
		('DE', "Germany"),
		('HU', "Hungary"),
		('IS', "Iceland"),
		('IT', "Italy"),
		('NI', "Northern Ireland"),
		('PL', "Poland"),
		('PT', "Portugal"),
		('RI', "Republic of Ireland"),
		('RO', "Romania"),
		('RU', "Russia"),
		('SK', "Slovakia"),
		('SP', "Spain"),
		('SE', "Sweden"),
		('SW', "Switzerland"),
		('TK', "Turkey"),
		('UK', "Ukraine"),
		('WA', "Wales"),
	)
	home_team = models.CharField(max_length = 2, choices = COUNTRY_CHOICES)
	away_team = models.CharField(max_length = 2, choices = COUNTRY_CHOICES)
	home_score = models.IntegerField(default = 0)
	away_score = models.IntegerField(default = 0)
	start_date = models.DateTimeField()
	is_finished = models.BooleanField(default = False)

	def __str__(self):
		return self.get_home_team_display() + "-" + self.get_away_team_display() + ": " + str(self.home_score) + "-" + str(self.away_score) + "___" + str(self.start_date)

class Tip(models.Model):
	user = models.ForeignKey(User, models.CASCADE)
	league = models.ForeignKey(League, models.CASCADE, null = True)
	match = models.ForeignKey(Match, models.CASCADE)
	home_score_tip = models.IntegerField()
	away_score_tip = models.IntegerField()

	def __str__(self):
		return self.user.username + " - " + self.league.league_name + ": " + self.match.home_team + " - " + self.match.away_team + ": " + str(self.home_score_tip) + " - " + str(self.away_score_tip)

<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User

# Create your models here.
class UserProfile(models.Model):
    # This line is required. Links UserProfile to a User model instance.
    user = models.OneToOneField(User)

    # The additional attributes we wish to include.
    picture = models.ImageField(upload_to='profile_images', blank=True)

    # Override the __unicode__() method to return out something meaningful!
    def __unicode__(self):
        return self.user.username
<commit_msg>added models for notes, files, and calendar<commit_after>from django.db import models
from django.contrib.auth.models import User

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    picture = models.ImageField(upload_to='profile_images', blank=True)

    def __unicode__(self):
        return self.user.username

class Project(models.Model):
    name = models.CharField(max_length=128)
    users = models.ManyToManyField(UserProfile)
    completed = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

class Notes(models.Model):
    project = models.ForeignKey('Project')
    date_added = models.DateTimeField()
    title = models.CharField(max_length=128)
    content = models.TextField()

    class Meta:
        verbose_name_plural = "Notes"

    def __unicode__(self):
        return self.title

class Files(models.Model):
    project = models.ForeignKey('Project')
    title = models.CharField(max_length=128)
    date_added = models.DateTimeField()
    file = models.FileField(upload_to='files')

    class Meta:
        verbose_name_plural = "Files"

    def __unicode__(self):
        return self.title

class Calendar(models.Model):
    project = models.ForeignKey('Project')
    date_added = models.DateTimeField()
    date_begin = models.DateTimeField()
    date_end = models.DateTimeField()
    title = models.CharField(max_length=128)
    place = models.CharField(max_length=128)
    description = models.TextField()

    def __unicode__(self):
        return self.title<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.core.urlresolvers import reverse_lazy
from models import Event

ANNOUNCEMENT_THRESHOLD = 14

def menu(request):
    menu_items = (
        {'title':u'События', 'url':'events'},
        {'title':u'Видео', 'url':'http://www.youtube.com/moscowdjangoru'},
        {'title':u'Группа Facebook', 'url':'http://www.facebook.com/groups/MoscowDjango/'},
        {'title':u'Люди', 'url':'speakers'},
        {'title':u'О проекте', 'url':'about'},
    )

    for item in menu_items:
        if item['url'] is not None:
            if item['url'].startswith('http'):
                item['external'] = True
            else:
                item['external'] = False
                item['url'] = reverse_lazy(item['url'])

    return {'menu_items': menu_items}

def all_events_processor(request):
    show_announcement = False
    try:
        days_to_next_event = Event.visible.all().latest().days_delta()
        if days_to_next_event <= ANNOUNCEMENT_THRESHOLD:
            show_announcement = True
    except Event.DoesNotExist:
        days_to_next_event = None
    return {
        'all_events': Event.visible.all(),
        'show_announcement': show_announcement,
        'days_to_next_event': days_to_next_event
    }
<commit_msg>No announcement for passed events<commit_after># -*- coding: utf-8 -*-
from django.core.urlresolvers import reverse_lazy
from models import Event

ANNOUNCEMENT_THRESHOLD = 14

def menu(request):
    menu_items = (
        {'title': u'События', 'url': 'events'},
        {'title': u'Видео', 'url': 'http://www.youtube.com/moscowdjangoru'},
        {'title': u'Группа Facebook', 'url': 'http://www.facebook.com/groups/MoscowDjango/'},
        {'title': u'Люди', 'url': 'speakers'},
        {'title': u'О проекте', 'url': 'about'},
    )

    for item in menu_items:
        if item['url'] is not None:
            if item['url'].startswith('http'):
                item['external'] = True
            else:
                item['external'] = False
                item['url'] = reverse_lazy(item['url'])

    return {'menu_items': menu_items}

def all_events_processor(request):
    show_announcement = False
    try:
        days_to_next_event = Event.visible.all().latest().days_delta()
        if days_to_next_event is not None and days_to_next_event <= ANNOUNCEMENT_THRESHOLD:
            show_announcement = True
    except Event.DoesNotExist:
        days_to_next_event = None
    return {
        'all_events': Event.visible.all(),
        'show_announcement': show_announcement,
        'days_to_next_event': days_to_next_event
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2014 Harvard
#
# Authors:
#          Xavier Antoviaque <xavier@antoviaque.org>
#
# This software's license gives you freedom; you can copy, convey,
# propagate, redistribute and/or modify this program under the terms of
# the GNU Affero General Public License (AGPL) as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version of the AGPL published by the FSF.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program in a file in the toplevel directory called
# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
#

# Imports ###########################################################

import logging

from lxml import etree

from xblock.fragment import Fragment

from .light_children import LightChild, Scope, String

# Globals ###########################################################

log = logging.getLogger(__name__)

# Classes ###########################################################

class HTMLBlock(LightChild):
    """
    A simplistic replacement for the HTML XModule, as a light XBlock child
    """
    content = String(help="HTML content", scope=Scope.content, default="")

    @classmethod
    def init_block_from_node(cls, block, node, attr):
        block.light_children = []

        node.tag = 'div'
        block.content = unicode(etree.tostring(node))
        node.tag = 'html'

        return block

    def student_view(self, context=None):
        return Fragment(self.content)

    def mentoring_view(self, context=None):
        return self.student_view(context)
<commit_msg>Fix missing HTMLBlock view<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2014 Harvard
#
# Authors:
#          Xavier Antoviaque <xavier@antoviaque.org>
#
# This software's license gives you freedom; you can copy, convey,
# propagate, redistribute and/or modify this program under the terms of
# the GNU Affero General Public License (AGPL) as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version of the AGPL published by the FSF.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program in a file in the toplevel directory called
# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
#

# Imports ###########################################################

import logging

from lxml import etree

from xblock.fragment import Fragment

from .light_children import LightChild, Scope, String

# Globals ###########################################################

log = logging.getLogger(__name__)

# Classes ###########################################################

class HTMLBlock(LightChild):
    """
    A simplistic replacement for the HTML XModule, as a light XBlock child
    """
    content = String(help="HTML content", scope=Scope.content, default="")

    @classmethod
    def init_block_from_node(cls, block, node, attr):
        block.light_children = []

        node.tag = 'div'
        block.content = unicode(etree.tostring(node))
        node.tag = 'html'

        return block

    def student_view(self, context=None):
        return Fragment(self.content)

    def mentoring_view(self, context=None):
        return self.student_view(context)

    def mentoring_table_view(self, context=None):
        return self.student_view(context)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import re
import sys

def visit_dict(root, parents, data, meta, coll, level=0):
    """Invoked when data is a dict."""
    next_level = level + 1
    for key, val in data.iteritems():
        meta_val = meta.get(key, None)
        if meta_val is None:
            log("warning: missing metadata entry at: %s; key: %s" %
                (parents, key))
            continue
        meta_inf = meta.get("-" + key, None)
        visit_entry(root, parents, data, meta, coll,
                    key, val, meta_val, meta_inf, level=next_level)

def visit_list(root, parents, data, meta, coll, level=0):
    """Invoked when data is a list."""
    next_level = level + 1
    if len(meta) <= 0:
        log("warning: missing list metadata entry at: %s" % (parents))
        return
    meta_val = meta[0] # Use only the first item for metadata.
    for idx, val in enumerate(data):
        visit_entry(root, parents, data, meta, coll,
                    idx, val, meta_val, None, level=next_level)

VISIT_COLLECTION_FUNCS = {"<type 'dict'>": visit_dict,
                          "<type 'list'>": visit_list}

def visit_url(context, path):
    """Recursively visits a ns_server URL, driven by metadata
       to follow links and process data."""
    root = dict(context) # Makes a copy.
    root["run"] = {}
    root["run"]["data"] = retrieve_data(context, path)
    root["run"]["meta"] = retrieve_meta(context, path)
    root["run"]["coll"] = type(root["run"]["meta"])() # Collection/hiearchy of slow data.
    root["run"]["tot_fast"] = 0
    root["run"]["tot_slow"] = 0
    func = root["collection_funcs"][str(type(root["run"]["meta"]))]
    func(root, [],
         root["run"]["data"],
         root["run"]["meta"],
         root["run"]["coll"])
    return root

EMIT_NONE = 0x00
EMIT_SLOW = 0x01
EMIT_FAST = 0x02

def visit_entry_default(root, parents, data, meta, coll,
                        key, val, meta_val, meta_inf, level=0,
                        emit_kind=None):
    """Records a scalar entry into the either the slow-changing or
       fast-changing time-series DB, depending on the emit_kind param.
       Recurses via visit_dict() or visit_list() to handle non-scalar entry."""
    path = parents + [str(key)]
    prefix = "  " * level

    t = type(val)
    if t != type(meta_val):
        if type(meta_val) == float or type(meta_val) == int:
            val = type(meta_val)(val)
        else:
            log("warning: unexpected type: %s; expected %s; at: %s" %
                (t, type(meta_val), path))
            return

    if t == str or t == unicode: # Scalar string entry.
        if emit_kind is None:
            emit_kind = EMIT_SLOW

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', '"%s"' % val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

            # Handle follow metadata, when val is URL/URI,
            # by pushing work onto the todo queue.
            if meta_inf and meta_inf.get('follow', None):
                context = dict(root) # Copy the context for recursion.
                del context["run"]   # But, not the current run info.
                root["todo"].append((context, val))

    elif t == float or t == int: # Scalar numeric entry.
        if emit_kind is None:
            emit_kind = EMIT_FAST

        if emit_kind & EMIT_FAST:
            units = ''
            if meta_inf:
                units = meta_inf.get("units", '')
            log(prefix, "ns-" + '-'.join(path), '=', val, units)
            root["store"]["fast"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

    elif t == bool: # Scalar boolean entry.
        if emit_kind is None:
            emit_kind = EMIT_SLOW

        if emit_kind & EMIT_FAST:
            log(prefix, "ns-" + '-'.join(path), '=', int(val))
            root["store"]["fast"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)
            root["run"]["tot_fast"] += 1

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

    elif t == dict or t == list: # Non-scalar entry.
        child_coll = t()
        debug(prefix, key, "= ...")
        if type(coll) == dict:
            coll[key] = child_coll
        else:
            coll.append(child_coll)

        func = VISIT_COLLECTION_FUNCS[str(t)]
        func(root, path, val, meta_val, child_coll, level=level)

    else:
        log("warning: unhandled type for val: %s; in key: %s" % (val, key))

def visit_entry_fast(root, parents, data, meta, coll,
                     key, val, meta_val, meta_inf, level=0):
    """Records a scalar entry into the fast-changing time-series DB."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, val, meta_val, meta_inf, level=level,
                               emit_kind=EMIT_FAST)

def visit_entry_slow(root, parents, data, meta, coll,
                     key, val, meta_val, meta_inf, level=0):
    """Records a scalar entry into the slow-changing time-series DB."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, val, meta_val, meta_inf, level=level,
                               emit_kind=EMIT_SLOW)

def visit_entry_int(root, parents, data, meta, coll,
                    key, val, meta_val, meta_inf, level=0):
    """Parse the val as an int, and then use default processing."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, int(val), int(meta_val), meta_inf, level=level)

def visit_entry_strip(root, parents, data, meta, coll,
                      key, val, meta_val, meta_inf, level=0):
    """This visit_entry_func strips the entry from results."""
    return

def visit_entry_collect_mc_stats(root, parents, data, meta, coll,
                                 key, val, meta_val, meta_inf, level=0):
    """Collects memcached stats from the val, which should be an array
       of "HOST:PORT", like ["couchbase-01:11210, couchbase-02:11211"].
       The root and parents path should have bucket and SASL auth info."""
    log("  " * level, "MC-STATS", val)
    return # TODO.

VISIT_ENTRY_FUNCS = {"default": visit_entry_default,
                     "fast": visit_entry_fast,
                     "slow": visit_entry_slow,
                     "int": visit_entry_int,
                     "strip": visit_entry_strip,
                     "collect_mc_stats": visit_entry_collect_mc_stats}

def visit_entry(root, parents, data, meta, coll,
                key, val, meta_val, meta_inf, level=0):
    """Invokes the right visit_entry_func on an entry."""
    if (type(key) == str or type(key) == unicode) and key.startswith('-'):
        if root.get("strip_meta", True):
            return

    if meta_inf:
        visit = meta_inf.get("visit", "default")
    else:
        visit = "default"

    visit_entry_func = root["entry_funcs"].get(visit)
    if not visit_entry_func:
        sys.exit("error: unknown visit function: %s; at %s" %
                 (meta_inf["visit"], parents + [key]))

    return visit_entry_func(root, parents, data, meta, coll,
                            key, val, meta_val, meta_inf, level=level)

def retrieve_data(context, path):
    # TODO: Fake implementation for now, since we can pretend
    # that a metadata file is actually a data result from ns_server.
    return retrieve_meta(context, path)

def retrieve_meta(context, path):
    """Retrieves the parsed json metadata that corresponds to
       the given parts of an ns_server URL.
       A path can look like '/pools/default?not=used&ignored=yes'."""
    with open(meta_path(context, path)) as f:
        return json.loads(f.read())

def meta_path(context, path):
    """Calculates the path of the metadata JSON file given a data/URL path."""
    path = path.split('?')[0]
    path = re.sub("/buckets/[^/]+/", "/buckets/BUCKET/", path)
    path = re.sub("/buckets/[^/]+$", "/buckets/BUCKET", path)
    path = re.sub("/nodes/[^/]+/", "/nodes/HOST%3APORT/", path)
    path = re.sub("/nodes/[^/]+$", "/nodes/HOST%3APORT", path)
    fname = "./ns_server/2.0.0/%s.json" % (path[1:].replace("/", "_"))
    return fname

def log(*args):
    print >> sys.stderr, " ".join([str(x) for x in args])

def debug(*args):
    return

def store_fast(root, parents, data, meta, coll,
               key, val, meta_val, meta_inf, level):
    # TODO: Store numeric or boolean metric into fast-changing time-series DB.
    root["run"]["tot_fast"] += 1

def store_slow(root, parents, data, meta, coll,
               key, val, meta_val, meta_inf, level):
    coll[key] = val
    root["run"]["tot_slow"] += 1

def url_before(context, path):
    log("=====", path)
    return context, path

def url_after(context, path, root):
    log("tot_fast =", root["run"]["tot_fast"])
    log("-----", path)
    log(json.dumps(root["run"]["coll"], sort_keys=True, indent=4))

def main(host, port, path, store, callbacks,
         collection_funcs=None,
         entry_funcs=None,
         strip_meta=True):
    if not collection_funcs:
        collection_funcs = VISIT_COLLECTION_FUNCS
    if not entry_funcs:
        entry_funcs = VISIT_ENTRY_FUNCS

    todo = []
    todo.append(({"host": host, "port": port, "store": store, "todo": todo,
                  "collection_funcs": collection_funcs,
                  "entry_funcs": entry_funcs,
                  "strip_meta": strip_meta},
                 path))

    while todo:
        context, path = todo.pop()
        context, path = callbacks["url_before"](context, path)
        root = visit_url(context, path)
        callbacks["url_after"](context, path, root)

if __name__ == '__main__':
    main("127.0.0.1", 8091, "/pools/default",
         {"fast": store_fast,
          "slow": store_slow},
         {"url_before": url_before,
          "url_after": url_after})

<commit_msg>proper defaulp params for visit_funcs<commit_after>#!/usr/bin/env python

import json
import re
import sys

def visit_dict(root, parents, data, meta, coll, level=0):
    """Invoked when data is a dict."""
    next_level = level + 1
    for key, val in data.iteritems():
        meta_val = meta.get(key, None)
        if meta_val is None:
            log("warning: missing metadata entry at: %s; key: %s" %
                (parents, key))
            continue
        meta_inf = meta.get("-" + key, None)
        visit_entry(root, parents, data, meta, coll,
                    key, val, meta_val, meta_inf, level=next_level)

def visit_list(root, parents, data, meta, coll, level=0):
    """Invoked when data is a list."""
    next_level = level + 1
    if len(meta) <= 0:
        log("warning: missing list metadata entry at: %s" % (parents))
        return
    meta_val = meta[0] # Use only the first item for metadata.
    for idx, val in enumerate(data):
        visit_entry(root, parents, data, meta, coll,
                    idx, val, meta_val, None, level=next_level)

VISIT_COLLECTION_FUNCS = {"<type 'dict'>": visit_dict,
                          "<type 'list'>": visit_list}

def visit_url(context, path):
    """Recursively visits a ns_server URL, driven by metadata
       to follow links and process data."""
    root = dict(context) # Makes a copy.
    root["run"] = {}
    root["run"]["data"] = retrieve_data(context, path)
    root["run"]["meta"] = retrieve_meta(context, path)
    root["run"]["coll"] = type(root["run"]["meta"])() # Collection/hiearchy of slow data.
    root["run"]["tot_fast"] = 0
    root["run"]["tot_slow"] = 0
    func = root["collection_funcs"][str(type(root["run"]["meta"]))]
    func(root, [],
         root["run"]["data"],
         root["run"]["meta"],
         root["run"]["coll"])
    return root

EMIT_NONE = 0x00
EMIT_SLOW = 0x01
EMIT_FAST = 0x02

def visit_entry_default(root, parents, data, meta, coll,
                        key, val, meta_val, meta_inf, level=0,
                        emit_kind=None):
    """Records a scalar entry into the either the slow-changing or
       fast-changing time-series DB, depending on the emit_kind param.
       Recurses via visit_dict() or visit_list() to handle non-scalar entry."""
    path = parents + [str(key)]
    prefix = "  " * level

    t = type(val)
    if t != type(meta_val):
        if type(meta_val) == float or type(meta_val) == int:
            val = type(meta_val)(val)
        else:
            log("warning: unexpected type: %s; expected %s; at: %s" %
                (t, type(meta_val), path))
            return

    if t == str or t == unicode: # Scalar string entry.
        if emit_kind is None:
            emit_kind = EMIT_SLOW

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', '"%s"' % val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

            # Handle follow metadata, when val is URL/URI,
            # by pushing work onto the todo queue.
            if meta_inf and meta_inf.get('follow', None):
                context = dict(root) # Copy the context for recursion.
                del context["run"]   # But, not the current run info.
                root["todo"].append((context, val))

    elif t == float or t == int: # Scalar numeric entry.
        if emit_kind is None:
            emit_kind = EMIT_FAST

        if emit_kind & EMIT_FAST:
            units = ''
            if meta_inf:
                units = meta_inf.get("units", '')
            log(prefix, "ns-" + '-'.join(path), '=', val, units)
            root["store"]["fast"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

    elif t == bool: # Scalar boolean entry.
        if emit_kind is None:
            emit_kind = EMIT_SLOW

        if emit_kind & EMIT_FAST:
            log(prefix, "ns-" + '-'.join(path), '=', int(val))
            root["store"]["fast"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)
            root["run"]["tot_fast"] += 1

        if emit_kind & EMIT_SLOW:
            debug(prefix, key, '=', val)
            root["store"]["slow"](root, parents, data, meta, coll,
                                  key, val, meta_val, meta_inf, level)

    elif t == dict or t == list: # Non-scalar entry.
        child_coll = t()
        debug(prefix, key, "= ...")
        if type(coll) == dict:
            coll[key] = child_coll
        else:
            coll.append(child_coll)

        func = VISIT_COLLECTION_FUNCS[str(t)]
        func(root, path, val, meta_val, child_coll, level=level)

    else:
        log("warning: unhandled type for val: %s; in key: %s" % (val, key))

def visit_entry_fast(root, parents, data, meta, coll,
                     key, val, meta_val, meta_inf, level=0):
    """Records a scalar entry into the fast-changing time-series DB."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, val, meta_val, meta_inf, level=level,
                               emit_kind=EMIT_FAST)

def visit_entry_slow(root, parents, data, meta, coll,
                     key, val, meta_val, meta_inf, level=0):
    """Records a scalar entry into the slow-changing time-series DB."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, val, meta_val, meta_inf, level=level,
                               emit_kind=EMIT_SLOW)

def visit_entry_int(root, parents, data, meta, coll,
                    key, val, meta_val, meta_inf, level=0):
    """Parse the val as an int, and then use default processing."""
    return visit_entry_default(root, parents, data, meta, coll,
                               key, int(val), int(meta_val), meta_inf, level=level)

def visit_entry_strip(root, parents, data, meta, coll,
                      key, val, meta_val, meta_inf, level=0):
    """This visit_entry_func strips the entry from results."""
    return

def visit_entry_collect_mc_stats(root, parents, data, meta, coll,
                                 key, val, meta_val, meta_inf, level=0):
    """Collects memcached stats from the val, which should be an array
       of "HOST:PORT", like ["couchbase-01:11210, couchbase-02:11211"].
       The root and parents path should have bucket and SASL auth info."""
    log("  " * level, "MC-STATS", val)
    return # TODO.

VISIT_ENTRY_FUNCS = {"default": visit_entry_default,
                     "fast": visit_entry_fast,
                     "slow": visit_entry_slow,
                     "int": visit_entry_int,
                     "strip": visit_entry_strip,
                     "collect_mc_stats": visit_entry_collect_mc_stats}

def visit_entry(root, parents, data, meta, coll,
                key, val, meta_val, meta_inf, level=0):
    """Invokes the right visit_entry_func on an entry."""
    if (type(key) == str or type(key) == unicode) and key.startswith('-'):
        if root.get("strip_meta", True):
            return

    if meta_inf:
        visit = meta_inf.get("visit", "default")
    else:
        visit = "default"

    visit_entry_func = root["entry_funcs"].get(visit)
    if not visit_entry_func:
        sys.exit("error: unknown visit function: %s; at %s" %
                 (meta_inf["visit"], parents + [key]))

    return visit_entry_func(root, parents, data, meta, coll,
                            key, val, meta_val, meta_inf, level=level)

def retrieve_data(context, path):
    # TODO: Fake implementation for now, since we can pretend
    # that a metadata file is actually a data result from ns_server.
    return retrieve_meta(context, path)

def retrieve_meta(context, path):
    """Retrieves the parsed json metadata that corresponds to
       the given parts of an ns_server URL.
       A path can look like '/pools/default?not=used&ignored=yes'."""
    with open(meta_path(context, path)) as f:
        return json.loads(f.read())

def meta_path(context, path):
    """Calculates the path of the metadata JSON file given a data/URL path."""
    path = path.split('?')[0]
    path = re.sub("/buckets/[^/]+/", "/buckets/BUCKET/", path)
    path = re.sub("/buckets/[^/]+$", "/buckets/BUCKET", path)
    path = re.sub("/nodes/[^/]+/", "/nodes/HOST%3APORT/", path)
    path = re.sub("/nodes/[^/]+$", "/nodes/HOST%3APORT", path)
    fname = "./ns_server/2.0.0/%s.json" % (path[1:].replace("/", "_"))
    return fname

def log(*args):
    print >> sys.stderr, " ".join([str(x) for x in args])

def debug(*args):
    return

def store_fast(root, parents, data, meta, coll,
               key, val, meta_val, meta_inf, level):
    # TODO: Store numeric or boolean metric into fast-changing time-series DB.
    root["run"]["tot_fast"] += 1

def store_slow(root, parents, data, meta, coll,
               key, val, meta_val, meta_inf, level):
    coll[key] = val
    root["run"]["tot_slow"] += 1

def url_before(context, path):
    log("=====", path)
    return context, path

def url_after(context, path, root):
    log("tot_fast =", root["run"]["tot_fast"])
    log("-----", path)
    log(json.dumps(root["run"]["coll"], sort_keys=True, indent=4))

def main(host, port, path, store, callbacks,
         collection_funcs=VISIT_COLLECTION_FUNCS,
         entry_funcs=VISIT_ENTRY_FUNCS,
         strip_meta=True):

    todo = []
    todo.append(({"host": host, "port": port, "store": store, "todo": todo,
                  "collection_funcs": collection_funcs,
                  "entry_funcs": entry_funcs,
                  "strip_meta": strip_meta},
                 path))

    while todo:
        context, path = todo.pop()
        context, path = callbacks["url_before"](context, path)
        root = visit_url(context, path)
        callbacks["url_after"](context, path, root)

if __name__ == '__main__':
    main("127.0.0.1", 8091, "/pools/default",
         {"fast": store_fast,
          "slow": store_slow},
         {"url_before": url_before,
          "url_after": url_after})

<|endoftext|>


<commit_before>'A collection of general purpose tools for reading files'
from __future__ import print_function
import logging
from collections import namedtuple
from struct import Struct

log = logging.getLogger("metpy.io.tools")
log.setLevel(logging.WARNING)

class NamedStruct(Struct):
    def __init__(self, info, prefmt='', tuple_name=None):
        if tuple_name is None:
            tuple_name = 'NamedStruct'
        names, fmts = zip(*info)
        self.converters = {}
        conv_off = 0
        for ind, i in enumerate(info):
            if len(i) > 2:
                self.converters[ind - conv_off] = i[-1]
            elif not i[0]:  # Skip items with no name
                conv_off += 1
        self._tuple = namedtuple(tuple_name, ' '.join(n for n in names if n))
        Struct.__init__(self, prefmt + ''.join(f for f in fmts if f))

    def _create(self, items):
        if self.converters:
            items = list(items)
            for ind, conv in self.converters.items():
                items[ind] = conv(items[ind])
            if len(items) < len(self._tuple._fields):
                items.extend([None] * (len(self._tuple._fields) - len(items)))
        return self._tuple(*items)

    def unpack(self, s):
        return self._create(Struct.unpack(self, s))

    def unpack_from(self, buff, offset=0):
        return self._create(Struct.unpack_from(self, buff, offset))

    def unpack_file(self, fobj):
        bytes = fobj.read(self.size)
        return self.unpack(bytes)

# This works around times when we have more than 255 items and can't use
# NamedStruct. This is a CPython limit for arguments.
class DictStruct(Struct):
    def __init__(self, info, prefmt=''):
        names, formats = zip(*info)

        # Remove empty names
        self._names = [n for n in names if n]

        Struct.__init__(self, prefmt + ''.join(f for f in formats if f))

    def _create(self, items):
        return dict(zip(self._names, items))

    def unpack(self, s):
        return self._create(Struct.unpack(self, s))

    def unpack_from(self, buff, offset=0):
        return self._create(Struct.unpack_from(self, buff, offset))

class Enum(object):
    def __init__(self, *args, **kwargs):
        self.val_map = dict()
        # Assign values for args in order starting at 0
        for ind, a in enumerate(args):
            self.val_map[ind] = a

        # Invert the kwargs dict so that we can map from value to name
        for k in kwargs:
            self.val_map[kwargs[k]] = k

    def __call__(self, val):
        return self.val_map.get(val, 'Unknown ({})'.format(val))

class Bits(object):
    def __init__(self, num_bits):
        self._bits = range(num_bits)

    def __call__(self, val):
        return [bool((val >> i) & 0x1) for i in self._bits]

class BitField(object):
    def __init__(self, *names):
        self._names = names

    def __call__(self, val):
        if not val:
            return None

        l = []
        for n in self._names:
            if val & 0x1:
                l.append(n)
            val = val >> 1
            if not val:
                break

        return l if len(l) > 1 else l[0]

class Array(object):
    def __init__(self, fmt):
        self._struct = Struct(fmt)

    def __call__(self, buf):
        return list(self._struct.unpack(buf))

class IOBuffer(object):
    def __init__(self, source):
        self._data = bytearray(source)
        self._offset = 0
        self.clear_marks()

    @classmethod
    def fromfile(cls, fobj):
        return cls(fobj.read())

    def set_mark(self):
        self._bookmarks.append(self._offset)
        return len(self._bookmarks) - 1

    def jump_to(self, mark, offset=0):
        self._offset = self._bookmarks[mark] + offset

    def offset_from(self, mark):
        return self._offset - self._bookmarks[mark]

    def clear_marks(self):
        self._bookmarks = []

    def splice(self, mark, newdata):
        self.jump_to(mark)
        self._data = self._data[:self._offset] + bytearray(newdata)

    def read_struct(self, struct_class):
        struct = struct_class.unpack_from(self._data, self._offset)
        self.skip(struct_class.size)
        return struct

    def read_func(self, func, num_bytes=None):
        # only advance if func succeeds
        res = func(self.get_next(num_bytes))
        self.skip(num_bytes)
        return res

    def read_ascii(self, num_bytes=None):
        return self.read(num_bytes).decode('ascii')

    def read_binary(self, num, type='B'):
        if 'B' in type:
            return self.read(num)

        if type[0] in ('@', '=', '<', '>', '!'):
            order = type[0]
            type = type[1:]
        else:
            order = '@'

        return list(self.read_struct(Struct(order + '%d' % num + type)))

    def read_int(self, code):
        return self.read_struct(Struct(code))[0]

    def read(self, num_bytes=None):
        res = self.get_next(num_bytes)
        self.skip(len(res))
        return res

    def get_next(self, num_bytes=None):
        if num_bytes is None:
            return self._data[self._offset:]
        else:
            return self._data[self._offset:self._offset + num_bytes]

    def skip(self, num_bytes):
        if num_bytes is None:
            self._offset = len(self._data)
        else:
            self._offset += num_bytes

    def check_remains(self, num_bytes):
        return len(self._data[self._offset:]) == num_bytes

    def truncate(self, num_bytes):
        self._data = self._data[:-num_bytes]

    def at_end(self):
        return self._offset >= len(self._data)

    def __getitem__(self, item):
        return self._data[item]

    def __str__(self):
        return 'Size: {} Offset: {}'.format(len(self._data), self._offset)

    def print_next(self, num_bytes):
        print(' '.join('%02x' % ord(c) for c in self.get_next(num_bytes)))

    def __len__(self):
        return len(self._data)

def bits_to_code(val):
    if val == 8:
        return 'B'
    elif val == 16:
        return 'H'
    else:
        log.warning('Unsupported bit size: %s. Returning "B"', val)
        return 'B'

# For debugging
def hexdump(buf, num_bytes, offset=0, width=32):
    ind = offset
    end = offset + num_bytes
    while ind < end:
        chunk = buf[ind:ind + width]
        actual_width = len(chunk)
        hexfmt = '%02X'
        blocksize = 4
        blocks = [hexfmt * blocksize for i in range(actual_width // blocksize)]

        # Need to get any partial lines
        num_left = actual_width % blocksize
        if num_left:
            blocks += [hexfmt * num_left + '--' * (blocksize - num_left)]
        blocks += ['--' * blocksize] * (width // blocksize - len(blocks))

        hexoutput = ' '.join(blocks)
        printable = tuple(chunk)
        print(hexoutput % printable, str(ind).ljust(len(str(end))),
              ''.join(chr(c) if 31 < c < 128 else '.' for c in chunk), sep='  ')
        ind += width
<commit_msg>Fix BitField processor when no values are decoded.<commit_after>'A collection of general purpose tools for reading files'
from __future__ import print_function
import logging
from collections import namedtuple
from struct import Struct

log = logging.getLogger("metpy.io.tools")
log.setLevel(logging.WARNING)

class NamedStruct(Struct):
    def __init__(self, info, prefmt='', tuple_name=None):
        if tuple_name is None:
            tuple_name = 'NamedStruct'
        names, fmts = zip(*info)
        self.converters = {}
        conv_off = 0
        for ind, i in enumerate(info):
            if len(i) > 2:
                self.converters[ind - conv_off] = i[-1]
            elif not i[0]:  # Skip items with no name
                conv_off += 1
        self._tuple = namedtuple(tuple_name, ' '.join(n for n in names if n))
        Struct.__init__(self, prefmt + ''.join(f for f in fmts if f))

    def _create(self, items):
        if self.converters:
            items = list(items)
            for ind, conv in self.converters.items():
                items[ind] = conv(items[ind])
            if len(items) < len(self._tuple._fields):
                items.extend([None] * (len(self._tuple._fields) - len(items)))
        return self._tuple(*items)

    def unpack(self, s):
        return self._create(Struct.unpack(self, s))

    def unpack_from(self, buff, offset=0):
        return self._create(Struct.unpack_from(self, buff, offset))

    def unpack_file(self, fobj):
        bytes = fobj.read(self.size)
        return self.unpack(bytes)

# This works around times when we have more than 255 items and can't use
# NamedStruct. This is a CPython limit for arguments.
class DictStruct(Struct):
    def __init__(self, info, prefmt=''):
        names, formats = zip(*info)

        # Remove empty names
        self._names = [n for n in names if n]

        Struct.__init__(self, prefmt + ''.join(f for f in formats if f))

    def _create(self, items):
        return dict(zip(self._names, items))

    def unpack(self, s):
        return self._create(Struct.unpack(self, s))

    def unpack_from(self, buff, offset=0):
        return self._create(Struct.unpack_from(self, buff, offset))

class Enum(object):
    def __init__(self, *args, **kwargs):
        self.val_map = dict()
        # Assign values for args in order starting at 0
        for ind, a in enumerate(args):
            self.val_map[ind] = a

        # Invert the kwargs dict so that we can map from value to name
        for k in kwargs:
            self.val_map[kwargs[k]] = k

    def __call__(self, val):
        return self.val_map.get(val, 'Unknown ({})'.format(val))

class Bits(object):
    def __init__(self, num_bits):
        self._bits = range(num_bits)

    def __call__(self, val):
        return [bool((val >> i) & 0x1) for i in self._bits]

class BitField(object):
    def __init__(self, *names):
        self._names = names

    def __call__(self, val):
        if not val:
            return None

        l = []
        for n in self._names:
            if val & 0x1:
                l.append(n)
            val = val >> 1
            if not val:
                break

        # Return whole list if empty or multiple items, otherwise just single item
        return l[0] if len(l) == 1 else l

class Array(object):
    def __init__(self, fmt):
        self._struct = Struct(fmt)

    def __call__(self, buf):
        return list(self._struct.unpack(buf))

class IOBuffer(object):
    def __init__(self, source):
        self._data = bytearray(source)
        self._offset = 0
        self.clear_marks()

    @classmethod
    def fromfile(cls, fobj):
        return cls(fobj.read())

    def set_mark(self):
        self._bookmarks.append(self._offset)
        return len(self._bookmarks) - 1

    def jump_to(self, mark, offset=0):
        self._offset = self._bookmarks[mark] + offset

    def offset_from(self, mark):
        return self._offset - self._bookmarks[mark]

    def clear_marks(self):
        self._bookmarks = []

    def splice(self, mark, newdata):
        self.jump_to(mark)
        self._data = self._data[:self._offset] + bytearray(newdata)

    def read_struct(self, struct_class):
        struct = struct_class.unpack_from(self._data, self._offset)
        self.skip(struct_class.size)
        return struct

    def read_func(self, func, num_bytes=None):
        # only advance if func succeeds
        res = func(self.get_next(num_bytes))
        self.skip(num_bytes)
        return res

    def read_ascii(self, num_bytes=None):
        return self.read(num_bytes).decode('ascii')

    def read_binary(self, num, type='B'):
        if 'B' in type:
            return self.read(num)

        if type[0] in ('@', '=', '<', '>', '!'):
            order = type[0]
            type = type[1:]
        else:
            order = '@'

        return list(self.read_struct(Struct(order + '%d' % num + type)))

    def read_int(self, code):
        return self.read_struct(Struct(code))[0]

    def read(self, num_bytes=None):
        res = self.get_next(num_bytes)
        self.skip(len(res))
        return res

    def get_next(self, num_bytes=None):
        if num_bytes is None:
            return self._data[self._offset:]
        else:
            return self._data[self._offset:self._offset + num_bytes]

    def skip(self, num_bytes):
        if num_bytes is None:
            self._offset = len(self._data)
        else:
            self._offset += num_bytes

    def check_remains(self, num_bytes):
        return len(self._data[self._offset:]) == num_bytes

    def truncate(self, num_bytes):
        self._data = self._data[:-num_bytes]

    def at_end(self):
        return self._offset >= len(self._data)

    def __getitem__(self, item):
        return self._data[item]

    def __str__(self):
        return 'Size: {} Offset: {}'.format(len(self._data), self._offset)

    def print_next(self, num_bytes):
        print(' '.join('%02x' % ord(c) for c in self.get_next(num_bytes)))

    def __len__(self):
        return len(self._data)

def bits_to_code(val):
    if val == 8:
        return 'B'
    elif val == 16:
        return 'H'
    else:
        log.warning('Unsupported bit size: %s. Returning "B"', val)
        return 'B'

# For debugging
def hexdump(buf, num_bytes, offset=0, width=32):
    ind = offset
    end = offset + num_bytes
    while ind < end:
        chunk = buf[ind:ind + width]
        actual_width = len(chunk)
        hexfmt = '%02X'
        blocksize = 4
        blocks = [hexfmt * blocksize for i in range(actual_width // blocksize)]

        # Need to get any partial lines
        num_left = actual_width % blocksize
        if num_left:
            blocks += [hexfmt * num_left + '--' * (blocksize - num_left)]
        blocks += ['--' * blocksize] * (width // blocksize - len(blocks))

        hexoutput = ' '.join(blocks)
        printable = tuple(chunk)
        print(hexoutput % printable, str(ind).ljust(len(str(end))),
              ''.join(chr(c) if 31 < c < 128 else '.' for c in chunk), sep='  ')
        ind += width
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import unicode_literals
from collections import defaultdict
from ajax_select.fields import AutoCompleteSelectMultipleField, \
    AutoCompleteWidget
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit, Field, HTML
from django.forms import ValidationError, ModelForm, Form, CharField, TextInput
from django.utils.translation import ugettext_lazy as _
from .models import (
    Oeuvre, Source, Individu, ElementDeProgramme, ElementDeDistribution,
    Ensemble)
from range_slider.fields import RangeSliderField

__all__ = (b'IndividuForm', b'EnsembleForm', b'OeuvreForm',
           b'ElementDeDistributionForm', b'ElementDeProgrammeForm',
           b'SourceForm', b'SaisonForm', b'EvenementListForm')

class IndividuForm(ModelForm):
    class Meta(object):
        model = Individu
        widgets = {
            b'prenoms':
                AutoCompleteWidget('individu__prenoms',
                                   attrs={'style': 'width: 300px;'})
        }

    def __init__(self, *args, **kwargs):
        super(IndividuForm, self).__init__(*args, **kwargs)

        def apply_style(fieldname, style):
            self.fields[fieldname].widget.attrs['style'] = style

        apply_style('particule_nom', 'width: 50px;')
        apply_style('particule_nom_naissance', 'width: 50px;')

    def clean_designation(self):
        # Anticipe si la désignation donnera un résultat nul.
        data = self.cleaned_data
        designation = data[b'designation']
        if designation == 'P' and not data[b'pseudonyme'] \
            or designation == 'B' and not data[b'nom_naissance'] \
                or designation == 'F' and not data[b'prenoms']:
            raise ValidationError(_('Il manque des données pour pouvoir '
                                    'choisir cette désignation.'))
        return designation

class EnsembleForm(ModelForm):
    class Meta(object):
        model = Ensemble
        widgets = {
            b'particule_nom':
                AutoCompleteWidget('ensemble__particule_nom',
                                   attrs={'style': 'width: 50px;'})
        }

class OeuvreForm(ModelForm):
    class Meta(object):
        model = Oeuvre
        widgets = {
            b'prefixe_titre':
                AutoCompleteWidget('oeuvre__prefixe_titre',
                                   attrs={'style': 'width: 50px;'}),
            b'coordination':
                AutoCompleteWidget('oeuvre__coordination',
                                   attrs={'style': 'width: 70px;'}),
            b'prefixe_titre_secondaire':
                AutoCompleteWidget('oeuvre__prefixe_titre_secondaire',
                                   attrs={'style': 'width: 50px;'}),
            b'coupe': AutoCompleteWidget('oeuvre__coupe',
                                         attrs={'style': 'width: 500px;'}),
        }

class ElementDeDistributionForm(ModelForm):
    class Meta(object):
        model = ElementDeDistribution

    def clean(self):
        data = super(ElementDeDistributionForm, self).clean()

        error_msgs = defaultdict(list)
        if not (data[b'individus'] or data[b'ensembles']):
            msg = _('Vous devez remplir au moins un individu ou un ensemble.')
            error_msgs[b'individus'].append(msg)
            error_msgs[b'ensembles'].append(msg)
        if data.get(b'pupitre') and data[b'profession']:
            msg = _('Vous ne pouvez remplir à la fois '
                    '« Pupitre » et « Profession ».')
            error_msgs[b'pupitre'].append(msg)
            error_msgs[b'profession'].append(msg)
        if data.get(b'pupitre', '') != '' \
                and data[b'profession'] \
                and data[b'profession'].parties.exists():
            msg = _('Au moins un rôle ou instrument est lié à cette '
                    'profession. Remplissez donc « Pupitre » à la place.')
            error_msgs[b'profession'].append(msg)

        for k, v in error_msgs.items():
            self._errors[k] = self.error_class(v)
            del data[k]

        return data

class ElementDeProgrammeForm(ModelForm):
    class Meta(object):
        model = ElementDeProgramme
        # FIXME: Rendre fonctionnel ce qui suit.
        # widgets = {
        #     b'autre': AutoCompleteWidget('elementdeprogramme__autre',
        #                                  attrs={'style': 'width: 600px;'}),
        # }

    def clean(self):
        data = super(ElementDeProgrammeForm, self).clean()

        if not (data[b'autre'] or data[b'oeuvre'] or data[b'distribution']):
            raise ValidationError(_('Vous devez remplir au moins « Œuvre », '
                                    '« Autre » ou « Distribution ».'))

        if data[b'autre'] and data[b'oeuvre']:
            raise ValidationError(_('Vous ne pouvez remplir à la fois '
                                    '« Œuvre » et « Autre ».'))

        return data

class SourceForm(ModelForm):
    class Meta(object):
        model = Source
        widgets = {
            b'titre': AutoCompleteWidget('source__titre',
                                         attrs={'style': 'width: 600px;'}),
            b'numero': TextInput(attrs={'cols': 10}),
            b'folio': TextInput(attrs={'cols': 10}),
            b'page': TextInput(attrs={'cols': 10}),
        }

    def clean(self):
        data = super(SourceForm, self).clean()

        if not (data['titre'] or (data['lieu_conservation'] and data['cote'])):
            raise ValidationError(_('Vous devez remplir « Titre » ou '
                                    '« Lieu de conservation » et « Cote ».'))

        return data

class SaisonForm(ModelForm):
    class Meta(object):
        model = Source

    def clean(self):
        data = super(SaisonForm, self).clean()

        if data['ensemble'] and data['lieu'] or \
                not data['ensemble'] and not data['lieu']:
            raise ValidationError(_('Vous devez remplir un seul des deux'
                                    ' champs : « Ensemble » ou « Lieu ou '
                                    'Institution »'))

        if data['debut'] and data['fin'] and data['debut'] > data['fin']:
            raise ValidationError(_('La fin ne peut précéder le début.'))
        return data

class EvenementListForm(Form):
    q = CharField(label=_('Recherche libre'), required=False)
    dates = RangeSliderField(required=False)
    lieu = AutoCompleteSelectMultipleField('lieu', label=_('Lieu'),
                                           required=False, help_text='')
    oeuvre = AutoCompleteSelectMultipleField(
        'oeuvre', required=False, label=_('Œuvre'), help_text='')
    individu = AutoCompleteSelectMultipleField(
        'individu', required=False, label=_('Individu'), help_text='')
    ensemble = AutoCompleteSelectMultipleField(
        'ensemble', required=False, label=_('Ensemble'), help_text='')

    def __init__(self, *args, **kwargs):
        queryset = kwargs.pop('queryset')

        self.helper = FormHelper()
        self.helper.form_method = 'GET'
        self.helper.form_class = 'well well-sm'
        self.helper.layout = Layout(
            Field('q', css_class='input-lg'),
            HTML('
'),
            'dates',
            HTML('
'),
            'lieu', 'oeuvre', 'individu', 'ensemble',
            HTML('
'),
            Submit('', _('Filtrer'), css_class='btn-lg btn-block',
                   data_loading_text=_('Chargement…')),
        )

        super(EvenementListForm, self).__init__(*args, **kwargs)

        for field in self.fields.values():
            field.widget.attrs[b'placeholder'] = (field.label or '') + '…'
            field.label = ''

        self.fields['dates'].widget.queryset = queryset
<commit_msg>Ajoute des contraintes de validation sur les saisons.<commit_after># coding: utf-8

from __future__ import unicode_literals
from collections import defaultdict
from ajax_select.fields import AutoCompleteSelectMultipleField, \
    AutoCompleteWidget
from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Submit, Field, HTML
from datetime import timedelta
from django.db.models import Q
from django.forms import ValidationError, ModelForm, Form, CharField, TextInput
from django.utils.translation import ugettext_lazy as _
from .models import (
    Oeuvre, Source, Individu, ElementDeProgramme, ElementDeDistribution,
    Ensemble, Saison)
from range_slider.fields import RangeSliderField

__all__ = (b'IndividuForm', b'EnsembleForm', b'OeuvreForm',
           b'ElementDeDistributionForm', b'ElementDeProgrammeForm',
           b'SourceForm', b'SaisonForm', b'EvenementListForm')

class IndividuForm(ModelForm):
    class Meta(object):
        model = Individu
        widgets = {
            b'prenoms':
                AutoCompleteWidget('individu__prenoms',
                                   attrs={'style': 'width: 300px;'})
        }

    def __init__(self, *args, **kwargs):
        super(IndividuForm, self).__init__(*args, **kwargs)

        def apply_style(fieldname, style):
            self.fields[fieldname].widget.attrs['style'] = style

        apply_style('particule_nom', 'width: 50px;')
        apply_style('particule_nom_naissance', 'width: 50px;')

    def clean_designation(self):
        # Anticipe si la désignation donnera un résultat nul.
        data = self.cleaned_data
        designation = data[b'designation']
        if designation == 'P' and not data[b'pseudonyme'] \
            or designation == 'B' and not data[b'nom_naissance'] \
                or designation == 'F' and not data[b'prenoms']:
            raise ValidationError(_('Il manque des données pour pouvoir '
                                    'choisir cette désignation.'))
        return designation

class EnsembleForm(ModelForm):
    class Meta(object):
        model = Ensemble
        widgets = {
            b'particule_nom':
                AutoCompleteWidget('ensemble__particule_nom',
                                   attrs={'style': 'width: 50px;'})
        }

class OeuvreForm(ModelForm):
    class Meta(object):
        model = Oeuvre
        widgets = {
            b'prefixe_titre':
                AutoCompleteWidget('oeuvre__prefixe_titre',
                                   attrs={'style': 'width: 50px;'}),
            b'coordination':
                AutoCompleteWidget('oeuvre__coordination',
                                   attrs={'style': 'width: 70px;'}),
            b'prefixe_titre_secondaire':
                AutoCompleteWidget('oeuvre__prefixe_titre_secondaire',
                                   attrs={'style': 'width: 50px;'}),
            b'coupe': AutoCompleteWidget('oeuvre__coupe',
                                         attrs={'style': 'width: 500px;'}),
        }

class ElementDeDistributionForm(ModelForm):
    class Meta(object):
        model = ElementDeDistribution

    def clean(self):
        data = super(ElementDeDistributionForm, self).clean()

        error_msgs = defaultdict(list)
        if not (data[b'individus'] or data[b'ensembles']):
            msg = _('Vous devez remplir au moins un individu ou un ensemble.')
            error_msgs[b'individus'].append(msg)
            error_msgs[b'ensembles'].append(msg)
        if data.get(b'pupitre') and data[b'profession']:
            msg = _('Vous ne pouvez remplir à la fois '
                    '« Pupitre » et « Profession ».')
            error_msgs[b'pupitre'].append(msg)
            error_msgs[b'profession'].append(msg)
        if data.get(b'pupitre', '') != '' \
                and data[b'profession'] \
                and data[b'profession'].parties.exists():
            msg = _('Au moins un rôle ou instrument est lié à cette '
                    'profession. Remplissez donc « Pupitre » à la place.')
            error_msgs[b'profession'].append(msg)

        for k, v in error_msgs.items():
            self._errors[k] = self.error_class(v)
            del data[k]

        return data

class ElementDeProgrammeForm(ModelForm):
    class Meta(object):
        model = ElementDeProgramme
        # FIXME: Rendre fonctionnel ce qui suit.
        # widgets = {
        #     b'autre': AutoCompleteWidget('elementdeprogramme__autre',
        #                                  attrs={'style': 'width: 600px;'}),
        # }

    def clean(self):
        data = super(ElementDeProgrammeForm, self).clean()

        if not (data[b'autre'] or data[b'oeuvre'] or data[b'distribution']):
            raise ValidationError(_('Vous devez remplir au moins « Œuvre », '
                                    '« Autre » ou « Distribution ».'))

        if data[b'autre'] and data[b'oeuvre']:
            raise ValidationError(_('Vous ne pouvez remplir à la fois '
                                    '« Œuvre » et « Autre ».'))

        return data

class SourceForm(ModelForm):
    class Meta(object):
        model = Source
        widgets = {
            b'titre': AutoCompleteWidget('source__titre',
                                         attrs={'style': 'width: 600px;'}),
            b'numero': TextInput(attrs={'cols': 10}),
            b'folio': TextInput(attrs={'cols': 10}),
            b'page': TextInput(attrs={'cols': 10}),
        }

    def clean(self):
        data = super(SourceForm, self).clean()

        if not (data['titre'] or (data['lieu_conservation'] and data['cote'])):
            raise ValidationError(_('Vous devez remplir « Titre » ou '
                                    '« Lieu de conservation » et « Cote ».'))

        return data

class SaisonForm(ModelForm):
    class Meta(object):
        model = Source

    def clean(self):
        data = super(SaisonForm, self).clean()

        if data['ensemble'] and data['lieu'] or \
                not data['ensemble'] and not data['lieu']:
            raise ValidationError(_('Vous devez remplir un seul des deux'
                                    ' champs : « Ensemble » ou « Lieu ou '
                                    'Institution »'))

        if data['debut'] and data['fin']:
            if data['debut'] > data['fin']:
                raise ValidationError(_('La fin ne peut précéder le début.'))
            elif data['fin'] - data['debut'] > timedelta(365):
                raise ValidationError(_('La durée d’une saison ne peut excéder '
                                        'un an.'))

        filters = Q()
        for s in ('debut', 'fin'):
            filters |= Q(**{'debut__lte': data[s]}) & Q(**{'fin__gte': data[s]})
        for k in ('ensemble', 'lieu'):
            if data[k]:
                filters &= Q(**{k: data[k]})
        if Saison.objects.filter(filters).exists():
            raise ValidationError(_('Les « Saison » liées au même  « Ensemble »'
                                    ' ou « Lieu ou Institution » ne peuvent se '
                                    'chevaucher.'))
        return data

class EvenementListForm(Form):
    q = CharField(label=_('Recherche libre'), required=False)
    dates = RangeSliderField(required=False)
    lieu = AutoCompleteSelectMultipleField('lieu', label=_('Lieu'),
                                           required=False, help_text='')
    oeuvre = AutoCompleteSelectMultipleField(
        'oeuvre', required=False, label=_('Œuvre'), help_text='')
    individu = AutoCompleteSelectMultipleField(
        'individu', required=False, label=_('Individu'), help_text='')
    ensemble = AutoCompleteSelectMultipleField(
        'ensemble', required=False, label=_('Ensemble'), help_text='')

    def __init__(self, *args, **kwargs):
        queryset = kwargs.pop('queryset')

        self.helper = FormHelper()
        self.helper.form_method = 'GET'
        self.helper.form_class = 'well well-sm'
        self.helper.layout = Layout(
            Field('q', css_class='input-lg'),
            HTML('
'),
            'dates',
            HTML('
'),
            'lieu', 'oeuvre', 'individu', 'ensemble',
            HTML('
'),
            Submit('', _('Filtrer'), css_class='btn-lg btn-block',
                   data_loading_text=_('Chargement…')),
        )

        super(EvenementListForm, self).__init__(*args, **kwargs)

        for field in self.fields.values():
            field.widget.attrs[b'placeholder'] = (field.label or '') + '…'
            field.label = ''

        self.fields['dates'].widget.queryset = queryset
<|endoftext|>