<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
from inspect import getcallargs

from mock import Mock, patch
from six.moves.urllib import parse as urlparse

from twisted.internet import defer, reactor

from synapse.api.errors import CodeMessageException, cs_error
from synapse.federation.transport import server
from synapse.http.server import HttpServer
from synapse.server import HomeServer
from synapse.storage import PostgresEngine
from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database
from synapse.util.logcontext import LoggingContext
from synapse.util.ratelimitutils import FederationRateLimiter

# set this to True to run the tests against postgres instead of sqlite.
# It requires you to have a local postgres database called synapse_test, within
# which ALL TABLES WILL BE DROPPED
USE_POSTGRES_FOR_TESTS = False

@defer.inlineCallbacks
def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None,
                          **kargs):
    """Setup a homeserver suitable for running tests against. Keyword arguments
    are passed to the Homeserver constructor. If no datastore is supplied a
    datastore backed by an in-memory sqlite db will be given to the HS.
    """
    if reactor is None:
        from twisted.internet import reactor

    if config is None:
        config = Mock()
        config.signing_key = [MockKey()]
        config.event_cache_size = 1
        config.enable_registration = True
        config.macaroon_secret_key = "not even a little secret"
        config.expire_access_token = False
        config.server_name = name
        config.trusted_third_party_id_servers = []
        config.room_invite_state_types = []
        config.password_providers = []
        config.worker_replication_url = ""
        config.worker_app = None
        config.email_enable_notifs = False
        config.block_non_admin_invites = False
        config.federation_domain_whitelist = None
        config.federation_rc_reject_limit = 10
        config.federation_rc_sleep_limit = 10
        config.federation_rc_sleep_delay = 100
        config.federation_rc_concurrent = 10
        config.filter_timeline_limit = 5000
        config.user_directory_search_all_users = False
        config.user_consent_server_notice_content = None
        config.block_events_without_consent_error = None
        config.media_storage_providers = []
        config.auto_join_rooms = []
        config.limit_usage_by_mau = False
        config.mau_limits_reserved_threepids = []

        # disable user directory updates, because they get done in the
        # background, which upsets the test runner.
        config.update_user_directory = False

    config.use_frozen_dicts = True
    config.ldap_enabled = False

    if "clock" not in kargs:
        kargs["clock"] = MockClock()

    if USE_POSTGRES_FOR_TESTS:
        config.database_config = {
            "name": "psycopg2",
            "args": {
                "database": "synapse_test",
                "cp_min": 1,
                "cp_max": 5,
            },
        }
    else:
        config.database_config = {
            "name": "sqlite3",
            "args": {
                "database": ":memory:",
                "cp_min": 1,
                "cp_max": 1,
            },
        }

    db_engine = create_engine(config.database_config)

    # we need to configure the connection pool to run the on_new_connection
    # function, so that we can test code that uses custom sqlite functions
    # (like rank).
    config.database_config["args"]["cp_openfun"] = db_engine.on_new_connection

    if datastore is None:
        hs = HomeServer(
            name, config=config,
            db_config=config.database_config,
            version_string="Synapse/tests",
            database_engine=db_engine,
            room_list_handler=object(),
            tls_server_context_factory=Mock(),
            reactor=reactor,
            **kargs
        )
        db_conn = hs.get_db_conn()
        # make sure that the database is empty
        if isinstance(db_engine, PostgresEngine):
            cur = db_conn.cursor()
            cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
            rows = cur.fetchall()
            for r in rows:
                cur.execute("DROP TABLE %s CASCADE" % r[0])
        yield prepare_database(db_conn, db_engine, config)
        hs.setup()
    else:
        hs = HomeServer(
            name, db_pool=None, datastore=datastore, config=config,
            version_string="Synapse/tests",
            database_engine=db_engine,
            room_list_handler=object(),
            tls_server_context_factory=Mock(),
            reactor=reactor,
            **kargs
        )

    # bcrypt is far too slow to be doing in unit tests
    # Need to let the HS build an auth handler and then mess with it
    # because AuthHandler's constructor requires the HS, so we can't make one
    # beforehand and pass it in to the HS's constructor (chicken / egg)
    hs.get_auth_handler().hash = lambda p: hashlib.md5(p).hexdigest()
    hs.get_auth_handler().validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h

    fed = kargs.get("resource_for_federation", None)
    if fed:
        server.register_servlets(
            hs,
            resource=fed,
            authenticator=server.Authenticator(hs),
            ratelimiter=FederationRateLimiter(
                hs.get_clock(),
                window_size=hs.config.federation_rc_window_size,
                sleep_limit=hs.config.federation_rc_sleep_limit,
                sleep_msec=hs.config.federation_rc_sleep_delay,
                reject_limit=hs.config.federation_rc_reject_limit,
                concurrent_requests=hs.config.federation_rc_concurrent
            ),
        )

    defer.returnValue(hs)

def get_mock_call_args(pattern_func, mock_func):
    """ Return the arguments the mock function was called with interpreted
    by the pattern functions argument list.
    """
    invoked_args, invoked_kargs = mock_func.call_args
    return getcallargs(pattern_func, *invoked_args, **invoked_kargs)

def mock_getRawHeaders(headers=None):
    headers = headers if headers is not None else {}

    def getRawHeaders(name, default=None):
        return headers.get(name, default)

    return getRawHeaders

# This is a mock /resource/ not an entire server
class MockHttpResource(HttpServer):

    def __init__(self, prefix=""):
        self.callbacks = []  # 3-tuple of method/pattern/function
        self.prefix = prefix

    def trigger_get(self, path):
        return self.trigger(b"GET", path, None)

    @patch('twisted.web.http.Request')
    @defer.inlineCallbacks
    def trigger(self, http_method, path, content, mock_request, federation_auth=False):
        """ Fire an HTTP event.

        Args:
            http_method : The HTTP method
            path : The HTTP path
            content : The HTTP body
            mock_request : Mocked request to pass to the event so it can get
                           content.
        Returns:
            A tuple of (code, response)
        Raises:
            KeyError If no event is found which will handle the path.
        """
        path = self.prefix + path

        # annoyingly we return a twisted http request which has chained calls
        # to get at the http content, hence mock it here.
        mock_content = Mock()
        config = {'read.return_value': content}
        mock_content.configure_mock(**config)
        mock_request.content = mock_content

        mock_request.method = http_method
        mock_request.uri = path

        mock_request.getClientIP.return_value = "-"

        headers = {}
        if federation_auth:
            headers[b"Authorization"] = [b"X-Matrix origin=test,key=,sig="]
        mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)

        # return the right path if the event requires it
        mock_request.path = path

        # add in query params to the right place
        try:
            mock_request.args = urlparse.parse_qs(path.split('?')[1])
            mock_request.path = path.split('?')[0]
            path = mock_request.path
        except Exception:
            pass

        if isinstance(path, bytes):
            path = path.decode('utf8')

        for (method, pattern, func) in self.callbacks:
            if http_method != method:
                continue

            matcher = pattern.match(path)
            if matcher:
                try:
                    args = [
                        urlparse.unquote(u)
                        for u in matcher.groups()
                    ]

                    (code, response) = yield func(
                        mock_request,
                        *args
                    )
                    defer.returnValue((code, response))
                except CodeMessageException as e:
                    defer.returnValue((e.code, cs_error(e.msg, code=e.errcode)))

        raise KeyError("No event can handle %s" % path)

    def register_paths(self, method, path_patterns, callback):
        for path_pattern in path_patterns:
            self.callbacks.append((method, path_pattern, callback))

class MockKey(object):
    alg = "mock_alg"
    version = "mock_version"
    signature = b"\x9a\x87$"

    @property
    def verify_key(self):
        return self

    def sign(self, message):
        return self

    def verify(self, message, sig):
        assert sig == b"\x9a\x87$"

class MockClock(object):
    now = 1000

    def __init__(self):
        # list of lists of [absolute_time, callback, expired] in no particular
        # order
        self.timers = []
        self.loopers = []

    def time(self):
        return self.now

    def time_msec(self):
        return self.time() * 1000

    def call_later(self, delay, callback, *args, **kwargs):
        current_context = LoggingContext.current_context()

        def wrapped_callback():
            LoggingContext.thread_local.current_context = current_context
            callback(*args, **kwargs)

        t = [self.now + delay, wrapped_callback, False]
        self.timers.append(t)

        return t

    def looping_call(self, function, interval):
        self.loopers.append([function, interval / 1000., self.now])

    def cancel_call_later(self, timer, ignore_errs=False):
        if timer[2]:
            if not ignore_errs:
                raise Exception("Cannot cancel an expired timer")

        timer[2] = True
        self.timers = [t for t in self.timers if t != timer]

    # For unit testing
    def advance_time(self, secs):
        self.now += secs

        timers = self.timers
        self.timers = []

        for t in timers:
            time, callback, expired = t

            if expired:
                raise Exception("Timer already expired")

            if self.now >= time:
                t[2] = True
                callback()
            else:
                self.timers.append(t)

        for looped in self.loopers:
            func, interval, last = looped
            if last + interval < self.now:
                func()
                looped[2] = self.now

    def advance_time_msec(self, ms):
        self.advance_time(ms / 1000.)

    def time_bound_deferred(self, d, *args, **kwargs):
        # We don't bother timing things out for now.
        return d

def _format_call(args, kwargs):
    return ", ".join(
        ["%r" % (a) for a in args] +
        ["%s=%r" % (k, v) for k, v in kwargs.items()]
    )

class DeferredMockCallable(object):
    """A callable instance that stores a set of pending call expectations and
    return values for them. It allows a unit test to assert that the given set
    of function calls are eventually made, by awaiting on them to be called.
    """

    def __init__(self):
        self.expectations = []
        self.calls = []

    def __call__(self, *args, **kwargs):
        self.calls.append((args, kwargs))

        if not self.expectations:
            raise ValueError("%r has no pending calls to handle call(%s)" % (
                self, _format_call(args, kwargs))
            )

        for (call, result, d) in self.expectations:
            if args == call[1] and kwargs == call[2]:
                d.callback(None)
                return result

        failure = AssertionError("Was not expecting call(%s)" % (
            _format_call(args, kwargs)
        ))

        for _, _, d in self.expectations:
            try:
                d.errback(failure)
            except Exception:
                pass

        raise failure

    def expect_call_and_return(self, call, result):
        self.expectations.append((call, result, defer.Deferred()))

    @defer.inlineCallbacks
    def await_calls(self, timeout=1000):
        deferred = defer.DeferredList(
            [d for _, _, d in self.expectations],
            fireOnOneErrback=True
        )

        timer = reactor.callLater(
            timeout / 1000,
            deferred.errback,
            AssertionError("%d pending calls left: %s" % (
                len([e for e in self.expectations if not e[2].called]),
                [e for e in self.expectations if not e[2].called]
            ))
        )

        yield deferred

        timer.cancel()

        self.calls = []

    def assert_had_no_calls(self):
        if self.calls:
            calls = self.calls
            self.calls = []

            raise AssertionError(
                "Expected not to received any calls, got:\n" + "\n".join([
                    "call(%s)" % _format_call(c[0], c[1]) for c in calls
                ])
            )
<commit_msg>mock config.max_mau_value<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
from inspect import getcallargs

from mock import Mock, patch
from six.moves.urllib import parse as urlparse

from twisted.internet import defer, reactor

from synapse.api.errors import CodeMessageException, cs_error
from synapse.federation.transport import server
from synapse.http.server import HttpServer
from synapse.server import HomeServer
from synapse.storage import PostgresEngine
from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database
from synapse.util.logcontext import LoggingContext
from synapse.util.ratelimitutils import FederationRateLimiter

# set this to True to run the tests against postgres instead of sqlite.
# It requires you to have a local postgres database called synapse_test, within
# which ALL TABLES WILL BE DROPPED
USE_POSTGRES_FOR_TESTS = False

@defer.inlineCallbacks
def setup_test_homeserver(name="test", datastore=None, config=None, reactor=None,
                          **kargs):
    """Setup a homeserver suitable for running tests against. Keyword arguments
    are passed to the Homeserver constructor. If no datastore is supplied a
    datastore backed by an in-memory sqlite db will be given to the HS.
    """
    if reactor is None:
        from twisted.internet import reactor

    if config is None:
        config = Mock()
        config.signing_key = [MockKey()]
        config.event_cache_size = 1
        config.enable_registration = True
        config.macaroon_secret_key = "not even a little secret"
        config.expire_access_token = False
        config.server_name = name
        config.trusted_third_party_id_servers = []
        config.room_invite_state_types = []
        config.password_providers = []
        config.worker_replication_url = ""
        config.worker_app = None
        config.email_enable_notifs = False
        config.block_non_admin_invites = False
        config.federation_domain_whitelist = None
        config.federation_rc_reject_limit = 10
        config.federation_rc_sleep_limit = 10
        config.federation_rc_sleep_delay = 100
        config.federation_rc_concurrent = 10
        config.filter_timeline_limit = 5000
        config.user_directory_search_all_users = False
        config.user_consent_server_notice_content = None
        config.block_events_without_consent_error = None
        config.media_storage_providers = []
        config.auto_join_rooms = []
        config.limit_usage_by_mau = False
        config.max_mau_value = 50
        config.mau_limits_reserved_threepids = []

        # disable user directory updates, because they get done in the
        # background, which upsets the test runner.
        config.update_user_directory = False

    config.use_frozen_dicts = True
    config.ldap_enabled = False

    if "clock" not in kargs:
        kargs["clock"] = MockClock()

    if USE_POSTGRES_FOR_TESTS:
        config.database_config = {
            "name": "psycopg2",
            "args": {
                "database": "synapse_test",
                "cp_min": 1,
                "cp_max": 5,
            },
        }
    else:
        config.database_config = {
            "name": "sqlite3",
            "args": {
                "database": ":memory:",
                "cp_min": 1,
                "cp_max": 1,
            },
        }

    db_engine = create_engine(config.database_config)

    # we need to configure the connection pool to run the on_new_connection
    # function, so that we can test code that uses custom sqlite functions
    # (like rank).
    config.database_config["args"]["cp_openfun"] = db_engine.on_new_connection

    if datastore is None:
        hs = HomeServer(
            name, config=config,
            db_config=config.database_config,
            version_string="Synapse/tests",
            database_engine=db_engine,
            room_list_handler=object(),
            tls_server_context_factory=Mock(),
            reactor=reactor,
            **kargs
        )
        db_conn = hs.get_db_conn()
        # make sure that the database is empty
        if isinstance(db_engine, PostgresEngine):
            cur = db_conn.cursor()
            cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
            rows = cur.fetchall()
            for r in rows:
                cur.execute("DROP TABLE %s CASCADE" % r[0])
        yield prepare_database(db_conn, db_engine, config)
        hs.setup()
    else:
        hs = HomeServer(
            name, db_pool=None, datastore=datastore, config=config,
            version_string="Synapse/tests",
            database_engine=db_engine,
            room_list_handler=object(),
            tls_server_context_factory=Mock(),
            reactor=reactor,
            **kargs
        )

    # bcrypt is far too slow to be doing in unit tests
    # Need to let the HS build an auth handler and then mess with it
    # because AuthHandler's constructor requires the HS, so we can't make one
    # beforehand and pass it in to the HS's constructor (chicken / egg)
    hs.get_auth_handler().hash = lambda p: hashlib.md5(p).hexdigest()
    hs.get_auth_handler().validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h

    fed = kargs.get("resource_for_federation", None)
    if fed:
        server.register_servlets(
            hs,
            resource=fed,
            authenticator=server.Authenticator(hs),
            ratelimiter=FederationRateLimiter(
                hs.get_clock(),
                window_size=hs.config.federation_rc_window_size,
                sleep_limit=hs.config.federation_rc_sleep_limit,
                sleep_msec=hs.config.federation_rc_sleep_delay,
                reject_limit=hs.config.federation_rc_reject_limit,
                concurrent_requests=hs.config.federation_rc_concurrent
            ),
        )

    defer.returnValue(hs)

def get_mock_call_args(pattern_func, mock_func):
    """ Return the arguments the mock function was called with interpreted
    by the pattern functions argument list.
    """
    invoked_args, invoked_kargs = mock_func.call_args
    return getcallargs(pattern_func, *invoked_args, **invoked_kargs)

def mock_getRawHeaders(headers=None):
    headers = headers if headers is not None else {}

    def getRawHeaders(name, default=None):
        return headers.get(name, default)

    return getRawHeaders

# This is a mock /resource/ not an entire server
class MockHttpResource(HttpServer):

    def __init__(self, prefix=""):
        self.callbacks = []  # 3-tuple of method/pattern/function
        self.prefix = prefix

    def trigger_get(self, path):
        return self.trigger(b"GET", path, None)

    @patch('twisted.web.http.Request')
    @defer.inlineCallbacks
    def trigger(self, http_method, path, content, mock_request, federation_auth=False):
        """ Fire an HTTP event.

        Args:
            http_method : The HTTP method
            path : The HTTP path
            content : The HTTP body
            mock_request : Mocked request to pass to the event so it can get
                           content.
        Returns:
            A tuple of (code, response)
        Raises:
            KeyError If no event is found which will handle the path.
        """
        path = self.prefix + path

        # annoyingly we return a twisted http request which has chained calls
        # to get at the http content, hence mock it here.
        mock_content = Mock()
        config = {'read.return_value': content}
        mock_content.configure_mock(**config)
        mock_request.content = mock_content

        mock_request.method = http_method
        mock_request.uri = path

        mock_request.getClientIP.return_value = "-"

        headers = {}
        if federation_auth:
            headers[b"Authorization"] = [b"X-Matrix origin=test,key=,sig="]
        mock_request.requestHeaders.getRawHeaders = mock_getRawHeaders(headers)

        # return the right path if the event requires it
        mock_request.path = path

        # add in query params to the right place
        try:
            mock_request.args = urlparse.parse_qs(path.split('?')[1])
            mock_request.path = path.split('?')[0]
            path = mock_request.path
        except Exception:
            pass

        if isinstance(path, bytes):
            path = path.decode('utf8')

        for (method, pattern, func) in self.callbacks:
            if http_method != method:
                continue

            matcher = pattern.match(path)
            if matcher:
                try:
                    args = [
                        urlparse.unquote(u)
                        for u in matcher.groups()
                    ]

                    (code, response) = yield func(
                        mock_request,
                        *args
                    )
                    defer.returnValue((code, response))
                except CodeMessageException as e:
                    defer.returnValue((e.code, cs_error(e.msg, code=e.errcode)))

        raise KeyError("No event can handle %s" % path)

    def register_paths(self, method, path_patterns, callback):
        for path_pattern in path_patterns:
            self.callbacks.append((method, path_pattern, callback))

class MockKey(object):
    alg = "mock_alg"
    version = "mock_version"
    signature = b"\x9a\x87$"

    @property
    def verify_key(self):
        return self

    def sign(self, message):
        return self

    def verify(self, message, sig):
        assert sig == b"\x9a\x87$"

class MockClock(object):
    now = 1000

    def __init__(self):
        # list of lists of [absolute_time, callback, expired] in no particular
        # order
        self.timers = []
        self.loopers = []

    def time(self):
        return self.now

    def time_msec(self):
        return self.time() * 1000

    def call_later(self, delay, callback, *args, **kwargs):
        current_context = LoggingContext.current_context()

        def wrapped_callback():
            LoggingContext.thread_local.current_context = current_context
            callback(*args, **kwargs)

        t = [self.now + delay, wrapped_callback, False]
        self.timers.append(t)

        return t

    def looping_call(self, function, interval):
        self.loopers.append([function, interval / 1000., self.now])

    def cancel_call_later(self, timer, ignore_errs=False):
        if timer[2]:
            if not ignore_errs:
                raise Exception("Cannot cancel an expired timer")

        timer[2] = True
        self.timers = [t for t in self.timers if t != timer]

    # For unit testing
    def advance_time(self, secs):
        self.now += secs

        timers = self.timers
        self.timers = []

        for t in timers:
            time, callback, expired = t

            if expired:
                raise Exception("Timer already expired")

            if self.now >= time:
                t[2] = True
                callback()
            else:
                self.timers.append(t)

        for looped in self.loopers:
            func, interval, last = looped
            if last + interval < self.now:
                func()
                looped[2] = self.now

    def advance_time_msec(self, ms):
        self.advance_time(ms / 1000.)

    def time_bound_deferred(self, d, *args, **kwargs):
        # We don't bother timing things out for now.
        return d

def _format_call(args, kwargs):
    return ", ".join(
        ["%r" % (a) for a in args] +
        ["%s=%r" % (k, v) for k, v in kwargs.items()]
    )

class DeferredMockCallable(object):
    """A callable instance that stores a set of pending call expectations and
    return values for them. It allows a unit test to assert that the given set
    of function calls are eventually made, by awaiting on them to be called.
    """

    def __init__(self):
        self.expectations = []
        self.calls = []

    def __call__(self, *args, **kwargs):
        self.calls.append((args, kwargs))

        if not self.expectations:
            raise ValueError("%r has no pending calls to handle call(%s)" % (
                self, _format_call(args, kwargs))
            )

        for (call, result, d) in self.expectations:
            if args == call[1] and kwargs == call[2]:
                d.callback(None)
                return result

        failure = AssertionError("Was not expecting call(%s)" % (
            _format_call(args, kwargs)
        ))

        for _, _, d in self.expectations:
            try:
                d.errback(failure)
            except Exception:
                pass

        raise failure

    def expect_call_and_return(self, call, result):
        self.expectations.append((call, result, defer.Deferred()))

    @defer.inlineCallbacks
    def await_calls(self, timeout=1000):
        deferred = defer.DeferredList(
            [d for _, _, d in self.expectations],
            fireOnOneErrback=True
        )

        timer = reactor.callLater(
            timeout / 1000,
            deferred.errback,
            AssertionError("%d pending calls left: %s" % (
                len([e for e in self.expectations if not e[2].called]),
                [e for e in self.expectations if not e[2].called]
            ))
        )

        yield deferred

        timer.cancel()

        self.calls = []

    def assert_had_no_calls(self):
        if self.calls:
            calls = self.calls
            self.calls = []

            raise AssertionError(
                "Expected not to received any calls, got:\n" + "\n".join([
                    "call(%s)" % _format_call(c[0], c[1]) for c in calls
                ])
            )
<|endoftext|>


<commit_before>'''
Script for collecting and processing the ZAP github stats
'''
import csv
import utils
import glob
import json
import os
import sys

tags = [
    "v2.11.1",
    "v2.11.0",
    "v2.10.0",
    "v2.9.0",
    "v2.8.0",
    ]

mappings = {
  "core": "core",
  "crossplatform": "cross",
  "linux": "linux",
  "unix": "unix",
  "deb": "deb",
  "dmg": "mac",
  "windows-x32": "win32",
  "windows.exe": "win64",
}

github_api = "https://api.github.com/repos/zaproxy/zaproxy/releases/tags/"

def collect():
    for tag in tags:
        utils.download_to_file(github_api + tag, utils.basedir() + "downloads/raw/" + utils.today() + '-' + tag + ".json")

def daily():
    # Process the download stats
    counts = {}
    assets = {}
    daily_files_to_write = set()
    monthly_files_to_write = set()
    files = sorted(glob.glob(utils.basedir() + "downloads/raw/*.json"))
    last_monthly_totals = {}
    for file in files:
        with open(file) as stats_file:
            stats = json.load(stats_file)
            date_str = os.path.basename(stats_file.name)[:10]
            daily_file = utils.basedir() + 'downloads/daily/' + date_str + '.csv'
            # Todays stats will be incomplete as this is run at the start of the day
            if not os.path.exists(daily_file) and not date_str == utils.today():
                daily_files_to_write.add(date_str)
                with open(daily_file, "a") as f:
                    print('Creating ' + daily_file)
                    f.write('date,version,name,tag,downloads\n')

            is_monthly = date_str.endswith('-01') or date_str == '2021-08-02' # No stats for 2021-08-01 :/
            daily_total = 0
            for asset in stats['assets']:
                tag = stats['name']
                name = asset['name']
                count = asset['download_count']
                if (name in counts):
                    # Ignore negative numbers - can happen when files are replaced
                    old_count = counts[name]
                else:
                    old_count = 0
                if (name in counts):
                    # Ignore negative numbers - can happen when files are replaced
                    assets[name] = max((count - counts[name]), 0)
                else:
                    assets[name] = count
                counts[name] = count
                if is_monthly:
                    daily_total += counts[name]
                mapping = "Unknown"
                for m in mappings:
                    if m in name.lower():
                        mapping = mappings[m]
                        break

                if old_count > 0:
                    # There can be multiple raw files with the same date, ie one per tag
                    if date_str in daily_files_to_write:
                        with open(daily_file, "a") as f:
                            f.write(date_str + ',' +  tag + ',' + asset['name'] + ',' + mapping + ',' + str(assets[name]) + '\n')

            if is_monthly:
                last_monthly_total = 0
                if tag in last_monthly_totals:
                    last_monthly_total = last_monthly_totals[tag]

                if tag in last_monthly_totals or tags.index(tag) == 0:
                    # we have a previous total or its the latest tag, which could be new
                    monthly_file = utils.basedir() + 'downloads/monthly/' + date_str + '.csv'
                    if not os.path.exists(monthly_file):
                        monthly_files_to_write.add(date_str)
                        with open(monthly_file, "a") as f:
                            print('Creating ' + monthly_file)
                            f.write('date,version,total,downloads\n')
                    # There can be multiple raw files with the same date, ie one per tag
                    if date_str in monthly_files_to_write:
                        with open(monthly_file, "a") as f:
                            f.write(date_str + ',' +  tag + ',' + str(daily_total) + ',' + str(daily_total - last_monthly_total) + '\n')

                last_monthly_totals[tag] = daily_total

def website():
    files = sorted(glob.glob(utils.basedir() + 'downloads/monthly/*.csv'))
    outfile = utils.websitedir() + 'site/data/charts/downloads.json'
    if not os.path.isfile(outfile):
        print('No existing file: ' + outfile)
        return

    versions = []
    map = {}

    for file in files:
        with open(file) as monthly_file:
            csv_reader = csv.reader(monthly_file)
            # Ignore the header
            next(csv_reader)
            for row in csv_reader:
                # Monthly stats tend to get recorded as the 2nd even though they really apply to the previous month
                date = row[0][:-2] + '01"'
                version = row[1]
                if len(row) > 3:
                    downloads = row[3]
                else:
                    downloads = row[2]
                if not version in versions:
                    versions.append(version)
                if not date in map:
                    map[date] = {}
                map[date][version] = downloads

    with open(outfile, 'w') as f:
        print('{', file=f)
        print('  "title": "Direct Downloads",', file=f)
        print('  "description": "Direct downloads since v2.4.3. It is worth noting that downloads have reduced since the Docker images have become more popular.",', file=f)
        print('  "columns": ["Version" ', end='', file=f)
        for l in versions:
            print(', "' + l + '"', end='', file=f)
        print('],', file=f)
        print('  "data": [', end='', file=f)

        first = True
        for date in sorted(map.keys()):
            if not first:
                print(',', end='', file=f)
            else:
                first = False
            print('\n    ["' + date, end='', file=f)
            for l in versions:
                if l in map[date] and len(map[date][l]) > 0:
                    print(', ' + map[date][l], end='', file=f)
                else:
                    print(', 0', end='', file=f)
            print(', ""]', end='', file=f)

        print('\n  ]', file=f)
        print('}', file=f)

    print('Updated: ' + outfile)
<commit_msg>Record 2.12.0 Downloads<commit_after>'''
Script for collecting and processing the ZAP github stats
'''
import csv
import utils
import glob
import json
import os
import sys

tags = [
    "v2.12.0",
    "v2.11.1",
    "v2.11.0",
    "v2.10.0",
    "v2.9.0",
    "v2.8.0",
    ]

mappings = {
  "core": "core",
  "crossplatform": "cross",
  "linux": "linux",
  "unix": "unix",
  "deb": "deb",
  "dmg": "mac",
  "windows-x32": "win32",
  "windows.exe": "win64",
}

github_api = "https://api.github.com/repos/zaproxy/zaproxy/releases/tags/"

def collect():
    for tag in tags:
        utils.download_to_file(github_api + tag, utils.basedir() + "downloads/raw/" + utils.today() + '-' + tag + ".json")

def daily():
    # Process the download stats
    counts = {}
    assets = {}
    daily_files_to_write = set()
    monthly_files_to_write = set()
    files = sorted(glob.glob(utils.basedir() + "downloads/raw/*.json"))
    last_monthly_totals = {}
    for file in files:
        with open(file) as stats_file:
            stats = json.load(stats_file)
            date_str = os.path.basename(stats_file.name)[:10]
            daily_file = utils.basedir() + 'downloads/daily/' + date_str + '.csv'
            # Todays stats will be incomplete as this is run at the start of the day
            if not os.path.exists(daily_file) and not date_str == utils.today():
                daily_files_to_write.add(date_str)
                with open(daily_file, "a") as f:
                    print('Creating ' + daily_file)
                    f.write('date,version,name,tag,downloads\n')

            is_monthly = date_str.endswith('-01') or date_str == '2021-08-02' # No stats for 2021-08-01 :/
            daily_total = 0
            for asset in stats['assets']:
                tag = stats['name']
                name = asset['name']
                count = asset['download_count']
                if (name in counts):
                    # Ignore negative numbers - can happen when files are replaced
                    old_count = counts[name]
                else:
                    old_count = 0
                if (name in counts):
                    # Ignore negative numbers - can happen when files are replaced
                    assets[name] = max((count - counts[name]), 0)
                else:
                    assets[name] = count
                counts[name] = count
                if is_monthly:
                    daily_total += counts[name]
                mapping = "Unknown"
                for m in mappings:
                    if m in name.lower():
                        mapping = mappings[m]
                        break

                if old_count > 0:
                    # There can be multiple raw files with the same date, ie one per tag
                    if date_str in daily_files_to_write:
                        with open(daily_file, "a") as f:
                            f.write(date_str + ',' +  tag + ',' + asset['name'] + ',' + mapping + ',' + str(assets[name]) + '\n')

            if is_monthly:
                last_monthly_total = 0
                if tag in last_monthly_totals:
                    last_monthly_total = last_monthly_totals[tag]

                if tag in last_monthly_totals or tags.index(tag) == 0:
                    # we have a previous total or its the latest tag, which could be new
                    monthly_file = utils.basedir() + 'downloads/monthly/' + date_str + '.csv'
                    if not os.path.exists(monthly_file):
                        monthly_files_to_write.add(date_str)
                        with open(monthly_file, "a") as f:
                            print('Creating ' + monthly_file)
                            f.write('date,version,total,downloads\n')
                    # There can be multiple raw files with the same date, ie one per tag
                    if date_str in monthly_files_to_write:
                        with open(monthly_file, "a") as f:
                            f.write(date_str + ',' +  tag + ',' + str(daily_total) + ',' + str(daily_total - last_monthly_total) + '\n')

                last_monthly_totals[tag] = daily_total

def website():
    files = sorted(glob.glob(utils.basedir() + 'downloads/monthly/*.csv'))
    outfile = utils.websitedir() + 'site/data/charts/downloads.json'
    if not os.path.isfile(outfile):
        print('No existing file: ' + outfile)
        return

    versions = []
    map = {}

    for file in files:
        with open(file) as monthly_file:
            csv_reader = csv.reader(monthly_file)
            # Ignore the header
            next(csv_reader)
            for row in csv_reader:
                # Monthly stats tend to get recorded as the 2nd even though they really apply to the previous month
                date = row[0][:-2] + '01"'
                version = row[1]
                if len(row) > 3:
                    downloads = row[3]
                else:
                    downloads = row[2]
                if not version in versions:
                    versions.append(version)
                if not date in map:
                    map[date] = {}
                map[date][version] = downloads

    with open(outfile, 'w') as f:
        print('{', file=f)
        print('  "title": "Direct Downloads",', file=f)
        print('  "description": "Direct downloads since v2.4.3. It is worth noting that downloads have reduced since the Docker images have become more popular.",', file=f)
        print('  "columns": ["Version" ', end='', file=f)
        for l in versions:
            print(', "' + l + '"', end='', file=f)
        print('],', file=f)
        print('  "data": [', end='', file=f)

        first = True
        for date in sorted(map.keys()):
            if not first:
                print(',', end='', file=f)
            else:
                first = False
            print('\n    ["' + date, end='', file=f)
            for l in versions:
                if l in map[date] and len(map[date][l]) > 0:
                    print(', ' + map[date][l], end='', file=f)
                else:
                    print(', 0', end='', file=f)
            print(', ""]', end='', file=f)

        print('\n  ]', file=f)
        print('}', file=f)

    print('Updated: ' + outfile)
<|endoftext|>


<commit_before>import os
import shutil
import tempfile
from contextlib import contextmanager

@contextmanager
def FakeProjectDirectory():
    tmpdir = tempfile.mkdtemp()
    current = os.getcwd()
    os.chdir(tmpdir)
    try:
        yield tmpdir
    finally:
        os.chdir(current)
        shutil.rmtree(tmpdir)

def add_scrapy_fake_config(tmpdir):
    # add fake scrapy.cfg
    config_path = os.path.join(tmpdir, 'scrapy.cfg')
    with open(config_path, 'w') as config_file:
        config_file.write("[settings]\ndefault=test.settings")

def add_sh_fake_config(tmpdir):
    # add fake SH config
    sh_config_path = os.path.join(tmpdir, 'scrapinghub.yml')
    with open(sh_config_path, 'w') as sh_config_file:
        sh_config_file.write('\n'.join([
            "projects:", "  dev: 12345", "images:",
            "  dev: registry/user/project",
            "  xyz: registry/xyz/project",
            "endpoints:", "  dev: https://dash-fake",
            "apikeys:", "  default: abcdef",
            "version: GIT"]))

def add_fake_requirements(tmpdir):
    """Add fake requirements"""
    reqs_path = os.path.join(tmpdir, 'fake-requirements.txt')
    with open(reqs_path, 'w') as reqs_file:
        reqs_file.write("mock\nrequests")

def add_fake_dockerfile(tmpdir):
    """Add fake Dockerfile"""
    docker_path = os.path.join(tmpdir, 'Dockerfile')
    with open(docker_path, 'w') as docker_file:
        docker_file.write("FROM python:2.7")

def add_fake_setup_py(tmpdir):
    """Add fake setup.py for extract scripts tests"""
    setup_path = os.path.join(tmpdir, 'setup.py')
    with open(setup_path, 'w') as setup_file:
        setup_file.write('\n'.join([
            "from setuptools import setup",
            "setup(name = 'project', version = '1.0',",
            "entry_points = {'scrapy': ['settings = test.settings']},",
            "scripts = ['bin/scriptA.py', 'scriptB.py'])"
        ]))
<commit_msg>Move fake setup.py/config to vars<commit_after>import os
import shutil
import tempfile
from contextlib import contextmanager

SH_CONFIG_FILE = """
projects:
  dev: 12345
images:
  dev: registry/user/project
  xyz: registry/xyz/project
endpoints:
  dev: https://dash-fake
apikeys:
  default: abcdef
version: GIT
"""

SH_SETUP_FILE = """
from setuptools import setup
setup(
    name = 'project', version = '1.0',
    entry_points = {'scrapy': ['settings = test.settings']},
    scripts = ['bin/scriptA.py', 'scriptB.py']
)
"""

@contextmanager
def FakeProjectDirectory():
    tmpdir = tempfile.mkdtemp()
    current = os.getcwd()
    os.chdir(tmpdir)
    try:
        yield tmpdir
    finally:
        os.chdir(current)
        shutil.rmtree(tmpdir)

def add_scrapy_fake_config(tmpdir):
    # add fake scrapy.cfg
    config_path = os.path.join(tmpdir, 'scrapy.cfg')
    with open(config_path, 'w') as config_file:
        config_file.write("[settings]\ndefault=test.settings")

def add_sh_fake_config(tmpdir):
    # add fake SH config
    sh_config_path = os.path.join(tmpdir, 'scrapinghub.yml')
    with open(sh_config_path, 'w') as sh_config_file:
        sh_config_file.write(SH_CONFIG_FILE)

def add_fake_requirements(tmpdir):
    """Add fake requirements"""
    reqs_path = os.path.join(tmpdir, 'fake-requirements.txt')
    with open(reqs_path, 'w') as reqs_file:
        reqs_file.write("mock\nrequests")

def add_fake_dockerfile(tmpdir):
    """Add fake Dockerfile"""
    docker_path = os.path.join(tmpdir, 'Dockerfile')
    with open(docker_path, 'w') as docker_file:
        docker_file.write("FROM python:2.7")

def add_fake_setup_py(tmpdir):
    """Add fake setup.py for extract scripts tests"""
    setup_path = os.path.join(tmpdir, 'setup.py')
    with open(setup_path, 'w') as setup_file:
        setup_file.write(SH_SETUP_FILE)
<|endoftext|>


<commit_before>import os
import pwd
import grp

# update pam_chroot configuration
def UpdateChroot(group='jail'):
	# update chroot pam configuration
	open('/etc/security/chroot.conf', 'w+').write(
		'\n'.join(
			map(lambda x: '{0} /jail/root/{0}'.format(x), grp.getgrnam(group).gr_mem))+'\n')

# update jail user directories
def UpdateUser(user):
	JailDirectories = (
		('/home/{0}',				True,		0750),
		('/jail/root/{0}',			False,		0755),
		('/jail/home/{0}',			False,		0750),
		('/jail/home/{0}/etc',			False,		0755),
		('/jail/home/{0}/var',			False,		0755),
		('/jail/home/{0}/var/log',		False,		0755),
		('/jail/home/{0}/var/run',		False,		0755),
		('/jail/home/{0}/tmp',			False,		1777),
		('/jail/home/{0}/home',			False,		0755),
		('/jail/home/{0}/home/{0}',		True,		0750),
		('/jail/home/{0}/home/{0}/logs',	True,		0755),
		('/jail/home/{0}/home/{0}/www',		True,		0750),
		('/jail/home/{0}/home/{0}/mail',	True,		0750),
		('/var/log/apache2/hosting/{0}',	False,		0750),
	)

	# 3. create jail directories
	for dirname, chown, mode in JailDirectories:
		dirname = dirname.format(user)
		if not os.path.exists(dirname):
			os.makedirs(dirname)
			if chown:
				os.chown(
					dirname,
					pwd.getpwnam(user).pw_uid,
					pwd.getpwnam(user).pw_gid)
		os.chmod(dirname, mode)

	# 4. create empty files
	for f in (os.path.join('/jail/home', user, 'var/log/wtmp'),
		os.path.join('/jail/home', user, 'var/log/utmp')):
		if not os.path.exists(f):
			open(f, 'w').close()
		os.chmod(f, 0644)

	# 5. generate custom passwd file
	passwd_path = os.path.join('/jail/home', user, 'etc/passwd')
	if not os.path.exists(passwd_path):
		userdata = [
			map(str, x) for x in
				[pw for pw in map(tuple, pwd.getpwall()) if
					pw[2] <= 100 or pw[0] == user]]
		open(passwd_path, 'w').write('\n'.join(map(':'.join, userdata))+'\n')
		os.chmod(passwd_path, 0644)

	# 6. generate custom group file
	group_path = os.path.join('/jail/home', user, 'etc/group')
	if not os.path.exists(group_path):
		groupdata = [
			map(str, gr[0:3]) + [','.join(gr[3])] for gr in
				map(tuple, grp.getgrall()) if
					gr[2] <= 100 or gr[0] == user or user in gr[3]]
		open(group_path, 'w').write('\n'.join(map(':'.join, groupdata))+'\n')
		os.chmod(group_path, 0644)

# update all jail user directories
def UpdateGroup(group='jail'):
	for user in grp.getgrnam(group).gr_mem:
		UpdateUser(user)
	UpdateChroot(group)
<commit_msg>Strict files and directories mode change on update<commit_after>import os
import pwd
import grp

# update pam_chroot configuration
def UpdateChroot(group='jail'):
	# update chroot pam configuration
	open('/etc/security/chroot.conf', 'w+').write(
		'\n'.join(
			map(lambda x: '{0} /jail/root/{0}'.format(x), grp.getgrnam(group).gr_mem))+'\n')

# update jail user directories
def UpdateUser(user):
	JailDirectories = (
		('/home/{0}',				True,		00750),
		('/jail/root/{0}',			False,		00755),
		('/jail/home/{0}',			False,		00750),
		('/jail/home/{0}/etc',			False,		00755),
		('/jail/home/{0}/var',			False,		00755),
		('/jail/home/{0}/var/log',		False,		00755),
		('/jail/home/{0}/var/run',		False,		00755),
		('/jail/home/{0}/tmp',			False,		01777),
		('/jail/home/{0}/home',			False,		00755),
		('/jail/home/{0}/home/{0}',		True,		00750),
		('/jail/home/{0}/home/{0}/logs',	True,		00755),
		('/jail/home/{0}/home/{0}/www',		True,		00750),
		('/jail/home/{0}/home/{0}/mail',	True,		00750),
		('/var/log/apache2/hosting/{0}',	False,		00750),
	)

	# 3. create jail directories
	for dirname, chown, mode in JailDirectories:
		dirname = dirname.format(user)
		if not os.path.exists(dirname):
			os.makedirs(dirname)
			if chown:
				os.chown(
					dirname,
					pwd.getpwnam(user).pw_uid,
					pwd.getpwnam(user).pw_gid)
		os.chmod(dirname, mode)

	# 4. create empty files
	for f in (os.path.join('/jail/home', user, 'var/log/wtmp'),
		os.path.join('/jail/home', user, 'var/log/utmp')):
		if not os.path.exists(f):
			open(f, 'w').close()
		os.chmod(f, 0644)

	# 5. generate custom passwd file
	passwd_path = os.path.join('/jail/home', user, 'etc/passwd')
	if not os.path.exists(passwd_path):
		userdata = [
			map(str, x) for x in
				[pw for pw in map(tuple, pwd.getpwall()) if
					pw[2] <= 100 or pw[0] == user]]
		open(passwd_path, 'w').write('\n'.join(map(':'.join, userdata))+'\n')
		os.chmod(passwd_path, 0644)

	# 6. generate custom group file
	group_path = os.path.join('/jail/home', user, 'etc/group')
	if not os.path.exists(group_path):
		groupdata = [
			map(str, gr[0:3]) + [','.join(gr[3])] for gr in
				map(tuple, grp.getgrall()) if
					gr[2] <= 100 or gr[0] == user or user in gr[3]]
		open(group_path, 'w').write('\n'.join(map(':'.join, groupdata))+'\n')
		os.chmod(group_path, 0644)

# update all jail user directories
def UpdateGroup(group='jail'):
	for user in grp.getgrnam(group).gr_mem:
		UpdateUser(user)
	UpdateChroot(group)
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models

from picklefield.fields import PickledObjectField

from annotations.models import Language, Tense, Corpus, Document, SubCorpus, Fragment, Label, LabelKey

class Scenario(models.Model):
    title = models.CharField(max_length=200)
    description = models.TextField()
    is_test = models.BooleanField(
        'Test scenario',
        default=False,
        help_text='Checking this box signals that the scenario should not be displayed in the standard overview.')

    corpus = models.ForeignKey(Corpus, on_delete=models.CASCADE)
    documents = models.ManyToManyField(Document, blank=True)
    subcorpora = models.ManyToManyField(SubCorpus, blank=True)

    formal_structure = models.PositiveIntegerField(
        'Formal structure', choices=Fragment.FORMAL_STRUCTURES, default=Fragment.FS_NONE)
    formal_structure_strict = models.BooleanField(
        'Require translations to be in the same formal structure', default=True)

    sentence_function = models.PositiveIntegerField(
        'Sentence function', choices=Fragment.SENTENCE_FUNCTIONS, default=Fragment.SF_NONE)

    mds_dimensions = models.PositiveIntegerField(
        'Number of dimensions',
        default=5,
        validators=[MinValueValidator(2), MaxValueValidator(5)],
        help_text='Number of dimensions to use in Multidimensional Scaling. Should be between 2 and 5.'
    )
    mds_model = PickledObjectField('MDS model', null=True)
    mds_matrix = PickledObjectField('MDS matrix', null=True)
    mds_fragments = PickledObjectField('MDS fragments', null=True)
    mds_labels = PickledObjectField('MDS labels', null=True)
    mds_stress = models.FloatField('MDS stress', null=True)
    mds_allow_partial = models.BooleanField(
        'Allow partial tuples in model', default=False,
        help_text='When enabled, the model will include tuples '
                  'where one or more of the target languages have no Annotation')

    last_run = models.DateTimeField(blank=True, null=True)

    owner = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='scenarios', null=True, on_delete=models.SET_NULL)

    def from_languages(self):
        languages = self.languages_from if hasattr(self, 'languages_from') else self.languages(as_from=True)
        return ', '.join([sl.language.title for sl in languages])

    def to_languages(self):
        languages = self.languages_to if hasattr(self, 'languages_to') else self.languages(as_to=True)
        return ', '.join([sl.language.title for sl in languages])

    def languages(self, **kwargs):
        return self.scenariolanguage_set.filter(**kwargs).select_related('language')

    def __str__(self):
        return self.title

    def get_labels(self):
        # format mds_labels in a way that works with the recent changes.
        # this prevents us from having to rerun all existing scenarios.
        result = dict()
        for language, values in self.mds_labels.items():
            if isinstance(values[0], int):
                result[language] = [('Tense:{}'.format(v),) for v in values]
            if isinstance(values[0], str):
                labels = []
                for v in values:
                    try:
                        keys = LabelKey.objects.filter(corpora=self.corpus)
                        label = Label.objects.get(key__in=keys, title=v)
                        labels.append(label)
                    except:  # If all else fails, just continue...
                        continue
                result[language] = [('Label:{}'.format(label.pk),) for label in labels]
            else:
                result[language] = values
        return result

class ScenarioLanguage(models.Model):
    scenario = models.ForeignKey(Scenario, on_delete=models.CASCADE)

    language = models.ForeignKey(Language, on_delete=models.CASCADE)
    as_from = models.BooleanField()
    as_to = models.BooleanField()
    tenses = models.ManyToManyField(Tense, blank=True)

    use_tenses = models.BooleanField(default=True)

    use_labels = models.BooleanField(default=False)
    include_keys = models.ManyToManyField(
        LabelKey, blank=True, related_name='+',
        help_text='Keys selected here will be used as components of the fragment tuples.')
    include_labels = models.ManyToManyField(
        Label, blank=True, related_name='+',
        verbose_name='Filter by labels',
        help_text='Fragments will be included in the scenario only '
                  'when they are assigned one of the selected labels. Leave emtpy to include all fragments.')

    # backward compatibility
    @property
    def use_other_label(self):
        return self.use_labels

    def __str__(self):
        return 'Details for language {} in scenario {}'.format(self.language.title, self.scenario.title)
<commit_msg>Added TODO<commit_after>from django.conf import settings
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models

from picklefield.fields import PickledObjectField

from annotations.models import Language, Tense, Corpus, Document, SubCorpus, Fragment, Label, LabelKey

class Scenario(models.Model):
    title = models.CharField(max_length=200)
    description = models.TextField()
    is_test = models.BooleanField(
        'Test scenario',
        default=False,
        help_text='Checking this box signals that the scenario should not be displayed in the standard overview.')

    corpus = models.ForeignKey(Corpus, on_delete=models.CASCADE)
    documents = models.ManyToManyField(Document, blank=True)
    subcorpora = models.ManyToManyField(SubCorpus, blank=True)

    formal_structure = models.PositiveIntegerField(
        'Formal structure', choices=Fragment.FORMAL_STRUCTURES, default=Fragment.FS_NONE)
    formal_structure_strict = models.BooleanField(
        'Require translations to be in the same formal structure', default=True)

    sentence_function = models.PositiveIntegerField(
        'Sentence function', choices=Fragment.SENTENCE_FUNCTIONS, default=Fragment.SF_NONE)

    mds_dimensions = models.PositiveIntegerField(
        'Number of dimensions',
        default=5,
        validators=[MinValueValidator(2), MaxValueValidator(5)],
        help_text='Number of dimensions to use in Multidimensional Scaling. Should be between 2 and 5.'
    )
    mds_model = PickledObjectField('MDS model', null=True)
    mds_matrix = PickledObjectField('MDS matrix', null=True)
    mds_fragments = PickledObjectField('MDS fragments', null=True)
    mds_labels = PickledObjectField('MDS labels', null=True)
    mds_stress = models.FloatField('MDS stress', null=True)
    mds_allow_partial = models.BooleanField(
        'Allow partial tuples in model', default=False,
        help_text='When enabled, the model will include tuples '
                  'where one or more of the target languages have no Annotation')

    last_run = models.DateTimeField(blank=True, null=True)

    owner = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='scenarios', null=True, on_delete=models.SET_NULL)

    def from_languages(self):
        languages = self.languages_from if hasattr(self, 'languages_from') else self.languages(as_from=True)
        return ', '.join([sl.language.title for sl in languages])

    def to_languages(self):
        languages = self.languages_to if hasattr(self, 'languages_to') else self.languages(as_to=True)
        return ', '.join([sl.language.title for sl in languages])

    def languages(self, **kwargs):
        return self.scenariolanguage_set.filter(**kwargs).select_related('language')

    def __str__(self):
        return self.title

    def get_labels(self):
        # format mds_labels in a way that works with the recent changes.
        # this prevents us from having to rerun all existing scenarios.
        result = dict()
        for language, values in self.mds_labels.items():
            if isinstance(values[0], int):
                result[language] = [('Tense:{}'.format(v),) for v in values]
            if isinstance(values[0], str):
                labels = []
                for v in values:
                    try:
                        # TODO: This generates a lot of queries! (only for old scenarios though)
                        keys = LabelKey.objects.filter(corpora=self.corpus)
                        label = Label.objects.get(key__in=keys, title=v)
                        labels.append(label)
                    except:  # If all else fails, just continue...
                        continue
                result[language] = [('Label:{}'.format(label.pk),) for label in labels]
            else:
                result[language] = values
        return result

class ScenarioLanguage(models.Model):
    scenario = models.ForeignKey(Scenario, on_delete=models.CASCADE)

    language = models.ForeignKey(Language, on_delete=models.CASCADE)
    as_from = models.BooleanField()
    as_to = models.BooleanField()
    tenses = models.ManyToManyField(Tense, blank=True)

    use_tenses = models.BooleanField(default=True)

    use_labels = models.BooleanField(default=False)
    include_keys = models.ManyToManyField(
        LabelKey, blank=True, related_name='+',
        help_text='Keys selected here will be used as components of the fragment tuples.')
    include_labels = models.ManyToManyField(
        Label, blank=True, related_name='+',
        verbose_name='Filter by labels',
        help_text='Fragments will be included in the scenario only '
                  'when they are assigned one of the selected labels. Leave emtpy to include all fragments.')

    # backward compatibility
    @property
    def use_other_label(self):
        return self.use_labels

    def __str__(self):
        return 'Details for language {} in scenario {}'.format(self.language.title, self.scenario.title)
<|endoftext|>


<commit_before># Copyright 2020 The cert-manager Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")

def helm_pkg(
    name,
    chart_name,
    chart_yaml,
    values_yaml,
    readme_file,
    tpl_files,
    srcs = [],
    helm_cmd = "//hack/bin:helm",
    version_file = "//:version",
    **kwargs):

    pkg_tar(
        name = "%s.template_files" % name,
        package_dir = "/%s/templates" % chart_name,
        srcs = srcs,
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.tpl_files" % name,
        package_dir = "/%s/templates" % chart_name,
        srcs = tpl_files,
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.chart_files" % name,
        package_dir = "/%s" % chart_name,
        srcs = [
            chart_yaml,
            values_yaml,
            readme_file,
        ],
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.dir_tar" % name,
        extension = "tar.gz",
        deps = [
            "%s.template_files" % name,
            "%s.tpl_files" % name,
            "%s.chart_files" % name,
        ],
        visibility = ["//visibility:private"],
    )

    cmds = []
    cmds = cmds + ["tar xf $(location %s.dir_tar)" % name]
    cmds = cmds + ["version=$$(cat $(location %s))" % version_file]
    cmds = cmds + [" ".join([
        "$(location %s)" % helm_cmd,
        "package",
        "--app-version=$$version",
        "--version=$$version > /dev/null 2>&1",
        "./%s" % chart_name,
    ])]
    cmds = cmds + ["mv \"%s-$$version.tgz\" $@" % chart_name]
    native.genrule(
        name = name,
        srcs = [
            version_file,
            ":%s.dir_tar" % name,
        ],
        stamp = 1,
        outs = ["%s.tgz" % name],
        cmd = "; ".join(cmds),
        tools = [helm_cmd],
        **kwargs
    )

def helm_tmpl(
        name,
        helm_pkg,
        release_namespace,
        release_name,
        additional_api_versions = "",
        values = {},
        helm_cmd = "//hack/bin:helm",
        **kwargs,
):
    cmds = []
    set_args = []
    for k, v in values.items():
        set_args = set_args + ["--set=%s=%s" % (k, v)]
    tmpl_cmd = [
        "$(location %s)" % helm_cmd,
        "template",
        "--api-versions=\"%s\"" % additional_api_versions,
        "--namespace=%s" % release_namespace,
        release_name,
        "$(location %s)" % helm_pkg,
    ] + set_args + ["> $@"]
    cmds = cmds + [" ".join(tmpl_cmd)]
    native.genrule(
        name = name,
        srcs = [helm_pkg, "//:version"],
        stamp = 1,
        outs = ["%s.yaml" % name],
        cmd = "; ".join(cmds),
        tools = [helm_cmd],
        **kwargs,
    )

def helm_chart_yaml(
    name,
    chart_yaml_template,
    version_file = "//:version",
    **kwargs,
):
    """
    We don't want alpha and beta releases to show on ArtifactHub as releases.
    To prevent pre-releases from appearing, we use the ArtifactHub-specific
    annotation:
      artifacthub.io/prerelease: "true"

    See https://artifacthub.io/docs/topics/annotations/helm/
    """
    native.genrule(
        name = name,
        srcs = [version_file, chart_yaml_template],
        stamp = 1,
        outs = ["Chart.yaml"],
        cmd = """
        awk '
          BEGIN{
            getline v < "$(location %s)"
            pr = match(v, "^v[0-9]+.[0-9]+.[0-9]+$$") == 0 ? "true" : "false"
          }
          /{PRERELEASE}/{
            gsub("{PRERELEASE}", pr)
          }1
        ' $(location %s) > $@
        """ % (version_file, chart_yaml_template),
        **kwargs,
    )
<commit_msg>Use awk regular expression delimeters and a more precise regexp for the version<commit_after># Copyright 2020 The cert-manager Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")

def helm_pkg(
    name,
    chart_name,
    chart_yaml,
    values_yaml,
    readme_file,
    tpl_files,
    srcs = [],
    helm_cmd = "//hack/bin:helm",
    version_file = "//:version",
    **kwargs):

    pkg_tar(
        name = "%s.template_files" % name,
        package_dir = "/%s/templates" % chart_name,
        srcs = srcs,
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.tpl_files" % name,
        package_dir = "/%s/templates" % chart_name,
        srcs = tpl_files,
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.chart_files" % name,
        package_dir = "/%s" % chart_name,
        srcs = [
            chart_yaml,
            values_yaml,
            readme_file,
        ],
        mode = "0644",
        visibility = ["//visibility:private"],
    )

    pkg_tar(
        name = "%s.dir_tar" % name,
        extension = "tar.gz",
        deps = [
            "%s.template_files" % name,
            "%s.tpl_files" % name,
            "%s.chart_files" % name,
        ],
        visibility = ["//visibility:private"],
    )

    cmds = []
    cmds = cmds + ["tar xf $(location %s.dir_tar)" % name]
    cmds = cmds + ["version=$$(cat $(location %s))" % version_file]
    cmds = cmds + [" ".join([
        "$(location %s)" % helm_cmd,
        "package",
        "--app-version=$$version",
        "--version=$$version > /dev/null 2>&1",
        "./%s" % chart_name,
    ])]
    cmds = cmds + ["mv \"%s-$$version.tgz\" $@" % chart_name]
    native.genrule(
        name = name,
        srcs = [
            version_file,
            ":%s.dir_tar" % name,
        ],
        stamp = 1,
        outs = ["%s.tgz" % name],
        cmd = "; ".join(cmds),
        tools = [helm_cmd],
        **kwargs
    )

def helm_tmpl(
        name,
        helm_pkg,
        release_namespace,
        release_name,
        additional_api_versions = "",
        values = {},
        helm_cmd = "//hack/bin:helm",
        **kwargs,
):
    cmds = []
    set_args = []
    for k, v in values.items():
        set_args = set_args + ["--set=%s=%s" % (k, v)]
    tmpl_cmd = [
        "$(location %s)" % helm_cmd,
        "template",
        "--api-versions=\"%s\"" % additional_api_versions,
        "--namespace=%s" % release_namespace,
        release_name,
        "$(location %s)" % helm_pkg,
    ] + set_args + ["> $@"]
    cmds = cmds + [" ".join(tmpl_cmd)]
    native.genrule(
        name = name,
        srcs = [helm_pkg, "//:version"],
        stamp = 1,
        outs = ["%s.yaml" % name],
        cmd = "; ".join(cmds),
        tools = [helm_cmd],
        **kwargs,
    )

def helm_chart_yaml(
    name,
    chart_yaml_template,
    version_file = "//:version",
    **kwargs,
):
    """
    We don't want alpha and beta releases to show on ArtifactHub as releases.
    To prevent pre-releases from appearing, we use the ArtifactHub-specific
    annotation:
      artifacthub.io/prerelease: "true"

    See https://artifacthub.io/docs/topics/annotations/helm/
    """
    native.genrule(
        name = name,
        srcs = [version_file, chart_yaml_template],
        stamp = 1,
        outs = ["Chart.yaml"],
        cmd = """
        awk '
          BEGIN{
            getline v < "$(location %s)"
            pr = match(v, /^v[0-9]+\.[0-9]+\.[0-9]+$$/) == 0 ? "true" : "false"
          }
          /{PRERELEASE}/{
            gsub(/{PRERELEASE}/, pr)
          }1
        ' $(location %s) > $@
        """ % (version_file, chart_yaml_template),
        **kwargs,
    )
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# Yet another NumerousApp atomicity test program.
# Can test atomicity of ADD operation and "OnlyIfChanged"
#
import argparse
import numerous
import queue
import threading

#
# arguments:
#    -c credspec  : the usual
#    -n num       : total number of requests to send to server
#    -m metric    : use metric as the test metric (see discussion below)
#    -p parallel  : number of threads to run in parallel (default is "-n" value)
#    -D           : debug flag
#    -q           : quiet - don't display all the individual results
#
# (Exactly) One non-optional argument must be given:
#        ADD - to run the ADD version of the test
#        ONLY - to run the OnlyIfChanged version of the test
#
# If no metric is supplied, the program will create its own metric and use it
# and delete that metric on normal exit (any abnormal exit won't delete it)
#
# If a metric argument is supplied, the program will:
#     - first attempt to use it as a metric ID. If that works, use that metric.
#     - then attempt to look it up ByLabel 'STRING'
#     - then attempt to look it up ByLabel 'ONE' (regexp processing will work)
#     - then simply use it as a label for a created metric
#
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--credspec')
parser.add_argument('-n', '--nwrites', type=int, default=50)
parser.add_argument('-D', '--debug', action="count")
parser.add_argument('-m', '--metric')
parser.add_argument('-p', '--parallel', type=int, default=-1)
parser.add_argument('-q', '--quiet', action="store_true")
parser.add_argument('testtype', choices=['ADD', 'ONLY'])
args = parser.parse_args()

if args.parallel < 1:
    args.parallel = args.nwrites

#
# This is the test function that is run in an individual thread, potentially
# in parallel (multiple threads running this function).
#
# Note in particular that we instantiate a local Numerous(), so it will be per-thread
# (versus the toplevel global one used to set up the test conditions).
# Similarly we instantiate a local metric (from the test metric's ID), again
# so it will be local to this thread vs shared across all the threads.
#
# The arguments are:
#    k      - the Numerous API key
#    mID    - the ID of the metric we should be banging on
#    n      - the number of calls to make
#    wargs  - arguments to pass to the metric.write() (aside from newval)
#    q      - place (queue) for output results that will be picked up in main thread
#
def tester(k, mID, n, wargs, q):
    # excessively clever lambda that bypasses the voluntary throttle aspect of the
    # system default policy while still using the 429/500 bit
    nr = numerous.Numerous(apiKey=k, throttle= lambda nr, tp, td, up:
                (tp['result-code'] in (429, 500)) and up[0](nr, tp, up[1], up[2]))

    testmetric = nr.metric(mID)
    for i in range(n):
        try:
            v = testmetric.write(1, **wargs)
        except numerous.NumerousMetricConflictError:
            v = "NoChange"
        q.put(v)

apiKey = numerous.numerousKey(args.credspec)
nrMain = numerous.Numerous(apiKey=apiKey)

testmetric = None

if args.metric:
    #  - first attempt to use it as a metric ID. If that works, use that metric.
    #  - then attempt to look it up ByLabel 'STRING'
    #  - then attempt to look it up ByLabel 'ONE' (regexp processing will work)
    for mt in [ 'ID', 'STRING', 'ONE' ]:
        try:
            testmetric = nrMain.metricByLabel(args.metric, matchType = mt)
            if testmetric:
                break
        except numerous.NumerousError:   # can potentially get "conflict"
            pass

# At this point either we found a metric (not None) or we have to create one.
# If we have to create it, use the args.metric as the label, if given.
deleteIt = False
if not testmetric:
    attrs = { 'private' : True,
              'description' : 'used by yatom testing program. Ok to delete'
            }
    testmetric = nrMain.createMetric(args.metric or 'yatom-temp', 0, attrs=attrs)
    deleteIt = True

if args.debug:
    nrMain.debug(10)

# always start the testmetric at zero
testmetric.write(0)

# the arguments to pass to the metric.write() routine
if args.testtype == "ADD":
    wargs = { 'add' : True }
elif args.testtype == "ONLY":
    wargs = { 'onlyIf' : True }
else:
    print("Bad testtype",args.testtype)
    exit(1)

#
# create one thread per "parallel" specified.
# Each thread will separately instantiate its own nr and its own copy
# of a metric object (all referring back to the same metric on the server)
# Each thread will put a resulting value on the outputQ when it is done.

#

outputQ = queue.Queue()

# create the threads...
theThreads = []
for i in range(args.parallel):
    nToWrite = args.nwrites//args.parallel
    # account for residue, some threads will write 1 extra
    if i < (args.nwrites%args.parallel):
        nToWrite += 1

    threadArgs = (apiKey, testmetric.id, nToWrite, wargs, outputQ)
    theThreads.append(threading.Thread(target=tester, args=threadArgs))

# start all the threads...
for t in theThreads:
    t.start()

# wait for all the threads
# we don't know that they will finish in this order (duh) but it doesn't
# matter (duh); the point is that we need to join all of them so we know they
# have all finished
for t in theThreads:
    t.join()

# and now just print all their results, if requested
if not args.quiet:
    while True:
        try:
            print(outputQ.get(block=False))
        except queue.Empty:
            break

if deleteIt:
    testmetric.crushKillDestroy()

<commit_msg>Fix up yatom.py atomicity tests<commit_after>#!/usr/bin/python3
#
# Yet another NumerousApp atomicity test program.
# Can test atomicity of ADD operation and "OnlyIfChanged"
#
import argparse
import numerous
import queue
import threading

#
# arguments:
#    -c credspec  : the usual
#    -n num       : total number of requests to send to server
#    -m metric    : use metric as the test metric (see discussion below)
#    -p parallel  : number of threads run in parallel (default is 10)
#    -D           : debug flag
#    -q           : quiet - don't display all the individual results
#
# (Exactly) One non-optional argument must be given:
#        ADD - to run the ADD version of the test
#        ONLY - to run the OnlyIfChanged version of the test
#
# If no metric is supplied, the program will create its own metric and use it
# and delete that metric on normal exit (any abnormal exit won't delete it)
#
# If a metric argument is supplied, the program will:
#     - first attempt to use it as a metric ID. If that works, use that metric.
#     - then attempt to look it up ByLabel 'STRING'
#     - then attempt to look it up ByLabel 'ONE' (regexp processing will work)
#     - then simply use it as a label for a created metric
#
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--credspec')
parser.add_argument('-n', '--nwrites', type=int, default=50)
parser.add_argument('-D', '--debug', action="count")
parser.add_argument('-m', '--metric')
parser.add_argument('-p', '--parallel', type=int, default=10)
parser.add_argument('-q', '--quiet', action="store_true")
parser.add_argument('testtype', choices=['ADD', 'ONLY'])
args = parser.parse_args()

# sanity on parallel threads ... can't be less than 1
if args.parallel < 1:
    args.parallel = args.nwrites

# can't usefully be more than nwrites
if args.parallel > args.nwrites:
    args.parallel = args.nwrites

#
# This is the test function that is run in an individual thread, potentially
# in parallel (multiple threads running this function).
#
# Note in particular that we instantiate a local Numerous(), so it will
# be per-thread (versus the toplevel global one used to set up the test
# conditions). Similarly we instantiate a local metric (from the
# test metric's ID), again so it will be local to this thread vs shared
# across all the threads.
#
# The arguments are:
#    k      - the Numerous API key
#    mID    - the ID of the metric we should be banging on
#    n      - the number of calls to make
#    wargs  - arguments to pass to the metric.write() (aside from newval)
#    q      - queue for output results that will be picked up in main thread
#
def tester(k, mID, n, wargs, q):
    # excessively clever lambda that bypasses the voluntary throttle of
    # system default policy while still using it for the 429
    nr = numerous.Numerous(apiKey=k, throttle= lambda nr, tp, td, up:
            (tp['result-code'] == 429) and up[0](nr, tp, up[1], up[2]))

    testmetric = nr.metric(mID)
    for i in range(n):
        try:
            v = testmetric.write(1, **wargs)
        except numerous.NumerousMetricConflictError:
            v = "NoChange"
        q.put(v)

apiKey = numerous.numerousKey(args.credspec)
nrMain = numerous.Numerous(apiKey=apiKey)

testmetric = None

if args.metric:
    #  - first attempt to use it as a metric ID. If works, use that metric.
    #  - then attempt to look it up ByLabel 'STRING'
    #  - then attempt to look it up ByLabel 'ONE' (regexp processing will work)
    for mt in [ 'ID', 'STRING', 'ONE' ]:
        try:
            testmetric = nrMain.metricByLabel(args.metric, matchType = mt)
            if testmetric:
                break
        except numerous.NumerousError:   # can potentially get "conflict"
            pass

# At this point either we found a metric (not None) or we have to create one.
# If we have to create it, use the args.metric as the label, if given.
deleteIt = False
if not testmetric:
    a = { 'private' : True,
          'description' : 'used by yatom testing program. Ok to delete'
        }
    testmetric = nrMain.createMetric(args.metric or 'yatom-temp', 0, attrs=a)
    deleteIt = True

if args.debug:
    if args.debug > 1:
        nrMain.debug(10)
    else:
        nrMain.debug(1)

# always start the testmetric at zero
testmetric.write(0)

# the arguments to pass to the metric.write() routine
if args.testtype == "ADD":
    wargs = { 'add' : True }
elif args.testtype == "ONLY":
    wargs = { 'onlyIf' : True }
else:
    print("Bad testtype",args.testtype)
    exit(1)

#
# create one thread per "parallel" specified.
# Each thread will separately instantiate its own nr and its own copy
# of a metric object (all referring back to the same metric on the server)
# Each thread will put a resulting value on the outputQ when it is done.

#

outputQ = queue.Queue()

# create the threads...
theThreads = []
for i in range(args.parallel):
    nToWrite = args.nwrites//args.parallel
    # account for residue, some threads will write 1 extra
    if i < (args.nwrites%args.parallel):
        nToWrite += 1

    threadArgs = (apiKey, testmetric.id, nToWrite, wargs, outputQ)
    theThreads.append(threading.Thread(target=tester, args=threadArgs))

# start all the threads...
for t in theThreads:
    t.start()

# wait for all the threads
# we don't know that they will finish in this order (duh) but it doesn't
# matter (duh); the point is that we need to join all of them so we know they
# have all finished
for t in theThreads:
    t.join()

# and now just print all their results, if requested
if not args.quiet:
    while True:
        try:
            print(outputQ.get(block=False))
        except queue.Empty:
            break

exitStatus = 0

if args.testtype == "ADD":
    if testmetric.read() != args.nwrites:
        exitStatus = 1
        if not args.quiet:
            print("Wrong final value: {}".format(testmetric.read()))

if deleteIt:
    testmetric.crushKillDestroy()

exit(exitStatus)

<|endoftext|>


<commit_before># Little helper to allow python modules in current jasylibrarys path
import sys, os.path, inspect
filename = inspect.getframeinfo(inspect.currentframe()).filename
path = os.path.dirname(os.path.abspath(filename))
sys.path.append(path)

import konstrukteur.Konstrukteur
import jasy.asset.Manager

@share
def build(profile, regenerate = False):
	""" Build static website """

	def getPartUrl(part, type):
		folder = ""
		if type == "css":
			folder = profile.getCssFolder()

		outputPath = os.path.relpath(os.path.join(profile.getDestinationPath(), folder), profile.getWorkingPath())
		filename = profile.expandFileName("%s/%s-{{id}}.%s" % (outputPath, part, type))

		return filename

	profile.addCommand("part.url", getPartUrl, "url")

	for permutation in profile.permutate():
		konstrukteur.Konstrukteur.build(regenerate, profile)
<commit_msg>Fix detecting css output folder in different jasy versions<commit_after># Little helper to allow python modules in current jasylibrarys path
import sys, os.path, inspect
filename = inspect.getframeinfo(inspect.currentframe()).filename
path = os.path.dirname(os.path.abspath(filename))
sys.path.append(path)

import konstrukteur.Konstrukteur
import jasy.asset.Manager

@share
def build(profile, regenerate = False):
	""" Build static website """

	def getPartUrl(part, type):
		folder = ""
		if type == "css":
			if hasattr(profile, "getCssFolder"):
				# Old jasy < 1.5-beta4
				folder = profile.getCssFolder()
			else:
				# New jasy >= 1.5-beta4
				folder = profile.getCssOutputFolder()

		outputPath = os.path.relpath(os.path.join(profile.getDestinationPath(), folder), profile.getWorkingPath())
		filename = profile.expandFileName("%s/%s-{{id}}.%s" % (outputPath, part, type))

		return filename

	profile.addCommand("part.url", getPartUrl, "url")

	for permutation in profile.permutate():
		konstrukteur.Konstrukteur.build(regenerate, profile)
<|endoftext|>