<commit_before>import sys
import struct
from datetime import datetime, time, timedelta

from gevent.server import DatagramServer

from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.sql.expression import and_, or_

from skylines.model import db, User, TrackingFix, Follower, Elevation
from skylines.tracking.crc import check_crc, set_crc

# More information about this protocol can be found in the XCSoar
# source code, source file src/Tracking/SkyLines/Protocol.hpp

MAGIC = 0x5df4b67b
TYPE_PING = 1
TYPE_ACK = 2
TYPE_FIX = 3
TYPE_TRAFFIC_REQUEST = 4
TYPE_TRAFFIC_RESPONSE = 5
TYPE_USER_NAME_REQUEST = 6
TYPE_USER_NAME_RESPONSE = 7

FLAG_ACK_BAD_KEY = 0x1

FLAG_LOCATION = 0x1
FLAG_TRACK = 0x2
FLAG_GROUND_SPEED = 0x4
FLAG_AIRSPEED = 0x8
FLAG_ALTITUDE = 0x10
FLAG_VARIO = 0x20
FLAG_ENL = 0x40

# for TYPE_TRAFFIC_REQUEST
TRAFFIC_FLAG_FOLLOWEES = 0x1
TRAFFIC_FLAG_CLUB = 0x2

USER_FLAG_NOT_FOUND = 0x1

def log(message):
    print message
    sys.stdout.flush()

class TrackingServer(DatagramServer):
    def init_app(self, app):
        self.app = app

    def pingReceived(self, host, port, key, payload):
        if len(payload) != 8: return
        id, reserved, reserved2 = struct.unpack('!HHI', payload)

        flags = 0

        pilot = User.by_tracking_key(key)
        if not pilot:
            log("%s PING unknown pilot (key: %x)" % (host, key))
            flags |= FLAG_ACK_BAD_KEY
        else:
            log("%s PING %s -> PONG" % (host, unicode(pilot).encode('utf8', 'ignore')))

        data = struct.pack('!IHHQHHI', MAGIC, 0, TYPE_ACK, 0,
                           id, 0, flags)
        data = set_crc(data)
        self.socket.sendto(data, (host, port))

    def fixReceived(self, host, key, payload):
        if len(payload) != 32: return

        pilot = User.by_tracking_key(key)
        if not pilot:
            log("%s FIX unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!IIiiIHHHhhH', payload)

        fix = TrackingFix()
        fix.ip = host
        fix.pilot = pilot

        # import the time stamp from the packet if it's within a
        # certain range
        time_of_day_ms = data[1] % (24 * 3600 * 1000)
        time_of_day_s = time_of_day_ms / 1000
        time_of_day = time(time_of_day_s / 3600,
                           (time_of_day_s / 60) % 60,
                           time_of_day_s % 60,
                           (time_of_day_ms % 1000) * 1000)
        now = datetime.utcnow()
        now_s = ((now.hour * 60) + now.minute) * 60 + now.second
        if now_s - 1800 < time_of_day_s < now_s + 180:
            fix.time = datetime.combine(now.date(), time_of_day)
        elif now_s < 1800 and time_of_day_s > 23 * 3600:
            # midnight rollover occurred
            fix.time = (datetime.combine(now.date(), time_of_day) -
                        timedelta(days=1))
        else:
            log("bad time stamp: " + str(time_of_day))

        flags = data[0]
        if flags & FLAG_LOCATION:
            latitude = data[2] / 1000000.
            longitude = data[3] / 1000000.
            fix.set_location(longitude, latitude)

            fix.elevation = Elevation.get(fix.location_wkt)

        if flags & FLAG_TRACK:
            fix.track = data[5]

        if flags & FLAG_GROUND_SPEED:
            fix.ground_speed = data[6] / 16.

        if flags & FLAG_AIRSPEED:
            fix.airspeed = data[7] / 16.

        if flags & FLAG_ALTITUDE:
            fix.altitude = data[8]

        if flags & FLAG_VARIO:
            fix.vario = data[9] / 256.

        if flags & FLAG_ENL:
            fix.engine_noise_level = data[10]

        log("{} FIX {} {} {}".format(
            host, unicode(pilot).encode('utf8', 'ignore'),
            fix.time and fix.time.time(), fix.location))

        db.session.add(fix)
        try:
            db.session.commit()
        except SQLAlchemyError, e:
            log('database error:' + str(e))
            db.session.rollback()

    def trafficRequestReceived(self, host, port, key, payload):
        if len(payload) != 8: return

        pilot = User.by_tracking_key(key)
        if pilot is None:
            log("%s TRAFFIC_REQUEST unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!II', payload)
        or_filters = []

        flags = data[0]
        if flags & TRAFFIC_FLAG_FOLLOWEES:
            subq = db.session \
                .query(Follower.destination_id) \
                .filter(Follower.source_id == pilot.id) \
                .subquery()

            or_filters.append(TrackingFix.pilot_id.in_(subq))

        if flags & TRAFFIC_FLAG_CLUB:
            subq = db.session \
                .query(User.id) \
                .filter(User.club_id == pilot.club_id) \
                .subquery()

            or_filters.append(TrackingFix.pilot_id.in_(subq))

        if len(or_filters) == 0:
            return

        query = TrackingFix.query() \
            .distinct(TrackingFix.pilot_id) \
            .filter(and_(TrackingFix.time >= datetime.utcnow() - timedelta(hours=2),
                         TrackingFix.pilot_id != pilot.id,
                         TrackingFix.location_wkt != None,
                         TrackingFix.altitude != None,
                         or_(*or_filters))) \
            .order_by(TrackingFix.pilot_id, TrackingFix.time.desc()) \
            .limit(32)

        response = ''
        count = 0
        for fix in query:
            location = fix.location
            if location is None: continue

            t = fix.time
            t = t.hour * 3600000 + t.minute * 60000 + t.second * 1000 + t.microsecond / 1000
            response += struct.pack('!IIiihHI', fix.pilot_id, t,
                                    int(location.latitude * 1000000),
                                    int(location.longitude * 1000000),
                                    int(fix.altitude), 0, 0)
            count += 1

        response = struct.pack('!HBBI', 0, 0, count, 0) + response
        response = struct.pack('!IHHQ', MAGIC, 0, TYPE_TRAFFIC_RESPONSE, 0) + response
        response = set_crc(response)
        self.socket.sendto(response, (host, port))

        log("%s TRAFFIC_REQUEST %s -> %d locations" %
            (host, unicode(pilot).encode('utf8', 'ignore'), count))

    def userNameRequestReceived(self, host, port, key, payload):
        """The client asks for the display name of a user account."""

        if len(payload) != 8: return

        pilot = User.by_tracking_key(key)
        if pilot is None:
            log("%s USER_NAME_REQUEST unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!II', payload)
        user_id = data[0]

        user = User.get(user_id)
        if user is None:
            response = struct.pack('!IHHQIIIBBBBII', MAGIC, 0, TYPE_USER_NAME_RESPONSE, 0,
                                   user_id, USER_FLAG_NOT_FOUND, 0,
                                   0, 0, 0, 0, 0, 0)
            response = set_crc(response)
            self.transport.write(response, (host, port))

            log("%s, USER_NAME_REQUEST %s -> NOT_FOUND" %
                (host, unicode(pilot).encode('utf8', 'ignore')))

            return

        name = user.name[:64].encode('utf8', 'ignore')
        club_id = user.club_id or 0

        response = struct.pack('!IHHQIIIBBBBII', MAGIC, 0, TYPE_USER_NAME_RESPONSE, 0,
                               user_id, 0, club_id,
                               len(name), 0, 0, 0, 0, 0)
        response += name
        response = set_crc(response)
        self.socket.sendto(response, (host, port))

        log("%s USER_NAME_REQUEST %s -> %s" %
            (host, unicode(pilot).encode('utf8', 'ignore'),
             unicode(user).encode('utf8', 'ignore')))

    def handle(self, data, (host, port)):
        if len(data) < 16: return

        header = struct.unpack('!IHHQ', data[:16])
        if header[0] != MAGIC: return
        if not check_crc(data): return

        with self.app.app_context():
            if header[2] == TYPE_FIX:
                self.fixReceived(host, header[3], data[16:])
            elif header[2] == TYPE_PING:
                self.pingReceived(host, port, header[3], data[16:])
            elif header[2] == TYPE_TRAFFIC_REQUEST:
                self.trafficRequestReceived(host, port, header[3], data[16:])
            elif header[2] == TYPE_USER_NAME_REQUEST:
                self.userNameRequestReceived(host, port, header[3], data[16:])

    def serve_forever(self):
        if not self.app:
            raise RuntimeError('application not registered on server instance')

        super(TrackingServer, self).serve_forever()
<commit_msg>tracking: Use TrackingFix.max_age_filter() method<commit_after>import sys
import struct
from datetime import datetime, time, timedelta

from gevent.server import DatagramServer

from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.sql.expression import and_, or_

from skylines.model import db, User, TrackingFix, Follower, Elevation
from skylines.tracking.crc import check_crc, set_crc

# More information about this protocol can be found in the XCSoar
# source code, source file src/Tracking/SkyLines/Protocol.hpp

MAGIC = 0x5df4b67b
TYPE_PING = 1
TYPE_ACK = 2
TYPE_FIX = 3
TYPE_TRAFFIC_REQUEST = 4
TYPE_TRAFFIC_RESPONSE = 5
TYPE_USER_NAME_REQUEST = 6
TYPE_USER_NAME_RESPONSE = 7

FLAG_ACK_BAD_KEY = 0x1

FLAG_LOCATION = 0x1
FLAG_TRACK = 0x2
FLAG_GROUND_SPEED = 0x4
FLAG_AIRSPEED = 0x8
FLAG_ALTITUDE = 0x10
FLAG_VARIO = 0x20
FLAG_ENL = 0x40

# for TYPE_TRAFFIC_REQUEST
TRAFFIC_FLAG_FOLLOWEES = 0x1
TRAFFIC_FLAG_CLUB = 0x2

USER_FLAG_NOT_FOUND = 0x1

def log(message):
    print message
    sys.stdout.flush()

class TrackingServer(DatagramServer):
    def init_app(self, app):
        self.app = app

    def pingReceived(self, host, port, key, payload):
        if len(payload) != 8: return
        id, reserved, reserved2 = struct.unpack('!HHI', payload)

        flags = 0

        pilot = User.by_tracking_key(key)
        if not pilot:
            log("%s PING unknown pilot (key: %x)" % (host, key))
            flags |= FLAG_ACK_BAD_KEY
        else:
            log("%s PING %s -> PONG" % (host, unicode(pilot).encode('utf8', 'ignore')))

        data = struct.pack('!IHHQHHI', MAGIC, 0, TYPE_ACK, 0,
                           id, 0, flags)
        data = set_crc(data)
        self.socket.sendto(data, (host, port))

    def fixReceived(self, host, key, payload):
        if len(payload) != 32: return

        pilot = User.by_tracking_key(key)
        if not pilot:
            log("%s FIX unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!IIiiIHHHhhH', payload)

        fix = TrackingFix()
        fix.ip = host
        fix.pilot = pilot

        # import the time stamp from the packet if it's within a
        # certain range
        time_of_day_ms = data[1] % (24 * 3600 * 1000)
        time_of_day_s = time_of_day_ms / 1000
        time_of_day = time(time_of_day_s / 3600,
                           (time_of_day_s / 60) % 60,
                           time_of_day_s % 60,
                           (time_of_day_ms % 1000) * 1000)
        now = datetime.utcnow()
        now_s = ((now.hour * 60) + now.minute) * 60 + now.second
        if now_s - 1800 < time_of_day_s < now_s + 180:
            fix.time = datetime.combine(now.date(), time_of_day)
        elif now_s < 1800 and time_of_day_s > 23 * 3600:
            # midnight rollover occurred
            fix.time = (datetime.combine(now.date(), time_of_day) -
                        timedelta(days=1))
        else:
            log("bad time stamp: " + str(time_of_day))

        flags = data[0]
        if flags & FLAG_LOCATION:
            latitude = data[2] / 1000000.
            longitude = data[3] / 1000000.
            fix.set_location(longitude, latitude)

            fix.elevation = Elevation.get(fix.location_wkt)

        if flags & FLAG_TRACK:
            fix.track = data[5]

        if flags & FLAG_GROUND_SPEED:
            fix.ground_speed = data[6] / 16.

        if flags & FLAG_AIRSPEED:
            fix.airspeed = data[7] / 16.

        if flags & FLAG_ALTITUDE:
            fix.altitude = data[8]

        if flags & FLAG_VARIO:
            fix.vario = data[9] / 256.

        if flags & FLAG_ENL:
            fix.engine_noise_level = data[10]

        log("{} FIX {} {} {}".format(
            host, unicode(pilot).encode('utf8', 'ignore'),
            fix.time and fix.time.time(), fix.location))

        db.session.add(fix)
        try:
            db.session.commit()
        except SQLAlchemyError, e:
            log('database error:' + str(e))
            db.session.rollback()

    def trafficRequestReceived(self, host, port, key, payload):
        if len(payload) != 8: return

        pilot = User.by_tracking_key(key)
        if pilot is None:
            log("%s TRAFFIC_REQUEST unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!II', payload)
        or_filters = []

        flags = data[0]
        if flags & TRAFFIC_FLAG_FOLLOWEES:
            subq = db.session \
                .query(Follower.destination_id) \
                .filter(Follower.source_id == pilot.id) \
                .subquery()

            or_filters.append(TrackingFix.pilot_id.in_(subq))

        if flags & TRAFFIC_FLAG_CLUB:
            subq = db.session \
                .query(User.id) \
                .filter(User.club_id == pilot.club_id) \
                .subquery()

            or_filters.append(TrackingFix.pilot_id.in_(subq))

        if len(or_filters) == 0:
            return

        query = TrackingFix.query() \
            .distinct(TrackingFix.pilot_id) \
            .filter(and_(TrackingFix.max_age_filter(2),
                         TrackingFix.pilot_id != pilot.id,
                         TrackingFix.location_wkt != None,
                         TrackingFix.altitude != None,
                         or_(*or_filters))) \
            .order_by(TrackingFix.pilot_id, TrackingFix.time.desc()) \
            .limit(32)

        response = ''
        count = 0
        for fix in query:
            location = fix.location
            if location is None: continue

            t = fix.time
            t = t.hour * 3600000 + t.minute * 60000 + t.second * 1000 + t.microsecond / 1000
            response += struct.pack('!IIiihHI', fix.pilot_id, t,
                                    int(location.latitude * 1000000),
                                    int(location.longitude * 1000000),
                                    int(fix.altitude), 0, 0)
            count += 1

        response = struct.pack('!HBBI', 0, 0, count, 0) + response
        response = struct.pack('!IHHQ', MAGIC, 0, TYPE_TRAFFIC_RESPONSE, 0) + response
        response = set_crc(response)
        self.socket.sendto(response, (host, port))

        log("%s TRAFFIC_REQUEST %s -> %d locations" %
            (host, unicode(pilot).encode('utf8', 'ignore'), count))

    def userNameRequestReceived(self, host, port, key, payload):
        """The client asks for the display name of a user account."""

        if len(payload) != 8: return

        pilot = User.by_tracking_key(key)
        if pilot is None:
            log("%s USER_NAME_REQUEST unknown pilot (key: %x)" % (host, key))
            return

        data = struct.unpack('!II', payload)
        user_id = data[0]

        user = User.get(user_id)
        if user is None:
            response = struct.pack('!IHHQIIIBBBBII', MAGIC, 0, TYPE_USER_NAME_RESPONSE, 0,
                                   user_id, USER_FLAG_NOT_FOUND, 0,
                                   0, 0, 0, 0, 0, 0)
            response = set_crc(response)
            self.transport.write(response, (host, port))

            log("%s, USER_NAME_REQUEST %s -> NOT_FOUND" %
                (host, unicode(pilot).encode('utf8', 'ignore')))

            return

        name = user.name[:64].encode('utf8', 'ignore')
        club_id = user.club_id or 0

        response = struct.pack('!IHHQIIIBBBBII', MAGIC, 0, TYPE_USER_NAME_RESPONSE, 0,
                               user_id, 0, club_id,
                               len(name), 0, 0, 0, 0, 0)
        response += name
        response = set_crc(response)
        self.socket.sendto(response, (host, port))

        log("%s USER_NAME_REQUEST %s -> %s" %
            (host, unicode(pilot).encode('utf8', 'ignore'),
             unicode(user).encode('utf8', 'ignore')))

    def handle(self, data, (host, port)):
        if len(data) < 16: return

        header = struct.unpack('!IHHQ', data[:16])
        if header[0] != MAGIC: return
        if not check_crc(data): return

        with self.app.app_context():
            if header[2] == TYPE_FIX:
                self.fixReceived(host, header[3], data[16:])
            elif header[2] == TYPE_PING:
                self.pingReceived(host, port, header[3], data[16:])
            elif header[2] == TYPE_TRAFFIC_REQUEST:
                self.trafficRequestReceived(host, port, header[3], data[16:])
            elif header[2] == TYPE_USER_NAME_REQUEST:
                self.userNameRequestReceived(host, port, header[3], data[16:])

    def serve_forever(self):
        if not self.app:
            raise RuntimeError('application not registered on server instance')

        super(TrackingServer, self).serve_forever()
<|endoftext|>


<commit_before># This file is part of Maker Keeper Framework.
#
# Copyright (C) 2017 reverendus
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from api.Address import Address
from api.Ray import Ray
from api.Wad import Wad
from api.otc import SimpleMarket
from api.otc.OfferInfo import OfferInfo
from api.sai import Tub
from keepers.arbitrage.Conversion import Conversion

class OasisTakeConversion(Conversion):
    def __init__(self, tub: Tub, market: SimpleMarket, offer: OfferInfo):
        self.tub = tub
        self.market = market
        self.offer = offer

        super().__init__(from_currency=self._currency_symbol(offer.buy_which_token.address),
                         to_currency=self._currency_symbol(offer.sell_which_token.address),
                         rate=Ray(offer.sell_how_much)/Ray(offer.buy_how_much),
                         min_from_amount=Wad.from_number(0),  #TODO will probably change after dust order limitation gets introduced
                         max_from_amount=offer.buy_how_much,
                         method=f"oasis-take-{self.offer.offer_id}")

    def _currency_symbol(self, address):
        if address == self.tub.sai():
            return "SAI"
        elif address == self.tub.skr():
            return "SKR"
        elif address == self.tub.gem():
            return "ETH"
        else:
            return "___"

    def perform(self):
        quantity = self.to_amount

        # if by any chance rounding makes us want to buy more quantity than is available,
        # we just buy the whole lot
        if quantity > self.offer.sell_how_much:
            quantity = self.offer.sell_how_much

        # if by any chance rounding makes us want to buy only slightly less than the available lot,
        # we buy everything as this is probably what we wanted in the first place
        if self.offer.sell_how_much - quantity < Wad.from_number(0.0000000001):
            quantity = self.offer.sell_how_much

        print(f"  Executing take({self.offer.offer_id}, '{quantity}') in order exchange {self.from_amount} {self.from_currency} to {self.to_amount} {self.to_currency}")
        take_result = self.market.take(self.offer.offer_id, quantity)
        if take_result:
            our_address = Address(self.tub.web3.eth.defaultAccount)
            incoming_transfer_on_take = next(filter(lambda transfer: transfer.token_address == self.offer.sell_which_token.address and transfer.to_address == our_address, take_result.transfers))
            outgoing_transfer_on_take = next(filter(lambda transfer: transfer.token_address == self.offer.buy_which_token.address and transfer.from_address == our_address, take_result.transfers))
            print(f"  Take was successful, exchanged {outgoing_transfer_on_take.value} {self.from_currency} to {incoming_transfer_on_take.value} {self.to_currency}")
            return take_result
        else:
            print(f"  Take failed!")
            return None
<commit_msg>F.ixed the output message<commit_after># This file is part of Maker Keeper Framework.
#
# Copyright (C) 2017 reverendus
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from api.Address import Address
from api.Ray import Ray
from api.Wad import Wad
from api.otc import SimpleMarket
from api.otc.OfferInfo import OfferInfo
from api.sai import Tub
from keepers.arbitrage.Conversion import Conversion

class OasisTakeConversion(Conversion):
    def __init__(self, tub: Tub, market: SimpleMarket, offer: OfferInfo):
        self.tub = tub
        self.market = market
        self.offer = offer

        super().__init__(from_currency=self._currency_symbol(offer.buy_which_token.address),
                         to_currency=self._currency_symbol(offer.sell_which_token.address),
                         rate=Ray(offer.sell_how_much)/Ray(offer.buy_how_much),
                         min_from_amount=Wad.from_number(0),  #TODO will probably change after dust order limitation gets introduced
                         max_from_amount=offer.buy_how_much,
                         method=f"oasis-take-{self.offer.offer_id}")

    def _currency_symbol(self, address):
        if address == self.tub.sai():
            return "SAI"
        elif address == self.tub.skr():
            return "SKR"
        elif address == self.tub.gem():
            return "ETH"
        else:
            return "___"

    def perform(self):
        quantity = self.to_amount

        # if by any chance rounding makes us want to buy more quantity than is available,
        # we just buy the whole lot
        if quantity > self.offer.sell_how_much:
            quantity = self.offer.sell_how_much

        # if by any chance rounding makes us want to buy only slightly less than the available lot,
        # we buy everything as this is probably what we wanted in the first place
        if self.offer.sell_how_much - quantity < Wad.from_number(0.0000000001):
            quantity = self.offer.sell_how_much

        print(f"  Executing take({self.offer.offer_id}, '{quantity}') in order to exchange {self.from_amount} {self.from_currency} to {self.to_amount} {self.to_currency}")
        take_result = self.market.take(self.offer.offer_id, quantity)
        if take_result:
            our_address = Address(self.tub.web3.eth.defaultAccount)
            incoming_transfer_on_take = next(filter(lambda transfer: transfer.token_address == self.offer.sell_which_token.address and transfer.to_address == our_address, take_result.transfers))
            outgoing_transfer_on_take = next(filter(lambda transfer: transfer.token_address == self.offer.buy_which_token.address and transfer.from_address == our_address, take_result.transfers))
            print(f"  Take was successful, exchanged {outgoing_transfer_on_take.value} {self.from_currency} to {incoming_transfer_on_take.value} {self.to_currency}")
            return take_result
        else:
            print(f"  Take failed!")
            return None
<|endoftext|>


<commit_before>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from smv import SmvApp
from smv.utils import smv_copy_array

from pyspark.sql import HiveContext, DataFrame

def ExactMatchPreFilter(colName, expr):
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.ExactMatchPreFilter(colName, expr._jc)

def NoOpPreFilter():
    return None

def GroupCondition(expr):
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.GroupCondition(expr._jc)

def NoOpGroupCondition():
    return None

def ExactLogic(colName, expr):
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.ExactLogic(colName, expr._jc)

def FuzzyLogic(colName, predicate, valueExpr, threshold):
    pjc = predicate._jc
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.FuzzyLogic(colName, pjc, valueExpr._jc, threshold)

class SmvEntityMatcher(object):
    def __init__(self,
        leftId,
        rightId,
        exactMatchFilter,
        groupCondition,
        levelLogics
    ):
        jlls = SmvApp.getInstance().sc._gateway.new_array(SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.LevelLogic, len(levelLogics))
        for i in range(0, len(jlls)):
            jlls[i] = levelLogics[i]

        self.jem = SmvApp.getInstance()._jvm.org.tresamigos.smv.python.SmvPythonHelper.createMatcher(
            leftId,
            rightId,
            exactMatchFilter,
            groupCondition,
            jlls
        )

    def doMatch(self, df1, df2, keepOriginalCols=True):
        jres = self.jem.doMatch(df1._jdf, df2._jdf, keepOriginalCols)
        return DataFrame(jres, SmvApp.getInstance().sqlContext)
<commit_msg>#576: add docstrings for matchers<commit_after>#
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from smv import SmvApp
from smv.utils import smv_copy_array

from pyspark.sql import HiveContext, DataFrame

def ExactMatchPreFilter(colName, expr):
    """Specify the top-level exact match

        Args:
            colName (string): level name used in the output DF
            expr (Column): match logic condition Column

        Example:
            >>> ExactMatchPreFilter("Full_Name_Match", col("full_name") == col("_full_name"))

        Returns:
            (ExactMatchPreFilter)
    """
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.ExactMatchPreFilter(colName, expr._jc)

def NoOpPreFilter():
    """Always returns None

        Returns:
            (None)
    """
    return None

def GroupCondition(expr):
    """Specify the shared matching condition of all the levels (except the top-level exact match)

        Args:
            expr (Column): shared matching condition

        Note:
            `expr` should be in "left == right" form so that it can really optimize the process by reducing searching space

        Example:
            >>> GroupCondition(soundex("first_name") == soundex("_first_name"))

        Returns:
            (GroupCondition)
    """
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.GroupCondition(expr._jc)

def NoOpGroupCondition():
    """Always returns None

        Returns:
            (None)
    """
    return None

def ExactLogic(colName, expr):
    """Level match with exact logic

        Args:
            colName (string): level name used in the output DF
            expr (Column): match logic colName

        Example:
            >>> ExactLogic("First_Name_Match", col("first_name") == col("_first_name"))

        Returns:
            (ExactLogic)
    """
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.ExactLogic(colName, expr._jc)

def FuzzyLogic(colName, predicate, valueExpr, threshold):
    """Level match with fuzzy logic

        Args:
            colName (string): level name used in the output DF
            predicate (Column): a condition column, no match if this condition evaluates as false
            valueExpr (Column): a value column, which typically return a score. Higher score means higher chance of matching
            threshold (float): No match if the evaluated `valueExpr` < threshold

        Example:
            >>> FuzzyLogic("Levenshtein_City", lit(True), normlevenshtein(col("city"),col("_city")), 0.9)

        Returns:
            (FuzzyLogic)
    """
    pjc = predicate._jc
    return SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.FuzzyLogic(colName, pjc, valueExpr._jc, threshold)

class SmvEntityMatcher(object):
    """Perform multiple level entity matching with exact and/or fuzzy logic

        Args:
            leftId (string): id column name of left DF (df1)
            rightId (string): id column name of right DF (df2)
            exactMatchFilter (ExactMatchPreFilter): exact match condition, if records matched no further tests will be performed
            groupCondition (GroupCondition): for exact match leftovers, a deterministic condition to narrow down the search space
            levelLogics (list(ExactLogic or FuzzyLogic)): A list of level match conditions (always weaker than exactMatchFilter), all of them will be tested

        Example:
            code::

                SmvEntityMatcher("id", "_id",
                    ExactMatchPreFilter("Full_Name_Match", col("full_name") == col("_full_name")),
                    GroupCondition(soundex("first_name") == soundex("_first_name")),
                    [
                        ExactLogic("First_Name_Match", col("first_name") == col("_first_name")),
                        FuzzyLogic("Levenshtein_City", lit(True), normlevenshtein(col("city"),col("_city")), 0.9)
                    ]
                )

        Returns:
            (XYZ)
    """
    def __init__(self,
        leftId,
        rightId,
        exactMatchFilter,
        groupCondition,
        levelLogics
    ):
        jlls = SmvApp.getInstance().sc._gateway.new_array(SmvApp.getInstance()._jvm.org.tresamigos.smv.matcher.LevelLogic, len(levelLogics))
        for i in range(0, len(jlls)):
            jlls[i] = levelLogics[i]

        self.jem = SmvApp.getInstance()._jvm.org.tresamigos.smv.python.SmvPythonHelper.createMatcher(
            leftId,
            rightId,
            exactMatchFilter,
            groupCondition,
            jlls
        )

    def doMatch(self, df1, df2, keepOriginalCols=True):
        """Apply `SmvEntityMatcher` to the 2 DataFrames

            Args:
                df1 (DataFrame): DataFrame 1 with an id column with name "id"
                df2 (DataFrame): DataFrame 2 with an id column with name "id"
                keepOriginalCols (boolean): whether to keep all input columns of df1 and df2, defaults to true

            Example:
                code::

                    SmvEntityMatcher("id", "_id",
                        ExactMatchPreFilter("Full_Name_Match", col("full_name") == col("_full_name")),
                        GroupCondition(soundex("first_name") == soundex("_first_name")),
                        [
                            ExactLogic("First_Name_Match", col("first_name") == col("_first_name")),
                            FuzzyLogic("Levenshtein_City", lit(True), normlevenshtein(col("city"),col("_city")), 0.9)
                        ]
                    ).doMatch(df1, df2, False)

            Returns:
                (DataFrame): a DataFrame with df1's id and df2's id and match flags of all the levels. For levels with fuzzy logic, the matching score is also provided. A column named "MatchBitmap" also provided to summarize all the matching flags. When keepOriginalCols is true, input columns are also kept
        """
        jres = self.jem.doMatch(df1._jdf, df2._jdf, keepOriginalCols)
        return DataFrame(jres, SmvApp.getInstance().sqlContext)
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

import frappe
from frappe import _
import frappe.defaults
from frappe.modules.import_file import get_file_path, read_doc_from_file
from frappe.translate import send_translations
from frappe.core.doctype.doctype.doctype import (clear_permissions_cache,
	validate_permissions_for_doctype)
from frappe.permissions import (reset_perms, get_linked_doctypes, get_all_perms,
	setup_custom_perms, add_permission, update_permission_property)
from frappe.utils.user import get_users_with_role as _get_user_with_role

not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"]

@frappe.whitelist()
def get_roles_and_doctypes():
	frappe.only_for("System Manager")
	send_translations(frappe.get_lang_dict("doctype", "DocPerm"))

	active_domains = frappe.get_active_domains()

	doctypes = frappe.get_all("DocType", filters={
		"istable": 0,
		"name": ("not in", ",".join(not_allowed_in_permission_manager)),
	}, or_filters={
		"ifnull(restrict_to_domain, '')": "",
		"restrict_to_domain": ("in", active_domains)
	}, fields=["name"])

	restricted_roles = ['Administrator']
	if frappe.session.user != 'Administrator':
		custom_user_type_roles = frappe.get_all('User Type', filters = {'is_standard': 0}, fields=['role'])
		for row in custom_user_type_roles:
			restricted_roles.append(row.role)

		restricted_roles.append('All')

	roles = frappe.get_all("Role", filters={
		"name": ("not in", restricted_roles),
		"disabled": 0,
	}, or_filters={
		"ifnull(restrict_to_domain, '')": "",
		"restrict_to_domain": ("in", active_domains)
	}, fields=["name"])

	doctypes_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in doctypes]
	roles_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in roles]

	return {
		"doctypes": sorted(doctypes_list, key=lambda d: d['label']),
		"roles": sorted(roles_list, key=lambda d: d['label'])
	}

@frappe.whitelist()
def get_permissions(doctype=None, role=None):
	frappe.only_for("System Manager")
	if role:
		out = get_all_perms(role)
		if doctype:
			out = [p for p in out if p.parent == doctype]
	else:
		filters=dict(parent = doctype)
		if frappe.session.user != 'Administrator':
			custom_roles = frappe.get_all('Role', filters={'is_custom': 1})
			filters['role'] = ['not in', [row.name for row in custom_roles]]

		out = frappe.get_all('Custom DocPerm', fields='*', filters=filters, order_by="permlevel")
		if not out:
			out = frappe.get_all('DocPerm', fields='*', filters=filters, order_by="permlevel")

	linked_doctypes = {}
	for d in out:
		if not d.parent in linked_doctypes:
			linked_doctypes[d.parent] = get_linked_doctypes(d.parent)
		d.linked_doctypes = linked_doctypes[d.parent]
		meta = frappe.get_meta(d.parent)
		if meta:
			d.is_submittable = meta.is_submittable
			d.in_create = meta.in_create

	return out

@frappe.whitelist()
def add(parent, role, permlevel):
	frappe.only_for("System Manager")
	add_permission(parent, role, permlevel)

@frappe.whitelist()
def update(doctype, role, permlevel, ptype, value=None):
	"""Update role permission params

	Args:
		doctype (str): Name of the DocType to update params for
		role (str): Role to be updated for, eg "Website Manager".
		permlevel (int): perm level the provided rule applies to
		ptype (str): permission type, example "read", "delete", etc.
		value (None, optional): value for ptype, None indicates False

	Returns:
		str: Refresh flag is permission is updated successfully
	"""
	frappe.only_for("System Manager")
	out = update_permission_property(doctype, role, permlevel, ptype, value)
	return 'refresh' if out else None

@frappe.whitelist()
def remove(doctype, role, permlevel):
	frappe.only_for("System Manager")
	setup_custom_perms(doctype)

	name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel))
	frappe.db.delete("Custom DocPerm", {
		"name": name
	})
	if not frappe.get_all('Custom DocPerm', dict(parent=doctype)):
		frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove'))

	validate_permissions_for_doctype(doctype, for_remove=True, alert=True)

@frappe.whitelist()
def reset(doctype):
	frappe.only_for("System Manager")
	reset_perms(doctype)
	clear_permissions_cache(doctype)

@frappe.whitelist()
def get_users_with_role(role):
	frappe.only_for("System Manager")
	return _get_user_with_role(role)

@frappe.whitelist()
def get_standard_permissions(doctype):
	frappe.only_for("System Manager")
	meta = frappe.get_meta(doctype)
	if meta.custom:
		doc = frappe.get_doc('DocType', doctype)
		return [p.as_dict() for p in doc.permissions]
	else:
		# also used to setup permissions via patch
		path = get_file_path(meta.module, "DocType", doctype)
		return read_doc_from_file(path).get("permissions")
<commit_msg>perf(minor): Permission Manager remove API<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

import frappe
from frappe import _
import frappe.defaults
from frappe.modules.import_file import get_file_path, read_doc_from_file
from frappe.translate import send_translations
from frappe.core.doctype.doctype.doctype import (clear_permissions_cache,
	validate_permissions_for_doctype)
from frappe.permissions import (reset_perms, get_linked_doctypes, get_all_perms,
	setup_custom_perms, add_permission, update_permission_property)
from frappe.utils.user import get_users_with_role as _get_user_with_role

not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"]

@frappe.whitelist()
def get_roles_and_doctypes():
	frappe.only_for("System Manager")
	send_translations(frappe.get_lang_dict("doctype", "DocPerm"))

	active_domains = frappe.get_active_domains()

	doctypes = frappe.get_all("DocType", filters={
		"istable": 0,
		"name": ("not in", ",".join(not_allowed_in_permission_manager)),
	}, or_filters={
		"ifnull(restrict_to_domain, '')": "",
		"restrict_to_domain": ("in", active_domains)
	}, fields=["name"])

	restricted_roles = ['Administrator']
	if frappe.session.user != 'Administrator':
		custom_user_type_roles = frappe.get_all('User Type', filters = {'is_standard': 0}, fields=['role'])
		for row in custom_user_type_roles:
			restricted_roles.append(row.role)

		restricted_roles.append('All')

	roles = frappe.get_all("Role", filters={
		"name": ("not in", restricted_roles),
		"disabled": 0,
	}, or_filters={
		"ifnull(restrict_to_domain, '')": "",
		"restrict_to_domain": ("in", active_domains)
	}, fields=["name"])

	doctypes_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in doctypes]
	roles_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in roles]

	return {
		"doctypes": sorted(doctypes_list, key=lambda d: d['label']),
		"roles": sorted(roles_list, key=lambda d: d['label'])
	}

@frappe.whitelist()
def get_permissions(doctype=None, role=None):
	frappe.only_for("System Manager")
	if role:
		out = get_all_perms(role)
		if doctype:
			out = [p for p in out if p.parent == doctype]
	else:
		filters=dict(parent = doctype)
		if frappe.session.user != 'Administrator':
			custom_roles = frappe.get_all('Role', filters={'is_custom': 1})
			filters['role'] = ['not in', [row.name for row in custom_roles]]

		out = frappe.get_all('Custom DocPerm', fields='*', filters=filters, order_by="permlevel")
		if not out:
			out = frappe.get_all('DocPerm', fields='*', filters=filters, order_by="permlevel")

	linked_doctypes = {}
	for d in out:
		if not d.parent in linked_doctypes:
			linked_doctypes[d.parent] = get_linked_doctypes(d.parent)
		d.linked_doctypes = linked_doctypes[d.parent]
		meta = frappe.get_meta(d.parent)
		if meta:
			d.is_submittable = meta.is_submittable
			d.in_create = meta.in_create

	return out

@frappe.whitelist()
def add(parent, role, permlevel):
	frappe.only_for("System Manager")
	add_permission(parent, role, permlevel)

@frappe.whitelist()
def update(doctype, role, permlevel, ptype, value=None):
	"""Update role permission params

	Args:
		doctype (str): Name of the DocType to update params for
		role (str): Role to be updated for, eg "Website Manager".
		permlevel (int): perm level the provided rule applies to
		ptype (str): permission type, example "read", "delete", etc.
		value (None, optional): value for ptype, None indicates False

	Returns:
		str: Refresh flag is permission is updated successfully
	"""
	frappe.only_for("System Manager")
	out = update_permission_property(doctype, role, permlevel, ptype, value)
	return 'refresh' if out else None

@frappe.whitelist()
def remove(doctype, role, permlevel):
	frappe.only_for("System Manager")
	setup_custom_perms(doctype)

	frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel})

	if not frappe.get_all('Custom DocPerm', {"parent": doctype}):
		frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove'))

	validate_permissions_for_doctype(doctype, for_remove=True, alert=True)

@frappe.whitelist()
def reset(doctype):
	frappe.only_for("System Manager")
	reset_perms(doctype)
	clear_permissions_cache(doctype)

@frappe.whitelist()
def get_users_with_role(role):
	frappe.only_for("System Manager")
	return _get_user_with_role(role)

@frappe.whitelist()
def get_standard_permissions(doctype):
	frappe.only_for("System Manager")
	meta = frappe.get_meta(doctype)
	if meta.custom:
		doc = frappe.get_doc('DocType', doctype)
		return [p.as_dict() for p in doc.permissions]
	else:
		# also used to setup permissions via patch
		path = get_file_path(meta.module, "DocType", doctype)
		return read_doc_from_file(path).get("permissions")
<|endoftext|>


<commit_before># Copyright (C) 2021 by frePPLe bv
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

from freppledb.input.models import Location
from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ("input", "0057_itemsupplier_extra_safety_lt"),
    ]

    operations = [
        migrations.RunSQL(
            """
            delete from buffer a
            using (
                select max(id) as id, item_id, location_id, batch
                from buffer
                group by item_id, location_id, batch
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.location_id is not distinct from b.location_id
            and a.batch is not distinct from b.batch
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update buffer set batch = '' where batch is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE buffer ALTER COLUMN batch SET DEFAULT ''",
        ),
        migrations.RunSQL(
            """
            delete from calendarbucket a
            using (
                select max(id) as id, calendar_id, priority, startdate, enddate
                from calendarbucket
                group by calendar_id, priority, startdate, enddate
                having count(*) > 1
                ) b
            where a.calendar_id is not distinct from b.calendar_id
            and a.priority is not distinct from b.priority
            and a.startdate is not distinct from b.startdate
            and a.enddate is not distinct from b.enddate
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update calendarbucket set startdate = '1971-01-01'::timestamp where startdate is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN startdate SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            "update calendarbucket set enddate = '2030-12-31'::timestamp where enddate is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN enddate SET DEFAULT '2030-12-31'::timestamp",
        ),
        migrations.RunSQL(
            "update calendarbucket set priority = 0 where priority is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN priority SET DEFAULT 0",
        ),
        migrations.RunSQL(
            """
            delete from suboperation a
            using (
                select max(id) as id, operation_id, suboperation_id, effective_start
                from suboperation
                group by operation_id, suboperation_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.suboperation_id is not distinct from b.suboperation_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update suboperation set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE suboperation ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from operationmaterial a
            using (
                select max(id) as id, operation_id, item_id, effective_start
                from operationmaterial
                group by operation_id, item_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.item_id is not distinct from b.item_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update operationmaterial set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE operationmaterial ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from operationresource a
            using (
                select max(id) as id, operation_id, resource_id, effective_start
                from operationresource
                group by operation_id, resource_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.resource_id is not distinct from b.resource_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update operationresource set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE operationresource ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from itemsupplier a
            using (
                select max(id) as id, item_id, supplier_id, location_id, effective_start
                from itemsupplier
                group by item_id, supplier_id, location_id, effective_start
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.supplier_id is not distinct from b.supplier_id
            and a.location_id is not distinct from b.location_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update itemsupplier set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE itemsupplier ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from itemdistribution a
            using (
                select max(id) as id, item_id, origin_id, location_id, effective_start
                from itemdistribution
                group by item_id, origin_id, location_id, effective_start
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.origin_id is not distinct from b.origin_id
            and a.location_id is not distinct from b.location_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update itemdistribution set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE itemdistribution ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
    ]
<commit_msg>cleaning unused imports<commit_after># Copyright (C) 2021 by frePPLe bv
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ("input", "0057_itemsupplier_extra_safety_lt"),
    ]

    operations = [
        migrations.RunSQL(
            """
            delete from buffer a
            using (
                select max(id) as id, item_id, location_id, batch
                from buffer
                group by item_id, location_id, batch
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.location_id is not distinct from b.location_id
            and a.batch is not distinct from b.batch
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update buffer set batch = '' where batch is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE buffer ALTER COLUMN batch SET DEFAULT ''",
        ),
        migrations.RunSQL(
            """
            delete from calendarbucket a
            using (
                select max(id) as id, calendar_id, priority, startdate, enddate
                from calendarbucket
                group by calendar_id, priority, startdate, enddate
                having count(*) > 1
                ) b
            where a.calendar_id is not distinct from b.calendar_id
            and a.priority is not distinct from b.priority
            and a.startdate is not distinct from b.startdate
            and a.enddate is not distinct from b.enddate
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update calendarbucket set startdate = '1971-01-01'::timestamp where startdate is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN startdate SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            "update calendarbucket set enddate = '2030-12-31'::timestamp where enddate is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN enddate SET DEFAULT '2030-12-31'::timestamp",
        ),
        migrations.RunSQL(
            "update calendarbucket set priority = 0 where priority is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE calendarbucket ALTER COLUMN priority SET DEFAULT 0",
        ),
        migrations.RunSQL(
            """
            delete from suboperation a
            using (
                select max(id) as id, operation_id, suboperation_id, effective_start
                from suboperation
                group by operation_id, suboperation_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.suboperation_id is not distinct from b.suboperation_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update suboperation set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE suboperation ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from operationmaterial a
            using (
                select max(id) as id, operation_id, item_id, effective_start
                from operationmaterial
                group by operation_id, item_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.item_id is not distinct from b.item_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update operationmaterial set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE operationmaterial ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from operationresource a
            using (
                select max(id) as id, operation_id, resource_id, effective_start
                from operationresource
                group by operation_id, resource_id, effective_start
                having count(*) > 1
                ) b
            where a.operation_id is not distinct from b.operation_id
            and a.resource_id is not distinct from b.resource_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update operationresource set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE operationresource ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from itemsupplier a
            using (
                select max(id) as id, item_id, supplier_id, location_id, effective_start
                from itemsupplier
                group by item_id, supplier_id, location_id, effective_start
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.supplier_id is not distinct from b.supplier_id
            and a.location_id is not distinct from b.location_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update itemsupplier set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE itemsupplier ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
        migrations.RunSQL(
            """
            delete from itemdistribution a
            using (
                select max(id) as id, item_id, origin_id, location_id, effective_start
                from itemdistribution
                group by item_id, origin_id, location_id, effective_start
                having count(*) > 1
                ) b
            where a.item_id is not distinct from b.item_id
            and a.origin_id is not distinct from b.origin_id
            and a.location_id is not distinct from b.location_id
            and a.effective_start is not distinct from b.effective_start
            and a.id is distinct from b.id
            """
        ),
        migrations.RunSQL(
            "update itemdistribution set effective_start = '1971-01-01'::timestamp where effective_start is null",
        ),
        migrations.RunSQL(
            "ALTER TABLE itemdistribution ALTER COLUMN effective_start SET DEFAULT '1971-01-01'::timestamp",
        ),
    ]
<|endoftext|>


<commit_before>class Convertor:
	def convert(self,score,settings):
		pass

class MidiConvertor(Convertor):
	# will need to hard-code volume levels for crescendos and decrescendos

	def dynamicRanges(self, score):
		for (instrument,music) in score['instruments'].items():
			dynamic = False
			for measure in music:
				for beat in measure['beats']:
					for note in beat:
						if 'dynamicChange' in note:
							dynamic = True
							# found the start, now we need to find the end
							# then calculate how much time would be in between
						pass

	def convert(self, score, settings):
		#score = self.dynamicRanges(score)

		instrumentProgramMap = {
			"bass": "0",
			"cymbal": "3",
			"snare": "2",
			"tenor": "1"
		}
		instrumentVolumeMap = {
			"bass": "127",
			"cymbal": "127",
			"snare": "100",
			"tenor": "127"
		}
		instrumentPanMap = {
			"bass": "30",
			"cymbal": "64",
			"snare": "64",
			"tenor": "98"
		}
		noteMap = {
			# snare
			"h": "c6",
			"x": "d6",

			# bass and tenor
			"a": "e6",
			"b": "c6",
			"c": "a5",
			"d": "f5",
			"e": "d5"
		}
		volumeMap = {
			"P": 32, # pianissimo
			"M": 64, # mezzo-forte
			"F": 95, # forte
			"G": 127 # ff
		}

		if 'tempo' in score:
			scoreTempo = int(score['tempo'])
			tempo = 60000000 / scoreTempo
		else:
			scoreTempo = 120
			tempo = 500000

		# MFile format tracks division
		out = "MFile 1 " + str(len(score['instruments']) + 1) + " 384\n" # +1 tracks because of tempo track
		# tempo track
		out += "MTrk\n"
		out += "0 Tempo " + str(tempo) + "\n"
		out += "0 TimeSig 4/4 18 8\n"
		out += "TrkEnd\n"

		# set counter a second into the future for blank space padding

		channel = 1
		flamPosition = -20 # calculate based on tempo
		accentIncrease = int(127/4)
		perBeat = 384
		startingCounter = 30 #(scoreTempo / 60) * 30 # calculate how much time would yield a second

		for (instrument,music) in score['instruments'].items():
			volume = volumeMap['M']
			counter = startingCounter
			nextBeat = counter + perBeat

			channelString = str(channel)

			out += "MTrk\n"
			# map instrument to a channel
			out += "0 PrCh ch=" + channelString + " prog=" + instrumentProgramMap[instrument] + "\n"
			# set main track volume
			out += "0 Par ch=" + channelString + " con=7 val=" + instrumentVolumeMap[instrument] + "\n"
			out += "0 Par ch=" + channelString + " con=10 val=" + instrumentPanMap[instrument] + "\n"

                        for measure in music:
                                for beat in measure['beats']:
					c1 = counter
					for note in beat:
						c2 = str(c1)
						if 'rest' in note:
							pass
						else:
							if 'flam' in note:
								# if surface is shot, flams should be on the drum head
								# annotate notes with proper flam surface
								#go back a bit, from current counter value
								out += str(c1 - 13) + " On ch=" + channelString + " n=" + noteMap[ note['flam'] ] + " v=" + str(volumeMap['P']) + "\n"
								#out += str(c1 - 5) + " Off ch=" + channelString + " n=" + noteMap[ note['surface'] ] + " v=0\n"

							# prepare volume
							if 'dynamic' in note:
								volume = volumeMap[ note['dynamic'] ]
							tempVolume = volume
							if 'accent' in note:
								tempVolume += accentIncrease # go up a quarter
								if tempVolume > 127:
									tempVolume = 127

							for surface in note['surface']:
								out += c2 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
								# expand diddle/tremolo
								# add the second note
								if 'diddle' in note:
									c3 = str(c1 + (perBeat / (note['duration'] * 2)))
									out += c3 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
							# when do we turn off
							# divide
							c3 = str(c1 + (perBeat / note['duration']))
							for surface in note['surface']:
								# why do i sometimes see the note off volume at 64?
								#out += c3 + " Off ch=" + channelString + " n=" + noteMap[ surface ] + " v=0\n"
								pass

							# i bet some cymbal notes we'll have to avoid turning off until we get an explicit choke note

						c1 += (perBeat / note['duration']) # how long does this note last?
					nextBeat += perBeat
					counter += perBeat
					# end note loop
				# end beat loop
			# end measure loop
			out += "TrkEnd\n"

			channel += 1
		# end instrument loop
		return out

class LilypondConvertor(Convertor):
	# lilypond specific
	# inserts flam spacers in the rest of the score
	# fucking hate that i have to do this
	def flams(self, parsed):
		instruments = parsed['instruments'].keys()
		measure = 0
		beat = 0
		note = 0
		done = False

		flams = []

		while not done:
			measureNotFound = False
			beatNotFound = False
			foundFlam = False
			for instrument in instruments:
				print(instrument)
				# if we've gone past beats
				# if we've gone past measures
				if measure >= len(parsed['instruments'][ instrument ] ):
					measureNotFound = True
					print('measure not found: ' + str(measure) )
					continue
				if beat >= len( parsed['instruments'][ instrument ][ measure ]['beats'] ):
					beatNotFound = True
					print('beat not found: ' + str(beat) )
					continue
				notes = parsed['instruments'][ instrument ][ measure ]['beats'][ beat ]

				i = 0
				z = len(notes)
				while i < z:
					note = notes[ i ]
					if type(note) == dict:
						if 'flam' in note:
							foundFlam = True
							flams.append( (instrument, measure, beat, i) )
					i += 1

			# somehow fill in flam spacers

			if measureNotFound and beatNotFound:
				done = True
			elif measureNotFound:
				done = True
			elif beatNotFound:
				beat = 0
				measure += 1
			else:
				beat += 1

		print( repr( flams ))
		return {}
		return parsed

	# or fixDurations
	def fixDurations(self, score):
		# fix durations and set tuplet flags
		# tuplet flags may already be set in some cases by left/right square brackets

		# 1=4, 2=8, 3=8tuplet, 4=16, 5=16tuplet, 6=16tuplet, 7=16tuplet, 8=32
		for (instrument,music) in score['instruments'].items():
			for measure in music:
				for beat in measure['beats']:
					for note in beat:
						# if tuplet and (last note in a beat, or no longer tuplet), close tuplet
						# is last beat?
						if note['duration'] == 3:
							duration = 8

						elif note['duration'] > 4 and note['duration'] < 8:
							duration = 16

						elif note['duration'] == 1 or note['duration'] == 2 or note['duration'] % 4 == 0:
							duration = note['duration'] * 4

						note['duration'] = duration
		return score

	def convert(self, parsed, settings={}):
		#a = self.flams(parsed)
		a = self.fixDurations(parsed)

		ret = '\\version "2.8.7"\n'

		ret += '\\header {\n'
		if 'title' in a:
			print('title')
			ret += '\ttitle="' + a['title'] + '"\n'
		if 'author' in a:
			ret += '\tauthor="' + a['author'] + '"\n'
			ret += '\tcopyright = \\markup {"Copyright" \\char ##x00A9 "' + a['author'] + '"}\n'
		ret += '}\n\n'

		ret += '\\score {\n'
		ret += '\t<<\n'

		mapping = {
			'h': 'c\'\'',
			'x': 'c\'\'', # rim shot

			'a': 'e\'\'',
			'b': 'c\'\'',
			'c': 'a\'',
			'd': 'f\'',
			'e': 'd\'',
			'f': 'b',

			'r': 'r', # rest
			'u': 'b\'', # unison b'

			# dynamics
			'P': '\\p',
			'M': '\\mf',
			'F': '\\f',
			'G': '\\ff',

			'<': '\\<',
			'>': '\\>',
			':': '\\!'
		}

		# set this flag when we encounter one of them
		# unset it when we encounter the first dynamic that's not one
		crescendoDecrescendo = False
		dynamic = 'M'

		for (instrument,music) in a['instruments'].items():

			if instrument == 'snare':
				ret += '\t% Snare\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "electric grand"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Snare "\n'

				#self.beaming()
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'tenor':
				ret += '\t% Tenor\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "bright acoustic"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Tenor "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'bass':
				ret += '\t% Bass\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "acoustic grand"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Bass "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'cymbal':
				ret += '\t% Cymbals\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "honky-tonk"\n'
				ret +='\t\t\\set Staff.instrumentName = #"Cymbals "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			ret += '\t\t\\stemUp\n'

			iMeasure = 1
			for measure in music:
				beats = len(measure['beats'])

				# if measure has a time signature, print it
				# but it's a bitch when doing it when a crescendo hasn't ended

				for beat in measure['beats']:
					for note in beat:
						if 'tupletStart' in note:
							if note['tupletStart'] == 3:
								ret += '\\times 2/3 { '
							else:
								ret += '\\times 4/6 { '

						if 'flam' in note:
							if instrument == 'snare':
								ret += '\\override Stem #\'length = #4 \\appoggiatura c\'\'8 \\revert Stem #\'length \stemUp '
							elif instrument == 'tenor': # tenor flam element has surface
								ret += '\\override Stem #\'length = #4 \\appoggiatura ' + mapping[ note['flam'] ] + '8 \\revert Stem #\'length \stemUp '

						# note or rest?
						if 'rest' in note:
							ret += 'r' + str(note['duration'])
						else:
							if len(note['surface']) > 1:
								ret += ' <<'
							for surface in note['surface']:
								if 'shot' in note:
									ret += '\\override NoteHead #\'style = #\'cross '
									ret += mapping[ surface ] + str(note['duration'])
									ret += ' \\revert NoteHead #\'style'
								else:
									ret += mapping[ surface ] + str(note['duration'])

								# diddle?
								# check flag for whether to expand tremolos
								if 'diddle' in note:
									ret += ':' + str(note['duration'] * 2)

								# fours?
								if 'fours' in note:
									ret += ':' + str(note['duration'] * 4)

								if 'dynamicChangeEnd' in note:
									ret += '\! '

								if 'dynamic' in note:
									ret += mapping[ note['dynamic'] ] + ' '

								if 'dynamicChange' in note:
									ret += mapping[ note['dynamicChange'] ] + ' '

								# should note be accented?
								if 'accent' in note:
									#ret += ' \\accent'
									ret += ' ^>'
								ret += ' '

							if len(note['surface']) > 1:
								ret += ' >>'

						if 'tupletStop' in note: # last note in a tuplet
							ret += '} '

						ret += ' '
					# end note loop

				# end beat loop

				ret += ' \n '
				if iMeasure == 4:
					ret += ' \\break '
					iMeasure = 1
				else:
					iMeasure += 1
			# end measure loop

			ret += '}\n'
		# end instrument loop

		ret += '>>\n'
		ret += '\t\\layout {\n'
		#print('\t\tindent = 0')
		ret += '\t}\n'
		ret += '}\n'
		return ret
<commit_msg>Output fours in midi file<commit_after>class Convertor:
	def convert(self,score,settings):
		pass

class MidiConvertor(Convertor):
	# will need to hard-code volume levels for crescendos and decrescendos

	def dynamicRanges(self, score):
		for (instrument,music) in score['instruments'].items():
			dynamic = False
			for measure in music:
				for beat in measure['beats']:
					for note in beat:
						if 'dynamicChange' in note:
							dynamic = True
							# found the start, now we need to find the end
							# then calculate how much time would be in between
						pass

	def convert(self, score, settings):
		#score = self.dynamicRanges(score)

		instrumentProgramMap = {
			"bass": "0",
			"cymbal": "3",
			"snare": "2",
			"tenor": "1"
		}
		instrumentVolumeMap = {
			"bass": "127",
			"cymbal": "127",
			"snare": "100",
			"tenor": "127"
		}
		instrumentPanMap = {
			"bass": "30",
			"cymbal": "64",
			"snare": "64",
			"tenor": "98"
		}
		noteMap = {
			# snare
			"h": "c6",
			"x": "d6",

			# bass and tenor
			"a": "e6",
			"b": "c6",
			"c": "a5",
			"d": "f5",
			"e": "d5"
		}
		volumeMap = {
			"P": 32, # pianissimo
			"M": 64, # mezzo-forte
			"F": 95, # forte
			"G": 127 # ff
		}

		if 'tempo' in score:
			scoreTempo = int(score['tempo'])
			tempo = 60000000 / scoreTempo
		else:
			scoreTempo = 120
			tempo = 500000

		# MFile format tracks division
		out = "MFile 1 " + str(len(score['instruments']) + 1) + " 384\n" # +1 tracks because of tempo track
		# tempo track
		out += "MTrk\n"
		out += "0 Tempo " + str(tempo) + "\n"
		out += "0 TimeSig 4/4 18 8\n"
		out += "TrkEnd\n"

		# set counter a second into the future for blank space padding

		channel = 1
		flamPosition = -20 # calculate based on tempo
		accentIncrease = int(127/4)
		perBeat = 384
		startingCounter = 30 #(scoreTempo / 60) * 30 # calculate how much time would yield a second

		for (instrument,music) in score['instruments'].items():
			volume = volumeMap['M']
			counter = startingCounter
			nextBeat = counter + perBeat

			channelString = str(channel)

			out += "MTrk\n"
			# map instrument to a channel
			out += "0 PrCh ch=" + channelString + " prog=" + instrumentProgramMap[instrument] + "\n"
			# set main track volume
			out += "0 Par ch=" + channelString + " con=7 val=" + instrumentVolumeMap[instrument] + "\n"
			out += "0 Par ch=" + channelString + " con=10 val=" + instrumentPanMap[instrument] + "\n"

                        for measure in music:
                                for beat in measure['beats']:
					c1 = counter
					for note in beat:
						c2 = str(c1)
						if 'rest' in note:
							pass
						else:
							if 'flam' in note:
								# if surface is shot, flams should be on the drum head
								# annotate notes with proper flam surface
								#go back a bit, from current counter value
								out += str(c1 - 13) + " On ch=" + channelString + " n=" + noteMap[ note['flam'] ] + " v=" + str(volumeMap['P']) + "\n"
								#out += str(c1 - 5) + " Off ch=" + channelString + " n=" + noteMap[ note['surface'] ] + " v=0\n"

							# prepare volume
							if 'dynamic' in note:
								volume = volumeMap[ note['dynamic'] ]
							tempVolume = volume
							if 'accent' in note:
								tempVolume += accentIncrease # go up a quarter
								if tempVolume > 127:
									tempVolume = 127

							for surface in note['surface']:
								out += c2 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
								# expand diddle/tremolo
								# add the second note
								if 'diddle' in note:
									c3 = str(c1 + (perBeat / (note['duration'] * 2)))
									out += c3 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
								if 'fours' in note:
									c3 = perBeat / (note['duration'] * 4)
									c4 = str(c1 + (c3))
									out += c4 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
									c4 = str(c1 + c3 + c3)
									out += c4 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
									c4 = str(c1 + c3 + c3 + c3)
									out += c4 + " On ch=" + channelString + " n=" + noteMap[ surface ] + " v=" + str(tempVolume) + "\n"
							# when do we turn off
							# divide
							c3 = str(c1 + (perBeat / note['duration']))
							for surface in note['surface']:
								# why do i sometimes see the note off volume at 64?
								#out += c3 + " Off ch=" + channelString + " n=" + noteMap[ surface ] + " v=0\n"
								pass

							# i bet some cymbal notes we'll have to avoid turning off until we get an explicit choke note

						c1 += (perBeat / note['duration']) # how long does this note last?
					nextBeat += perBeat
					counter += perBeat
					# end note loop
				# end beat loop
			# end measure loop
			out += "TrkEnd\n"

			channel += 1
		# end instrument loop
		return out

class LilypondConvertor(Convertor):
	# lilypond specific
	# inserts flam spacers in the rest of the score
	# fucking hate that i have to do this
	def flams(self, parsed):
		instruments = parsed['instruments'].keys()
		measure = 0
		beat = 0
		note = 0
		done = False

		flams = []

		while not done:
			measureNotFound = False
			beatNotFound = False
			foundFlam = False
			for instrument in instruments:
				print(instrument)
				# if we've gone past beats
				# if we've gone past measures
				if measure >= len(parsed['instruments'][ instrument ] ):
					measureNotFound = True
					print('measure not found: ' + str(measure) )
					continue
				if beat >= len( parsed['instruments'][ instrument ][ measure ]['beats'] ):
					beatNotFound = True
					print('beat not found: ' + str(beat) )
					continue
				notes = parsed['instruments'][ instrument ][ measure ]['beats'][ beat ]

				i = 0
				z = len(notes)
				while i < z:
					note = notes[ i ]
					if type(note) == dict:
						if 'flam' in note:
							foundFlam = True
							flams.append( (instrument, measure, beat, i) )
					i += 1

			# somehow fill in flam spacers

			if measureNotFound and beatNotFound:
				done = True
			elif measureNotFound:
				done = True
			elif beatNotFound:
				beat = 0
				measure += 1
			else:
				beat += 1

		print( repr( flams ))
		return {}
		return parsed

	# or fixDurations
	def fixDurations(self, score):
		# fix durations and set tuplet flags
		# tuplet flags may already be set in some cases by left/right square brackets

		# 1=4, 2=8, 3=8tuplet, 4=16, 5=16tuplet, 6=16tuplet, 7=16tuplet, 8=32
		for (instrument,music) in score['instruments'].items():
			for measure in music:
				for beat in measure['beats']:
					for note in beat:
						# if tuplet and (last note in a beat, or no longer tuplet), close tuplet
						# is last beat?
						if note['duration'] == 3:
							duration = 8

						elif note['duration'] > 4 and note['duration'] < 8:
							duration = 16

						elif note['duration'] == 1 or note['duration'] == 2 or note['duration'] % 4 == 0:
							duration = note['duration'] * 4

						note['duration'] = duration
		return score

	def convert(self, parsed, settings={}):
		#a = self.flams(parsed)
		a = self.fixDurations(parsed)

		ret = '\\version "2.8.7"\n'

		ret += '\\header {\n'
		if 'title' in a:
			ret += '\ttitle="' + a['title'] + '"\n'
		if 'author' in a:
			ret += '\tauthor="' + a['author'] + '"\n'
			ret += '\tcopyright = \\markup {"Copyright" \\char ##x00A9 "' + a['author'] + '"}\n'
		ret += '}\n\n'

		ret += '\\score {\n'
		ret += '\t<<\n'

		mapping = {
			'h': 'c\'\'',
			'x': 'c\'\'', # rim shot

			'a': 'e\'\'',
			'b': 'c\'\'',
			'c': 'a\'',
			'd': 'f\'',
			'e': 'd\'',
			'f': 'b',

			'r': 'r', # rest
			'u': 'b\'', # unison b'

			# dynamics
			'P': '\\p',
			'M': '\\mf',
			'F': '\\f',
			'G': '\\ff',

			'<': '\\<',
			'>': '\\>',
			':': '\\!'
		}

		# set this flag when we encounter one of them
		# unset it when we encounter the first dynamic that's not one
		crescendoDecrescendo = False
		dynamic = 'M'

		for (instrument,music) in a['instruments'].items():

			if instrument == 'snare':
				ret += '\t% Snare\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "electric grand"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Snare "\n'

				#self.beaming()
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'tenor':
				ret += '\t% Tenor\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "bright acoustic"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Tenor "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'bass':
				ret += '\t% Bass\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "acoustic grand"\n'
				ret += '\t\t\\set Staff.instrumentName = #"Bass "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			elif instrument == 'cymbal':
				ret += '\t% Cymbals\n'
				ret += '\t\\new Staff {\n'
				#ret += '\t\t\\set Staff.midiInstrument = "honky-tonk"\n'
				ret +='\t\t\\set Staff.instrumentName = #"Cymbals "\n'
				if 'timesignature' in a:
					ret += '\t\t\\time ' + a['timesignature'] + '\n'

			ret += '\t\t\\stemUp\n'

			iMeasure = 1
			for measure in music:
				beats = len(measure['beats'])

				# if measure has a time signature, print it
				# but it's a bitch when doing it when a crescendo hasn't ended

				for beat in measure['beats']:
					for note in beat:
						if 'tupletStart' in note:
							if note['tupletStart'] == 3:
								ret += '\\times 2/3 { '
							else:
								ret += '\\times 4/6 { '

						if 'flam' in note:
							if instrument == 'snare':
								ret += '\\override Stem #\'length = #4 \\appoggiatura c\'\'8 \\revert Stem #\'length \stemUp '
							elif instrument == 'tenor': # tenor flam element has surface
								ret += '\\override Stem #\'length = #4 \\appoggiatura ' + mapping[ note['flam'] ] + '8 \\revert Stem #\'length \stemUp '

						# note or rest?
						if 'rest' in note:
							ret += 'r' + str(note['duration'])
						else:
							if len(note['surface']) > 1:
								ret += ' <<'
							for surface in note['surface']:
								if 'shot' in note:
									ret += '\\override NoteHead #\'style = #\'cross '
									ret += mapping[ surface ] + str(note['duration'])
									ret += ' \\revert NoteHead #\'style'
								else:
									ret += mapping[ surface ] + str(note['duration'])

								# diddle?
								# check flag for whether to expand tremolos
								if 'diddle' in note:
									ret += ':' + str(note['duration'] * 2)

								# fours?
								if 'fours' in note:
									ret += ':' + str(note['duration'] * 4)

								if 'dynamicChangeEnd' in note:
									ret += '\! '

								if 'dynamic' in note:
									ret += mapping[ note['dynamic'] ] + ' '

								if 'dynamicChange' in note:
									ret += mapping[ note['dynamicChange'] ] + ' '

								# should note be accented?
								if 'accent' in note:
									#ret += ' \\accent'
									ret += ' ^>'
								ret += ' '

							if len(note['surface']) > 1:
								ret += ' >>'

						if 'tupletStop' in note: # last note in a tuplet
							ret += '} '

						ret += ' '
					# end note loop

				# end beat loop

				ret += ' \n '
				if iMeasure == 4:
					ret += ' \\break '
					iMeasure = 1
				else:
					iMeasure += 1
			# end measure loop

			ret += '}\n'
		# end instrument loop

		ret += '>>\n'
		ret += '\t\\layout {\n'
		#print('\t\tindent = 0')
		ret += '\t}\n'
		ret += '}\n'
		return ret
<|endoftext|>


<commit_before>import os, re
from subprocess import *
from time import strftime, gmtime, ctime, localtime, asctime

TERM_COLS = 110

## Run a command and return the output, or ERROR: + output if retcode != 0
def runCommand(cmd):
  p = Popen([cmd],stdout=PIPE,stderr=STDOUT, close_fds=True, shell=True)
  output = p.communicate()[0]
  if (p.returncode != 0):
    output = 'ERROR: ' + output
  return output

## print an optionally colorified test result
#
# The test will not be colored if
# 1) options.colored is False,
# 2) the environment variable BITTEN_NOCOLOR is true, or
# 3) the color parameter is False.
def printResult(test_name, result, timing, start, end, options, color=True):
  f_result = ''

  cnt = (TERM_COLS-2) - len(test_name + result)
  if color:
    f_result = test_name + '.'*cnt + ' ' + colorify(result, options)
  else:
    f_result = test_name + '.'*cnt + ' ' + result

  # Tack on the timing if it exists
  if timing:
    f_result += ' [' + '%0.3f' % float(timing) + 's]'
  if options.debug_harness:
    f_result += ' Start: ' + '%0.3f' % start + ' End: ' + '%0.3f' % end
  return f_result

## Color the error messages if the options permit, also do not color in bitten scripts because
# it messes up the trac output.
# supports weirded html for more advanced coloring schemes. <r>,<g>,<y>,<b> All colors are bolded.
def colorify(str, options, html=False):
  # ANSI color codes for colored terminal output
  RESET  = '\033[0m'
  BOLD   = '\033[1m'
  RED    = '\033[31m'
  GREEN  = '\033[32m'
  CYAN   = '\033[36m'
  YELLOW = '\033[33m'

  if options.colored and not (os.environ.has_key('BITTEN_NOCOLOR') and os.environ['BITTEN_NOCOLOR'] == 'true'):
    if html:
      str = str.replace('<r>', BOLD+RED)
      str = str.replace('<g>', BOLD+GREEN)
      str = str.replace('<y>', BOLD+YELLOW)
      str = str.replace('<b>', BOLD)
      str = re.sub(r'</[rgyb]>', RESET, str)
    else:
      str = str.replace('OK', BOLD+GREEN+'OK'+RESET)
      str = re.sub(r'(\[.*?\])', BOLD+CYAN+'\\1'+RESET, str)
      str = str.replace('skipped', BOLD+'skipped'+RESET)
      str = str.replace('deleted', BOLD+RED+'deleted'+RESET)
      if str.find('FAILED (EXODIFF)') != -1:
        str = str.replace('FAILED (EXODIFF)', BOLD+YELLOW+'FAILED (EXODIFF)'+RESET)
      elif str.find('FAILED (CSVDIFF') != -1:
        str = str.replace('FAILED (CSVDIFF)', BOLD+YELLOW+'FAILED (CSVDIFF)'+RESET)
      else:
        str = re.sub(r'(FAILED \([A-Za-z ]*\))', BOLD+RED+'\\1'+RESET, str)
  elif html:
    str = re.sub(r'</?[rgyb]>', '', str)    # strip all "html" tags

  return str

def getPlatforms():
  # We'll use uname to figure this out
  # Supported platforms are LINUX, DARWIN, SL, LION or ALL
  platforms = set()
  platforms.add('ALL')
  raw_uname = os.uname()
  if raw_uname[0].upper() == 'DARWIN':
    platforms.add('DARWIN')
    if re.match("10\.", raw_uname[2]):
      platforms.add('SL')
    if re.match("11\.", raw_uname[2]):
      platforms.add("LION")
  else:
    platforms.add(raw_uname[0].upper())
  return platforms

def getCompilers(libmesh_dir):
  # We'll use the GXX-VERSION string from LIBMESH's Make.common
  # to figure this out
  # Supported compilers are GCC, INTEL or ALL
  compilers = set()
  compilers.add('ALL')
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('GXX-VERSION') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        raw_compiler = m.group(1)
        if re.search('intel', raw_compiler, re.I) != None:
          compilers.add("INTEL")
        elif re.search('gcc', raw_compiler, re.I) != None:
          compilers.add("GCC")
        elif re.search('clang', raw_compiler, re.I) != None:
          compilers.add("CLANG")
        break
  f.close()
  return compilers

def getPetscVersion(libmesh_dir):
  # We'll use the petsc-major string from $LIBMESH_DIR/Make.common
  # to figure this out
  # Supported versions are 2, 3
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('petsc-version') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        raw_version = m.group(1)
        petsc_version = raw_version
        break
  f.close()
  return petsc_version

def getParmeshOption(libmesh_dir):
  # Some tests work differently with parallel mesh enabled
  # We need to detect this condition
  parmesh = set()
  f = open(libmesh_dir + '/include/base/libmesh_config.h')
  for line in f.readlines():
    m = re.search(r'#define\s+LIBMESH_ENABLE_PARMESH\s+(\d+)', line)
    if m != None:
      if m.group(1) == '1':
        parmesh.add('PARALLEL')
      else:
        parmesh.add('SERIAL')
      break
  # If we didn't find the #define indicated by having no entries in our set, then parmesh is off
  if not len(parmesh):
    parmesh.add('SERIAL')
  parmesh.add('ALL')
  f.close()
  return parmesh

def getSharedOption(libmesh_dir):
  # Some tests may only run properly with shared libraries on/off
  # We need to detect this condition
  shared_option = set()
  shared_option.add('ALL')
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('enable-shared') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        if m.group(1) == 'yes':
          shared_option.add('DYNAMIC')
        else:
          shared_option.add('STATIC')
        break
  f.close()
  return shared_option
<commit_msg>Make reading libmesh_config.h more robust to the file being in different locations.<commit_after>import os, re
from subprocess import *
from time import strftime, gmtime, ctime, localtime, asctime

TERM_COLS = 110

## Run a command and return the output, or ERROR: + output if retcode != 0
def runCommand(cmd):
  p = Popen([cmd],stdout=PIPE,stderr=STDOUT, close_fds=True, shell=True)
  output = p.communicate()[0]
  if (p.returncode != 0):
    output = 'ERROR: ' + output
  return output

## print an optionally colorified test result
#
# The test will not be colored if
# 1) options.colored is False,
# 2) the environment variable BITTEN_NOCOLOR is true, or
# 3) the color parameter is False.
def printResult(test_name, result, timing, start, end, options, color=True):
  f_result = ''

  cnt = (TERM_COLS-2) - len(test_name + result)
  if color:
    f_result = test_name + '.'*cnt + ' ' + colorify(result, options)
  else:
    f_result = test_name + '.'*cnt + ' ' + result

  # Tack on the timing if it exists
  if timing:
    f_result += ' [' + '%0.3f' % float(timing) + 's]'
  if options.debug_harness:
    f_result += ' Start: ' + '%0.3f' % start + ' End: ' + '%0.3f' % end
  return f_result

## Color the error messages if the options permit, also do not color in bitten scripts because
# it messes up the trac output.
# supports weirded html for more advanced coloring schemes. <r>,<g>,<y>,<b> All colors are bolded.
def colorify(str, options, html=False):
  # ANSI color codes for colored terminal output
  RESET  = '\033[0m'
  BOLD   = '\033[1m'
  RED    = '\033[31m'
  GREEN  = '\033[32m'
  CYAN   = '\033[36m'
  YELLOW = '\033[33m'

  if options.colored and not (os.environ.has_key('BITTEN_NOCOLOR') and os.environ['BITTEN_NOCOLOR'] == 'true'):
    if html:
      str = str.replace('<r>', BOLD+RED)
      str = str.replace('<g>', BOLD+GREEN)
      str = str.replace('<y>', BOLD+YELLOW)
      str = str.replace('<b>', BOLD)
      str = re.sub(r'</[rgyb]>', RESET, str)
    else:
      str = str.replace('OK', BOLD+GREEN+'OK'+RESET)
      str = re.sub(r'(\[.*?\])', BOLD+CYAN+'\\1'+RESET, str)
      str = str.replace('skipped', BOLD+'skipped'+RESET)
      str = str.replace('deleted', BOLD+RED+'deleted'+RESET)
      if str.find('FAILED (EXODIFF)') != -1:
        str = str.replace('FAILED (EXODIFF)', BOLD+YELLOW+'FAILED (EXODIFF)'+RESET)
      elif str.find('FAILED (CSVDIFF') != -1:
        str = str.replace('FAILED (CSVDIFF)', BOLD+YELLOW+'FAILED (CSVDIFF)'+RESET)
      else:
        str = re.sub(r'(FAILED \([A-Za-z ]*\))', BOLD+RED+'\\1'+RESET, str)
  elif html:
    str = re.sub(r'</?[rgyb]>', '', str)    # strip all "html" tags

  return str

def getPlatforms():
  # We'll use uname to figure this out
  # Supported platforms are LINUX, DARWIN, SL, LION or ALL
  platforms = set()
  platforms.add('ALL')
  raw_uname = os.uname()
  if raw_uname[0].upper() == 'DARWIN':
    platforms.add('DARWIN')
    if re.match("10\.", raw_uname[2]):
      platforms.add('SL')
    if re.match("11\.", raw_uname[2]):
      platforms.add("LION")
  else:
    platforms.add(raw_uname[0].upper())
  return platforms

def getCompilers(libmesh_dir):
  # We'll use the GXX-VERSION string from LIBMESH's Make.common
  # to figure this out
  # Supported compilers are GCC, INTEL or ALL
  compilers = set()
  compilers.add('ALL')
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('GXX-VERSION') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        raw_compiler = m.group(1)
        if re.search('intel', raw_compiler, re.I) != None:
          compilers.add("INTEL")
        elif re.search('gcc', raw_compiler, re.I) != None:
          compilers.add("GCC")
        elif re.search('clang', raw_compiler, re.I) != None:
          compilers.add("CLANG")
        break
  f.close()
  return compilers

def getPetscVersion(libmesh_dir):
  # We'll use the petsc-major string from $LIBMESH_DIR/Make.common
  # to figure this out
  # Supported versions are 2, 3
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('petsc-version') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        raw_version = m.group(1)
        petsc_version = raw_version
        break
  f.close()
  return petsc_version

def getParmeshOption(libmesh_dir):
  # Some tests work differently with parallel mesh enabled
  # We need to detect this condition
  parmesh = set()

  # We check libmesh_config.h for LIBMESH_ENABLE_PARMESH
  filenames = [
    libmesh_dir + '/include/base/libmesh_config.h',   # Old location
    libmesh_dir + '/include/libmesh/libmesh_config.h' # New location
    ];

  success = 0

  for filename in filenames:
    if success == 1:
      break

    try:
      f = open(filename)

      for line in f.readlines():
        m = re.search(r'#define\s+LIBMESH_ENABLE_PARMESH\s+(\d+)', line)
        if m != None:
          if m.group(1) == '1':
            parmesh.add('PARALLEL')
          else:
            parmesh.add('SERIAL')
          break

      f.close()
      success = 1

    except IOError as e:
      #print "Warning: I/O Error trying to read", filename, ":", e.strerror, "... Will try other locations."
      pass

  if success == 0:
    print "Error! Could not find libmesh_config.h in any of the usual locations!"
    exit(1)

  # If we didn't find the #define indicated by having no entries in our set, then parmesh is off
  if not len(parmesh):
    parmesh.add('SERIAL')
  parmesh.add('ALL')

  return parmesh

def getSharedOption(libmesh_dir):
  # Some tests may only run properly with shared libraries on/off
  # We need to detect this condition
  shared_option = set()
  shared_option.add('ALL')
  f = open(libmesh_dir + '/Make.common')
  for line in f.readlines():
    if line.find('enable-shared') != -1:
      m = re.search(r'=\s*(\S+)', line)
      if m != None:
        if m.group(1) == 'yes':
          shared_option.add('DYNAMIC')
        else:
          shared_option.add('STATIC')
        break
  f.close()
  return shared_option
<|endoftext|>