<commit_before>import csv
import json
import logging
import os
from bottle import request, route
from buttervolume import btrfs
from datetime import datetime
from os.path import join, basename, exists, dirname
from subprocess import check_call, CalledProcessError
from subprocess import run, PIPE
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()

# absolute path to the volumes
VOLUMES_PATH = "/var/lib/docker/volumes/"
SNAPSHOTS_PATH = "/var/lib/docker/snapshots/"
TEST_REMOTE_PATH = "/var/lib/docker/received/"
SCHEDULE = "/etc/buttervolume/schedule.csv"
SCHEDULE_LOG = {'snapshot': {}, 'replicate': {}, 'synchronize': {}}

def jsonloads(stuff):
    return json.loads(stuff.decode())

@route('/Plugin.Activate', ['POST'])
def plugin_activate():
    return json.dumps({'Implements': ['VolumeDriver']})

@route('/VolumeDriver.Create', ['POST'])
def volume_create():
    name = jsonloads(request.body.read())['Name']
    if '@' in name:
        return json.dumps({'Err': '"@" is illegal in the name of the volume'})
    volpath = join(VOLUMES_PATH, name)
    # volume already exists?
    if name in [v['Name']for v in json.loads(volume_list())['Volumes']]:
        return json.dumps({'Err': ''})
    try:
        btrfs.Subvolume(volpath).create()
    except Exception as e:
        return json.dumps({'Err': e.strerror})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Mount', ['POST'])
def volume_mount():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if exists(join(path, '_data', '.nocow')) or exists(join(path, '.nocow')):
        try:
            check_call("chattr +C '{}'".format(join(path)), shell=True)
            log.info("disabled COW on %s", path)
        except Exception:
            return json.dumps(
                {'Err': 'could not disable COW on {}'.format(path)})
    if exists(join(path, '_data', '.nocow')):
        os.remove(join(path, '_data', '.nocow'))
    if exists(join(path, '.nocow')):
        os.remove(join(path, '.nocow'))
    return volume_path()

@route('/VolumeDriver.Path', ['POST'])
def volume_path():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if not btrfs.Subvolume(path).exists():
        return json.dumps({'Err': '{}: no such volume'.format(path)})
    return json.dumps({'Mountpoint': path, 'Err': ''})

@route('/VolumeDriver.Unmount', ['POST'])
def volume_unmount():
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Get', ['POST'])
def volume_get():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if not btrfs.Subvolume(path).exists():
        return json.dumps({'Err': '{}: no such volume'.format(path)})
    return json.dumps(
        {'Volume': {'Name': name, 'Mountpoint': path}, 'Err': ''})

@route('/VolumeDriver.Remove', ['POST'])
def volume_remove():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    try:
        btrfs.Subvolume(path).delete()
    except Exception:
        log.error('%s: no such volume', name)
        return json.dumps({'Err': '{}: no such volume'.format(name)})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.List', ['POST'])
def volume_list():
    volumes = []
    for p in [join(VOLUMES_PATH, v) for v in os.listdir(VOLUMES_PATH)
              if v != 'metadata.db']:
        if not btrfs.Subvolume(p).exists():
            continue
        volumes.append(p)
    return json.dumps({'Volumes': [{'Name': basename(v)} for v in volumes],
                       'Err': ''})

@route('/VolumeDriver.Volume.Sync', ['POST'])
def volume_sync():
    """Rsync between two nodes"""
    test = jsonloads(request.body.read()).get('Test', False)
    remote_volumes = VOLUMES_PATH if not test else TEST_REMOTE_PATH
    volumes = jsonloads(request.body.read())['Volumes']
    remote_hosts = jsonloads(request.body.read())['Hosts']
    port = os.getenv("SSH_PORT", '1122')
    errors = list()
    for volume_name in volumes:
        local_volume_path = join(VOLUMES_PATH, volume_name)
        remote_volume_path = join(remote_volumes, volume_name)
        for remote_host in remote_hosts:
            log.debug(
                "Rsync volume: %s from host: %s",
                local_volume_path, remote_host
            )
            try:
                cmd = [
                    'rsync', '-v', '-r', '-a', '-z', '-h', '-P',
                    '--update',
                    '-e', "ssh -p {}".format(port),
                    '{}:{}/'.format(remote_host, remote_volume_path),
                    local_volume_path,
                ]
                log.debug("Running %r", cmd)
                run(cmd, check=True, stdout=PIPE, stderr=PIPE)
            except Exception as ex:
                err = getattr(ex, 'stderr', ex)
                error_message = "Error while rsync {} from {} (cmd: {}): " \
                                "{}".format(volume_name, remote_host, cmd, err)
                log.error(error_message)
                errors.append(error_message)

    return json.dumps({'Err': '\n'.join(errors)})

@route('/VolumeDriver.Snapshot.Send', ['POST'])
def snapshot_send():
    """The last sent snapshot is remembered by adding a suffix with the target
    """
    test = jsonloads(request.body.read()).get('Test', False)
    snapshot_name = jsonloads(request.body.read())['Name']
    snapshot_path = join(SNAPSHOTS_PATH, snapshot_name)
    remote_host = jsonloads(request.body.read())['Host']
    remote_snapshots = SNAPSHOTS_PATH if not test else TEST_REMOTE_PATH
    # take the latest snapshot suffixed with the target host
    sent_snapshots = sorted(
        [s for s in os.listdir(SNAPSHOTS_PATH)
         if len(s.split('@')) == 3 and
         s.split('@')[0] == snapshot_name.split('@')[0] and
         s.split('@')[2] == remote_host])
    latest = sent_snapshots[-1] if len(sent_snapshots) > 0 else None
    if latest and len(latest.rsplit('@')) == 3:
        latest = latest.rsplit('@', 1)[0]
    parent = '-p "{}"'.format(join(SNAPSHOTS_PATH, latest)) if latest else ''
    port = os.getenv("SSH_PORT", '1122')
    run('sync', shell=True)  # needed by a current issue with send
    cmd = ('btrfs send {parent} "{snapshot_path}"'
           ' | ssh -p {port} {remote_host} "btrfs receive {remote_snapshots}"')
    try:
        log.info(cmd.format(**locals()))
        run(cmd.format(**locals()),
            shell=True, check=True, stdout=PIPE, stderr=PIPE)
    except CalledProcessError as e:
        log.warn('Failed using parent %s. Sending full snapshot %s '
                 '(stdout: %s, stderr: %s)',
                 latest, snapshot_path, e.stdout, e.stderr)
        parent = ''
        try:
            rmcmd = (
                'ssh -p {port} {remote_host} '
                '"btrfs subvolume delete {remote_snapshots}/{snapshot_name}"')
            log.info(rmcmd.format(**locals()))
            run(rmcmd.format(**locals()), shell=True, stdout=PIPE, stderr=PIPE)
            log.info(cmd.format(**locals()))
            run(cmd.format(**locals()),
                shell=True, check=True, stdout=PIPE, stderr=PIPE)
        except CalledProcessError as e:
            log.error('Failed sending full snapshot '
                      '(stdout: %s, stderr: %s)',
                      e.stdout, e.stderr)
            return json.dumps({'Err': str(e.stderr)})
    btrfs.Subvolume(snapshot_path).snapshot(
        '{}@{}'.format(snapshot_path, remote_host), readonly=True)
    for old_snapshot in sent_snapshots:
        btrfs.Subvolume(old_snapshot).delete
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Snapshot', ['POST'])
def volume_snapshot():
    """snapshot a volume in the SNAPSHOTS dir
    """
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    timestamped = '{}@{}'.format(name, datetime.now().isoformat())
    snapshot_path = join(SNAPSHOTS_PATH, timestamped)
    if not os.path.exists(path):
        return json.dumps({'Err': 'No such volume: {}'.format(name)})
    try:
        btrfs.Subvolume(path).snapshot(snapshot_path, readonly=True)
    except Exception as e:
        log.error("Error creating snapshot: %s", str(e))
        return json.dumps({'Err': str(e)})
    return json.dumps({'Err': '', 'Snapshot': timestamped})

@route('/VolumeDriver.Snapshot.List', ['POST'])
def snapshot_list():
    name = jsonloads(request.body.read()).get('Name')
    snapshots = os.listdir(SNAPSHOTS_PATH)
    if name:
        snapshots = [s for s in snapshots if s.startswith(name + '@')]
    return json.dumps({'Err': '', 'Snapshots': snapshots})

@route('/VolumeDriver.Snapshot.Remove', ['POST'])
def snapshot_delete():
    name = jsonloads(request.body.read())['Name']
    path = join(SNAPSHOTS_PATH, name)
    if not os.path.exists(path):
        return json.dumps({'Err': 'No such snapshot'})
    try:
        btrfs.Subvolume(path).delete()
    except Exception as e:
        log.error("Error deleting snapshot: %s", str(e))
        return json.dumps({'Err': str(e)})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Schedule', ['POST'])
def schedule():
    """Schedule or unschedule a job
    TODO add a lock
    """
    name = jsonloads(request.body.read())['Name']
    timer = jsonloads(request.body.read())['Timer']
    action = jsonloads(request.body.read())['Action']
    schedule = []
    if timer:  # 0 means unschedule!
        schedule.append((name, action, timer))
    if os.path.exists(SCHEDULE):
        with open(SCHEDULE) as f:
            for n, a, t in csv.reader(f):
                # skip the line we want to write
                if n == name and a == action:
                    continue
                schedule.append((n, a, t))
    os.makedirs(dirname(SCHEDULE), exist_ok=True)
    with open(SCHEDULE, 'w') as f:
        for line in schedule:
            csv.writer(f).writerow(line)
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Schedule.List', ['GET'])
def schedule_list():
    """List scheduled jobs
    """
    schedule = []
    if os.path.exists(SCHEDULE):
        with open(SCHEDULE) as f:
            for n, a, t in csv.reader(f):
                schedule.append({'Name': n, 'Timer': t, 'Action': a})
    return json.dumps({'Err': '', 'Schedule': schedule})

@route('/VolumeDriver.Snapshot.Restore', ['POST'])
def snapshot_restore():
    """
    Snapshot a volume and overwrite it with the specified snapshot.
    """
    snapshot_name = jsonloads(request.body.read())['Name']
    if '@' not in snapshot_name:
        # we're passing the name of the volume. Use the latest snapshot.
        volume_name = snapshot_name
        snapshots = os.listdir(SNAPSHOTS_PATH)
        snapshots = [s for s in snapshots if s.startswith(volume_name + '@')]
        if not snapshots:
            return json.dumps({'Err': ''})
        snapshot_name = sorted(snapshots)[-1]
    snapshot_path = join(SNAPSHOTS_PATH, snapshot_name)
    snapshot = btrfs.Subvolume(snapshot_path)
    volume_name = snapshot_name.split('@')[0]
    volume_path = join(VOLUMES_PATH, volume_name)
    volume = btrfs.Subvolume(volume_path)
    res = {'Err': ''}
    if snapshot.exists():
        if volume.exists():
            # backup and delete
            timestamp = datetime.now().isoformat()
            stamped_name = '{}@{}'.format(volume_name, timestamp)
            stamped_path = join(SNAPSHOTS_PATH, stamped_name)
            volume.snapshot(stamped_path, readonly=True)
            res['VolumeBackup'] = stamped_name
            volume.delete()
        snapshot.snapshot(volume_path)
    else:
        res['Err'] = 'No such snapshot'
    return json.dumps(res)

@route('/VolumeDriver.Snapshots.Purge', ['POST'])
def snapshots_purge():
    """
    Purge snapshots with a retention pattern
    (see cli help)
    """
    params = jsonloads(request.body.read())
    volume_name = params['Name']
    dryrun = params.get('Dryrun', False)
    units = {'m': 1, 'h': 60, 'd': 60*24, 'w': 60*24*7, 'y': 60*24*365}
    try:
        pattern = sorted(int(i[:-1])*units[i[-1]]
                         for i in params['Pattern'].split(':'))
        assert(len(pattern) >= 2)
        max_age = pattern[-1]
    except:
        log.error("Invalid purge pattern: %s", params['Pattern'])
        return json.dumps({'Err': 'Invalid purge pattern'})
    # snapshots related to the volume, more recents first
    snapshots = sorted((s for s in os.listdir(SNAPSHOTS_PATH)
                        if s.startswith(volume_name + '@')), reverse=True)
    now = datetime.now()
    # Age of the snapshots in minutes.
    # Example : [30, 70, 90, 150, 210, ..., 4000]
    snapshots_age = []
    for s in snapshots:
        try:
            snapshots_age.append(
                int((now - datetime.strptime(
                    s.split('@')[1], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds()
                    )//60)
        except:
            log.info("Skipping purge of %s with invalid date format", s)
            continue
    if not snapshots:
        return json.dumps({'Err': ''})
    # pattern = 60:180:3600
    # age segments = [(60, 180), (180, 3600)]
    try:
        for age_segment in [(pattern[i], pattern[i+1])
                            for i, p in enumerate(pattern[:-1])]:
            last_timeframe = -1
            for i, age in enumerate(snapshots_age):
                # if the age is outside the age_segment, delete nothing.
                # Only 70 and 90 are inside the age_segment (60, 180)
                if age <= age_segment[0] or age >= age_segment[1] < max_age:
                    continue
                # Now get the timeframe of the snapshot.
                # Ages 70 and 90 are in the same timeframe (70//60 == 90//60)
                timeframe = age // age_segment[0]
                # delete if we already had a snapshot in the same timeframe
                # or if the snapshot is very old
                if timeframe == last_timeframe or age > max_age:
                    snapshot = snapshots[i]
                    if dryrun:
                        log.info('(Dry run) Would delete snapshot {}'
                                 .format(snapshot))
                    else:
                        btrfs.Subvolume(
                            join(SNAPSHOTS_PATH, snapshot)).delete()
                        log.info('Deleted snapshot {}'.format(snapshot))
                last_timeframe = timeframe
    except Exception as e:
        log.error("Error purging snapshots: %s", e.strerror)
        return json.dumps({'Err': e.strerror})
    return json.dumps({'Err': ''})
<commit_msg>don't delete at max age<commit_after>import csv
import json
import logging
import os
from bottle import request, route
from buttervolume import btrfs
from datetime import datetime
from os.path import join, basename, exists, dirname
from subprocess import check_call, CalledProcessError
from subprocess import run, PIPE
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()

# absolute path to the volumes
VOLUMES_PATH = "/var/lib/docker/volumes/"
SNAPSHOTS_PATH = "/var/lib/docker/snapshots/"
TEST_REMOTE_PATH = "/var/lib/docker/received/"
SCHEDULE = "/etc/buttervolume/schedule.csv"
SCHEDULE_LOG = {'snapshot': {}, 'replicate': {}, 'synchronize': {}}

def jsonloads(stuff):
    return json.loads(stuff.decode())

@route('/Plugin.Activate', ['POST'])
def plugin_activate():
    return json.dumps({'Implements': ['VolumeDriver']})

@route('/VolumeDriver.Create', ['POST'])
def volume_create():
    name = jsonloads(request.body.read())['Name']
    if '@' in name:
        return json.dumps({'Err': '"@" is illegal in the name of the volume'})
    volpath = join(VOLUMES_PATH, name)
    # volume already exists?
    if name in [v['Name']for v in json.loads(volume_list())['Volumes']]:
        return json.dumps({'Err': ''})
    try:
        btrfs.Subvolume(volpath).create()
    except Exception as e:
        return json.dumps({'Err': e.strerror})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Mount', ['POST'])
def volume_mount():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if exists(join(path, '_data', '.nocow')) or exists(join(path, '.nocow')):
        try:
            check_call("chattr +C '{}'".format(join(path)), shell=True)
            log.info("disabled COW on %s", path)
        except Exception:
            return json.dumps(
                {'Err': 'could not disable COW on {}'.format(path)})
    if exists(join(path, '_data', '.nocow')):
        os.remove(join(path, '_data', '.nocow'))
    if exists(join(path, '.nocow')):
        os.remove(join(path, '.nocow'))
    return volume_path()

@route('/VolumeDriver.Path', ['POST'])
def volume_path():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if not btrfs.Subvolume(path).exists():
        return json.dumps({'Err': '{}: no such volume'.format(path)})
    return json.dumps({'Mountpoint': path, 'Err': ''})

@route('/VolumeDriver.Unmount', ['POST'])
def volume_unmount():
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Get', ['POST'])
def volume_get():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    if not btrfs.Subvolume(path).exists():
        return json.dumps({'Err': '{}: no such volume'.format(path)})
    return json.dumps(
        {'Volume': {'Name': name, 'Mountpoint': path}, 'Err': ''})

@route('/VolumeDriver.Remove', ['POST'])
def volume_remove():
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    try:
        btrfs.Subvolume(path).delete()
    except Exception:
        log.error('%s: no such volume', name)
        return json.dumps({'Err': '{}: no such volume'.format(name)})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.List', ['POST'])
def volume_list():
    volumes = []
    for p in [join(VOLUMES_PATH, v) for v in os.listdir(VOLUMES_PATH)
              if v != 'metadata.db']:
        if not btrfs.Subvolume(p).exists():
            continue
        volumes.append(p)
    return json.dumps({'Volumes': [{'Name': basename(v)} for v in volumes],
                       'Err': ''})

@route('/VolumeDriver.Volume.Sync', ['POST'])
def volume_sync():
    """Rsync between two nodes"""
    test = jsonloads(request.body.read()).get('Test', False)
    remote_volumes = VOLUMES_PATH if not test else TEST_REMOTE_PATH
    volumes = jsonloads(request.body.read())['Volumes']
    remote_hosts = jsonloads(request.body.read())['Hosts']
    port = os.getenv("SSH_PORT", '1122')
    errors = list()
    for volume_name in volumes:
        local_volume_path = join(VOLUMES_PATH, volume_name)
        remote_volume_path = join(remote_volumes, volume_name)
        for remote_host in remote_hosts:
            log.debug(
                "Rsync volume: %s from host: %s",
                local_volume_path, remote_host
            )
            try:
                cmd = [
                    'rsync', '-v', '-r', '-a', '-z', '-h', '-P',
                    '--update',
                    '-e', "ssh -p {}".format(port),
                    '{}:{}/'.format(remote_host, remote_volume_path),
                    local_volume_path,
                ]
                log.debug("Running %r", cmd)
                run(cmd, check=True, stdout=PIPE, stderr=PIPE)
            except Exception as ex:
                err = getattr(ex, 'stderr', ex)
                error_message = "Error while rsync {} from {} (cmd: {}): " \
                                "{}".format(volume_name, remote_host, cmd, err)
                log.error(error_message)
                errors.append(error_message)

    return json.dumps({'Err': '\n'.join(errors)})

@route('/VolumeDriver.Snapshot.Send', ['POST'])
def snapshot_send():
    """The last sent snapshot is remembered by adding a suffix with the target
    """
    test = jsonloads(request.body.read()).get('Test', False)
    snapshot_name = jsonloads(request.body.read())['Name']
    snapshot_path = join(SNAPSHOTS_PATH, snapshot_name)
    remote_host = jsonloads(request.body.read())['Host']
    remote_snapshots = SNAPSHOTS_PATH if not test else TEST_REMOTE_PATH
    # take the latest snapshot suffixed with the target host
    sent_snapshots = sorted(
        [s for s in os.listdir(SNAPSHOTS_PATH)
         if len(s.split('@')) == 3 and
         s.split('@')[0] == snapshot_name.split('@')[0] and
         s.split('@')[2] == remote_host])
    latest = sent_snapshots[-1] if len(sent_snapshots) > 0 else None
    if latest and len(latest.rsplit('@')) == 3:
        latest = latest.rsplit('@', 1)[0]
    parent = '-p "{}"'.format(join(SNAPSHOTS_PATH, latest)) if latest else ''
    port = os.getenv("SSH_PORT", '1122')
    run('sync', shell=True)  # needed by a current issue with send
    cmd = ('btrfs send {parent} "{snapshot_path}"'
           ' | ssh -p {port} {remote_host} "btrfs receive {remote_snapshots}"')
    try:
        log.info(cmd.format(**locals()))
        run(cmd.format(**locals()),
            shell=True, check=True, stdout=PIPE, stderr=PIPE)
    except CalledProcessError as e:
        log.warn('Failed using parent %s. Sending full snapshot %s '
                 '(stdout: %s, stderr: %s)',
                 latest, snapshot_path, e.stdout, e.stderr)
        parent = ''
        try:
            rmcmd = (
                'ssh -p {port} {remote_host} '
                '"btrfs subvolume delete {remote_snapshots}/{snapshot_name}"')
            log.info(rmcmd.format(**locals()))
            run(rmcmd.format(**locals()), shell=True, stdout=PIPE, stderr=PIPE)
            log.info(cmd.format(**locals()))
            run(cmd.format(**locals()),
                shell=True, check=True, stdout=PIPE, stderr=PIPE)
        except CalledProcessError as e:
            log.error('Failed sending full snapshot '
                      '(stdout: %s, stderr: %s)',
                      e.stdout, e.stderr)
            return json.dumps({'Err': str(e.stderr)})
    btrfs.Subvolume(snapshot_path).snapshot(
        '{}@{}'.format(snapshot_path, remote_host), readonly=True)
    for old_snapshot in sent_snapshots:
        btrfs.Subvolume(old_snapshot).delete
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Snapshot', ['POST'])
def volume_snapshot():
    """snapshot a volume in the SNAPSHOTS dir
    """
    name = jsonloads(request.body.read())['Name']
    path = join(VOLUMES_PATH, name)
    timestamped = '{}@{}'.format(name, datetime.now().isoformat())
    snapshot_path = join(SNAPSHOTS_PATH, timestamped)
    if not os.path.exists(path):
        return json.dumps({'Err': 'No such volume: {}'.format(name)})
    try:
        btrfs.Subvolume(path).snapshot(snapshot_path, readonly=True)
    except Exception as e:
        log.error("Error creating snapshot: %s", str(e))
        return json.dumps({'Err': str(e)})
    return json.dumps({'Err': '', 'Snapshot': timestamped})

@route('/VolumeDriver.Snapshot.List', ['POST'])
def snapshot_list():
    name = jsonloads(request.body.read()).get('Name')
    snapshots = os.listdir(SNAPSHOTS_PATH)
    if name:
        snapshots = [s for s in snapshots if s.startswith(name + '@')]
    return json.dumps({'Err': '', 'Snapshots': snapshots})

@route('/VolumeDriver.Snapshot.Remove', ['POST'])
def snapshot_delete():
    name = jsonloads(request.body.read())['Name']
    path = join(SNAPSHOTS_PATH, name)
    if not os.path.exists(path):
        return json.dumps({'Err': 'No such snapshot'})
    try:
        btrfs.Subvolume(path).delete()
    except Exception as e:
        log.error("Error deleting snapshot: %s", str(e))
        return json.dumps({'Err': str(e)})
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Schedule', ['POST'])
def schedule():
    """Schedule or unschedule a job
    TODO add a lock
    """
    name = jsonloads(request.body.read())['Name']
    timer = jsonloads(request.body.read())['Timer']
    action = jsonloads(request.body.read())['Action']
    schedule = []
    if timer:  # 0 means unschedule!
        schedule.append((name, action, timer))
    if os.path.exists(SCHEDULE):
        with open(SCHEDULE) as f:
            for n, a, t in csv.reader(f):
                # skip the line we want to write
                if n == name and a == action:
                    continue
                schedule.append((n, a, t))
    os.makedirs(dirname(SCHEDULE), exist_ok=True)
    with open(SCHEDULE, 'w') as f:
        for line in schedule:
            csv.writer(f).writerow(line)
    return json.dumps({'Err': ''})

@route('/VolumeDriver.Schedule.List', ['GET'])
def schedule_list():
    """List scheduled jobs
    """
    schedule = []
    if os.path.exists(SCHEDULE):
        with open(SCHEDULE) as f:
            for n, a, t in csv.reader(f):
                schedule.append({'Name': n, 'Timer': t, 'Action': a})
    return json.dumps({'Err': '', 'Schedule': schedule})

@route('/VolumeDriver.Snapshot.Restore', ['POST'])
def snapshot_restore():
    """
    Snapshot a volume and overwrite it with the specified snapshot.
    """
    snapshot_name = jsonloads(request.body.read())['Name']
    if '@' not in snapshot_name:
        # we're passing the name of the volume. Use the latest snapshot.
        volume_name = snapshot_name
        snapshots = os.listdir(SNAPSHOTS_PATH)
        snapshots = [s for s in snapshots if s.startswith(volume_name + '@')]
        if not snapshots:
            return json.dumps({'Err': ''})
        snapshot_name = sorted(snapshots)[-1]
    snapshot_path = join(SNAPSHOTS_PATH, snapshot_name)
    snapshot = btrfs.Subvolume(snapshot_path)
    volume_name = snapshot_name.split('@')[0]
    volume_path = join(VOLUMES_PATH, volume_name)
    volume = btrfs.Subvolume(volume_path)
    res = {'Err': ''}
    if snapshot.exists():
        if volume.exists():
            # backup and delete
            timestamp = datetime.now().isoformat()
            stamped_name = '{}@{}'.format(volume_name, timestamp)
            stamped_path = join(SNAPSHOTS_PATH, stamped_name)
            volume.snapshot(stamped_path, readonly=True)
            res['VolumeBackup'] = stamped_name
            volume.delete()
        snapshot.snapshot(volume_path)
    else:
        res['Err'] = 'No such snapshot'
    return json.dumps(res)

@route('/VolumeDriver.Snapshots.Purge', ['POST'])
def snapshots_purge():
    """
    Purge snapshots with a retention pattern
    (see cli help)
    """
    params = jsonloads(request.body.read())
    volume_name = params['Name']
    dryrun = params.get('Dryrun', False)
    units = {'m': 1, 'h': 60, 'd': 60*24, 'w': 60*24*7, 'y': 60*24*365}
    try:
        pattern = sorted(int(i[:-1])*units[i[-1]]
                         for i in params['Pattern'].split(':'))
        assert(len(pattern) >= 2)
        max_age = pattern[-1]
    except:
        log.error("Invalid purge pattern: %s", params['Pattern'])
        return json.dumps({'Err': 'Invalid purge pattern'})
    # snapshots related to the volume, more recents first
    snapshots = sorted((s for s in os.listdir(SNAPSHOTS_PATH)
                        if s.startswith(volume_name + '@')), reverse=True)
    now = datetime.now()
    # Age of the snapshots in minutes.
    # Example : [30, 70, 90, 150, 210, ..., 4000]
    snapshots_age = []
    for s in snapshots:
        try:
            snapshots_age.append(
                int((now - datetime.strptime(
                    s.split('@')[1], "%Y-%m-%dT%H:%M:%S.%f")).total_seconds()
                    )//60)
        except:
            log.info("Skipping purge of %s with invalid date format", s)
            continue
    if not snapshots:
        return json.dumps({'Err': ''})
    # pattern = 60:180:3600
    # age segments = [(60, 180), (180, 3600)]
    try:
        for age_segment in [(pattern[i], pattern[i+1])
                            for i, p in enumerate(pattern[:-1])]:
            last_timeframe = -1
            for i, age in enumerate(snapshots_age):
                # if the age is outside the age_segment, delete nothing.
                # Only 70 and 90 are inside the age_segment (60, 180)
                if age <= age_segment[0] or age >= age_segment[1] <= max_age:
                    continue
                # Now get the timeframe number of the snapshot.
                # Ages 70 and 90 are in the same timeframe (70//60 == 90//60)
                timeframe = age // age_segment[0]
                # delete if we already had a snapshot in the same timeframe
                # or if the snapshot is very old
                if timeframe == last_timeframe or age > max_age:
                    snapshot = snapshots[i]
                    if dryrun:
                        log.info('(Dry run) Would delete snapshot {}'
                                 .format(snapshot))
                    else:
                        btrfs.Subvolume(
                            join(SNAPSHOTS_PATH, snapshot)).delete()
                        log.info('Deleted snapshot {}'.format(snapshot))
                last_timeframe = timeframe
    except Exception as e:
        log.error("Error purging snapshots: %s", e.strerror)
        return json.dumps({'Err': e.strerror})
    return json.dumps({'Err': ''})
<|endoftext|>


<commit_before>"""Models for main lutris app"""
# pylint: disable=E1002, E0202
import yaml
import json
import datetime
from django.core.mail import send_mail
from django.db import models
from django.db.models import Q, Count
from django.conf import settings
from django.utils.translation import ugettext as _
from django.template.defaultfilters import slugify
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.core.urlresolvers import reverse
from bitfield import BitField
from jsonfield import JSONField

from games import managers
from games.util import steam

DEFAULT_INSTALLER = {
    'files': [
        {'file_id': "http://location"},
        {'unredistribuable_file': "N/A"}
    ],
    'installer': [
        {'move': {'src': 'file_id', 'dst': '$GAMEDIR'}}
    ]
}

class Platform(models.Model):
    """Gaming platform"""
    name = models.CharField(_('Name'), max_length=127)
    slug = models.SlugField(unique=True)
    icon = models.ImageField(upload_to='platforms/icons', blank=True)
    default_installer = JSONField(null=True, blank=True)

    # pylint: disable=W0232, R0903
    class Meta:
        ordering = ('name', )

    def __unicode__(self):
        return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        return super(Platform, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class Company(models.Model):
    """Gaming company"""
    name = models.CharField(_('Name'), max_length=127)
    slug = models.SlugField(unique=True)
    logo = models.ImageField(upload_to='companies/logos', blank=True)
    website = models.CharField(max_length=128, blank=True)

    # pylint: disable=W0232, R0903
    class Meta(object):
        """Additional configuration for model"""
        verbose_name_plural = "companies"
        ordering = ['name']

    def get_absolute_url(self):
        return reverse("games_by_company", args=(self.slug, ))

    def __unicode__(self):
        return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        return super(Company, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', 'slug__icontains')

class Runner(models.Model):
    """ Model definition for the runners """
    name = models.CharField(_("Name"), max_length=127)
    slug = models.SlugField(unique=True)
    website = models.CharField(_("Website"), max_length=127, blank=True)
    icon = models.ImageField(upload_to='runners/icons', blank=True)
    platforms = models.ManyToManyField(Platform)

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        return super(Runner, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class RunnerVersion(models.Model):
    runner = models.ForeignKey(Runner)
    version = models.CharField(max_length=32)
    path = models.CharField(max_length=128, blank=True)

class Genre(models.Model):
    """Gaming genre"""
    name = models.CharField(max_length=50)
    slug = models.SlugField(unique=True)

    # pylint: disable=W0232, R0903
    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        return super(Genre, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class GameManager(models.Manager):
    def published(self):
        return self.get_query_set().filter(is_public=True)

    def with_installer(self):
        return (
            self.get_query_set()
            .filter(is_public=True)
            .filter(
                Q(installer__published=True) |
                Q(platforms__default_installer__isnull=False)
            )
            .order_by('name')
            .annotate(installer_count=Count('installer'))
            .annotate(default_installer_count=Count('platforms'))
        )

class Game(models.Model):
    """Game model"""
    name = models.CharField(max_length=200)
    slug = models.SlugField(unique=True, blank=False)
    year = models.IntegerField(null=True, blank=True)
    platforms = models.ManyToManyField(Platform, null=True, blank=True)
    genres = models.ManyToManyField(Genre, null=True, blank=True)
    publisher = models.ForeignKey(
        Company, related_name='published_game', null=True, blank=True
    )
    developer = models.ForeignKey(
        Company, related_name='developed_game', null=True, blank=True
    )
    website = models.CharField(max_length=200, blank=True)
    icon = models.ImageField(upload_to='games/icons', blank=True)
    title_logo = models.ImageField(upload_to='games/banners', blank=True)
    description = models.TextField(blank=True)
    is_public = models.BooleanField("Published on website", default=False)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    steamid = models.PositiveIntegerField(null=True, blank=True)
    flags = BitField(flags=(
        'open_source',
        'open_engine',
        'freeware',
    ))

    objects = GameManager()

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']
        permissions = (
            ('can_publish_game', "Can make the game visible"),
        )

    def __unicode__(self):
        return self.name

    @staticmethod
    def autocomplete_search_fields():
        return ("name__icontains",)

    def has_installer(self):
        return self.installer_set.count() > 0 \
            or bool(self.get_default_installers())

    def get_absolute_url(self):
        """Return the absolute url for a game"""
        return "/games/%s/" % self.slug

    def download_steam_capsule(self):
        if self.title_logo or not self.steamid:
            return
        else:
            self.title_logo = ContentFile(steam.get_capsule(self.steamid),
                                          "%d.jpg" % self.steamid)

    def get_steam_logo(self, img_url):
        self.title_logo = ContentFile(steam.get_image(self.steamid, img_url),
                                      "%d.jpg" % self.steamid)

    def get_steam_icon(self, img_url):
        self.icon = ContentFile(steam.get_image(self.steamid, img_url),
                                "%d.jpg" % self.steamid)

    def steam_support(self):
        """ Return the platform supported by Steam """
        if not self.steamid:
            return False
        platforms = [p.slug for p in self.platforms.all()]
        if 'linux' in platforms:
            return 'linux'
        elif 'windows' in platforms:
            return 'windows'
        else:
            return True

    def get_default_installers(self):
        installers = []
        for platform in self.platforms.all():
            if platform.default_installer:
                installer = platform.default_installer
                installer['name'] = self.name
                installer['version'] = platform.slug
                installer['slug'] = "-".join((self.slug[:30],
                                              platform.slug[:20]))
                installer['platform'] = platform.slug
                installer['description'] = platform.name + " version"
                installer['published'] = True
                installer['auto'] = True
                installers.append(installer)
        return installers

    def check_for_submission(self):

        # Skip freshly created and unpublished objects
        if not self.pk or not self.is_public:
            return

        # Skip objects that were already published
        original = Game.objects.get(pk=self.pk)
        if original.is_public:
            return

        try:
            submission = GameSubmission.objects.get(game=self,
                                                    accepted_at__isnull=True)
        except GameSubmission.DoesNotExist:
            pass
        else:
            submission.accept()

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)[:50]
        self.download_steam_capsule()
        self.check_for_submission()
        return super(Game, self).save(*args, **kwargs)

class GameMetadata(models.Model):
    game = models.ForeignKey(Game)
    key = models.CharField(max_length=16)
    value = models.CharField(max_length=255)

class Screenshot(models.Model):
    """Screenshots for games"""
    game = models.ForeignKey(Game)
    image = models.ImageField(upload_to="games/screenshots")
    uploaded_at = models.DateTimeField(auto_now_add=True)
    uploaded_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    description = models.CharField(max_length=256, null=True, blank=True)
    published = models.BooleanField(default=False)

    objects = managers.ScreenshotManager()

    def __unicode__(self):
        desc = self.description if self.description else self.game.name
        return "%s: %s (uploaded by %s)" % (self.game, desc, self.uploaded_by)

    def game_link(self):
        return u"{1}".format(
            reverse("admin:games_game_change", args=(self.game.id, )),
            self.game
        )
    game_link.allow_tags = True
    game_link.short_description = "Game (link)"

class InstallerManager(models.Manager):
    def published(self, user=None, is_staff=False):
        if is_staff:
            return self.get_query_set()
        elif user:
            return self.get_query_set().filter(models.Q(published=True)
                                               | models.Q(user=user))
        else:
            return self.get_query_set().filter(published=True)

    def fuzzy_get(self, slug):
        """Return either the installer that matches exactly 'slug' or the
        installers with game matching slug.
        Installers are always returned in a list.
        """
        try:
            installer = self.get_query_set().get(slug=slug)
            return [installer]
        except ObjectDoesNotExist:
            installers = self.get_query_set().filter(game__slug=slug,
                                                     published=True)
            if not installers:
                raise
            else:
                return installers

    def get_json(self, slug):
        try:
            installers = self.fuzzy_get(slug)
        except ObjectDoesNotExist:
            installer_data = []
        else:
            installer_data = [installer.as_dict() for installer in installers]
        try:
            game = Game.objects.get(slug=slug)
            installer_data += game.get_default_installers()
        except ObjectDoesNotExist:
            pass
        if not installer_data:
            raise Installer.DoesNotExist
        return json.dumps(installer_data)

class Installer(models.Model):
    """Game installer model"""
    game = models.ForeignKey(Game)
    user = models.ForeignKey(settings.AUTH_USER_MODEL)
    runner = models.ForeignKey(Runner)

    slug = models.SlugField(unique=True)
    version = models.CharField(max_length=32)
    description = models.CharField(max_length=512, blank=True, null=True)
    content = models.TextField()
    created_at = models.DateTimeField(auto_now_add=True, null=True)
    updated_at = models.DateTimeField(auto_now=True)
    published = models.BooleanField(default=False)
    objects = InstallerManager()

    def __unicode__(self):
        return self.slug

    def game_link(self):
        return u"{1}".format(
            reverse("admin:games_game_change", args=(self.game.id, )),
            self.game
        )
    game_link.allow_tags = True
    game_link.short_description = "Game (link)"

    def set_default_installer(self):
        if self.game and self.game.steam_support():
            installer_data = {'game': {'appid': self.game.steamid}}
            self.version = 'steam'
        else:
            installer_data = DEFAULT_INSTALLER
        self.content = yaml.safe_dump(installer_data, default_flow_style=False)

    def as_dict(self):
        yaml_content = yaml.safe_load(self.content) or {}
        yaml_content['version'] = self.version
        yaml_content['name'] = self.game.name
        yaml_content['year'] = self.game.year
        yaml_content['steamid'] = self.game.steamid
        try:
            yaml_content['runner'] = self.runner.slug
        except Runner.DoesNotExist:
            yaml_content['runner'] = ''
        yaml_content['slug'] = self.slug
        return yaml_content

    def as_yaml(self):
        return yaml.safe_dump(self.as_dict())

    def as_json(self):
        return json.dumps(self.as_dict())

    def build_slug(self, version):
        return "%s-%s" % (slugify(self.game.name)[:30],
                          slugify(version)[:20])

    def save(self, *args, **kwargs):
        self.slug = self.build_slug(self.version)
        return super(Installer, self).save(*args, **kwargs)

class GameLibrary(models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL)
    games = models.ManyToManyField(Game)

    # pylint: disable=W0232, R0903
    class Meta:
        verbose_name_plural = "game libraries"

    def __unicode__(self):
        return "%s's library" % self.user.username

class Featured(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey('content_type', 'object_id')
    image = models.ImageField(upload_to='featured', max_length=100)
    description = models.CharField(max_length=255, blank=True, null=True)
    created_at = models.DateTimeField(auto_now_add=True)

    # pylint: disable=W0232, R0903
    class Meta:
        verbose_name = "Featured content"

    def __unicode__(self):
        return "[%s] %s" % (self.content_type, str(self.content_object), )

class GameSubmission(models.Model):
    user = models.ForeignKey(settings.AUTH_USER_MODEL)
    game = models.ForeignKey(Game)
    created_at = models.DateTimeField(auto_now_add=True)
    accepted_at = models.DateTimeField(null=True)

    class Meta:
        verbose_name = "User submitted game"

    def __unicode__(self):
        return u"{0} submitted {1} on {2}".format(self.user, self.game,
                                                  self.created_at)

    def accept(self):
        self.accepted_at = datetime.datetime.now()
        self.save()
        subject = u"{} Your game submission for '{}' as been accepted!".format(
            settings.EMAIL_SUBJECT_PREFIX,
            self.game.name
        )
        body = u"""
Hello {0}!

Your submission for {1} has been reviewed by a moderator and approved!

The game's page is available at https://lutris.net{2}
You can submit an installer script for it if you haven't done so already. The
scripting details are explained on the installer submission page, you can have
a look at other scripts to see how they are written. If you get confused or
have any questions about the scripting process, please drop us a line at
admin@lutris.net or on IRC: #lutris on Freenode.

Have a great day!

The Lutris team
        """.format(
            self.user.username,
            self.game.name,
            self.game.get_absolute_url()
        )
        send_mail(subject, body, settings.DEFAULT_FROM_EMAIL, [self.user.email])
<commit_msg>Fix installer slug too long error<commit_after>"""Models for main lutris app"""
# pylint: disable=E1002, E0202
import yaml
import json
import datetime
from django.core.mail import send_mail
from django.db import models
from django.db.models import Q, Count
from django.conf import settings
from django.utils.translation import ugettext as _
from django.template.defaultfilters import slugify
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.core.urlresolvers import reverse
from bitfield import BitField
from jsonfield import JSONField

from games import managers
from games.util import steam

DEFAULT_INSTALLER = {
    'files': [
        {'file_id': "http://location"},
        {'unredistribuable_file': "N/A"}
    ],
    'installer': [
        {'move': {'src': 'file_id', 'dst': '$GAMEDIR'}}
    ]
}

class Platform(models.Model):
    """Gaming platform"""
    name = models.CharField(_('Name'), max_length=127)
    slug = models.SlugField(unique=True)
    icon = models.ImageField(upload_to='platforms/icons', blank=True)
    default_installer = JSONField(null=True, blank=True)

    # pylint: disable=W0232, R0903
    class Meta:
        ordering = ('name', )

    def __unicode__(self):
        return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        return super(Platform, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class Company(models.Model):
    """Gaming company"""
    name = models.CharField(_('Name'), max_length=127)
    slug = models.SlugField(unique=True)
    logo = models.ImageField(upload_to='companies/logos', blank=True)
    website = models.CharField(max_length=128, blank=True)

    # pylint: disable=W0232, R0903
    class Meta(object):
        """Additional configuration for model"""
        verbose_name_plural = "companies"
        ordering = ['name']

    def get_absolute_url(self):
        return reverse("games_by_company", args=(self.slug, ))

    def __unicode__(self):
        return "%s" % self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(self.name)
        return super(Company, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', 'slug__icontains')

class Runner(models.Model):
    """ Model definition for the runners """
    name = models.CharField(_("Name"), max_length=127)
    slug = models.SlugField(unique=True)
    website = models.CharField(_("Website"), max_length=127, blank=True)
    icon = models.ImageField(upload_to='runners/icons', blank=True)
    platforms = models.ManyToManyField(Platform)

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        return super(Runner, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class RunnerVersion(models.Model):
    runner = models.ForeignKey(Runner)
    version = models.CharField(max_length=32)
    path = models.CharField(max_length=128, blank=True)

class Genre(models.Model):
    """Gaming genre"""
    name = models.CharField(max_length=50)
    slug = models.SlugField(unique=True)

    # pylint: disable=W0232, R0903
    class Meta:
        ordering = ['name']

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        return super(Genre, self).save(*args, **kwargs)

    @staticmethod
    def autocomplete_search_fields():
        return ('name__icontains', )

class GameManager(models.Manager):
    def published(self):
        return self.get_query_set().filter(is_public=True)

    def with_installer(self):
        return (
            self.get_query_set()
            .filter(is_public=True)
            .filter(
                Q(installer__published=True) |
                Q(platforms__default_installer__isnull=False)
            )
            .order_by('name')
            .annotate(installer_count=Count('installer'))
            .annotate(default_installer_count=Count('platforms'))
        )

class Game(models.Model):
    """Game model"""
    name = models.CharField(max_length=200)
    slug = models.SlugField(unique=True, blank=False)
    year = models.IntegerField(null=True, blank=True)
    platforms = models.ManyToManyField(Platform, null=True, blank=True)
    genres = models.ManyToManyField(Genre, null=True, blank=True)
    publisher = models.ForeignKey(
        Company, related_name='published_game', null=True, blank=True
    )
    developer = models.ForeignKey(
        Company, related_name='developed_game', null=True, blank=True
    )
    website = models.CharField(max_length=200, blank=True)
    icon = models.ImageField(upload_to='games/icons', blank=True)
    title_logo = models.ImageField(upload_to='games/banners', blank=True)
    description = models.TextField(blank=True)
    is_public = models.BooleanField("Published on website", default=False)
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    steamid = models.PositiveIntegerField(null=True, blank=True)
    flags = BitField(flags=(
        'open_source',
        'open_engine',
        'freeware',
    ))

    objects = GameManager()

    # pylint: disable=W0232, R0903
    class Meta(object):
        ordering = ['name']
        permissions = (
            ('can_publish_game', "Can make the game visible"),
        )

    def __unicode__(self):
        return self.name

    @staticmethod
    def autocomplete_search_fields():
        return ("name__icontains",)

    def has_installer(self):
        return self.installer_set.count() > 0 \
            or bool(self.get_default_installers())

    def get_absolute_url(self):
        """Return the absolute url for a game"""
        return "/games/%s/" % self.slug

    def download_steam_capsule(self):
        if self.title_logo or not self.steamid:
            return
        else:
            self.title_logo = ContentFile(steam.get_capsule(self.steamid),
                                          "%d.jpg" % self.steamid)

    def get_steam_logo(self, img_url):
        self.title_logo = ContentFile(steam.get_image(self.steamid, img_url),
                                      "%d.jpg" % self.steamid)

    def get_steam_icon(self, img_url):
        self.icon = ContentFile(steam.get_image(self.steamid, img_url),
                                "%d.jpg" % self.steamid)

    def steam_support(self):
        """ Return the platform supported by Steam """
        if not self.steamid:
            return False
        platforms = [p.slug for p in self.platforms.all()]
        if 'linux' in platforms:
            return 'linux'
        elif 'windows' in platforms:
            return 'windows'
        else:
            return True

    def get_default_installers(self):
        installers = []
        for platform in self.platforms.all():
            if platform.default_installer:
                installer = platform.default_installer
                installer['name'] = self.name
                installer['version'] = platform.slug
                installer['slug'] = "-".join((self.slug[:30],
                                              platform.slug[:20]))
                installer['platform'] = platform.slug
                installer['description'] = platform.name + " version"
                installer['published'] = True
                installer['auto'] = True
                installers.append(installer)
        return installers

    def check_for_submission(self):

        # Skip freshly created and unpublished objects
        if not self.pk or not self.is_public:
            return

        # Skip objects that were already published
        original = Game.objects.get(pk=self.pk)
        if original.is_public:
            return

        try:
            submission = GameSubmission.objects.get(game=self,
                                                    accepted_at__isnull=True)
        except GameSubmission.DoesNotExist:
            pass
        else:
            submission.accept()

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)[:50]
        self.download_steam_capsule()
        self.check_for_submission()
        return super(Game, self).save(*args, **kwargs)

class GameMetadata(models.Model):
    game = models.ForeignKey(Game)
    key = models.CharField(max_length=16)
    value = models.CharField(max_length=255)

class Screenshot(models.Model):
    """Screenshots for games"""
    game = models.ForeignKey(Game)
    image = models.ImageField(upload_to="games/screenshots")
    uploaded_at = models.DateTimeField(auto_now_add=True)
    uploaded_by = models.ForeignKey(settings.AUTH_USER_MODEL)
    description = models.CharField(max_length=256, null=True, blank=True)
    published = models.BooleanField(default=False)

    objects = managers.ScreenshotManager()

    def __unicode__(self):
        desc = self.description if self.description else self.game.name
        return "%s: %s (uploaded by %s)" % (self.game, desc, self.uploaded_by)

    def game_link(self):
        return u"{1}".format(
            reverse("admin:games_game_change", args=(self.game.id, )),
            self.game
        )
    game_link.allow_tags = True
    game_link.short_description = "Game (link)"

class InstallerManager(models.Manager):
    def published(self, user=None, is_staff=False):
        if is_staff:
            return self.get_query_set()
        elif user:
            return self.get_query_set().filter(models.Q(published=True)
                                               | models.Q(user=user))
        else:
            return self.get_query_set().filter(published=True)

    def fuzzy_get(self, slug):
        """Return either the installer that matches exactly 'slug' or the
        installers with game matching slug.
        Installers are always returned in a list.
        """
        try:
            installer = self.get_query_set().get(slug=slug)
            return [installer]
        except ObjectDoesNotExist:
            installers = self.get_query_set().filter(game__slug=slug,
                                                     published=True)
            if not installers:
                raise
            else:
                return installers

    def get_json(self, slug):
        try:
            installers = self.fuzzy_get(slug)
        except ObjectDoesNotExist:
            installer_data = []
        else:
            installer_data = [installer.as_dict() for installer in installers]
        try:
            game = Game.objects.get(slug=slug)
            installer_data += game.get_default_installers()
        except ObjectDoesNotExist:
            pass
        if not installer_data:
            raise Installer.DoesNotExist
        return json.dumps(installer_data)

class Installer(models.Model):
    """Game installer model"""
    game = models.ForeignKey(Game)
    user = models.ForeignKey(settings.AUTH_USER_MODEL)
    runner = models.ForeignKey(Runner)

    slug = models.SlugField(unique=True)
    version = models.CharField(max_length=32)
    description = models.CharField(max_length=512, blank=True, null=True)
    content = models.TextField()
    created_at = models.DateTimeField(auto_now_add=True, null=True)
    updated_at = models.DateTimeField(auto_now=True)
    published = models.BooleanField(default=False)
    objects = InstallerManager()

    def __unicode__(self):
        return self.slug

    def game_link(self):
        return u"{1}".format(
            reverse("admin:games_game_change", args=(self.game.id, )),
            self.game
        )
    game_link.allow_tags = True
    game_link.short_description = "Game (link)"

    def set_default_installer(self):
        if self.game and self.game.steam_support():
            installer_data = {'game': {'appid': self.game.steamid}}
            self.version = 'steam'
        else:
            installer_data = DEFAULT_INSTALLER
        self.content = yaml.safe_dump(installer_data, default_flow_style=False)

    def as_dict(self):
        yaml_content = yaml.safe_load(self.content) or {}
        yaml_content['version'] = self.version
        yaml_content['name'] = self.game.name
        yaml_content['year'] = self.game.year
        yaml_content['steamid'] = self.game.steamid
        try:
            yaml_content['runner'] = self.runner.slug
        except Runner.DoesNotExist:
            yaml_content['runner'] = ''
        yaml_content['slug'] = self.slug
        return yaml_content

    def as_yaml(self):
        return yaml.safe_dump(self.as_dict())

    def as_json(self):
        return json.dumps(self.as_dict())

    def build_slug(self, version):
        return "%s-%s" % (slugify(self.game.name)[:29],
                          slugify(version)[:20])

    def save(self, *args, **kwargs):
        self.slug = self.build_slug(self.version)
        return super(Installer, self).save(*args, **kwargs)

class GameLibrary(models.Model):
    user = models.OneToOneField(settings.AUTH_USER_MODEL)
    games = models.ManyToManyField(Game)

    # pylint: disable=W0232, R0903
    class Meta:
        verbose_name_plural = "game libraries"

    def __unicode__(self):
        return "%s's library" % self.user.username

class Featured(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey('content_type', 'object_id')
    image = models.ImageField(upload_to='featured', max_length=100)
    description = models.CharField(max_length=255, blank=True, null=True)
    created_at = models.DateTimeField(auto_now_add=True)

    # pylint: disable=W0232, R0903
    class Meta:
        verbose_name = "Featured content"

    def __unicode__(self):
        return "[%s] %s" % (self.content_type, str(self.content_object), )

class GameSubmission(models.Model):
    user = models.ForeignKey(settings.AUTH_USER_MODEL)
    game = models.ForeignKey(Game)
    created_at = models.DateTimeField(auto_now_add=True)
    accepted_at = models.DateTimeField(null=True)

    class Meta:
        verbose_name = "User submitted game"

    def __unicode__(self):
        return u"{0} submitted {1} on {2}".format(self.user, self.game,
                                                  self.created_at)

    def accept(self):
        self.accepted_at = datetime.datetime.now()
        self.save()
        subject = u"{} Your game submission for '{}' as been accepted!".format(
            settings.EMAIL_SUBJECT_PREFIX,
            self.game.name
        )
        body = u"""
Hello {0}!

Your submission for {1} has been reviewed by a moderator and approved!

The game's page is available at https://lutris.net{2}
You can submit an installer script for it if you haven't done so already. The
scripting details are explained on the installer submission page, you can have
a look at other scripts to see how they are written. If you get confused or
have any questions about the scripting process, please drop us a line at
admin@lutris.net or on IRC: #lutris on Freenode.

Have a great day!

The Lutris team
        """.format(
            self.user.username,
            self.game.name,
            self.game.get_absolute_url()
        )
        send_mail(subject, body, settings.DEFAULT_FROM_EMAIL, [self.user.email])
<|endoftext|>


<commit_before>from .struct import Struct
from .types import Array, Bytes, Int16, Int32, Schema, String

class JoinGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16),
        ('generation_id', Int32),
        ('group_protocol', String('utf-8')),
        ('leader_id', String('utf-8')),
        ('member_id', String('utf-8')),
        ('members', Array(
            ('member_id', String('utf-8')),
            ('member_metadata', Bytes)))
    )

class JoinGroupRequest(Struct):
    API_KEY = 11
    API_VERSION = 0
    RESPONSE_TYPE = JoinGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('session_timeout', Int32),
        ('member_id', String('utf-8')),
        ('protocol_type', String('utf-8')),
        ('group_protocols', Array(
            ('protocol_name', String('utf-8')),
            ('protocol_metadata', Bytes)))
    )

class ProtocolMetadata(Struct):
    SCHEMA = Schema(
        ('version', Int16),
        ('subscription', Array(String('utf-8'))), # topics list
        ('user_data', Bytes)
    )

class SyncGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16),
        ('member_assignment', Bytes)
    )

class SyncGroupRequest(Struct):
    API_KEY = 14
    API_VERSION = 0
    RESPONSE_TYPE = SyncGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('generation_id', Int32),
        ('member_id', String('utf-8')),
        ('group_assignment', Array(
            ('member_id', String('utf-8')),
            ('member_metadata', Bytes)))
    )

class MemberAssignment(Struct):
    SCHEMA = Schema(
        ('version', Int16),
        ('partition_assignment', Array(
            ('topic', String('utf-8')),
            ('partitions', Array(Int32)))),
        ('user_data', Bytes)
    )

class HeartbeatResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16)
    )

class HeartbeatRequest(Struct):
    API_KEY = 12
    API_VERSION = 0
    RESPONSE_TYPE = HeartbeatResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('generation_id', Int32),
        ('member_id', String('utf-8'))
    )

class LeaveGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16)
    )

class LeaveGroupRequest(Struct):
    API_KEY = 13
    API_VERSION = 0
    RESPONSE_TYPE = LeaveGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('member_id', String('utf-8'))
    )
<commit_msg>Add UNKNOWN_MEMBER_ID to JoinGroupRequest<commit_after>from .struct import Struct
from .types import Array, Bytes, Int16, Int32, Schema, String

class JoinGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16),
        ('generation_id', Int32),
        ('group_protocol', String('utf-8')),
        ('leader_id', String('utf-8')),
        ('member_id', String('utf-8')),
        ('members', Array(
            ('member_id', String('utf-8')),
            ('member_metadata', Bytes)))
    )

class JoinGroupRequest(Struct):
    API_KEY = 11
    API_VERSION = 0
    RESPONSE_TYPE = JoinGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('session_timeout', Int32),
        ('member_id', String('utf-8')),
        ('protocol_type', String('utf-8')),
        ('group_protocols', Array(
            ('protocol_name', String('utf-8')),
            ('protocol_metadata', Bytes)))
    )
    UNKNOWN_MEMBER_ID = ''

class ProtocolMetadata(Struct):
    SCHEMA = Schema(
        ('version', Int16),
        ('subscription', Array(String('utf-8'))), # topics list
        ('user_data', Bytes)
    )

class SyncGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16),
        ('member_assignment', Bytes)
    )

class SyncGroupRequest(Struct):
    API_KEY = 14
    API_VERSION = 0
    RESPONSE_TYPE = SyncGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('generation_id', Int32),
        ('member_id', String('utf-8')),
        ('group_assignment', Array(
            ('member_id', String('utf-8')),
            ('member_metadata', Bytes)))
    )

class MemberAssignment(Struct):
    SCHEMA = Schema(
        ('version', Int16),
        ('partition_assignment', Array(
            ('topic', String('utf-8')),
            ('partitions', Array(Int32)))),
        ('user_data', Bytes)
    )

class HeartbeatResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16)
    )

class HeartbeatRequest(Struct):
    API_KEY = 12
    API_VERSION = 0
    RESPONSE_TYPE = HeartbeatResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('generation_id', Int32),
        ('member_id', String('utf-8'))
    )

class LeaveGroupResponse(Struct):
    SCHEMA = Schema(
        ('error_code', Int16)
    )

class LeaveGroupRequest(Struct):
    API_KEY = 13
    API_VERSION = 0
    RESPONSE_TYPE = LeaveGroupResponse
    SCHEMA = Schema(
        ('group', String('utf-8')),
        ('member_id', String('utf-8'))
    )
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Define templates for RVS.'''

from typing import Dict, Sequence, Text

import nltk
from nltk import CFG, Production
from nltk.parse.generate import Nonterminal
import pandas as pd

from cabby.rvs import item

# Terminals in the grammar.
MAIN_NO_V = [
  "CARDINAL_DIRECTION from MAIN_PIVOT for NUMBER_INTERSECTIONS intersections",
  "CARDINAL_DIRECTION from MAIN_PIVOT for NUMBER_BLOCKS blocks",
  "NUMBER_BLOCKS blocks past MAIN_PIVOT",
  "NUMBER_INTERSECTIONS intersections past MAIN_PIVOT",
  "past MAIN_PIVOT",
  "CARDINAL_DIRECTION and past MAIN_PIVOT",
  "past MAIN_PIVOT and continue to the next intersection",
  "past MAIN_PIVOT and continue to the next block",
  "to MAIN_PIVOT and go CARDINAL_DIRECTION",
  "to MAIN_PIVOT and turn CARDINAL_DIRECTION"
  ""
]

MAIN = [
  ". When you pass MAIN_PIVOT, you'll be just NUMBER_INTERSECTIONS intersections away",
  ". When you pass MAIN_PIVOT, you'll be just NUMBER_BLOCKS blocks away",
  ". MAIN_PIVOT is NUMBER_INTERSECTIONS intersections away",
  ". MAIN_PIVOT is NUMBER_BLOCKS blocks away",
  ". Head to MAIN_PIVOT and go NUMBER_INTERSECTIONS intersections further",
  ". Go to MAIN_PIVOT and walk NUMBER_INTERSECTIONS intersections past it",
  ". Travel to MAIN_PIVOT and continue NUMBER_BLOCKS blocks further",
  ". Walk to MAIN_PIVOT and proceed NUMBER_BLOCKS blocks past it",
  ". After you reach MAIN_PIVOT, you'll need to go NUMBER_INTERSECTIONS intersections further",
  ". When you get to MAIN_PIVOT, you have NUMBER_INTERSECTIONS intersections more to walk",
  ". After you pass MAIN_PIVOT, go NUMBER_BLOCKS blocks more",
  ". Once you reach MAIN_PIVOT, continue for NUMBER_BLOCKS blocks"
]

V1 = ['Go', 'Walk', 'Head', 'Proceed', 'Travel']

V2 = [
  'Meet at GOAL_LOCATION.',
  'Come to GOAL_LOCATION.',
  'Head over to GOAL_LOCATION.',
  'GOAL_LOCATION is the meeting point.'
]

NEAR_GOAL_END = [
  ". GOAL_LOCATION will be near NEAR_PIVOT.",
  ". NEAR_PIVOT is quite close to GOAL_LOCATION.",
  ". Meet at GOAL_LOCATION, which is right next to NEAR_PIVOT.",
  ". If you see NEAR_PIVOT, you should find GOAL_LOCATION close by."
]
NEAR_GOAL_START = [
  ". It will be near NEAR_PIVOT.",
  ". It is close to NEAR_PIVOT.",
  ". NEAR_PIVOT is close by."
]
AVOID = [
  ". If you reach BEYOND_PIVOT, you have gone too far.",
  ". You've overshot the meeting point if you reach BEYOND_PIVOT.",
  ". If you pass BEYOND_PIVOT, you've gone too far.",
  ""]

GOAL_END = [
  'and meet at GOAL_LOCATION.',
  'and come to GOAL_LOCATION.',
  'to reach GOAL_LOCATION.',
  'to arrive at GOAL_LOCATION.'
]

GOAL = ["GOAL_LOCATION"]

def add_rules(nonterminal_name: Text,
        list_terminals: Sequence[Text]) -> Sequence[Production]:
  """Create the production rules for a givn nonterminal and a
   list of terminals corresponding to it.
  Args:
    nonterminal_name: The name of the nonterminal.
    list_terminals: The list of terminals that for each one a rule with
    the nonterminal will be produced.
  Returns:
    A sequence of productions rules.
  """
  prods = []
  for phrase in list_terminals:
    rule = Production(Nonterminal(nonterminal_name), (phrase,))
    prods.append(rule)
  return prods

def create_templates():
  """Creates the templates from the grammar."""

  prods = [
    # Specific verb with goal and the rest of instruction body.
    Production(Nonterminal('S'), (Nonterminal('V2'),
                    Nonterminal('V2_BODY'))),
    # A verb and rest of the instruction body assuming goal already mentioned.
    Production(Nonterminal('V2_BODY'), (Nonterminal('V1'),
                      Nonterminal('M_G_ALREADY_V'))),
    # A verb and the rest of the instruction body assuming the goal wasn't
    # mentioned before.
    Production(Nonterminal('S'), (Nonterminal(
      'V1'), Nonterminal('NO_GOAL'))),
    # The goal in the begining and the rest of the instruction body assuming
    # goal already mentioned.
    Production(Nonterminal('S'), (Nonterminal('V1_GOAL'),
                    Nonterminal('WITH_GOAL'))),
    # A verb and 'to the' and then goal mention and the rest of the instruction
    # body.
    Production(Nonterminal('V1_GOAL'), (Nonterminal('V1'),
                      Nonterminal('V1_CON'))),
    # A goal mention and the rest of the instruction body.
    Production(Nonterminal('WITH_GOAL'), (Nonterminal('GOAL'),
                        Nonterminal('M_G'))),
    # Main part of the instruction without verb in begining and resuming
    # sentence.
    Production(Nonterminal('M_G_ALREADY_V'), (Nonterminal('MAIN_NO_V'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # # Main part of the instruction, adding a new sentence.
    Production(Nonterminal('M_G'), (Nonterminal('MAIN'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # End part - (1) near pivot assuming goal already mentioned; and (2) avoid
    # sentence.
    Production(
      Nonterminal('END_NEAR_GOAL_KNOWN'), (Nonterminal('NEAR_GOAL_START'),
                    Nonterminal('AVOID'))),
    # End part - (1) near pivot assuming goal not mentioned yet; and (2) avoid
    # sentence.
    Production(Nonterminal(
      'END_NEAR_GOAL_KNOWN'), (Nonterminal('NEAR_GOAL_END'),
                    Nonterminal('AVOID'))),
    # Main part of the instruction without verb in begining and resuming
    # sentence assuming no goal mentioned before.
    Production(Nonterminal('NO_GOAL'), (Nonterminal('MAIN_NO_V'),
                      Nonterminal('END_NEAR_GOAL_UNKNOWN'))),
    # Add Goal to main part and then resume instruction by adding an
    # ending(near+avoid).
    Production(Nonterminal('END_NEAR_GOAL_UNKNOWN'), (Nonterminal('GOAL_END'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # Add Goal with near and then add an avoid sentenece.
    Production(
      Nonterminal('END_NEAR_GOAL_UNKNOWN'), (Nonterminal('NEAR_GOAL_END'),
                    Nonterminal('AVOID'))),
    # Termial for IN+DT after verb.
    Production(Nonterminal('V1_CON'), ('to the',)),

  ]

  prods += add_rules('V2', V2)
  prods += add_rules('AVOID', AVOID)
  prods += add_rules('NEAR_GOAL_START', NEAR_GOAL_START)
  prods += add_rules('NEAR_GOAL_END', NEAR_GOAL_END)
  prods += add_rules('GOAL', GOAL)
  prods += add_rules('GOAL_END', GOAL_END)
  prods += add_rules('MAIN_NO_V', MAIN_NO_V)
  prods += add_rules('MAIN', MAIN)
  prods += add_rules('V1', V1)

  grammar = CFG(Nonterminal('S'), prods)

  # Generate templates.
  templates = []
  for sentence in nltk.parse.generate.generate(grammar):

    sentence = ' '.join(sentence)

    if sentence[-1] != '.':
      sentence += '.'
    sentence = sentence.replace(" .", ".")
    sentence = sentence.replace(" ,", ",")
    sentence = sentence.replace("..", ".")
    sentence = sentence.replace("   ", " ")
    sentence = sentence.replace("  ", " ")
    templates.append(sentence)

  templates_df = pd.DataFrame(
    templates, columns=['sentence']).drop_duplicates()
  # Save templates
  templates_df.to_csv('templates.csv', index=False, header = False)

  # Flag features.
  templates_df['cardinal_direction'] = templates_df['sentence'].apply(
    lambda x: 'CARDINAL_DIRECTION' in x)
  templates_df['blocks'] = templates_df['sentence'].apply(
    lambda x: 'NUMBER_BLOCKS' in x)
  templates_df['intersections'] = templates_df['sentence'].apply(
    lambda x: 'NUMBER_INTERSECTIONS' in x)
  templates_df['beyond_pivot'] = templates_df['sentence'].apply(
    lambda x: 'BEYOND_PIVOT' in x)
  templates_df['next_block'] = templates_df['sentence'].apply(
    lambda x: 'next block' in x)
  templates_df['next_intersection'] = templates_df['sentence'].apply(
    lambda x: 'next intersection' in x)

  return templates_df

def add_features_to_template(template: Text, entity: item.RVSPath) -> Text:
  '''Add the entity features to the picked template to create an instruction:
    template: The choosen template.
    entity: The features of the path to add to the template.
  '''
  intersections = int(entity.intersections)
  blocks = str(intersections-1)
  intersections = str(intersections)
  template = template.replace('GOAL_LOCATION', entity.end_point.main_tag)
  template = template.replace('MAIN_PIVOT', entity.main_pivot.main_tag)
  template = template.replace('NEAR_PIVOT', entity.near_pivot.main_tag)
  template = template.replace('BEYOND_PIVOT', entity.beyond_pivot.main_tag)
  template = template.replace('NUMBER_INTERSECTIONS', intersections)
  template = template.replace('NUMBER_BLOCKS', blocks)
  template = template.replace(
    'CARDINAL_DIRECTION', entity.cardinal_direction)

  # Fix text.
  template = template.replace('The The', 'The')
  template = template.replace('the The', 'the')

  return template.capitalize()<commit_msg>fixed The and A for name entities<commit_after># coding=utf-8
# Copyright 2020 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Define templates for RVS.'''

from typing import Dict, Sequence, Text

import nltk
from nltk import CFG, Production
from nltk.parse.generate import Nonterminal
import pandas as pd

from cabby.rvs import item

# Terminals in the grammar.
MAIN_NO_V = [
  "CARDINAL_DIRECTION from MAIN_PIVOT for NUMBER_INTERSECTIONS intersections",
  "CARDINAL_DIRECTION from MAIN_PIVOT for NUMBER_BLOCKS blocks",
  "NUMBER_BLOCKS blocks past MAIN_PIVOT",
  "NUMBER_INTERSECTIONS intersections past MAIN_PIVOT",
  "past MAIN_PIVOT",
  "CARDINAL_DIRECTION and past MAIN_PIVOT",
  "past MAIN_PIVOT and continue to the next intersection",
  "past MAIN_PIVOT and continue to the next block",
  "to MAIN_PIVOT and go CARDINAL_DIRECTION",
  "to MAIN_PIVOT and turn CARDINAL_DIRECTION"
  ""
]

MAIN = [
  ". When you pass MAIN_PIVOT, you'll be just NUMBER_INTERSECTIONS intersections away",
  ". When you pass MAIN_PIVOT, you'll be just NUMBER_BLOCKS blocks away",
  ". MAIN_PIVOT is NUMBER_INTERSECTIONS intersections away",
  ". MAIN_PIVOT is NUMBER_BLOCKS blocks away",
  ". Head to MAIN_PIVOT and go NUMBER_INTERSECTIONS intersections further",
  ". Go to MAIN_PIVOT and walk NUMBER_INTERSECTIONS intersections past it",
  ". Travel to MAIN_PIVOT and continue NUMBER_BLOCKS blocks further",
  ". Walk to MAIN_PIVOT and proceed NUMBER_BLOCKS blocks past it",
  ". After you reach MAIN_PIVOT, you'll need to go NUMBER_INTERSECTIONS intersections further",
  ". When you get to MAIN_PIVOT, you have NUMBER_INTERSECTIONS intersections more to walk",
  ". After you pass MAIN_PIVOT, go NUMBER_BLOCKS blocks more",
  ". Once you reach MAIN_PIVOT, continue for NUMBER_BLOCKS blocks"
]

V1 = ['Go', 'Walk', 'Head', 'Proceed', 'Travel']

V2 = [
  'Meet at the GOAL_LOCATION.',
  'Come to the GOAL_LOCATION.',
  'Head over to the GOAL_LOCATION.',
  'The GOAL_LOCATION is the meeting point.'
]

NEAR_GOAL_END = [
  ". The GOAL_LOCATION will be near a NEAR_PIVOT.",
  ". A NEAR_PIVOT is quite close to the GOAL_LOCATION.",
  ". Meet at the GOAL_LOCATION, which is right next to a NEAR_PIVOT.",
  ". If you see a NEAR_PIVOT, you should find the GOAL_LOCATION close by."
]
NEAR_GOAL_START = [
  ". It will be near a NEAR_PIVOT.",
  ". It is close to a NEAR_PIVOT.",
  ". A NEAR_PIVOT is close by."
]
AVOID = [
  ". If you reach BEYOND_PIVOT, you have gone too far.",
  ". You've overshot the meeting point if you reach BEYOND_PIVOT.",
  ". If you pass BEYOND_PIVOT, you've gone too far.",
  ""]

GOAL_END = [
  'and meet at the GOAL_LOCATION.',
  'and come to the GOAL_LOCATION.',
  'to reach the GOAL_LOCATION.',
  'to arrive at the GOAL_LOCATION.'
]

GOAL = ["the GOAL_LOCATION"]

def add_rules(nonterminal_name: Text,
        list_terminals: Sequence[Text]) -> Sequence[Production]:
  """Create the production rules for a givn nonterminal and a
   list of terminals corresponding to it.
  Args:
    nonterminal_name: The name of the nonterminal.
    list_terminals: The list of terminals that for each one a rule with
    the nonterminal will be produced.
  Returns:
    A sequence of productions rules.
  """
  prods = []
  for phrase in list_terminals:
    rule = Production(Nonterminal(nonterminal_name), (phrase,))
    prods.append(rule)
  return prods

def create_templates():
  """Creates the templates from the grammar."""

  prods = [
    # Specific verb with goal and the rest of instruction body.
    Production(Nonterminal('S'), (Nonterminal('V2'),
                    Nonterminal('V2_BODY'))),
    # A verb and rest of the instruction body assuming goal already mentioned.
    Production(Nonterminal('V2_BODY'), (Nonterminal('V1'),
                      Nonterminal('M_G_ALREADY_V'))),
    # A verb and the rest of the instruction body assuming the goal wasn't
    # mentioned before.
    Production(Nonterminal('S'), (Nonterminal(
      'V1'), Nonterminal('NO_GOAL'))),
    # The goal in the begining and the rest of the instruction body assuming
    # goal already mentioned.
    Production(Nonterminal('S'), (Nonterminal('V1_GOAL'),
                    Nonterminal('WITH_GOAL'))),
    # A verb and 'to the' and then goal mention and the rest of the instruction
    # body.
    Production(Nonterminal('V1_GOAL'), (Nonterminal('V1'),
                      Nonterminal('V1_CON'))),
    # A goal mention and the rest of the instruction body.
    Production(Nonterminal('WITH_GOAL'), (Nonterminal('GOAL'),
                        Nonterminal('M_G'))),
    # Main part of the instruction without verb in begining and resuming
    # sentence.
    Production(Nonterminal('M_G_ALREADY_V'), (Nonterminal('MAIN_NO_V'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # # Main part of the instruction, adding a new sentence.
    Production(Nonterminal('M_G'), (Nonterminal('MAIN'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # End part - (1) near pivot assuming goal already mentioned; and (2) avoid
    # sentence.
    Production(
      Nonterminal('END_NEAR_GOAL_KNOWN'), (Nonterminal('NEAR_GOAL_START'),
                    Nonterminal('AVOID'))),
    # End part - (1) near pivot assuming goal not mentioned yet; and (2) avoid
    # sentence.
    Production(Nonterminal(
      'END_NEAR_GOAL_KNOWN'), (Nonterminal('NEAR_GOAL_END'),
                    Nonterminal('AVOID'))),
    # Main part of the instruction without verb in begining and resuming
    # sentence assuming no goal mentioned before.
    Production(Nonterminal('NO_GOAL'), (Nonterminal('MAIN_NO_V'),
                      Nonterminal('END_NEAR_GOAL_UNKNOWN'))),
    # Add Goal to main part and then resume instruction by adding an
    # ending(near+avoid).
    Production(Nonterminal('END_NEAR_GOAL_UNKNOWN'), (Nonterminal('GOAL_END'),
                    Nonterminal('END_NEAR_GOAL_KNOWN'))),
    # Add Goal with near and then add an avoid sentenece.
    Production(
      Nonterminal('END_NEAR_GOAL_UNKNOWN'), (Nonterminal('NEAR_GOAL_END'),
                    Nonterminal('AVOID'))),
    # Termial for IN+DT after verb.
    Production(Nonterminal('V1_CON'), ('to the',)),

  ]

  prods += add_rules('V2', V2)
  prods += add_rules('AVOID', AVOID)
  prods += add_rules('NEAR_GOAL_START', NEAR_GOAL_START)
  prods += add_rules('NEAR_GOAL_END', NEAR_GOAL_END)
  prods += add_rules('GOAL', GOAL)
  prods += add_rules('GOAL_END', GOAL_END)
  prods += add_rules('MAIN_NO_V', MAIN_NO_V)
  prods += add_rules('MAIN', MAIN)
  prods += add_rules('V1', V1)

  grammar = CFG(Nonterminal('S'), prods)

  # Generate templates.
  templates = []
  for sentence in nltk.parse.generate.generate(grammar):

    sentence = ' '.join(sentence)

    if sentence[-1] != '.':
      sentence += '.'
    sentence = sentence.replace(" .", ".")
    sentence = sentence.replace(" ,", ",")
    sentence = sentence.replace("..", ".")
    sentence = sentence.replace("   ", " ")
    sentence = sentence.replace("  ", " ")
    templates.append(sentence)

  templates_df = pd.DataFrame(
    templates, columns=['sentence']).drop_duplicates()
  # Save templates
  templates_df.to_csv('templates.csv', index=False, header = False)

  # Flag features.
  templates_df['cardinal_direction'] = templates_df['sentence'].apply(
    lambda x: 'CARDINAL_DIRECTION' in x)
  templates_df['blocks'] = templates_df['sentence'].apply(
    lambda x: 'NUMBER_BLOCKS' in x)
  templates_df['intersections'] = templates_df['sentence'].apply(
    lambda x: 'NUMBER_INTERSECTIONS' in x)
  templates_df['beyond_pivot'] = templates_df['sentence'].apply(
    lambda x: 'BEYOND_PIVOT' in x)
  templates_df['next_block'] = templates_df['sentence'].apply(
    lambda x: 'next block' in x)
  templates_df['next_intersection'] = templates_df['sentence'].apply(
    lambda x: 'next intersection' in x)

  return templates_df

def add_features_to_template(template: Text, entity: item.RVSPath) -> Text:
  '''Add the entity features to the picked template to create an instruction:
    template: The choosen template.
    entity: The features of the path to add to the template.
  '''
  intersections = int(entity.intersections)
  blocks = str(intersections-1)
  intersections = str(intersections)
  if entity.end_point.main_tag[0].isupper():
    template = template.replace('The GOAL_LOCATION', 'GOAL_LOCATION')
    template = template.replace('the GOAL_LOCATION', 'GOAL_LOCATION')
  if entity.near_pivot.main_tag[0].isupper():
    template = template.replace('A NEAR_PIVOT', 'NEAR_PIVOT')
    template = template.replace('a NEAR_PIVOT', 'NEAR_PIVOT')
  template = template.replace('GOAL_LOCATION', entity.end_point.main_tag)
  template = template.replace('MAIN_PIVOT', entity.main_pivot.main_tag)
  template = template.replace('NEAR_PIVOT', entity.near_pivot.main_tag)
  template = template.replace('BEYOND_PIVOT', entity.beyond_pivot.main_tag)
  template = template.replace('NUMBER_INTERSECTIONS', intersections)
  template = template.replace('NUMBER_BLOCKS', blocks)
  template = template.replace(
    'CARDINAL_DIRECTION', entity.cardinal_direction)

  # Fix text.
  template = template.replace('The The', 'The')
  template = template.replace('the The', 'the')

  return template<|endoftext|>