<commit_before>#! /usr/bin/env python3.6
# coding: utf-8
from kamera.logger import log

from pathlib import Path
import datetime as dt
from functools import partial
from io import BytesIO
import re

import pytz
from timezonefinderL import TimezoneFinder
import dropbox
import requests
import redis
from PIL import Image
import imagehash

from kamera import config
from kamera import image_processing

from typing import Callable, Optional, Dict

seconds_in_fortnight = int(dt.timedelta(weeks=1).total_seconds())

class FoundBetterDuplicateException(Exception):
    pass

class Task:
    dbx_cache: Dict[str, dropbox.Dropbox] = {}
    settings_cache: Dict[str, config.Settings] = {}
    redis_client: redis.Redis = None

    def __init__(
        self,
        account_id: str,
        entry: dropbox.files.FileMetadata,
        metadata: Optional[dropbox.files.PhotoMetadata],
        review_dir: Path,
        backup_dir: Path,
        error_dir: Path,
    ) -> None:
        self.account_id: str = account_id
        self.path: Path = Path(entry.path_display)
        self.name: str = self.path.name
        self.client_modified: dt.datetime = entry.client_modified
        self.time_taken: Optional[dt.datetime] = None
        self.dimensions: Optional[dropbox.files.Dimensions] = None
        self.coordinates: Optional[dropbox.files.GpsCoordinates] = None
        if metadata is not None:
            self.time_taken = metadata.time_taken
            if metadata.dimensions is not None:
                self.dimensions = metadata.dimensions
            if metadata.location is not None:
                self.coordinates = metadata.location

        self.review_dir: Path = review_dir
        self.backup_dir: Path = backup_dir
        self.error_dir: Path = error_dir

    def __repr__(self):
        return repr(self.name)

    @classmethod
    def connect_redis(cls, account_id: str) -> None:
        log.debug(cls.dbx_cache)
        log.debug(cls.settings_cache)
        if cls.redis_client is None:
            cls.redis_client = redis.from_url(config.redis_url)

    @classmethod
    def load_dbx_from_cache(cls, account_id: str) -> dropbox.Dropbox:
        try:
            dbx = cls.dbx_cache[account_id]
        except KeyError:
            dbx = dropbox.Dropbox(config.get_dbx_token(cls.redis_client, account_id))
            cls.dbx_cache[account_id] = dbx
        log.debug(cls.dbx_cache)
        return dbx

    @classmethod
    def load_settings_from_cache(
        cls, account_id: str, dbx: dropbox.Dropbox
    ) -> config.Settings:
        try:
            settings = cls.settings_cache[account_id]
            log.debug("Settings loaded from cache")
        except KeyError:
            settings = config.Settings(dbx)
            cls.settings_cache[account_id] = settings
            log.debug("Settings loaded from dbx")
        log.debug(cls.settings_cache)
        return settings

    def process_entry(self) -> None:
        start_time = dt.datetime.now()
        log.info(f"{self.name}: Processing")
        Task.connect_redis(self.account_id)
        dbx = Task.load_dbx_from_cache(self.account_id)
        settings = Task.load_settings_from_cache(self.account_id, dbx)
        try:
            date = parse_date(
                self.time_taken,
                self.client_modified,
                self.coordinates,
                settings.default_tz,
            )
            out_name = get_out_name(self.path.stem, self.path.suffix, date)
            subfolder = Path(str(date.year), settings.folder_names[date.month])
            review_path = self.review_dir / subfolder / out_name
            backup_path = self.backup_dir / subfolder / out_name

            if self.path.suffix.lower() in config.video_extensions:
                copy_entry(dbx, self.path, review_path)
                move_entry(dbx, self.path, backup_path)
                return

            elif self.path.suffix.lower() not in config.image_extensions:
                return

            _, response = download_entry(dbx, self.path.as_posix())
            in_data = response.raw.data
            img_hash = get_hash(data=in_data)
            handle_duplication(
                account_id_and_img_hash=f"user:{self.account_id}, hash:{img_hash}",
                file_path=review_path,
                dbx=dbx,
                redis_client=self.redis_client,
                dimensions=self.dimensions,
            )

            new_data = image_processing.main(
                data=in_data,
                filepath=self.path,
                date=date,
                settings=settings,
                coordinates=self.coordinates,
                dimensions=self.dimensions,
            )

            if new_data is None:
                copy_entry(dbx, self.path, review_path)
            else:
                upload_entry(dbx, new_data, review_path)

            move_entry(dbx, self.path, backup_path)
        except FoundBetterDuplicateException:
            log.info(f"{self.name}: Found better duplicate, finishing")
            move_entry(dbx, self.path, backup_path)
        except Exception:
            log.exception(f"Exception occured, moving to Error subfolder: {self.name}")
            move_entry(dbx, self.path, (self.error_dir / self.name))
        finally:
            end_time = dt.datetime.now()
            duration = end_time - start_time
            log.info(f"{self.name}, duration: {duration}")
            log.info("\n")

def handle_duplication(
    account_id_and_img_hash: str,
    file_path: Path,
    dbx: dropbox.Dropbox,
    redis_client: redis.Redis,
    dimensions: dropbox.files.Dimensions,
) -> None:
    dup_file_path = redis_client.get(account_id_and_img_hash)
    # need to check for duplicate before storing the hash
    store_hash(account_id_and_img_hash, file_path, redis_client)
    if dup_file_path is None:
        return
    dup_file_path = dup_file_path.decode()

    try:
        dup_entry = dbx.files_get_metadata(dup_file_path, include_media_info=True)
    except dropbox.exceptions.ApiError:
        delete_hash(account_id_and_img_hash, redis_client)
        return
    dup_metadata = dup_entry.media_info.get_metadata() if dup_entry.media_info else None
    try:
        duplicate_better = (
            dup_metadata.dimensions.height * dup_metadata.dimensions.width
        ) > (dimensions.height * dimensions.width)
    except AttributeError:
        duplicate_better = (
            dup_metadata is not None and dup_metadata.dimensions is not None
        ) and (dimensions is None)
    if duplicate_better:
        raise FoundBetterDuplicateException
    else:
        delete_entry(dup_entry, dbx)
        delete_hash(account_id_and_img_hash, redis_client)

def get_hash(data: bytes) -> str:
    img = Image.open(BytesIO(data))
    img_hash = imagehash.whash(img)
    return img_hash

def store_hash(
    account_id_and_img_hash: str, file_path: Path, redis_client: redis.Redis
) -> None:
    redis_client.set(account_id_and_img_hash, file_path.as_posix())
    redis_client.expire(account_id_and_img_hash, seconds_in_fortnight)

def delete_hash(account_id_and_img_hash: str, redis_client: redis.Redis) -> None:
    redis_client.delete(account_id_and_img_hash)

def delete_entry(entry: dropbox.files.FileMetadata, dbx: dropbox.Dropbox) -> None:
    dbx.files_delete(entry.path_display)

def _execute_transfer(
    dbx: dropbox.Dropbox, transfer_func: Callable, destination_folder: Path
) -> None:
    try:
        transfer_func()
    except requests.exceptions.SSLError:
        log.info("Encountered SSL error during transfer. Trying again")
        transfer_func()
    except dropbox.exceptions.BadInputError:
        log.info(f"Making folder: {destination_folder}")
        dbx.files_create_folder(destination_folder.as_posix())
        transfer_func()

def move_entry(dbx: dropbox.Dropbox, from_path: Path, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_move,
        from_path=from_path.as_posix(),
        to_path=to_path.as_posix(),
        autorename=True,
    )
    log.info(f"{from_path.name}: Moving to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def copy_entry(dbx: dropbox.Dropbox, from_path: Path, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_copy,
        from_path=from_path.as_posix(),
        to_path=to_path.as_posix(),
        autorename=True,
    )
    log.info(f"{from_path.name}: Copying to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def upload_entry(dbx: dropbox.Dropbox, new_data: bytes, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_upload, f=new_data, path=to_path.as_posix(), autorename=True
    )
    log.info(f"{to_path.name}: Uploading to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def download_entry(dbx, path_str: str):
    try:
        return dbx.files_download(path_str)
    except requests.exceptions.SSLError:
        log.info("Encountered SSL error during transfer. Trying again")
        return dbx.files_download(path_str)

def parse_date(
    time_taken: Optional[dt.datetime],
    client_modified: dt.datetime,
    coordinates: Optional[dropbox.files.GpsCoordinates],
    default_tz: str,
) -> dt.datetime:
    naive_date = time_taken if time_taken is not None else client_modified
    utc_date = naive_date.replace(tzinfo=dt.timezone.utc)
    if coordinates is not None:
        img_tz = TimezoneFinder().timezone_at(
            lat=coordinates.latitude, lng=coordinates.longitude
        )
        if img_tz:
            local_date = utc_date.astimezone(tz=pytz.timezone(img_tz))
            return local_date
    local_date = utc_date.astimezone(tz=pytz.timezone(default_tz))
    return local_date

def get_out_name(stem: str, suffix: str, date: dt.datetime) -> str:
    ymd = date.strftime("%Y%m%d")
    hms = date.strftime("%H%M%S")
    if ymd in stem:
        out_name = re.sub(rf"(.*)_+({ymd}.*)", r"\2_\1", stem)
    else:
        out_name = f"{ymd}_{hms}_{stem}"
    out_suffix = ".jpg" if suffix == ".png" else suffix
    return out_name + out_suffix
<commit_msg>Resize before hashing for duplicate check<commit_after>#! /usr/bin/env python3.6
# coding: utf-8
from kamera.logger import log

from pathlib import Path
import datetime as dt
from functools import partial
from io import BytesIO
import re

import pytz
from timezonefinderL import TimezoneFinder
import dropbox
import requests
import redis
from PIL import Image
import imagehash
from resizeimage import resizeimage

from kamera import config
from kamera import image_processing

from typing import Callable, Optional, Dict

seconds_in_fortnight = int(dt.timedelta(weeks=1).total_seconds())

class FoundBetterDuplicateException(Exception):
    pass

class Task:
    dbx_cache: Dict[str, dropbox.Dropbox] = {}
    settings_cache: Dict[str, config.Settings] = {}
    redis_client: redis.Redis = None

    def __init__(
        self,
        account_id: str,
        entry: dropbox.files.FileMetadata,
        metadata: Optional[dropbox.files.PhotoMetadata],
        review_dir: Path,
        backup_dir: Path,
        error_dir: Path,
    ) -> None:
        self.account_id: str = account_id
        self.path: Path = Path(entry.path_display)
        self.name: str = self.path.name
        self.client_modified: dt.datetime = entry.client_modified
        self.time_taken: Optional[dt.datetime] = None
        self.dimensions: Optional[dropbox.files.Dimensions] = None
        self.coordinates: Optional[dropbox.files.GpsCoordinates] = None
        if metadata is not None:
            self.time_taken = metadata.time_taken
            if metadata.dimensions is not None:
                self.dimensions = metadata.dimensions
            if metadata.location is not None:
                self.coordinates = metadata.location

        self.review_dir: Path = review_dir
        self.backup_dir: Path = backup_dir
        self.error_dir: Path = error_dir

    def __repr__(self):
        return repr(self.name)

    @classmethod
    def connect_redis(cls, account_id: str) -> None:
        log.debug(cls.dbx_cache)
        log.debug(cls.settings_cache)
        if cls.redis_client is None:
            cls.redis_client = redis.from_url(config.redis_url)

    @classmethod
    def load_dbx_from_cache(cls, account_id: str) -> dropbox.Dropbox:
        try:
            dbx = cls.dbx_cache[account_id]
        except KeyError:
            dbx = dropbox.Dropbox(config.get_dbx_token(cls.redis_client, account_id))
            cls.dbx_cache[account_id] = dbx
        log.debug(cls.dbx_cache)
        return dbx

    @classmethod
    def load_settings_from_cache(
        cls, account_id: str, dbx: dropbox.Dropbox
    ) -> config.Settings:
        try:
            settings = cls.settings_cache[account_id]
            log.debug("Settings loaded from cache")
        except KeyError:
            settings = config.Settings(dbx)
            cls.settings_cache[account_id] = settings
            log.debug("Settings loaded from dbx")
        log.debug(cls.settings_cache)
        return settings

    def process_entry(self) -> None:
        start_time = dt.datetime.now()
        log.info(f"{self.name}: Processing")
        Task.connect_redis(self.account_id)
        dbx = Task.load_dbx_from_cache(self.account_id)
        settings = Task.load_settings_from_cache(self.account_id, dbx)
        try:
            date = parse_date(
                self.time_taken,
                self.client_modified,
                self.coordinates,
                settings.default_tz,
            )
            out_name = get_out_name(self.path.stem, self.path.suffix, date)
            subfolder = Path(str(date.year), settings.folder_names[date.month])
            review_path = self.review_dir / subfolder / out_name
            backup_path = self.backup_dir / subfolder / out_name

            if self.path.suffix.lower() in config.video_extensions:
                copy_entry(dbx, self.path, review_path)
                move_entry(dbx, self.path, backup_path)
                return

            elif self.path.suffix.lower() not in config.image_extensions:
                return

            _, response = download_entry(dbx, self.path.as_posix())
            in_data = response.raw.data
            img_hash = get_hash(data=in_data)
            handle_duplication(
                account_id_and_img_hash=f"user:{self.account_id}, hash:{img_hash}",
                file_path=review_path,
                dbx=dbx,
                redis_client=self.redis_client,
                dimensions=self.dimensions,
            )

            new_data = image_processing.main(
                data=in_data,
                filepath=self.path,
                date=date,
                settings=settings,
                coordinates=self.coordinates,
                dimensions=self.dimensions,
            )

            if new_data is None:
                copy_entry(dbx, self.path, review_path)
            else:
                upload_entry(dbx, new_data, review_path)

            move_entry(dbx, self.path, backup_path)
        except FoundBetterDuplicateException:
            log.info(f"{self.name}: Found better duplicate, finishing")
            move_entry(dbx, self.path, backup_path)
        except Exception:
            log.exception(f"Exception occured, moving to Error subfolder: {self.name}")
            move_entry(dbx, self.path, (self.error_dir / self.name))
        finally:
            end_time = dt.datetime.now()
            duration = end_time - start_time
            log.info(f"{self.name}, duration: {duration}")
            log.info("\n")

def handle_duplication(
    account_id_and_img_hash: str,
    file_path: Path,
    dbx: dropbox.Dropbox,
    redis_client: redis.Redis,
    dimensions: dropbox.files.Dimensions,
) -> None:
    dup_file_path = redis_client.get(account_id_and_img_hash)
    # need to check for duplicate before storing the hash
    store_hash(account_id_and_img_hash, file_path, redis_client)
    if dup_file_path is None:
        return
    dup_file_path = dup_file_path.decode()

    try:
        dup_entry = dbx.files_get_metadata(dup_file_path, include_media_info=True)
    except dropbox.exceptions.ApiError:
        log.info("Duplicate hash found, but image not in dbx")
        delete_hash(account_id_and_img_hash, redis_client)
        return
    dup_metadata = dup_entry.media_info.get_metadata() if dup_entry.media_info else None
    try:
        duplicate_better = (
            dup_metadata.dimensions.height * dup_metadata.dimensions.width
        ) > (dimensions.height * dimensions.width)
    except AttributeError:
        duplicate_better = (
            dup_metadata is not None and dup_metadata.dimensions is not None
        ) and (dimensions is None)
    if duplicate_better:
        raise FoundBetterDuplicateException
    else:
        log.info(f"{self.name}: Found worse duplicate, deleting: {dup_entry.path_display}")
        delete_entry(dup_entry, dbx)
        delete_hash(account_id_and_img_hash, redis_client)

def get_hash(data: bytes) -> str:
    img = Image.open(BytesIO(data))
    if img.height > 500:
        small_img = resizeimage.resize_height(img, size=500)
    else:
        small_img = img
    img_hash = imagehash.whash(small_img)
    return img_hash

def store_hash(
    account_id_and_img_hash: str, file_path: Path, redis_client: redis.Redis
) -> None:
    redis_client.set(account_id_and_img_hash, file_path.as_posix())
    redis_client.expire(account_id_and_img_hash, seconds_in_fortnight)

def delete_hash(account_id_and_img_hash: str, redis_client: redis.Redis) -> None:
    redis_client.delete(account_id_and_img_hash)

def delete_entry(entry: dropbox.files.FileMetadata, dbx: dropbox.Dropbox) -> None:
    dbx.files_delete(entry.path_display)

def _execute_transfer(
    dbx: dropbox.Dropbox, transfer_func: Callable, destination_folder: Path
) -> None:
    try:
        transfer_func()
    except requests.exceptions.SSLError:
        log.info("Encountered SSL error during transfer. Trying again")
        transfer_func()
    except dropbox.exceptions.BadInputError:
        log.info(f"Making folder: {destination_folder}")
        dbx.files_create_folder(destination_folder.as_posix())
        transfer_func()

def move_entry(dbx: dropbox.Dropbox, from_path: Path, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_move,
        from_path=from_path.as_posix(),
        to_path=to_path.as_posix(),
        autorename=True,
    )
    log.info(f"{from_path.name}: Moving to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def copy_entry(dbx: dropbox.Dropbox, from_path: Path, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_copy,
        from_path=from_path.as_posix(),
        to_path=to_path.as_posix(),
        autorename=True,
    )
    log.info(f"{from_path.name}: Copying to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def upload_entry(dbx: dropbox.Dropbox, new_data: bytes, to_path: Path) -> None:
    transfer_func = partial(
        dbx.files_upload, f=new_data, path=to_path.as_posix(), autorename=True
    )
    log.info(f"{to_path.name}: Uploading to dest: {to_path.as_posix()}")
    _execute_transfer(dbx, transfer_func, to_path.parent)

def download_entry(dbx, path_str: str):
    try:
        return dbx.files_download(path_str)
    except requests.exceptions.SSLError:
        log.info("Encountered SSL error during transfer. Trying again")
        return dbx.files_download(path_str)

def parse_date(
    time_taken: Optional[dt.datetime],
    client_modified: dt.datetime,
    coordinates: Optional[dropbox.files.GpsCoordinates],
    default_tz: str,
) -> dt.datetime:
    naive_date = time_taken if time_taken is not None else client_modified
    utc_date = naive_date.replace(tzinfo=dt.timezone.utc)
    if coordinates is not None:
        img_tz = TimezoneFinder().timezone_at(
            lat=coordinates.latitude, lng=coordinates.longitude
        )
        if img_tz:
            local_date = utc_date.astimezone(tz=pytz.timezone(img_tz))
            return local_date
    local_date = utc_date.astimezone(tz=pytz.timezone(default_tz))
    return local_date

def get_out_name(stem: str, suffix: str, date: dt.datetime) -> str:
    ymd = date.strftime("%Y%m%d")
    hms = date.strftime("%H%M%S")
    if ymd in stem:
        out_name = re.sub(rf"(.*)_+({ymd}.*)", r"\2_\1", stem)
    else:
        out_name = f"{ymd}_{hms}_{stem}"
    out_suffix = ".jpg" if suffix == ".png" else suffix
    return out_name + out_suffix
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs
import os.path
import re
import sys

from cStringIO import StringIO
from pkg_resources import resource_listdir, resource_string
import html2rest

from trac.loader import load_components
from trac.test import EnvironmentStub, Mock, MockPerm
from trac.util.text import printout
from trac.web.chrome import web_context
from trac.web.href import Href
from trac.wiki.formatter import format_to_html
from trac.wiki.model import WikiPage

class Parser(html2rest.Parser):

    def __init__(self, writer=sys.stdout, encoding='utf8', relroot=None,
                 relpath=None):
        html2rest.Parser.__init__(self, writer, encoding, relroot, relpath)
        self.links = {}

    def end_a(self):
        if '#pending' in self.hrefs:
            href = self.hrefs['#pending']
            label = self.hrefs[href]
            key = label.lower()
            if key not in self.links:
                self.links[key] = (label, href)
            elif href != self.links[key][1]:
                alt = label
                while True:
                    alt += '*'
                    if alt not in self.links:
                        break
                    continue
                self.data(alt[len(label):])
                self.hrefs[href] = alt
                self.links[alt] = (alt, href)
            self.data('`_')
            del self.hrefs['#pending']

    def end_body(self):
        self.end_p()
        for label, href in self.links.itervalues():
            if href[0] != '#':
                self.writeline('.. _%s: %s' % (label, href))
        self.end_p()

def wiki2rest(env, context, wiki):
    text = re.sub('\r?\n', '\n', wiki.text)
    text = re.sub(r'\[\[TracGuideToc\]\]\r?\n?', '', text)
    html = format_to_html(env, context, text)
    html = html.replace(u'\u200b', '')
    html = re.sub(r'\s*([^<]*?)\s*', r'\1', html)
    html = '%s' % html
    writer = StringIO()
    parser = Parser(writer, 'utf-8', None, None)
    parser.feed(html)
    parser.close()
    rst = writer.getvalue().strip('\n')
    rst = re.sub('\n{4,}', '\n\n\n', rst)
    # sort links
    rst = re.sub(r'(?:\n\.\. _[^\n]*)+\Z',
                 lambda m: '\n'.join(sorted(m.group(0).split('\n'),
                                            key=lambda v: v.lower())),
                 rst)
    if any(ord(c) > 0x7f for c in rst):
        # Trac detects utf-8 using BOM
        rst = '%s.. charset=utf-8\n\n%s' % (codecs.BOM_UTF8, rst)
    return rst + '\n'

def main():
    names = sorted(name for name in resource_listdir('trac.wiki',
                                                     'default-pages')
                        if not name.startswith('.'))

    env = EnvironmentStub()
    load_components(env)
    with env.db_transaction:
        for name in names:
            wiki = WikiPage(env, name)
            wiki.text = resource_string('trac.wiki', 'default-pages/' +
                                        name).decode('utf-8')
            if wiki.text:
                wiki.save('trac', '')
            else:
                printout('%s: Skipped empty page' % name)

    req = Mock(href=Href('/'), abs_href=Href('http://trac.edgewall.org/'),
               perm=MockPerm(), chrome={})
    for name in sys.argv[1:]:
        name = os.path.basename(name)
        wiki = WikiPage(env, name)
        if not wiki.exists:
            continue
        context = web_context(req, wiki.resource, absurls=True)
        rst = wiki2rest(env, context, wiki)
        sys.stdout.write(rst)

if __name__ == '__main__':
    main()
<commit_msg>1.0.13dev: Print error message when dependency is missing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import codecs
import os.path
import re
import sys

from cStringIO import StringIO
from pkg_resources import resource_listdir, resource_string

from trac.loader import load_components
from trac.test import EnvironmentStub, Mock, MockPerm
from trac.util.text import printerr, printout
from trac.web.chrome import web_context
from trac.web.href import Href
from trac.wiki.formatter import format_to_html
from trac.wiki.model import WikiPage

try:
    import html2rest
except ImportError:
    printerr("The html2rest package must be installed.")
    sys.exit(1)

class Parser(html2rest.Parser):

    def __init__(self, writer=sys.stdout, encoding='utf8', relroot=None,
                 relpath=None):
        html2rest.Parser.__init__(self, writer, encoding, relroot, relpath)
        self.links = {}

    def end_a(self):
        if '#pending' in self.hrefs:
            href = self.hrefs['#pending']
            label = self.hrefs[href]
            key = label.lower()
            if key not in self.links:
                self.links[key] = (label, href)
            elif href != self.links[key][1]:
                alt = label
                while True:
                    alt += '*'
                    if alt not in self.links:
                        break
                    continue
                self.data(alt[len(label):])
                self.hrefs[href] = alt
                self.links[alt] = (alt, href)
            self.data('`_')
            del self.hrefs['#pending']

    def end_body(self):
        self.end_p()
        for label, href in self.links.itervalues():
            if href[0] != '#':
                self.writeline('.. _%s: %s' % (label, href))
        self.end_p()

def wiki2rest(env, context, wiki):
    text = re.sub('\r?\n', '\n', wiki.text)
    text = re.sub(r'\[\[TracGuideToc\]\]\r?\n?', '', text)
    html = format_to_html(env, context, text)
    html = html.replace(u'\u200b', '')
    html = re.sub(r'\s*([^<]*?)\s*', r'\1', html)
    html = '%s' % html
    writer = StringIO()
    parser = Parser(writer, 'utf-8', None, None)
    parser.feed(html)
    parser.close()
    rst = writer.getvalue().strip('\n')
    rst = re.sub('\n{4,}', '\n\n\n', rst)
    # sort links
    rst = re.sub(r'(?:\n\.\. _[^\n]*)+\Z',
                 lambda m: '\n'.join(sorted(m.group(0).split('\n'),
                                            key=lambda v: v.lower())),
                 rst)
    if any(ord(c) > 0x7f for c in rst):
        # Trac detects utf-8 using BOM
        rst = '%s.. charset=utf-8\n\n%s' % (codecs.BOM_UTF8, rst)
    return rst + '\n'

def main():
    names = sorted(name for name in resource_listdir('trac.wiki',
                                                     'default-pages')
                        if not name.startswith('.'))

    env = EnvironmentStub()
    load_components(env)
    with env.db_transaction:
        for name in names:
            wiki = WikiPage(env, name)
            wiki.text = resource_string('trac.wiki', 'default-pages/' +
                                        name).decode('utf-8')
            if wiki.text:
                wiki.save('trac', '')
            else:
                printout('%s: Skipped empty page' % name)

    req = Mock(href=Href('/'), abs_href=Href('http://trac.edgewall.org/'),
               perm=MockPerm(), chrome={})
    for name in sys.argv[1:]:
        name = os.path.basename(name)
        wiki = WikiPage(env, name)
        if not wiki.exists:
            continue
        context = web_context(req, wiki.resource, absurls=True)
        rst = wiki2rest(env, context, wiki)
        sys.stdout.write(rst)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from threading import Thread
import cv2
from scipy.ndimage.interpolate import rotate

#Actions: N = 0   W = 1   S = 2   E = 3
#         NW = 4  SW = 5  SE = 6  NE = 7
class AStar(Thread):
  rotate = np.array([(0,1),(-1,0),(0,-1),(1,0),(-1,1),(-1,-1),(1,-1),(1,1)])
  angle  = np.array([math.atan2(p[1], p[0]) * 180 / math.pi for p in rotate])

  def __init__(self, pathmap):
    Thread.__init__(self)
    self.stopstate = False
    self.c_space = None
    self.energyCost = 0.01
    self.nextGoal = None
    #self.self = cv2.imread('robot.png')
    self.xPotential = np.zeros(pathmap.shape[:2]) # flow map
    self.yPotential = np.zeros(pathmap.shape[:2])
    self.pathmap = pathmap

  def setNextGoal(self, pose):
    self.nextGoal = pose

  def reachedGoal(self, pose):
    return np.array_equal(pose[0:2], self.nextGoal[0:2])

  def getValidActions(self, pose, c_space):
    actions = np.zeros(8)
    for action in range(8):
      cost = getCost(pose,action,1,c_space)
      if cost < 0.5:
        actions[action] = 1
    return actions

  def getSuccessors(self, pose, action, dt):
    return (pose[0] + rotation[action][0], \
            pose[1] + rotation[action][1], \
            angle[action])

  def getCost(self, pose, action, dt, c_space):
    #cost = np.zeros(2)
    #newPose = getSuccessors(pose,action,dt)
    #rotatedSelf = rotate(self.self,newPose[2])
    #rotSizex,rotSize_y = rotatedSelf.shape[0]/2,rotatedSelf.shape[1]/2

    #cost[0] = np.multiply(rotatedSelf, self.c_space[pose[0]-rotSize_x:pose[0]+rotSize_x, pose[1]-rotSize_y:pose[1]+rotSize_y])

    #cost[1] = np.multiply(rotatedSelf, self.c_space[pose[0]-rotSize_x + rotate[theta,0]:pose[0]+rotSize_x+ rotate[theta,0], pose[1]-rotSize_y+rotate[theta,1]:pose[1]+rotSize_y]+rotate[theta,1])

    # make it simple in order to speed things up, do not account for theta :(
    x = pose[0] + rotate[action][0]
    y = pose[1] + rotate[action][1]
    return c_space[y, x] * self.energy_cost

  def distanceCost(self, pose1, pose2):
    return np.linalg.norm(pose2[0:2]-pose1[0:2]) * self.energyCost

  def retraceParents(self,node):
    path = np.empty(0)
    while node != None:
      np.append(path,node)
      node = node.parent
    return path

  def computePath(self, pose):
    closed = np.zeros(self.c_space.shape)
    opened = []
    gScore = np.zeros(self.c_space.shape)
    fScore = np.zeros(self.c_shape.shape)
    numleft = np.prod(self.c_space.shape)
    xPotential = np.zeros(self.pathmap.shape[:2])
    yPotential = np.zeros(self.pathmap.shape[:2])
    c_space = np.copy(self.c_space)

    opened.append((self.nextGoal[:2], 0))
    while len(opened) > 0:
      # this can be parallelizable on a GPU, instead of 1 do k kernel calls
      minitem = min(opened, key=lambda s: s[1])
      opened.remove(minitem)
      current = minitem[0]
      #if np.array_equal(current, pose[:2]):
      #  return retraceParents(current)
      closed[current[1], current[0]] = 1
      validActions = self.getValidActions(pose, c_space)
      for action in range(len(validActions)):
        if validActions[action] != 1:
          continue
        successor = getSuccessors(current, action, 1)[:2]
        if closed[successor[1], successor[0]] == 1:
          continue
        tentative_gScore = gScore[current[1], current[0]] + \
            self.getCost(current, successor)
        tentative_fScore = tentative_gScore + \
            self.distanceCost(successor, pose[:2])
        if sum([np.array_equal(successor, n) for n in opened]) == 0:
          opened.append((successor, tentative_fScore))
        elif tentative_gScore >= gScore[successor[1], successor[0]]:
          continue
        xPotential = current[0] - successor[0]
        yPotential = current[1] - successor[1]
        gScore[successor[1], successor[0]] = tentative_gScore
        fScore[successor[1], successor[0]] = tentative_fScore

    self.xPotential = xPotential
    self.yPotential = yPotential

  def getNextState(pose):
    return (xPotential[pose[0]] + pose[0], yPotential[pose[1]] + pose[1]])

  def setLocalizer(self, localizer):
    self.localizer = localizer

  def setConstraintSpace(self, collisionSpace):
    # hack for the planner to plan faster
    robotMask = np.ones((3, 3))
    collisionSpace = convolve2d(collisionSpace, robotMask, mode="same")
    self.c_space = pathmap + collisionSpace
    # set up walls
    self.c_space[:,0] = 1
    self.c_space[:,self.c_space.shape[1]-1] = 1
    self.c_space[0,:] = 1
    self.c_space[self.c_space.shape[0]-1,:] = 1

  def run(self):
    lastTime = time.time()
    while not self.stopstate:
      currTime = time.time()
      # grab the distribution from the localizer
      distribution = self.localizer.predict()
      # select the most likely position to start planning on
      pose = list(np.argwhere(distribution == np.amax(distribution))[0, :])
      # plan the flow map on that position
      self.computePath(pose)
      print "[PLANNING] process time:", time.time() - currTime

  def stop(self):
    self.stopstate = True
<commit_msg>explicitly define datatypes for arrays in planner<commit_after>from threading import Thread
import cv2
from scipy.ndimage.interpolate import rotate

#Actions: N = 0   W = 1   S = 2   E = 3
#         NW = 4  SW = 5  SE = 6  NE = 7
class AStar(Thread):
  rotate = np.array([(0,1),(-1,0),(0,-1),(1,0),(-1,1),(-1,-1),(1,-1),(1,1)])
  angle  = np.array([math.atan2(p[1], p[0]) * 180 / math.pi for p in rotate])

  def __init__(self, pathmap):
    Thread.__init__(self)
    self.stopstate = False
    self.c_space = None
    self.energyCost = 0.01
    self.nextGoal = None
    #self.self = cv2.imread('robot.png')
    self.xPotential = np.zeros(pathmap.shape[:2]) # flow map
    self.yPotential = np.zeros(pathmap.shape[:2])
    self.pathmap = pathmap

  def setNextGoal(self, pose):
    self.nextGoal = pose

  def reachedGoal(self, pose):
    return np.array_equal(pose[0:2], self.nextGoal[0:2])

  def getValidActions(self, pose, c_space):
    actions = np.zeros(8)
    for action in range(8):
      cost = getCost(pose,action,1,c_space)
      if cost < 0.5:
        actions[action] = 1
    return actions

  def getSuccessors(self, pose, action, dt):
    return (pose[0] + rotation[action][0], \
            pose[1] + rotation[action][1], \
            angle[action])

  def getCost(self, pose, action, dt, c_space):
    #cost = np.zeros(2)
    #newPose = getSuccessors(pose,action,dt)
    #rotatedSelf = rotate(self.self,newPose[2])
    #rotSizex,rotSize_y = rotatedSelf.shape[0]/2,rotatedSelf.shape[1]/2

    #cost[0] = np.multiply(rotatedSelf, self.c_space[pose[0]-rotSize_x:pose[0]+rotSize_x, pose[1]-rotSize_y:pose[1]+rotSize_y])

    #cost[1] = np.multiply(rotatedSelf, self.c_space[pose[0]-rotSize_x + rotate[theta,0]:pose[0]+rotSize_x+ rotate[theta,0], pose[1]-rotSize_y+rotate[theta,1]:pose[1]+rotSize_y]+rotate[theta,1])

    # make it simple in order to speed things up, do not account for theta :(
    x = pose[0] + rotate[action][0]
    y = pose[1] + rotate[action][1]
    return c_space[y, x] * self.energy_cost

  def distanceCost(self, pose1, pose2):
    return np.linalg.norm(pose2[0:2]-pose1[0:2]) * self.energyCost

  def retraceParents(self,node):
    path = np.empty(0)
    while node != None:
      np.append(path,node)
      node = node.parent
    return path

  def computePath(self, pose):
    closed = np.zeros(self.c_space.shape, dtype=np.uint8)
    opened = []
    gScore = np.zeros(self.c_space.shape, dtype=np.float32)
    fScore = np.zeros(self.c_shape.shape, dtype=np.float32)
    xPotential = np.zeros(self.pathmap.shape[:2], dtype=np.int8)
    yPotential = np.zeros(self.pathmap.shape[:2], dtype=np.int8)
    c_space = np.copy(self.c_space)

    opened.append((self.nextGoal[:2], 0))
    while len(opened) > 0:
      # this can be parallelizable on a GPU, instead of 1 do k kernel calls
      minitem = min(opened, key=lambda s: s[1])
      opened.remove(minitem)
      current = minitem[0]
      #if np.array_equal(current, pose[:2]):
      #  return retraceParents(current)
      closed[current[1], current[0]] = 1
      validActions = self.getValidActions(pose, c_space)
      for action in range(len(validActions)):
        if validActions[action] != 1:
          continue
        successor = getSuccessors(current, action, 1)[:2]
        if closed[successor[1], successor[0]] == 1:
          continue
        tentative_gScore = gScore[current[1], current[0]] + \
            self.getCost(current, successor)
        #tentative_fScore = tentative_gScore + \
        #    self.distanceCost(successor, pose[:2])
        tentative_fScore = tentative_gScore
        if sum([np.array_equal(successor, n) for n in opened]) == 0:
          opened.append((successor, tentative_fScore))
        elif tentative_gScore >= gScore[successor[1], successor[0]]:
          continue
        xPotential = current[0] - successor[0]
        yPotential = current[1] - successor[1]
        gScore[successor[1], successor[0]] = tentative_gScore
        fScore[successor[1], successor[0]] = tentative_fScore

    self.xPotential = xPotential
    self.yPotential = yPotential

  def getNextState(pose):
    return (xPotential[pose[0]] + pose[0], yPotential[pose[1]] + pose[1]])

  def setLocalizer(self, localizer):
    self.localizer = localizer

  def setConstraintSpace(self, collisionSpace):
    # hack for the planner to plan faster
    robotMask = np.ones((3, 3))
    collisionSpace = convolve2d(collisionSpace, robotMask, mode="same")
    self.c_space = pathmap + collisionSpace
    # set up walls
    self.c_space[:,0] = 1
    self.c_space[:,self.c_space.shape[1]-1] = 1
    self.c_space[0,:] = 1
    self.c_space[self.c_space.shape[0]-1,:] = 1

  def run(self):
    lastTime = time.time()
    while not self.stopstate:
      currTime = time.time()
      # grab the distribution from the localizer
      distribution = self.localizer.predict()
      # select the most likely position to start planning on
      pose = list(np.argwhere(distribution == np.amax(distribution))[0, :])
      # plan the flow map on that position
      self.computePath(pose)
      print "[PLANNING] process time:", time.time() - currTime

  def stop(self):
    self.stopstate = True
<|endoftext|>


<commit_before>import os
import tempfile
import time
import shutil
from typing import Text, Optional, List

import pytest
from _pytest.tmpdir import TempdirFactory

import rasa
import rasa.data as data
import rasa.core
import rasa.nlu
from rasa.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH
from rasa.core.domain import Domain
from rasa.model import (
    FINGERPRINT_CONFIG_KEY,
    FINGERPRINT_DOMAIN_KEY,
    FINGERPRINT_FILE_PATH,
    FINGERPRINT_NLU_DATA_KEY,
    FINGERPRINT_RASA_VERSION_KEY,
    FINGERPRINT_STORIES_KEY,
    FINGERPRINT_TRAINED_AT_KEY,
    core_fingerprint_changed,
    create_package_rasa,
    get_latest_model,
    get_model,
    get_model_subdirectories,
    model_fingerprint,
    nlu_fingerprint_changed,
    Fingerprint,
    should_retrain,
    FINGERPRINT_CONFIG_CORE_KEY,
    FINGERPRINT_CONFIG_NLU_KEY,
)
from rasa.exceptions import ModelNotFound

def test_get_latest_model(trained_model):
    import shutil

    path_of_latest = os.path.join(os.path.dirname(trained_model), "latest.tar.gz")
    shutil.copy(trained_model, path_of_latest)

    model_directory = os.path.dirname(path_of_latest)

    assert get_latest_model(model_directory) == path_of_latest

def test_get_model_from_directory(trained_model):
    unpacked = get_model(trained_model)

    assert os.path.exists(os.path.join(unpacked, "core"))
    assert os.path.exists(os.path.join(unpacked, "nlu"))

def test_get_model_context_manager(trained_model):
    with get_model(trained_model) as unpacked:
        assert os.path.exists(unpacked)

    assert not os.path.exists(unpacked)

@pytest.mark.parametrize("model_path", ["foobar", "rasa", "README.md", None])
def test_get_model_exception(model_path):
    with pytest.raises(ModelNotFound):
        get_model(model_path)

def test_get_model_from_directory_with_subdirectories(
    trained_model, tmpdir_factory: TempdirFactory
):
    unpacked = get_model(trained_model)
    unpacked_core, unpacked_nlu = get_model_subdirectories(unpacked)

    assert unpacked_core
    assert unpacked_nlu

    directory = tmpdir_factory.mktemp("empty_model_dir")
    with pytest.raises(ModelNotFound):
        get_model_subdirectories(directory)

def test_get_model_from_directory_nlu_only(trained_model):
    unpacked = get_model(trained_model)
    shutil.rmtree(os.path.join(unpacked, "core"))
    unpacked_core, unpacked_nlu = get_model_subdirectories(unpacked)

    assert not unpacked_core
    assert unpacked_nlu

def _fingerprint(
    config: Optional[Text] = None,
    config_nlu: Optional[Text] = None,
    config_core: Optional[Text] = None,
    domain: Optional[List[Text]] = None,
    rasa_version: Text = "1.0",
    stories: Optional[List[Text]] = None,
    nlu: Optional[List[Text]] = None,
):
    return {
        FINGERPRINT_CONFIG_KEY: config if config is not None else ["test"],
        FINGERPRINT_CONFIG_CORE_KEY: config_core
        if config_core is not None
        else ["test"],
        FINGERPRINT_CONFIG_NLU_KEY: config_nlu if config_nlu is not None else ["test"],
        FINGERPRINT_DOMAIN_KEY: domain if domain is not None else ["test"],
        FINGERPRINT_TRAINED_AT_KEY: time.time(),
        FINGERPRINT_RASA_VERSION_KEY: rasa_version,
        FINGERPRINT_STORIES_KEY: stories if stories is not None else ["test"],
        FINGERPRINT_NLU_DATA_KEY: nlu if nlu is not None else ["test"],
    }

def test_persist_and_load_fingerprint():
    from rasa.model import persist_fingerprint, fingerprint_from_path

    fingerprint = _fingerprint()
    output_directory = tempfile.mkdtemp()

    persist_fingerprint(output_directory, fingerprint)
    actual = fingerprint_from_path(output_directory)

    assert actual == fingerprint

@pytest.mark.parametrize(
    "fingerprint2",
    [
        _fingerprint(config=["other"]),
        _fingerprint(domain=["other"]),
        _fingerprint(domain=Domain.empty()),
        _fingerprint(stories=["test", "other"]),
        _fingerprint(rasa_version="100"),
        _fingerprint(config=["other"], domain=["other"]),
    ],
)
def test_core_fingerprint_changed(fingerprint2):
    fingerprint1 = _fingerprint()
    assert core_fingerprint_changed(fingerprint1, fingerprint2)

@pytest.mark.parametrize(
    "fingerprint2",
    [
        _fingerprint(config=["other"]),
        _fingerprint(nlu=["test", "other"]),
        _fingerprint(rasa_version="100"),
        _fingerprint(rasa_version="100", config=["other"]),
    ],
)
def test_nlu_fingerprint_changed(fingerprint2):
    fingerprint1 = _fingerprint()
    assert nlu_fingerprint_changed(fingerprint1, fingerprint2)

def _project_files(
    project,
    config_file=DEFAULT_CONFIG_PATH,
    domain="domain.yml",
    training_files=DEFAULT_DATA_PATH,
):

    if training_files is None:
        core_directory = None
        nlu_directory = None
    else:
        core_directory, nlu_directory = data.get_core_nlu_directories(
            os.path.join(project, training_files)
        )
    paths = {
        "config_file": config_file,
        "domain": domain,
        "nlu_data": core_directory,
        "stories": nlu_directory,
    }

    return {k: v if v is None else os.path.join(project, v) for k, v in paths.items()}

def test_create_fingerprint_from_paths(project):
    project_files = _project_files(project)

    assert model_fingerprint(**project_files)

@pytest.mark.parametrize(
    "project_files", [["invalid", "invalid", "invalid"], [None, None, None]]
)
def test_create_fingerprint_from_invalid_paths(project, project_files):
    project_files = _project_files(project, *project_files)

    expected = _fingerprint(
        config="",
        config_nlu="",
        config_core="",
        domain=[],
        rasa_version=rasa.__version__,
        stories=[],
        nlu=[],
    )

    actual = model_fingerprint(**project_files)
    assert actual[FINGERPRINT_TRAINED_AT_KEY] is not None

    del actual[FINGERPRINT_TRAINED_AT_KEY]
    del expected[FINGERPRINT_TRAINED_AT_KEY]

    assert actual == expected

@pytest.mark.parametrize("use_fingerprint", [True, False])
def test_rasa_packaging(trained_model, project, use_fingerprint):
    unpacked_model_path = get_model(trained_model)

    os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH))
    if use_fingerprint:
        fingerprint = model_fingerprint(**_project_files(project))
    else:
        fingerprint = None

    tempdir = tempfile.mkdtemp()
    output_path = os.path.join(tempdir, "test.tar.gz")

    create_package_rasa(unpacked_model_path, output_path, fingerprint)

    unpacked = get_model(output_path)

    assert (
        os.path.exists(os.path.join(unpacked, FINGERPRINT_FILE_PATH)) == use_fingerprint
    )
    assert os.path.exists(os.path.join(unpacked, "core"))
    assert os.path.exists(os.path.join(unpacked, "nlu"))

    assert not os.path.exists(unpacked_model_path)

@pytest.mark.parametrize(
    "fingerprint",
    [
        {
            "new": _fingerprint(),
            "old": _fingerprint(stories=["others"]),
            "retrain_core": True,
            "retrain_nlu": False,
        },
        {
            "new": _fingerprint(nlu=["others"]),
            "old": _fingerprint(),
            "retrain_core": False,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(config="others"),
            "old": _fingerprint(),
            "retrain_core": True,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(config_core="others"),
            "old": _fingerprint(),
            "retrain_core": True,
            "retrain_nlu": False,
        },
        {
            "new": _fingerprint(),
            "old": _fingerprint(config_nlu="others"),
            "retrain_core": False,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(),
            "old": _fingerprint(),
            "retrain_core": False,
            "retrain_nlu": False,
        },
    ],
)
def test_should_retrain(trained_model, fingerprint):
    old_model = set_fingerprint(trained_model, fingerprint["old"])

    retrain_core, retrain_nlu = should_retrain(
        fingerprint["new"], old_model, tempfile.mkdtemp()
    )

    assert retrain_core == fingerprint["retrain_core"]
    assert retrain_nlu == fingerprint["retrain_nlu"]

def set_fingerprint(
    trained_model: Text, fingerprint: Fingerprint, use_fingerprint: bool = True
) -> Text:
    unpacked_model_path = get_model(trained_model)

    os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH))
    if use_fingerprint:
        fingerprint = fingerprint
    else:
        fingerprint = None

    tempdir = tempfile.mkdtemp()
    output_path = os.path.join(tempdir, "test.tar.gz")

    create_package_rasa(unpacked_model_path, output_path, fingerprint)

    return output_path
<commit_msg>Fix test on test_model.py.<commit_after>import os
import tempfile
import time
import shutil
from typing import Text, Optional, List

import pytest
from _pytest.tmpdir import TempdirFactory

import rasa
import rasa.data as data
import rasa.core
import rasa.nlu
from rasa.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH
from rasa.core.domain import Domain
from rasa.model import (
    FINGERPRINT_CONFIG_KEY,
    FINGERPRINT_DOMAIN_KEY,
    FINGERPRINT_FILE_PATH,
    FINGERPRINT_NLU_DATA_KEY,
    FINGERPRINT_RASA_VERSION_KEY,
    FINGERPRINT_STORIES_KEY,
    FINGERPRINT_TRAINED_AT_KEY,
    core_fingerprint_changed,
    create_package_rasa,
    get_latest_model,
    get_model,
    get_model_subdirectories,
    model_fingerprint,
    nlu_fingerprint_changed,
    Fingerprint,
    should_retrain,
    FINGERPRINT_CONFIG_CORE_KEY,
    FINGERPRINT_CONFIG_NLU_KEY,
)
from rasa.exceptions import ModelNotFound

def test_get_latest_model(trained_model):
    import shutil

    path_of_latest = os.path.join(os.path.dirname(trained_model), "latest.tar.gz")
    shutil.copy(trained_model, path_of_latest)

    model_directory = os.path.dirname(path_of_latest)

    assert get_latest_model(model_directory) == path_of_latest

def test_get_model_from_directory(trained_model):
    unpacked = get_model(trained_model)

    assert os.path.exists(os.path.join(unpacked, "core"))
    assert os.path.exists(os.path.join(unpacked, "nlu"))

def test_get_model_context_manager(trained_model):
    with get_model(trained_model) as unpacked:
        assert os.path.exists(unpacked)

    assert not os.path.exists(unpacked)

@pytest.mark.parametrize("model_path", ["foobar", "rasa", "README.md", None])
def test_get_model_exception(model_path):
    with pytest.raises(ModelNotFound):
        get_model(model_path)

def test_get_model_from_directory_with_subdirectories(
    trained_model, tmpdir_factory: TempdirFactory
):
    unpacked = get_model(trained_model)
    unpacked_core, unpacked_nlu = get_model_subdirectories(unpacked)

    assert unpacked_core
    assert unpacked_nlu

    directory = tmpdir_factory.mktemp("empty_model_dir").strpath
    with pytest.raises(ModelNotFound):
        get_model_subdirectories(directory)

def test_get_model_from_directory_nlu_only(trained_model):
    unpacked = get_model(trained_model)
    shutil.rmtree(os.path.join(unpacked, "core"))
    unpacked_core, unpacked_nlu = get_model_subdirectories(unpacked)

    assert not unpacked_core
    assert unpacked_nlu

def _fingerprint(
    config: Optional[Text] = None,
    config_nlu: Optional[Text] = None,
    config_core: Optional[Text] = None,
    domain: Optional[List[Text]] = None,
    rasa_version: Text = "1.0",
    stories: Optional[List[Text]] = None,
    nlu: Optional[List[Text]] = None,
):
    return {
        FINGERPRINT_CONFIG_KEY: config if config is not None else ["test"],
        FINGERPRINT_CONFIG_CORE_KEY: config_core
        if config_core is not None
        else ["test"],
        FINGERPRINT_CONFIG_NLU_KEY: config_nlu if config_nlu is not None else ["test"],
        FINGERPRINT_DOMAIN_KEY: domain if domain is not None else ["test"],
        FINGERPRINT_TRAINED_AT_KEY: time.time(),
        FINGERPRINT_RASA_VERSION_KEY: rasa_version,
        FINGERPRINT_STORIES_KEY: stories if stories is not None else ["test"],
        FINGERPRINT_NLU_DATA_KEY: nlu if nlu is not None else ["test"],
    }

def test_persist_and_load_fingerprint():
    from rasa.model import persist_fingerprint, fingerprint_from_path

    fingerprint = _fingerprint()
    output_directory = tempfile.mkdtemp()

    persist_fingerprint(output_directory, fingerprint)
    actual = fingerprint_from_path(output_directory)

    assert actual == fingerprint

@pytest.mark.parametrize(
    "fingerprint2",
    [
        _fingerprint(config=["other"]),
        _fingerprint(domain=["other"]),
        _fingerprint(domain=Domain.empty()),
        _fingerprint(stories=["test", "other"]),
        _fingerprint(rasa_version="100"),
        _fingerprint(config=["other"], domain=["other"]),
    ],
)
def test_core_fingerprint_changed(fingerprint2):
    fingerprint1 = _fingerprint()
    assert core_fingerprint_changed(fingerprint1, fingerprint2)

@pytest.mark.parametrize(
    "fingerprint2",
    [
        _fingerprint(config=["other"]),
        _fingerprint(nlu=["test", "other"]),
        _fingerprint(rasa_version="100"),
        _fingerprint(rasa_version="100", config=["other"]),
    ],
)
def test_nlu_fingerprint_changed(fingerprint2):
    fingerprint1 = _fingerprint()
    assert nlu_fingerprint_changed(fingerprint1, fingerprint2)

def _project_files(
    project,
    config_file=DEFAULT_CONFIG_PATH,
    domain="domain.yml",
    training_files=DEFAULT_DATA_PATH,
):

    if training_files is None:
        core_directory = None
        nlu_directory = None
    else:
        core_directory, nlu_directory = data.get_core_nlu_directories(
            os.path.join(project, training_files)
        )
    paths = {
        "config_file": config_file,
        "domain": domain,
        "nlu_data": core_directory,
        "stories": nlu_directory,
    }

    return {k: v if v is None else os.path.join(project, v) for k, v in paths.items()}

def test_create_fingerprint_from_paths(project):
    project_files = _project_files(project)

    assert model_fingerprint(**project_files)

@pytest.mark.parametrize(
    "project_files", [["invalid", "invalid", "invalid"], [None, None, None]]
)
def test_create_fingerprint_from_invalid_paths(project, project_files):
    project_files = _project_files(project, *project_files)

    expected = _fingerprint(
        config="",
        config_nlu="",
        config_core="",
        domain=[],
        rasa_version=rasa.__version__,
        stories=[],
        nlu=[],
    )

    actual = model_fingerprint(**project_files)
    assert actual[FINGERPRINT_TRAINED_AT_KEY] is not None

    del actual[FINGERPRINT_TRAINED_AT_KEY]
    del expected[FINGERPRINT_TRAINED_AT_KEY]

    assert actual == expected

@pytest.mark.parametrize("use_fingerprint", [True, False])
def test_rasa_packaging(trained_model, project, use_fingerprint):
    unpacked_model_path = get_model(trained_model)

    os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH))
    if use_fingerprint:
        fingerprint = model_fingerprint(**_project_files(project))
    else:
        fingerprint = None

    tempdir = tempfile.mkdtemp()
    output_path = os.path.join(tempdir, "test.tar.gz")

    create_package_rasa(unpacked_model_path, output_path, fingerprint)

    unpacked = get_model(output_path)

    assert (
        os.path.exists(os.path.join(unpacked, FINGERPRINT_FILE_PATH)) == use_fingerprint
    )
    assert os.path.exists(os.path.join(unpacked, "core"))
    assert os.path.exists(os.path.join(unpacked, "nlu"))

    assert not os.path.exists(unpacked_model_path)

@pytest.mark.parametrize(
    "fingerprint",
    [
        {
            "new": _fingerprint(),
            "old": _fingerprint(stories=["others"]),
            "retrain_core": True,
            "retrain_nlu": False,
        },
        {
            "new": _fingerprint(nlu=["others"]),
            "old": _fingerprint(),
            "retrain_core": False,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(config="others"),
            "old": _fingerprint(),
            "retrain_core": True,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(config_core="others"),
            "old": _fingerprint(),
            "retrain_core": True,
            "retrain_nlu": False,
        },
        {
            "new": _fingerprint(),
            "old": _fingerprint(config_nlu="others"),
            "retrain_core": False,
            "retrain_nlu": True,
        },
        {
            "new": _fingerprint(),
            "old": _fingerprint(),
            "retrain_core": False,
            "retrain_nlu": False,
        },
    ],
)
def test_should_retrain(trained_model, fingerprint):
    old_model = set_fingerprint(trained_model, fingerprint["old"])

    retrain_core, retrain_nlu = should_retrain(
        fingerprint["new"], old_model, tempfile.mkdtemp()
    )

    assert retrain_core == fingerprint["retrain_core"]
    assert retrain_nlu == fingerprint["retrain_nlu"]

def set_fingerprint(
    trained_model: Text, fingerprint: Fingerprint, use_fingerprint: bool = True
) -> Text:
    unpacked_model_path = get_model(trained_model)

    os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH))
    if use_fingerprint:
        fingerprint = fingerprint
    else:
        fingerprint = None

    tempdir = tempfile.mkdtemp()
    output_path = os.path.join(tempdir, "test.tar.gz")

    create_package_rasa(unpacked_model_path, output_path, fingerprint)

    return output_path
<|endoftext|>


<commit_before># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Oppia base constants and handlers."""

__author__ = 'Sean Lip'

import json
import logging
import sys
import traceback
import webapp2

import feconf
from models.exploration import Exploration
from models.state import State
import utils

from google.appengine.api import users

# The JS and CSS code to include in the header of each response.
JS_LIB_CODE = '\n'.join([
    utils.GetFileContents(feconf.THIRD_PARTY_DIR, filepath)
    for filepath in feconf.THIRD_PARTY_JS_LIBS])

CSS_LIB_CODE = '\n'.join([
    utils.GetFileContents('', filepath)
    for filepath in feconf.ALL_CSS_LIBS])

def require_user(handler):
    """Decorator that checks if a user is associated to the current session."""
    def test_login(self, **kwargs):
        """Checks if the user for the current session is logged in."""
        user = users.get_current_user()
        if not user:
            self.redirect(users.create_login_url(self.request.uri))
            return
        return handler(self, user, **kwargs)

    return test_login

def require_editor(handler):
    """Decorator that checks if the user can edit the given entity."""
    def test_editor(self, exploration_id, state_id=None, **kwargs):
        """Gets the user and exploration id if the user can edit it.

        Returns:
            The user and exploration instance, if the user is authorized to edit
            this exploration. Also, the state instance, if one is supplied.

        Raises:
            self.NotLoggedInException: if there is no current user.
            self.UnauthorizedUserException: if the user exists but does not have
                the right credentials.
        """
        user = users.get_current_user()
        if not user:
            self.redirect(users.create_login_url(self.request.uri))
            return

        exploration = utils.GetEntity(Exploration, exploration_id)
        if not utils.CheckAuthorship(user, exploration):
            raise self.UnauthorizedUserException(
                '%s does not have the credentials to edit this exploration.',
                user)

        if not state_id:
            return handler(self, user, exploration, **kwargs)
        state = utils.GetEntity(State, state_id)
        return handler(self, user, exploration, state, **kwargs)

    return test_editor

class BaseHandler(webapp2.RequestHandler):
    """Base class for all Oppia handlers."""
    def __init__(self, request, response):
        # Set self.request, self.response and self.app.
        self.initialize(request, response)

        # Initializes the return dicts for the handlers.
        self.values = {
            'css': CSS_LIB_CODE,
            'debug': feconf.DEBUG,
            'js_lib_code': JS_LIB_CODE,
        }
        self.data_values = {'debug': feconf.DEBUG}

        user = users.get_current_user()
        if user:
            self.values.update({
                'logout_url': users.create_logout_url(self.request.uri),
                'user': str(user),
            })
            self.data_values.update({'user': str(user)})
        else:
            login_url = users.create_login_url(self.request.uri)
            self.values['login_url'] = login_url
            self.data_values['login_url'] = login_url

    def handle_exception(self, exception, debug_mode):
        """Overwrites the default exception handler."""
        logging.info(''.join(traceback.format_exception(*sys.exc_info())))
        logging.error('Exception raised: %s', exception)

        if isinstance(exception, self.NotLoggedInException):
            self.redirect(users.create_login_url(self.request.uri))
            return

        if isinstance(exception, self.UnauthorizedUserException):
            self.error(401)
            self.response.out.write(json.dumps(
                {'code': '401 Unauthorized', 'error': str(exception)}))
            return

        if isinstance(exception, self.InvalidInputException):
            self.error(400)
            self.response.out.write(json.dumps(
                {'code': '400 Bad Request', 'error': str(exception)}))
            return

        webapp2.RequestHandler.handle_exception(self, exception, debug_mode)
        logging.error('Exception was not handled: %s', exception)

    class UnauthorizedUserException(Exception):
        """Error class for unauthorized access."""

    class NotLoggedInException(Exception):
        """Error class for users that are not logged in (error code 401)."""

    class InvalidInputException(Exception):
        """Error class for invalid input on the user's side (error code 400)."""
<commit_msg>Add TODO<commit_after># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Oppia base constants and handlers."""

__author__ = 'Sean Lip'

import json
import logging
import sys
import traceback
import webapp2

import feconf
from models.exploration import Exploration
from models.state import State
import utils

from google.appengine.api import users

# The JS and CSS code to include in the header of each response.
# TODO(sll): Cache these so that they aren't recomputed each time.
JS_LIB_CODE = '\n'.join([
    utils.GetFileContents(feconf.THIRD_PARTY_DIR, filepath)
    for filepath in feconf.THIRD_PARTY_JS_LIBS])

CSS_LIB_CODE = '\n'.join([
    utils.GetFileContents('', filepath)
    for filepath in feconf.ALL_CSS_LIBS])

def require_user(handler):
    """Decorator that checks if a user is associated to the current session."""
    def test_login(self, **kwargs):
        """Checks if the user for the current session is logged in."""
        user = users.get_current_user()
        if not user:
            self.redirect(users.create_login_url(self.request.uri))
            return
        return handler(self, user, **kwargs)

    return test_login

def require_editor(handler):
    """Decorator that checks if the user can edit the given entity."""
    def test_editor(self, exploration_id, state_id=None, **kwargs):
        """Gets the user and exploration id if the user can edit it.

        Returns:
            The user and exploration instance, if the user is authorized to edit
            this exploration. Also, the state instance, if one is supplied.

        Raises:
            self.NotLoggedInException: if there is no current user.
            self.UnauthorizedUserException: if the user exists but does not have
                the right credentials.
        """
        user = users.get_current_user()
        if not user:
            self.redirect(users.create_login_url(self.request.uri))
            return

        exploration = utils.GetEntity(Exploration, exploration_id)
        if not utils.CheckAuthorship(user, exploration):
            raise self.UnauthorizedUserException(
                '%s does not have the credentials to edit this exploration.',
                user)

        if not state_id:
            return handler(self, user, exploration, **kwargs)
        state = utils.GetEntity(State, state_id)
        return handler(self, user, exploration, state, **kwargs)

    return test_editor

class BaseHandler(webapp2.RequestHandler):
    """Base class for all Oppia handlers."""
    def __init__(self, request, response):
        # Set self.request, self.response and self.app.
        self.initialize(request, response)

        # Initializes the return dicts for the handlers.
        self.values = {
            'css': CSS_LIB_CODE,
            'debug': feconf.DEBUG,
            'js_lib_code': JS_LIB_CODE,
        }
        self.data_values = {'debug': feconf.DEBUG}

        user = users.get_current_user()
        if user:
            self.values.update({
                'logout_url': users.create_logout_url(self.request.uri),
                'user': str(user),
            })
            self.data_values.update({'user': str(user)})
        else:
            login_url = users.create_login_url(self.request.uri)
            self.values['login_url'] = login_url
            self.data_values['login_url'] = login_url

    def handle_exception(self, exception, debug_mode):
        """Overwrites the default exception handler."""
        logging.info(''.join(traceback.format_exception(*sys.exc_info())))
        logging.error('Exception raised: %s', exception)

        if isinstance(exception, self.NotLoggedInException):
            self.redirect(users.create_login_url(self.request.uri))
            return

        if isinstance(exception, self.UnauthorizedUserException):
            self.error(401)
            self.response.out.write(json.dumps(
                {'code': '401 Unauthorized', 'error': str(exception)}))
            return

        if isinstance(exception, self.InvalidInputException):
            self.error(400)
            self.response.out.write(json.dumps(
                {'code': '400 Bad Request', 'error': str(exception)}))
            return

        webapp2.RequestHandler.handle_exception(self, exception, debug_mode)
        logging.error('Exception was not handled: %s', exception)

    class UnauthorizedUserException(Exception):
        """Error class for unauthorized access."""

    class NotLoggedInException(Exception):
        """Error class for users that are not logged in (error code 401)."""

    class InvalidInputException(Exception):
        """Error class for invalid input on the user's side (error code 400)."""
<|endoftext|>


<commit_before>from rdflib.term import Literal, URIRef
from rdflib.namespace import Namespace

__test__ = False

test_data = """
#  Definitions of terms describing the n3 model
#

@keywords a.

@prefix n3: <#>.
@prefix log: <log.n3#> .
@prefix rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <#> .

@forAll :s, :p, :x, :y, :z.

n3:Statement    a rdf:Class .
n3:StatementSet a rdf:Class .

n3:includes     a rdfs:Property .   # Cf rdf:li

n3:predicate    a rdf:Property; rdfs:domain n3:statement .
n3:subject      a rdf:Property; rdfs:domain n3:statement .
n3:object       a rdf:Property; rdfs:domain n3:statement .

n3:context      a rdf:Property; rdfs:domain n3:statement;
                rdfs:range n3:StatementSet .

########### Rules

{ :x :p :y . } log:means { [
                n3:subject :x;
                n3:predicate :p;
                n3:object :y ] a log:Truth}.

# Needs more thought ... ideally, we have the implcit AND rules of
# juxtaposition (introduction and elimination)

{
    {
        {  :x n3:includes :s. } log:implies { :y n3:includes :s. } .
    } forall :s1 .
} log:implies { :x log:implies :y } .

{
    {
        {  :x n3:includes :s. } log:implies { :y n3:includes :s. } .
    } forall :s1
} log:implies { :x log:implies :y } .

# I think n3:includes has to be axiomatic builtin. - unless you go to syntax description.
# syntax.n3?
"""

import unittest

from rdflib.graph import Graph, ConjunctiveGraph

class N3TestCase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def testFileName(self):
        """
        Test that the n3 parser throws an Exception when using the identifier
        ":foo.txt", as this is not valid as per the rdf spec.
        """
        test_data = """
@prefix : <http://www.example.com/> .

:foo.txt :p :q .
"""
        g = Graph()
        # TODO: determine what's correct here, either:
        # seeAlso <http://www.w3.org/TeamSubmission/turtle/#name>
        self.assertRaises(Exception, g.parse, data=test_data, format="n3")
        # or.. (challenging comment below):
        # This isn't the expected result based on my reading of n3 bits
        #g.parse(data=test_data, format="n3")
        #s = g.value(predicate=URIRef("http://www.example.com/p"), object=URIRef("http://www.example.com/q"))
        #self.assertEquals(s, URIRef("http://www.example.org/foo.txt"))

    def testModel(self):
        g = ConjunctiveGraph()
        g.parse(data=test_data, format="n3")
        i = 0
        for s, p, o in g:
            if isinstance(s, Graph):
                i += 1
        self.assertEquals(i, 3)
        self.assertEquals(len(list(g.contexts())), 13)

        g.close()

    def testParse(self):
        g = ConjunctiveGraph()
        g.parse("http://groups.csail.mit.edu/dig/2005/09/rein/examples/troop42-policy.n3", format="n3")

cases = ['no quotes',
         "single ' quote",
         'double " quote',
         '"',
         "'",
         '"\'"',
         '\\', # len 1
         '\\"', # len 2
         '\\\\"', # len 3
         '\\"\\', # len 3
         'here',
         ]

class N3Quoting(unittest.TestCase):
    def test(self):
        g = Graph()
        NS = Namespace("http://quoting.test/")
        for i, case in enumerate(cases):
            g.add((NS['subj'], NS['case%s' % i], Literal(case)))
        n3txt = g.serialize(format="n3")
        #print n3txt

        g2 = Graph()
        g2.parse(data=n3txt, format="n3")
        for i, case in enumerate(cases):
            l = g2.value(NS['subj'], NS['case%s' % i])
            #print repr(l), repr(case)
            self.assertEqual(l, Literal(case))

if __name__ == '__main__':
    unittest.main()
<commit_msg>added some n3 tests for known issues;<commit_after>from rdflib.term import Literal, URIRef
from rdflib.namespace import Namespace

__test__ = False

test_data = """
#  Definitions of terms describing the n3 model
#

@keywords a.

@prefix n3: <#>.
@prefix log: <log.n3#> .
@prefix rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <#> .

@forAll :s, :p, :x, :y, :z.

n3:Statement    a rdf:Class .
n3:StatementSet a rdf:Class .

n3:includes     a rdfs:Property .   # Cf rdf:li

n3:predicate    a rdf:Property; rdfs:domain n3:statement .
n3:subject      a rdf:Property; rdfs:domain n3:statement .
n3:object       a rdf:Property; rdfs:domain n3:statement .

n3:context      a rdf:Property; rdfs:domain n3:statement;
                rdfs:range n3:StatementSet .

########### Rules

{ :x :p :y . } log:means { [
                n3:subject :x;
                n3:predicate :p;
                n3:object :y ] a log:Truth}.

# Needs more thought ... ideally, we have the implcit AND rules of
# juxtaposition (introduction and elimination)

{
    {
        {  :x n3:includes :s. } log:implies { :y n3:includes :s. } .
    } forall :s1 .
} log:implies { :x log:implies :y } .

{
    {
        {  :x n3:includes :s. } log:implies { :y n3:includes :s. } .
    } forall :s1
} log:implies { :x log:implies :y } .

# I think n3:includes has to be axiomatic builtin. - unless you go to syntax description.
# syntax.n3?
"""

import unittest

from rdflib.graph import Graph, ConjunctiveGraph

class N3TestCase(unittest.TestCase):

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def testFileName(self):
        """
        Test that the n3 parser throws an Exception when using the identifier
        ":foo.txt", as this is not valid as per the rdf spec.
        """
        test_data = """
@prefix : <http://www.example.com/> .

:foo.txt :p :q .
"""
        g = Graph()
        # TODO: determine what's correct here, either:
        # seeAlso <http://www.w3.org/TeamSubmission/turtle/#name>
        self.assertRaises(Exception, g.parse, data=test_data, format="n3")
        # or.. (challenging comment below):
        # This isn't the expected result based on my reading of n3 bits
        #g.parse(data=test_data, format="n3")
        #s = g.value(predicate=URIRef("http://www.example.com/p"), object=URIRef("http://www.example.com/q"))
        #self.assertEquals(s, URIRef("http://www.example.org/foo.txt"))

    def testBase(self):
        """
        Test that the n3 parser supports base declarations
        This is issue #22
        """

        input = """
@prefix : <http://example.com> .
# default base
<foo> :name "Foo" .
# change it
@base <http://example.com/doc/> .
<bar> :name "Bar" .
# and change it more - they are cummalative
@base <doc2/> .
<bing> :name "Bing" .
"""
        g = Graph()
        g.parse(data=input, format="n3")

    def testIssue23(self):
        input="""<http://example.com/article1> <http://example.com/title> "this word is in \u201Cquotes\u201D"."""

        g = Graph()
        g.parse(data=input, format="n3")

        # Note difference in case of hex code, cwm allows lower-case
        input="""<http://example.com/article1> <http://example.com/title> "this word is in \u201cquotes\u201d"."""

        g.parse(data=input, format="n3")

    def testIssue29(self):
        input="""@prefix foo-bar: <http://example.org/> .

foo-bar:Ex foo-bar:name "Test" . """

        g = Graph()
        g.parse(data=input, format="n3")

    def testModel(self):
        g = ConjunctiveGraph()
        g.parse(data=test_data, format="n3")
        i = 0
        for s, p, o in g:
            if isinstance(s, Graph):
                i += 1
        self.assertEquals(i, 3)
        self.assertEquals(len(list(g.contexts())), 13)

        g.close()

    def testParse(self):
        g = ConjunctiveGraph()
        g.parse("http://groups.csail.mit.edu/dig/2005/09/rein/examples/troop42-policy.n3", format="n3")

cases = ['no quotes',
         "single ' quote",
         'double " quote',
         '"',
         "'",
         '"\'"',
         '\\', # len 1
         '\\"', # len 2
         '\\\\"', # len 3
         '\\"\\', # len 3
         'here',
         ]

class N3Quoting(unittest.TestCase):
    def test(self):
        g = Graph()
        NS = Namespace("http://quoting.test/")
        for i, case in enumerate(cases):
            g.add((NS['subj'], NS['case%s' % i], Literal(case)))
        n3txt = g.serialize(format="n3")
        #print n3txt

        g2 = Graph()
        g2.parse(data=n3txt, format="n3")
        for i, case in enumerate(cases):
            l = g2.value(NS['subj'], NS['case%s' % i])
            #print repr(l), repr(case)
            self.assertEqual(l, Literal(case))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># This file is part of beets.
# Copyright 2010, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Tests for the command-line interface.
"""

import unittest
import os
import shutil
import textwrap
from StringIO import StringIO
import logging

import _common
from beets import library
from beets import ui
from beets.ui import commands
from beets import autotag
from beets import mediafile

TEST_TITLES = ('The Opener','The Second Track','The Last Track')
class ImportTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()

        # Suppress logging output.
        log = logging.getLogger('beets')
        log.setLevel(logging.CRITICAL)

        self.lib = library.Library(':memory:')
        self.libdir = os.path.join('rsrc', 'testlibdir')
        self.lib.directory = self.libdir
        self.lib.path_formats = {
            'default': os.path.join('$artist', '$album', '$title')
        }

        self.srcdir = os.path.join('rsrc', 'testsrcdir')

    def tearDown(self):
        self.io.restore()
        if os.path.exists(self.libdir):
            shutil.rmtree(self.libdir)
        if os.path.exists(self.srcdir):
            shutil.rmtree(self.srcdir)

    def _create_test_file(self, filepath, metadata):
        """Creates an mp3 file at the given path within self.srcdir.
        filepath is given as an array of folder names, ending with the
        file name. Sets the file's metadata from the provided dict.
        Returns the full, real path to the file.
        """
        realpath = os.path.join(self.srcdir, *filepath)
        if not os.path.exists(os.path.dirname(realpath)):
            os.makedirs(os.path.dirname(realpath))
        shutil.copy(os.path.join('rsrc', 'full.mp3'), realpath)

        f = mediafile.MediaFile(realpath)
        for attr in metadata:
            setattr(f, attr, metadata[attr])
        f.save()

        return realpath

    def _run_import(self, titles=TEST_TITLES):
        # Make a bunch of tracks to import.
        for i, title in enumerate(titles):
            path = self._create_test_file(
                ['the_album', 'track_%s.mp3' % (i+1)],
                {
                    'track': (i+1),
                    'artist': 'The Artist',
                    'album': 'The Album',
                    'title': title,
                })

        # Run the UI "beet import" command!
        commands.import_files(
                lib=self.lib,
                paths=[os.path.dirname(path)],
                copy=True,
                write=True,
                autot=False,
                logpath=None,
                art=False,
                threaded=False,
                color=False,
                delete=False,
                quiet=True,
                resume=False,
                quiet_fallback='skip',
        )

    def test_album_created_with_track_artist(self):
        self._run_import()
        albums = self.lib.albums()
        self.assertEqual(len(albums), 1)
        self.assertEqual(albums[0].albumartist, 'The Artist')

    def test_import_copy_arrives(self):
        self._run_import()
        artist_folder = os.path.join(self.libdir, 'The Artist')
        album_folder = os.path.join(artist_folder, 'The Album')
        self.assertEqual(len(os.listdir(artist_folder)), 1)
        self.assertEqual(len(os.listdir(album_folder)), 3)

        filenames = set(os.listdir(album_folder))
        destinations = set('%s.mp3' % title for title in TEST_TITLES)
        self.assertEqual(filenames, destinations)

class ListTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()

        self.lib = library.Library(':memory:')
        i = _common.item()
        self.lib.add(i)
        self.lib.add_album([i])

    def tearDown(self):
        self.io.restore()

    def test_list_outputs_item(self):
        commands.list_items(self.lib, '', False)
        out = self.io.getoutput()
        self.assertTrue(u'the title' in out)

    def test_list_album_outputs_something(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertGreater(len(out), 0)

    def test_list_album_omits_title(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertTrue(u'the title' not in out)

    def test_list_uses_track_artist(self):
        commands.list_items(self.lib, '', False)
        out = self.io.getoutput()
        self.assertTrue(u'the artist' in out)
        self.assertTrue(u'the album artist' not in out)

    def test_list_album_uses_album_artist(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertTrue(u'the artist' not in out)
        self.assertTrue(u'the album artist' in out)

class PrintTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def test_print_without_locale(self):
        lang = os.environ.get('LANG')
        if lang:
            del os.environ['LANG']

        try:
            ui.print_(u'something')
        except TypeError:
            self.fail('TypeError during print')
        finally:
            if lang:
                os.environ['LANG'] = lang

    def test_print_with_invalid_locale(self):
        old_lang = os.environ.get('LANG')
        os.environ['LANG'] = ''
        old_ctype = os.environ.get('LC_CTYPE')
        os.environ['LC_CTYPE'] = 'UTF-8'

        try:
            ui.print_(u'something')
        except ValueError:
            self.fail('ValueError during print')
        finally:
            if old_lang:
                os.environ['LANG'] = old_lang
            else:
                del os.environ['LANG']
            if old_ctype:
                os.environ['LC_CTYPE'] = old_ctype
            else:
                del os.environ['LC_CTYPE']

class AutotagTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def _no_candidates_test(self, result):
        res = commands.choose_match(
            'path',
            [_common.item()], # items
            'artist',
            'album',
            [], # candidates
            autotag.RECOMMEND_NONE,
            True, False, commands.CHOICE_SKIP
        )
        self.assertEqual(res, result)
        self.assertTrue('No match' in self.io.getoutput())

    def test_choose_match_with_no_candidates_skip(self):
        self.io.addinput('s')
        self._no_candidates_test(commands.CHOICE_SKIP)

    def test_choose_match_with_no_candidates_asis(self):
        self.io.addinput('u')
        self._no_candidates_test(commands.CHOICE_ASIS)

class InputTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def test_manual_search_gets_unicode(self):
        self.io.addinput('\xc3\x82me')
        self.io.addinput('\xc3\x82me')
        artist, album = commands.manual_search()
        self.assertEqual(artist, u'\xc2me')
        self.assertEqual(album, u'\xc2me')

class ConfigTest(unittest.TestCase):
    def setUp(self):
        self.test_cmd = ui.Subcommand('test', help='test')
        commands.default_commands.append(self.test_cmd)
    def tearDown(self):
        commands.default_commands.pop()
    def _run_main(self, args, config, func):
        self.test_cmd.func = func
        ui.main(args + ['test'], StringIO(config))

    def test_paths_section_respected(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['x'], 'y')
        self._run_main([], textwrap.dedent("""
            [paths]
            x=y"""), func)

    def test_default_paths_preserved(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'],
                             ui.DEFAULT_PATH_FORMATS['default'])
        self._run_main([], textwrap.dedent("""
            [paths]
            x=y"""), func)

    def test_default_paths_overriden_by_legacy_path_format(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'], 'x')
            self.assertEqual(len(lib.path_formats), 1)
        self._run_main([], textwrap.dedent("""
            [beets]
            path_format=x"""), func)

    def test_paths_section_overriden_by_cli_switch(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'], 'z')
            self.assertEqual(len(lib.path_formats), 1)
        self._run_main(['-p', 'z'], textwrap.dedent("""
            [paths]
            x=y"""), func)
def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<commit_msg>tests for deletion during import<commit_after># This file is part of beets.
# Copyright 2010, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Tests for the command-line interface.
"""

import unittest
import os
import shutil
import textwrap
from StringIO import StringIO
import logging

import _common
from beets import library
from beets import ui
from beets.ui import commands
from beets import autotag
from beets import mediafile

TEST_TITLES = ('The Opener','The Second Track','The Last Track')
class ImportTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()

        # Suppress logging output.
        log = logging.getLogger('beets')
        log.setLevel(logging.CRITICAL)

        self.lib = library.Library(':memory:')
        self.libdir = os.path.join('rsrc', 'testlibdir')
        self.lib.directory = self.libdir
        self.lib.path_formats = {
            'default': os.path.join('$artist', '$album', '$title')
        }

        self.srcdir = os.path.join('rsrc', 'testsrcdir')

    def tearDown(self):
        self.io.restore()
        if os.path.exists(self.libdir):
            shutil.rmtree(self.libdir)
        if os.path.exists(self.srcdir):
            shutil.rmtree(self.srcdir)

    def _create_test_file(self, filepath, metadata):
        """Creates an mp3 file at the given path within self.srcdir.
        filepath is given as an array of folder names, ending with the
        file name. Sets the file's metadata from the provided dict.
        Returns the full, real path to the file.
        """
        realpath = os.path.join(self.srcdir, *filepath)
        if not os.path.exists(os.path.dirname(realpath)):
            os.makedirs(os.path.dirname(realpath))
        shutil.copy(os.path.join('rsrc', 'full.mp3'), realpath)

        f = mediafile.MediaFile(realpath)
        for attr in metadata:
            setattr(f, attr, metadata[attr])
        f.save()

        return realpath

    def _run_import(self, titles=TEST_TITLES, delete=False):
        # Make a bunch of tracks to import.
        paths = []
        for i, title in enumerate(titles):
            paths.append(self._create_test_file(
                ['the_album', 'track_%s.mp3' % (i+1)],
                {
                    'track': (i+1),
                    'artist': 'The Artist',
                    'album': 'The Album',
                    'title': title,
                }))

        # Run the UI "beet import" command!
        commands.import_files(
                lib=self.lib,
                paths=[os.path.dirname(paths[0])],
                copy=True,
                write=True,
                autot=False,
                logpath=None,
                art=False,
                threaded=False,
                color=False,
                delete=delete,
                quiet=True,
                resume=False,
                quiet_fallback='skip',
        )

        return paths

    def test_album_created_with_track_artist(self):
        self._run_import()
        albums = self.lib.albums()
        self.assertEqual(len(albums), 1)
        self.assertEqual(albums[0].albumartist, 'The Artist')

    def test_import_copy_arrives(self):
        self._run_import()
        artist_folder = os.path.join(self.libdir, 'The Artist')
        album_folder = os.path.join(artist_folder, 'The Album')
        self.assertEqual(len(os.listdir(artist_folder)), 1)
        self.assertEqual(len(os.listdir(album_folder)), 3)

        filenames = set(os.listdir(album_folder))
        destinations = set('%s.mp3' % title for title in TEST_TITLES)
        self.assertEqual(filenames, destinations)

    def test_import_no_delete(self):
        paths = self._run_import(['sometrack'], delete=False)
        self.assertTrue(os.path.exists(paths[0]))

    def test_import_with_delete(self):
        paths = self._run_import(['sometrack'], delete=True)
        self.assertFalse(os.path.exists(paths[0]))

class ImportApplyTest(unittest.TestCase):
    def setUp(self):
        self.libdir = os.path.join('rsrc', 'testlibdir')
        os.mkdir(self.libdir)
        self.lib = library.Library(':memory:', self.libdir)

        self.srcpath = os.path.join(self.libdir, 'srcfile.mp3')
        shutil.copy(os.path.join('rsrc', 'full.mp3'), self.srcpath)
        self.i = library.Item.from_path(self.srcpath)

        trackinfo = {'title': 'one', 'artist': 'some artist',
                     'track': 1, 'length': 1, 'id': 'trackid'}
        self.info = {
            'artist': 'some artist',
            'album': 'some album',
            'tracks': [trackinfo],
            'va': False,
            'album_id': 'albumid',
            'artist_id': 'artistid',
            'albumtype': 'soundtrack',
        }

    def tearDown(self):
        shutil.rmtree(self.libdir)

    def call_apply(self, coro, items, info):
        coro.send((None, None, # Only used for progress.
                   items, info))

    def test_apply_no_delete(self):
        coro = commands.apply_choices(self.lib, True, False, False,
                                      False, False)
        coro.next() # Prime coroutine.
        self.call_apply(coro, [self.i], self.info)
        self.assertTrue(os.path.exists(self.srcpath))

    def test_apply_with_delete(self):
        coro = commands.apply_choices(self.lib, True, False, False,
                                      True, False)
        coro.next() # Prime coroutine.
        self.call_apply(coro, [self.i], self.info)
        self.assertFalse(os.path.exists(self.srcpath))

class ListTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()

        self.lib = library.Library(':memory:')
        i = _common.item()
        self.lib.add(i)
        self.lib.add_album([i])

    def tearDown(self):
        self.io.restore()

    def test_list_outputs_item(self):
        commands.list_items(self.lib, '', False)
        out = self.io.getoutput()
        self.assertTrue(u'the title' in out)

    def test_list_album_outputs_something(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertGreater(len(out), 0)

    def test_list_album_omits_title(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertTrue(u'the title' not in out)

    def test_list_uses_track_artist(self):
        commands.list_items(self.lib, '', False)
        out = self.io.getoutput()
        self.assertTrue(u'the artist' in out)
        self.assertTrue(u'the album artist' not in out)

    def test_list_album_uses_album_artist(self):
        commands.list_items(self.lib, '', True)
        out = self.io.getoutput()
        self.assertTrue(u'the artist' not in out)
        self.assertTrue(u'the album artist' in out)

class RemoveTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()

        self.libdir = os.path.join('rsrc', 'testlibdir')
        os.mkdir(self.libdir)

        # Copy a file into the library.
        self.lib = library.Library(':memory:', self.libdir)
        self.i = library.Item.from_path(os.path.join('rsrc', 'full.mp3'))
        self.lib.add(self.i, True)

    def tearDown(self):
        self.io.restore()
        shutil.rmtree(self.libdir)

    def test_remove_items_no_delete(self):
        self.io.addinput('y')
        commands.remove_items(self.lib, '', False, False)
        items = self.lib.items()
        self.assertEqual(len(list(items)), 0)
        self.assertTrue(os.path.exists(self.i.path))

    def test_remove_items_with_delete(self):
        self.io.addinput('y')
        commands.remove_items(self.lib, '', False, True)
        items = self.lib.items()
        self.assertEqual(len(list(items)), 0)
        self.assertFalse(os.path.exists(self.i.path))

class PrintTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def test_print_without_locale(self):
        lang = os.environ.get('LANG')
        if lang:
            del os.environ['LANG']

        try:
            ui.print_(u'something')
        except TypeError:
            self.fail('TypeError during print')
        finally:
            if lang:
                os.environ['LANG'] = lang

    def test_print_with_invalid_locale(self):
        old_lang = os.environ.get('LANG')
        os.environ['LANG'] = ''
        old_ctype = os.environ.get('LC_CTYPE')
        os.environ['LC_CTYPE'] = 'UTF-8'

        try:
            ui.print_(u'something')
        except ValueError:
            self.fail('ValueError during print')
        finally:
            if old_lang:
                os.environ['LANG'] = old_lang
            else:
                del os.environ['LANG']
            if old_ctype:
                os.environ['LC_CTYPE'] = old_ctype
            else:
                del os.environ['LC_CTYPE']

class AutotagTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def _no_candidates_test(self, result):
        res = commands.choose_match(
            'path',
            [_common.item()], # items
            'artist',
            'album',
            [], # candidates
            autotag.RECOMMEND_NONE,
            True, False, commands.CHOICE_SKIP
        )
        self.assertEqual(res, result)
        self.assertTrue('No match' in self.io.getoutput())

    def test_choose_match_with_no_candidates_skip(self):
        self.io.addinput('s')
        self._no_candidates_test(commands.CHOICE_SKIP)

    def test_choose_match_with_no_candidates_asis(self):
        self.io.addinput('u')
        self._no_candidates_test(commands.CHOICE_ASIS)

class InputTest(unittest.TestCase):
    def setUp(self):
        self.io = _common.DummyIO()
        self.io.install()
    def tearDown(self):
        self.io.restore()

    def test_manual_search_gets_unicode(self):
        self.io.addinput('\xc3\x82me')
        self.io.addinput('\xc3\x82me')
        artist, album = commands.manual_search()
        self.assertEqual(artist, u'\xc2me')
        self.assertEqual(album, u'\xc2me')

class ConfigTest(unittest.TestCase):
    def setUp(self):
        self.test_cmd = ui.Subcommand('test', help='test')
        commands.default_commands.append(self.test_cmd)
    def tearDown(self):
        commands.default_commands.pop()
    def _run_main(self, args, config, func):
        self.test_cmd.func = func
        ui.main(args + ['test'], StringIO(config))

    def test_paths_section_respected(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['x'], 'y')
        self._run_main([], textwrap.dedent("""
            [paths]
            x=y"""), func)

    def test_default_paths_preserved(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'],
                             ui.DEFAULT_PATH_FORMATS['default'])
        self._run_main([], textwrap.dedent("""
            [paths]
            x=y"""), func)

    def test_default_paths_overriden_by_legacy_path_format(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'], 'x')
            self.assertEqual(len(lib.path_formats), 1)
        self._run_main([], textwrap.dedent("""
            [beets]
            path_format=x"""), func)

    def test_paths_section_overriden_by_cli_switch(self):
        def func(lib, config, opts, args):
            self.assertEqual(lib.path_formats['default'], 'z')
            self.assertEqual(len(lib.path_formats), 1)
        self._run_main(['-p', 'z'], textwrap.dedent("""
            [paths]
            x=y"""), func)

def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<|endoftext|>


<commit_before>from django.db import models
from django.db.models.signals import post_save, post_delete

from wagtail.wagtailcore.models import Page

from wagtail.contrib.wagtailfrontendcache.utils import purge_page_from_cache

def post_save_signal_handler(instance, **kwargs):
    purge_page_from_cache(instance)

def post_delete_signal_handler(instance, **kwargs):
    purge_page_from_cache(instance)

def register_signal_handlers():
    # Get list of models that are page types
    indexed_models = [model for model in models.get_models() if issubclass(model, Page)]

    # Loop through list and register signal handlers for each one
    for model in indexed_models:
        post_save.connect(post_save_signal_handler, sender=model)
        post_delete.connect(post_delete_signal_handler, sender=model)
<commit_msg>Use page_published signal for cache invalidation instead of post_save<commit_after>from django.db import models
from django.db.models.signals import post_delete

from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore.signals import page_published

from wagtail.contrib.wagtailfrontendcache.utils import purge_page_from_cache

def page_published_signal_handler(instance, **kwargs):
    purge_page_from_cache(instance)

def post_delete_signal_handler(instance, **kwargs):
    purge_page_from_cache(instance)

def register_signal_handlers():
    # Get list of models that are page types
    indexed_models = [model for model in models.get_models() if issubclass(model, Page)]

    # Loop through list and register signal handlers for each one
    for model in indexed_models:
        page_published.connect(page_published_signal_handler, sender=model)
        post_delete.connect(post_delete_signal_handler, sender=model)
<|endoftext|>