<commit_before># Copyright 2009 Wayne See
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import nested_scopes
from pychmlib.chm import chm

import server
import appuifw
import e32
import chm_filebrowser
import os
import e32dbm
import hhc

CONF_FILE = u"E:\\Data\\chompy\\chompy.cfg"
INIT_FILE = u"E:\\Data\\chompy\\tmp.html"
SEPARATOR = u"/"

INIT_HTML = u"""<html>
<body>
<script type="text/javascript">
document.location = "http://localhost:""" + unicode(server.PORT) + """/%s"
</script>
</body>
</html>
"""

if not os.path.exists("E:\\Data\\chompy"):
    os.makedirs("E:\\Data\\chompy")

class Chompy:

    def __init__(self):
        self.app_lock = e32.Ao_lock()
        self.fb = chm_filebrowser.Filebrowser()
        self.load_recent()
        self.hhc_callback = e32.ao_callgate(self.load_hhc_viewer)

    def load_recent(self):
        try:
            db = e32dbm.open(CONF_FILE, "c")
            recent = db["recent"]
            if recent:
                self.recent = recent.split(SEPARATOR)
            else:
                self.recent = []
            db.close()
        except:
            self.recent = []

    def save_recent(self):
        db = e32dbm.open(CONF_FILE, "wf")
        try:
            db["recent"] = SEPARATOR.join(self.recent)
        finally:
            db.close()

    def browse(self):
        self.fb.show()
        selected = self.fb.selected
        if selected:
            file = unicode(selected)
            if file not in self.recent:
                self.recent.append(file)
                self.update_list(len(self.recent) - 1)
            self.open(file)
        else:
            self.refresh()

    def to_display(self, filename):
        return unicode(os.path.basename(filename))

    def update_list(self, selected_index=None):
        if self.recent:
            self.lb.set_list(self.get_list(), selected_index)
        else:
            self.lb.set_list(self.get_list())

    def get_list(self):
        if self.recent:
            return map(self.to_display, self.recent)
        else:
            return [u"Select file"]

    def lb_observe(self, index=None):
        if index is None:
            index = self.lb.current()
        if not self.recent:
            self.browse()
        else:
            self.open(self.recent[index])

    def open(self, filename=None):
        if filename is None:
            filename = self.recent[self.lb.current()]
        server.start(filename, self.hhc_callback)
        self.stall()

    def load_hhc_viewer(self, filename=None, contents=None, encoding=None, error=None):
        if not error:
            viewer = HHCViewer(filename, contents, encoding)
            viewer.show()
            server.stop() #if there is an error, no need to stop server
        else:
            if error == server.ERR_INVALID_CHM:
                appuifw.note(u"CHM File cannot be read", "error")
            elif error == server.ERR_NO_HHC:
                appuifw.note(u"CHM File contains no HHC file", "error")
        self.refresh()

    def remove(self):
        index = self.lb.current()
        del self.recent[index]
        self.update_list(index)

    def quit(self):
        self.save_recent()
        self.app_lock.signal()

    def stall(self):
        appuifw.app.menu = []
        appuifw.app.title = u"Loading..."
        text = appuifw.Text()
        text.style = appuifw.STYLE_ITALIC
        text.set(u"Please wait while CHM file is being read...")
        appuifw.app.body = text
        appuifw.app.exit_key_handler = None

    def refresh(self):
        menu_list = [(u"Browse for file", self.browse), (u"Exit", self.quit)]
        if self.recent:
            menu_list.insert(0, (u"Open", self.open))
            menu_list.insert(2, (u"Remove", self.remove))
        appuifw.app.menu = menu_list
        appuifw.app.exit_key_handler = self.quit
        appuifw.app.title = u"chompy"
        appuifw.app.body = self.lb

    def show(self):
        self.lb = appuifw.Listbox(self.get_list(), self.lb_observe)
        self.refresh()
        self.app_lock.wait()
        self.lb = None
        self.hhc_callback = None
        self.fb = None
        appuifw.app.body = None
        appuifw.app.exit_key_handler = None

class HHCViewer:

    def __init__(self, filename, hhc_obj, encoding):
        self.title = os.path.basename(filename)
        self.current_context = hhc_obj
        self.encoding = encoding
        self.app_lock = e32.Ao_lock()

    def to_displayable_list(self):
        entries = map(lambda x: x.name.decode(self.encoding), self.current_context.children)
        if not self.current_context.is_root:
            entries.insert(0, u"..")
        return entries

    def lb_observe(self, index=None):
        if index is None:
            index = self.lb.current()
        if index == 0 and not self.current_context.is_root:
            #go back up
            selected = self.current_context.parent
        else:
            selected_index = index
            if not self.current_context.is_root:
                selected_index -= 1
            selected = self.current_context.children[selected_index]
        if selected.is_inner_node:
            self.current_context = selected
            entries = self.to_displayable_list()
            self.lb.set_list(entries)
        else:
            self.load_in_viewer(selected)

    def load_in_viewer(self, entry):
        f = file(INIT_FILE, "w")
        f.write(INIT_HTML % entry.local)
        f.close()
        lock = e32.Ao_lock()
        viewer = appuifw.Content_handler(lock.signal)
        viewer.open(INIT_FILE)
        lock.wait()

    def quit(self):
        appuifw.app.exit_key_handler = None
        self.app_lock.signal()

    def open(self):
        self.lb_observe()

    def refresh(self):
        appuifw.app.menu = [(u"Open", self.open), (u"Exit", self.quit)]
        appuifw.app.exit_key_handler = self.quit
        appuifw.app.title = self.title
        appuifw.app.body = self.lb

    def show(self):
        entries = self.to_displayable_list()
        self.lb = appuifw.Listbox(entries, self.lb_observe)
        self.refresh()
        self.app_lock.wait()
        self.lb = None
        appuifw.app.body = None

if __name__ == '__main__':
    Chompy().show()<commit_msg>disabled exiting when program is stalled<commit_after># Copyright 2009 Wayne See
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import nested_scopes
from pychmlib.chm import chm

import server
import appuifw
import e32
import chm_filebrowser
import os
import e32dbm
import hhc

CONF_FILE = u"E:\\Data\\chompy\\chompy.cfg"
INIT_FILE = u"E:\\Data\\chompy\\tmp.html"
SEPARATOR = u"/"

INIT_HTML = u"""<html>
<body>
<script type="text/javascript">
document.location = "http://localhost:""" + unicode(server.PORT) + """/%s"
</script>
</body>
</html>
"""

if not os.path.exists("E:\\Data\\chompy"):
    os.makedirs("E:\\Data\\chompy")

class Chompy:

    def __init__(self):
        self.app_lock = e32.Ao_lock()
        self.fb = chm_filebrowser.Filebrowser()
        self.load_recent()
        self.hhc_callback = e32.ao_callgate(self.load_hhc_viewer)

    def load_recent(self):
        try:
            db = e32dbm.open(CONF_FILE, "c")
            recent = db["recent"]
            if recent:
                self.recent = recent.split(SEPARATOR)
            else:
                self.recent = []
            db.close()
        except:
            self.recent = []

    def save_recent(self):
        db = e32dbm.open(CONF_FILE, "wf")
        try:
            db["recent"] = SEPARATOR.join(self.recent)
        finally:
            db.close()

    def browse(self):
        self.fb.show()
        selected = self.fb.selected
        if selected:
            file = unicode(selected)
            if file not in self.recent:
                self.recent.append(file)
                self.update_list(len(self.recent) - 1)
            self.open(file)
        else:
            self.refresh()

    def to_display(self, filename):
        return unicode(os.path.basename(filename))

    def update_list(self, selected_index=None):
        if self.recent:
            self.lb.set_list(self.get_list(), selected_index)
        else:
            self.lb.set_list(self.get_list())

    def get_list(self):
        if self.recent:
            return map(self.to_display, self.recent)
        else:
            return [u"Select file"]

    def lb_observe(self, index=None):
        if index is None:
            index = self.lb.current()
        if not self.recent:
            self.browse()
        else:
            self.open(self.recent[index])

    def open(self, filename=None):
        if filename is None:
            filename = self.recent[self.lb.current()]
        server.start(filename, self.hhc_callback)
        self.stall()

    def load_hhc_viewer(self, filename=None, contents=None, encoding=None, error=None):
        if not error:
            viewer = HHCViewer(filename, contents, encoding)
            viewer.show()
            server.stop() #if there is an error, no need to stop server
        else:
            if error == server.ERR_INVALID_CHM:
                appuifw.note(u"CHM File cannot be read", "error")
            elif error == server.ERR_NO_HHC:
                appuifw.note(u"CHM File contains no HHC file", "error")
        self.refresh()

    def remove(self):
        index = self.lb.current()
        del self.recent[index]
        self.update_list(index)

    def quit(self):
        self.save_recent()
        self.app_lock.signal()

    def stall(self):
        appuifw.app.menu = []
        appuifw.app.title = u"Loading..."
        text = appuifw.Text()
        text.style = appuifw.STYLE_ITALIC
        text.set(u"Please wait while CHM file is being read...")
        appuifw.app.body = text
        appuifw.app.exit_key_handler = self.stop_quit

    def stop_quit(self):
        appuifw.note(u"Cannot exit until process has finished", "info")

    def refresh(self):
        menu_list = [(u"Browse for file", self.browse), (u"Exit", self.quit)]
        if self.recent:
            menu_list.insert(0, (u"Open", self.open))
            menu_list.insert(2, (u"Remove", self.remove))
        appuifw.app.menu = menu_list
        appuifw.app.exit_key_handler = self.quit
        appuifw.app.title = u"chompy"
        appuifw.app.body = self.lb

    def show(self):
        self.lb = appuifw.Listbox(self.get_list(), self.lb_observe)
        self.refresh()
        self.app_lock.wait()
        self.lb = None
        self.hhc_callback = None
        self.fb = None
        appuifw.app.body = None
        appuifw.app.exit_key_handler = None

class HHCViewer:

    def __init__(self, filename, hhc_obj, encoding):
        self.title = os.path.basename(filename)
        self.current_context = hhc_obj
        self.encoding = encoding
        self.app_lock = e32.Ao_lock()

    def to_displayable_list(self):
        entries = map(lambda x: x.name.decode(self.encoding), self.current_context.children)
        if not self.current_context.is_root:
            entries.insert(0, u"..")
        return entries

    def lb_observe(self, index=None):
        if index is None:
            index = self.lb.current()
        if index == 0 and not self.current_context.is_root:
            #go back up
            selected = self.current_context.parent
        else:
            selected_index = index
            if not self.current_context.is_root:
                selected_index -= 1
            selected = self.current_context.children[selected_index]
        if selected.is_inner_node:
            self.current_context = selected
            entries = self.to_displayable_list()
            self.lb.set_list(entries)
        else:
            self.load_in_viewer(selected)

    def load_in_viewer(self, entry):
        f = file(INIT_FILE, "w")
        f.write(INIT_HTML % entry.local)
        f.close()
        lock = e32.Ao_lock()
        viewer = appuifw.Content_handler(lock.signal)
        viewer.open(INIT_FILE)
        lock.wait()

    def quit(self):
        appuifw.app.exit_key_handler = None
        self.app_lock.signal()

    def open(self):
        self.lb_observe()

    def refresh(self):
        appuifw.app.menu = [(u"Open", self.open), (u"Exit", self.quit)]
        appuifw.app.exit_key_handler = self.quit
        appuifw.app.title = self.title
        appuifw.app.body = self.lb

    def show(self):
        entries = self.to_displayable_list()
        self.lb = appuifw.Listbox(entries, self.lb_observe)
        self.refresh()
        self.app_lock.wait()
        self.lb = None
        appuifw.app.body = None

if __name__ == '__main__':
    Chompy().show()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ('zerver', '0126_prereg_remove_users_without_realm'),
    ]

    operations = [
        # There was a migration here, which wasn't ready for wide deployment
        # and was backed out.  This placeholder is left behind to avoid
        # confusing the migration engine on any installs that applied the
        # migration.  (Fortunately no reverse migration is needed.)
    ]
<commit_msg>migrations: Fix mypy error in placeholder migration.<commit_after># -*- coding: utf-8 -*-
from typing import Any, List

from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ('zerver', '0126_prereg_remove_users_without_realm'),
    ]

    operations = [
        # There was a migration here, which wasn't ready for wide deployment
        # and was backed out.  This placeholder is left behind to avoid
        # confusing the migration engine on any installs that applied the
        # migration.  (Fortunately no reverse migration is needed.)
    ]  # type: List[Any]
<|endoftext|>


<commit_before>import base64
import os
basedir = os.path.abspath(os.path.dirname(__file__))

def base64ToJGP(base64Image, name):
    imageLen = len(base64Image)
    image = base64Image[22:imageLen]
    imgdata = base64.b64decode(image)
    path = '/static/images'+name+'.jpg'
    fullpath = basedir + path
    with open(fullpath, 'wb') as f:
        f.write(imgdata)
        return path
    return 'failed the image convertion'<commit_msg>Fixed function when base64 its out of padding<commit_after>import base64
import os
basedir = os.path.abspath(os.path.dirname(__file__))

def base64ToJGP(base64Image, name):
	base64Image += "=" * ((4 - len(base64Image) % 4) % 4)
	imageLen = len(base64Image)
	image = base64Image[22:imageLen]
	imgdata = base64.b64decode(image)
	path = '/static/images'+name+'.jpg'
	fullpath = basedir + path
	with open(fullpath, 'wb') as f:
		f.write(imgdata)
		return path
	return 'failed the image convertion'<|endoftext|>


<commit_before>""" ImageKit image specifications

All imagekit specifications must inherit from the ImageSpec class. Models
inheriting from ImageModel will be modified with a descriptor/accessor for each
spec found.

"""
import os
from StringIO import StringIO
from imagekit import processors
from imagekit.lib import *
from imagekit.utils import img_to_fobj
from django.core.files.base import ContentFile

class ImageSpec(object):
    pre_cache = False
    quality = 70
    increment_count = False
    processors = []

    @classmethod
    def name(cls):
        return getattr(cls, 'access_as', cls.__name__.lower())

    @classmethod
    def process(cls, image, obj):
        fmt = image.format
        img = image.copy()
        for proc in cls.processors:
            img, fmt = proc.process(img, fmt, obj)
        img.format = fmt
        return img, fmt

class Accessor(object):
    def __init__(self, obj, spec):
        self._img = None
        self._fmt = None
        self._obj = obj
        self.spec = spec

    def _get_imgfile(self):
        format = self._img.format or 'JPEG'
        if format != 'JPEG':
            imgfile = img_to_fobj(self._img, format)
        else:
            imgfile = img_to_fobj(self._img, format,
                                  quality=int(self.spec.quality),
                                  optimize=True)
        return imgfile

    def _create(self):
        if self._exists():
            return
        # process the original image file
        try:
            fp = self._obj._imgfield.storage.open(self._obj._imgfield.name)
        except IOError:
            return
        fp.seek(0)
        fp = StringIO(fp.read())
        self._img, self._fmt = self.spec.process(Image.open(fp), self._obj)
        # save the new image to the cache
        content = ContentFile(self._get_imgfile().read())
        self._obj._storage.save(self.name, content)

    def _delete(self):
        self._obj._storage.delete(self.name)

    def _exists(self):
        return self._obj._storage.exists(self.name)

    @property
    def name(self):
        filepath, basename = os.path.split(self._obj._imgfield.name)
        filename, extension = os.path.splitext(basename)
        for processor in self.spec.processors:
            if issubclass(processor, processors.Format):
                extension = processor.extension
        cache_filename = self._obj._ik.cache_filename_format % \
            {'filename': filename,
             'specname': self.spec.name(),
             'extension': extension.lstrip('.')}
        if callable(self._obj._ik.cache_dir):
            return self._obj._ik.cache_dir(self._obj, filepath,
                                           cache_filename)
        else:
            return os.path.join(self._obj._ik.cache_dir, filepath,
                                cache_filename)

    @property
    def url(self):
        self._create()
        if self.spec.increment_count:
            fieldname = self._obj._ik.save_count_as
            if fieldname is not None:
                current_count = getattr(self._obj, fieldname)
                setattr(self._obj, fieldname, current_count + 1)
                self._obj.save(clear_cache=False)
        return self._obj._storage.url(self.name)

    @property
    def file(self):
        self._create()
        return self._obj._storage.open(self.name)

    @property
    def image(self):
        if self._img is None:
            self._create()
            if self._img is None:
                self._img = Image.open(self.file)
        return self._img

    @property
    def width(self):
        return self.image.size[0]

    @property
    def height(self):
        return self.image.size[1]

class Descriptor(object):
    def __init__(self, spec):
        self._spec = spec

    def __get__(self, obj, type=None):
        return Accessor(obj, self._spec)
<commit_msg>Imagekit not longer uses the appends the upload_to path of the image when generating the cache filename. The new default format is [media_root]/[cache_dir]/[cache_filename].<commit_after>""" ImageKit image specifications

All imagekit specifications must inherit from the ImageSpec class. Models
inheriting from ImageModel will be modified with a descriptor/accessor for each
spec found.

"""
import os
from StringIO import StringIO
from imagekit import processors
from imagekit.lib import *
from imagekit.utils import img_to_fobj
from django.core.files.base import ContentFile

class ImageSpec(object):
    pre_cache = False
    quality = 70
    increment_count = False
    processors = []

    @classmethod
    def name(cls):
        return getattr(cls, 'access_as', cls.__name__.lower())

    @classmethod
    def process(cls, image, obj):
        fmt = image.format
        img = image.copy()
        for proc in cls.processors:
            img, fmt = proc.process(img, fmt, obj)
        img.format = fmt
        return img, fmt

class Accessor(object):
    def __init__(self, obj, spec):
        self._img = None
        self._fmt = None
        self._obj = obj
        self.spec = spec

    def _get_imgfile(self):
        format = self._img.format or 'JPEG'
        if format != 'JPEG':
            imgfile = img_to_fobj(self._img, format)
        else:
            imgfile = img_to_fobj(self._img, format,
                                  quality=int(self.spec.quality),
                                  optimize=True)
        return imgfile

    def _create(self):
        if self._exists():
            return
        # process the original image file
        try:
            fp = self._obj._imgfield.storage.open(self._obj._imgfield.name)
        except IOError:
            return
        fp.seek(0)
        fp = StringIO(fp.read())
        self._img, self._fmt = self.spec.process(Image.open(fp), self._obj)
        # save the new image to the cache
        content = ContentFile(self._get_imgfile().read())
        self._obj._storage.save(self.name, content)

    def _delete(self):
        self._obj._storage.delete(self.name)

    def _exists(self):
        return self._obj._storage.exists(self.name)

    @property
    def name(self):
        filepath, basename = os.path.split(self._obj._imgfield.name)
        filename, extension = os.path.splitext(basename)
        for processor in self.spec.processors:
            if issubclass(processor, processors.Format):
                extension = processor.extension
        cache_filename = self._obj._ik.cache_filename_format % \
            {'filename': filename,
             'specname': self.spec.name(),
             'extension': extension.lstrip('.')}
        if callable(self._obj._ik.cache_dir):
            return self._obj._ik.cache_dir(self._obj, filepath,
                                           cache_filename)
        else:
            return os.path.join(self._obj._ik.cache_dir, cache_filename)

    @property
    def url(self):
        self._create()
        if self.spec.increment_count:
            fieldname = self._obj._ik.save_count_as
            if fieldname is not None:
                current_count = getattr(self._obj, fieldname)
                setattr(self._obj, fieldname, current_count + 1)
                self._obj.save(clear_cache=False)
        return self._obj._storage.url(self.name)

    @property
    def file(self):
        self._create()
        return self._obj._storage.open(self.name)

    @property
    def image(self):
        if self._img is None:
            self._create()
            if self._img is None:
                self._img = Image.open(self.file)
        return self._img

    @property
    def width(self):
        return self.image.size[0]

    @property
    def height(self):
        return self.image.size[1]

class Descriptor(object):
    def __init__(self, spec):
        self._spec = spec

    def __get__(self, obj, type=None):
        return Accessor(obj, self._spec)
<|endoftext|>


<commit_before>import os
import mimetypes
import sys
import types

from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import ContentFile
from django.db.models.loading import cache
from django.utils.encoding import smart_str, smart_unicode
from django.utils.functional import wraps
from django.utils.importlib import import_module

from .exceptions import UnknownExtensionError, UnknownFormatError
from .lib import Image, ImageFile, StringIO

RGBA_TRANSPARENCY_FORMATS = ['PNG']
PALETTE_TRANSPARENCY_FORMATS = ['PNG', 'GIF']

class IKContentFile(ContentFile):
    """
    Wraps a ContentFile in a file-like object with a filename and a
    content_type. A PIL image format can be optionally be provided as a content
    type hint.

    """
    def __init__(self, filename, content, format=None):
        self.file = ContentFile(content)
        self.file.name = filename
        mimetype = getattr(self.file, 'content_type', None)
        if format and not mimetype:
            mimetype = format_to_mimetype(format)
        if not mimetype:
            ext = os.path.splitext(filename or '')[1]
            mimetype = extension_to_mimetype(ext)
        self.file.content_type = mimetype

    def __str__(self):
        return smart_str(self.file.name or '')

    def __unicode__(self):
        return smart_unicode(self.file.name or u'')

def img_to_fobj(img, format, autoconvert=True, **options):
    return save_image(img, StringIO(), format, options, autoconvert)

def get_spec_files(instance):
    try:
        return instance._ik.spec_files
    except AttributeError:
        return []

def open_image(target):
    target.seek(0)
    img = Image.open(target)
    img.copy = types.MethodType(_wrap_copy(img.copy), img, img.__class__)
    return img

def _wrap_copy(f):
    @wraps(f)
    def copy(self):
        img = f()
        try:
            img.app = self.app
        except AttributeError:
            pass
        try:
            img._getexif = self._getexif
        except AttributeError:
            pass
        return img
    return copy

_pil_init = 0

def _preinit_pil():
    """Loads the standard PIL file format drivers. Returns True if ``preinit()``
    was called (and there's a potential that more drivers were loaded) or False
    if there is no possibility that new drivers were loaded.

    """
    global _pil_init
    if _pil_init < 1:
        Image.preinit()
        _pil_init = 1
        return True
    return False

def _init_pil():
    """Loads all PIL file format drivers. Returns True if ``init()`` was called
    (and there's a potential that more drivers were loaded) or False if there is
    no possibility that new drivers were loaded.

    """
    global _pil_init
    _preinit_pil()
    if _pil_init < 2:
        Image.init()
        _pil_init = 2
        return True
    return False

def _extension_to_format(extension):
    return Image.EXTENSION.get(extension.lower())

def _format_to_extension(format):
    if format:
        for k, v in Image.EXTENSION.iteritems():
            if v == format.upper():
                return k
    return None

def extension_to_mimetype(ext):
    try:
        filename = 'a%s' % (ext or '')  # guess_type requires a full filename, not just an extension
        mimetype = mimetypes.guess_type(filename)[0]
    except IndexError:
        mimetype = None
    return mimetype

def format_to_mimetype(format):
    return extension_to_mimetype(format_to_extension(format))

def extension_to_format(extension):
    """Returns the format that corresponds to the provided extension.

    """
    format = _extension_to_format(extension)
    if not format and _preinit_pil():
        format = _extension_to_format(extension)
    if not format and _init_pil():
        format = _extension_to_format(extension)
    if not format:
        raise UnknownExtensionError(extension)
    return format

def format_to_extension(format):
    """Returns the first extension that matches the provided format.

    """
    extension = None
    if format:
        extension = _format_to_extension(format)
        if not extension and _preinit_pil():
            extension = _format_to_extension(format)
        if not extension and _init_pil():
            extension = _format_to_extension(format)
    if not extension:
        raise UnknownFormatError(format)
    return extension

def _get_models(apps):
    models = []
    for app_label in apps or []:
        app = cache.get_app(app_label)
        models += [m for m in cache.get_models(app)]
    return models

def invalidate_app_cache(apps):
    for model in _get_models(apps):
        print 'Invalidating cache for "%s.%s"' % (model._meta.app_label, model.__name__)
        for obj in model._default_manager.order_by('-pk'):
            for f in get_spec_files(obj):
                f.invalidate()

def validate_app_cache(apps, force_revalidation=False):
    for model in _get_models(apps):
        for obj in model._default_manager.order_by('-pk'):
            model_name = '%s.%s' % (model._meta.app_label, model.__name__)
            if force_revalidation:
                print 'Invalidating & validating cache for "%s"' % model_name
            else:
                print 'Validating cache for "%s"' % model_name
            for f in get_spec_files(obj):
                if force_revalidation:
                    f.invalidate()
                f.validate()

def suggest_extension(name, format):
    original_extension = os.path.splitext(name)[1]
    try:
        suggested_extension = format_to_extension(format)
    except UnknownFormatError:
        extension = original_extension
    else:
        if suggested_extension.lower() == original_extension.lower():
            extension = original_extension
        else:
            try:
                original_format = extension_to_format(original_extension)
            except UnknownExtensionError:
                extension = suggested_extension
            else:
                # If the formats match, give precedence to the original extension.
                if format.lower() == original_format.lower():
                    extension = original_extension
                else:
                    extension = suggested_extension
    return extension

def save_image(img, outfile, format, options=None, autoconvert=True):
    """
    Wraps PIL's ``Image.save()`` method. There are two main benefits of using
    this function over PIL's:

    1. It gracefully handles the infamous "Suspension not allowed here" errors.
    2. It prepares the image for saving using ``prepare_image()``, which will do
        some common-sense processing given the target format.

    """
    options = options or {}

    if autoconvert:
        img, save_kwargs = prepare_image(img, format)
        options = dict(save_kwargs.items() + options.items())

    # Attempt to reset the file pointer.
    try:
        outfile.seek(0)
    except AttributeError:
        pass

    try:
        with quiet():
            img.save(outfile, format, **options)
    except IOError:
        # PIL can have problems saving large JPEGs if MAXBLOCK isn't big enough,
        # So if we have a problem saving, we temporarily increase it. See
        # http://github.com/jdriscoll/django-imagekit/issues/50
        old_maxblock = ImageFile.MAXBLOCK
        ImageFile.MAXBLOCK = img.size[0] * img.size[1]
        try:
            img.save(outfile, format, **options)
        finally:
            ImageFile.MAXBLOCK = old_maxblock

    try:
        outfile.seek(0)
    except AttributeError:
        pass

    return outfile

class quiet(object):
    """
    A context manager for suppressing the stderr activity of PIL's C libraries.
    Based on http://stackoverflow.com/a/978264/155370

    """
    def __enter__(self):
        self.stderr_fd = sys.__stderr__.fileno()
        self.null_fd = os.open(os.devnull, os.O_RDWR)
        self.old = os.dup(self.stderr_fd)
        os.dup2(self.null_fd, self.stderr_fd)

    def __exit__(self, *args, **kwargs):
        os.dup2(self.old, self.stderr_fd)
        os.close(self.null_fd)
        os.close(self.old)

def prepare_image(img, format):
    """
    Prepares the image for saving to the provided format by doing some
    common-sense conversions. This includes things like preserving transparency
    and quantizing. This function is used automatically by ``save_image()``
    (and classes like ``ImageSpecField`` and ``ProcessedImageField``)
    immediately before saving unless you specify ``autoconvert=False``. It is
    provided as a utility for those doing their own processing.

    :param img: The image to prepare for saving.
    :param format: The format that the image will be saved to.

    """
    matte = False
    save_kwargs = {}

    if img.mode == 'RGBA':
        if format in RGBA_TRANSPARENCY_FORMATS:
            pass
        elif format in PALETTE_TRANSPARENCY_FORMATS:
            # If you're going from a format with alpha transparency to one
            # with palette transparency, transparency values will be
            # snapped: pixels that are more opaque than not will become
            # fully opaque; pixels that are more transparent than not will
            # become fully transparent. This will not produce a good-looking
            # result if your image contains varying levels of opacity; in
            # that case, you'll probably want to use a processor to matte
            # the image on a solid color. The reason we don't matte by
            # default is because not doing so allows processors to treat
            # RGBA-format images as a super-type of P-format images: if you
            # have an RGBA-format image with only a single transparent
            # color, and save it as a GIF, it will retain its transparency.
            # In other words, a P-format image converted to an
            # RGBA-formatted image by a processor and then saved as a
            # P-format image will give the expected results.

            # Work around a bug in PIL: split() doesn't check to see if
            # img is loaded.
            img.load()

            alpha = img.split()[-1]
            mask = Image.eval(alpha, lambda a: 255 if a <= 128 else 0)
            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE,
                    colors=255)
            img.paste(255, mask)
            save_kwargs['transparency'] = 255
        else:
            # Simply converting an RGBA-format image to an RGB one creates a
            # gross result, so we matte the image on a white background. If
            # that's not what you want, that's fine: use a processor to deal
            # with the transparency however you want. This is simply a
            # sensible default that will always produce something that looks
            # good. Or at least, it will look better than just a straight
            # conversion.
            matte = True
    elif img.mode == 'P':
        if format in PALETTE_TRANSPARENCY_FORMATS:
            try:
                save_kwargs['transparency'] = img.info['transparency']
            except KeyError:
                pass
        elif format in RGBA_TRANSPARENCY_FORMATS:
            # Currently PIL doesn't support any RGBA-mode formats that
            # aren't also P-mode formats, so this will never happen.
            img = img.convert('RGBA')
        else:
            matte = True
    else:
        img = img.convert('RGB')

        # GIFs are always going to be in palette mode, so we can do a little
        # optimization. Note that the RGBA sources also use adaptive
        # quantization (above). Images that are already in P mode don't need
        # any quantization because their colors are already limited.
        if format == 'GIF':
            img = img.convert('P', palette=Image.ADAPTIVE)

    if matte:
        img = img.convert('RGBA')
        bg = Image.new('RGBA', img.size, (255, 255, 255))
        bg.paste(img, img)
        img = bg.convert('RGB')

    if format == 'JPEG':
        save_kwargs['optimize'] = True

    return img, save_kwargs

def get_class(path, desc):
    try:
        dot = path.rindex('.')
    except ValueError:
        raise ImproperlyConfigured("%s isn't a %s module." % (path, desc))
    module, classname = path[:dot], path[dot + 1:]
    try:
        mod = import_module(module)
    except ImportError, e:
        raise ImproperlyConfigured('Error importing %s module %s: "%s"' %
                (desc, module, e))
    try:
        cls = getattr(mod, classname)
        return cls
    except AttributeError:
        raise ImproperlyConfigured('%s module "%s" does not define a "%s"'
                ' class.' % (desc[0].upper() + desc[1:], module, classname))

_singletons = {}

def get_singleton(class_path, desc):
    global _singletons
    cls = get_class(class_path, desc)
    instance = _singletons.get(cls)
    if not instance:
        instance = _singletons[cls] = cls()
    return instance

def autodiscover():
    """
    Auto-discover INSTALLED_APPS imagespecs.py modules and fail silently when
    not present. This forces an import on them to register any admin bits they
    may want.

    Copied from django.contrib.admin
    """

    import copy
    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule
    from .templatetags import imagekit_tags

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            import_module('%s.imagespecs' % app)
        except:
            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'imagespecs'):
                raise

class SpecWrapper(object):
    """
    Wraps a user-defined spec object so we can access properties that don't
    exist without errors.

    """
    def __init__(self, spec):
        self.processors = getattr(spec, 'processors', None)
        self.format = getattr(spec, 'format', None)
        self.options = getattr(spec, 'options', None)
        self.autoconvert = getattr(spec, 'autoconvert', True)
        self.storage = getattr(spec, 'storage', None)
        self.image_cache_backend = getattr(spec, 'image_cache_backend', None) \
                or get_default_image_cache_backend()
<commit_msg>Fix circular import utils > imagecache.backends<commit_after>import os
import mimetypes
import sys
import types

from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import ContentFile
from django.db.models.loading import cache
from django.utils.encoding import smart_str, smart_unicode
from django.utils.functional import wraps
from django.utils.importlib import import_module

from .exceptions import UnknownExtensionError, UnknownFormatError
from .lib import Image, ImageFile, StringIO

RGBA_TRANSPARENCY_FORMATS = ['PNG']
PALETTE_TRANSPARENCY_FORMATS = ['PNG', 'GIF']

class IKContentFile(ContentFile):
    """
    Wraps a ContentFile in a file-like object with a filename and a
    content_type. A PIL image format can be optionally be provided as a content
    type hint.

    """
    def __init__(self, filename, content, format=None):
        self.file = ContentFile(content)
        self.file.name = filename
        mimetype = getattr(self.file, 'content_type', None)
        if format and not mimetype:
            mimetype = format_to_mimetype(format)
        if not mimetype:
            ext = os.path.splitext(filename or '')[1]
            mimetype = extension_to_mimetype(ext)
        self.file.content_type = mimetype

    def __str__(self):
        return smart_str(self.file.name or '')

    def __unicode__(self):
        return smart_unicode(self.file.name or u'')

def img_to_fobj(img, format, autoconvert=True, **options):
    return save_image(img, StringIO(), format, options, autoconvert)

def get_spec_files(instance):
    try:
        return instance._ik.spec_files
    except AttributeError:
        return []

def open_image(target):
    target.seek(0)
    img = Image.open(target)
    img.copy = types.MethodType(_wrap_copy(img.copy), img, img.__class__)
    return img

def _wrap_copy(f):
    @wraps(f)
    def copy(self):
        img = f()
        try:
            img.app = self.app
        except AttributeError:
            pass
        try:
            img._getexif = self._getexif
        except AttributeError:
            pass
        return img
    return copy

_pil_init = 0

def _preinit_pil():
    """Loads the standard PIL file format drivers. Returns True if ``preinit()``
    was called (and there's a potential that more drivers were loaded) or False
    if there is no possibility that new drivers were loaded.

    """
    global _pil_init
    if _pil_init < 1:
        Image.preinit()
        _pil_init = 1
        return True
    return False

def _init_pil():
    """Loads all PIL file format drivers. Returns True if ``init()`` was called
    (and there's a potential that more drivers were loaded) or False if there is
    no possibility that new drivers were loaded.

    """
    global _pil_init
    _preinit_pil()
    if _pil_init < 2:
        Image.init()
        _pil_init = 2
        return True
    return False

def _extension_to_format(extension):
    return Image.EXTENSION.get(extension.lower())

def _format_to_extension(format):
    if format:
        for k, v in Image.EXTENSION.iteritems():
            if v == format.upper():
                return k
    return None

def extension_to_mimetype(ext):
    try:
        filename = 'a%s' % (ext or '')  # guess_type requires a full filename, not just an extension
        mimetype = mimetypes.guess_type(filename)[0]
    except IndexError:
        mimetype = None
    return mimetype

def format_to_mimetype(format):
    return extension_to_mimetype(format_to_extension(format))

def extension_to_format(extension):
    """Returns the format that corresponds to the provided extension.

    """
    format = _extension_to_format(extension)
    if not format and _preinit_pil():
        format = _extension_to_format(extension)
    if not format and _init_pil():
        format = _extension_to_format(extension)
    if not format:
        raise UnknownExtensionError(extension)
    return format

def format_to_extension(format):
    """Returns the first extension that matches the provided format.

    """
    extension = None
    if format:
        extension = _format_to_extension(format)
        if not extension and _preinit_pil():
            extension = _format_to_extension(format)
        if not extension and _init_pil():
            extension = _format_to_extension(format)
    if not extension:
        raise UnknownFormatError(format)
    return extension

def _get_models(apps):
    models = []
    for app_label in apps or []:
        app = cache.get_app(app_label)
        models += [m for m in cache.get_models(app)]
    return models

def invalidate_app_cache(apps):
    for model in _get_models(apps):
        print 'Invalidating cache for "%s.%s"' % (model._meta.app_label, model.__name__)
        for obj in model._default_manager.order_by('-pk'):
            for f in get_spec_files(obj):
                f.invalidate()

def validate_app_cache(apps, force_revalidation=False):
    for model in _get_models(apps):
        for obj in model._default_manager.order_by('-pk'):
            model_name = '%s.%s' % (model._meta.app_label, model.__name__)
            if force_revalidation:
                print 'Invalidating & validating cache for "%s"' % model_name
            else:
                print 'Validating cache for "%s"' % model_name
            for f in get_spec_files(obj):
                if force_revalidation:
                    f.invalidate()
                f.validate()

def suggest_extension(name, format):
    original_extension = os.path.splitext(name)[1]
    try:
        suggested_extension = format_to_extension(format)
    except UnknownFormatError:
        extension = original_extension
    else:
        if suggested_extension.lower() == original_extension.lower():
            extension = original_extension
        else:
            try:
                original_format = extension_to_format(original_extension)
            except UnknownExtensionError:
                extension = suggested_extension
            else:
                # If the formats match, give precedence to the original extension.
                if format.lower() == original_format.lower():
                    extension = original_extension
                else:
                    extension = suggested_extension
    return extension

def save_image(img, outfile, format, options=None, autoconvert=True):
    """
    Wraps PIL's ``Image.save()`` method. There are two main benefits of using
    this function over PIL's:

    1. It gracefully handles the infamous "Suspension not allowed here" errors.
    2. It prepares the image for saving using ``prepare_image()``, which will do
        some common-sense processing given the target format.

    """
    options = options or {}

    if autoconvert:
        img, save_kwargs = prepare_image(img, format)
        options = dict(save_kwargs.items() + options.items())

    # Attempt to reset the file pointer.
    try:
        outfile.seek(0)
    except AttributeError:
        pass

    try:
        with quiet():
            img.save(outfile, format, **options)
    except IOError:
        # PIL can have problems saving large JPEGs if MAXBLOCK isn't big enough,
        # So if we have a problem saving, we temporarily increase it. See
        # http://github.com/jdriscoll/django-imagekit/issues/50
        old_maxblock = ImageFile.MAXBLOCK
        ImageFile.MAXBLOCK = img.size[0] * img.size[1]
        try:
            img.save(outfile, format, **options)
        finally:
            ImageFile.MAXBLOCK = old_maxblock

    try:
        outfile.seek(0)
    except AttributeError:
        pass

    return outfile

class quiet(object):
    """
    A context manager for suppressing the stderr activity of PIL's C libraries.
    Based on http://stackoverflow.com/a/978264/155370

    """
    def __enter__(self):
        self.stderr_fd = sys.__stderr__.fileno()
        self.null_fd = os.open(os.devnull, os.O_RDWR)
        self.old = os.dup(self.stderr_fd)
        os.dup2(self.null_fd, self.stderr_fd)

    def __exit__(self, *args, **kwargs):
        os.dup2(self.old, self.stderr_fd)
        os.close(self.null_fd)
        os.close(self.old)

def prepare_image(img, format):
    """
    Prepares the image for saving to the provided format by doing some
    common-sense conversions. This includes things like preserving transparency
    and quantizing. This function is used automatically by ``save_image()``
    (and classes like ``ImageSpecField`` and ``ProcessedImageField``)
    immediately before saving unless you specify ``autoconvert=False``. It is
    provided as a utility for those doing their own processing.

    :param img: The image to prepare for saving.
    :param format: The format that the image will be saved to.

    """
    matte = False
    save_kwargs = {}

    if img.mode == 'RGBA':
        if format in RGBA_TRANSPARENCY_FORMATS:
            pass
        elif format in PALETTE_TRANSPARENCY_FORMATS:
            # If you're going from a format with alpha transparency to one
            # with palette transparency, transparency values will be
            # snapped: pixels that are more opaque than not will become
            # fully opaque; pixels that are more transparent than not will
            # become fully transparent. This will not produce a good-looking
            # result if your image contains varying levels of opacity; in
            # that case, you'll probably want to use a processor to matte
            # the image on a solid color. The reason we don't matte by
            # default is because not doing so allows processors to treat
            # RGBA-format images as a super-type of P-format images: if you
            # have an RGBA-format image with only a single transparent
            # color, and save it as a GIF, it will retain its transparency.
            # In other words, a P-format image converted to an
            # RGBA-formatted image by a processor and then saved as a
            # P-format image will give the expected results.

            # Work around a bug in PIL: split() doesn't check to see if
            # img is loaded.
            img.load()

            alpha = img.split()[-1]
            mask = Image.eval(alpha, lambda a: 255 if a <= 128 else 0)
            img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE,
                    colors=255)
            img.paste(255, mask)
            save_kwargs['transparency'] = 255
        else:
            # Simply converting an RGBA-format image to an RGB one creates a
            # gross result, so we matte the image on a white background. If
            # that's not what you want, that's fine: use a processor to deal
            # with the transparency however you want. This is simply a
            # sensible default that will always produce something that looks
            # good. Or at least, it will look better than just a straight
            # conversion.
            matte = True
    elif img.mode == 'P':
        if format in PALETTE_TRANSPARENCY_FORMATS:
            try:
                save_kwargs['transparency'] = img.info['transparency']
            except KeyError:
                pass
        elif format in RGBA_TRANSPARENCY_FORMATS:
            # Currently PIL doesn't support any RGBA-mode formats that
            # aren't also P-mode formats, so this will never happen.
            img = img.convert('RGBA')
        else:
            matte = True
    else:
        img = img.convert('RGB')

        # GIFs are always going to be in palette mode, so we can do a little
        # optimization. Note that the RGBA sources also use adaptive
        # quantization (above). Images that are already in P mode don't need
        # any quantization because their colors are already limited.
        if format == 'GIF':
            img = img.convert('P', palette=Image.ADAPTIVE)

    if matte:
        img = img.convert('RGBA')
        bg = Image.new('RGBA', img.size, (255, 255, 255))
        bg.paste(img, img)
        img = bg.convert('RGB')

    if format == 'JPEG':
        save_kwargs['optimize'] = True

    return img, save_kwargs

def get_class(path, desc):
    try:
        dot = path.rindex('.')
    except ValueError:
        raise ImproperlyConfigured("%s isn't a %s module." % (path, desc))
    module, classname = path[:dot], path[dot + 1:]
    try:
        mod = import_module(module)
    except ImportError, e:
        raise ImproperlyConfigured('Error importing %s module %s: "%s"' %
                (desc, module, e))
    try:
        cls = getattr(mod, classname)
        return cls
    except AttributeError:
        raise ImproperlyConfigured('%s module "%s" does not define a "%s"'
                ' class.' % (desc[0].upper() + desc[1:], module, classname))

_singletons = {}

def get_singleton(class_path, desc):
    global _singletons
    cls = get_class(class_path, desc)
    instance = _singletons.get(cls)
    if not instance:
        instance = _singletons[cls] = cls()
    return instance

def autodiscover():
    """
    Auto-discover INSTALLED_APPS imagespecs.py modules and fail silently when
    not present. This forces an import on them to register any admin bits they
    may want.

    Copied from django.contrib.admin
    """

    import copy
    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule
    from .templatetags import imagekit_tags

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            import_module('%s.imagespecs' % app)
        except:
            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'imagespecs'):
                raise

class SpecWrapper(object):
    """
    Wraps a user-defined spec object so we can access properties that don't
    exist without errors.

    """
    def __init__(self, spec):
        self.processors = getattr(spec, 'processors', None)
        self.format = getattr(spec, 'format', None)
        self.options = getattr(spec, 'options', None)
        self.autoconvert = getattr(spec, 'autoconvert', True)
        self.storage = getattr(spec, 'storage', None)
        self.image_cache_backend = getattr(spec, 'image_cache_backend', None)

        if not self.image_cache_backend:
            from .imagecache.backends import get_default_image_cache_backend
            self.image_cache_backend = get_default_image_cache_backend()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""Loads a GeoTIFF file representing Landsat data into a triple store via
SPARQL."""

from argparse import ArgumentParser, FileType
from base64 import b64encode
from collections import namedtuple
from datetime import datetime
from io import StringIO, BytesIO
from itertools import islice, chain
import logging
import re
from urllib.parse import quote_plus

from osgeo import gdal
import numpy as np
import pytz
from scipy.misc import toimage
from screw_rdflib import (ConjunctiveGraph, Literal, Namespace, OWL, RDF, RDFS,
                          XSD, URIRef, BNode, Graph, OWL)

# RDF namespaces
GEO = Namespace('http://www.w3.org/2003/01/geo/wgs84_pos#')
LED = Namespace('http://www.example.org/ANU-LED#')
QB = Namespace('http://purl.org/linked-data/cube#')
SDMXC = Namespace('http://purl.org/linked-data/sdmx/2009/concept#')
SDMXD = Namespace('http://purl.org/linked-data/sdmx/2009/dimension#')
SDMXM = Namespace('http://purl.org/linked-data/sdmx/2009/measure#')
OGC = Namespace('http://www.opengis.net/ont/geosparql#')
# For parsing AGDC filenames
AGDC_RE = re.compile(
    r'^(?P<sat_id>[^_]+)_(?P<sensor_id>[^_]+)_(?P<prod_code>[^_]+)_'
    r'(?P<lon>[^_]+)_(?P<lat>[^_]+)_(?P<year>\d{4})-(?P<month>\d{2})-'
    r'(?P<day>\d{2})T(?P<hour>\d+)-(?P<minute>\d+)-(?P<second>\d+(\.\d+)?)'
    r'\.tif$'
)
# To select default graph in Jena
DEFAULT = 'urn:x-arq:DefaultGraph'
# Boilerplate turtles are the best turtles
BOILERPLATE_TURTLE = """
@prefix : <{LED}> .
@prefix rdf: <{RDF}> .
@prefix rdfs: <{RDFS}> .
@prefix xsd: <{XSD}> .
@prefix qb: <{QB}> .
@prefix sdmx-concept: <{SDMXC}> .
@prefix sdmx-dimension: <{SDMXD}> .
@prefix sdmx-measure: <{SDMXM}> .
@prefix geo: <{GEO}> .
@prefix owl: <{OWL}> .
@prefix ogc: <{OGC}> .
@prefix gcmd-platform: <http://geobrain.laits.gmu.edu/ontology/2004/11/gcmd-platform.owl#> .
@prefix gcmd-instrument: <http://geobrain.laits.gmu.edu/ontology/2004/11/gcmd-instrument.owl#> .

:landsatDSD a qb:DataStructureDefinition ;
    qb:componentProperty :instrumentComponent
                       , :positionComponent
                       , :satelliteComponent
                       , :timeComponent
                       , :dataComponent
                       , :etmBandComponent .

:landsatDS a owl:NamedIndividual, qb:DataSet ;
    rdfs:label "Landsat sensor data"@en ;
    rdfs:comment "Some data from LandSat, retrieved from AGDC"@en ;
    qb:structure :landsatDSD ;
    :instrument gcmd-instrument:SCANNER ;
    :satellite gcmd-platform:LANDSAT-7 .

:instrumentComponent a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:attribute :instrument .

:positionComponent a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:dimension :location .

:satelliteComponent a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:attribute :satellite .

:timeComponent a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:dimension sdmx-dimension:timePeriod .

:dataComponent a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:dimension :imageData .

:etmBandComponnet a owl:NamedIndividual , qb:ComponentSpecification ;
    qb:dimension :etmBand .

:etmBand a rdf:Property , qb:DimensionProperty;
    rdfs:label "LandSat ETM observation band"@en;
    rdfs:range xsd:integer .

:instrument a owl:ObjectProperty, qb:AttributeProperty ;
    rdfs:range gcmd-instrument:Instrument .

:satellite a owl:ObjectProperty, qb:AttributeProperty ;
    rdfs:range gcmd-platform:PLATFORM .
""".format(QB=QB, SDMXD=SDMXD, SDMXM=SDMXM, LED=LED, GEO=GEO, SDMXC=SDMXC,
           RDF=RDF, RDFS=RDFS, XSD=XSD, OWL=OWL, OGC=OGC)

def parse_agdc_fn(fn):
    """Parses a filename in the format used by the AGDC Landsat archive. For
    example, ``LS7_ETM_NBAR_149_-036_2012-02-10T23-50-47.650696.tif`` is a
    Landsat 7 observation in GeoTIFF format taken on Februrary 10 2012 at
    11:50pm (amongst other things).

    >>> fn = 'LS7_ETM_NBAR_149_-036_2012-02-10T23-50-47.650696.tif'
    >>> sorted_results = sorted(parse_agdc_fn(fn).items())
    >>> print('\\n'.join('{}: {}'.format(k, v) for k, v in sorted_results))
    datetime: 2012-02-10 23:50:47.650696+00:00
    lat: -36.0
    lon: 149.0
    prod_code: NBAR
    sat_id: LS7
    sensor_id: ETM

    :param string fn: Filename for observation.
    :return: Dictionary of extracted metdata from filename.
    """
    match = AGDC_RE.match(fn)
    if match is None:
        raise ValueError('Invalid AGDC filename: "{}"'.format(fn))
    info = match.groupdict()
    raw_sec = float(info['second'])
    int_sec = int(raw_sec)
    microsecond = int(1e6 * (raw_sec % 1))
    dt = pytz.datetime.datetime(
        year=int(info['year']), month=int(info['month']), day=int(info['day']),
        hour=int(info['hour']), minute=int(info['minute']),
        second=int_sec, microsecond=microsecond, tzinfo=pytz.utc
    )
    return {
        'lat': float(info['lat']), 'lon': float(info['lon']), 'datetime': dt,
        'prod_code': info['prod_code'], 'sensor_id': info['sensor_id'],
        'sat_id': info['sat_id']
    }

def slow(generator, suffix, interval=500, total=None):
    """Used to annotate slow generators. Will print progress every ``interval``
    yields."""
    tot_str = '/' + str(total) if total is not None else ''
    for idx, val in enumerate(generator):
        if idx % interval == 0:
            print('{}{} {}'.format(idx, tot_str, suffix))
        yield val

def tile_iterator(band, tile_size):
    """Iterator of (top left row, top left col, data) for valid tiles in band
    (starting at left side, rather than using some more sophisticated
    scheme)."""
    value_arr = band.ReadAsArray()
    valid_mask = band.GetMaskBand().ReadAsArray().astype('bool')
    for row in range(0, value_arr.shape[0], tile_size):
        for col in range(0, value_arr.shape[1], tile_size):
            if valid_mask[row:row+tile_size, col:col+tile_size].any():
                vals = value_arr[row:row+tile_size, col:col+tile_size]
                yield (row, col, vals)

def undo_transform(row, col, t):
    """Recover (lat, lon) pair from row and column of a GeoTIFF file with
    transform t. See GDAL GetGeoTransform docs for formula."""
    return (
        t[3] + col * t[4] + row * t[5],
        t[0] + col * t[1] + row * t[2]
    )

def array_to_png(array):
    """Turn a 2D array into a data: URI filled with PNG goodies :)"""
    assert array.ndim == 2
    im = toimage(array)
    fp = BytesIO()
    im.save(fp, format='png')
    data = b64encode(fp.getvalue()).decode('utf-8')
    return 'data:image/png;base64,' + data

def loc_triples(subj, prop, lat, lon):
    """Yield a bunch of triples indicating that something is at a given
    latitude and longitude. This is actually really painful because of
    blank nodes :("""
    loc_bnode = BNode()
    yield (subj, prop, loc_bnode)
    yield (loc_bnode, GEO.lat, Literal(lat, datatype=XSD.decimal))
    yield (loc_bnode, GEO.lon, Literal(lon, datatype=XSD.decimal))

def graph_for_band(band, band_num, tile_size, gt_meta, transform):
    """Process a single band in a GeoTIFF file."""
    it = tile_iterator(band, tile_size)
    for row, col, tile in slow(it, 'tiles processed', interval=10):
        start_lat, start_lon = undo_transform(row, col, transform)
        end_lat, end_lon = undo_transform(
            row + tile_size, col + tile_size, transform
        )
        # Bounding box for the tile, which we'll convert into WKT
        bbox_corners = [
            (start_lon, start_lat), (start_lon, end_lat),
            (end_lon, end_lat), (end_lon, start_lat)
        ]
        loc_wkt = Literal('POLYGON({0}, {1}, {2}, {3}, {0})'.format(
            *['{} {}'.format(lon, lat) for lat, lon in bbox_corners]
        ), datatype=OGC.wktLiteral)
        png_tile = URIRef(array_to_png(tile))
        ident_str = 'lat/{}/lon/{}/tile-size/{}/band/{}'.format(
            start_lat, start_lon, tile_size, band_num
        )
        ident = URIRef(LED['observation/' + ident_str])

        # First add data describing the accident
        yield from [
            (ident, RDF.type, QB.Observation),
            (ident, QB.dataSet, LED.landsatDS),
            (ident, OGC.asWKT, loc_wkt),
            (ident, LED.etmBand, Literal(band_num, datatype=XSD.integer)),
            (ident, SDMXD.timePeriod, Literal(gt_meta['datetime'])),
            (ident, LED.imageData, png_tile),
            (ident, LED.pixelWidth,
                Literal(tile.shape[1], datatype=XSD.integer)),
            (ident, LED.pixelHeight,
                Literal(tile.shape[0], datatype=XSD.integer)),
        ]

        # Yield the centre point
        yield from loc_triples(
            ident, LED.location,
            (end_lat + start_lat) / 2,
            (end_lon + start_lon) / 2
        )

        # Yield each of the corners
        for lon, lat in bbox_corners:
            yield from loc_triples(ident, LED.corner, lat, lon)

def build_graph(geotiff, tile_sizes):
    """Generator producing all observation triples for each band."""
    boilerplate_graph = Graph().parse(
        StringIO(BOILERPLATE_TURTLE), format='turtle'
    )
    yield from boilerplate_graph.triples((None, None, None))

    gt = geotiff.GetGeoTransform()
    filename, = geotiff.GetFileList()
    gt_meta = parse_agdc_fn(filename)

    for band_num in range(1, geotiff.RasterCount + 1):
        band = geotiff.GetRasterBand(band_num)
        for tile_size in tile_sizes:
            print('Processing band {}/{}, tile size {}'.format(
                band_num, geotiff.RasterCount, tile_size
            ))
            yield from graph_for_band(band, band_num, tile_size, gt_meta, gt)

def iterchunk(iterator, n):
    """Chunk iterator into blocks of size n"""
    it = iter(iterator)
    while True:
        chunk = islice(it, n)
        try:
            fst = next(chunk)
        except StopIteration:
            return
        yield chain([fst], chunk)

parser = ArgumentParser()
parser.add_argument(
    'geotiff_file', type=str, help='Path to GeoTIFF file to load'
)
parser.add_argument(
    '--query-url', type=str, default='http://localhost:3030/landsat/query',
    dest='query_url', help='Query URL for SPARQL endpoint'
)
parser.add_argument(
    '--update-url', type=str, default='http://localhost:3030/landsat/update',
    dest='update_url', help='Update URL for SPARQL endpoint'
)
parser.add_argument(
    '--tile-sizes', type=int, default=[64, 128], nargs='+',
    dest='tile_sizes', help='Sizes of tiles to output'
)

if __name__ == '__main__':
    args = parser.parse_args()

    geotiff_fp = gdal.Open(args.geotiff_file)
    graph_triples = build_graph(geotiff_fp, args.tile_sizes)

    # Batch the triples so that Python doesn't asplode
    for triples in slow(iterchunk(graph_triples, 100), 'chunks'):
        fuseki = ConjunctiveGraph(store='SPARQLUpdateStore')
        fuseki.open((args.query_url, args.update_url))
        fuseki.addN((s, p, o, DEFAULT) for s, p, o in triples)
        fuseki.close()

    print('Done')
<commit_msg>Update batch loader to match ontology<commit_after>#!/usr/bin/env python3

"""Loads a GeoTIFF file representing Landsat data into a triple store via
SPARQL."""

from argparse import ArgumentParser, FileType
from base64 import b64encode
from collections import namedtuple
from datetime import datetime
from io import StringIO, BytesIO
from itertools import islice, chain
import logging
import re
from urllib.parse import quote_plus

from osgeo import gdal
import numpy as np
import pytz
from scipy.misc import toimage
from screw_rdflib import (ConjunctiveGraph, Literal, Namespace, OWL, RDF, RDFS,
                          XSD, URIRef, BNode, Graph, OWL)

# RDF namespaces
GEO = Namespace('http://www.w3.org/2003/01/geo/wgs84_pos#')
LED = Namespace('http://www.example.org/ANU-LED#')
QB = Namespace('http://purl.org/linked-data/cube#')
SDMXC = Namespace('http://purl.org/linked-data/sdmx/2009/concept#')
SDMXD = Namespace('http://purl.org/linked-data/sdmx/2009/dimension#')
SDMXM = Namespace('http://purl.org/linked-data/sdmx/2009/measure#')
OGC = Namespace('http://www.opengis.net/ont/geosparql#')
# For parsing AGDC filenames
AGDC_RE = re.compile(
    r'^(?P<sat_id>[^_]+)_(?P<sensor_id>[^_]+)_(?P<prod_code>[^_]+)_'
    r'(?P<lon>[^_]+)_(?P<lat>[^_]+)_(?P<year>\d{4})-(?P<month>\d{2})-'
    r'(?P<day>\d{2})T(?P<hour>\d+)-(?P<minute>\d+)-(?P<second>\d+(\.\d+)?)'
    r'\.tif$'
)
# To select default graph in Jena
DEFAULT = 'urn:x-arq:DefaultGraph'
# Boilerplate turtles are the best turtles
BOILERPLATE_TURTLE = """
@prefix : <{LED}> .
@prefix rdf: <{RDF}> .
@prefix rdfs: <{RDFS}> .
@prefix xsd: <{XSD}> .
@prefix qb: <{QB}> .
@prefix sdmx-concept: <{SDMXC}> .
@prefix sdmx-dimension: <{SDMXD}> .
@prefix sdmx-measure: <{SDMXM}> .
@prefix geo: <{GEO}> .
@prefix owl: <{OWL}> .
@prefix ogc: <{OGC}> .
@prefix gcmd-platform: <http://geobrain.laits.gmu.edu/ontology/2004/11/gcmd-platform.owl#> .
@prefix gcmd-instrument: <http://geobrain.laits.gmu.edu/ontology/2004/11/gcmd-instrument.owl#> .

:landsatDSD a qb:DataStructureDefinition ;
    qb:component :instrumentComponent
               , :positionComponent
               , :satelliteComponent
               , :timeComponent
               , :dataComponent
               , :etmBandComponent .

:landsatDS a qb:DataSet ;
    rdfs:label "Landsat sensor data"@en ;
    rdfs:comment "Some data from LandSat, retrieved from AGDC"@en ;
    qb:structure :landsatDSD ;
    :instrument gcmd-instrument:SCANNER ;
    :satellite gcmd-platform:LANDSAT-7 .

:instrumentComponent a qb:ComponentSpecification ;
    qb:attribute :instrument .

:positionComponent a qb:ComponentSpecification ;
    qb:dimension :location .

:satelliteComponent a qb:ComponentSpecification ;
    qb:attribute :satellite .

:timeComponent a qb:ComponentSpecification ;
    qb:dimension :time .

:dataComponent a qb:ComponentSpecification ;
    qb:dimension :imageData .

:etmBandComponnet a qb:ComponentSpecification ;
    qb:dimension :etmBand .

:etmBand a qb:AttributeProperty ;
    rdfs:label "LandSat ETM observation band"@en;
    rdfs:range xsd:integer .

:instrument a qb:AttributeProperty ;
    rdfs:range gcmd-instrument:Instrument .

:satellite a qb:AttributeProperty ;
    rdfs:range gcmd-platform:PLATFORM .

:time a qb:AttributeProperty ;
    rdfs:range xsd:dateTime .
""".format(QB=QB, SDMXD=SDMXD, SDMXM=SDMXM, LED=LED, GEO=GEO, SDMXC=SDMXC,
           RDF=RDF, RDFS=RDFS, XSD=XSD, OWL=OWL, OGC=OGC)

def parse_agdc_fn(fn):
    """Parses a filename in the format used by the AGDC Landsat archive. For
    example, ``LS7_ETM_NBAR_149_-036_2012-02-10T23-50-47.650696.tif`` is a
    Landsat 7 observation in GeoTIFF format taken on Februrary 10 2012 at
    11:50pm (amongst other things).

    >>> fn = 'LS7_ETM_NBAR_149_-036_2012-02-10T23-50-47.650696.tif'
    >>> sorted_results = sorted(parse_agdc_fn(fn).items())
    >>> print('\\n'.join('{}: {}'.format(k, v) for k, v in sorted_results))
    datetime: 2012-02-10 23:50:47.650696+00:00
    lat: -36.0
    lon: 149.0
    prod_code: NBAR
    sat_id: LS7
    sensor_id: ETM

    :param string fn: Filename for observation.
    :return: Dictionary of extracted metdata from filename.
    """
    match = AGDC_RE.match(fn)
    if match is None:
        raise ValueError('Invalid AGDC filename: "{}"'.format(fn))
    info = match.groupdict()
    raw_sec = float(info['second'])
    int_sec = int(raw_sec)
    microsecond = int(1e6 * (raw_sec % 1))
    dt = pytz.datetime.datetime(
        year=int(info['year']), month=int(info['month']), day=int(info['day']),
        hour=int(info['hour']), minute=int(info['minute']),
        second=int_sec, microsecond=microsecond, tzinfo=pytz.utc
    )
    return {
        'lat': float(info['lat']), 'lon': float(info['lon']), 'datetime': dt,
        'prod_code': info['prod_code'], 'sensor_id': info['sensor_id'],
        'sat_id': info['sat_id']
    }

def slow(generator, suffix, interval=500, total=None):
    """Used to annotate slow generators. Will print progress every ``interval``
    yields."""
    tot_str = '/' + str(total) if total is not None else ''
    for idx, val in enumerate(generator):
        if idx % interval == 0:
            print('{}{} {}'.format(idx, tot_str, suffix))
        yield val

def tile_iterator(band, tile_size):
    """Iterator of (top left row, top left col, data) for valid tiles in band
    (starting at left side, rather than using some more sophisticated
    scheme)."""
    value_arr = band.ReadAsArray()
    valid_mask = band.GetMaskBand().ReadAsArray().astype('bool')
    for row in range(0, value_arr.shape[0], tile_size):
        for col in range(0, value_arr.shape[1], tile_size):
            if valid_mask[row:row+tile_size, col:col+tile_size].any():
                vals = value_arr[row:row+tile_size, col:col+tile_size]
                yield (row, col, vals)

def undo_transform(row, col, t):
    """Recover (lat, lon) pair from row and column of a GeoTIFF file with
    transform t. See GDAL GetGeoTransform docs for formula."""
    return (
        t[3] + col * t[4] + row * t[5],
        t[0] + col * t[1] + row * t[2]
    )

def array_to_png(array):
    """Turn a 2D array into a data: URI filled with PNG goodies :)"""
    assert array.ndim == 2
    im = toimage(array)
    fp = BytesIO()
    im.save(fp, format='png')
    data = b64encode(fp.getvalue()).decode('utf-8')
    return 'data:image/png;base64,' + data

def loc_triples(subj, prop, lat, lon):
    """Yield a bunch of triples indicating that something is at a given
    latitude and longitude. This is actually really painful because of
    blank nodes :("""
    loc_bnode = BNode()
    yield (subj, prop, loc_bnode)
    yield (loc_bnode, GEO.lat, Literal(lat, datatype=XSD.decimal))
    yield (loc_bnode, GEO.lon, Literal(lon, datatype=XSD.decimal))

def graph_for_band(band, band_num, tile_size, gt_meta, transform):
    """Process a single band in a GeoTIFF file."""
    it = tile_iterator(band, tile_size)
    for row, col, tile in slow(it, 'tiles processed', interval=10):
        start_lat, start_lon = undo_transform(row, col, transform)
        end_lat, end_lon = undo_transform(
            row + tile_size, col + tile_size, transform
        )
        # Bounding box for the tile, which we'll convert into WKT
        bbox_corners = [
            (start_lon, start_lat), (start_lon, end_lat),
            (end_lon, end_lat), (end_lon, start_lat)
        ]
        loc_wkt = Literal('POLYGON({0}, {1}, {2}, {3}, {0})'.format(
            *['{} {}'.format(lon, lat) for lat, lon in bbox_corners]
        ), datatype=OGC.wktLiteral)
        png_tile = URIRef(array_to_png(tile))
        ident_str = 'lat/{}/lon/{}/tile-size/{}/band/{}'.format(
            start_lat, start_lon, tile_size, band_num
        )
        ident = URIRef(LED['observation/' + ident_str])
        # Woooo this makes no sense
        res = np.mean(np.abs([
            tile.shape[1] / (start_lat - end_lat),
            tile.shape[0] / (start_lon - end_lon)
        ]))

        # First add data describing the accident
        yield from [
            (ident, RDF.type, QB.Observation),
            (ident, QB.dataSet, LED.landsatDS),
            (ident, LED.bounds, loc_wkt),
            (ident, LED.etmBand, Literal(band_num, datatype=XSD.integer)),
            (ident, LED.time, Literal(gt_meta['datetime'])),
            (ident, LED.imageData, png_tile),
            (ident, LED.resolution,
                Literal(res, datatype=XSD.decimal))
        ]

        # Yield the centre point
        yield from loc_triples(
            ident, LED.location,
            (end_lat + start_lat) / 2,
            (end_lon + start_lon) / 2
        )

def build_graph(geotiff, tile_sizes):
    """Generator producing all observation triples for each band."""
    boilerplate_graph = Graph().parse(
        StringIO(BOILERPLATE_TURTLE), format='turtle'
    )
    yield from boilerplate_graph.triples((None, None, None))

    gt = geotiff.GetGeoTransform()
    filename, = geotiff.GetFileList()
    gt_meta = parse_agdc_fn(filename)

    for band_num in range(1, geotiff.RasterCount + 1):
        band = geotiff.GetRasterBand(band_num)
        for tile_size in tile_sizes:
            print('Processing band {}/{}, tile size {}'.format(
                band_num, geotiff.RasterCount, tile_size
            ))
            yield from graph_for_band(band, band_num, tile_size, gt_meta, gt)

def iterchunk(iterator, n):
    """Chunk iterator into blocks of size n"""
    it = iter(iterator)
    while True:
        chunk = islice(it, n)
        try:
            fst = next(chunk)
        except StopIteration:
            return
        yield chain([fst], chunk)

parser = ArgumentParser()
parser.add_argument(
    'geotiff_file', type=str, help='Path to GeoTIFF file to load'
)
parser.add_argument(
    '--query-url', type=str, default='http://localhost:3030/landsat/query',
    dest='query_url', help='Query URL for SPARQL endpoint'
)
parser.add_argument(
    '--update-url', type=str, default='http://localhost:3030/landsat/update',
    dest='update_url', help='Update URL for SPARQL endpoint'
)
parser.add_argument(
    '--tile-sizes', type=int, default=[64, 128], nargs='+',
    dest='tile_sizes', help='Sizes of tiles to output'
)

if __name__ == '__main__':
    args = parser.parse_args()

    geotiff_fp = gdal.Open(args.geotiff_file)
    graph_triples = build_graph(geotiff_fp, args.tile_sizes)

    # Batch the triples so that Python doesn't asplode
    for triples in slow(iterchunk(graph_triples, 100), 'chunks'):
        fuseki = ConjunctiveGraph(store='SPARQLUpdateStore')
        fuseki.open((args.query_url, args.update_url))
        fuseki.addN((s, p, o, DEFAULT) for s, p, o in triples)
        fuseki.close()

    print('Done')
<|endoftext|>


<commit_before># Copyright (c) 2017-2018 Martin Olejar
#
# SPDX-License-Identifier: BSD-3-Clause
# The BSD-3-Clause license for this file can be found in the LICENSE file included with this distribution
# or at https://spdx.org/licenses/BSD-3-Clause.html#licenseText

from easy_enum import EEnum as Enum
from struct import pack, unpack_from, calcsize

########################################################################################################################
# Enums
########################################################################################################################

class SegTag(Enum):
    """ Segments Tag """
    DCD = (0xD2, 'Device Configuration Data')
    CSF = (0xD4, 'Command Sequence File Data')
    # i.MX6, i.MX7, i.MX8M
    IVT2 = (0xD1, 'Image Vector Table (Version 2)')
    CRT = (0xD7, 'Certificate')
    SIG = (0xD8, 'Signature')
    EVT = (0xDB, 'Event')
    RVT = (0xDD, 'ROM Vector Table')
    WRP = (0x81, 'Wrapped Key')
    MAC = (0xAC, 'Message Authentication Code')
    # i.MX8QXP_A0, i.MX8QM_A0
    IVT3 = (0xDE, 'Image Vector Table (Version 3)')
    # i.MX8QXP_B0, i.MX8QM_B0
    BIC1 = (0x87, 'Boot Images Container')
    SIGB = (0x90, 'Signature block')

class CmdTag(Enum):
    """ Commands Tag """
    SET = (0xB1, 'Set')
    INS_KEY = (0xBE, 'Install Key')
    AUT_DAT = (0xCA, 'Authenticate Data')
    WRT_DAT = (0xCC, 'Write Data')
    CHK_DAT = (0xCF, 'Check Data')
    NOP = (0xC0, 'No Operation')
    INIT = (0xB4, 'Initialize')
    UNLK = (0xB2, 'Unlock')

########################################################################################################################
# Exceptions
########################################################################################################################

class UnparsedException(Exception):
    pass

class CorruptedException(Exception):
    pass

########################################################################################################################
# Classes
########################################################################################################################

class Header(object):
    """ header element type """
    FORMAT = ">BHB"
    SIZE = calcsize(FORMAT)

    @property
    def size(self):
        """ Header Size """
        return self.SIZE

    def __init__(self, tag, param=0, length=None):
        self.tag = tag
        self.param = param
        self.length = self.SIZE if length is None else length

    def __repr__(self):
        return "Header <TAG:0x{:02X}, PARAM:0x{:02X}, LEN:{:d}B>".format(self.tag, self.param, self.length)

    def info(self):
        return "HEADER < TAG: 0x{:02X}, PARAM: 0x{:02X}, DLEN: {:d} Bytes >\n".format(self.tag, self.param, self.length)

    def export(self):
        return pack(self.FORMAT, self.tag, self.length, self.param)

    @classmethod
    def parse(cls, data, offset=0, required_tag=None):
        """ Parse header
        :param data: Raw data as bytes or bytearray
        :param offset: Offset of input data
        :param required_tag: Check header TAG if specified value or ignore if is None
        :return: Header object
        """
        tag, length, param = unpack_from(cls.FORMAT, data, offset)
        if required_tag is not None and tag != required_tag:
            raise UnparsedException(" Invalid header tag: '0x{:02X}' expected '0x{:02X}' ".format(tag, required_tag))

        return cls(tag, param, length)

class Header2(Header):
    """ header element type """
    FORMAT = "<BHB"

    def __repr__(self):
        return "Header2 <TAG:0x{:02X}, PARAM:0x{:02X}, LEN:{:d}B>".format(self.tag, self.param, self.length)

    def export(self):
        return pack(self.FORMAT, self.param, self.length, self.tag)

    @classmethod
    def parse(cls, data, offset=0, required_tag=None):
        """ Parse header
        :param data: Raw data as bytes or bytearray
        :param offset: Offset of input data
        :param required_tag: Check header TAG if specified value or ignore if is None
        :return: Header2 object
        """
        param, length, tag = unpack_from(cls.FORMAT, data, offset)
        if required_tag is not None and tag != required_tag:
            raise UnparsedException(" Invalid header tag: '0x{:02X}' expected '0x{:02X}' ".format(tag, required_tag))

        return cls(tag, param, length)
<commit_msg>Fix pytest issue<commit_after># Copyright (c) 2017-2018 Martin Olejar
#
# SPDX-License-Identifier: BSD-3-Clause
# The BSD-3-Clause license for this file can be found in the LICENSE file included with this distribution
# or at https://spdx.org/licenses/BSD-3-Clause.html#licenseText

from easy_enum import EEnum as Enum
from struct import pack, unpack_from, calcsize

########################################################################################################################
# Enums
########################################################################################################################

class SegTag(Enum):
    """ Segments Tag """
    DCD = (0xD2, 'Device Configuration Data')
    CSF = (0xD4, 'Command Sequence File Data')
    # i.MX6, i.MX7, i.MX8M
    IVT2 = (0xD1, 'Image Vector Table (Version 2)')
    CRT = (0xD7, 'Certificate')
    SIG = (0xD8, 'Signature')
    EVT = (0xDB, 'Event')
    RVT = (0xDD, 'ROM Vector Table')
    WRP = (0x81, 'Wrapped Key')
    MAC = (0xAC, 'Message Authentication Code')
    # i.MX8QXP_A0, i.MX8QM_A0
    IVT3 = (0xDE, 'Image Vector Table (Version 3)')
    # i.MX8QXP_B0, i.MX8QM_B0
    BIC1 = (0x87, 'Boot Images Container')
    SIGB = (0x90, 'Signature block')

class CmdTag(Enum):
    """ Commands Tag """
    SET = (0xB1, 'Set')
    INS_KEY = (0xBE, 'Install Key')
    AUT_DAT = (0xCA, 'Authenticate Data')
    WRT_DAT = (0xCC, 'Write Data')
    CHK_DAT = (0xCF, 'Check Data')
    NOP = (0xC0, 'No Operation')
    INIT = (0xB4, 'Initialize')
    UNLK = (0xB2, 'Unlock')

########################################################################################################################
# Exceptions
########################################################################################################################

class UnparsedException(Exception):
    pass

class CorruptedException(Exception):
    pass

########################################################################################################################
# Classes
########################################################################################################################

class Header(object):
    """ header element type """
    FORMAT = ">BHB"
    SIZE = calcsize(FORMAT)

    @property
    def size(self):
        """ Header Size """
        return self.SIZE

    def __init__(self, tag, param=0, length=None):
        self.tag = tag
        self.param = param
        self.length = self.SIZE if length is None else length

    def __repr__(self):
        return "Header <TAG:0x{:02X}, PARAM:0x{:02X}, LEN:{:d}B>".format(self.tag, self.param, self.length)

    def __eq__(self, header):
        if not isinstance(header, Header):
            return False
        if self.tag != header.tag or self.param != header.param or self.length != header.length:
            return False
        return True

    def __ne__(self, header):
        return not self.__eq__(header)

    def info(self):
        return "HEADER < TAG: 0x{:02X}, PARAM: 0x{:02X}, DLEN: {:d} Bytes >\n".format(self.tag, self.param, self.length)

    def export(self):
        return pack(self.FORMAT, self.tag, self.length, self.param)

    @classmethod
    def parse(cls, data, offset=0, required_tag=None):
        """ Parse header
        :param data: Raw data as bytes or bytearray
        :param offset: Offset of input data
        :param required_tag: Check header TAG if specified value or ignore if is None
        :return: Header object
        """
        tag, length, param = unpack_from(cls.FORMAT, data, offset)
        if required_tag is not None and tag != required_tag:
            raise UnparsedException(" Invalid header tag: '0x{:02X}' expected '0x{:02X}' ".format(tag, required_tag))

        return cls(tag, param, length)

class Header2(Header):
    """ header element type """
    FORMAT = "<BHB"

    def __repr__(self):
        return "Header2 <TAG:0x{:02X}, PARAM:0x{:02X}, LEN:{:d}B>".format(self.tag, self.param, self.length)

    def __eq__(self, header):
        if not isinstance(header, Header2):
            return False
        if self.tag != header.tag or self.param != header.param or self.length != header.length:
            return False
        return True

    def export(self):
        return pack(self.FORMAT, self.param, self.length, self.tag)

    @classmethod
    def parse(cls, data, offset=0, required_tag=None):
        """ Parse header
        :param data: Raw data as bytes or bytearray
        :param offset: Offset of input data
        :param required_tag: Check header TAG if specified value or ignore if is None
        :return: Header2 object
        """
        param, length, tag = unpack_from(cls.FORMAT, data, offset)
        if required_tag is not None and tag != required_tag:
            raise UnparsedException(" Invalid header tag: '0x{:02X}' expected '0x{:02X}' ".format(tag, required_tag))

        return cls(tag, param, length)
<|endoftext|>