<commit_before>import re
import sublime

from . import persist, util
from .style import HighlightStyleStore
from .const import PROTECTED_REGIONS_KEY, WARNING, ERROR, WARN_ERR, INBUILT_ICONS

UNDERLINE_FLAGS = sublime.DRAW_NO_FILL | sublime.DRAW_NO_OUTLINE | sublime.DRAW_EMPTY_AS_OVERWRITE

MARK_STYLES = {
    'outline': sublime.DRAW_NO_FILL,
    'fill': sublime.DRAW_NO_OUTLINE,
    'solid_underline': sublime.DRAW_SOLID_UNDERLINE | UNDERLINE_FLAGS,
    'squiggly_underline': sublime.DRAW_SQUIGGLY_UNDERLINE | UNDERLINE_FLAGS,
    'stippled_underline': sublime.DRAW_STIPPLED_UNDERLINE | UNDERLINE_FLAGS,
    'none': sublime.HIDDEN
}

WORD_RE = re.compile(r'^([-\w]+)')
NEAR_RE_TEMPLATE = r'(?<!"){}({}){}(?!")'

class RegionStore:
    _memory = None

    @property
    def memory(self):
        """structure: {"view.id": [region_keys ... ]}."""
        memory = self._memory
        if not memory:
            memory = sublime.load_settings("sl_regions.sublime-settings")
            views = memory.get("views")
            if not views:
                memory.set("views", {})
            self._memory = memory
        return memory

    def add_region_keys(self, view, new_keys):
        view_id = view.id()
        saved_keys = self._get_views(view_id)
        saved_keys.extend(new_keys)
        self._set_views(view_id, saved_keys)

    def del_regions(self, view):
        view_id = view.id()
        saved_keys = self._get_views(view_id)
        for key in saved_keys:
            view.erase_regions(key)
        self._set_views(view_id)

    def _get_views(self, view_id):
        return self.memory.get("views").get(str(view_id), [])

    def _set_views(self, view_id, region_keys=None):
        view_id = str(view_id)
        views = self.memory.get("views")

        if not region_keys:
            if view_id in views:
                del views[view_id]
        else:
            views[view_id] = region_keys
        self.memory.set("views", views)

region_store = RegionStore()

def clear_view(view):
    """Clear all marks in the given view."""
    region_store.del_regions(view)

class HighlightSet:
    """This class maintains a set of Highlight objects and performs bulk operations on them."""

    def __init__(self):
        self.all = set()

    def add(self, highlight):
        """Add a Highlight to the set."""
        self.all.add(highlight)

    def draw(self, view):
        """
        Draw all of the Highlight objects in our set.

        Rather than draw each Highlight object individually, the marks in each
        object are aggregated into a new Highlight object, and that object
        is then drawn for the given view.
        """
        if not self.all:
            return

        all = Highlight()

        for highlight in self.all:
            all.update(highlight)

        all.draw(view)

    def line_type(self, line):
        """Return the primary error type for the given line number."""
        if not self.all:
            return None

        line_type = None
        for highlight in self.all:
            if line_type == ERROR:
                continue
            _line_type = highlight.lines.get(line)
            if _line_type != WARNING and line_type == WARNING:
                continue
            line_type = _line_type
        return line_type

class Highlight:
    """This class maintains error marks and knows how to draw them."""

    def __init__(self, code=''):
        self.code = code
        self.marks = util.get_new_dict()
        self.mark_style = 'outline'
        self.style_store = HighlightStyleStore()

        # Every line that has a mark is kept in this dict, so we know which
        # lines to mark in the gutter.
        self.lines = util.get_new_dict()

        # These are used when highlighting embedded code
        # The embedded code is linted as if it begins
        # at (0, 0), but we need to keep track of where
        # the actual start is within the source.
        self.line_offset = 0
        self.char_offset = 0

        # Linting runs asynchronously on a snapshot of the code.
        # Marks are added to the code during that asynchronous linting,
        # and the markup code needs to calculate character positions given
        # a line + column. By the time marks are added, the actual buffer
        # may have changed, so we can't reliably use the plugin API to
        # calculate character positions.
        # The solution is to calculate and store the character positions for
        # every line when this object is created, then reference that
        # when needed.
        self.newlines = newlines = [0]
        last = -1

        while True:
            last = code.find('\n', last + 1)

            if last == -1:
                break

            newlines.append(last + 1)

        newlines.append(len(code))

    @staticmethod
    def strip_quotes(text):
        """Return text stripped of enclosing single/double quotes."""
        first = text[0]

        if first in ('\'', '"') and text[-1] == first:
            text = text[1:-1]

        return text

    def full_line(self, line):
        """
        Return the start/end character positions for the given line.

        This returns *real* character positions (relative to the beginning
        of self.code) base on the *virtual* line number (adjusted by the
        self.line_offset).
        """
        if line == 0:
            char_offset = self.char_offset
        else:
            char_offset = 0

        line += self.line_offset
        start = self.newlines[line] + char_offset

        end = self.newlines[min(line + 1, len(self.newlines) - 1)]

        return start, end

    def range(
        self,
        line,
        pos,
        length=-1,
        near=None,
        error_type=ERROR,
        word_re=None,
        style=None
    ):
        """
        Mark a range of text.

        line and pos should be zero-based. The pos and length argument can be
        used to control marking:

        - If pos < 0, the entire line is marked and length is ignored.

        - If near is not None, it is stripped of quotes and length = len(near)

        - If length < 0, the nearest word starting at pos is marked, and if
          no word is matched, the character at pos is marked.

        - If length == 0, no text is marked,
          but a gutter mark will appear on that line.

        error_type determines what type mark will be drawn (ERROR or WARNING).

        When length < 0, this method attempts to mark the closest word at
        pos on the given line.
        If you want to customize the word matching regex, pass it in word_re.

        If the error_type is WARNING and an identical ERROR region exists,
        it is not added.
        If the error_type is ERROR and an identical WARNING region exists,
        the warning region is removed and the error region is added.
        """
        start, end = self.full_line(line)

        if pos < 0:
            pos = 0
            length = (end - start) - 1
        elif near:
            near = self.strip_quotes(near)
            length = len(near)
        elif length < 0:
            code = self.code[start:end][pos:]
            match = (word_re or WORD_RE).search(code)

            if match:
                length = len(match.group())
            else:
                length = 1

        pos += start
        region = sublime.Region(pos, pos + length)
        other_type = ERROR if error_type == WARNING else WARNING

        for scope, marks in self.marks[other_type].items():
            i_offset = 0
            for i, mark in enumerate(marks):
                if (mark.a, mark.b) == (region.a, region.b):
                    if error_type == WARNING:
                        return length
                    else:
                        self.marks[other_type][scope].pop(i - i_offset)
                        i_offset += 1

        self.marks[error_type].setdefault(style, []).append(region)
        return length

    def regex(self, line, regex, error_type=ERROR,
              line_match=None, word_match=None, word_re=None):
        """
        Mark a range of text that matches a regex.

        line, error_type and word_re are the same as in range().

        line_match may be a string pattern or a compiled regex.
        If provided, it must have a named group called 'match' that
        determines which part of the source line will be considered
        for marking.

        word_match may be a string pattern or a compiled regex.
        If provided, it must have a named group called 'mark' that
        determines which part of the source line will actually be marked.
        Multiple portions of the source line may match.
        """
        offset = 0

        start, end = self.full_line(line)
        line_text = self.code[start:end]

        if line_match:
            match = re.match(line_match, line_text)

            if match:
                line_text = match.group('match')
                offset = match.start('match')
            else:
                return

        it = re.finditer(regex, line_text)
        results = [
            result.span('mark')
            for result in it
            if word_match is None or result.group('mark') == word_match
        ]

        for start, end in results:
            self.range(line, start + offset, end -
                       start, error_type=error_type)

    def near(self, line, near, error_type=ERROR, word_re=None, style=None):
        """
        Mark a range of text near a given word.

        line, error_type and word_re are the same as in range().

        If near is enclosed by quotes, they are stripped. The first occurrence
        of near in the given line of code is matched. If the first and last
        characters of near are word characters, a match occurs only if near
        is a complete word.

        The position at which near is found is returned, or zero if there
        is no match.
        """
        if not near:
            return

        start, end = self.full_line(line)
        text = self.code[start:end]
        near = self.strip_quotes(near)

        if near == '':
            return 0

        # Add \b fences around the text if it begins/ends with a word character
        fence = ['', '']

        for i, pos in enumerate((0, -1)):
            if near[pos].isalnum() or near[pos] == '_':
                fence[i] = r'\b'

        pattern = NEAR_RE_TEMPLATE.format(fence[0], re.escape(near), fence[1])
        match = re.search(pattern, text)

        if match:
            start = match.start(1)
        else:
            start = -1

        if start != -1:
            length = self.range(
                line,
                start,
                len(near),
                error_type=error_type,
                word_re=word_re,
                style=style
            )
            return start, length
        else:
            return 0, 0

    def update(self, other):
        """
        Update this object with another Highlight.

        It is assumed that other.code == self.code.

        other's marks and error positions are merged, and this
        object takes the newlines array from other.

        """
        for error_type in WARN_ERR:
            self.marks[error_type].update(other.marks[error_type])

            for line, style in other.lines[error_type].items():
                self.overwrite_line(line, error_type, style)

        self.newlines = other.newlines

    def draw(self, view):
        """
        Draw code and gutter marks in the given view.

        Error, warning and gutter marks are drawn with separate regions,
        since each one potentially needs a different color.

        """
        from .style import GUTTER_ICONS

        drawn_regions = []
        protected_regions = []

        for error_type in WARN_ERR:
            if not self.marks[error_type]:
                continue

            for style, regions in self.marks[error_type].items():
                if not self.style_store.has_style(style):
                    continue

                scope = self.style_store.get_val("scope", style, error_type)
                mark_style = self.style_store.get_val(
                    "mark_style",
                    style,
                    error_type
                )

                flags = MARK_STYLES[mark_style]
                if not persist.settings.get('show_marks_in_minimap'):
                    flags |= sublime.HIDE_ON_MINIMAP

                view.add_regions(style, regions, scope=scope, flags=flags)
                drawn_regions.append(style)

            # gutter marks
            if not persist.settings.has('gutter_theme'):
                continue

            gutter_regions = {}
            # collect regions of error type
            for line, style in self.lines[error_type].items():
                if not self.style_store.has_style(style):
                    continue
                pos = self.newlines[line]
                region = sublime.Region(pos, pos)
                gutter_regions.setdefault(style, []).append(region)

            # draw gutter marks for
            for style, regions in gutter_regions.items():
                icon = self.style_store.get_val("icon", style, error_type)
                if not icon or icon == "none":  # do not draw icon
                    continue

                if GUTTER_ICONS.get('colorize', True) or icon in INBUILT_ICONS:
                    scope = self.style_store.get_val("scope", style, error_type)
                else:
                    scope = " "  # set scope to non-existent one

                k = style.rfind(".")
                gutter_key = style[:k] + ".gutter." + style[k + 1:]

                view.add_regions(
                    gutter_key,
                    regions,
                    scope=scope,
                    icon=icon
                )
                drawn_regions.append(gutter_key)
                protected_regions.extend(regions)

            # overlaying all gutter regions with common invisible one,
            # to create unified handle for GitGutter and other plugins
            # flag might not be neccessary
            view.add_regions(
                PROTECTED_REGIONS_KEY,
                protected_regions,
                flags=sublime.HIDDEN
            )
            drawn_regions.append(PROTECTED_REGIONS_KEY)

        # persisting region keys for later clearance
        region_store.add_region_keys(view, drawn_regions)

    def line(self, line, error_type, style=None):
        """Record the given line as having the given error type."""
        line += self.line_offset
        self.overwrite_line(line, error_type, style)

    def overwrite_line(self, line, error_type, style):
        # Errors override warnings on the same line
        if error_type == WARNING:
            if line in self.lines[ERROR]:
                return
        else:  # ensure no warning icons on same line as error
            self.lines[WARNING].pop(line, None)

        # Styles with higher priority override those of lower one
        # on the same line
        existing = self.lines[error_type].get(line)
        if existing:
            scope_ex = self.style_store.get(existing).get("priority", 0)
            scope_new = self.style_store.get(style).get("priority", 0)
            if scope_ex > scope_new:
                return

        self.lines[error_type][line] = style

    def move_to(self, line, char_offset):
        """
        Move the highlight to the given line and character offset.

        The character offset is relative to the start of the line.
        This method is used to create virtual line numbers
        and character positions when linting embedded code.

        """
        self.line_offset = line
        self.char_offset = char_offset
<commit_msg>Remove RegionStore and KISS<commit_after>from collections import defaultdict
import re
import sublime

from . import persist, util
from .style import HighlightStyleStore
from .const import PROTECTED_REGIONS_KEY, WARNING, ERROR, WARN_ERR, INBUILT_ICONS

UNDERLINE_FLAGS = sublime.DRAW_NO_FILL | sublime.DRAW_NO_OUTLINE | sublime.DRAW_EMPTY_AS_OVERWRITE

MARK_STYLES = {
    'outline': sublime.DRAW_NO_FILL,
    'fill': sublime.DRAW_NO_OUTLINE,
    'solid_underline': sublime.DRAW_SOLID_UNDERLINE | UNDERLINE_FLAGS,
    'squiggly_underline': sublime.DRAW_SQUIGGLY_UNDERLINE | UNDERLINE_FLAGS,
    'stippled_underline': sublime.DRAW_STIPPLED_UNDERLINE | UNDERLINE_FLAGS,
    'none': sublime.HIDDEN
}

WORD_RE = re.compile(r'^([-\w]+)')
NEAR_RE_TEMPLATE = r'(?<!"){}({}){}(?!")'

# Dict[view_id, region_keys]
regions = defaultdict(list)

def remember_drawn_regions(view, regions_keys):
    """Remember draw regions for later clearance."""
    view_id = view.id()
    regions[view_id].extend(regions_keys)

def clear_view(view):
    """Clear all marks in the given view."""
    view_id = view.id()
    region_keys = regions.pop(view_id, [])
    for key in region_keys:
        view.erase_regions(key)

class HighlightSet:
    """This class maintains a set of Highlight objects and performs bulk operations on them."""

    def __init__(self):
        self.all = set()

    def add(self, highlight):
        """Add a Highlight to the set."""
        self.all.add(highlight)

    def draw(self, view):
        """
        Draw all of the Highlight objects in our set.

        Rather than draw each Highlight object individually, the marks in each
        object are aggregated into a new Highlight object, and that object
        is then drawn for the given view.
        """
        if not self.all:
            return

        all = Highlight()

        for highlight in self.all:
            all.update(highlight)

        all.draw(view)

    def line_type(self, line):
        """Return the primary error type for the given line number."""
        if not self.all:
            return None

        line_type = None
        for highlight in self.all:
            if line_type == ERROR:
                continue
            _line_type = highlight.lines.get(line)
            if _line_type != WARNING and line_type == WARNING:
                continue
            line_type = _line_type
        return line_type

class Highlight:
    """This class maintains error marks and knows how to draw them."""

    def __init__(self, code=''):
        self.code = code
        self.marks = util.get_new_dict()
        self.mark_style = 'outline'
        self.style_store = HighlightStyleStore()

        # Every line that has a mark is kept in this dict, so we know which
        # lines to mark in the gutter.
        self.lines = util.get_new_dict()

        # These are used when highlighting embedded code
        # The embedded code is linted as if it begins
        # at (0, 0), but we need to keep track of where
        # the actual start is within the source.
        self.line_offset = 0
        self.char_offset = 0

        # Linting runs asynchronously on a snapshot of the code.
        # Marks are added to the code during that asynchronous linting,
        # and the markup code needs to calculate character positions given
        # a line + column. By the time marks are added, the actual buffer
        # may have changed, so we can't reliably use the plugin API to
        # calculate character positions.
        # The solution is to calculate and store the character positions for
        # every line when this object is created, then reference that
        # when needed.
        self.newlines = newlines = [0]
        last = -1

        while True:
            last = code.find('\n', last + 1)

            if last == -1:
                break

            newlines.append(last + 1)

        newlines.append(len(code))

    @staticmethod
    def strip_quotes(text):
        """Return text stripped of enclosing single/double quotes."""
        first = text[0]

        if first in ('\'', '"') and text[-1] == first:
            text = text[1:-1]

        return text

    def full_line(self, line):
        """
        Return the start/end character positions for the given line.

        This returns *real* character positions (relative to the beginning
        of self.code) base on the *virtual* line number (adjusted by the
        self.line_offset).
        """
        if line == 0:
            char_offset = self.char_offset
        else:
            char_offset = 0

        line += self.line_offset
        start = self.newlines[line] + char_offset

        end = self.newlines[min(line + 1, len(self.newlines) - 1)]

        return start, end

    def range(
        self,
        line,
        pos,
        length=-1,
        near=None,
        error_type=ERROR,
        word_re=None,
        style=None
    ):
        """
        Mark a range of text.

        line and pos should be zero-based. The pos and length argument can be
        used to control marking:

        - If pos < 0, the entire line is marked and length is ignored.

        - If near is not None, it is stripped of quotes and length = len(near)

        - If length < 0, the nearest word starting at pos is marked, and if
          no word is matched, the character at pos is marked.

        - If length == 0, no text is marked,
          but a gutter mark will appear on that line.

        error_type determines what type mark will be drawn (ERROR or WARNING).

        When length < 0, this method attempts to mark the closest word at
        pos on the given line.
        If you want to customize the word matching regex, pass it in word_re.

        If the error_type is WARNING and an identical ERROR region exists,
        it is not added.
        If the error_type is ERROR and an identical WARNING region exists,
        the warning region is removed and the error region is added.
        """
        start, end = self.full_line(line)

        if pos < 0:
            pos = 0
            length = (end - start) - 1
        elif near:
            near = self.strip_quotes(near)
            length = len(near)
        elif length < 0:
            code = self.code[start:end][pos:]
            match = (word_re or WORD_RE).search(code)

            if match:
                length = len(match.group())
            else:
                length = 1

        pos += start
        region = sublime.Region(pos, pos + length)
        other_type = ERROR if error_type == WARNING else WARNING

        for scope, marks in self.marks[other_type].items():
            i_offset = 0
            for i, mark in enumerate(marks):
                if (mark.a, mark.b) == (region.a, region.b):
                    if error_type == WARNING:
                        return length
                    else:
                        self.marks[other_type][scope].pop(i - i_offset)
                        i_offset += 1

        self.marks[error_type].setdefault(style, []).append(region)
        return length

    def regex(self, line, regex, error_type=ERROR,
              line_match=None, word_match=None, word_re=None):
        """
        Mark a range of text that matches a regex.

        line, error_type and word_re are the same as in range().

        line_match may be a string pattern or a compiled regex.
        If provided, it must have a named group called 'match' that
        determines which part of the source line will be considered
        for marking.

        word_match may be a string pattern or a compiled regex.
        If provided, it must have a named group called 'mark' that
        determines which part of the source line will actually be marked.
        Multiple portions of the source line may match.
        """
        offset = 0

        start, end = self.full_line(line)
        line_text = self.code[start:end]

        if line_match:
            match = re.match(line_match, line_text)

            if match:
                line_text = match.group('match')
                offset = match.start('match')
            else:
                return

        it = re.finditer(regex, line_text)
        results = [
            result.span('mark')
            for result in it
            if word_match is None or result.group('mark') == word_match
        ]

        for start, end in results:
            self.range(line, start + offset, end -
                       start, error_type=error_type)

    def near(self, line, near, error_type=ERROR, word_re=None, style=None):
        """
        Mark a range of text near a given word.

        line, error_type and word_re are the same as in range().

        If near is enclosed by quotes, they are stripped. The first occurrence
        of near in the given line of code is matched. If the first and last
        characters of near are word characters, a match occurs only if near
        is a complete word.

        The position at which near is found is returned, or zero if there
        is no match.
        """
        if not near:
            return

        start, end = self.full_line(line)
        text = self.code[start:end]
        near = self.strip_quotes(near)

        if near == '':
            return 0

        # Add \b fences around the text if it begins/ends with a word character
        fence = ['', '']

        for i, pos in enumerate((0, -1)):
            if near[pos].isalnum() or near[pos] == '_':
                fence[i] = r'\b'

        pattern = NEAR_RE_TEMPLATE.format(fence[0], re.escape(near), fence[1])
        match = re.search(pattern, text)

        if match:
            start = match.start(1)
        else:
            start = -1

        if start != -1:
            length = self.range(
                line,
                start,
                len(near),
                error_type=error_type,
                word_re=word_re,
                style=style
            )
            return start, length
        else:
            return 0, 0

    def update(self, other):
        """
        Update this object with another Highlight.

        It is assumed that other.code == self.code.

        other's marks and error positions are merged, and this
        object takes the newlines array from other.

        """
        for error_type in WARN_ERR:
            self.marks[error_type].update(other.marks[error_type])

            for line, style in other.lines[error_type].items():
                self.overwrite_line(line, error_type, style)

        self.newlines = other.newlines

    def draw(self, view):
        """
        Draw code and gutter marks in the given view.

        Error, warning and gutter marks are drawn with separate regions,
        since each one potentially needs a different color.

        """
        from .style import GUTTER_ICONS

        drawn_regions = []
        protected_regions = []

        for error_type in WARN_ERR:
            if not self.marks[error_type]:
                continue

            for style, regions in self.marks[error_type].items():
                if not self.style_store.has_style(style):
                    continue

                scope = self.style_store.get_val("scope", style, error_type)
                mark_style = self.style_store.get_val(
                    "mark_style",
                    style,
                    error_type
                )

                flags = MARK_STYLES[mark_style]
                if not persist.settings.get('show_marks_in_minimap'):
                    flags |= sublime.HIDE_ON_MINIMAP

                view.add_regions(style, regions, scope=scope, flags=flags)
                drawn_regions.append(style)

            # gutter marks
            if not persist.settings.has('gutter_theme'):
                continue

            gutter_regions = {}
            # collect regions of error type
            for line, style in self.lines[error_type].items():
                if not self.style_store.has_style(style):
                    continue
                pos = self.newlines[line]
                region = sublime.Region(pos, pos)
                gutter_regions.setdefault(style, []).append(region)

            # draw gutter marks for
            for style, regions in gutter_regions.items():
                icon = self.style_store.get_val("icon", style, error_type)
                if not icon or icon == "none":  # do not draw icon
                    continue

                if GUTTER_ICONS.get('colorize', True) or icon in INBUILT_ICONS:
                    scope = self.style_store.get_val("scope", style, error_type)
                else:
                    scope = " "  # set scope to non-existent one

                k = style.rfind(".")
                gutter_key = style[:k] + ".gutter." + style[k + 1:]

                view.add_regions(
                    gutter_key,
                    regions,
                    scope=scope,
                    icon=icon
                )
                drawn_regions.append(gutter_key)
                protected_regions.extend(regions)

            # overlaying all gutter regions with common invisible one,
            # to create unified handle for GitGutter and other plugins
            # flag might not be neccessary
            view.add_regions(
                PROTECTED_REGIONS_KEY,
                protected_regions,
                flags=sublime.HIDDEN
            )
            drawn_regions.append(PROTECTED_REGIONS_KEY)

        # persisting region keys for later clearance
        remember_drawn_regions(view, drawn_regions)

    def line(self, line, error_type, style=None):
        """Record the given line as having the given error type."""
        line += self.line_offset
        self.overwrite_line(line, error_type, style)

    def overwrite_line(self, line, error_type, style):
        # Errors override warnings on the same line
        if error_type == WARNING:
            if line in self.lines[ERROR]:
                return
        else:  # ensure no warning icons on same line as error
            self.lines[WARNING].pop(line, None)

        # Styles with higher priority override those of lower one
        # on the same line
        existing = self.lines[error_type].get(line)
        if existing:
            scope_ex = self.style_store.get(existing).get("priority", 0)
            scope_new = self.style_store.get(style).get("priority", 0)
            if scope_ex > scope_new:
                return

        self.lines[error_type][line] = style

    def move_to(self, line, char_offset):
        """
        Move the highlight to the given line and character offset.

        The character offset is relative to the start of the line.
        This method is used to create virtual line numbers
        and character positions when linting embedded code.

        """
        self.line_offset = line
        self.char_offset = char_offset
<|endoftext|>


<commit_before>from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.orm import relationship

from models.base import Base
from models.user import User
from models.store import Store

class Product(Base):
	__tablename__ = "product"
	__table_args__ = {'sqlite_autoincrement': True}

	id = Column(Integer, primary_key=True)
	name = Column(String(250), nullable=False)
	category = Column(String(80), nullable=False)
	description = Column(String(250))
	price = Column(String(10))
	store_id = Column(Integer, ForeignKey('store.id'))
	store = relationship(Store)
	user_id = Column(Integer, ForeignKey('user.id'))
	user = relationship(User)

	@property
	def serialize(self):
		"""Return product type object data in easily serializeable format"""
		return {
			'id': self.id,
			'name': self.name,
			'category': self.category,
			'description': self.description,
			'price': self.price,
			'store': self.store_id,
			'user': self.user_id,
		}
<commit_msg>bug: Correct the character limit for descriptions<commit_after>from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.orm import relationship

from models.base import Base
from models.user import User
from models.store import Store

class Product(Base):
	__tablename__ = "product"
	__table_args__ = {'sqlite_autoincrement': True}

	id = Column(Integer, primary_key=True)
	name = Column(String(250), nullable=False)
	category = Column(String(80), nullable=False)
	description = Column(String(2000))
	price = Column(String(10))
	store_id = Column(Integer, ForeignKey('store.id'))
	store = relationship(Store)
	user_id = Column(Integer, ForeignKey('user.id'))
	user = relationship(User)

	@property
	def serialize(self):
		"""Return product type object data in easily serializeable format"""
		return {
			'id': self.id,
			'name': self.name,
			'category': self.category,
			'description': self.description,
			'price': self.price,
			'store': self.store_id,
			'user': self.user_id,
		}
<|endoftext|>


<commit_before>from app import db
import json
import shortuuid

class NoDoiException(Exception):
    pass

def make_product(product_dict):
    product = Product(id=shortuuid.uuid()[0:10])

    # get the DOI
    doi = None
    if product_dict.get('work-external-identifiers', []):
        for x in product_dict.get('work-external-identifiers', []):
            for eid in product_dict['work-external-identifiers']['work-external-identifier']:
                if eid['work-external-identifier-type'] == 'DOI':
                    doi = str(eid['work-external-identifier-id']['value'].encode('utf-8')).lower()

    if not doi:  # this should become actual validation check in the future.
        raise NoDoiException("all products need a DOI.")

    # AIP journals tend to have a \n in the DOI, and the doi is the second line.
    # we get that here. put this in the validation function later.
    if len(doi.split('\n')) == 2:
        doi = doi.split('\n')[1]

    product.doi = doi

    # get the title
    try:
        product.title = str(product_dict['work-title']['title']['value'].encode('utf-8'))
    except TypeError:
        product.title = None

    # get the publication date
    pub_date = product_dict.get('publication-date', None)
    if pub_date:
        product.year = pub_date.get('year', None).get('value').encode('utf-8')
    else:
        product.year = None

    product.api_raw = json.dumps(product_dict)

    return product

class Product(db.Model):
    id = db.Column(db.Text, primary_key=True)
    title = db.Column(db.Text)
    year = db.Column(db.Text)
    doi = db.Column(db.Text)
    api_raw = db.Column(db.Text)
    orcid = db.Column(db.Text, db.ForeignKey('profile.id'))

    @property
    def display_title(self):
        if self.title:
            return self.title
        else:
            return "No title"

    def __repr__(self):
        return u'<Product ({id}) "{title}" >'.format(
            id=self.id,
            title=self.display_title
        )

    def to_dict(self):
        return {
            "id": self.id,
            "doi": self.doi,
            "orcid": self.orcid,
            "title": self.title,
            "year": self.year
        }

<commit_msg>adds some altmetric api stuff.<commit_after>from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.mutable import MutableDict

from app import db
import json
import shortuuid
import requests
import os

class NoDoiException(Exception):
    pass

def make_product(product_dict):
    product = Product(id=shortuuid.uuid()[0:10])

    # get the DOI
    doi = None
    if product_dict.get('work-external-identifiers', []):
        for x in product_dict.get('work-external-identifiers', []):
            for eid in product_dict['work-external-identifiers']['work-external-identifier']:
                if eid['work-external-identifier-type'] == 'DOI':
                    doi = str(eid['work-external-identifier-id']['value'].encode('utf-8')).lower()

    if not doi:  # this should become actual validation check in the future.
        raise NoDoiException("all products need a DOI.")

    # AIP journals tend to have a \n in the DOI, and the doi is the second line.
    # we get that here. put this in the validation function later.
    if len(doi.split('\n')) == 2:
        doi = doi.split('\n')[1]

    product.doi = doi

    # get the title
    try:
        product.title = str(product_dict['work-title']['title']['value'].encode('utf-8'))
    except TypeError:
        product.title = None

    # get the publication date
    pub_date = product_dict.get('publication-date', None)
    if pub_date:
        product.year = pub_date.get('year', None).get('value').encode('utf-8')
    else:
        product.year = None

    product.api_raw = json.dumps(product_dict)

    return product

class Product(db.Model):
    id = db.Column(db.Text, primary_key=True)
    title = db.Column(db.Text)
    year = db.Column(db.Text)
    doi = db.Column(db.Text)
    api_raw = db.Column(db.Text)
    orcid = db.Column(db.Text, db.ForeignKey('profile.id'))

    altmetric_api_raw = db.Column(JSONB)
    altmetric_counts = db.Column(JSONB)

    def set_altmetric(self):
        url = "http://api.altmetric.com/v1/doi/{doi}?key={key}".format(
            doi=self.doi,
            key=os.getenv("ALTMETRIC_KEY")
        )

        r = requests.get(url)
        if r.status_code == 404:
            self.altmetric_api_raw = None
            self.altmetric_counts = {}
        else:
            self.altmetric_api_raw = r.text

            # add this later.
            self.altmetric_counts = {}

    @property
    def display_title(self):
        if self.title:
            return self.title
        else:
            return "No title"

    def __repr__(self):
        return u'<Product ({id}) "{title}" >'.format(
            id=self.id,
            title=self.display_title
        )

    def to_dict(self):
        return {
            "id": self.id,
            "doi": self.doi,
            "orcid": self.orcid,
            "title": self.title,
            "year": self.year
        }

<|endoftext|>


<commit_before>"""Various utilities."""

import tensorflow as tf

def as_scope(scope):
    """Get the proper variable scope.

    Given an object that can represent a `tf.VariableScope`,
    namely a `str` or a `tf.VariableScope`, performs type checking
    and return a proper `tf.VariableScope` object. Such function is
    hancy when a function accepts an argument serving as a variable
    scope but doesn's know its proper type.

    Arguments:
      scope: a `str` or a `tf.VariableScope` representing a variable scope.

    Returns:
      a `tf.VariableScope` instance.

    Raises:
      ValueError: if `scope` is `None`.
      TypeError: if `scope` is neither `str` or `tf.VariableScope`.

    Example:
    ```python
    from dket import utils

    def do_something(scope):
        scope = utils.as_scope(scope or 'DefaultScope')
        with tf.variable_scope(scope) as scope:
            # do something
            pass
    ```
    """
    if scope is None:
        raise ValueError('Cannot create a scope from a None.')
    if isinstance(scope, str):
        return next(tf.variable_scope(scope).gen)  # pylint: disable=I0011,E1101
    if isinstance(scope, tf.VariableScope):
        return scope
    raise TypeError("""`scope` argument can be of type str, """
                    """tf.VariableScope, while %s found.""",
                    (str(type(scope))))

def get_variables(prefix=None):
    """Get variables by their name prefix.

    Arguments:
      prefix: a `str` or a `tf.VariableScope` instance.

    Returns:
      a list of `tf.Variable` with their name starting with the
        given prefix, i.e. all those variables under the scope
        specified by the prefix.
    """
    prefix = prefix or tf.get_variable_scope().name
    return [var for var in tf.global_variables()
            if var.name.startswith(prefix)]
<commit_msg>added minor utility functions.<commit_after>"""Various utilities."""

import tensorflow as tf

def as_scope(scope):
    """Get the proper variable scope.

    Given an object that can represent a `tf.VariableScope`,
    namely a `str` or a `tf.VariableScope`, performs type checking
    and return a proper `tf.VariableScope` object. Such function is
    hancy when a function accepts an argument serving as a variable
    scope but doesn's know its proper type.

    Arguments:
      scope: a `str` or a `tf.VariableScope` representing a variable scope.

    Returns:
      a `tf.VariableScope` instance.

    Raises:
      ValueError: if `scope` is `None`.
      TypeError: if `scope` is neither `str` or `tf.VariableScope`.

    Example:
    ```python
    from dket import utils

    def do_something(scope):
        scope = utils.as_scope(scope or 'DefaultScope')
        with tf.variable_scope(scope) as scope:
            # do something
            pass
    ```
    """
    if scope is None:
        raise ValueError('Cannot create a scope from a None.')
    if isinstance(scope, str):
        return next(tf.variable_scope(scope).gen)  # pylint: disable=I0011,E1101
    if isinstance(scope, tf.VariableScope):
        return scope
    raise TypeError("""`scope` argument can be of type str, """
                    """tf.VariableScope, while %s found.""",
                    (str(type(scope))))

def get_variables(prefix=None):
    """Get variables by their name prefix.

    Arguments:
      prefix: a `str` or a `tf.VariableScope` instance.

    Returns:
      a list of `tf.Variable` with their name starting with the
        given prefix, i.e. all those variables under the scope
        specified by the prefix.
    """
    prefix = prefix or tf.get_variable_scope().name
    return [var for var in tf.global_variables()
            if var.name.startswith(prefix)]

def dypes(tensors):
    """Get the `dtype` for tensors in a list.

    Arguments:
      tensors: an iterable of `tf.Tensor`.

    Returns:
      a `list` of `dtype`s, one for each tensor in `tensors`,
        representing their `dtype`.
    """
    return [t.dtype for t in tensors]

def shapes(tensors):
    """Get the static shapes of tensors in a list.

    Arguments:
      tensors: an iterable of `tf.Tensor`.

    Returns:
      a `list` of `tf.TensorShape`, one for each tensor in `tensors`,
        representing their static shape (via `tf.Tensor.get_shape()`).
    """
    return [t.get_shape() for t in tensors]
<|endoftext|>


<commit_before>
import numpy as np
import urllib2
import urllib
import os
import os.path
import argparse
import logging
import pandas as pd
from astropy.table import Table as tab
from astropy import coordinates as coord
from astropy import units as u
from astropy.io import votable
def load_SDSS_phot_dr7(ra,dec,search_radius,pandas=None,ver=None):
    '''
    ra in degrees, dec in degrees, search_radius in arcmin.
    '''
    def gen_SDSS_sql(ra, dec, search_radius):
        query_out  = ' p.objID, p.type, p.ra, p.dec, p.u, p.g, p.r, p.i, p.z, p.Err_u, p.Err_g, p.Err_r, p.Err_i, p.Err_z, p.extinction_u, p.extinction_i, p.extinction_r, p.extinction_g, p.extinction_z, p.petroMag_u, p.petroMag_g, p.petroMag_r, p.petroMag_i, p.petroMag_z, petroMagErr_u, petroMagErr_g, petroMagErr_r, petroMagErr_i, petroMagErr_z'
        query_from = ' FROM fGetNearbyObjEq({ra},{dec},{search_radius}) n, PhotoPrimary p WHERE n.objID=p.objID'.format(ra=str(ra),dec=str(dec),search_radius=str(search_radius))
        query_str='SELECT'+query_out+query_from
        return query_str

    def query_SDSS(sSQL_query):
    	sURL = 'http://cas.sdss.org/dr7/en/tools/search/x_sql.asp'
    	# for POST request
    	values = {'cmd': sSQL_query,
    						'format': 'csv'}
    	data = urllib.urlencode(values)
    	request = urllib2.Request(sURL, data)
    	response = urllib2.urlopen(request)
    	return response.read()

    sql_str=gen_SDSS_sql(ra,dec,search_radius)
    sdss_ds=query_SDSS(sql_str)
    lines=sdss_ds.split('\n')
    nobj=len(lines)-2
    if ver:print(str(nobj)+' SDSS objects found')
    if nobj >0:
        cols=lines[0].split(',')
        #pop columnes and the EOF line
        lines.pop(0)
        lines.pop(-1)
        rows=[]
        for i in lines:
            tt=i.split(',')
            tt=map(float,tt)
            rows.append(tt)
        tab_out=tab(rows=rows,names=cols)
        if pandas:
            tab_out=pd.DataFrame.from_records(tab_out._data)
            radec=coord.SkyCoord(ra,dec,unit=(u.degree,u.degree),frame='icrs')
            sdss=coord.SkyCoord(tab_out.ra, tab_out.dec, unit=(u.degree,u.degree),frame='icrs')
            tab_out['dist_arcsec'] = radec.separation(sdss).arcsec
        #should fix objID to string
        #should find out what p.type means
        return tab_out
    else:
        return []

def load_SDSS_phot_dr12(ra,dec,search_radius,pandas=None, limit=None):
    '''
    ra in degrees, dec in degrees, search_radius in arcmin.
    '''
    #creat url
    sURL='http://skyserver.sdss.org/dr12/en/tools/search/x_radial.aspx?'
    ra_str='ra='+str(ra)+'&'
    dec_str='dec='+str(dec)+'&'
    radius_str='radius='+str(search_radius)+'&'
    if not limit: limit=20
    limit_str='format=csv&limit='+str(limit)
    # for POST request
    urlquery=sURL+ra_str+dec_str+radius_str+limit_str
    response=urllib2.urlopen(urlquery)
    if pandas:
        tab_out=pd.read_csv(response,comment='#')
    else:
        tab_out=ascii.read(response, delimiter=',')
    #should fix objID to string
    #should find out what p.type means
    return tab_out

def IRSA_TAP(ra,dec,search_radius,irsa_table,pandas=False,sql_SELECT=None,sql_WHERE=None,verbose=False):
    '''
    '''
    j2000 ="'"+"J2000"+"'"
    sURL = 'http://irsa.ipac.caltech.edu/TAP/sync?QUERY='
    str_from = 'FROM+'+irsa_table+'+'
    if sql_SELECT == None :
        str_select = 'SELECT+*+' #select all
    else:
        str_select = 'SELECT+'+sql_SELECT+'+'
    if sql_WHERE == None :
        str_where = 'WHERE+'
    else:
        str_where = 'WHERE+'+sql_WHERE+'+'
    str_coord1 = 'CONTAINS(POINT('+j2000+',ra,dec),'
    str_coord2 = 'CIRCLE('+j2000+','+str(ra)+','+str(dec)+','+str(search_radius)+'))=1'
    str_coord=str_coord1+str_coord2
    urlquery = sURL+str_select+str_from+str_where+str_coord
    if verbose:
        print(urlquery)
    response=urllib2.urlopen(urlquery)
    if pandas:
        tab_out = votble.parse_single_table(response).to_pandas()
    else:
        tab_out = votable.parse_single_table(response)
    return tab_out
<commit_msg>updated load_SDSS_phot, now the default is p.* for both DR7 and DR12<commit_after>
import numpy as np
import urllib2
import urllib
import os
import os.path
import argparse
import logging
import pandas as pd
from astropy.table import Table as tab
from astropy import coordinates as coord
from astropy import units as u
from astropy.io import votable
def load_SDSS_phot_dr7(ra,dec,search_radius,pandas=None,ver=None,columns=None):
    '''
    ra in degrees, dec in degrees, search_radius in arcmin.
    '''
    def gen_SDSS_sql(ra, dec, search_radius,columns=columns):
        if columns is None:
            query_out  = ' p.*'
        else:
            query_out = columns+',p.ra,p.dec'
        query_from = ' FROM fGetNearbyObjEq({ra},{dec},{search_radius}) n, PhotoPrimary p WHERE n.objID=p.objID'.format(ra=str(ra),dec=str(dec),search_radius=str(search_radius))
        query_str='SELECT'+query_out+query_from
        return query_str

    def query_SDSS(sSQL_query):
        sURL = 'http://cas.sdss.org/dr7/en/tools/search/x_sql.asp'
        # for POST request
        values = {'cmd': sSQL_query,
                            'format': 'csv'}
        data = urllib.urlencode(values)
        request = urllib2.Request(sURL, data)
        response = urllib2.urlopen(request)
        return response.read()

    sql_str=gen_SDSS_sql(ra,dec,search_radius)
    sdss_ds=query_SDSS(sql_str)
    lines=sdss_ds.split('\n')
    nobj=len(lines)-2
    if ver:print(str(nobj)+' SDSS objects found')
    if nobj >0:
        cols=lines[0].split(',')
        #pop columnes and the EOF line
        lines.pop(0)
        lines.pop(-1)
        rows=[]
        for i in lines:
            tt=i.split(',')
            tt=map(float,tt)
            rows.append(tt)
        tab_out=tab(rows=rows,names=cols)
        if pandas:
            tab_out=pd.DataFrame.from_records(tab_out._data)
            radec=coord.SkyCoord(ra,dec,unit=(u.degree,u.degree),frame='icrs')
            sdss=coord.SkyCoord(tab_out.ra, tab_out.dec, unit=(u.degree,u.degree),frame='icrs')
            tab_out['dist_arcsec'] = radec.separation(sdss).arcsec
        #should fix objID to string
        #should find out what p.type means
        return tab_out
    else:
        return []

def load_SDSS_phot_dr12(ra,dec,search_radius,pandas=None,ver=None,columns=None):
    '''
    ra in degrees, dec in degrees, search_radius in arcmin.
    '''
    def gen_SDSS_sql(ra, dec, search_radius,columns=columns):
        if columns is None:
            query_out  = ' p.*'
        else:
            query_out = columns+',p.ra,p.dec'
        query_from = ' FROM fGetNearbyObjEq({ra},{dec},{search_radius}) n, PhotoPrimary p WHERE n.objID=p.objID'.format(ra=str(ra),dec=str(dec),search_radius=str(search_radius))
        query_str='SELECT'+query_out+query_from
        return query_str

    def query_SDSS(sSQL_query):
        sURL = 'http://skyserver.sdss.org/dr12/en/tools/search/x_sql.aspx?'
        # for POST request
        values = {'cmd': sSQL_query,
                            'format': 'csv'}
        data = urllib.urlencode(values)
        request = urllib2.Request(sURL, data)
        response = urllib2.urlopen(request)
        return response.read()

    sql_str=gen_SDSS_sql(ra,dec,search_radius)
    sdss_ds=query_SDSS(sql_str)
    lines=sdss_ds.split('\n')
    nobj=len(lines)-2
    if ver:print(str(nobj)+' SDSS objects found')
    if nobj >0:
        #pop table name and the EOF line
        lines.pop(0)
        lines.pop(-1)
        cols=lines[0].split(',')
        #pop column
        lines.pop(0)
        rows=[]
        for i in lines:
            tt=i.split(',')
            tt=map(float,tt)
            rows.append(tt)
        tab_out=tab(rows=rows,names=cols)
        if pandas:
            tab_out=pd.DataFrame.from_records(tab_out._data)
            radec=coord.SkyCoord(ra,dec,unit=(u.degree,u.degree),frame='icrs')
            sdss=coord.SkyCoord(tab_out.ra, tab_out.dec, unit=(u.degree,u.degree),frame='icrs')
            tab_out['dist_arcsec'] = radec.separation(sdss).arcsec
        #should fix objID to string
        #should find out what p.type means
        return tab_out
    else:
        return []

def IRSA_TAP(ra,dec,search_radius,irsa_table,pandas=False,sql_SELECT=None,sql_WHERE=None,verbose=False):
    '''
    '''
    j2000 ="'"+"J2000"+"'"
    sURL = 'http://irsa.ipac.caltech.edu/TAP/sync?QUERY='
    str_from = 'FROM+'+irsa_table+'+'
    if sql_SELECT == None :
        str_select = 'SELECT+*+' #select all
    else:
        str_select = 'SELECT+'+sql_SELECT+'+'
    if sql_WHERE == None :
        str_where = 'WHERE+'
    else:
        str_where = 'WHERE+'+sql_WHERE+'+'
    str_coord1 = 'CONTAINS(POINT('+j2000+',ra,dec),'
    str_coord2 = 'CIRCLE('+j2000+','+str(ra)+','+str(dec)+','+str(search_radius)+'))=1'
    str_coord=str_coord1+str_coord2
    urlquery = sURL+str_select+str_from+str_where+str_coord
    if verbose:
        print(urlquery)
    response=urllib2.urlopen(urlquery)
    if pandas:
        tab_out = votble.parse_single_table(response).to_pandas()
    else:
        tab_out = votable.parse_single_table(response)
    return tab_out
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import changeset
import filecontents
import files
import log
import update
import os
import repository
import versions

# Many comments in this file only make sense if you consider the case
# where something is being updated from version A to B, and the local
# branches are called A.local and B.local; this terminology is used throughout
# the comments in this file

# Every item which is inserted into the database really goes in twice;
# once as is, and then a LocalBranch is immediately created which reflects
# what's in the file system. The "as is" copy doesn't include data for
# any file except configuration files (which means while they can be used
# as the source for a change set, they can't be used as the target)
#
# The LocalBranch for each item in the database has normal header information,
# but the on-disk files are the real source of information on the package,
# including file types, hashes, etc

class Database(repository.LocalRepository):

    createBranches = 1

    # If the request is on the local branch, pull the package from the
    # filesystem
    def getPackageVersion(self, name, version):
	if not version.isLocal():
	    return repository.LocalRepository.getPackageVersion(self, name,
								version)
	parent = version.parent()
	pkg = repository.LocalRepository.getPackageVersion(self, name, parent)
	pkg.changeVersion(self.pkgGetFullVersion(pkg.getName(),
			  pkg.getVersion()))

	version.timeStamp = 1
	rc = update.buildLocalChanges(self, [(pkg, pkg, version)])
	if not rc:
	    raise DatabaseError, \
		    "failed to build local package for %s" % pkg.getName()

	assert(len(rc[1]) == 1)
	return rc[1][0][1]

    # takes an abstract change set and creates a differential change set
    # against a branch of the repository
    def rootChangeSet(self, absSet, branch):
	assert(absSet.isAbstract())

	# this has an empty source path template, which is only used to
	# construct the eraseFiles list anyway
	job = repository.ChangeSetJob(self, absSet)

	# abstract change sets cannot have eraseLists
	#assert(not eraseList)
	#assert(not eraseFiles)

	cs = changeset.ChangeSetFromAbstractChangeSet(absSet)

	for newPkg in job.newPackageList():
	    # FIXME
	    #
	    # this shouldn't be against branch, it should be against
	    # the version of the package already installed on the
	    # system. unfortunately we can't represent that yet.
	    pkgName = newPkg.getName()
	    oldVersion = self.pkgLatestVersion(pkgName, branch)
	    if not oldVersion:
		# new package; the Package.diff() right after this never
		# sets the abstract flag, so the right thing happens
		old = None
	    else:
		old = self.getPackageVersion(pkgName, oldVersion)

	    # we ignore pkgsNeeded; it doesn't mean much in this case
	    (pkgChgSet, filesNeeded, pkgsNeeded) =	    \
		    newPkg.diff(old, abstract = 0)
	    cs.newPackage(pkgChgSet)

	    for (fileId, oldVersion, newVersion, newPath) in filesNeeded:
		fileObj = job.getFile(fileId)
		assert(newVersion.equal(fileObj.version()))

		oldFile = None
		if oldVersion:
		    (oldFile, oldCont) = self.getFileVersion(fileId,
					    oldVersion, withContents = 1)

		(filecs, hash) = changeset.fileChangeSet(fileId, oldFile,
							 fileObj.file())

		cs.addFile(fileId, oldVersion, newVersion, filecs)
		if hash:
		    contType = changeset.ChangedFileTypes.file
		    cont = filecontents.FromChangeSet(absSet, fileId)
		    if oldVersion:
			(contType, cont) = changeset.fileContentsDiff(oldFile,
				    oldCont, fileObj.file(), cont)

		    cs.addFileContents(fileId, contType, cont)

	assert(not cs.validate())

	return cs

    # local changes includes the A->A.local portion of a rollback; if it
    # doesn't exist we need to compute that and save a rollback for this
    # transaction
    def commitChangeSet(self, cs, localRollback = None):
	assert(not cs.isAbstract())

	for pkg in cs.getNewPackageList():
	    if pkg.name.endswith(":sources"): raise SourcePackageInstall

	if not localRollback:
	    # create the change set from A->A.local
	    for newPkg in cs.getNewPackageList():
		name = newPkg.getName()
		old = newPkg.getOldVersion()
		pkgList = []
		if self.hasPackage(name) and old:
		    ver = old.fork(versions.LocalBranch(), sameVerRel = 1)
		    pkg = self.getPackageVersion(name, old)
		    assert(pkg)
		    pkgList.append((pkg, pkg, ver))
	    result = update.buildLocalChanges(self, pkgList, root = self.root)
	    if not result: return

	    (localChanges, retList) = result
	    fsPkgDict = {}
	    for (changed, fsPkg) in retList:
		fsPkgDict[fsPkg.getName()] = fsPkg

	    inverse = cs.makeRollback(self, configFiles = 1)
	else:
	    assert(0)
	    localChanges = localRollback

	# Build and commit A->B
	job = DatabaseChangeSetJob(self, cs)
	undo = repository.ChangeSetUndo(self)

	try:
	    # add new packages
	    job.commit(undo)
	    # build the list of things to commit
	    fsJob = update.FilesystemJob(self, cs, fsPkgDict, self.root)
	    # remove old packages
	    errList = fsJob.getErrorList()
	    if errList:
		for err in errList: log.error(err)
		undo.undo()
		# FIXME need a --force for this
		#return

	    job.removals(undo)
	except:
	    # this won't work it things got too far, but it won't hurt
	    # anything either
	    undo.undo()
	    raise

	# everything is in the database... save this so we can undo
	# it later.
	if not localRollback:
	    self.addRollback(inverse, localChanges)

	fsJob.apply()

    # this is called when a Repository wants to store a file; we never
    # want to do this; we copy files onto the filesystem after we've
    # created the LocalBranch
    def storeFileFromContents(self, contents, file, restoreContents):
	if file.isConfig():
	    return repository.LocalRepository.storeFileFromContents(self,
				contents, file, restoreContents)

    def close(self):
	repository.LocalRepository.close(self)

    def open(self, mode):
	repository.LocalRepository.open(self, mode)
	self.rollbackCache = self.top + "/rollbacks"
	self.rollbackStatus = self.rollbackCache + "/status"
	if not os.path.exists(self.rollbackCache):
	    os.mkdir(self.rollbackCache)
	if not os.path.exists(self.rollbackStatus):
	    self.firstRollback = 0
	    self.lastRollback = -1
	    self.writeRollbackStatus()
	else:
	    self.readRollbackStatus()

    def addRollback(self, reposChangeset, localChangeset):
	rpFn = self.rollbackCache + ("/rb.r.%d" % (self.lastRollback + 1))
	reposChangeset.writeToFile(rpFn)

	localFn = self.rollbackCache + ("/rb.l.%d" % (self.lastRollback + 1))
	localChangeset.writeToFile(localFn)

	self.lastRollback += 1
	self.writeRollbackStatus()

    # name looks like "r.%d"
    def removeRollback(self, name):
	rollback = int(name[2:])
	os.unlink(self.rollbackCache + "/rb.r.%d" % rollback)
	os.unlink(self.rollbackCache + "/rb.l.%d" % rollback)
	if rollback == self.lastRollback:
	    self.lastRollback -= 1
	    self.writeRollbackStatus()

    def writeRollbackStatus(self):
	newStatus = self.rollbackCache + ".new"

	f = open(newStatus, "w")
	f.write("%s %d\n" % (self.firstRollback, self.lastRollback))
	f.close()

	os.rename(newStatus, self.rollbackStatus)

    def getRollbackList(self):
	list = []
	for i in range(self.firstRollback, self.lastRollback + 1):
	    list.append("r.%d" % i)

	return list

    def readRollbackStatus(self):
	f = open(self.rollbackStatus)
	(first, last) = f.read()[:-1].split()
	self.firstRollback = int(first)
	self.lastRollback = int(last)
	f.close()

    def hasRollback(self, name):
	try:
	    num = int(name[2:])
	except ValueError:
	    return False

	if (num >= self.firstRollback and num <= self.lastRollback):
	    return True

	return False

    def getRollback(self, name):
	if not self.hasRollback(name): return None

	num = int(name[2:])

	rc = []
	for ch in [ "r", "l" ]:
	    name = self.rollbackCache + "/" + "rb.%c.%d" % (ch, num)
	    rc.append(changeset.ChangeSetFromFile(name,
						  justContentsForConfig = 1))

	return rc

    def applyRollbackList(self, names):
	last = self.lastRollback
	for name in names:
	    if not self.hasRollback(name):
		raise RollbackDoesNotExist(name)

	    num = int(name[2:])
	    if num != last:
		raise RollbackOrderError(name)
	    last -= 1

	for name in names:
	    (repostCs, localCs) = self.getRollback(name)
	    self.commitChangeSet(repostCs, localRollback = localCs)
	    self.removeRollback(name)

    def __init__(self, root, path, mode = "r"):
	self.root = root
	fullPath = root + "/" + path
	repository.LocalRepository.__init__(self, fullPath, mode)

# This builds a job which applies both a change set and the local changes
# which are needed.
class DatabaseChangeSetJob(repository.ChangeSetJob):

    def removals(self, undo):
	for pkg in self.oldPackageList():
	    self.repos.erasePackageVersion(pkg.getName(), pkg.getVersion())
	    undo.removedPackage(pkg)

	for (fileId, fileVersion, fileObj) in self.oldFileList():
	    self.repos.eraseFileVersion(fileId, fileVersion)
	    undo.removedFile(fileId, fileVersion, fileObj)

	### FIXME: we don't restore the contents of config files!

    # remove the specified file and it's local branch
    def removeFile(self, fileId, version):
	# we need this object in case of an undo
	fileObj = self.repos.getFileVersion(fileId, version)
	self.oldFile(fileId, version, fileObj)

    # If retargetLocal is set, then localCs is for A->A.local whlie
    # origJob is A->B, so localCs needs to be changed to be B->B.local.
    # Otherwise, we're applying a rollback and origJob is B->A and
    # localCs is A->A.local, so it doesn't need retargeting.
    def __init__(self, repos, cs):
	assert(not cs.isAbstract())

	repository.ChangeSetJob.__init__(self, repos, cs)

	# remove old versions of the packages which are being added
	#
	# while we're here, package change sets may mark some files as removed;
	# we need to remember to remove those files, and make the paths for
	# those files candidates for removal package change sets also know
	# when file paths have changed, and those old paths are also candidates
	# for removal
	for csPkg in cs.getNewPackageList():
	    name = csPkg.getName()
	    oldVersion = csPkg.getOldVersion()

	    if not oldVersion:
		# we know this isn't an abstract change set (since this
		# class can't handle abstract change sets, and asserts
		# the away at the top of __init__() ), so this must be
		# a new package. no need to erase any old stuff then!
		continue

	    assert(repos.hasPackageVersion(name, oldVersion))

	    self.oldPackage(repos.getPackageVersion(name, oldVersion))

	    for fileId in csPkg.getOldFileList():
		(oldPath, oldFileVersion) = pkg.getFile(fileId)
		self.removeFile(fileId, oldFileVersion)

	# for each file which has changed, erase the old version of that
	# file from the repository
	for f in self.newFileList():
	    oldVersion = cs.getFileOldVersion(f.fileId())
	    if not oldVersion:
		# this is a new file; there is no old version to erase
		continue

	    self.removeFile(f.fileId(), oldVersion)

# Exception classes

class DatabaseError(Exception):
    """Base class for exceptions from the system database"""

    def __str__(self):
	return self.str

    def __init__(self, str = None):
	self.str = str

class RollbackError(Database):

    """Base class for exceptions related to applying rollbacks"""

class RollbackOrderError(RollbackError):

    """Raised when an attempt is made to apply rollbacks in the
       wrong order"""

    def __repr__(self):
	return "rollback %s can not be applied out of order" % self.name

    def __str__(self):
	return repr(self)

    def __init__(self, rollbackName):
	"""Create new new RollbackOrderError
	@param rollbackName: string represeting the name of the rollback
	which was trying to be applied out of order"""
	self.name = rollbackName

class RollbackDoesNotExist(RollbackError):

    """Raised when the system tries to access a rollback which isn't in
       the database"""

    def __repr__(self):
	return "rollback %s does not exist" % self.name

    def __str__(self):
	return repr(self)

    def __init__(self, rollbackName):
	"""Create new new RollbackOrderError
	@param rollbackName: string represeting the name of the rollback
	which does not exist"""
	self.name = rollbackName

class SourcePackageInstall(DatabaseError):

    def __str__(self):
	return "cannot install a source package onto the local system"

<commit_msg>remove unused files import, sort imports<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import changeset
import filecontents
import log
import os
import repository
import update
import versions

# Many comments in this file only make sense if you consider the case
# where something is being updated from version A to B, and the local
# branches are called A.local and B.local; this terminology is used throughout
# the comments in this file

# Every item which is inserted into the database really goes in twice;
# once as is, and then a LocalBranch is immediately created which reflects
# what's in the file system. The "as is" copy doesn't include data for
# any file except configuration files (which means while they can be used
# as the source for a change set, they can't be used as the target)
#
# The LocalBranch for each item in the database has normal header information,
# but the on-disk files are the real source of information on the package,
# including file types, hashes, etc

class Database(repository.LocalRepository):

    createBranches = 1

    # If the request is on the local branch, pull the package from the
    # filesystem
    def getPackageVersion(self, name, version):
	if not version.isLocal():
	    return repository.LocalRepository.getPackageVersion(self, name,
								version)
	parent = version.parent()
	pkg = repository.LocalRepository.getPackageVersion(self, name, parent)
	pkg.changeVersion(self.pkgGetFullVersion(pkg.getName(),
			  pkg.getVersion()))

	version.timeStamp = 1
	rc = update.buildLocalChanges(self, [(pkg, pkg, version)])
	if not rc:
	    raise DatabaseError, \
		    "failed to build local package for %s" % pkg.getName()

	assert(len(rc[1]) == 1)
	return rc[1][0][1]

    # takes an abstract change set and creates a differential change set
    # against a branch of the repository
    def rootChangeSet(self, absSet, branch):
	assert(absSet.isAbstract())

	# this has an empty source path template, which is only used to
	# construct the eraseFiles list anyway
	job = repository.ChangeSetJob(self, absSet)

	# abstract change sets cannot have eraseLists
	#assert(not eraseList)
	#assert(not eraseFiles)

	cs = changeset.ChangeSetFromAbstractChangeSet(absSet)

	for newPkg in job.newPackageList():
	    # FIXME
	    #
	    # this shouldn't be against branch, it should be against
	    # the version of the package already installed on the
	    # system. unfortunately we can't represent that yet.
	    pkgName = newPkg.getName()
	    oldVersion = self.pkgLatestVersion(pkgName, branch)
	    if not oldVersion:
		# new package; the Package.diff() right after this never
		# sets the abstract flag, so the right thing happens
		old = None
	    else:
		old = self.getPackageVersion(pkgName, oldVersion)

	    # we ignore pkgsNeeded; it doesn't mean much in this case
	    (pkgChgSet, filesNeeded, pkgsNeeded) =	    \
		    newPkg.diff(old, abstract = 0)
	    cs.newPackage(pkgChgSet)

	    for (fileId, oldVersion, newVersion, newPath) in filesNeeded:
		fileObj = job.getFile(fileId)
		assert(newVersion.equal(fileObj.version()))

		oldFile = None
		if oldVersion:
		    (oldFile, oldCont) = self.getFileVersion(fileId,
					    oldVersion, withContents = 1)

		(filecs, hash) = changeset.fileChangeSet(fileId, oldFile,
							 fileObj.file())

		cs.addFile(fileId, oldVersion, newVersion, filecs)
		if hash:
		    contType = changeset.ChangedFileTypes.file
		    cont = filecontents.FromChangeSet(absSet, fileId)
		    if oldVersion:
			(contType, cont) = changeset.fileContentsDiff(oldFile,
				    oldCont, fileObj.file(), cont)

		    cs.addFileContents(fileId, contType, cont)

	assert(not cs.validate())

	return cs

    # local changes includes the A->A.local portion of a rollback; if it
    # doesn't exist we need to compute that and save a rollback for this
    # transaction
    def commitChangeSet(self, cs, localRollback = None):
	assert(not cs.isAbstract())

	for pkg in cs.getNewPackageList():
	    if pkg.name.endswith(":sources"): raise SourcePackageInstall

	if not localRollback:
	    # create the change set from A->A.local
	    for newPkg in cs.getNewPackageList():
		name = newPkg.getName()
		old = newPkg.getOldVersion()
		pkgList = []
		if self.hasPackage(name) and old:
		    ver = old.fork(versions.LocalBranch(), sameVerRel = 1)
		    pkg = self.getPackageVersion(name, old)
		    assert(pkg)
		    pkgList.append((pkg, pkg, ver))
	    result = update.buildLocalChanges(self, pkgList, root = self.root)
	    if not result: return

	    (localChanges, retList) = result
	    fsPkgDict = {}
	    for (changed, fsPkg) in retList:
		fsPkgDict[fsPkg.getName()] = fsPkg

	    inverse = cs.makeRollback(self, configFiles = 1)
	else:
	    assert(0)
	    localChanges = localRollback

	# Build and commit A->B
	job = DatabaseChangeSetJob(self, cs)
	undo = repository.ChangeSetUndo(self)

	try:
	    # add new packages
	    job.commit(undo)
	    # build the list of things to commit
	    fsJob = update.FilesystemJob(self, cs, fsPkgDict, self.root)
	    # remove old packages
	    errList = fsJob.getErrorList()
	    if errList:
		for err in errList: log.error(err)
		undo.undo()
		# FIXME need a --force for this
		#return

	    job.removals(undo)
	except:
	    # this won't work it things got too far, but it won't hurt
	    # anything either
	    undo.undo()
	    raise

	# everything is in the database... save this so we can undo
	# it later.
	if not localRollback:
	    self.addRollback(inverse, localChanges)

	fsJob.apply()

    # this is called when a Repository wants to store a file; we never
    # want to do this; we copy files onto the filesystem after we've
    # created the LocalBranch
    def storeFileFromContents(self, contents, file, restoreContents):
	if file.isConfig():
	    return repository.LocalRepository.storeFileFromContents(self,
				contents, file, restoreContents)

    def close(self):
	repository.LocalRepository.close(self)

    def open(self, mode):
	repository.LocalRepository.open(self, mode)
	self.rollbackCache = self.top + "/rollbacks"
	self.rollbackStatus = self.rollbackCache + "/status"
	if not os.path.exists(self.rollbackCache):
	    os.mkdir(self.rollbackCache)
	if not os.path.exists(self.rollbackStatus):
	    self.firstRollback = 0
	    self.lastRollback = -1
	    self.writeRollbackStatus()
	else:
	    self.readRollbackStatus()

    def addRollback(self, reposChangeset, localChangeset):
	rpFn = self.rollbackCache + ("/rb.r.%d" % (self.lastRollback + 1))
	reposChangeset.writeToFile(rpFn)

	localFn = self.rollbackCache + ("/rb.l.%d" % (self.lastRollback + 1))
	localChangeset.writeToFile(localFn)

	self.lastRollback += 1
	self.writeRollbackStatus()

    # name looks like "r.%d"
    def removeRollback(self, name):
	rollback = int(name[2:])
	os.unlink(self.rollbackCache + "/rb.r.%d" % rollback)
	os.unlink(self.rollbackCache + "/rb.l.%d" % rollback)
	if rollback == self.lastRollback:
	    self.lastRollback -= 1
	    self.writeRollbackStatus()

    def writeRollbackStatus(self):
	newStatus = self.rollbackCache + ".new"

	f = open(newStatus, "w")
	f.write("%s %d\n" % (self.firstRollback, self.lastRollback))
	f.close()

	os.rename(newStatus, self.rollbackStatus)

    def getRollbackList(self):
	list = []
	for i in range(self.firstRollback, self.lastRollback + 1):
	    list.append("r.%d" % i)

	return list

    def readRollbackStatus(self):
	f = open(self.rollbackStatus)
	(first, last) = f.read()[:-1].split()
	self.firstRollback = int(first)
	self.lastRollback = int(last)
	f.close()

    def hasRollback(self, name):
	try:
	    num = int(name[2:])
	except ValueError:
	    return False

	if (num >= self.firstRollback and num <= self.lastRollback):
	    return True

	return False

    def getRollback(self, name):
	if not self.hasRollback(name): return None

	num = int(name[2:])

	rc = []
	for ch in [ "r", "l" ]:
	    name = self.rollbackCache + "/" + "rb.%c.%d" % (ch, num)
	    rc.append(changeset.ChangeSetFromFile(name,
						  justContentsForConfig = 1))

	return rc

    def applyRollbackList(self, names):
	last = self.lastRollback
	for name in names:
	    if not self.hasRollback(name):
		raise RollbackDoesNotExist(name)

	    num = int(name[2:])
	    if num != last:
		raise RollbackOrderError(name)
	    last -= 1

	for name in names:
	    (repostCs, localCs) = self.getRollback(name)
	    self.commitChangeSet(repostCs, localRollback = localCs)
	    self.removeRollback(name)

    def __init__(self, root, path, mode = "r"):
	self.root = root
	fullPath = root + "/" + path
	repository.LocalRepository.__init__(self, fullPath, mode)

# This builds a job which applies both a change set and the local changes
# which are needed.
class DatabaseChangeSetJob(repository.ChangeSetJob):

    def removals(self, undo):
	for pkg in self.oldPackageList():
	    self.repos.erasePackageVersion(pkg.getName(), pkg.getVersion())
	    undo.removedPackage(pkg)

	for (fileId, fileVersion, fileObj) in self.oldFileList():
	    self.repos.eraseFileVersion(fileId, fileVersion)
	    undo.removedFile(fileId, fileVersion, fileObj)

	### FIXME: we don't restore the contents of config files!

    # remove the specified file and it's local branch
    def removeFile(self, fileId, version):
	# we need this object in case of an undo
	fileObj = self.repos.getFileVersion(fileId, version)
	self.oldFile(fileId, version, fileObj)

    # If retargetLocal is set, then localCs is for A->A.local whlie
    # origJob is A->B, so localCs needs to be changed to be B->B.local.
    # Otherwise, we're applying a rollback and origJob is B->A and
    # localCs is A->A.local, so it doesn't need retargeting.
    def __init__(self, repos, cs):
	assert(not cs.isAbstract())

	repository.ChangeSetJob.__init__(self, repos, cs)

	# remove old versions of the packages which are being added
	#
	# while we're here, package change sets may mark some files as removed;
	# we need to remember to remove those files, and make the paths for
	# those files candidates for removal package change sets also know
	# when file paths have changed, and those old paths are also candidates
	# for removal
	for csPkg in cs.getNewPackageList():
	    name = csPkg.getName()
	    oldVersion = csPkg.getOldVersion()

	    if not oldVersion:
		# we know this isn't an abstract change set (since this
		# class can't handle abstract change sets, and asserts
		# the away at the top of __init__() ), so this must be
		# a new package. no need to erase any old stuff then!
		continue

	    assert(repos.hasPackageVersion(name, oldVersion))

	    self.oldPackage(repos.getPackageVersion(name, oldVersion))

	    for fileId in csPkg.getOldFileList():
		(oldPath, oldFileVersion) = pkg.getFile(fileId)
		self.removeFile(fileId, oldFileVersion)

	# for each file which has changed, erase the old version of that
	# file from the repository
	for f in self.newFileList():
	    oldVersion = cs.getFileOldVersion(f.fileId())
	    if not oldVersion:
		# this is a new file; there is no old version to erase
		continue

	    self.removeFile(f.fileId(), oldVersion)

# Exception classes

class DatabaseError(Exception):
    """Base class for exceptions from the system database"""

    def __str__(self):
	return self.str

    def __init__(self, str = None):
	self.str = str

class RollbackError(Database):

    """Base class for exceptions related to applying rollbacks"""

class RollbackOrderError(RollbackError):

    """Raised when an attempt is made to apply rollbacks in the
       wrong order"""

    def __repr__(self):
	return "rollback %s can not be applied out of order" % self.name

    def __str__(self):
	return repr(self)

    def __init__(self, rollbackName):
	"""Create new new RollbackOrderError
	@param rollbackName: string represeting the name of the rollback
	which was trying to be applied out of order"""
	self.name = rollbackName

class RollbackDoesNotExist(RollbackError):

    """Raised when the system tries to access a rollback which isn't in
       the database"""

    def __repr__(self):
	return "rollback %s does not exist" % self.name

    def __str__(self):
	return repr(self)

    def __init__(self, rollbackName):
	"""Create new new RollbackOrderError
	@param rollbackName: string represeting the name of the rollback
	which does not exist"""
	self.name = rollbackName

class SourcePackageInstall(DatabaseError):

    def __str__(self):
	return "cannot install a source package onto the local system"

<|endoftext|>