<commit_before>import time
from server_common.channel_access import caget, caput
from server_common.utilities import print_and_log

class Gateway(object):
    """A class for interacting with the EPICS gateway that creates the aliases used for implementing blocks"""

    def __init__(self, prefix, block_prefix, pvlist_file, pv_prefix):
        """Constructor.

        Args:
            prefix (string) : The prefix for the gateway
            block_prefix (string) : The block prefix
            pvlist_file (string) : Where to write the gateway file
            pv_prefix (string) : Prefix for instrument PVs
        """
        self._prefix = prefix
        self._block_prefix = block_prefix
        self._pvlist_file = pvlist_file
        self._pv_prefix = pv_prefix

    def exists(self):
        """Checks the gateway exists by querying on of the PVs.

        Returns:
            bool : Whether the gateway is running and is accessible
        """
        val = caget(self._prefix + "pvtotal")
        if val is None:
            return False
        else:
            return True

    def _reload(self):
        print_and_log("Reloading gateway")
        try:
            # Have to wait after put as the gateway does not do completion callbacks (it is not an IOC)
            caput(self._prefix + "newAsFlag", 1, False)
            while(caget(self._prefix + "newAsFlag") == 1) :
                time.sleep(1)
            print_and_log("Gateway reloaded")
        except Exception as err:
            print_and_log("Problem with reloading the gateway %s" % err)

    def _generate_alias_file(self, blocks=None):
        # Generate blocks.pvlist for gateway
        f = open(self._pvlist_file, 'w')
        header = """\
## Make ALLOW rules override DENY rules
EVALUATION ORDER DENY, ALLOW
## serve nothing by default, this is to avoid gateway loops
.*												DENY
## serve blockserver internal variables, including Flag variables needed by blockserver process to restart gateway
%sCS:GATEWAY:BLOCKSERVER:.*    				    ALLOW	ANYBODY	    1
## allow anybody to generate gateway reports
%sCS:GATEWAY:BLOCKSERVER:report[1-9]Flag		ALLOW	ANYBODY		1
""" % ( self._pv_prefix, self._pv_prefix )
        f.write(header)
        if blocks is not None:
            for name, value in blocks.iteritems():
                lines = self._generate_alias(value.name, value.pv, value.local)
                for l in lines:
                    f.write(l)
        # Add a blank line at the end!
        f.write("\n")
        f.close()

    def _generate_alias(self, blockname, pv, local):
        print("Creating block: %s for %s" % (blockname, pv))
        lines = list()
        if pv.endswith(".VAL"):
            # Strip off the .VAL
            pv = pv.rstrip(".VAL")
        if pv.endswith(":SP"):
            # The block points at a setpoint
            lines.append("## The block points at a :SP, so it needs an optional group as genie_python will append an additional :SP\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s%s\\2\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s\\2\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        elif pv.endswith(".RBV"):
            # The block points at a readback value (most likely for a motor)
            lines.append("## The block points at a .RBV, so it needs entries for both reading the RBV and for the rest\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv.rstrip(".RBV")))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname, pv.rstrip(".RBV")))
        else:
            # Standard case
            lines.append("## Standard block with entries for matching :SP and :SP:RBV as well as .EGU\n")
            if local:
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
        return lines

    def set_new_aliases(self, blocks):
        """Creates the aliases for the blocks and restarts the gateway.

        Args:
            blocks (OrderedDict) : The blocks that belong to the configuration
        """
        self._generate_alias_file(blocks)
        self._reload()<commit_msg>Wrapped some of the lines of code to match PEP8<commit_after>import time
from server_common.channel_access import caget, caput
from server_common.utilities import print_and_log

class Gateway(object):
    """A class for interacting with the EPICS gateway that creates the aliases used for implementing blocks"""

    def __init__(self, prefix, block_prefix, pvlist_file, pv_prefix):
        """Constructor.

        Args:
            prefix (string) : The prefix for the gateway
            block_prefix (string) : The block prefix
            pvlist_file (string) : Where to write the gateway file
            pv_prefix (string) : Prefix for instrument PVs
        """
        self._prefix = prefix
        self._block_prefix = block_prefix
        self._pvlist_file = pvlist_file
        self._pv_prefix = pv_prefix

    def exists(self):
        """Checks the gateway exists by querying on of the PVs.

        Returns:
            bool : Whether the gateway is running and is accessible
        """
        val = caget(self._prefix + "pvtotal")
        if val is None:
            return False
        else:
            return True

    def _reload(self):
        print_and_log("Reloading gateway")
        try:
            # Have to wait after put as the gateway does not do completion callbacks (it is not an IOC)
            caput(self._prefix + "newAsFlag", 1, False)
            while(caget(self._prefix + "newAsFlag") == 1) :
                time.sleep(1)
            print_and_log("Gateway reloaded")
        except Exception as err:
            print_and_log("Problem with reloading the gateway %s" % err)

    def _generate_alias_file(self, blocks=None):
        # Generate blocks.pvlist for gateway
        f = open(self._pvlist_file, 'w')
        header = """\
## Make ALLOW rules override DENY rules
EVALUATION ORDER DENY, ALLOW
## serve nothing by default, this is to avoid gateway loops
.*												DENY
## serve blockserver internal variables, including Flag variables needed by blockserver process to restart gateway
%sCS:GATEWAY:BLOCKSERVER:.*    				    ALLOW	ANYBODY	    1
## allow anybody to generate gateway reports
%sCS:GATEWAY:BLOCKSERVER:report[1-9]Flag		ALLOW	ANYBODY		1
""" % (self._pv_prefix, self._pv_prefix)
        f.write(header)
        if blocks is not None:
            for name, value in blocks.iteritems():
                lines = self._generate_alias(value.name, value.pv, value.local)
                for l in lines:
                    f.write(l)
        # Add a blank line at the end!
        f.write("\n")
        f.close()

    def _generate_alias(self, blockname, pv, local):
        print("Creating block: %s for %s" % (blockname, pv))
        lines = list()
        if pv.endswith(".VAL"):
            # Strip off the .VAL
            pv = pv.rstrip(".VAL")
        if pv.endswith(":SP"):
            # The block points at a setpoint
            lines.append("## The block points at a :SP, so it needs an optional group as genie_python will append an additional :SP\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s%s\n' %
                             (self._pv_prefix, self._block_prefix, blockname, self._pv_prefix, pv))
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s%s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                                blockname, self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s\(:SP\)?    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\(:SP\)?\([.:].*\)    ALIAS    %s\\2\n' % (self._pv_prefix, self._block_prefix,
                                                                               blockname, pv))
        elif pv.endswith(".RBV"):
            # The block points at a readback value (most likely for a motor)
            lines.append("## The block points at a .RBV, so it needs entries for both reading the RBV and for the rest\n")
            if local:
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv.rstrip(".RBV")))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any extras like :RBV or .EGU
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv.rstrip(".RBV")))
        else:
            # Standard case
            lines.append("## Standard block with entries for matching :SP and :SP:RBV as well as .EGU\n")
            if local:
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s    ALIAS    %s%s\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                            self._pv_prefix, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s%s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                         self._pv_prefix, pv))
            else:
                # pv_prefix is hard-coded for non-local PVs
                # Pattern match is for picking up any any SP or SP:RBV
                lines.append('%s%s%s    ALIAS    %s\n' % (self._pv_prefix, self._block_prefix, blockname, pv))
                lines.append('%s%s%s\([.:].*\)    ALIAS    %s\\1\n' % (self._pv_prefix, self._block_prefix, blockname,
                                                                       pv))
        return lines

    def set_new_aliases(self, blocks):
        """Creates the aliases for the blocks and restarts the gateway.

        Args:
            blocks (OrderedDict) : The blocks that belong to the configuration
        """
        self._generate_alias_file(blocks)
        self._reload()<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.contrib.messages.views import SuccessMessageMixin
from django.core.urlresolvers import reverse_lazy
from django.forms.util import ErrorList
from django.forms import forms
from django.http import HttpResponseNotFound
from django.http.response import HttpResponseForbidden
from django.utils import timezone
from django.utils.translation import ugettext as _
from django.views import generic
from rest_framework import permissions

from braces.views import LoginRequiredMixin
from rest_framework import mixins
from rest_framework.filters import SearchFilter
from rest_framework.viewsets import GenericViewSet

from ..listings.seralizers import ListingSerializer
from ..settings import bazaar_settings
from .forms import ListingForm, PublishingForm
from .models import Listing, ListingSet, Publishing
from .stores import stores_loader
from ..mixins import BazaarPrefixMixin, FilterMixin, FilterSortableListView

class ListingListView(LoginRequiredMixin, BazaarPrefixMixin, FilterMixin, generic.ListView):
    model = Listing
    paginate_by = 50
    template_name = "bazaar/listings/listing_list.html"
    filter_name = "listing_filter"

    filter_class = bazaar_settings.LISTING_FILTER

    def get_queryset(self):
        qs = super(ListingListView, self).get_queryset()
        # prefetch list of values, populated by inherit items, plus items from each store
        prefetch_list = ["listing_sets__product", "publishings__store"]
        for manager in stores_loader.get_all_store_managers():
            prefetch_list.extend(manager.get_store_extra("prefetch_list"))
        return qs.prefetch_related(*prefetch_list)

    def get_context_data(self, **kwargs):
        context = super(ListingListView, self).get_context_data(**kwargs)

        # populate forms and actions (tasks).
        tasks = []
        forms = []
        for manager in stores_loader.get_all_store_managers():
            forms.extend(manager.get_store_forms())
            tasks.append((manager.get_store_name(), manager.get_store_actions()))
        context['tasks'] = tasks
        # from template create a menu for every store with title == store_name
        # add all forms to context
        for form in forms:
            context[form.name] = form.form

        return context

class ListingDetailView(LoginRequiredMixin, generic.DetailView):
    model = Listing
    template_name = "bazaar/listings/listing_detail_view.html"

class ListingDeleteView(LoginRequiredMixin, generic.DeleteView):
    model = Listing
    success_url = reverse_lazy("bazaar:listings-list")

    def delete(self, request, *args, **kwargs):

        listing = self.get_object()
        has_publishings = Publishing.objects.filter(listing=listing).exists()
        if has_publishings:
            return HttpResponseForbidden()

        return super(ListingDeleteView, self).delete(request, *args, **kwargs)

class ListingUpdateView(LoginRequiredMixin, generic.FormView):
    form_class = ListingForm
    template_name = "bazaar/listings/listing_form.html"
    listing_to_update = None
    error_response = None
    success_url = reverse_lazy("bazaar:listings-list")

    def get_context_data(self, **kwargs):
        context = super(ListingUpdateView, self).get_context_data(**kwargs)
        context["listing"] = self.listing_to_update
        return context

    def get(self, request, *args, **kwargs):
        get_result = super(ListingUpdateView, self).get(request, *args, **kwargs)
        return self.error_response or get_result

    def get_initial(self):
        # Populate ticks in BooleanFields
        initial = {}
        listing_id = self.kwargs.get("pk", None)
        if listing_id:
            try:
                self.listing_to_update = Listing.objects.get(pk=listing_id)
                initial["title"] = self.listing_to_update.title
                initial["picture_url"] = self.listing_to_update.picture_url
                initial["description"] = self.listing_to_update.description
                initial["quantity"] = int(self.listing_to_update.listing_sets.first().quantity)
                initial["product"] = self.listing_to_update.listing_sets.first().product.id
            except Listing.DoesNotExist:
                self.error_response = HttpResponseNotFound()
        return initial

    def get_success_url(self):
        return reverse_lazy("bazaar:listings-detail", kwargs={'pk': self.object.id})

    def _retrieve_listingset(self, form):
        listing_id = self.kwargs.get("pk", None)
        if listing_id:
            return ListingSet.objects.get(listing_id=listing_id)
        return None

    def form_valid(self, form):
        """
        Even if it's a valid form, it's not possible edit/update the listing set when there are associated publishings
        """
        # TODO: WARNING: This valid form assumed that only one-type product and one listingset per listing.
        if self.listing_to_update:
            # Update Listing
            listing = Listing(
                title=form.cleaned_data.get("title"),
                picture_url=form.cleaned_data.get("picture_url", None),
                description=form.cleaned_data.get("description", None),
                id=self.listing_to_update.id
            )
            publishings_exist = Publishing.objects.select_related('listing')\
                .filter(listing__id=self.listing_to_update.id).exists()
        else:
            # Create listing
            listing = Listing.objects.create(
                title=form.cleaned_data.get("title"),
                picture_url=form.cleaned_data.get("picture_url", None),
                description=form.cleaned_data.get("description", None)
            )
            publishings_exist = False
        try:
            listing_set = self._retrieve_listingset(form)
        except ListingSet.DoesNotExist:
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Update is denied. No associated ListingSet exists."))
            return self.form_invalid(form)
        except ListingSet.MultipleObjectsReturned:
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Update is denied. Allowed only one-product and one listingset per listing."))
            return self.form_invalid(form)

        product = form.get_product()
        if not listing_set:
            listing_set, is_created = ListingSet.objects.get_or_create(listing=listing,
                                                                       product=product,
                                                                       quantity=int(form.cleaned_data.get("quantity")))

        if publishings_exist and \
                (listing_set.quantity != int(form.cleaned_data.get("quantity")) or listing_set.product != product):
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Updating listingset is denied. It's not allowed update/edit listingset"
                            " when listing is associated at least to one publishing."))
            return self.form_invalid(form)

        self.object = listing
        listing.listing_sets = [listing_set]
        listing.save()

        return super(ListingUpdateView, self).form_valid(form)

class PublishingTagsMixin(object):
        def get_context_data(self, **kwargs):
            # Call the base implementation first to get a context
            context = super(PublishingTagsMixin, self).get_context_data(**kwargs)
            context['PUBLISHING_STATUS_CHOICES'] = dict(Publishing.PUBLISHING_STATUS_CHOICES)
            return context

class PublishingListView(LoginRequiredMixin, PublishingTagsMixin, FilterSortableListView):
    template_name = 'bazaar/listings/publishing_list.html'
    model = Publishing
    paginate_by = 100
    sort_fields = (
        'external_id', 'id', 'last_modified',
        'pub_date', 'status', 'store__name', 'is_active', 'listing__title'
    )

class PublishingCreateView(SuccessMessageMixin, LoginRequiredMixin, PublishingTagsMixin, generic.CreateView):
    model = Publishing
    form_class = PublishingForm
    success_url = reverse_lazy("bazaar:publishings-list")
    template_name = 'bazaar/listings/publishing_form.html'

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.last_modified = timezone.now()
        self.object.save()
        self.success_url = reverse_lazy("publishings-update", kwargs={'pk': self.object.id})
        return super(PublishingCreateView, self).form_valid(form)

class PublishingDeleteView(LoginRequiredMixin, generic.DeleteView):
    model = Publishing
    success_url = reverse_lazy("bazaar:publishings-list")

class PublishingUpdateView(SuccessMessageMixin, LoginRequiredMixin, PublishingTagsMixin, generic.UpdateView):
    model = Publishing
    form_class = PublishingForm
    template_name = 'bazaar/listings/publishing_form.html'

    def get_success_url(self):
        return reverse_lazy("bazaar:publishings-update", kwargs={'pk': self.object.id})

class ListingViewSet(mixins.ListModelMixin, GenericViewSet):
    model = Listing
    serializer_class = ListingSerializer
    paginate_by = 10
    permission_classes = (permissions.IsAuthenticated,)
    filter_backends = (SearchFilter,)
    search_fields = ('title', 'products__name')
<commit_msg>Fixed listing view due new validation logic in listing form<commit_after>from __future__ import unicode_literals

from django.contrib.messages.views import SuccessMessageMixin
from django.core.urlresolvers import reverse_lazy
from django.forms.util import ErrorList
from django.forms import forms
from django.http import HttpResponseNotFound
from django.http.response import HttpResponseForbidden
from django.utils import timezone
from django.utils.translation import ugettext as _
from django.views import generic
from rest_framework import permissions

from braces.views import LoginRequiredMixin
from rest_framework import mixins
from rest_framework.filters import SearchFilter
from rest_framework.viewsets import GenericViewSet

from ..goods.models import Product
from ..listings.seralizers import ListingSerializer
from ..settings import bazaar_settings
from .forms import ListingForm, PublishingForm
from .models import Listing, ListingSet, Publishing
from .stores import stores_loader
from ..mixins import BazaarPrefixMixin, FilterMixin, FilterSortableListView

class ListingListView(LoginRequiredMixin, BazaarPrefixMixin, FilterMixin, generic.ListView):
    model = Listing
    paginate_by = 50
    template_name = "bazaar/listings/listing_list.html"
    filter_name = "listing_filter"

    filter_class = bazaar_settings.LISTING_FILTER

    def get_queryset(self):
        qs = super(ListingListView, self).get_queryset()
        # prefetch list of values, populated by inherit items, plus items from each store
        prefetch_list = ["listing_sets__product", "publishings__store"]
        for manager in stores_loader.get_all_store_managers():
            prefetch_list.extend(manager.get_store_extra("prefetch_list"))
        return qs.prefetch_related(*prefetch_list)

    def get_context_data(self, **kwargs):
        context = super(ListingListView, self).get_context_data(**kwargs)

        # populate forms and actions (tasks).
        tasks = []
        forms = []
        for manager in stores_loader.get_all_store_managers():
            forms.extend(manager.get_store_forms())
            tasks.append((manager.get_store_name(), manager.get_store_actions()))
        context['tasks'] = tasks
        # from template create a menu for every store with title == store_name
        # add all forms to context
        for form in forms:
            context[form.name] = form.form

        return context

class ListingDetailView(LoginRequiredMixin, generic.DetailView):
    model = Listing
    template_name = "bazaar/listings/listing_detail_view.html"

class ListingDeleteView(LoginRequiredMixin, generic.DeleteView):
    model = Listing
    success_url = reverse_lazy("bazaar:listings-list")

    def delete(self, request, *args, **kwargs):

        listing = self.get_object()
        has_publishings = Publishing.objects.filter(listing=listing).exists()
        if has_publishings:
            return HttpResponseForbidden()

        return super(ListingDeleteView, self).delete(request, *args, **kwargs)

class ListingUpdateView(LoginRequiredMixin, generic.FormView):
    form_class = ListingForm
    template_name = "bazaar/listings/listing_form.html"
    listing_to_update = None
    error_response = None
    success_url = reverse_lazy("bazaar:listings-list")

    def get_context_data(self, **kwargs):
        context = super(ListingUpdateView, self).get_context_data(**kwargs)
        context["listing"] = self.listing_to_update
        return context

    def get(self, request, *args, **kwargs):
        get_result = super(ListingUpdateView, self).get(request, *args, **kwargs)
        return self.error_response or get_result

    def get_initial(self):
        # Populate ticks in BooleanFields
        initial = {}
        listing_id = self.kwargs.get("pk", None)
        if listing_id:
            try:
                self.listing_to_update = Listing.objects.get(pk=listing_id)
                initial["title"] = self.listing_to_update.title
                initial["picture_url"] = self.listing_to_update.picture_url
                initial["description"] = self.listing_to_update.description
                initial["quantity"] = int(self.listing_to_update.listing_sets.first().quantity)
                initial["product"] = self.listing_to_update.listing_sets.first().product.id
            except Listing.DoesNotExist:
                self.error_response = HttpResponseNotFound()
        return initial

    def get_success_url(self):
        return reverse_lazy("bazaar:listings-detail", kwargs={'pk': self.object.id})

    def _retrieve_listingset(self, form):
        listing_id = self.kwargs.get("pk", None)
        if listing_id:
            return ListingSet.objects.get(listing_id=listing_id)
        return None

    def _retrieve_product(self, form):
        product_id = form.clean_product()
        return Product.objects.get(id=product_id)

    def form_valid(self, form):
        """
        Even if it's a valid form, it's not possible edit/update the listing set when there are associated publishings
        """
        # TODO: WARNING: This valid form assumed that only one-type product and one listingset per listing.
        if self.listing_to_update:
            # Update Listing
            listing = Listing(
                title=form.cleaned_data.get("title"),
                picture_url=form.cleaned_data.get("picture_url", None),
                description=form.cleaned_data.get("description", None),
                id=self.listing_to_update.id
            )
            publishings_exist = Publishing.objects.select_related('listing')\
                .filter(listing__id=self.listing_to_update.id).exists()
        else:
            # Create listing
            listing = Listing.objects.create(
                title=form.cleaned_data.get("title"),
                picture_url=form.cleaned_data.get("picture_url", None),
                description=form.cleaned_data.get("description", None)
            )
            publishings_exist = False
        try:
            listing_set = self._retrieve_listingset(form)
        except ListingSet.DoesNotExist:
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Update is denied. No associated ListingSet exists."))
            return self.form_invalid(form)
        except ListingSet.MultipleObjectsReturned:
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Update is denied. Allowed only one-product and one listingset per listing."))
            return self.form_invalid(form)

        product = self._retrieve_product(form)
        if not listing_set:
            listing_set, is_created = ListingSet.objects.get_or_create(listing=listing,
                                                                       product=product,
                                                                       quantity=int(form.cleaned_data.get("quantity")))

        if publishings_exist and \
                (listing_set.quantity != int(form.cleaned_data.get("quantity")) or listing_set.product != product):
            errors = form._errors.setdefault(forms.NON_FIELD_ERRORS, ErrorList())
            errors.append(_("Updating listingset is denied. It's not allowed update/edit listingset"
                            " when listing is associated at least to one publishing."))
            return self.form_invalid(form)

        self.object = listing
        listing.listing_sets = [listing_set]
        listing.save()

        return super(ListingUpdateView, self).form_valid(form)

class PublishingTagsMixin(object):
        def get_context_data(self, **kwargs):
            # Call the base implementation first to get a context
            context = super(PublishingTagsMixin, self).get_context_data(**kwargs)
            context['PUBLISHING_STATUS_CHOICES'] = dict(Publishing.PUBLISHING_STATUS_CHOICES)
            return context

class PublishingListView(LoginRequiredMixin, PublishingTagsMixin, FilterSortableListView):
    template_name = 'bazaar/listings/publishing_list.html'
    model = Publishing
    paginate_by = 100
    sort_fields = (
        'external_id', 'id', 'last_modified',
        'pub_date', 'status', 'store__name', 'is_active', 'listing__title'
    )

class PublishingCreateView(SuccessMessageMixin, LoginRequiredMixin, PublishingTagsMixin, generic.CreateView):
    model = Publishing
    form_class = PublishingForm
    success_url = reverse_lazy("bazaar:publishings-list")
    template_name = 'bazaar/listings/publishing_form.html'

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.last_modified = timezone.now()
        self.object.save()
        self.success_url = reverse_lazy("publishings-update", kwargs={'pk': self.object.id})
        return super(PublishingCreateView, self).form_valid(form)

class PublishingDeleteView(LoginRequiredMixin, generic.DeleteView):
    model = Publishing
    success_url = reverse_lazy("bazaar:publishings-list")

class PublishingUpdateView(SuccessMessageMixin, LoginRequiredMixin, PublishingTagsMixin, generic.UpdateView):
    model = Publishing
    form_class = PublishingForm
    template_name = 'bazaar/listings/publishing_form.html'

    def get_success_url(self):
        return reverse_lazy("bazaar:publishings-update", kwargs={'pk': self.object.id})

class ListingViewSet(mixins.ListModelMixin, GenericViewSet):
    model = Listing
    serializer_class = ListingSerializer
    paginate_by = 10
    permission_classes = (permissions.IsAuthenticated,)
    filter_backends = (SearchFilter,)
    search_fields = ('title', 'products__name')
<|endoftext|>


<commit_before># Copyright (C) 2014 Forest and Biomass Romania
# Copyright (C) 2020 NextERP Romania
# Copyright (C) 2020 Terrabit
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from odoo import api, fields, models

class StockValuationLayer(models.Model):
    _inherit = "stock.valuation.layer"

    valued_type = fields.Char()
    invoice_line_id = fields.Many2one("account.move.line", string="Invoice Line")
    invoice_id = fields.Many2one("account.move", string="Invoice")
    account_id = fields.Many2one(
        "account.account", compute="_compute_account", store=True
    )

    @api.depends("product_id", "account_move_id")
    def _compute_account(self):
        for svl in self:
            account = self.env["account.account"]
            if not svl.account_move_id:
                loc_dest = svl.stock_move_id.location_dest_id
                loc_scr = svl.stock_move_id.location_id
                account = (
                    svl.product_id.property_stock_valuation_account_id
                    or svl.product_id.categ_id.property_stock_valuation_account_id
                )
                if svl.value > 0 and loc_dest.property_stock_valuation_account_id:
                    account = loc_dest.property_stock_valuation_account_id
                if svl.value < 0 and loc_scr.property_stock_valuation_account_id:
                    account = loc_scr.property_stock_valuation_account_id
            else:
                for aml in svl.account_move_id.line_ids:
                    if aml.account_id.code[0] in ["2", "3"]:
                        if aml.balance < 0 and svl.value < 0:
                            account = aml.account_id
                            break
                        if aml.balance > 0 and svl.value > 0:
                            account = aml.account_id
                            break
            svl.account_id = account

    def init(self):
        """ This method will compute values for valuation layer valued_type"""
        val_layers = self.search(
            ["|", ("valued_type", "=", False), ("valued_type", "=", "")]
        )
        val_types = self.env["stock.move"]._get_valued_types()
        val_types = [
            val
            for val in val_types
            if val not in ["in", "out", "dropshipped", "dropshipped_returned"]
        ]
        for layer in val_layers:
            if layer.stock_move_id:
                for valued_type in val_types:
                    if getattr(layer.stock_move_id, "_is_%s" % valued_type)():
                        layer.valued_type = valued_type
                        continue

    @api.model_create_multi
    def create(self, vals_list):
        for values in vals_list:
            if "valued_type" not in values and "stock_valuation_layer_id" in values:
                svl = self.env["stock.valuation.layer"].browse(
                    values["stock_valuation_layer_id"]
                )
                if svl:
                    values["valued_type"] = svl.valued_type
        return super(StockValuationLayer, self).create(vals_list)

    def _compute_invoice_line_id(self):
        for svl in self:
            invoice_lines = self.env["account.move.line"]
            stock_move = svl.stock_move_id
            if "reception" in svl.valued_type:
                invoice_lines = stock_move.purchase_line_id.invoice_lines
            if "delivery" in svl.valued_type:
                invoice_lines = stock_move.sale_line_id.invoice_lines

            if len(invoice_lines) == 1:
                svl.invoice_line_id = invoice_lines
            else:
                for line in invoice_lines:
                    if stock_move.date.date() == line.move_id.date:
                        svl.invoice_line_id = line
<commit_msg>Update account.<commit_after># Copyright (C) 2014 Forest and Biomass Romania
# Copyright (C) 2020 NextERP Romania
# Copyright (C) 2020 Terrabit
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
from odoo import api, fields, models

class StockValuationLayer(models.Model):
    _inherit = "stock.valuation.layer"

    valued_type = fields.Char()
    invoice_line_id = fields.Many2one("account.move.line", string="Invoice Line")
    invoice_id = fields.Many2one("account.move", string="Invoice")
    account_id = fields.Many2one(
        "account.account", compute="_compute_account", store=True
    )

    @api.depends("product_id", "account_move_id")
    def _compute_account(self):
        for svl in self:
            account = self.env["account.account"]
            if not svl.account_move_id:
                loc_dest = svl.stock_move_id.location_dest_id
                loc_scr = svl.stock_move_id.location_id
                account = (
                    svl.product_id.property_stock_valuation_account_id
                    or svl.product_id.categ_id.property_stock_valuation_account_id
                )
                if svl.value > 0 and loc_dest.property_stock_valuation_account_id:
                    account = loc_dest.property_stock_valuation_account_id
                if svl.value < 0 and loc_scr.property_stock_valuation_account_id:
                    account = loc_scr.property_stock_valuation_account_id
            else:
                for aml in svl.account_move_id.line_ids:
                    if aml.account_id.code[0] in ["2", "3"]:
                        if aml.balance <= 0 and svl.value <= 0:
                            account = aml.account_id
                            break
                        if aml.balance > 0 and svl.value > 0:
                            account = aml.account_id
                            break
            svl.account_id = account

    def init(self):
        """ This method will compute values for valuation layer valued_type"""
        val_layers = self.search(
            ["|", ("valued_type", "=", False), ("valued_type", "=", "")]
        )
        val_types = self.env["stock.move"]._get_valued_types()
        val_types = [
            val
            for val in val_types
            if val not in ["in", "out", "dropshipped", "dropshipped_returned"]
        ]
        for layer in val_layers:
            if layer.stock_move_id:
                for valued_type in val_types:
                    if getattr(layer.stock_move_id, "_is_%s" % valued_type)():
                        layer.valued_type = valued_type
                        continue

    @api.model_create_multi
    def create(self, vals_list):
        for values in vals_list:
            if "valued_type" not in values and "stock_valuation_layer_id" in values:
                svl = self.env["stock.valuation.layer"].browse(
                    values["stock_valuation_layer_id"]
                )
                if svl:
                    values["valued_type"] = svl.valued_type
        return super(StockValuationLayer, self).create(vals_list)

    def _compute_invoice_line_id(self):
        for svl in self:
            invoice_lines = self.env["account.move.line"]
            stock_move = svl.stock_move_id
            if "reception" in svl.valued_type:
                invoice_lines = stock_move.purchase_line_id.invoice_lines
            if "delivery" in svl.valued_type:
                invoice_lines = stock_move.sale_line_id.invoice_lines

            if len(invoice_lines) == 1:
                svl.invoice_line_id = invoice_lines
            else:
                for line in invoice_lines:
                    if stock_move.date.date() == line.move_id.date:
                        svl.invoice_line_id = line
<|endoftext|>


<commit_before>"""Provide analysis of input files by chromosomal regions.

Handle splitting and analysis of files from chromosomal subsets separated by
no-read regions.
"""
import collections
import os

import toolz as tz

from bcbio.distributed.split import parallel_split_combine

def get_max_counts(samples):
    """Retrieve number of regions that can be processed in parallel from current samples.
    """
    counts = []
    for data in (x[0] for x in samples):
        count = tz.get_in(["config", "algorithm", "callable_count"], data, 1)
        vcs = tz.get_in(["config", "algorithm", "variantcaller"], data, [])
        if isinstance(vcs, basestring):
            vcs = [vcs]
        if vcs:
            count *= len(vcs)
        counts.append(count)
    return max(counts)

# ## BAM preparation

def to_safestr(region):
    if region[0] in ["nochrom", "noanalysis"]:
        return region[0]
    else:
        return "_".join([str(x) for x in region])

# ## Split and delayed BAM combine

def _split_by_regions(dirname, out_ext, in_key):
    """Split a BAM file data analysis into chromosomal regions.
    """
    def _do_work(data):
        # XXX Need to move retrieval of regions into preparation to avoid
        # need for files when running in non-shared filesystems
        with open(data["config"]["algorithm"]["callable_regions"]) as in_handle:
            regions = [(xs[0], int(xs[1]), int(xs[2])) for xs in
                       (l.rstrip().split("\t") for l in in_handle) if (len(xs) >= 3 and
                                                                       not xs[0].startswith(("track", "browser",)))]
        bam_file = data[in_key]
        if bam_file is None:
            return None, []
        part_info = []
        base_out = os.path.splitext(os.path.basename(bam_file))[0]
        nowork = [["nochrom"], ["noanalysis", data["config"]["algorithm"]["non_callable_regions"]]]
        for region in regions + nowork:
            out_dir = os.path.join(data["dirs"]["work"], dirname, data["name"][-1], region[0])
            region_outfile = os.path.join(out_dir, "%s-%s%s" %
                                          (base_out, to_safestr(region), out_ext))
            part_info.append((region, region_outfile))
        out_file = os.path.join(data["dirs"]["work"], dirname, data["name"][-1],
                                "%s%s" % (base_out, out_ext))
        return out_file, part_info
    return _do_work

def _add_combine_info(output, combine_map, file_key):
    """Do not actually combine, but add details for later combining work.

    Each sample will contain information on the out file and additional files
    to merge, enabling other splits and recombines without losing information.
    """
    files_per_output = collections.defaultdict(list)
    for part_file, out_file in combine_map.items():
        files_per_output[out_file].append(part_file)
    out_by_file = collections.defaultdict(list)
    out = []
    for data in output:
        # Do not pass along nochrom, noanalysis regions
        if data["region"][0] not in ["nochrom", "noanalysis"]:
            cur_file = data[file_key]
            # If we didn't process, no need to add combine information
            if cur_file in combine_map:
                out_file = combine_map[cur_file]
                if "combine" not in data:
                    data["combine"] = {}
                data["combine"][file_key] = {"out": out_file,
                                             "extras": files_per_output.get(out_file, [])}
                out_by_file[out_file].append(data)
            elif cur_file:
                out_by_file[cur_file].append(data)
            else:
                out.append([data])
    for samples in out_by_file.values():
        regions = [x["region"] for x in samples]
        region_bams = [x["work_bam"] for x in samples]
        assert len(regions) == len(region_bams)
        if len(set(region_bams)) == 1:
            region_bams = [region_bams[0]]
        data = samples[0]
        data["region_bams"] = region_bams
        data["region"] = regions
        out.append([data])
    return out

def parallel_prep_region(samples, run_parallel):
    """Perform full pre-variant calling BAM prep work on regions.
    """
    file_key = "work_bam"
    split_fn = _split_by_regions("bamprep", "-prep.bam", file_key)
    # identify samples that do not need preparation -- no recalibration or realignment
    extras = []
    torun = []
    for data in [x[0] for x in samples]:
        if data.get("work_bam"):
            data["align_bam"] = data["work_bam"]
        a = data["config"]["algorithm"]
        if (not a.get("recalibrate") and not a.get("realign") and not a.get("variantcaller", "gatk")):
            extras.append([data])
        elif not data.get(file_key):
            extras.append([data])
        else:
            torun.append([data])
    return extras + parallel_split_combine(torun, split_fn, run_parallel,
                                           "piped_bamprep", _add_combine_info, file_key, ["config"])

def delayed_bamprep_merge(samples, run_parallel):
    """Perform a delayed merge on regional prepared BAM files.
    """
    needs_merge = False
    for data in samples:
        if (data[0]["config"]["algorithm"].get("merge_bamprep", True) and
              "combine" in data[0]):
            needs_merge = True
            break
    if needs_merge:
        return run_parallel("delayed_bam_merge", samples)
    else:
        return samples

# ## Utilities

def clean_sample_data(samples):
    """Clean unnecessary information from sample data, reducing size for message passing.
    """
    out = []
    for data in (x[0] for x in samples):
        if "dirs" in data:
            data["dirs"] = {"work": data["dirs"]["work"], "galaxy": data["dirs"]["galaxy"],
                            "fastq": data["dirs"].get("fastq")}
        data["config"] = {"algorithm": data["config"]["algorithm"],
                          "resources": data["config"]["resources"]}
        for remove_attr in ["config_file", "regions", "algorithm"]:
            data.pop(remove_attr, None)
        out.append([data])
    return out
<commit_msg>Useful error message when missing callable_regions<commit_after>"""Provide analysis of input files by chromosomal regions.

Handle splitting and analysis of files from chromosomal subsets separated by
no-read regions.
"""
import collections
import os

import toolz as tz

from bcbio.distributed.split import parallel_split_combine
from bcbio.pipeline import datadict as dd

def get_max_counts(samples):
    """Retrieve number of regions that can be processed in parallel from current samples.
    """
    counts = []
    for data in (x[0] for x in samples):
        count = tz.get_in(["config", "algorithm", "callable_count"], data, 1)
        vcs = tz.get_in(["config", "algorithm", "variantcaller"], data, [])
        if isinstance(vcs, basestring):
            vcs = [vcs]
        if vcs:
            count *= len(vcs)
        counts.append(count)
    return max(counts)

# ## BAM preparation

def to_safestr(region):
    if region[0] in ["nochrom", "noanalysis"]:
        return region[0]
    else:
        return "_".join([str(x) for x in region])

# ## Split and delayed BAM combine

def _split_by_regions(dirname, out_ext, in_key):
    """Split a BAM file data analysis into chromosomal regions.
    """
    def _do_work(data):
        # XXX Need to move retrieval of regions into preparation to avoid
        # need for files when running in non-shared filesystems
        callable_regions = tz.get_in(["config", "algorithm", "callable_regions"], data)
        if not callable_regions:
            raise ValueError("Did not find any callable regions for sample: %s\n"
                             "Check 'align/%s/*-callableblocks.bed' and 'regions' to examine callable regions"
                             % (dd.get_sample_name(data), dd.get_sample_name(data)))
        with open(callable_regions) as in_handle:
            regions = [(xs[0], int(xs[1]), int(xs[2])) for xs in
                       (l.rstrip().split("\t") for l in in_handle) if (len(xs) >= 3 and
                                                                       not xs[0].startswith(("track", "browser",)))]
        bam_file = data[in_key]
        if bam_file is None:
            return None, []
        part_info = []
        base_out = os.path.splitext(os.path.basename(bam_file))[0]
        nowork = [["nochrom"], ["noanalysis", data["config"]["algorithm"]["non_callable_regions"]]]
        for region in regions + nowork:
            out_dir = os.path.join(data["dirs"]["work"], dirname, data["name"][-1], region[0])
            region_outfile = os.path.join(out_dir, "%s-%s%s" %
                                          (base_out, to_safestr(region), out_ext))
            part_info.append((region, region_outfile))
        out_file = os.path.join(data["dirs"]["work"], dirname, data["name"][-1],
                                "%s%s" % (base_out, out_ext))
        return out_file, part_info
    return _do_work

def _add_combine_info(output, combine_map, file_key):
    """Do not actually combine, but add details for later combining work.

    Each sample will contain information on the out file and additional files
    to merge, enabling other splits and recombines without losing information.
    """
    files_per_output = collections.defaultdict(list)
    for part_file, out_file in combine_map.items():
        files_per_output[out_file].append(part_file)
    out_by_file = collections.defaultdict(list)
    out = []
    for data in output:
        # Do not pass along nochrom, noanalysis regions
        if data["region"][0] not in ["nochrom", "noanalysis"]:
            cur_file = data[file_key]
            # If we didn't process, no need to add combine information
            if cur_file in combine_map:
                out_file = combine_map[cur_file]
                if "combine" not in data:
                    data["combine"] = {}
                data["combine"][file_key] = {"out": out_file,
                                             "extras": files_per_output.get(out_file, [])}
                out_by_file[out_file].append(data)
            elif cur_file:
                out_by_file[cur_file].append(data)
            else:
                out.append([data])
    for samples in out_by_file.values():
        regions = [x["region"] for x in samples]
        region_bams = [x["work_bam"] for x in samples]
        assert len(regions) == len(region_bams)
        if len(set(region_bams)) == 1:
            region_bams = [region_bams[0]]
        data = samples[0]
        data["region_bams"] = region_bams
        data["region"] = regions
        out.append([data])
    return out

def parallel_prep_region(samples, run_parallel):
    """Perform full pre-variant calling BAM prep work on regions.
    """
    file_key = "work_bam"
    split_fn = _split_by_regions("bamprep", "-prep.bam", file_key)
    # identify samples that do not need preparation -- no recalibration or realignment
    extras = []
    torun = []
    for data in [x[0] for x in samples]:
        if data.get("work_bam"):
            data["align_bam"] = data["work_bam"]
        a = data["config"]["algorithm"]
        if (not a.get("recalibrate") and not a.get("realign") and not a.get("variantcaller", "gatk")):
            extras.append([data])
        elif not data.get(file_key):
            extras.append([data])
        else:
            torun.append([data])
    return extras + parallel_split_combine(torun, split_fn, run_parallel,
                                           "piped_bamprep", _add_combine_info, file_key, ["config"])

def delayed_bamprep_merge(samples, run_parallel):
    """Perform a delayed merge on regional prepared BAM files.
    """
    needs_merge = False
    for data in samples:
        if (data[0]["config"]["algorithm"].get("merge_bamprep", True) and
              "combine" in data[0]):
            needs_merge = True
            break
    if needs_merge:
        return run_parallel("delayed_bam_merge", samples)
    else:
        return samples

# ## Utilities

def clean_sample_data(samples):
    """Clean unnecessary information from sample data, reducing size for message passing.
    """
    out = []
    for data in (x[0] for x in samples):
        if "dirs" in data:
            data["dirs"] = {"work": data["dirs"]["work"], "galaxy": data["dirs"]["galaxy"],
                            "fastq": data["dirs"].get("fastq")}
        data["config"] = {"algorithm": data["config"]["algorithm"],
                          "resources": data["config"]["resources"]}
        for remove_attr in ["config_file", "regions", "algorithm"]:
            data.pop(remove_attr, None)
        out.append([data])
    return out
<|endoftext|>


<commit_before>
COMMON_REACTION_LIMIT = 5

class Compound:
    """
    A compound is the essential metabolic compound that is involved in a reaction.

    The compound by itself does not have a location. See PyFBA.metabolism.CompoundWithLocation for that detail.
    This abstraction allows us to create Compound objects and then create separate objects with a location
    that are required for the FBA

    Other variables associated with the Compound class:
    :ivar name: the name of the  compound
    :ivar reactions: a set of reaction objects that this compound is connected to
    :ivar model_seed_id: the compound id from the model seed.
    :ivar abbreviation: a short name for the compound
    :ivar formula: the compounds formula
    :ivar mw: the molecular weight of the compound
    :ivar common: Boolean: this is a common compound. This means the coompound is in > COMMON_REACTION_LIMIT reactions
    :ivar charge: the charge associated with the compound

    """

    def __init__(self, cpd_id, name):
        """
        Initiate the object
        :param cpd_id: The id of the compound
        :type cpd_id: str
        :param name: The name of the compound
        :type name: str
        :return:
        :rtype:
        """
        self.id = cpd_id
        self.name = name
        self.reactions = set()
        self.model_seed_id = id
        self.alternate_seed_ids = set()
        self.abbreviation = None
        self.formula = None
        self.mw = 0
        self.common = False
        self.charge = 0
        self.is_cofactor = False
        self.linked_compound = False
        self.pka = 0
        self.pkb = 0
        self.is_obsolete = False
        self.abstract_compound = False
        self.is_core = False
        self.inchikey = 0

    def __eq__(self, other):
        """
        Two compounds are equal if they have the same id or the same name

        :param other: The other compound
        :type other: Compound
        :return: If they are equal
        :rtype: bool
        """
        if isinstance(other, Compound):
            return self.id == other.id or self.name == other.name
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __cmp__(self, other):
        """
        Compare whether two things are the same.

        :param other: The other compound
        :type other: Compound
        :return: An int, zero if they are the same
        :rtype: int
        """
        if isinstance(other, Compound):
            if __eq__(other):
                return 0
            else:
                return 1
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __ne__(self, other):
        """
        Are these not equal?

        :param other: The other compound
        :type other: Compound
        :return: If they are not equal
        :rtype: bool
        """
        try:
            result = self.__eq__(other)
        except NotImplementedError:
            return True
        return not result

    def __hash__(self):
        """
        The hash function is based on the name of the compound.

        :rtype: int
        """
        return hash((self.id, self.name))

    def __str__(self):
        """
        The to string function.
        :rtype: str
        """
        return f"{self.id}: {self.name}"

    def add_reactions(self, rxns):
        """
        Add a reaction that this compound is involved in. You  can add a set of reactions. See the note above about the
        number of reactions.

        :param rxns: A set of reactions
        :type rxns: set
        """
        if isinstance(rxns, set):
            self.reactions.update(rxns)
        else:
            raise TypeError("You need to add a set of reactions to a compound")

    def has_reaction(self, rxn):
        """
        Is this compound involved in this reaction?

        :param rxn: A Reaction object
        :type rxn: Reaction
        :return: Whether the reaction is present
        :rtype: bool
        """
        return rxn in self.reactions

    def number_of_reactions(self):
        """
        How many reactions is this compound involved in?

        :rtype: int
        """
        return len(self.reactions)

    def all_reactions(self):
        """
        Return a set of all the reactions that this compound is involved in

        :rtype: int
        """
        return self.reactions

    def is_common(self, rct_limit=COMMON_REACTION_LIMIT):
        """
        Is this a common compound? This requires that you have
        added reactions to this compound.

        You can either specify the number of reactions or use our
        default that is currently 50.

        :param rct_limit: The limit for a compound to be considered common
        :type rct_limit: int
        :return: Whether this is a common reaction
        :rtype: bool
        """
        if self.number_of_reactions() > rct_limit:
            self.common = True
        else:
            self.common = False
        return self.common

    def calculate_molecular_weight(self):
        """
        Calculate and return the molecular weight of this compound

        :return: The molecular weight
        :rtype: float
        """

        raise NotImplementedError("Sorry. Calculate molecular weight has not yet been implemented.")

    def add_attribute(self, key, value):
        """
        Add an attribute to this class
        """
        setattr(self, key, value)

    def get_attribute(self, key):
        """
        Retrieve an attribute
        """
        return getattr(self, key)

class CompoundWithLocation(Compound):
    """
    Compounds can have several locations:

    A compound has at the very minimum a name and a location. The location is typically one of:
       * e: extracellular
       * c: cytoplasmic
       * h: chloroplast
       * p: periplasm

    We extend the Compound class to add a location, and override a few of the methods

    :ivar location: the location of the compound.
    """

    def calculate_molecular_weight(self):
        pass

    def __init__(self, cpd_id, name, location):
        """
        Initiate the object

        :param cpd_id: The id of the compound
        :type cpd_id: str
        :param name: The name of the compound
        :type name: str
        :param location: The location of the compound
        :type location: str
        :return:
        :rtype:
        """
        super().__init__(cpd_id, name)
        self.location = location

    def __eq__(self, other):
        """
        Two compounds are equal if they have the same name and the same location

        :param other: The other compound
        :type other: Compound
        :return: If they are equal
        :rtype: bool
        """
        if isinstance(other, CompoundWithLocation):
            return self.id == other.id or (self.name, self.location) == (other.name, other.location)
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __cmp__(self, other):
        """
        Compare whether two things are the same.

        :param other: The other compound
        :type other: Compound
        :return: An int, zero if they are the same
        :rtype: int
        """
        if isinstance(other, CompoundWithLocation):
            if __eq__(other):
                return 0
            else:
                return 1
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __ne__(self, other):
        """
        Are these not equal?

        :param other: The other compound
        :type other: Compound
        :return: If they are not equal
        :rtype: bool
        """
        try:
            result = self.__eq__(other)
        except NotImplementedError:
            return True
        return not result

    def __hash__(self):
        """
        The hash function is based on the name of the compound.

        :rtype: int
        """
        return hash((self.id, self.name, self.location))

    def __str__(self):
        """
        The to string function.
        :rtype: str
        """
        return f"{self.id}: {self.name} (location: {self.location})"
<commit_msg>Enabling subclassing an existing object<commit_after>
COMMON_REACTION_LIMIT = 5

class Compound:
    """
    A compound is the essential metabolic compound that is involved in a reaction.

    The compound by itself does not have a location. See PyFBA.metabolism.CompoundWithLocation for that detail.
    This abstraction allows us to create Compound objects and then create separate objects with a location
    that are required for the FBA

    Other variables associated with the Compound class:
    :ivar name: the name of the  compound
    :ivar reactions: a set of reaction objects that this compound is connected to
    :ivar model_seed_id: the compound id from the model seed.
    :ivar abbreviation: a short name for the compound
    :ivar formula: the compounds formula
    :ivar mw: the molecular weight of the compound
    :ivar common: Boolean: this is a common compound. This means the coompound is in > COMMON_REACTION_LIMIT reactions
    :ivar charge: the charge associated with the compound

    """

    def __init__(self, cpd_id, name):
        """
        Initiate the object
        :param cpd_id: The id of the compound
        :type cpd_id: str
        :param name: The name of the compound
        :type name: str
        :return:
        :rtype:
        """
        self.id = cpd_id
        self.name = name
        self.reactions = set()
        self.model_seed_id = id
        self.alternate_seed_ids = set()
        self.abbreviation = None
        self.formula = None
        self.mw = 0
        self.common = False
        self.charge = 0
        self.is_cofactor = False
        self.linked_compound = False
        self.pka = 0
        self.pkb = 0
        self.is_obsolete = False
        self.abstract_compound = False
        self.is_core = False
        self.inchikey = 0

    def __eq__(self, other):
        """
        Two compounds are equal if they have the same id or the same name

        :param other: The other compound
        :type other: Compound
        :return: If they are equal
        :rtype: bool
        """
        if isinstance(other, Compound):
            return self.id == other.id or self.name == other.name
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __cmp__(self, other):
        """
        Compare whether two things are the same.

        :param other: The other compound
        :type other: Compound
        :return: An int, zero if they are the same
        :rtype: int
        """
        if isinstance(other, Compound):
            if __eq__(other):
                return 0
            else:
                return 1
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __ne__(self, other):
        """
        Are these not equal?

        :param other: The other compound
        :type other: Compound
        :return: If they are not equal
        :rtype: bool
        """
        try:
            result = self.__eq__(other)
        except NotImplementedError:
            return True
        return not result

    def __hash__(self):
        """
        The hash function is based on the name of the compound.

        :rtype: int
        """
        return hash((self.id, self.name))

    def __str__(self):
        """
        The to string function.
        :rtype: str
        """
        return f"{self.id}: {self.name}"

    def __iter__(self):
        for i in self.__dict__.items():
            yield i

    def add_reactions(self, rxns):
        """
        Add a reaction that this compound is involved in. You  can add a set of reactions. See the note above about the
        number of reactions.

        :param rxns: A set of reactions
        :type rxns: set
        """
        if isinstance(rxns, set):
            self.reactions.update(rxns)
        else:
            raise TypeError("You need to add a set of reactions to a compound")

    def has_reaction(self, rxn):
        """
        Is this compound involved in this reaction?

        :param rxn: A Reaction object
        :type rxn: Reaction
        :return: Whether the reaction is present
        :rtype: bool
        """
        return rxn in self.reactions

    def number_of_reactions(self):
        """
        How many reactions is this compound involved in?

        :rtype: int
        """
        return len(self.reactions)

    def all_reactions(self):
        """
        Return a set of all the reactions that this compound is involved in

        :rtype: int
        """
        return self.reactions

    def is_common(self, rct_limit=COMMON_REACTION_LIMIT):
        """
        Is this a common compound? This requires that you have
        added reactions to this compound.

        You can either specify the number of reactions or use our
        default that is currently 50.

        :param rct_limit: The limit for a compound to be considered common
        :type rct_limit: int
        :return: Whether this is a common reaction
        :rtype: bool
        """
        if self.number_of_reactions() > rct_limit:
            self.common = True
        else:
            self.common = False
        return self.common

    def calculate_molecular_weight(self):
        """
        Calculate and return the molecular weight of this compound

        :return: The molecular weight
        :rtype: float
        """

        raise NotImplementedError("Sorry. Calculate molecular weight has not yet been implemented.")

    def add_attribute(self, key, value):
        """
        Add an attribute to this class
        """
        setattr(self, key, value)

    def get_attribute(self, key):
        """
        Retrieve an attribute
        """
        return getattr(self, key)

class CompoundWithLocation(Compound):
    """
    Compounds can have several locations:

    A compound has at the very minimum a name and a location. The location is typically one of:
       * e: extracellular
       * c: cytoplasmic
       * h: chloroplast
       * p: periplasm

    We extend the Compound class to add a location, and override a few of the methods

    :ivar location: the location of the compound.
    """

    def calculate_molecular_weight(self):
        pass

    def __init__(self, compound, location):
        """
        Initiate the object

        :param compound: the parent compound. Note you should create this first if it doesn't exist!
        :type cpd_id: PyFBA.metabolism.Compound
        :param location: The location of the compound
        :type location: str
        :return:
        :rtype:
        """
        super().__init__(compound.id, compound.name)
        for it in super():
            self.add_attribute(*it)
        self.location = location

    def __eq__(self, other):
        """
        Two compounds are equal if they have the same name and the same location

        :param other: The other compound
        :type other: Compound
        :return: If they are equal
        :rtype: bool
        """
        if isinstance(other, CompoundWithLocation):
            return self.id == other.id or (self.name, self.location) == (other.name, other.location)
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __cmp__(self, other):
        """
        Compare whether two things are the same.

        :param other: The other compound
        :type other: Compound
        :return: An int, zero if they are the same
        :rtype: int
        """
        if isinstance(other, CompoundWithLocation):
            if __eq__(other):
                return 0
            else:
                return 1
        else:
            raise NotImplementedError(f"Comparing a Compound with {type(other)} has not been implemented")

    def __ne__(self, other):
        """
        Are these not equal?

        :param other: The other compound
        :type other: Compound
        :return: If they are not equal
        :rtype: bool
        """
        try:
            result = self.__eq__(other)
        except NotImplementedError:
            return True
        return not result

    def __hash__(self):
        """
        The hash function is based on the name of the compound.

        :rtype: int
        """
        return hash((self.id, self.name, self.location))

    def __str__(self):
        """
        The to string function.
        :rtype: str
        """
        return f"{self.id}: {self.name} (location: {self.location})"
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import sys
import time
import json
import shutil
import zipfile
import tempfile
import subprocess
import collections
import urllib.request
import distutils.version
from urllib.parse import unquote

from pathlib import Path

# You can adjust some setting here. Default is for QOwnNotes app.
links_as_URI = True  # True for file://link%20target style links, False for /link target style links
absolute_links = False  # True for absolute links, False for relative links
media_dir_name = 'media'  # name of the directory inside the produced directory where all images and attachments will be stored
md_file_ext = 'md'  # extension for produced markdown syntax note files
insert_title = True  # True to insert note title as a markdown heading at the first line, False to disable
insert_ctime = False  # True to insert note creation time to the beggining of the note text, False to disable
insert_mtime = False  # True to insert note modifictation time to the beggining of the note text, False to disable
creation_date_in_filename = False  # True to insert note creation time to the note file name, False to disable

tag_prepend = ''  # string to prepend each tag in a tag list inside the note, default is empty
tag_delimiter = ', '  # string to delimit tags, default is comma separated list
no_spaces_in_tags = False  # True to replace spaces in tag names with '_', False to keep spaces

############################################################################

Notebook = collections.namedtuple('Notebook', ['path', 'media_path'])

def sanitise_path_string(path_str):
    for char in (':', '/', '\\', '|'):
        path_str = path_str.replace(char, '-')
    for char in ('?', '*'):
        path_str = path_str.replace(char, '')
    path_str = path_str.replace('<', '(')
    path_str = path_str.replace('>', ')')
    path_str = path_str.replace('"', "'")
    path_str = urllib.parse.unquote(path_str)

    return path_str[:240]

work_path = Path.cwd()
media_dir_name = sanitise_path_string(media_dir_name)
pandoc_input_file = tempfile.NamedTemporaryFile(delete=False)
pandoc_output_file = tempfile.NamedTemporaryFile(delete=False)

if not shutil.which('pandoc') and not os.path.isfile('pandoc'):
    print('Can\'t find pandoc. Please install pandoc or place it to the directory, where the script is.')
    exit(1)

try:
    pandoc_ver = subprocess.check_output(['pandoc', '-v'], timeout=3).decode('utf-8')[7:].split('\n', 1)[0].strip()
    print('Found pandoc ' + pandoc_ver)

    if distutils.version.LooseVersion(pandoc_ver) < distutils.version.LooseVersion('1.16'):
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--no-wrap', '-o', pandoc_output_file.name, pandoc_input_file.name]
    elif distutils.version.LooseVersion(pandoc_ver) < distutils.version.LooseVersion('1.19'):
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--wrap=none', '-o', pandoc_output_file.name, pandoc_input_file.name]
    else:
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--wrap=none', '--atx-headers', '-o',
                       pandoc_output_file.name, pandoc_input_file.name]
except Exception:
    pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                   '--wrap=none', '--atx-headers', '-o',
                   pandoc_output_file.name, pandoc_input_file.name]

if len(sys.argv) > 1:
    files_to_convert = [Path(path) for path in sys.argv[1:]]
else:
    files_to_convert = Path(work_path).glob('*.nsx')

if not files_to_convert:
    print('No .nsx files found')
    exit(1)

for file in files_to_convert:
    nsx_file = zipfile.ZipFile(str(file))
    config_data = json.loads(nsx_file.read('config.json').decode('utf-8'))
    notebook_id_to_path_index = {}

    recycle_bin_path = work_path / Path('Recycle bin')

    n = 1
    while recycle_bin_path.is_dir():
        recycle_bin_path = work_path / Path('{}_{}'.format('Recycle bin', n))
        n += 1

    recycle_bin_media_path = recycle_bin_path / media_dir_name
    recycle_bin_media_path.mkdir(parents=True)
    notebook_id_to_path_index['1027_#00000000'] = Notebook(recycle_bin_path, recycle_bin_media_path)

    print('Extracting notes from "{}"'.format(file.name))

    for notebook_id in config_data['notebook']:
        notebook_data = json.loads(nsx_file.read(notebook_id).decode('utf-8'))
        notebook_title = notebook_data['title'] or 'Untitled'
        notebook_path = work_path / Path(sanitise_path_string(notebook_title))

        n = 1
        while notebook_path.is_dir():
            notebook_path = work_path / Path('{}_{}'.format(sanitise_path_string(notebook_title), n))
            n += 1

        notebook_media_path = Path(notebook_path / media_dir_name)
        notebook_media_path.mkdir(parents=True)

        notebook_id_to_path_index[notebook_id] = Notebook(notebook_path, notebook_media_path)

    note_id_to_title_index = {}
    converted_note_ids = []

    for note_id in config_data['note']:
        note_data = json.loads(nsx_file.read(note_id).decode('utf-8'))

        note_title = note_data.get('title', 'Untitled')
        note_ctime = note_data.get('ctime', '')
        note_mtime = note_data.get('mtime', '')

        note_id_to_title_index[note_id] = note_title

        try:
            parent_notebook_id = note_data['parent_id']
            parent_notebook = notebook_id_to_path_index[parent_notebook_id]
        except KeyError:
            continue

        print('Converting note "{}"'.format(note_title))

        content = re.sub(']*syno-notestation-image-object[^>]*src=[^>]*ref=',
                         '<img src=', note_data.get('content', ''))

        Path(pandoc_input_file.name).write_text(content, 'utf-8')
        pandoc = subprocess.Popen(pandoc_args)
        pandoc.wait(20)
        content = Path(pandoc_output_file.name).read_text('utf-8')

        attachments_data = note_data.get('attachment')
        attachment_list = []

        if attachments_data:
            for attachment_id in note_data.get('attachment', ''):

                ref = note_data['attachment'][attachment_id].get('ref', '')
                md5 = note_data['attachment'][attachment_id]['md5']
                source = note_data['attachment'][attachment_id].get('source', '')
                name = sanitise_path_string(note_data['attachment'][attachment_id]['name'])
                name = name.replace('ns_attach_image_', '')

                n = 1
                while Path(parent_notebook.media_path / name).is_file():
                    name_parts = name.rpartition('.')
                    name = ''.join((name_parts[0], '_{}'.format(n), name_parts[1], name_parts[2]))
                    n += 1

                if links_as_URI:
                    if absolute_links:
                        link_path = Path(parent_notebook.media_path / name).as_uri()
                    else:
                        link_path = 'file://{}/{}'.format(urllib.request.pathname2url(media_dir_name),
                                                          urllib.request.pathname2url(name))
                else:
                    if absolute_links:
                        link_path = str(Path(parent_notebook.media_path / name))
                    else:
                        link_path = '{}/{}'.format(media_dir_name, name)

                try:
                    Path(parent_notebook.media_path / name).write_bytes(nsx_file.read('file_' + md5))
                    attachment_link = '[{}]({})'.format(name, link_path)
                except Exception:
                    if source:
                        attachment_link = '[{}]({})'.format(name, source)
                    else:
                        print('Can\'t find attachment "{}" of note "{}"'.format(name, note_title))
                        attachment_link = '[NOT FOUND]({})'.format(link_path)

                if ref and source:
                    content = content.replace(ref, source)
                elif ref:
                    content = content.replace(ref, link_path)
                else:
                    attachment_list.append(attachment_link)

        if note_data.get('tag', '') or attachment_list or insert_title \
                or insert_ctime or insert_mtime:
            content = '\n' + content

        if insert_mtime and note_mtime:
            text_mtime = time.strftime('%Y-%m-%d %H:%M', time.localtime(note_mtime))
            content = 'Modified: {}  \n{}'.format(text_mtime, content)
        if insert_ctime and note_ctime:
            text_ctime = time.strftime('%Y-%m-%d %H:%M', time.localtime(note_ctime))
            content = 'Created: {}  \n{}'.format(text_ctime, content)
        if attachment_list:
            content = 'Attachments: {}  \n{}'.format(', '.join(attachment_list), content)
        if note_data.get('tag', ''):
            if no_spaces_in_tags:
                note_data['tag'] = [tag.replace(' ', '_') for tag in note_data['tag']]
            tag_list = tag_delimiter.join(''.join((tag_prepend, tag)) for tag in note_data['tag'])
            content = 'Tags: {}  \n{}'.format(tag_list, content)
        if insert_title:
            content = '{}\n{}\n{}'.format(note_title, '=' * len(note_title), content)

        if creation_date_in_filename and note_ctime:
            note_title = time.strftime('%Y-%m-%d ', time.localtime(note_ctime)) + note_title

        md_file_name = sanitise_path_string(note_title) or 'Untitled'
        md_file_path = Path(parent_notebook.path / '{}.{}'.format(md_file_name, md_file_ext))

        n = 1
        while md_file_path.is_file():
            md_file_path = Path(parent_notebook.path / ('{}_{}.{}'.format(
                                            sanitise_path_string(note_title), n, md_file_ext)))
            n += 1

        md_file_path.write_text(content, 'utf-8')

        converted_note_ids.append(note_id)

    for notebook in notebook_id_to_path_index.values():
        try:
            notebook.media_path.rmdir()
        except OSError:
            pass

    not_converted_note_ids = set(note_id_to_title_index.keys()) - set(converted_note_ids)

    if not_converted_note_ids:
        print('Failed to convert notes:',
              '\n'.join(('    {} (ID: {})'.format(note_id_to_title_index[note_id], note_id)
                         for note_id in not_converted_note_ids)),
              sep='\n')

    if len(config_data['notebook']) == 1:
        notebook_log_str = 'notebook'
    else:
        notebook_log_str = 'notebooks'

    print('Converted {} {} and {} out of {} notes.\n'.format(len(config_data['notebook']),
                                                             notebook_log_str,
                                                             len(converted_note_ids),
                                                             len(note_id_to_title_index.keys())))
    try:
        recycle_bin_media_path.rmdir()
    except OSError:
        pass

    try:
        recycle_bin_path.rmdir()
    except OSError:
        pass

pandoc_input_file.close()
pandoc_output_file.close()
os.unlink(pandoc_input_file.name)
os.unlink(pandoc_output_file.name)

input('Press Enter to quit...')
<commit_msg>Fixed pandoc options for pandoc ver > 2.11.2<commit_after>#!/usr/bin/env python

import os
import re
import sys
import time
import json
import shutil
import zipfile
import tempfile
import subprocess
import collections
import urllib.request
import distutils.version
from urllib.parse import unquote

from pathlib import Path

# You can adjust some setting here. Default is for QOwnNotes app.
links_as_URI = True  # True for file://link%20target style links, False for /link target style links
absolute_links = False  # True for absolute links, False for relative links
media_dir_name = 'media'  # name of the directory inside the produced directory where all images and attachments will be stored
md_file_ext = 'md'  # extension for produced markdown syntax note files
insert_title = True  # True to insert note title as a markdown heading at the first line, False to disable
insert_ctime = False  # True to insert note creation time to the beggining of the note text, False to disable
insert_mtime = False  # True to insert note modifictation time to the beggining of the note text, False to disable
creation_date_in_filename = False  # True to insert note creation time to the note file name, False to disable

tag_prepend = ''  # string to prepend each tag in a tag list inside the note, default is empty
tag_delimiter = ', '  # string to delimit tags, default is comma separated list
no_spaces_in_tags = False  # True to replace spaces in tag names with '_', False to keep spaces

############################################################################

Notebook = collections.namedtuple('Notebook', ['path', 'media_path'])

def sanitise_path_string(path_str):
    for char in (':', '/', '\\', '|'):
        path_str = path_str.replace(char, '-')
    for char in ('?', '*'):
        path_str = path_str.replace(char, '')
    path_str = path_str.replace('<', '(')
    path_str = path_str.replace('>', ')')
    path_str = path_str.replace('"', "'")
    path_str = urllib.parse.unquote(path_str)

    return path_str[:240]

work_path = Path.cwd()
media_dir_name = sanitise_path_string(media_dir_name)
pandoc_input_file = tempfile.NamedTemporaryFile(delete=False)
pandoc_output_file = tempfile.NamedTemporaryFile(delete=False)

if not shutil.which('pandoc') and not os.path.isfile('pandoc'):
    print('Can\'t find pandoc. Please install pandoc or place it to the directory, where the script is.')
    exit(1)

try:
    pandoc_ver = subprocess.check_output(['pandoc', '-v'], timeout=3).decode('utf-8')[7:].split('\n', 1)[0].strip()
    print('Found pandoc ' + pandoc_ver)

    if distutils.version.LooseVersion(pandoc_ver) < distutils.version.LooseVersion('1.16'):
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--no-wrap', '-o', pandoc_output_file.name, pandoc_input_file.name]
    elif distutils.version.LooseVersion(pandoc_ver) < distutils.version.LooseVersion('1.19'):
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--wrap=none', '-o', pandoc_output_file.name, pandoc_input_file.name]
    elif distutils.version.LooseVersion(pandoc_ver) < distutils.version.LooseVersion('2.11.2'):
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--wrap=none', '--atx-headers', '-o',
                       pandoc_output_file.name, pandoc_input_file.name]
    else:
        pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                       '--wrap=none', '--markdown-headings=atx', '-o',
                       pandoc_output_file.name, pandoc_input_file.name]
except Exception:
    pandoc_args = ['pandoc', '-f', 'html', '-t', 'markdown_strict+pipe_tables-raw_html',
                   '--wrap=none', '--markdown-headings=atx', '-o',
                   pandoc_output_file.name, pandoc_input_file.name]

if len(sys.argv) > 1:
    files_to_convert = [Path(path) for path in sys.argv[1:]]
else:
    files_to_convert = Path(work_path).glob('*.nsx')

if not files_to_convert:
    print('No .nsx files found')
    exit(1)

for file in files_to_convert:
    nsx_file = zipfile.ZipFile(str(file))
    config_data = json.loads(nsx_file.read('config.json').decode('utf-8'))
    notebook_id_to_path_index = {}

    recycle_bin_path = work_path / Path('Recycle bin')

    n = 1
    while recycle_bin_path.is_dir():
        recycle_bin_path = work_path / Path('{}_{}'.format('Recycle bin', n))
        n += 1

    recycle_bin_media_path = recycle_bin_path / media_dir_name
    recycle_bin_media_path.mkdir(parents=True)
    notebook_id_to_path_index['1027_#00000000'] = Notebook(recycle_bin_path, recycle_bin_media_path)

    print('Extracting notes from "{}"'.format(file.name))

    for notebook_id in config_data['notebook']:
        notebook_data = json.loads(nsx_file.read(notebook_id).decode('utf-8'))
        notebook_title = notebook_data['title'] or 'Untitled'
        notebook_path = work_path / Path(sanitise_path_string(notebook_title))

        n = 1
        while notebook_path.is_dir():
            notebook_path = work_path / Path('{}_{}'.format(sanitise_path_string(notebook_title), n))
            n += 1

        notebook_media_path = Path(notebook_path / media_dir_name)
        notebook_media_path.mkdir(parents=True)

        notebook_id_to_path_index[notebook_id] = Notebook(notebook_path, notebook_media_path)

    note_id_to_title_index = {}
    converted_note_ids = []

    for note_id in config_data['note']:
        note_data = json.loads(nsx_file.read(note_id).decode('utf-8'))

        note_title = note_data.get('title', 'Untitled')
        note_ctime = note_data.get('ctime', '')
        note_mtime = note_data.get('mtime', '')

        note_id_to_title_index[note_id] = note_title

        try:
            parent_notebook_id = note_data['parent_id']
            parent_notebook = notebook_id_to_path_index[parent_notebook_id]
        except KeyError:
            continue

        print('Converting note "{}"'.format(note_title))

        content = re.sub(']*syno-notestation-image-object[^>]*src=[^>]*ref=',
                         '<img src=', note_data.get('content', ''))

        Path(pandoc_input_file.name).write_text(content, 'utf-8')
        pandoc = subprocess.Popen(pandoc_args)
        pandoc.wait(20)
        content = Path(pandoc_output_file.name).read_text('utf-8')

        attachments_data = note_data.get('attachment')
        attachment_list = []

        if attachments_data:
            for attachment_id in note_data.get('attachment', ''):

                ref = note_data['attachment'][attachment_id].get('ref', '')
                md5 = note_data['attachment'][attachment_id]['md5']
                source = note_data['attachment'][attachment_id].get('source', '')
                name = sanitise_path_string(note_data['attachment'][attachment_id]['name'])
                name = name.replace('ns_attach_image_', '')

                n = 1
                while Path(parent_notebook.media_path / name).is_file():
                    name_parts = name.rpartition('.')
                    name = ''.join((name_parts[0], '_{}'.format(n), name_parts[1], name_parts[2]))
                    n += 1

                if links_as_URI:
                    if absolute_links:
                        link_path = Path(parent_notebook.media_path / name).as_uri()
                    else:
                        link_path = 'file://{}/{}'.format(urllib.request.pathname2url(media_dir_name),
                                                          urllib.request.pathname2url(name))
                else:
                    if absolute_links:
                        link_path = str(Path(parent_notebook.media_path / name))
                    else:
                        link_path = '{}/{}'.format(media_dir_name, name)

                try:
                    Path(parent_notebook.media_path / name).write_bytes(nsx_file.read('file_' + md5))
                    attachment_link = '[{}]({})'.format(name, link_path)
                except Exception:
                    if source:
                        attachment_link = '[{}]({})'.format(name, source)
                    else:
                        print('Can\'t find attachment "{}" of note "{}"'.format(name, note_title))
                        attachment_link = '[NOT FOUND]({})'.format(link_path)

                if ref and source:
                    content = content.replace(ref, source)
                elif ref:
                    content = content.replace(ref, link_path)
                else:
                    attachment_list.append(attachment_link)

        if note_data.get('tag', '') or attachment_list or insert_title \
                or insert_ctime or insert_mtime:
            content = '\n' + content

        if insert_mtime and note_mtime:
            text_mtime = time.strftime('%Y-%m-%d %H:%M', time.localtime(note_mtime))
            content = 'Modified: {}  \n{}'.format(text_mtime, content)
        if insert_ctime and note_ctime:
            text_ctime = time.strftime('%Y-%m-%d %H:%M', time.localtime(note_ctime))
            content = 'Created: {}  \n{}'.format(text_ctime, content)
        if attachment_list:
            content = 'Attachments: {}  \n{}'.format(', '.join(attachment_list), content)
        if note_data.get('tag', ''):
            if no_spaces_in_tags:
                note_data['tag'] = [tag.replace(' ', '_') for tag in note_data['tag']]
            tag_list = tag_delimiter.join(''.join((tag_prepend, tag)) for tag in note_data['tag'])
            content = 'Tags: {}  \n{}'.format(tag_list, content)
        if insert_title:
            content = '{}\n{}\n{}'.format(note_title, '=' * len(note_title), content)

        if creation_date_in_filename and note_ctime:
            note_title = time.strftime('%Y-%m-%d ', time.localtime(note_ctime)) + note_title

        md_file_name = sanitise_path_string(note_title) or 'Untitled'
        md_file_path = Path(parent_notebook.path / '{}.{}'.format(md_file_name, md_file_ext))

        n = 1
        while md_file_path.is_file():
            md_file_path = Path(parent_notebook.path / ('{}_{}.{}'.format(
                                            sanitise_path_string(note_title), n, md_file_ext)))
            n += 1

        md_file_path.write_text(content, 'utf-8')

        converted_note_ids.append(note_id)

    for notebook in notebook_id_to_path_index.values():
        try:
            notebook.media_path.rmdir()
        except OSError:
            pass

    not_converted_note_ids = set(note_id_to_title_index.keys()) - set(converted_note_ids)

    if not_converted_note_ids:
        print('Failed to convert notes:',
              '\n'.join(('    {} (ID: {})'.format(note_id_to_title_index[note_id], note_id)
                         for note_id in not_converted_note_ids)),
              sep='\n')

    if len(config_data['notebook']) == 1:
        notebook_log_str = 'notebook'
    else:
        notebook_log_str = 'notebooks'

    print('Converted {} {} and {} out of {} notes.\n'.format(len(config_data['notebook']),
                                                             notebook_log_str,
                                                             len(converted_note_ids),
                                                             len(note_id_to_title_index.keys())))
    try:
        recycle_bin_media_path.rmdir()
    except OSError:
        pass

    try:
        recycle_bin_path.rmdir()
    except OSError:
        pass

pandoc_input_file.close()
pandoc_output_file.close()
os.unlink(pandoc_input_file.name)
os.unlink(pandoc_output_file.name)

input('Press Enter to quit...')
<|endoftext|>