<commit_before>#
# Copyright (C) 2010-2020 by frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import re
from io import BytesIO
from os.path import basename
from datetime import datetime
from time import localtime, strftime
from zipfile import ZipFile, ZIP_DEFLATED

from django.core.management.base import BaseCommand, CommandError
from django.core.mail import EmailMessage
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS
from django.utils.translation import gettext_lazy as _
from django.template import Template, RequestContext

from freppledb.execute.models import Task
from freppledb.common.middleware import _thread_locals
from freppledb.common.models import User
from freppledb import VERSION

class Command(BaseCommand):
    help = """
  This command will email reports found in the export folder to specific recipients.
  """

    requires_system_checks = False

    def get_version(self):
        return VERSION

    def add_arguments(self, parser):
        parser.add_argument("--user", help="User running the command"),
        parser.add_argument(
            "--task",
            type=int,
            help="Task identifier (generated automatically if not provided)",
        )
        parser.add_argument(
            "--database",
            default=DEFAULT_DB_ALIAS,
            help="database where the task should be launched",
        )
        parser.add_argument(
            "--sender", default=settings.DEFAULT_FROM_EMAIL, help="sender of the email"
        )
        parser.add_argument("--recipient", help="a comma separated list of recipients")
        parser.add_argument(
            "--report", help="a comma separated list of reports to email"
        )

    def handle(self, **options):
        now = datetime.now()
        database = options["database"]
        if database not in settings.DATABASES:
            raise CommandError("No database settings known for '%s'" % database)

        # Pick up options
        if options["user"]:
            try:
                user = User.objects.all().get(username=options["user"])
            except Exception:
                raise CommandError("User '%s' not found" % options["user"])
        else:
            user = None

        task = None
        try:
            setattr(_thread_locals, "database", database)
            if "task" in options and options["task"]:
                try:
                    task = Task.objects.all().using(database).get(pk=options["task"])
                except Exception:
                    raise CommandError("Task identifier not found")
                if (
                    task.started
                    or task.finished
                    or task.status != "Waiting"
                    or task.name not in ("emailreport")
                ):
                    raise CommandError("Invalid task identifier")
                task.status = "0%"
                task.started = now
            else:
                task = Task(
                    name="emailreport",
                    submitted=now,
                    started=now,
                    status="0%",
                    user=user,
                )
            task.processid = os.getpid()
            task.save(using=database)

            if not settings.EMAIL_HOST:
                raise CommandError(
                    "No SMTP mail server is configured in your djangosettings.py file"
                )

            sender = options["sender"]
            recipient = options["recipient"]
            report = options["report"]

            if not sender:
                raise CommandError("No sender has been defined")

            if not recipient:
                raise CommandError("No recipient has been defined")

            if not report:
                raise CommandError("No report to email has been defined")

            # Make sure file exist in the export folder
            reports = report.split(",")
            correctedReports = []
            missingFiles = []
            for r in reports:
                if len(r.strip()) == 0:
                    continue
                path = os.path.join(
                    settings.DATABASES[database]["FILEUPLOADFOLDER"],
                    "export",
                    r.strip(),
                )
                if not os.path.isfile(path):
                    missingFiles.append(r.strip())
                else:
                    correctedReports.append(path)

            if len(missingFiles) > 0:
                raise CommandError(
                    "Following files are missing in export folder: %s"
                    % (",".join(str(x) for x in missingFiles))
                )

            if len(correctedReports) == 0:
                raise CommandError("No report defined in options")

            # Validate email adresses
            recipients = recipient.split(",")
            correctedRecipients = []
            invalidEmails = []
            for r in recipients:
                if len(r.strip()) == 0:
                    continue
                if not re.fullmatch(r"[^@]+@[^@]+\.[^@]+", r.strip()):
                    invalidEmails.append(r.strip())
                else:
                    correctedRecipients.append(r.strip())

            if len(invalidEmails) > 0:
                raise CommandError(
                    "Invalid email formatting for following addresses: %s"
                    % (",".join(str(x) for x in invalidEmails))
                )
            if len(correctedRecipients) == 0:
                raise CommandError("No recipient defined in options")

            task.arguments = "--recipient=%s --report=%s" % (recipient, report)
            task.save(using=database)

            # create message
            message = EmailMessage(
                subject="Exported reports",
                body="",
                from_email=sender,
                to=correctedRecipients,
            )

            b = BytesIO()
            with ZipFile(file=b, mode="w", compression=ZIP_DEFLATED) as zf:
                processedFiles = 0
                for f in correctedReports:
                    task.message = "Compressing file %s" % basename(f)
                    task.status = (
                        str(int(processedFiles / len(correctedReports) * 90.0)) + "%"
                    )
                    task.save(using=database)
                    zf.write(filename=f, arcname=basename(f))
                    processedFiles = processedFiles + 1
                zf.close()

                # attach zip file
                task.status = "90%"
                task.message = "Sending email"
                task.save(using=database)
                message.attach("reports.zip", b.getvalue(), "application/zip")
                # send email
                message.send()

            b.close()

            # Logging message
            task.processid = None
            task.message = ""
            task.status = "Done"
            task.finished = datetime.now()

        except Exception as e:
            if task:
                task.status = "Failed"
                task.message = "%s" % e
                task.finished = datetime.now()
            raise e

        finally:
            setattr(_thread_locals, "database", None)
            if task:
                task.processid = None
                task.save(using=database)

    # accordion template
    title = _("Email exported reports")
    index = 1250
    help_url = "user-guide/command-reference.html#emailreport"

    @staticmethod
    def getHTML(request):

        if (
            "FILEUPLOADFOLDER" in settings.DATABASES[request.database]
            and request.user.is_superuser
        ):
            # Function to convert from bytes to human readabl format
            def sizeof_fmt(num):
                for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
                    if abs(num) < 1024.0:
                        return "%3.1f%sB" % (num, unit)
                    num /= 1024.0
                return "%.1f%sB" % (num, "Yi")

            # List available data files
            filesexported = []
            all_reports = []
            if "FILEUPLOADFOLDER" in settings.DATABASES[request.database]:
                exportfolder = os.path.join(
                    settings.DATABASES[request.database]["FILEUPLOADFOLDER"], "export"
                )
                if os.path.isdir(exportfolder):
                    for file in os.listdir(exportfolder):
                        if file.endswith((".csv", ".csv.gz", ".log")):
                            all_reports.append(file)
                            filesexported.append(
                                [
                                    file,
                                    strftime(
                                        "%Y-%m-%d %I:%M%p",
                                        localtime(
                                            os.stat(
                                                os.path.join(exportfolder, file)
                                            ).st_mtime
                                        ),
                                    ),
                                    sizeof_fmt(
                                        os.stat(
                                            os.path.join(exportfolder, file)
                                        ).st_size
                                    ),
                                    file.split(".")[0],
                                ]
                            )

            context = RequestContext(
                request,
                {
                    "filesexported": filesexported,
                    "user_email": request.user.email,
                    "all_reports": ",".join(map(str, all_reports)),
                },
            )

            template = Template(
                """
        {% load i18n %}
        <form role="form" method="post" action="{{request.prefix}}/execute/launch/emailreport/">{% csrf_token %}

                <button type="submit" class="btn btn-primary" id="emailreport" value="email">{% trans "email"|capfirst %}</button>

{% trans "Emails the selected reports to a comma separated list of recipients. Files are zipped and attached to email." %}

                   <input type="checkbox" id="allcheckboxes" checked>
                   {% trans 'file name'|capfirst %}

              {% trans 'size'|capfirst %}
              {% trans 'last modified'|capfirst %}

</form>
            {% for j in filesexported %}

                  <input type="checkbox" id="{{j.3}}" checked>
                  {{j.0}}

              {{j.2}}
              {{j.1}}

            {% endfor %}

                {% trans 'emails'|capfirst %}:

                <input type="email" class="form-control" id="emails" name="recipient" multiple value="{{user_email}}">

          <input type="hidden" name="report" id="report" value="{{all_reports}}">
        </form>

        """
            )
            return template.render(context)
            # A list of translation strings from the above
            translated = (
                _("email"),
                _(
                    "Emails the selected reports to a comma separated list of recipients. Files are zipped and attached to email."
                ),
                _("file name"),
                _("size"),
                _("please correct invalid email addresses"),
                _("please enter email addresses"),
            )
        else:
            return None
<commit_msg>Make sure email button is initially disabled if no report exists.<commit_after>#
# Copyright (C) 2010-2020 by frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import re
from io import BytesIO
from os.path import basename
from datetime import datetime
from time import localtime, strftime
from zipfile import ZipFile, ZIP_DEFLATED

from django.core.management.base import BaseCommand, CommandError
from django.core.mail import EmailMessage
from django.conf import settings
from django.db import DEFAULT_DB_ALIAS
from django.utils.translation import gettext_lazy as _
from django.template import Template, RequestContext

from freppledb.execute.models import Task
from freppledb.common.middleware import _thread_locals
from freppledb.common.models import User
from freppledb import VERSION

class Command(BaseCommand):
    help = """
  This command will email reports found in the export folder to specific recipients.
  """

    requires_system_checks = False

    def get_version(self):
        return VERSION

    def add_arguments(self, parser):
        parser.add_argument("--user", help="User running the command"),
        parser.add_argument(
            "--task",
            type=int,
            help="Task identifier (generated automatically if not provided)",
        )
        parser.add_argument(
            "--database",
            default=DEFAULT_DB_ALIAS,
            help="database where the task should be launched",
        )
        parser.add_argument(
            "--sender", default=settings.DEFAULT_FROM_EMAIL, help="sender of the email"
        )
        parser.add_argument("--recipient", help="a comma separated list of recipients")
        parser.add_argument(
            "--report", help="a comma separated list of reports to email"
        )

    def handle(self, **options):
        now = datetime.now()
        database = options["database"]
        if database not in settings.DATABASES:
            raise CommandError("No database settings known for '%s'" % database)

        # Pick up options
        if options["user"]:
            try:
                user = User.objects.all().get(username=options["user"])
            except Exception:
                raise CommandError("User '%s' not found" % options["user"])
        else:
            user = None

        task = None
        try:
            setattr(_thread_locals, "database", database)
            if "task" in options and options["task"]:
                try:
                    task = Task.objects.all().using(database).get(pk=options["task"])
                except Exception:
                    raise CommandError("Task identifier not found")
                if (
                    task.started
                    or task.finished
                    or task.status != "Waiting"
                    or task.name not in ("emailreport")
                ):
                    raise CommandError("Invalid task identifier")
                task.status = "0%"
                task.started = now
            else:
                task = Task(
                    name="emailreport",
                    submitted=now,
                    started=now,
                    status="0%",
                    user=user,
                )
            task.processid = os.getpid()
            task.save(using=database)

            if not settings.EMAIL_HOST:
                raise CommandError(
                    "No SMTP mail server is configured in your djangosettings.py file"
                )

            sender = options["sender"]
            recipient = options["recipient"]
            report = options["report"]

            if not sender:
                raise CommandError("No sender has been defined")

            if not recipient:
                raise CommandError("No recipient has been defined")

            if not report:
                raise CommandError("No report to email has been defined")

            # Make sure file exist in the export folder
            reports = report.split(",")
            correctedReports = []
            missingFiles = []
            for r in reports:
                if len(r.strip()) == 0:
                    continue
                path = os.path.join(
                    settings.DATABASES[database]["FILEUPLOADFOLDER"],
                    "export",
                    r.strip(),
                )
                if not os.path.isfile(path):
                    missingFiles.append(r.strip())
                else:
                    correctedReports.append(path)

            if len(missingFiles) > 0:
                raise CommandError(
                    "Following files are missing in export folder: %s"
                    % (",".join(str(x) for x in missingFiles))
                )

            if len(correctedReports) == 0:
                raise CommandError("No report defined in options")

            # Validate email adresses
            recipients = recipient.split(",")
            correctedRecipients = []
            invalidEmails = []
            for r in recipients:
                if len(r.strip()) == 0:
                    continue
                if not re.fullmatch(r"[^@]+@[^@]+\.[^@]+", r.strip()):
                    invalidEmails.append(r.strip())
                else:
                    correctedRecipients.append(r.strip())

            if len(invalidEmails) > 0:
                raise CommandError(
                    "Invalid email formatting for following addresses: %s"
                    % (",".join(str(x) for x in invalidEmails))
                )
            if len(correctedRecipients) == 0:
                raise CommandError("No recipient defined in options")

            task.arguments = "--recipient=%s --report=%s" % (recipient, report)
            task.save(using=database)

            # create message
            message = EmailMessage(
                subject="Exported reports",
                body="",
                from_email=sender,
                to=correctedRecipients,
            )

            b = BytesIO()
            with ZipFile(file=b, mode="w", compression=ZIP_DEFLATED) as zf:
                processedFiles = 0
                for f in correctedReports:
                    task.message = "Compressing file %s" % basename(f)
                    task.status = (
                        str(int(processedFiles / len(correctedReports) * 90.0)) + "%"
                    )
                    task.save(using=database)
                    zf.write(filename=f, arcname=basename(f))
                    processedFiles = processedFiles + 1
                zf.close()

                # attach zip file
                task.status = "90%"
                task.message = "Sending email"
                task.save(using=database)
                message.attach("reports.zip", b.getvalue(), "application/zip")
                # send email
                message.send()

            b.close()

            # Logging message
            task.processid = None
            task.message = ""
            task.status = "Done"
            task.finished = datetime.now()

        except Exception as e:
            if task:
                task.status = "Failed"
                task.message = "%s" % e
                task.finished = datetime.now()
            raise e

        finally:
            setattr(_thread_locals, "database", None)
            if task:
                task.processid = None
                task.save(using=database)

    # accordion template
    title = _("Email exported reports")
    index = 1250
    help_url = "user-guide/command-reference.html#emailreport"

    @staticmethod
    def getHTML(request):

        if (
            "FILEUPLOADFOLDER" in settings.DATABASES[request.database]
            and request.user.is_superuser
        ):
            # Function to convert from bytes to human readabl format
            def sizeof_fmt(num):
                for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
                    if abs(num) < 1024.0:
                        return "%3.1f%sB" % (num, unit)
                    num /= 1024.0
                return "%.1f%sB" % (num, "Yi")

            # List available data files
            filesexported = []
            all_reports = []
            if "FILEUPLOADFOLDER" in settings.DATABASES[request.database]:
                exportfolder = os.path.join(
                    settings.DATABASES[request.database]["FILEUPLOADFOLDER"], "export"
                )
                if os.path.isdir(exportfolder):
                    for file in os.listdir(exportfolder):
                        if file.endswith((".csv", ".csv.gz", ".log")):
                            all_reports.append(file)
                            filesexported.append(
                                [
                                    file,
                                    strftime(
                                        "%Y-%m-%d %I:%M%p",
                                        localtime(
                                            os.stat(
                                                os.path.join(exportfolder, file)
                                            ).st_mtime
                                        ),
                                    ),
                                    sizeof_fmt(
                                        os.stat(
                                            os.path.join(exportfolder, file)
                                        ).st_size
                                    ),
                                    file.split(".")[0],
                                ]
                            )

            context = RequestContext(
                request,
                {
                    "filesexported": filesexported,
                    "user_email": request.user.email,
                    "all_reports": ",".join(map(str, all_reports)),
                    "initially_disabled": "" if len(all_reports) > 0 else "disabled",
                },
            )

            template = Template(
                """
        {% load i18n %}
        <form role="form" method="post" action="{{request.prefix}}/execute/launch/emailreport/">{% csrf_token %}

                <button type="submit" class="btn btn-primary" id="emailreport" value="email" {{initially_disabled}}>{% trans "email"|capfirst %}</button>

{% trans "Emails the selected reports to a comma separated list of recipients. Files are zipped and attached to email." %}

                   <input type="checkbox" id="allcheckboxes" checked>
                   {% trans 'file name'|capfirst %}

              {% trans 'size'|capfirst %}
              {% trans 'last modified'|capfirst %}

</form>
            {% for j in filesexported %}

                  <input type="checkbox" id="{{j.3}}" checked>
                  {{j.0}}

              {{j.2}}
              {{j.1}}

            {% endfor %}

                {% trans 'emails'|capfirst %}:

                <input type="email" class="form-control" id="emails" name="recipient" multiple value="{{user_email}}">

          <input type="hidden" name="report" id="report" value="{{all_reports}}">
        </form>

        """
            )
            return template.render(context)
            # A list of translation strings from the above
            translated = (
                _("email"),
                _(
                    "Emails the selected reports to a comma separated list of recipients. Files are zipped and attached to email."
                ),
                _("file name"),
                _("size"),
                _("please correct invalid email addresses"),
                _("please enter email addresses"),
            )
        else:
            return None
<|endoftext|>


<commit_before>#
# Copyright (C) 2007-2013 by frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
from datetime import datetime
import subprocess
import re

from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import force_text
from django.template import Template, RequestContext

from freppledb.common.commands import PlanTaskRegistry
from freppledb.common.models import User
from freppledb.execute.models import Task
from freppledb import VERSION

class Command(BaseCommand):

  help = "Runs frePPLe to generate a plan"

  requires_system_checks = False

  def get_version(self):
    return VERSION

  def add_arguments(self, parser):
    parser.add_argument(
      '--user', dest='user',
      help='User running the command'
      )
    parser.add_argument(
      '--constraint', dest='constraint', type=int, default=15,
      choices=range(0, 16),
      help='Constraints to be considered: 1=lead time, 2=material, 4=capacity, 8=release fence'
      )
    parser.add_argument(
      '--plantype', dest='plantype', type=int, choices=[1, 2],
      default=1, help='Plan type: 1=constrained, 2=unconstrained'
      )
    parser.add_argument(
      '--database', dest='database', default=DEFAULT_DB_ALIAS,
      help='Nominates a specific database to load data from and export results into'
      )
    parser.add_argument(
      '--task', dest='task', type=int,
      help='Task identifier (generated automatically if not provided)'
      )
    parser.add_argument(
      '--env', dest='env',
      help='A comma separated list of extra settings passed as environment variables to the engine'
      )
    parser.add_argument(
      '--background', dest='background', action='store_true', default=False,
      help='Run the planning engine in the background (default = False)'
      )

  def handle(self, **options):
    # Pick up the options
    now = datetime.now()

    if 'database' in options:
      database = options['database'] or DEFAULT_DB_ALIAS
    else:
      database = DEFAULT_DB_ALIAS
    if database not in settings.DATABASES:
      raise CommandError("No database settings known for '%s'" % database )
    if 'user' in options and options['user']:
      try:
        user = User.objects.all().using(database).get(username=options['user'])
      except:
        raise CommandError("User '%s' not found" % options['user'] )
    else:
      user = None

    timestamp = now.strftime("%Y%m%d%H%M%S")
    if database == DEFAULT_DB_ALIAS:
      logfile = 'frepple-%s.log' % timestamp
    else:
      logfile = 'frepple_%s-%s.log' % (database, timestamp)

    task = None
    try:
      # Initialize the task
      if 'task' in options and options['task']:
        try:
          task = Task.objects.all().using(database).get(pk=options['task'])
        except:
          raise CommandError("Task identifier not found")
        if task.started or task.finished or task.status != "Waiting" or task.name != 'generate plan':
          raise CommandError("Invalid task identifier")
        task.status = '0%'
        task.started = now
        task.logfile = logfile
      else:
        task = Task(name='generate plan', submitted=now, started=now, status='0%', user=user, logfile=logfile)

      # Validate options
      if 'constraint' in options:
        constraint = int(options['constraint'])
        if constraint < 0 or constraint > 15:
          raise ValueError("Invalid constraint: %s" % options['constraint'])
      else:
        constraint = 15
      if 'plantype' in options:
        plantype = int(options['plantype'])
      else:
        plantype = 1

      # Reset environment variables
      # TODO avoid having to delete the environment variables. Use options directly?
      PlanTaskRegistry.autodiscover()
      for i in PlanTaskRegistry.reg:
        if 'env' in options:
          # Options specified
          if i.label and i.label[0] in os.environ:
            del os.environ[i.label[0]]
        elif i.label:
          # No options specified - default to activate them all
          os.environ[i.label[0]] = '1'

      # Set environment variables
      if options['env']:
        task.arguments = "--constraint=%d --plantype=%d --env=%s" % (constraint, plantype, options['env'])
        for i in options['env'].split(','):
          j = i.split('=')
          if len(j) == 1:
            os.environ[j[0]] = '1'
          else:
            os.environ[j[0]] = j[1]
      else:
        task.arguments = "--constraint=%d --plantype=%d" % (constraint, plantype)
      if options['background']:
        task.arguments += " --background"

      # Log task
      task.save(using=database)

      # Locate commands.py
      import freppledb.common.commands
      cmd = freppledb.common.commands.__file__

      # Prepare environment
      os.environ['FREPPLE_PLANTYPE'] = str(plantype)
      os.environ['FREPPLE_CONSTRAINT'] = str(constraint)
      os.environ['FREPPLE_TASKID'] = str(task.id)
      os.environ['FREPPLE_DATABASE'] = database
      os.environ['FREPPLE_LOGFILE'] = logfile
      os.environ['PATH'] = settings.FREPPLE_HOME + os.pathsep + os.environ['PATH'] + os.pathsep + settings.FREPPLE_APP
      if os.path.isfile(os.path.join(settings.FREPPLE_HOME, 'libfrepple.so')):
        os.environ['LD_LIBRARY_PATH'] = settings.FREPPLE_HOME
      if 'DJANGO_SETTINGS_MODULE' not in os.environ:
        os.environ['DJANGO_SETTINGS_MODULE'] = 'freppledb.settings'
      os.environ['PYTHONPATH'] = os.path.normpath(settings.FREPPLE_APP)

      if options['background']:
        # Execute as background process on Windows
        if os.name == 'nt':
          subprocess.Popen(['frepple', cmd], creationflags=0x08000000)
        else:
          # Execute as background process on Linux
          subprocess.Popen(['frepple', cmd])
      else:
        # Execute in foreground
        ret = subprocess.call(['frepple', cmd])
        if ret != 0 and ret != 2:
          # Return code 0 is a successful run
          # Return code is 2 is a run cancelled by a user. That's shown in the status field.
          raise Exception('Failed with exit code %d' % ret)

        # Task update
        task.status = 'Done'
        task.finished = datetime.now()

    except Exception as e:
      if task:
        task.status = 'Failed'
        task.message = '%s' % e
        task.finished = datetime.now()
      raise e

    finally:
      if task:
        task.save(using=database)

  # accordion template
  title = _('Create a plan')
  index = 0

  @ staticmethod
  def getHTML(request):

    if request.user.has_perm('auth.generate_plan'):
      # Collect optional tasks
      PlanTaskRegistry.autodiscover()
      planning_options = PlanTaskRegistry.getLabels()
      current_options = request.session.get('env', [ i[0] for i in planning_options ])

      try:
        constraint = int(request.session['constraint'])
      except:
        constraint = 15

      context = RequestContext(request, {
        'planning_options': planning_options,
        'current_options': current_options,
        'capacityconstrained': constraint & 4,
        'materialconstrained': constraint & 2,
        'leadtimeconstrained': constraint & 1,
        'fenceconstrained': constraint & 8
        })

      template = Template('''
        {% load i18n %}
        <form role="form" method="post" action="{{request.prefix}}/execute/launch/frepple_run/">{% csrf_token %}

                <button type="submit" class="btn btn-primary">{% trans "launch"|capfirst %}</button>

				        {% blocktrans %}
										Load frePPLe from the database and live data sources...

										and create a plan in frePPLe...

										and export results.{% endblocktrans %}

              {% if planning_options %}
              <p {% if planning_options|length <= 1 %}style="display: none"{% endif %}>{% trans "optional planning steps"|capfirst %}

              {% for b in planning_options %}
              <input type="checkbox" name="env" {% if b.0 in current_options %}checked {% endif %}value="{{b.0}}" id="option_{{b.0}}"/>  {{b.1}}

              {% endfor %}

              {% endif %}

Plan type

              <input type="radio" id="plantype1" name="plantype" {% ifnotequal request.session.plantype '2' %}checked {% endifnotequal %}value="1"/>
              {% blocktrans %}Constrained plan{% endblocktrans %}

              <input type="radio" id="plantype2" name="plantype" {% ifequal  request.session.plantype '2' %}checked {% endifequal %}value="2"/>
              {% blocktrans %}Unconstrained plan{% endblocktrans %}

              {% trans "constraints"|capfirst %}

              <input type="checkbox" name="constraint" {% if capacityconstrained %}checked {% endif %}value="4" id="cb4"/>  {% trans "Capacity: respect capacity limits" %}

              <input type="checkbox" name="constraint" {% if materialconstrained %}checked {% endif %}value="2" id="cb2"/>  {% trans "Material: respect procurement limits" %}

              <input type="checkbox" name="constraint" {% if leadtimeconstrained %}checked {% endif %}value="1" id="cb1"/>  {% trans "Lead time: do not plan in the past" %}

              <input type="checkbox" name="constraint" {% if fenceconstrained %}checked {% endif %}value="8" id="cb8"/>  {% trans "Release fence: do not plan within the release time window" %}

              {% trans "View log file" %}

        </form>
      ''')
      return template.render(context)
    else:
      return None
<commit_msg>logfile changes: remove view logfile button from frepple_run.<commit_after>#
# Copyright (C) 2007-2013 by frePPLe bvba
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
from datetime import datetime
import subprocess
import re

from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import force_text
from django.template import Template, RequestContext

from freppledb.common.commands import PlanTaskRegistry
from freppledb.common.models import User
from freppledb.execute.models import Task
from freppledb import VERSION

class Command(BaseCommand):

  help = "Runs frePPLe to generate a plan"

  requires_system_checks = False

  def get_version(self):
    return VERSION

  def add_arguments(self, parser):
    parser.add_argument(
      '--user', dest='user',
      help='User running the command'
      )
    parser.add_argument(
      '--constraint', dest='constraint', type=int, default=15,
      choices=range(0, 16),
      help='Constraints to be considered: 1=lead time, 2=material, 4=capacity, 8=release fence'
      )
    parser.add_argument(
      '--plantype', dest='plantype', type=int, choices=[1, 2],
      default=1, help='Plan type: 1=constrained, 2=unconstrained'
      )
    parser.add_argument(
      '--database', dest='database', default=DEFAULT_DB_ALIAS,
      help='Nominates a specific database to load data from and export results into'
      )
    parser.add_argument(
      '--task', dest='task', type=int,
      help='Task identifier (generated automatically if not provided)'
      )
    parser.add_argument(
      '--env', dest='env',
      help='A comma separated list of extra settings passed as environment variables to the engine'
      )
    parser.add_argument(
      '--background', dest='background', action='store_true', default=False,
      help='Run the planning engine in the background (default = False)'
      )

  def handle(self, **options):
    # Pick up the options
    now = datetime.now()

    if 'database' in options:
      database = options['database'] or DEFAULT_DB_ALIAS
    else:
      database = DEFAULT_DB_ALIAS
    if database not in settings.DATABASES:
      raise CommandError("No database settings known for '%s'" % database )
    if 'user' in options and options['user']:
      try:
        user = User.objects.all().using(database).get(username=options['user'])
      except:
        raise CommandError("User '%s' not found" % options['user'] )
    else:
      user = None

    timestamp = now.strftime("%Y%m%d%H%M%S")
    if database == DEFAULT_DB_ALIAS:
      logfile = 'frepple-%s.log' % timestamp
    else:
      logfile = 'frepple_%s-%s.log' % (database, timestamp)

    task = None
    try:
      # Initialize the task
      if 'task' in options and options['task']:
        try:
          task = Task.objects.all().using(database).get(pk=options['task'])
        except:
          raise CommandError("Task identifier not found")
        if task.started or task.finished or task.status != "Waiting" or task.name != 'generate plan':
          raise CommandError("Invalid task identifier")
        task.status = '0%'
        task.started = now
        task.logfile = logfile
      else:
        task = Task(name='generate plan', submitted=now, started=now, status='0%', user=user, logfile=logfile)

      # Validate options
      if 'constraint' in options:
        constraint = int(options['constraint'])
        if constraint < 0 or constraint > 15:
          raise ValueError("Invalid constraint: %s" % options['constraint'])
      else:
        constraint = 15
      if 'plantype' in options:
        plantype = int(options['plantype'])
      else:
        plantype = 1

      # Reset environment variables
      # TODO avoid having to delete the environment variables. Use options directly?
      PlanTaskRegistry.autodiscover()
      for i in PlanTaskRegistry.reg:
        if 'env' in options:
          # Options specified
          if i.label and i.label[0] in os.environ:
            del os.environ[i.label[0]]
        elif i.label:
          # No options specified - default to activate them all
          os.environ[i.label[0]] = '1'

      # Set environment variables
      if options['env']:
        task.arguments = "--constraint=%d --plantype=%d --env=%s" % (constraint, plantype, options['env'])
        for i in options['env'].split(','):
          j = i.split('=')
          if len(j) == 1:
            os.environ[j[0]] = '1'
          else:
            os.environ[j[0]] = j[1]
      else:
        task.arguments = "--constraint=%d --plantype=%d" % (constraint, plantype)
      if options['background']:
        task.arguments += " --background"

      # Log task
      task.save(using=database)

      # Locate commands.py
      import freppledb.common.commands
      cmd = freppledb.common.commands.__file__

      # Prepare environment
      os.environ['FREPPLE_PLANTYPE'] = str(plantype)
      os.environ['FREPPLE_CONSTRAINT'] = str(constraint)
      os.environ['FREPPLE_TASKID'] = str(task.id)
      os.environ['FREPPLE_DATABASE'] = database
      os.environ['FREPPLE_LOGFILE'] = logfile
      os.environ['PATH'] = settings.FREPPLE_HOME + os.pathsep + os.environ['PATH'] + os.pathsep + settings.FREPPLE_APP
      if os.path.isfile(os.path.join(settings.FREPPLE_HOME, 'libfrepple.so')):
        os.environ['LD_LIBRARY_PATH'] = settings.FREPPLE_HOME
      if 'DJANGO_SETTINGS_MODULE' not in os.environ:
        os.environ['DJANGO_SETTINGS_MODULE'] = 'freppledb.settings'
      os.environ['PYTHONPATH'] = os.path.normpath(settings.FREPPLE_APP)

      if options['background']:
        # Execute as background process on Windows
        if os.name == 'nt':
          subprocess.Popen(['frepple', cmd], creationflags=0x08000000)
        else:
          # Execute as background process on Linux
          subprocess.Popen(['frepple', cmd])
      else:
        # Execute in foreground
        ret = subprocess.call(['frepple', cmd])
        if ret != 0 and ret != 2:
          # Return code 0 is a successful run
          # Return code is 2 is a run cancelled by a user. That's shown in the status field.
          raise Exception('Failed with exit code %d' % ret)

        # Task update
        task.status = 'Done'
        task.finished = datetime.now()

    except Exception as e:
      if task:
        task.status = 'Failed'
        task.message = '%s' % e
        task.finished = datetime.now()
      raise e

    finally:
      if task:
        task.save(using=database)

  # accordion template
  title = _('Create a plan')
  index = 0

  @ staticmethod
  def getHTML(request):

    if request.user.has_perm('auth.generate_plan'):
      # Collect optional tasks
      PlanTaskRegistry.autodiscover()
      planning_options = PlanTaskRegistry.getLabels()
      current_options = request.session.get('env', [ i[0] for i in planning_options ])

      try:
        constraint = int(request.session['constraint'])
      except:
        constraint = 15

      context = RequestContext(request, {
        'planning_options': planning_options,
        'current_options': current_options,
        'capacityconstrained': constraint & 4,
        'materialconstrained': constraint & 2,
        'leadtimeconstrained': constraint & 1,
        'fenceconstrained': constraint & 8
        })

      template = Template('''
        {% load i18n %}
        <form role="form" method="post" action="{{request.prefix}}/execute/launch/frepple_run/">{% csrf_token %}

                <button type="submit" class="btn btn-primary">{% trans "launch"|capfirst %}</button>

				        {% blocktrans %}
										Load frePPLe from the database and live data sources...

										and create a plan in frePPLe...

										and export results.{% endblocktrans %}

              {% if planning_options %}
              <p {% if planning_options|length <= 1 %}style="display: none"{% endif %}>{% trans "optional planning steps"|capfirst %}

              {% for b in planning_options %}
              <input type="checkbox" name="env" {% if b.0 in current_options %}checked {% endif %}value="{{b.0}}" id="option_{{b.0}}"/>  {{b.1}}

              {% endfor %}

              {% endif %}

Plan type

              <input type="radio" id="plantype1" name="plantype" {% ifnotequal request.session.plantype '2' %}checked {% endifnotequal %}value="1"/>
              {% blocktrans %}Constrained plan{% endblocktrans %}

              <input type="radio" id="plantype2" name="plantype" {% ifequal  request.session.plantype '2' %}checked {% endifequal %}value="2"/>
              {% blocktrans %}Unconstrained plan{% endblocktrans %}

              {% trans "constraints"|capfirst %}

              <input type="checkbox" name="constraint" {% if capacityconstrained %}checked {% endif %}value="4" id="cb4"/>  {% trans "Capacity: respect capacity limits" %}

              <input type="checkbox" name="constraint" {% if materialconstrained %}checked {% endif %}value="2" id="cb2"/>  {% trans "Material: respect procurement limits" %}

              <input type="checkbox" name="constraint" {% if leadtimeconstrained %}checked {% endif %}value="1" id="cb1"/>  {% trans "Lead time: do not plan in the past" %}

              <input type="checkbox" name="constraint" {% if fenceconstrained %}checked {% endif %}value="8" id="cb8"/>  {% trans "Release fence: do not plan within the release time window" %}

        </form>
      ''')
      return template.render(context)
    else:
      return None
<|endoftext|>


<commit_before># MANSEDS Lunar Rover -- Continuous Servo Controller
# Author: Ethan Ramsay

# Abbreviations:
# dc = duty cycle
# pl = pulse length
# pin = pulse width modulation pin no.

# Import dependencies
import RPi.GPIO as GPIO
import logging
import Adafruit_PCA9685
import time

# Logging config
logging.basicConfig(filename='servo.log', level=logging.WARNING)

# System variables
pi = 3.14159
servo = 0
arm_angle = 0 # degrees
zero_motion_pl = 410
compensation_factor = 1.027
ccw_full_rot_time = 1.1 # s / 360 deg
base_angle_data_filename = "base_angle.dat" # External file storing base angle value

# GPIO setup function
def GPIO_set(pin, dc):
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(pin, GPIO.OUT)
    servo = GPIO.PWM(pin, 50)
    servo.start(dc)

def GPIO_clear(servo):
    servo.stop()
    GPIO.cleanup()

# Adafruit setup
pwm = Adafruit_PCA9685.PCA9685()
pwm.set_pwm_freq(60)

# Funcs
def determine_zero_pl(pl_min, pl_max, channel):
    mid_range_pl = (pl_min + pl_max)/2
    initial_pl = mid_range_pl - 100
    pl = int(initial_pl)
    while pl < (mid_range_pl+100):
        pwm.set_pwm(channel, 0, pl)
        print(pl)
        pl += 10
        time.sleep(1)

def speed_loop(pl_min, pl_max, channel):
    pwm.set_pwm(channel,0,410)
    time.sleep(2)
    pwm.set_pwm(channel,0,440)
    time.sleep(1.1)
    pwm.set_pwm(channel,0,410)
    time.sleep(5)
    pwm.set_pwm(channel,0,220)
    time.sleep(1.1297)
    pwm.set_pwm(channel,0,410)
    time.sleep(5)

def rotational_positioning(desired_angle, channel):
    if desired_angle > 45 or desired_angle < -45:
        raise ValueError("Desired angle exceeds current configuration range: min = -45 deg; max  \
        = 45 deg")
    current_angle = 0
    with open(base_angle_data_filename, 'r') as f:
        current_angle = f.read()
    print(current_angle)
    perc_full_rot = 100 * (desired_angle - current_angle) / 360
    print(perc_full_rot)
    if desired_angle < current_angle:
        rot_time = ccw_full_rot_time * perc_full_rot / 100
        print(rot_time)
        pwm.set_pwm(channel, 0, 440)
        time.sleep(rot_time)
        pwm.set_pwm(channel, 0, 410)
        with open(base_angle_data_filename, 'w') as f:
            f.write(desired_angle)
        time.sleep(100000)
    elif desired_angle > current_angle:
        rot_time = ccw_full_rot_time * compensation_factor * perc_full_rot / 100
        pwm.set_pwm(channel, 0, 220)
        time.sleep(rot.time)
        pwm.set_pwm(channel, 0, 410)
        with open(base_angle_data_filename, 'w') as f:
            f.write(desired_angle)
        time.sleep(1000000)
    else:
        pass # cuurent angle must be equal to desired angle

if __name__ == "__main__":
    while True:
        import argparse

        # Arguments
        parser = argparse.ArgumentParser()
        parser.add_argument("angle", help="Angle of turn (deg)")
        parser.add_argument("direction", help="Direction of turn - True = CCW")

        # Arguments for direct GPIO control from Pi
        # parser.add_argument("dc_min", help="Minimum Duty Cycle")
        # parser.add_argument("dc_max", help="Maximum Duty Cycle")
        # parser.add_argument("pin", help="PWM Pin No. (BCM)")

        # Arguments for Adafruit PWM hat control
        parser.add_argument("pl_min", help="Minimum Pulse Length")
        parser.add_argument("pl_max", help="Maximum Pulse Length")
        parser.add_argument("channel", help="Channel No. (Adafruit PWM Hat)")

        # Parse arguments
        args = parser.parse_args()
        angle = int(args.angle)
        direction = args.direction
        # dc_min = float(args.dc_min)
        # dc_max = float(args.dc_max)
        # pin = int(args.pin)
        pl_min = float(args.pl_min)
        pl_max = float(args.pl_max)
        channel = int(args.channel)
        logging.warning("Channel: %s", channel)

        # Actuate servo
        # GPIO_set(pin, dc)
        # GPIO_clear()

        rotational_positioning(direction, angle, channel)
<commit_msg>further continuous servo tweaking<commit_after># MANSEDS Lunar Rover -- Continuous Servo Controller
# Author: Ethan Ramsay

# Abbreviations:
# dc = duty cycle
# pl = pulse length
# pin = pulse width modulation pin no.

# Import dependencies
import RPi.GPIO as GPIO
import logging
import Adafruit_PCA9685
import time

# Logging config
logging.basicConfig(filename='servo.log', level=logging.WARNING)

# System variables
pi = 3.14159
servo = 0
arm_angle = 0 # degrees
zero_motion_pl = 410
compensation_factor = 1.027
ccw_full_rot_time = 1.1 # s / 360 deg
base_angle_data_filename = "base_angle.dat" # External file storing base angle value

# GPIO setup function
def GPIO_set(pin, dc):
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(pin, GPIO.OUT)
    servo = GPIO.PWM(pin, 50)
    servo.start(dc)

def GPIO_clear(servo):
    servo.stop()
    GPIO.cleanup()

# Adafruit setup
pwm = Adafruit_PCA9685.PCA9685()
pwm.set_pwm_freq(60)

# Funcs
def determine_zero_pl(pl_min, pl_max, channel):
    mid_range_pl = (pl_min + pl_max)/2
    initial_pl = mid_range_pl - 100
    pl = int(initial_pl)
    while pl < (mid_range_pl+100):
        pwm.set_pwm(channel, 0, pl)
        print(pl)
        pl += 10
        time.sleep(1)

def speed_loop(pl_min, pl_max, channel):
    pwm.set_pwm(channel,0,410)
    time.sleep(2)
    pwm.set_pwm(channel,0,440)
    time.sleep(1.1)
    pwm.set_pwm(channel,0,410)
    time.sleep(5)
    pwm.set_pwm(channel,0,220)
    time.sleep(1.1297)
    pwm.set_pwm(channel,0,410)
    time.sleep(5)

def rotational_positioning(desired_angle, channel):
    if desired_angle > 45 or desired_angle < -45:
        raise ValueError("Desired angle exceeds current configuration range: min = -45 deg; max  \
        = 45 deg")
    current_angle = 0
    with open(base_angle_data_filename, 'r') as f:
        current_angle = f.read()
    print(current_angle)
    perc_full_rot = 100 * (desired_angle - current_angle) / 360
    print(perc_full_rot)
    if desired_angle < current_angle:
        rot_time = ccw_full_rot_time * perc_full_rot / 100
        print(rot_time)
        pwm.set_pwm(channel, 0, 440)
        time.sleep(rot_time)
        pwm.set_pwm(channel, 0, 410)
        with open(base_angle_data_filename, 'w') as f:
            f.write(desired_angle)
        time.sleep(100000)
    elif desired_angle > current_angle:
        rot_time = ccw_full_rot_time * compensation_factor * perc_full_rot / 100
        pwm.set_pwm(channel, 0, 220)
        time.sleep(rot.time)
        pwm.set_pwm(channel, 0, 410)
        with open(base_angle_data_filename, 'w') as f:
            f.write(desired_angle)
        time.sleep(1000000)
    else:
        pwm.set_pwm(channel, 0, 410)
        time.sleep(100000)
        # cuurent angle must be equal to desired angle

if __name__ == "__main__":
    while True:
        import argparse

        # Arguments
        parser = argparse.ArgumentParser()
        parser.add_argument("angle", help="Angle of turn (deg)")

        # Arguments for direct GPIO control from Pi
        # parser.add_argument("dc_min", help="Minimum Duty Cycle")
        # parser.add_argument("dc_max", help="Maximum Duty Cycle")
        # parser.add_argument("pin", help="PWM Pin No. (BCM)")

        # Arguments for Adafruit PWM hat control
        parser.add_argument("pl_min", help="Minimum Pulse Length")
        parser.add_argument("pl_max", help="Maximum Pulse Length")
        parser.add_argument("channel", help="Channel No. (Adafruit PWM Hat)")

        # Parse arguments
        args = parser.parse_args()
        angle = int(args.angle)
        # dc_min = float(args.dc_min)
        # dc_max = float(args.dc_max)
        # pin = int(args.pin)
        pl_min = float(args.pl_min)
        pl_max = float(args.pl_max)
        channel = int(args.channel)
        logging.warning("Channel: %s", channel)

        # Actuate servo
        # GPIO_set(pin, dc)
        # GPIO_clear()

        rotational_positioning(angle, channel)
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
from astromodels import clone_model

from threeML.classicMLE.joint_likelihood import JointLikelihood
from threeML.classicMLE.joint_likelihood_set import JointLikelihoodSet
from threeML.data_list import DataList
from threeML.plugins.OGIPLike import OGIPLike
from threeML.plugins.OGIP.pha import PHAWrite
from threeML.exceptions.custom_exceptions import custom_warnings

class LikelihoodRatioTest(object):

    def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):

        self._joint_likelihood_instance0 = joint_likelihood_instance0  # type: JointLikelihood
        self._joint_likelihood_instance1 = joint_likelihood_instance1  # type: JointLikelihood

        # Restore best fit and store the reference value for the likelihood
        self._joint_likelihood_instance0.restore_best_fit()
        self._joint_likelihood_instance1.restore_best_fit()

        self._reference_TS = 2 * (self._joint_likelihood_instance0.current_minimum -
                                  self._joint_likelihood_instance1.current_minimum)

        # Safety check that the user has provided the models in the right order
        if self._reference_TS < 0:

            custom_warnings.warn("The reference TS is negative, either you specified the likelihood objects "
                                 "in the wrong order, or the fit for the alternative hyp. has failed. Since the "
                                 "two hyp. are nested, by definition the more complex hypothesis should give a "
                                 "better or equal fit with respect to the null hypothesis.")

        # Check that the dataset is the same

        if self._joint_likelihood_instance1.data_list != self._joint_likelihood_instance0.data_list:

            # Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
            # an exception.

            custom_warnings.warn("The data lists for the null hyp. and for the alternative hyp. seems to be different."
                                 " If you loaded twice the same data and made the same data selections, disregard this "
                                 "message. Otherwise, consider the fact that the LRT is meaningless if the two data "
                                 "sets are not exactly the same. We will use the data loaded as part of the null "
                                 "hypothesis JointLikelihood object", RuntimeWarning)

        # For saving pha files
        self._save_pha = False
        self._data_container = []

    def get_simulated_data(self, id):

        # Generate a new data set for each plugin contained in the data list

        new_datas = []

        for dataset in self._joint_likelihood_instance0.data_list.values():

            # Make sure that the active likelihood model is the null hypothesis
            # This is needed if the user has used the same DataList instance for both
            # JointLikelihood instances
            dataset.set_model(self._joint_likelihood_instance0.likelihood_model)

            new_data = dataset.get_simulated_dataset("%s_sim" % dataset.name)

            new_datas.append(new_data)

        new_data_list = DataList(*new_datas)

        if self._save_pha:

            self._data_container.append(new_data_list)

        return new_data_list

    def get_models(self, id):

        # Make a copy of the best fit models, so that we don't touch the original models during the fit, and we
        # also always restart from the best fit (instead of the last iteration)

        new_model0 = clone_model(self._joint_likelihood_instance0.likelihood_model)
        new_model1 = clone_model(self._joint_likelihood_instance1.likelihood_model)

        return new_model0, new_model1

    def by_mc(self, n_iterations=1000, continue_on_failure=False, save_pha=False):
        """
        Compute the Likelihood Ratio Test by generating Monte Carlo datasets and fitting the current models on them.
        The fraction of synthetic datasets which have a value for the TS larger or equal to the observed one gives
        the null-hypothesis probability (i.e., the probability that the observed TS is obtained by chance from the
        null hypothesis)

        :param n_iterations: number of MC iterations to perform (default: 1000)
        :param continue_of_failure: whether to continue in the case a fit fails (False by default)
        :param save_pha: Saves pha files for reading into XSPEC as a cross check.
         Currently only supports OGIP data. This can become slow! (False by default)
        :return: tuple (null. hyp. probability, frame with all results, frame with all likelihood values)
        """

        self._save_pha = save_pha

        # Create the joint likelihood set
        jl_set = JointLikelihoodSet(self.get_simulated_data, self.get_models, n_iterations, iteration_name='simulation')

        # Use the same minimizer as in the first joint likelihood object

        minimizer_name, algorithm = self._joint_likelihood_instance0.minimizer_in_use
        jl_set.set_minimizer(minimizer_name, algorithm)

        # Run the set
        data_frame, like_data_frame = jl_set.go(continue_on_failure=continue_on_failure)

        # Get the TS values

        TS_ = 2 * (like_data_frame['-log(likelihood)'][:, 'model_0', 'total'] -
                   like_data_frame['-log(likelihood)'][:, 'model_1', 'total'])  # type: pd.Series

        TS = pd.Series(TS_.values, name='TS')

        # Compute the null hyp probability
        idx = TS >= self._reference_TS  # type: np.ndarray

        null_hyp_prob = np.sum(idx) / float(n_iterations)

        # Save the sims to phas if requested
        if self._save_pha:

            self._process_saved_data()

        return null_hyp_prob, TS, data_frame, like_data_frame

    def _process_saved_data(self):
        """

        :return:
        """

        for plugin in self._data_container[0].values():

            assert isinstance(plugin, OGIPLike), 'Saving simulations is only supported for OGIP plugins currently'

        # The first entry is always a test by the JL Set class.
        # so we do not use it

        for key in self._data_container[0].keys():

            per_plugin_list = []

            for data in self._data_container[1:]:

                per_plugin_list.append(data[key])

            # Now write them

            pha_writer = PHAWrite(*per_plugin_list)

            pha_writer.write("%s" % key, overwrite=True)
<commit_msg>doc string<commit_after>import numpy as np
import pandas as pd
from astromodels import clone_model

from threeML.classicMLE.joint_likelihood import JointLikelihood
from threeML.classicMLE.joint_likelihood_set import JointLikelihoodSet
from threeML.data_list import DataList
from threeML.plugins.OGIPLike import OGIPLike
from threeML.plugins.OGIP.pha import PHAWrite
from threeML.exceptions.custom_exceptions import custom_warnings

class LikelihoodRatioTest(object):

    def __init__(self, joint_likelihood_instance0, joint_likelihood_instance1):

        self._joint_likelihood_instance0 = joint_likelihood_instance0  # type: JointLikelihood
        self._joint_likelihood_instance1 = joint_likelihood_instance1  # type: JointLikelihood

        # Restore best fit and store the reference value for the likelihood
        self._joint_likelihood_instance0.restore_best_fit()
        self._joint_likelihood_instance1.restore_best_fit()

        self._reference_TS = 2 * (self._joint_likelihood_instance0.current_minimum -
                                  self._joint_likelihood_instance1.current_minimum)

        # Safety check that the user has provided the models in the right order
        if self._reference_TS < 0:

            custom_warnings.warn("The reference TS is negative, either you specified the likelihood objects "
                                 "in the wrong order, or the fit for the alternative hyp. has failed. Since the "
                                 "two hyp. are nested, by definition the more complex hypothesis should give a "
                                 "better or equal fit with respect to the null hypothesis.")

        # Check that the dataset is the same

        if self._joint_likelihood_instance1.data_list != self._joint_likelihood_instance0.data_list:

            # Since this check might fail if the user loaded twice the same data, only issue a warning, instead of
            # an exception.

            custom_warnings.warn("The data lists for the null hyp. and for the alternative hyp. seems to be different."
                                 " If you loaded twice the same data and made the same data selections, disregard this "
                                 "message. Otherwise, consider the fact that the LRT is meaningless if the two data "
                                 "sets are not exactly the same. We will use the data loaded as part of the null "
                                 "hypothesis JointLikelihood object", RuntimeWarning)

        # For saving pha files
        self._save_pha = False
        self._data_container = []

    def get_simulated_data(self, id):

        # Generate a new data set for each plugin contained in the data list

        new_datas = []

        for dataset in self._joint_likelihood_instance0.data_list.values():

            # Make sure that the active likelihood model is the null hypothesis
            # This is needed if the user has used the same DataList instance for both
            # JointLikelihood instances
            dataset.set_model(self._joint_likelihood_instance0.likelihood_model)

            new_data = dataset.get_simulated_dataset("%s_sim" % dataset.name)

            new_datas.append(new_data)

        new_data_list = DataList(*new_datas)

        if self._save_pha:

            self._data_container.append(new_data_list)

        return new_data_list

    def get_models(self, id):

        # Make a copy of the best fit models, so that we don't touch the original models during the fit, and we
        # also always restart from the best fit (instead of the last iteration)

        new_model0 = clone_model(self._joint_likelihood_instance0.likelihood_model)
        new_model1 = clone_model(self._joint_likelihood_instance1.likelihood_model)

        return new_model0, new_model1

    def by_mc(self, n_iterations=1000, continue_on_failure=False, save_pha=False):
        """
        Compute the Likelihood Ratio Test by generating Monte Carlo datasets and fitting the current models on them.
        The fraction of synthetic datasets which have a value for the TS larger or equal to the observed one gives
        the null-hypothesis probability (i.e., the probability that the observed TS is obtained by chance from the
        null hypothesis)

        :param n_iterations: number of MC iterations to perform (default: 1000)
        :param continue_of_failure: whether to continue in the case a fit fails (False by default)
        :param save_pha: Saves pha files for reading into XSPEC as a cross check.
         Currently only supports OGIP data. This can become slow! (False by default)
        :return: tuple (null. hyp. probability, frame with all results, frame with all likelihood values)
        """

        self._save_pha = save_pha

        # Create the joint likelihood set
        jl_set = JointLikelihoodSet(self.get_simulated_data, self.get_models, n_iterations, iteration_name='simulation')

        # Use the same minimizer as in the first joint likelihood object

        minimizer_name, algorithm = self._joint_likelihood_instance0.minimizer_in_use
        jl_set.set_minimizer(minimizer_name, algorithm)

        # Run the set
        data_frame, like_data_frame = jl_set.go(continue_on_failure=continue_on_failure)

        # Get the TS values

        TS_ = 2 * (like_data_frame['-log(likelihood)'][:, 'model_0', 'total'] -
                   like_data_frame['-log(likelihood)'][:, 'model_1', 'total'])  # type: pd.Series

        TS = pd.Series(TS_.values, name='TS')

        # Compute the null hyp probability
        idx = TS >= self._reference_TS  # type: np.ndarray

        null_hyp_prob = np.sum(idx) / float(n_iterations)

        # Save the sims to phas if requested
        if self._save_pha:

            self._process_saved_data()

        return null_hyp_prob, TS, data_frame, like_data_frame

    def _process_saved_data(self):
        """

        Saves data sets for each plugin to PHAs for OGIP data.

        :return:
        """

        for plugin in self._data_container[0].values():

            assert isinstance(plugin, OGIPLike), 'Saving simulations is only supported for OGIP plugins currently'

        # The first entry is always a test by the JL Set class.
        # so we do not use it

        for key in self._data_container[0].keys():

            per_plugin_list = []

            for data in self._data_container[1:]:

                per_plugin_list.append(data[key])

            # Now write them

            pha_writer = PHAWrite(*per_plugin_list)

            pha_writer.write("%s" % key, overwrite=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import datetime
import argparse
import smtplib
import email
import email.mime.text
import email.mime.multipart

import elasticsearch
import elasticsearch.helpers

ES_NODES = ['uct2-es-head.mwt2.org:9200', 'uct2-es-door.mwt2.org:9200']
TZ_NAME = "UTC"
INDEX_BASE_NAME = "oasis-module-usage"
REPORT_EMAIL = ["sthapa@ci.uchicago.edu"]
EMAIL_SERVER = "mail.ci-connect.net"

def parse_date(date=None):
    """
    Parse a string in YYYY-MM-DD format into a datetime.date object.
    Throws ValueError if input is invalid

    :param date: string in YYYY-MM-DD format giving a date
    :return: a datetime.date object corresponding to the date given
    """
    if date is None:
        raise ValueError
    fields = date.split('-')
    if len(fields) != 3:
        raise ValueError
    return datetime.date(year=int(fields[0]),
                         month=int(fields[1]),
                         day=int(fields[2]))

def get_week_start(date=None):
    """
    Return a datetime corresponding to the start of the week for the given date
    E.g. if tuesday 7/8/2014 is passed in, monday 7/7/2014 00:00:00 would
    be returned
    """
    if date is None:
        start_date = datetime.date.today()
    else:
        start_date = date
    week_day = start_date.isoweekday()
    if week_day not in [0, 7]:
        week_start = start_date - datetime.timedelta(days=week_day)
    else:
        week_start = start_date
    return week_start

def get_indices(date=None):
    """
    Get the relevant indices for a weekly report using the specified date

    :param date: specifies the week the report will be on
    :return: a list with the indices that should be used
    """

    start_date = get_week_start()
    end_date = start_date + datetime.timedelta(days=7)
    if start_date.month != end_date.month:
        index_list = ["{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               start_date.year,
                                               start_date.month),
                      "{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               end_date.year,
                                               end_date.month)]
    else:
        index_list = ["{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               start_date.year,
                                               start_date.month)]
    es = get_es_client()
    index_client = elasticsearch.client.IndicesClient(es)
    validated_indices = []
    for index in index_list:
        if index_client.exists(index=index):
            validated_indices.append(index)
    return validated_indices

def get_user_modulelist(start_date=None, top=None):
    """
    Give up to the top N modules used per user
    """
    db = get_mongodb_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10

    user_list = {}
    for record in db.weekly_count.find({"date": start_date.isoformat()},
                                       sort=[('count', -1)]):
        user = record['user']
        if user not in user_list:
            user_list[user] = []
        if len(user_list[user]) < top:
            user_list[user].append((record['module'], record['count']))
    return user_list

def get_project_modulelist(start_date=None, top=None):
    """
    Give up to the top N modules used per project
    """
    db = get_es_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10
    end_date = start_date + datetime.timedelta(days=7)
    indices = ",".join(get_indices(start_date))
    query = {"query": {
                "filtered": {
                    "filter": {
                        "range": {
                            "@timestamp": {
                                "gte": start_date.isoformat(),
                                "lt": end_date.isoformat()}}}}},
             "size": 0,
             "aggs": {
                 "projects": {
                     "terms": {
                         "field": "project",
                         "size": 50},
                     "aggs": {
                        "modules": {
                            "terms": {
                                "field": "module",
                                "size": top}}}}}}
    project_list = []
    results = db.search(body=query, size=0, index=indices)
    doc_count = results['hits']['total']
    if doc_count == 0:
        return project_list

    for record in results['aggregations']['projects']['buckets']:
        project_name = record['key']
        for x in record['modules']['buckets']:
            project_list.append((project_name, x['key'], x['doc_count']))
    return project_list

def get_top_modules(start_date, top=None):
    """
    Return a json representation of the top N modules used in the last week
    """
    db = get_es_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10
    end_date = start_date + datetime.timedelta(days=7)
    indices = ",".join(get_indices(start_date))
    query = {"query": {
                "filtered": {
                    "filter": {
                        "range": {
                            "@timestamp": {
                                "gte": start_date.isoformat(),
                                "lt": end_date.isoformat()}}}}},
             "size": 0,
             "aggs": {
                 "modules":
                     {"terms":
                         {"field": "module",
                          "size": top}}}}

    module_list = []
    results = db.search(body=query, size=0, index=indices)
    doc_count = results['hits']['total']
    if doc_count == 0:
        return module_list

    for record in results['aggregations']['modules']['buckets']:
        module_list.append((record['key'], record['doc_count']))
    return module_list

def get_moduleloads(start_date, top=None):
    """
    Return a json representation of the number of times modules where used
    in the week containing start_date
    """
    db = get_mongodb_client()
    if top is None:
        top = 10
    return db.weekly_count.aggregate([{"$match": {"date": start_date.isoformat()}},
                                      {"$group": {"_id": "$date",
                                                         "sum": {"$sum": 1}}},
                                      {"$sort": {"sum": -1}},
                                      {"$limit": top}])['result']['sum']

def generate_report(start_date):
    """
    Generate text and html report of module usage over the week

    :param start_date: timedate.date object giving the date to start from
    :return: a tuple with the text and html report -- (text, html)
    """
    start_date = get_week_start(start_date)
    iso_start = start_date.isoformat()
    report_title = "Modules usage report for week of {0}".format(iso_start)
    report_html = "<title>{0}</title>".format(report_title)
    report_html += ""

    report_html += "
{0}
\n".format(report_title)
    report_text = "{0:^80}\n".format(report_title)

    report_text += "{0:-^80}\n".format('Top 10 modules used')
    report_text += "\n\n"
    report_text += "|{0:^30}|{1:^30}|\n".format('Module', '# of times used')
    module_list = get_top_modules(start_date)
    for module, count in module_list:
        report_text += "|{0:^30}|{1:^30}|\n".format(module, count)
    report_text += "\n\n"

    report_html += "

{0}
\n".format('Top 10 modules used')
    report_html += "

Module	# of times used

\n"
    report_html += "
"
    for module, count in module_list:
        report_text += "
{0}	{1
\n".format(module, count)
    report_html += "

"

    report_text += "{0:-^80}\n".format(' Top 10 modules used by each project')
    report_text += "\n\n"
    report_text += "|{0:^20}|{1:^20}|{2:^20}|\n".format('Project', 'Module', '# of times used')
    project_module_list = get_project_modulelist(start_date)
    for (project, module, count) in project_module_list:
        report_text += "|{0:^20}|{1:^20}|{2:^20}|\n".format(project, module, count)
    report_text += "\n\n"

    report_html += "

{0}
\n".format('Top 10 modules used by each project')
    report_html += "

Project	Module	"
    report_html += "# of times used

\n"
    report_html += "
"
    for (project, module, count) in project_module_list:
        report_text += "
{0}	{1}	{2}
\n".format(project, module, count)
    report_html += "

"

    report_html += '<iframe src="http://kibana.mwt2.org/#/dashboard/' \
                   'Module-usage-dashboard?embed&_g=' \
                   '(refreshInterval:(display:Off,section:0,value:0),' \
                   'time:(from:now-7d,mode:quick,to:now)' \
                   ')&_a=(filters:!(),panels:!((col:1,id:Top-5-Modules-used,' \
                   'row:1,size_x:12,size_y:6,type:visualization),(col:1,id:' \
                   'Module-Usage-locations,row:12,size_x:12,size_y:6,type:' \
                   'visualization),(col:1,id:Module-usage-by-site,row:7,' \
                   'size_x:6,size_y:5,type:visualization),(col:7,' \
                   'id:Module-usage-by-project,row:7,size_x:6,size_y:5,' \
                   'type:visualization)),query:(query_string:(analyze_wildcard:' \
                   '!t,query:\'*\')),title:\'Module%20usage%20dashboard\')" ' \
                   'height="600" width="800">'
    return report_text, report_html

def send_email(text_body, html_body, recipient=REPORT_EMAIL, server=EMAIL_SERVER):
    """
    Send email based on text given in email_body

    :param text_body: text version of email
    :param html_body: html version of email
    :param recipient: list of email addresses to send email to
    :param server: SMTP server to use
    :return: No return
    """
    msg = email.mime.multipart.MIMEMultipart()
    msg['Subject'] = 'Weekly module usage report'
    msg['From'] = 'sthapa@ci.uchicago.edu'
    to_addresses = ",".join(recipient)
    msg['To'] = to_addresses
    msg.attach(html_body)
    msg.attach(text_body)
    server_handle = smtplib.SMTP(server)
    server_handle.sendmail('sthapa@ci.uchicago.edu', to_addresses, msg.as_string())
    server_handle.quit()

def get_es_client():
    """ Instantiate DB client and pass connection back """
    return elasticsearch.Elasticsearch(hosts=ES_NODES,
                                       retry_on_timeout=True,
                                       max_retries=10,
                                       timeout=300)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create a report of module usage for given week')
    parser.add_argument('--start-date', dest='start_date', default=datetime.date.today().isoformat(),
                        help='Date specifying ISO week that the report should be on')
    parser.add_argument('--email', dest='email', default=False,
                        type=bool, help='Email address to send report to')
    args = parser.parse_args(sys.argv[1:])
    args.start_date = parse_date(args.start_date)

    text_report, html_report = generate_report(args.start_date)
    if args.email:
        send_email(text_report, html_report)
    else:
        sys.stdout.write(text_report)
    sys.exit(0)
<commit_msg>Update table html to use css, remove kibana iframe<commit_after>#!/usr/bin/env python

import sys
import datetime
import argparse
import smtplib
import email
import email.mime.text
import email.mime.multipart

import elasticsearch
import elasticsearch.helpers

ES_NODES = ['uct2-es-head.mwt2.org:9200', 'uct2-es-door.mwt2.org:9200']
TZ_NAME = "UTC"
INDEX_BASE_NAME = "oasis-module-usage"
REPORT_EMAIL = ["sthapa@ci.uchicago.edu"]
EMAIL_SERVER = "mail.ci-connect.net"

def parse_date(date=None):
    """
    Parse a string in YYYY-MM-DD format into a datetime.date object.
    Throws ValueError if input is invalid

    :param date: string in YYYY-MM-DD format giving a date
    :return: a datetime.date object corresponding to the date given
    """
    if date is None:
        raise ValueError
    fields = date.split('-')
    if len(fields) != 3:
        raise ValueError
    return datetime.date(year=int(fields[0]),
                         month=int(fields[1]),
                         day=int(fields[2]))

def get_week_start(date=None):
    """
    Return a datetime corresponding to the start of the week for the given date
    E.g. if tuesday 7/8/2014 is passed in, monday 7/7/2014 00:00:00 would
    be returned
    """
    if date is None:
        start_date = datetime.date.today()
    else:
        start_date = date
    week_day = start_date.isoweekday()
    if week_day not in [0, 7]:
        week_start = start_date - datetime.timedelta(days=week_day)
    else:
        week_start = start_date
    return week_start

def get_indices(date=None):
    """
    Get the relevant indices for a weekly report using the specified date

    :param date: specifies the week the report will be on
    :return: a list with the indices that should be used
    """

    start_date = get_week_start()
    end_date = start_date + datetime.timedelta(days=7)
    if start_date.month != end_date.month:
        index_list = ["{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               start_date.year,
                                               start_date.month),
                      "{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               end_date.year,
                                               end_date.month)]
    else:
        index_list = ["{0}-{1}-{2:0>2}".format(INDEX_BASE_NAME,
                                               start_date.year,
                                               start_date.month)]
    es = get_es_client()
    index_client = elasticsearch.client.IndicesClient(es)
    validated_indices = []
    for index in index_list:
        if index_client.exists(index=index):
            validated_indices.append(index)
    return validated_indices

def get_user_modulelist(start_date=None, top=None):
    """
    Give up to the top N modules used per user
    """
    db = get_mongodb_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10

    user_list = {}
    for record in db.weekly_count.find({"date": start_date.isoformat()},
                                       sort=[('count', -1)]):
        user = record['user']
        if user not in user_list:
            user_list[user] = []
        if len(user_list[user]) < top:
            user_list[user].append((record['module'], record['count']))
    return user_list

def get_project_modulelist(start_date=None, top=None):
    """
    Give up to the top N modules used per project
    """
    db = get_es_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10
    end_date = start_date + datetime.timedelta(days=7)
    indices = ",".join(get_indices(start_date))
    query = {"query": {
                "filtered": {
                    "filter": {
                        "range": {
                            "@timestamp": {
                                "gte": start_date.isoformat(),
                                "lt": end_date.isoformat()}}}}},
             "size": 0,
             "aggs": {
                 "projects": {
                     "terms": {
                         "field": "project",
                         "size": 50},
                     "aggs": {
                        "modules": {
                            "terms": {
                                "field": "module",
                                "size": top}}}}}}
    project_list = []
    results = db.search(body=query, size=0, index=indices)
    doc_count = results['hits']['total']
    if doc_count == 0:
        return project_list

    for record in results['aggregations']['projects']['buckets']:
        project_name = record['key']
        for x in record['modules']['buckets']:
            project_list.append((project_name, x['key'], x['doc_count']))
    return project_list

def get_top_modules(start_date, top=None):
    """
    Return a json representation of the top N modules used in the last week
    """
    db = get_es_client()
    if start_date is None:
        start_date = get_week_start()
    else:
        start_date = get_week_start(start_date)

    if top is None:
        top = 10
    end_date = start_date + datetime.timedelta(days=7)
    indices = ",".join(get_indices(start_date))
    query = {"query": {
                "filtered": {
                    "filter": {
                        "range": {
                            "@timestamp": {
                                "gte": start_date.isoformat(),
                                "lt": end_date.isoformat()}}}}},
             "size": 0,
             "aggs": {
                 "modules":
                     {"terms":
                         {"field": "module",
                          "size": top}}}}

    module_list = []
    results = db.search(body=query, size=0, index=indices)
    doc_count = results['hits']['total']
    if doc_count == 0:
        return module_list

    for record in results['aggregations']['modules']['buckets']:
        module_list.append((record['key'], record['doc_count']))
    return module_list

def get_moduleloads(start_date, top=None):
    """
    Return a json representation of the number of times modules where used
    in the week containing start_date
    """
    db = get_mongodb_client()
    if top is None:
        top = 10
    return db.weekly_count.aggregate([{"$match": {"date": start_date.isoformat()}},
                                      {"$group": {"_id": "$date",
                                                         "sum": {"$sum": 1}}},
                                      {"$sort": {"sum": -1}},
                                      {"$limit": top}])['result']['sum']

def generate_report(start_date):
    """
    Generate text and html report of module usage over the week

    :param start_date: timedate.date object giving the date to start from
    :return: a tuple with the text and html report -- (text, html)
    """
    start_date = get_week_start(start_date)
    iso_start = start_date.isoformat()
    report_title = "Modules usage report for week of {0}".format(iso_start)
    report_html = ""
    report_html += "<title>{0}</title>\n".format(report_title)"
    report_html += "\n"
    report_html += ""
    report_html += ""

    report_html += "
{0}
\n".format(report_title)
    report_text = "{0:^80}\n".format(report_title)

    report_text += "{0:-^80}\n".format('Top 10 modules used')
    report_text += "\n\n"
    report_text += "|{0:^30}|{1:^30}|\n".format('Module', '# of times used')
    module_list = get_top_modules(start_date)
    for module, count in module_list:
        report_text += "|{0:^30}|{1:^30}|\n".format(module, count)
    report_text += "\n\n"

    report_html += "

{0}
\n".format('Top 10 modules used')
    report_html += "

Module	# of times used

\n"
    report_html += "
"
    for module, count in module_list:
        report_text += "
{0}	{1
\n".format(module, count)
    report_html += "

"

    report_text += "{0:-^80}\n".format(' Top 10 modules used by each project')
    report_text += "\n\n"
    report_text += "|{0:^20}|{1:^20}|{2:^20}|\n".format('Project', 'Module', '# of times used')
    project_module_list = get_project_modulelist(start_date)
    for (project, module, count) in project_module_list:
        report_text += "|{0:^20}|{1:^20}|{2:^20}|\n".format(project, module, count)
    report_text += "\n\n"

    report_html += "

{0}
\n".format('Top 10 modules used by each project')
    report_html += "

Project	Module	"
    report_html += "# of times used

\n"
    report_html += "
"
    for (project, module, count) in project_module_list:
        report_text += "
{0}	{1}	{2}
\n".format(project, module, count)
    report_html += "

"
    return report_text, report_html

def send_email(text_body, html_body, recipient=REPORT_EMAIL, server=EMAIL_SERVER):
    """
    Send email based on text given in email_body

    :param text_body: text version of email
    :param html_body: html version of email
    :param recipient: list of email addresses to send email to
    :param server: SMTP server to use
    :return: No return
    """
    msg = email.mime.multipart.MIMEMultipart()
    msg['Subject'] = 'Weekly module usage report'
    msg['From'] = 'sthapa@ci.uchicago.edu'
    to_addresses = ",".join(recipient)
    msg['To'] = to_addresses
    msg.attach(html_body)
    msg.attach(text_body)
    server_handle = smtplib.SMTP(server)
    server_handle.sendmail('sthapa@ci.uchicago.edu', to_addresses, msg.as_string())
    server_handle.quit()

def get_es_client():
    """ Instantiate DB client and pass connection back """
    return elasticsearch.Elasticsearch(hosts=ES_NODES,
                                       retry_on_timeout=True,
                                       max_retries=10,
                                       timeout=300)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create a report of module usage for given week')
    parser.add_argument('--start-date', dest='start_date', default=datetime.date.today().isoformat(),
                        help='Date specifying ISO week that the report should be on')
    parser.add_argument('--email', dest='email', default=False,
                        type=bool, help='Email address to send report to')
    args = parser.parse_args(sys.argv[1:])
    args.start_date = parse_date(args.start_date)

    text_report, html_report = generate_report(args.start_date)
    if args.email:
        send_email(text_report, html_report)
    else:
        sys.stdout.write(text_report)
    sys.exit(0)
<|endoftext|>


<commit_before><commit_msg>[fix] Command invocation<commit_after><|endoftext|>


<commit_before># coding: utf-8
# Copyright (c) 2010-2012 Raphaël Barrois

from __future__ import unicode_literals, absolute_import

"""Abstract the filesystem layer."""

import codecs
from fs import osfs, multifs, memoryfs
from fs.wrapfs import readonlyfs
import io
import os
import stat

from . import helpers

class FSConfig(object):
    """Setup for the filesystem layout."""
    def __init__(self, source_root, target_root, chroot='/', dry_run=False):
        self.source_root = helpers.get_absolute_path(source_root)
        self.target_root = helpers.get_absolute_path(target_root, base=source_root)
        self.chroot = helpers.get_absolute_path(chroot)
        self.dry_run = dry_run

    def _add_target_fs(self, full_fs, target, dry_run=False):
        target_fs = helpers.rebase_fs(target, osfs.OSFS(target))

        if dry_run:
            full_fs.addfs('target',
                helpers.rebase_fs(target, memory.MemoryFS()),
                write=True)
            full_fs.addfs('protected_target', target_fs)
        else:
            full_fs.addfs('target', target_fs, write=True)

    def _make_fs(self, target, chroot='/', dry_run=False):
        """Prepare a fs.FS object."""
        base_fs = readonlyfs.ReadOnlyFS(osfs.OSFS(chroot))
        full_fs = multifs.MultiFS()

        self._add_target_fs(full_fs, target, dry_run=dry_run)
        full_fs.addfs('base', base_fs)
        return full_fs

    def get_forward_fs(self):
        wrapped_fs = self._make_fs(self.target_root, self.chroot, self.dry_run)
        return FileSystem(wrapped_fs)

    def get_backward_fs(self):
        wrapped_fs = self._make_fs(self.source_root, self.chroot, self.dry_run)
        return FileSystem(wrapped_fs)

class FileSystem(object):
    def __init__(self, fs, default_encoding=None):
        self.fs = fs
        self.default_encoding = default_encoding

    def __getattr__(self, name):
        return getattr(self.fs, name)

    def copy(self, source, destination, allow_overwrite=True):
        needs_overwrite = self.exists(destination)
        overwrite = needs_overwrite and allow_overwrite
        return self.fs.copy(source, destination, overwrite=overwrite)

    def open(self, filename, mode='r', encoding=None, **kwargs):
        encoding = encoding or self.default_encoding

        if encoding and 'b' not in mode:
            # Text mode requested, and some encoding was given.
            # We don't trust the underlying layers for unicode handling.
            if 't' in mode:
                mode = mode.replace('t', 'b')
            else:
                mode += 'b'

            writer = codecs.getwriter(encoding)
            reader = codecs.getreader(encoding)

            wrapper = lambda stream: reader(writer(stream))
        else:
            wrapper = lambda stream: stream

        base_file = self.fs.open(filename, mode, **kwargs)
        return wrapper(base_file)

    def read_one_line(self, filename, encoding=None):
        """Read one (stripped) line from a file.

        Typically used to read a password.
        """
        with self.open(filename, 'rt', encoding=encoding) as f:
            return f.readline().strip()

    def readlines(self, filename, encoding=None):
        """Read all lines from a file.

        Yields lines of the file, stripping the terminating \n.
        """
        with self.open(filename, 'rt', encoding=encoding) as f:
            for line in f:
                if line and line[-1] == '\n':
                    # Strip final \n
                    line = line[:-1]
                yield line

    def writelines(self, filename, lines, encoding=None):
        """Write a set of lines to a file.

        A \n will be appended to lines.
        """
        with self.open(filename, 'wt', encoding=encoding) as f:
            for line in lines:
                f.write(line)
                # Add final \n.
                f.write('\n')

    # Complex commands
    def create_symlink(self, link_name, target, relative=False, force=False):
        if self.access(link_name):
            file_stat = self.stat(link_name)
            if not stat.S_ISLNK(file_stat.st_mode) and not force:
                raise IOError(
                    "File at %s exists and is not a symlink." % link_name)
            elif stat.S_ISDIR(file_stat.st_mode):
                raise IOError("%s exists and is a directory.")
            else:
                self.remove(link_name)

        # TODO: Handle relative=True
        self.symlink(link_name, target)
<commit_msg>Rewrite FSConfig setup.<commit_after># coding: utf-8
# Copyright (c) 2010-2012 Raphaël Barrois

from __future__ import unicode_literals, absolute_import

"""Abstract the filesystem layer."""

import codecs
from fs import osfs, multifs, mountfs, memoryfs
from fs.wrapfs import readonlyfs
import io
import os
import stat

from . import helpers

class FSConfig(object):
    """Setup for the filesystem layout."""
    def __init__(self, source_root, target_root, chroot='/', dry_run=False):
        self.source_root = helpers.get_absolute_path(source_root)
        self.target_root = helpers.get_absolute_path(target_root, base=source_root)
        self.chroot = helpers.get_absolute_path(chroot)
        self.dry_run = dry_run
        self._forward_fs = self._backward_fs = None

    def get_forward_fs(self, **kwargs):
        if self._forward_fs is None:
            self._forward_fs = FileSystem(self.target_root, dry_run=self.dry_run,
                **kwargs)
        return self._forward_fs

    def get_backward_fs(self, **kwargs):
        if self._backward_fs is None:
            self._backward_fs = FileSystem(self.source_root, dry_run=self.dry_run,
                **kwargs)
        return self._backward_fs

class FileSystem(object):
    def __init__(self, *write_paths, **kwargs):
        self.dry_run = kwargs.pop('dry_run', False)
        self.default_encoding = kwargs.pop('default_encoding', None)
        self.write_paths = write_paths
        self.fs, self.subfs = self._prepare_fs(write_paths, dry_run=self.dry_run)

    def _make_merged_fs(self, paths, memory=False):
        """Make a (merged) filesystem for a given set of paths.

        If ``memory`` is True, the resulting filesystem will only use
        memoryfs; otherwise, it will be backed by actual osfs.OSFS.

        Returns:
            (fs, dict(path => fs)): the resulting filesystem, and a dict mapping
                each path to the related filesystem
        """
        if memory:
            fs_class = lambda path: memoryfs.MemoryFS()
        else:
            fs_class = lambda path: osfs.OSFS(path)

        merged_fs = mountfs.MountFS()
        filesystems = {}
        for path in paths:
            fs = fs_class(path)
            merged_fs.mount(path, fs)
            filesystems[path] = fs

        return merged_fs, filesystems

    def _prepare_fs(self, paths, dry_run=False):
        """Prepare the filesystem for a set of writable paths."""
        base_fs = multifs.MultiFS()
        merged_fs, sub_filesystems = self._make_merged_fs(paths)
        if dry_run:
            memory_fs, sub_filesystems = self._make_merged_fs(paths, memory=True)
            base_fs.addfs('memory', memory_fs, write=True)

        base_fs.addfs('targets', merged_fs, write=not dry_run)
        base_fs.addfs('base', readonlyfs.ReadOnlyFS(osfs.OSFS('/')))

        return base_fs, sub_filesystems

    def get_changes(self):
        if self.dry_run:
            for path, fs in self.subfs.items():
                for subpath in fs.ilistdir(full=True):
                    yield path, subpath

    def __getattr__(self, name):
        return getattr(self.fs, name)

    def copy(self, source, destination, allow_overwrite=True):
        needs_overwrite = self.exists(destination)
        overwrite = needs_overwrite and allow_overwrite
        return self.fs.copy(source, destination, overwrite=overwrite)

    def open(self, filename, mode='r', encoding=None, **kwargs):
        encoding = encoding or self.default_encoding

        if encoding and 'b' not in mode:
            # Text mode requested, and some encoding was given.
            # We don't trust the underlying layers for unicode handling.
            if 't' in mode:
                mode = mode.replace('t', 'b')
            else:
                mode += 'b'

            writer = codecs.getwriter(encoding)
            reader = codecs.getreader(encoding)

            wrapper = lambda stream: reader(writer(stream))
        else:
            wrapper = lambda stream: stream

        base_file = self.fs.open(filename, mode, **kwargs)
        return wrapper(base_file)

    def read_one_line(self, filename, encoding=None):
        """Read one (stripped) line from a file.

        Typically used to read a password.
        """
        with self.open(filename, 'rt', encoding=encoding) as f:
            return f.readline().strip()

    def readlines(self, filename, encoding=None):
        """Read all lines from a file.

        Yields lines of the file, stripping the terminating \n.
        """
        with self.open(filename, 'rt', encoding=encoding) as f:
            for line in f:
                if line and line[-1] == '\n':
                    # Strip final \n
                    line = line[:-1]
                yield line

    def writelines(self, filename, lines, encoding=None):
        """Write a set of lines to a file.

        A \n will be appended to lines.
        """
        with self.open(filename, 'wt', encoding=encoding) as f:
            for line in lines:
                f.write(line)
                # Add final \n.
                f.write('\n')

    # Complex commands
    def create_symlink(self, link_name, target, relative=False, force=False):
        if self.access(link_name):
            file_stat = self.stat(link_name)
            if not stat.S_ISLNK(file_stat.st_mode) and not force:
                raise IOError(
                    "File at %s exists and is not a symlink." % link_name)
            elif stat.S_ISDIR(file_stat.st_mode):
                raise IOError("%s exists and is a directory.")
            else:
                self.remove(link_name)

        # TODO: Handle relative=True
        self.symlink(link_name, target)
<|endoftext|>


<commit_before>from arguments import make_app
app = make_app()
from arguments import db
from arguments.database.datamodel import *
s = db.session
q = s.query

ip = get_ipython()

ip.magic("autocall 2")
<commit_msg>fix consoleenv<commit_after>from munch import Munch
from arguments.app import make_wsgi_app
app = make_wsgi_app("config.yml")
from arguments import database
from arguments.app import App
from arguments.database.datamodel import *

s = database.Session()
q = s.query

ip = get_ipython()

ip.magic("autocall 2")
<|endoftext|>