<commit_before>import logging
import os
import sys
import unittest2 as unittest

from multiprocessing import get_logger, current_process
from StringIO import StringIO

from celery import Celery
from celery import platforms
from celery import signals
from celery.app import app_or_default
from celery.apps import worker as cd
from celery.bin.celeryd import WorkerCommand, main as celeryd_main
from celery.exceptions import ImproperlyConfigured
from celery.utils import patch
from celery.utils.functional import wraps

from celery.tests.compat import catch_warnings
from celery.tests.utils import execute_context

patch.ensure_process_aware_logger()

def disable_stdouts(fun):

    @wraps(fun)
    def disable(*args, **kwargs):
        sys.stdout, sys.stderr = StringIO(), StringIO()
        try:
            return fun(*args, **kwargs)
        finally:
            sys.stdout = sys.__stdout__
            sys.stderr = sys.__stderr__

    return disable

class _WorkController(object):

    def __init__(self, *args, **kwargs):
        pass

    def start(self):
        pass

class Worker(cd.Worker):
    WorkController = _WorkController

class test_Worker(unittest.TestCase):
    Worker = Worker

    @disable_stdouts
    def test_queues_string(self):
        celery = Celery(set_as_current=False)
        worker = celery.Worker(queues="foo,bar,baz")
        worker.init_queues()
        self.assertEqual(worker.use_queues, ["foo", "bar", "baz"])
        self.assertTrue("foo" in celery.amqp.queues)

    @disable_stdouts
    def test_loglevel_string(self):
        worker = self.Worker(loglevel="INFO")
        self.assertEqual(worker.loglevel, logging.INFO)

    def test_run_worker(self):
        handlers = {}

        def i(sig, handler):
            handlers[sig] = handler

        p = platforms.install_signal_handler
        platforms.install_signal_handler = i
        try:
            w = self.Worker()
            w._isatty = False
            w.run_worker()
            for sig in "SIGINT", "SIGHUP", "SIGTERM":
                self.assertIn(sig, handlers)

            handlers.clear()
            w = self.Worker()
            w._isatty = True
            w.run_worker()
            for sig in "SIGINT", "SIGTERM":
                self.assertIn(sig, handlers)
            self.assertNotIn("SIGHUP", handlers)
        finally:
            platforms.install_signal_handler = p

    @disable_stdouts
    def test_startup_info(self):
        worker = self.Worker()
        worker.run()
        self.assertTrue(worker.startup_info())
        worker.loglevel = logging.DEBUG
        self.assertTrue(worker.startup_info())
        worker.loglevel = logging.INFO
        self.assertTrue(worker.startup_info())

    @disable_stdouts
    def test_run(self):
        self.Worker().run()
        self.Worker(discard=True).run()

        worker = self.Worker()
        worker.init_loader()
        worker.settings.DEBUG = True

        def with_catch_warnings(log):
            worker.run()
            self.assertIn("memory leak", log[0].message.args[0])

        context = catch_warnings(record=True)
        execute_context(context, with_catch_warnings)
        worker.settings.DEBUG = False

    @disable_stdouts
    def test_purge_messages(self):
        self.Worker().purge_messages()

    @disable_stdouts
    def test_init_queues(self):
        app = app_or_default()
        c = app.conf
        p, app.amqp.queues = app.amqp.queues, {
                "celery": {"exchange": "celery",
                           "binding_key": "celery"},
                "video": {"exchange": "video",
                           "binding_key": "video"}}
        try:
            worker = self.Worker(queues=["video"])
            worker.init_queues()
            self.assertIn("video", worker.queues)
            self.assertNotIn("celery", worker.queues)

            c.CELERY_CREATE_MISSING_QUEUES = False
            self.assertRaises(ImproperlyConfigured,
                    self.Worker(queues=["image"]).init_queues)
            c.CELERY_CREATE_MISSING_QUEUES = True
            worker = self.Worker(queues=["image"])
            worker.init_queues()
            self.assertIn("image", worker.queues)
            self.assertDictContainsSubset({"exchange": "image",
                                           "routing_key": "image",
                                           "binding_key": "image",
                                           "exchange_type": "direct"},
                                            worker.queues["image"])
        finally:
            app.amqp.queues = p

    @disable_stdouts
    def test_on_listener_ready(self):
        worker_ready_sent = [False]

        def on_worker_ready(**kwargs):
            worker_ready_sent[0] = True

        signals.worker_ready.connect(on_worker_ready)

        self.Worker().on_listener_ready(object())
        self.assertTrue(worker_ready_sent[0])

class test_funs(unittest.TestCase):

    @disable_stdouts
    def test_set_process_status(self):
        prev1, sys.argv = sys.argv, ["Arg0"]
        try:
            st = cd.set_process_status("Running")
            self.assertIn("celeryd", st)
            self.assertIn("Running", st)
            prev2, sys.argv = sys.argv, ["Arg0", "Arg1"]
            try:
                st = cd.set_process_status("Running")
                self.assertIn("celeryd", st)
                self.assertIn("Running", st)
                self.assertIn("Arg1", st)
            finally:
                sys.argv = prev2
        finally:
            sys.argv = prev1

    @disable_stdouts
    def test_parse_options(self):
        cmd = WorkerCommand()
        cmd.app = app_or_default()
        opts, args = cmd.parse_options("celeryd", ["--concurrency=512"])
        self.assertEqual(opts.concurrency, 512)

    @disable_stdouts
    def test_run_worker(self):
        p, cd.Worker = cd.Worker, Worker
        try:
            cd.run_worker(discard=True)
        finally:
            cd.Worker = p

    @disable_stdouts
    def test_main(self):
        p, cd.Worker = cd.Worker, Worker
        s, sys.argv = sys.argv, ["celeryd", "--discard"]
        try:
            celeryd_main()
        finally:
            cd.Worker = p
            sys.argv = s

class test_signal_handlers(unittest.TestCase):

    class _Worker(object):
        stopped = False
        terminated = False
        logger = get_logger()

        def stop(self):
            self.stopped = True

        def terminate(self):
            self.terminated = True

    def psig(self, fun, *args, **kwargs):
        handlers = {}

        def i(sig, handler):
            handlers[sig] = handler

        p, platforms.install_signal_handler = \
                platforms.install_signal_handler, i
        try:
            fun(*args, **kwargs)
            return handlers
        finally:
            platforms.install_signal_handler = p

    @disable_stdouts
    def test_worker_int_handler(self):
        worker = self._Worker()
        handlers = self.psig(cd.install_worker_int_handler, worker)
        next_handlers = {}

        def i(sig, handler):
            next_handlers[sig] = handler

        p = platforms.install_signal_handler
        platforms.install_signal_handler = i
        try:
            self.assertRaises(SystemExit, handlers["SIGINT"],
                              "SIGINT", object())
            self.assertTrue(worker.stopped)
        finally:
            platforms.install_signal_handler = p

        self.assertRaises(SystemExit, next_handlers["SIGINT"],
                          "SIGINT", object())
        self.assertTrue(worker.terminated)

    @disable_stdouts
    def test_worker_int_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_int_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGINT"],
                            "SIGINT", object())
            self.assertFalse(worker.stopped)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_int_again_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_int_again_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGINT"],
                            "SIGINT", object())
            self.assertFalse(worker.terminated)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_term_handler(self):
        worker = self._Worker()
        handlers = self.psig(cd.install_worker_term_handler, worker)
        self.assertRaises(SystemExit, handlers["SIGTERM"],
                          "SIGTERM", object())
        self.assertTrue(worker.stopped)

    @disable_stdouts
    def test_worker_term_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_term_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGTERM"],
                          "SIGTERM", object())
            self.assertFalse(worker.stopped)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_restart_handler(self):
        argv = []

        def _execv(*args):
            argv.extend(args)

        execv, os.execv = os.execv, _execv
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_restart_handler, worker)
            handlers["SIGHUP"]("SIGHUP", object())
            self.assertTrue(worker.stopped)
            self.assertTrue(argv)
        finally:
            os.execv = execv
<commit_msg>Fixed broken tests<commit_after>import logging
import os
import sys
import unittest2 as unittest

from multiprocessing import get_logger, current_process
from StringIO import StringIO

from celery import Celery
from celery import platforms
from celery import signals
from celery.app import app_or_default
from celery.apps import worker as cd
from celery.bin.celeryd import WorkerCommand, main as celeryd_main
from celery.exceptions import ImproperlyConfigured
from celery.utils import patch
from celery.utils.functional import wraps

from celery.tests.compat import catch_warnings
from celery.tests.utils import execute_context

patch.ensure_process_aware_logger()

def disable_stdouts(fun):

    @wraps(fun)
    def disable(*args, **kwargs):
        sys.stdout, sys.stderr = StringIO(), StringIO()
        try:
            return fun(*args, **kwargs)
        finally:
            sys.stdout = sys.__stdout__
            sys.stderr = sys.__stderr__

    return disable

class _WorkController(object):

    def __init__(self, *args, **kwargs):
        self.logger = log.get_default_logger()

    def start(self):
        pass

class Worker(cd.Worker):
    WorkController = _WorkController

class test_Worker(unittest.TestCase):
    Worker = Worker

    @disable_stdouts
    def test_queues_string(self):
        celery = Celery(set_as_current=False)
        worker = celery.Worker(queues="foo,bar,baz")
        worker.init_queues()
        self.assertEqual(worker.use_queues, ["foo", "bar", "baz"])
        self.assertTrue("foo" in celery.amqp.queues)

    @disable_stdouts
    def test_loglevel_string(self):
        worker = self.Worker(loglevel="INFO")
        self.assertEqual(worker.loglevel, logging.INFO)

    def test_run_worker(self):
        handlers = {}

        def i(sig, handler):
            handlers[sig] = handler

        p = platforms.install_signal_handler
        platforms.install_signal_handler = i
        try:
            w = self.Worker()
            w._isatty = False
            w.run_worker()
            for sig in "SIGINT", "SIGHUP", "SIGTERM":
                self.assertIn(sig, handlers)

            handlers.clear()
            w = self.Worker()
            w._isatty = True
            w.run_worker()
            for sig in "SIGINT", "SIGTERM":
                self.assertIn(sig, handlers)
            self.assertNotIn("SIGHUP", handlers)
        finally:
            platforms.install_signal_handler = p

    @disable_stdouts
    def test_startup_info(self):
        worker = self.Worker()
        worker.run()
        self.assertTrue(worker.startup_info())
        worker.loglevel = logging.DEBUG
        self.assertTrue(worker.startup_info())
        worker.loglevel = logging.INFO
        self.assertTrue(worker.startup_info())

    @disable_stdouts
    def test_run(self):
        self.Worker().run()
        self.Worker(discard=True).run()

        worker = self.Worker()
        worker.init_loader()
        worker.settings.DEBUG = True

        def with_catch_warnings(log):
            worker.run()
            self.assertIn("memory leak", log[0].message.args[0])

        context = catch_warnings(record=True)
        execute_context(context, with_catch_warnings)
        worker.settings.DEBUG = False

    @disable_stdouts
    def test_purge_messages(self):
        self.Worker().purge_messages()

    @disable_stdouts
    def test_init_queues(self):
        app = app_or_default()
        c = app.conf
        p, app.amqp.queues = app.amqp.queues, {
                "celery": {"exchange": "celery",
                           "binding_key": "celery"},
                "video": {"exchange": "video",
                           "binding_key": "video"}}
        try:
            worker = self.Worker(queues=["video"])
            worker.init_queues()
            self.assertIn("video", worker.queues)
            self.assertNotIn("celery", worker.queues)

            c.CELERY_CREATE_MISSING_QUEUES = False
            self.assertRaises(ImproperlyConfigured,
                    self.Worker(queues=["image"]).init_queues)
            c.CELERY_CREATE_MISSING_QUEUES = True
            worker = self.Worker(queues=["image"])
            worker.init_queues()
            self.assertIn("image", worker.queues)
            self.assertDictContainsSubset({"exchange": "image",
                                           "routing_key": "image",
                                           "binding_key": "image",
                                           "exchange_type": "direct"},
                                            worker.queues["image"])
        finally:
            app.amqp.queues = p

    @disable_stdouts
    def test_on_listener_ready(self):
        worker_ready_sent = [False]

        def on_worker_ready(**kwargs):
            worker_ready_sent[0] = True

        signals.worker_ready.connect(on_worker_ready)

        self.Worker().on_listener_ready(object())
        self.assertTrue(worker_ready_sent[0])

class test_funs(unittest.TestCase):

    @disable_stdouts
    def test_set_process_status(self):
        worker = Worker(hostname="xyzza")
        prev1, sys.argv = sys.argv, ["Arg0"]
        try:
            st = worker.set_process_status("Running")
            self.assertIn("celeryd", st)
            self.assertIn("xyzza", st)
            self.assertIn("Running", st)
            prev2, sys.argv = sys.argv, ["Arg0", "Arg1"]
            try:
                st = worker.set_process_status("Running")
                self.assertIn("celeryd", st)
                self.assertIn("xyzza", st)
                self.assertIn("Running", st)
                self.assertIn("Arg1", st)
            finally:
                sys.argv = prev2
        finally:
            sys.argv = prev1

    @disable_stdouts
    def test_parse_options(self):
        cmd = WorkerCommand()
        cmd.app = app_or_default()
        opts, args = cmd.parse_options("celeryd", ["--concurrency=512"])
        self.assertEqual(opts.concurrency, 512)

    @disable_stdouts
    def test_run_worker(self):
        p, cd.Worker = cd.Worker, Worker
        try:
            cd.run_worker(discard=True)
        finally:
            cd.Worker = p

    @disable_stdouts
    def test_main(self):
        p, cd.Worker = cd.Worker, Worker
        s, sys.argv = sys.argv, ["celeryd", "--discard"]
        try:
            celeryd_main()
        finally:
            cd.Worker = p
            sys.argv = s

class test_signal_handlers(unittest.TestCase):

    class _Worker(object):
        stopped = False
        terminated = False
        logger = get_logger()

        def stop(self):
            self.stopped = True

        def terminate(self):
            self.terminated = True

    def psig(self, fun, *args, **kwargs):
        handlers = {}

        def i(sig, handler):
            handlers[sig] = handler

        p, platforms.install_signal_handler = \
                platforms.install_signal_handler, i
        try:
            fun(*args, **kwargs)
            return handlers
        finally:
            platforms.install_signal_handler = p

    @disable_stdouts
    def test_worker_int_handler(self):
        worker = self._Worker()
        handlers = self.psig(cd.install_worker_int_handler, worker)
        next_handlers = {}

        def i(sig, handler):
            next_handlers[sig] = handler

        p = platforms.install_signal_handler
        platforms.install_signal_handler = i
        try:
            self.assertRaises(SystemExit, handlers["SIGINT"],
                              "SIGINT", object())
            self.assertTrue(worker.stopped)
        finally:
            platforms.install_signal_handler = p

        self.assertRaises(SystemExit, next_handlers["SIGINT"],
                          "SIGINT", object())
        self.assertTrue(worker.terminated)

    @disable_stdouts
    def test_worker_int_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_int_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGINT"],
                            "SIGINT", object())
            self.assertFalse(worker.stopped)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_int_again_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_int_again_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGINT"],
                            "SIGINT", object())
            self.assertFalse(worker.terminated)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_term_handler(self):
        worker = self._Worker()
        handlers = self.psig(cd.install_worker_term_handler, worker)
        self.assertRaises(SystemExit, handlers["SIGTERM"],
                          "SIGTERM", object())
        self.assertTrue(worker.stopped)

    @disable_stdouts
    def test_worker_term_handler_only_stop_MainProcess(self):
        process = current_process()
        name, process.name = process.name, "OtherProcess"
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_term_handler, worker)
            self.assertRaises(SystemExit, handlers["SIGTERM"],
                          "SIGTERM", object())
            self.assertFalse(worker.stopped)
        finally:
            process.name = name

    @disable_stdouts
    def test_worker_restart_handler(self):
        argv = []

        def _execv(*args):
            argv.extend(args)

        execv, os.execv = os.execv, _execv
        try:
            worker = self._Worker()
            handlers = self.psig(cd.install_worker_restart_handler, worker)
            handlers["SIGHUP"]("SIGHUP", object())
            self.assertTrue(worker.stopped)
            self.assertTrue(argv)
        finally:
            os.execv = execv
<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the view for the site menus."""

import datetime

from google.appengine.api import users

from django.core.urlresolvers import reverse

from soc.logic import accounts
from soc.views.template import Template

from soc.modules.gsoc.logic.program import getMostRecentProgram

from soc.modules.gci.models.task import ACTIVE_CLAIMED_TASK
from soc.modules.gci.models.task import GCITask
from soc.modules.gci.views.helper import url_names

def siteMenuContext(data):
  """Generates URL links for the hard-coded GCI site menu items.
  """
  redirect = data.redirect
  program = data.program

  from soc.modules.gci.models.program import GCIProgram
  about_page = GCIProgram.about_page.get_value_for_datastore(program)
  connect = GCIProgram.connect_with_us_page.get_value_for_datastore(program)
  help_page = GCIProgram.help_page.get_value_for_datastore(program)
  terms = GCIProgram.terms_and_conditions.get_value_for_datastore(program)

  context = {
      'about_link': redirect.document(about_page).url(),
      'terms_link': redirect.document(terms).url(),
      'events_link': redirect.events().url(),
      'connect_link': redirect.document(connect).url(),
      'help_link': redirect.document(help_page).url(),
  }

  if users.get_current_user():
    context['logout_link'] = redirect.logout().url()
  else:
    context['login_link'] = redirect.login().url()

  if data.user:
    context['dashboard_link'] = redirect.dashboard().url()

  if data.timeline.tasksPubliclyVisible():
    context['tasks_link'] = redirect.program().urlOf('gci_list_tasks')
    if not data.user:
      context['register_as_student_link'] = redirect.createProfile(
          'student').urlOf('create_gci_profile', secure=True)

  return context

class Header(Template):
  """MainMenu template.
  """

  def __init__(self, data):
    self.data = data

  def templatePath(self):
    return "v2/modules/gci/_header.html"

  def context(self):
    # Need this import to make sponsor visible for sponsor link_id
    from soc.models.sponsor import Sponsor

    gsoc_link = ''
    key_name = getMostRecentProgram(self.data)

    if key_name:
      sponsor, program = key_name.split('/')
      gsoc_kwargs = {
          'sponsor': sponsor,
          'program': program,
      }
      # We have to use reverse method instead of the redirect helper
      # because we have to get the URL for a program of the other module
      # that is not part of the request data. So we cannot directly use
      # the redirect helper, since a module's redirect helper doesn't
      # resolve to the correct module URL prefix.
      gsoc_link = reverse('gsoc_homepage', kwargs=gsoc_kwargs)

    return {
        'home_link': self.data.redirect.homepage().url(),
        'gsoc_link': gsoc_link,
    }

class MainMenu(Template):
  """MainMenu template.
  """

  def __init__(self, data):
    self.data = data

  def context(self):
    context = siteMenuContext(self.data)
    context.update({
        'home_link': self.data.redirect.homepage().url(),
    })

    if self.data.profile:
      self.data.redirect.program()
      if self.data.profile.status == 'active':
        if self.data.timeline.programActive():
          context['profile_link'] = self.data.redirect.urlOf(
              'edit_gci_profile', secure=True)
        else:
          context['profile_link'] = self.data.redirect.urlOf(
              url_names.GCI_PROFILE_SHOW, secure=True)

        if self.data.is_org_admin:
          # Add org admin dashboard link if the user has active
          # org admin profile and is an org admin of some organization
          context['org_dashboard_link'] = self.data.redirect.urlOf(
              'gci_dashboard')

    if self.data.is_host:
      self.data.redirect.program()
      context['admin_link'] = self.data.redirect.urlOf('gci_admin_dashboard')

    return context

  def templatePath(self):
    return "v2/modules/gci/_mainmenu.html"

class Footer(Template):
  """Footer template.
  """

  def __init__(self, data):
    self.data = data

  def context(self):
    context = siteMenuContext(self.data)
    program = self.data.program

    context.update({
        'privacy_policy_link': program.privacy_policy_url,
        'blogger_link': program.blogger,
        'email_id': program.email,
        'irc_link': program.irc,
        })

    return context

  def templatePath(self):
    return "v2/modules/gci/_footer.html"

class Status(Template):
  """Template to render the status block.
  """

  def __init__(self, data):
    self.data = data

  def getTimeLeftForTask(self, task):
    """Time left to complete the task in human readable format.
    """
    if not task.deadline:
      return ""

    time_now = datetime.datetime.utcnow()
    time_left = task.deadline - time_now
    days_left = time_left.days
    hours_left = time_left.seconds/3600
    minutes_left = (time_left.seconds/60)%60
    return "%s days %s hrs %s min" % (days_left, hours_left, minutes_left)

  def context(self):
    context = {
        'user_email': accounts.denormalizeAccount(self.data.user.account).email(),
        'link_id': self.data.user.link_id,
        'logout_link': self.data.redirect.logout().url(),
        'dashboard_link': self.data.redirect.dashboard().url()
    }

    if self.data.profile:
      if self.data.is_student and self.data.profile.status == 'active':
        q = GCITask.all()
        q.filter('student', self.data.profile)
        q.filter('status IN', ACTIVE_CLAIMED_TASK)
        task = q.get()
        if task:
          context['task'] = task
          context['time_left'] = self.getTimeLeftForTask(task)
          task_url = self.data.redirect.id(
              task.key().id()).urlOf('gci_view_task')
          context['task_url'] = task_url
    return context

  def templatePath(self):
    return "v2/modules/gci/_status_block.html"
<commit_msg>'program_id' added to Header template for GCI.<commit_after>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the view for the site menus."""

import datetime

from google.appengine.api import users

from django.core.urlresolvers import reverse

from soc.logic import accounts
from soc.views.template import Template

from soc.modules.gsoc.logic.program import getMostRecentProgram

from soc.modules.gci.models.task import ACTIVE_CLAIMED_TASK
from soc.modules.gci.models.task import GCITask
from soc.modules.gci.views.helper import url_names

def siteMenuContext(data):
  """Generates URL links for the hard-coded GCI site menu items.
  """
  redirect = data.redirect
  program = data.program

  from soc.modules.gci.models.program import GCIProgram
  about_page = GCIProgram.about_page.get_value_for_datastore(program)
  connect = GCIProgram.connect_with_us_page.get_value_for_datastore(program)
  help_page = GCIProgram.help_page.get_value_for_datastore(program)
  terms = GCIProgram.terms_and_conditions.get_value_for_datastore(program)

  context = {
      'about_link': redirect.document(about_page).url(),
      'terms_link': redirect.document(terms).url(),
      'events_link': redirect.events().url(),
      'connect_link': redirect.document(connect).url(),
      'help_link': redirect.document(help_page).url(),
  }

  if users.get_current_user():
    context['logout_link'] = redirect.logout().url()
  else:
    context['login_link'] = redirect.login().url()

  if data.user:
    context['dashboard_link'] = redirect.dashboard().url()

  if data.timeline.tasksPubliclyVisible():
    context['tasks_link'] = redirect.program().urlOf('gci_list_tasks')
    if not data.user:
      context['register_as_student_link'] = redirect.createProfile(
          'student').urlOf('create_gci_profile', secure=True)

  return context

class Header(Template):
  """MainMenu template.
  """

  def __init__(self, data):
    self.data = data

  def templatePath(self):
    return "v2/modules/gci/_header.html"

  def context(self):
    # Need this import to make sponsor visible for sponsor link_id
    from soc.models.sponsor import Sponsor

    gsoc_link = ''
    key_name = getMostRecentProgram(self.data)

    if key_name:
      sponsor, program = key_name.split('/')
      gsoc_kwargs = {
          'sponsor': sponsor,
          'program': program,
      }
      # We have to use reverse method instead of the redirect helper
      # because we have to get the URL for a program of the other module
      # that is not part of the request data. So we cannot directly use
      # the redirect helper, since a module's redirect helper doesn't
      # resolve to the correct module URL prefix.
      gsoc_link = reverse('gsoc_homepage', kwargs=gsoc_kwargs)

    return {
        'home_link': self.data.redirect.homepage().url(),
        'gsoc_link': gsoc_link,
        'program_id': self.data.program.link_id,
    }

class MainMenu(Template):
  """MainMenu template.
  """

  def __init__(self, data):
    self.data = data

  def context(self):
    context = siteMenuContext(self.data)
    context.update({
        'home_link': self.data.redirect.homepage().url(),
    })

    if self.data.profile:
      self.data.redirect.program()
      if self.data.profile.status == 'active':
        if self.data.timeline.programActive():
          context['profile_link'] = self.data.redirect.urlOf(
              'edit_gci_profile', secure=True)
        else:
          context['profile_link'] = self.data.redirect.urlOf(
              url_names.GCI_PROFILE_SHOW, secure=True)

        if self.data.is_org_admin:
          # Add org admin dashboard link if the user has active
          # org admin profile and is an org admin of some organization
          context['org_dashboard_link'] = self.data.redirect.urlOf(
              'gci_dashboard')

    if self.data.is_host:
      self.data.redirect.program()
      context['admin_link'] = self.data.redirect.urlOf('gci_admin_dashboard')

    return context

  def templatePath(self):
    return "v2/modules/gci/_mainmenu.html"

class Footer(Template):
  """Footer template.
  """

  def __init__(self, data):
    self.data = data

  def context(self):
    context = siteMenuContext(self.data)
    program = self.data.program

    context.update({
        'privacy_policy_link': program.privacy_policy_url,
        'blogger_link': program.blogger,
        'email_id': program.email,
        'irc_link': program.irc,
        })

    return context

  def templatePath(self):
    return "v2/modules/gci/_footer.html"

class Status(Template):
  """Template to render the status block.
  """

  def __init__(self, data):
    self.data = data

  def getTimeLeftForTask(self, task):
    """Time left to complete the task in human readable format.
    """
    if not task.deadline:
      return ""

    time_now = datetime.datetime.utcnow()
    time_left = task.deadline - time_now
    days_left = time_left.days
    hours_left = time_left.seconds/3600
    minutes_left = (time_left.seconds/60)%60
    return "%s days %s hrs %s min" % (days_left, hours_left, minutes_left)

  def context(self):
    context = {
        'user_email': accounts.denormalizeAccount(self.data.user.account).email(),
        'link_id': self.data.user.link_id,
        'logout_link': self.data.redirect.logout().url(),
        'dashboard_link': self.data.redirect.dashboard().url()
    }

    if self.data.profile:
      if self.data.is_student and self.data.profile.status == 'active':
        q = GCITask.all()
        q.filter('student', self.data.profile)
        q.filter('status IN', ACTIVE_CLAIMED_TASK)
        task = q.get()
        if task:
          context['task'] = task
          context['time_left'] = self.getTimeLeftForTask(task)
          task_url = self.data.redirect.id(
              task.key().id()).urlOf('gci_view_task')
          context['task_url'] = task_url
    return context

  def templatePath(self):
    return "v2/modules/gci/_status_block.html"
<|endoftext|>


<commit_before>import itertools
import os
import unittest
from pymclevel import mclevel
from templevel import TempLevel, mktemp
from pymclevel.schematic import MCSchematic
from pymclevel.box import BoundingBox

__author__ = 'Rio'

class TestSchematics(unittest.TestCase):
    def setUp(self):
        # self.alphaLevel = TempLevel("Dojo_64_64_128.dat")
        self.indevLevel = TempLevel("hell.mclevel")
        self.anvilLevel = TempLevel("AnvilWorld")

    def testCreate(self):
        # log.info("Schematic from indev")

        size = (64, 64, 64)
        temp = mktemp("testcreate.schematic")
        schematic = MCSchematic(shape=size, filename=temp, mats='Classic')
        level = self.indevLevel.level

        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (0, 0, 0))
        assert((schematic.Blocks[0:64, 0:64, 0:64] == level.Blocks[0:64, 0:64, 0:64]).all())

        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (-32, -32, -32))
        assert((schematic.Blocks[0:32, 0:32, 0:32] == level.Blocks[32:64, 32:64, 32:64]).all())

        schematic.saveInPlace()

        schem = mclevel.fromFile("schematics/CreativeInABox.schematic")
        tempSchematic = MCSchematic(shape=(1, 1, 3))
        tempSchematic.copyBlocksFrom(schem, BoundingBox((0, 0, 0), (1, 1, 3)), (0, 0, 0))

        level = self.anvilLevel.level
        for cx, cz in itertools.product(xrange(0, 4), xrange(0, 4)):
            try:
                level.createChunk(cx, cz)
            except ValueError:
                pass
        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (0, 0, 0))
        schematic.close()
        os.remove(temp)

    def testRotate(self):
        level = self.indevLevel.level
        schematic = level.extractSchematic(level.bounds)
        schematic.rotateLeft()
        schematic.flipNorthSouth()
        schematic.flipVertical()

    def testZipSchematic(self):
        level = self.anvilLevel.level

        x, y, z = level.bounds.origin
        x += level.bounds.size[0]/2 & ~15
        z += level.bounds.size[2]/2 & ~15

        box = BoundingBox((x, y, z), (64, 64, 64,))
        zs = level.extractZipSchematic(box)
        assert(box.chunkCount == zs.chunkCount)
        zs.close()
        os.remove(zs.filename)

    def testINVEditChests(self):
        invFile = mclevel.fromFile("schematics/Chests/TinkerersBox.inv")
        assert invFile.Blocks.any()
        assert not invFile.Data.any()
        assert len(invFile.Entities) == 0
        assert len(invFile.TileEntities) == 1
        # raise SystemExit
<commit_msg>Test: Schematic.testRotate now fails when importing rotated biomes<commit_after>import itertools
import os
import unittest
from pymclevel import mclevel
from templevel import TempLevel, mktemp
from pymclevel.schematic import MCSchematic
from pymclevel.box import BoundingBox

__author__ = 'Rio'

class TestSchematics(unittest.TestCase):
    def setUp(self):
        # self.alphaLevel = TempLevel("Dojo_64_64_128.dat")
        self.indevLevel = TempLevel("hell.mclevel")
        self.anvilLevel = TempLevel("AnvilWorld")

    def testCreate(self):
        # log.info("Schematic from indev")

        size = (64, 64, 64)
        temp = mktemp("testcreate.schematic")
        schematic = MCSchematic(shape=size, filename=temp, mats='Classic')
        level = self.indevLevel.level

        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (0, 0, 0))
        assert((schematic.Blocks[0:64, 0:64, 0:64] == level.Blocks[0:64, 0:64, 0:64]).all())

        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (-32, -32, -32))
        assert((schematic.Blocks[0:32, 0:32, 0:32] == level.Blocks[32:64, 32:64, 32:64]).all())

        schematic.saveInPlace()

        schem = mclevel.fromFile("schematics/CreativeInABox.schematic")
        tempSchematic = MCSchematic(shape=(1, 1, 3))
        tempSchematic.copyBlocksFrom(schem, BoundingBox((0, 0, 0), (1, 1, 3)), (0, 0, 0))

        level = self.anvilLevel.level
        for cx, cz in itertools.product(xrange(0, 4), xrange(0, 4)):
            try:
                level.createChunk(cx, cz)
            except ValueError:
                pass
        schematic.copyBlocksFrom(level, BoundingBox((0, 0, 0), (64, 64, 64,)), (0, 0, 0))
        schematic.close()
        os.remove(temp)

    def testRotate(self):
        level = self.anvilLevel.level
        schematic = level.extractSchematic(BoundingBox((0, 0, 0), (21, 11, 8)))
        schematic.rotateLeft()

        level.copyBlocksFrom(schematic, schematic.bounds, level.bounds.origin, biomes=True, create=True)

        schematic.flipEastWest()
        level.copyBlocksFrom(schematic, schematic.bounds, level.bounds.origin, biomes=True, create=True)

        schematic.flipVertical()
        level.copyBlocksFrom(schematic, schematic.bounds, level.bounds.origin, biomes=True, create=True)

    def testZipSchematic(self):
        level = self.anvilLevel.level

        x, y, z = level.bounds.origin
        x += level.bounds.size[0]/2 & ~15
        z += level.bounds.size[2]/2 & ~15

        box = BoundingBox((x, y, z), (64, 64, 64,))
        zs = level.extractZipSchematic(box)
        assert(box.chunkCount == zs.chunkCount)
        zs.close()
        os.remove(zs.filename)

    def testINVEditChests(self):
        invFile = mclevel.fromFile("schematics/Chests/TinkerersBox.inv")
        assert invFile.Blocks.any()
        assert not invFile.Data.any()
        assert len(invFile.Entities) == 0
        assert len(invFile.TileEntities) == 1
        # raise SystemExit
<|endoftext|>


<commit_before>import base64
from flask import jsonify
from flask import render_template
from flask import request
from io import BytesIO

from mosaic import app
from .controller.Image2Mosaic import create_mosaic

@app.route('/', methods=['GET', 'POST'])
@app.route('/index', methods=['GET', 'POST'])
def index():
    def allowed_file(filename):
        return '.' in filename and \
               filename.lower().rsplit('.', 1)[1] in app.config['ALLOWED_EXTENSIONS']

    if request.method == 'POST':
        if 'file' not in request.files:
            return jsonify(message='No file part')

        file = request.files['file']
        if file.filename == '':
            return jsonify(message='No selected file')

        if file and allowed_file(file.filename):
            # filename = secure_filename(file.filename)
            image = create_mosaic(file, app.config['COLOR_MAP'])

            buffer = BytesIO()
            image.save(buffer, format='JPEG')
            buffer.seek(0)
            base64_image = base64.b64encode(buffer.getvalue()).decode('utf-8')

            return jsonify(result="data:image/png;base64, %s" % base64_image)

        else:
            return jsonify(message='file extension not allowed')

    else:
        return render_template('index.html',
                               title='Upload')
<commit_msg>some code is moved to utils.py: allowed_file -> utils.allowed_file image convertion -> utils.image_to_string<commit_after>from flask import jsonify
from flask import render_template
from flask import request

from mosaic import app
from .utils.image_2_mosaic import create_mosaic
from .utils.utils import image_to_string, allowed_file

@app.route('/', methods=['GET', 'POST'])
@app.route('/index', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        if 'file' not in request.files:
            return jsonify(message='No file part')
        file = request.files['file']
        if file.filename == '':
            return jsonify(message='No selected file')
        if file and allowed_file(file.filename):
            image = create_mosaic(file, app.config['COLOR_MAP'])
            base64_image = image_to_string(image)
            return jsonify(result="data:image/png;base64, %s" % base64_image)
        else:
            return jsonify(message='file extension not allowed')
    else:
        return render_template('index.html',
                               title='Upload')

<|endoftext|>


<commit_before>import os
import angr
import time
import shutil
import signal
import socket
import logging
import resource
import tempfile
import subprocess
import contextlib
import shellphish_qemu

from .tracerpov import TracerPoV
from .tracer import TracerEnvironmentError, TracerInstallError

l = logging.getLogger("tracer.Runner")

class Runner(object):
    """
    Trace an angr path with a concrete input
    """

    def __init__(self, binary, input=None, pov_file=None, record_trace=False, record_stdout=False, record_magic=False,
                 seed=None, memory_limit="8G"):
        """
        :param binary: path to the binary to be traced
        :param input: concrete input string to feed to binary
        :param pov_file: CGC PoV describing the input to trace
        :param record_trace: whether or not to record the basic block trace
        """

        self.binary = binary
        self.input = input
        self.pov_file = pov_file
        self._record_magic = record_magic
        self._record_trace = record_trace
        self.trace = None
        self.reg_vals = None
        self._state = None
        self.memory = None
        self.seed = seed
        self.memory_limit = self._memory_limit_to_int(memory_limit)

        if self.pov_file is None and self.input is None:
            raise ValueError("must specify input or pov_file")

        if self.pov_file is not None and self.input is not None:
            raise ValueError("cannot specify both a pov_file and an input")

        # a PoV was provided
        if self.pov_file is not None:
            self.pov_file = TracerPoV(self.pov_file)
            self.pov = True
        else:
            self.pov = False

        self.base = os.path.join(os.path.dirname(__file__), "..", "..")

        self.tracer_qemu = None
        self.tracer_qemu_path = None

        self._setup()

        l.debug("accumulating basic block trace...")
        l.debug("self.tracer_qemu_path: %s", self.tracer_qemu_path)

        # does the input cause a crash?
        self.crash_mode = False
        # if the input causes a crash, what address does it crash at?
        self.crash_addr = None

        self.stdout = None
        self.magic = None

        self._set_memory_limit(self.memory_limit)

        if record_stdout:
            tmp = tempfile.mktemp(prefix="stdout_" + os.path.basename(binary))
            # will set crash_mode correctly
            self.dynamic_trace(stdout_file=tmp)
            with open(tmp, "rb") as f:
                self.stdout = f.read()
            os.remove(tmp)
        else:
            # will set crash_mode correctly
            self.dynamic_trace()

### SETUP

    @staticmethod
    def _memory_limit_to_int(ms):

        if not isinstance(ms, str):
            raise ValueError("memory_limit must be a string such as \"8G\"")

        if ms.endswith('k'):
            return int(ms[:-1]) * 1024
        elif ms.endswith('M'):
            return int(ms[:-1]) * 1024 * 1024
        elif ms.endswith('G'):
            return int(ms[:-1]) * 1024 * 1024 * 1024

        raise ValueError("unrecognized size, should be 'k', 'M', or 'G'")

    @staticmethod
    def _set_memory_limit(ml):

        resource.setrlimit(resource.RLIMIT_AS, (ml, ml))

    def _setup(self):
        """
        make sure the environment is sane and we have everything we need to do a trace
        """
        # check the binary
        if not os.access(self.binary, os.X_OK):
            if os.path.isfile(self.binary):
                l.error("\"%s\" binary is not executable", self.binary)
                raise TracerEnvironmentError
            else:
                l.error("\"%s\" binary does not exist", self.binary)
                raise TracerEnvironmentError

        # hack for the OS
        with open(self.binary, "rb") as f:
            header = f.read(4)
            header = header[1:]
        if header == "CGC":
            self.os = "cgc"
        else:
            self.os = None

        if self.os != "cgc":
            l.error("\"%s\" runs on an OS not supported by the runner (only cgc at the moment)", self.binary)
            raise TracerEnvironmentError

        # try to find the install base
        self.base = shellphish_qemu.qemu_base()
        self._check_qemu_install()
        return True

    def _check_qemu_install(self):
        """
        check the install location of qemu
        """

        if self.os == "cgc":
            self.tracer_qemu = "shellphish-qemu-cgc-%s" % ("tracer" if self._record_trace else "base")

        self.tracer_qemu_path = shellphish_qemu.qemu_path(self.tracer_qemu)

        if not os.access(self.tracer_qemu_path, os.X_OK):
            if os.path.isfile(self.tracer_qemu_path):
                l.error("%s is not executable", self.tracer_qemu)
                raise TracerEnvironmentError
            else:
                l.error("\"%s\" does not exist", self.tracer_qemu_path)
                raise TracerEnvironmentError

### DYNAMIC TRACING

    # create a tmp dir in /dev/shm, chdir into it, set rlimit, save the current self.binary
    # at the end, it restores everything
    @contextlib.contextmanager
    def _setup_env(self):
        # TODO switch back to /dev/shm, when this is fixed:
        # https://trello.com/c/zTruwWry/247-dev-shm-size-issues
        # prefix = "/dev/shm/"
        prefix = "/tmp/tracer_"
        curdir = os.getcwd()
        tmpdir = tempfile.mkdtemp(prefix=prefix)
        # allow cores to be dumped
        saved_limit = resource.getrlimit(resource.RLIMIT_CORE)
        resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
        binary_old = self.binary
        binary_replacement_fname = os.path.join(tmpdir,"binary_replacement")
        shutil.copy(self.binary, binary_replacement_fname)
        self.binary = binary_replacement_fname
        os.chdir(tmpdir)
        try:
            yield (tmpdir,binary_replacement_fname)
        finally:
            assert tmpdir.startswith(prefix)
            shutil.rmtree(tmpdir)
            os.chdir(curdir)
            resource.setrlimit(resource.RLIMIT_CORE, saved_limit)
            self.binary = binary_old

    def dynamic_trace(self, stdout_file=None):
        binary_name = os.path.basename(self.binary)

        with self._setup_env() as (tmpdir,binary_replacement_fname):
            # get the dynamic trace
            self._run_trace(stdout_file=stdout_file)

            if self.crash_mode:
                # find core file
                unique_prefix = "qemu_{}".format(os.path.basename(binary_replacement_fname))
                core_files = filter(
                        lambda x: x.startswith(unique_prefix) and x.endswith('.core'),
                        os.listdir('.')
                        )

                a_mesg = "No core files found for binary, this shouldn't happen"
                assert len(core_files) > 0, a_mesg
                a_mesg = "Multiple core files found for binary, this shouldn't happen"
                assert len(core_files) < 2, a_mesg
                core_file = core_files[0]
                a_mesg = "Empty core file generated"
                assert os.path.getsize(core_file) > 0, a_mesg
                self._load_core_values(core_file)

    def _run_trace(self, stdout_file=None):
        """
        accumulate a basic block trace using qemu
        """

        logname = tempfile.mktemp(dir="/dev/shm/", prefix="tracer-log-")
        args = [self.tracer_qemu_path]
        if self.seed is not None:
            args.append("-seed")
            args.append(str(self.seed))
        mname = tempfile.mktemp(dir="/dev/shm/", prefix="tracer-magic-")
        if self._record_magic:
            args += ["-magicdump", mname]

        if self._record_trace:
            args += ["-d", "exec", "-D", logname]
        else:
            args += ["-enable_double_empty_exiting"]

        args += [self.binary]

        with open('/dev/null', 'wb') as devnull:
            stdout_f = devnull
            if stdout_file is not None:
                stdout_f = open(stdout_file, 'wb')

            # we assume qemu with always exit and won't block
            if self.pov_file is None:
                l.info("tracing as raw input")
                p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=stdout_f, stderr=devnull)
                _, _ = p.communicate(self.input)
            else:
                l.info("tracing as pov file")
                in_s, out_s = socket.socketpair()
                p = subprocess.Popen(args, stdin=in_s, stdout=stdout_f, stderr=devnull)

                for write in self.pov_file.writes:
                    out_s.send(write)
                    time.sleep(.01)

            ret = p.wait()
            self.returncode = p.returncode
            # did a crash occur?
            if ret < 0:
                if abs(ret) == signal.SIGSEGV or abs(ret) == signal.SIGILL:
                    l.info("input caused a crash (signal %d) during dynamic tracing", abs(ret))
                    l.info("entering crash mode")
                    self.crash_mode = True

            if stdout_file is not None:
                stdout_f.close()

        if self._record_trace:
            trace = open(logname).read()
            addrs = [int(v.split('[')[1].split(']')[0], 16)
                     for v in trace.split('\n')
                     if v.startswith('Trace')]

            # grab the faulting address
            if self.crash_mode:
                self.crash_addr = int(trace.split('\n')[-2].split('[')[1].split(']')[0], 16)

            os.remove(logname)
            self.trace = addrs
            l.debug("trace consists of %d basic blocks", len(self.trace))

        if self._record_magic:
            magic = open(mname).read()
            self.magic = magic
            os.remove(mname)

    def _load_core_values(self, core_file):
        p = angr.Project(core_file)
        self.reg_vals = {reg:val for (reg, val) in p.loader.main_bin.initial_register_values()}
        self._state = p.factory.entry_state()
        self.memory = self._state.memory
<commit_msg>No memory limit on the runner by default<commit_after>import os
import angr
import time
import shutil
import signal
import socket
import logging
import resource
import tempfile
import subprocess
import contextlib
import shellphish_qemu

from .tracerpov import TracerPoV
from .tracer import TracerEnvironmentError, TracerInstallError

l = logging.getLogger("tracer.Runner")

class Runner(object):
    """
    Trace an angr path with a concrete input
    """

    def __init__(self, binary, input=None, pov_file=None, record_trace=False, record_stdout=False, record_magic=False,
                 seed=None, memory_limit=None):
        """
        :param binary: path to the binary to be traced
        :param input: concrete input string to feed to binary
        :param pov_file: CGC PoV describing the input to trace
        :param record_trace: whether or not to record the basic block trace
        """

        self.binary = binary
        self.input = input
        self.pov_file = pov_file
        self._record_magic = record_magic
        self._record_trace = record_trace
        self.trace = None
        self.reg_vals = None
        self._state = None
        self.memory = None
        self.seed = seed
        self.memory_limit = self._memory_limit_to_int(memory_limit) if memory_limit is not None else None

        if self.pov_file is None and self.input is None:
            raise ValueError("must specify input or pov_file")

        if self.pov_file is not None and self.input is not None:
            raise ValueError("cannot specify both a pov_file and an input")

        # a PoV was provided
        if self.pov_file is not None:
            self.pov_file = TracerPoV(self.pov_file)
            self.pov = True
        else:
            self.pov = False

        self.base = os.path.join(os.path.dirname(__file__), "..", "..")

        self.tracer_qemu = None
        self.tracer_qemu_path = None

        self._setup()

        l.debug("accumulating basic block trace...")
        l.debug("self.tracer_qemu_path: %s", self.tracer_qemu_path)

        # does the input cause a crash?
        self.crash_mode = False
        # if the input causes a crash, what address does it crash at?
        self.crash_addr = None

        self.stdout = None
        self.magic = None

        if self.memory_limit:
            self._set_memory_limit(self.memory_limit)

        if record_stdout:
            tmp = tempfile.mktemp(prefix="stdout_" + os.path.basename(binary))
            # will set crash_mode correctly
            self.dynamic_trace(stdout_file=tmp)
            with open(tmp, "rb") as f:
                self.stdout = f.read()
            os.remove(tmp)
        else:
            # will set crash_mode correctly
            self.dynamic_trace()

### SETUP

    @staticmethod
    def _memory_limit_to_int(ms):

        if not isinstance(ms, str):
            raise ValueError("memory_limit must be a string such as \"8G\"")

        if ms.endswith('k'):
            return int(ms[:-1]) * 1024
        elif ms.endswith('M'):
            return int(ms[:-1]) * 1024 * 1024
        elif ms.endswith('G'):
            return int(ms[:-1]) * 1024 * 1024 * 1024

        raise ValueError("unrecognized size, should be 'k', 'M', or 'G'")

    @staticmethod
    def _set_memory_limit(ml):

        resource.setrlimit(resource.RLIMIT_AS, (ml, ml))

    def _setup(self):
        """
        make sure the environment is sane and we have everything we need to do a trace
        """
        # check the binary
        if not os.access(self.binary, os.X_OK):
            if os.path.isfile(self.binary):
                l.error("\"%s\" binary is not executable", self.binary)
                raise TracerEnvironmentError
            else:
                l.error("\"%s\" binary does not exist", self.binary)
                raise TracerEnvironmentError

        # hack for the OS
        with open(self.binary, "rb") as f:
            header = f.read(4)
            header = header[1:]
        if header == "CGC":
            self.os = "cgc"
        else:
            self.os = None

        if self.os != "cgc":
            l.error("\"%s\" runs on an OS not supported by the runner (only cgc at the moment)", self.binary)
            raise TracerEnvironmentError

        # try to find the install base
        self.base = shellphish_qemu.qemu_base()
        self._check_qemu_install()
        return True

    def _check_qemu_install(self):
        """
        check the install location of qemu
        """

        if self.os == "cgc":
            self.tracer_qemu = "shellphish-qemu-cgc-%s" % ("tracer" if self._record_trace else "base")

        self.tracer_qemu_path = shellphish_qemu.qemu_path(self.tracer_qemu)

        if not os.access(self.tracer_qemu_path, os.X_OK):
            if os.path.isfile(self.tracer_qemu_path):
                l.error("%s is not executable", self.tracer_qemu)
                raise TracerEnvironmentError
            else:
                l.error("\"%s\" does not exist", self.tracer_qemu_path)
                raise TracerEnvironmentError

### DYNAMIC TRACING

    # create a tmp dir in /dev/shm, chdir into it, set rlimit, save the current self.binary
    # at the end, it restores everything
    @contextlib.contextmanager
    def _setup_env(self):
        # TODO switch back to /dev/shm, when this is fixed:
        # https://trello.com/c/zTruwWry/247-dev-shm-size-issues
        # prefix = "/dev/shm/"
        prefix = "/tmp/tracer_"
        curdir = os.getcwd()
        tmpdir = tempfile.mkdtemp(prefix=prefix)
        # allow cores to be dumped
        saved_limit = resource.getrlimit(resource.RLIMIT_CORE)
        resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
        binary_old = self.binary
        binary_replacement_fname = os.path.join(tmpdir,"binary_replacement")
        shutil.copy(self.binary, binary_replacement_fname)
        self.binary = binary_replacement_fname
        os.chdir(tmpdir)
        try:
            yield (tmpdir,binary_replacement_fname)
        finally:
            assert tmpdir.startswith(prefix)
            shutil.rmtree(tmpdir)
            os.chdir(curdir)
            resource.setrlimit(resource.RLIMIT_CORE, saved_limit)
            self.binary = binary_old

    def dynamic_trace(self, stdout_file=None):
        binary_name = os.path.basename(self.binary)

        with self._setup_env() as (tmpdir,binary_replacement_fname):
            # get the dynamic trace
            self._run_trace(stdout_file=stdout_file)

            if self.crash_mode:
                # find core file
                unique_prefix = "qemu_{}".format(os.path.basename(binary_replacement_fname))
                core_files = filter(
                        lambda x: x.startswith(unique_prefix) and x.endswith('.core'),
                        os.listdir('.')
                        )

                a_mesg = "No core files found for binary, this shouldn't happen"
                assert len(core_files) > 0, a_mesg
                a_mesg = "Multiple core files found for binary, this shouldn't happen"
                assert len(core_files) < 2, a_mesg
                core_file = core_files[0]
                a_mesg = "Empty core file generated"
                assert os.path.getsize(core_file) > 0, a_mesg
                self._load_core_values(core_file)

    def _run_trace(self, stdout_file=None):
        """
        accumulate a basic block trace using qemu
        """

        logname = tempfile.mktemp(dir="/dev/shm/", prefix="tracer-log-")
        args = [self.tracer_qemu_path]
        if self.seed is not None:
            args.append("-seed")
            args.append(str(self.seed))
        mname = tempfile.mktemp(dir="/dev/shm/", prefix="tracer-magic-")
        if self._record_magic:
            args += ["-magicdump", mname]

        if self._record_trace:
            args += ["-d", "exec", "-D", logname]
        else:
            args += ["-enable_double_empty_exiting"]

        args += [self.binary]

        with open('/dev/null', 'wb') as devnull:
            stdout_f = devnull
            if stdout_file is not None:
                stdout_f = open(stdout_file, 'wb')

            # we assume qemu with always exit and won't block
            if self.pov_file is None:
                l.info("tracing as raw input")
                p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=stdout_f, stderr=devnull)
                _, _ = p.communicate(self.input)
            else:
                l.info("tracing as pov file")
                in_s, out_s = socket.socketpair()
                p = subprocess.Popen(args, stdin=in_s, stdout=stdout_f, stderr=devnull)

                for write in self.pov_file.writes:
                    out_s.send(write)
                    time.sleep(.01)

            ret = p.wait()
            self.returncode = p.returncode
            # did a crash occur?
            if ret < 0:
                if abs(ret) == signal.SIGSEGV or abs(ret) == signal.SIGILL:
                    l.info("input caused a crash (signal %d) during dynamic tracing", abs(ret))
                    l.info("entering crash mode")
                    self.crash_mode = True

            if stdout_file is not None:
                stdout_f.close()

        if self._record_trace:
            trace = open(logname).read()
            addrs = [int(v.split('[')[1].split(']')[0], 16)
                     for v in trace.split('\n')
                     if v.startswith('Trace')]

            # grab the faulting address
            if self.crash_mode:
                self.crash_addr = int(trace.split('\n')[-2].split('[')[1].split(']')[0], 16)

            os.remove(logname)
            self.trace = addrs
            l.debug("trace consists of %d basic blocks", len(self.trace))

        if self._record_magic:
            magic = open(mname).read()
            self.magic = magic
            os.remove(mname)

    def _load_core_values(self, core_file):
        p = angr.Project(core_file)
        self.reg_vals = {reg:val for (reg, val) in p.loader.main_bin.initial_register_values()}
        self._state = p.factory.entry_state()
        self.memory = self._state.memory
<|endoftext|>


<commit_before># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SimpleSwitch(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, in_port, dst, instructions):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        match = datapath.ofproto_parser.OFPMatch()
        match.set_in_port(in_port)
        match.set_dl_dst(dst)

        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, match=match, cookie=0,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000,
            flags=ofproto.OFPFF_SEND_FLOW_REM, instructions=instructions)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields because WHY IS THIS SO HARD
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # learn the mac address to avoid FLOOD next time.
        self.mac_to_port[dpid][src] = in_port

        # update on our list of mac addresses
        print self.mac_to_port[dpid]

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
        instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]

        # install a flow to avoid packet_in next time
        if out_port != ofproto_v1_2.OFPP_FLOOD:
            self.add_flow(datapath, in_port, dst, actions)

        out = datapath.ofproto_parser.OFPPacketOut(
            datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
            instructions=instructions)
        datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<commit_msg>packet_out gets actions, flow gets instructions<commit_after># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SimpleSwitch(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, in_port, dst, instructions):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        match = datapath.ofproto_parser.OFPMatch()
        match.set_in_port(in_port)
        match.set_dl_dst(dst)

        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, match=match, cookie=0,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000,
            flags=ofproto.OFPFF_SEND_FLOW_REM, instructions=instructions)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields because WHY IS THIS SO HARD
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # learn the mac address to avoid FLOOD next time.
        self.mac_to_port[dpid][src] = in_port

        # update on our list of mac addresses
        print self.mac_to_port[dpid]

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]
        instructions = [datapath.ofproto_parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)]

        # install a flow to avoid packet_in next time
        if out_port != ofproto_v1_2.OFPP_FLOOD:
            self.add_flow(datapath, in_port, dst, instructions)

        out = datapath.ofproto_parser.OFPPacketOut(
            datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
            actions=actions)
        datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os.path

import github3
import unidiff

from inlineplz.interfaces.base import InterfaceBase
from inlineplz.util import git

class GitHubInterface(InterfaceBase):
    def __init__(self, owner, repo, pr, token, url=None):
        # TODO: support non-PR runs
        if pr is None:
            return
        if not url:
            self.gh = github3.GitHub(token=token)
        else:
            self.gh = github3.GitHubEnterprise(url, token=token)
        self.pull_request = self.gh.pull_request(owner, repo, pr)
        # github3 has naming/compatibility issues
        try:
            self.commits = [c for c in self.pull_request.commits()]
        except (AttributeError, TypeError):
            self.commits = [c for c in self.pull_request.iter_commits()]
        self.last_sha = self.commits[-1].sha
        self.first_sha = self.commits[0].sha
        self.parent_sha = git.parent_sha(self.first_sha)
        self.diff = git.diff(self.parent_sha, self.last_sha)

    def post_messages(self, messages):
        messages_to_post = 0
        for msg in messages:
            if not msg.comments:
                continue
            msg_position = self.position(msg)
            if msg_position:
                messages_to_post += 1
                if not self.is_duplicate(msg, msg_position):
                    self.pull_request.create_review_comment(
                        self.format_message(msg),
                        self.last_sha,
                        msg.path,
                        msg_position
                    )
        return messages_to_post

    def is_duplicate(self, message, position):
        for comment in self.pull_request.review_comments():
            if (comment.position == position and
                    comment.path == message.path and
                    comment.body.strip() == self.format_message(message).strip()):
                return True
        return False

    @staticmethod
    def format_message(message):
        if not message.comments:
            return ''
        if len(message.comments) > 1:
            return (
                '```\n' +
                '\n'.join(sorted(list(message.comments))) +
                '\n```'
            )
        return '`{0}`'.format(list(message.comments)[0].strip())

    def position(self, message):
        """Calculate position within the PR, which is not the line number"""
        patch = unidiff.PatchSet(self.diff.split('\n'))
        for patched_file in patch:
            target = patched_file.target_file.lstrip('b/')
            if target == message.path:
                offset = 1
                for hunk_no, hunk in enumerate(patched_file):
                    for position, hunk_line in enumerate(hunk):
                        if '+' not in hunk_line.line_type:
                            continue
                        if hunk_line.target_line_no == message.line_number:
                            return position + offset
                    offset += len(hunk) + 1
<commit_msg>just check pr truthiness<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

import os.path

import github3
import unidiff

from inlineplz.interfaces.base import InterfaceBase
from inlineplz.util import git

class GitHubInterface(InterfaceBase):
    def __init__(self, owner, repo, pr, token, url=None):
        # TODO: support non-PR runs
        if not pr:
            return
        if not url:
            self.gh = github3.GitHub(token=token)
        else:
            self.gh = github3.GitHubEnterprise(url, token=token)
        self.pull_request = self.gh.pull_request(owner, repo, pr)
        # github3 has naming/compatibility issues
        try:
            self.commits = [c for c in self.pull_request.commits()]
        except (AttributeError, TypeError):
            self.commits = [c for c in self.pull_request.iter_commits()]
        self.last_sha = self.commits[-1].sha
        self.first_sha = self.commits[0].sha
        self.parent_sha = git.parent_sha(self.first_sha)
        self.diff = git.diff(self.parent_sha, self.last_sha)

    def post_messages(self, messages):
        messages_to_post = 0
        for msg in messages:
            if not msg.comments:
                continue
            msg_position = self.position(msg)
            if msg_position:
                messages_to_post += 1
                if not self.is_duplicate(msg, msg_position):
                    self.pull_request.create_review_comment(
                        self.format_message(msg),
                        self.last_sha,
                        msg.path,
                        msg_position
                    )
        return messages_to_post

    def is_duplicate(self, message, position):
        for comment in self.pull_request.review_comments():
            if (comment.position == position and
                    comment.path == message.path and
                    comment.body.strip() == self.format_message(message).strip()):
                return True
        return False

    @staticmethod
    def format_message(message):
        if not message.comments:
            return ''
        if len(message.comments) > 1:
            return (
                '```\n' +
                '\n'.join(sorted(list(message.comments))) +
                '\n```'
            )
        return '`{0}`'.format(list(message.comments)[0].strip())

    def position(self, message):
        """Calculate position within the PR, which is not the line number"""
        patch = unidiff.PatchSet(self.diff.split('\n'))
        for patched_file in patch:
            target = patched_file.target_file.lstrip('b/')
            if target == message.path:
                offset = 1
                for hunk_no, hunk in enumerate(patched_file):
                    for position, hunk_line in enumerate(hunk):
                        if '+' not in hunk_line.line_type:
                            continue
                        if hunk_line.target_line_no == message.line_number:
                            return position + offset
                    offset += len(hunk) + 1
<|endoftext|>