<commit_before># -*- coding: utf-8 -*-
import logging
import secrets

import webapp2
from webapp2_extras import auth, sessions, jinja2
from jinja2.runtime import TemplateNotFound

from simpleauth import SimpleAuthHandler

import tweepy
import random
from tweepy.error import TweepError

# Consumer keys and access tokens, used for OAuth
consumer_key = secrets.consumer_key
consumer_secret = secrets.consumer_secret

class BaseRequestHandler(webapp2.RequestHandler):
  def dispatch(self):
    # Get a session store for this request.
    self.session_store = sessions.get_store(request=self.request)

    try:
      # Dispatch the request.
      webapp2.RequestHandler.dispatch(self)
    finally:
      # Save all sessions.
      self.session_store.save_sessions(self.response)

  @webapp2.cached_property
  def jinja2(self):
    """Returns a Jinja2 renderer cached in the app registry"""
    return jinja2.get_jinja2(app=self.app)

  @webapp2.cached_property
  def session(self):
    """Returns a session using the default cookie key"""
    return self.session_store.get_session()

  @webapp2.cached_property
  def auth(self):
      return auth.get_auth()

  @webapp2.cached_property
  def current_user(self):
    """Returns currently logged in user"""
    user_dict = self.auth.get_user_by_session()
    return self.auth.store.user_model.get_by_id(user_dict['user_id'])

  @webapp2.cached_property
  def logged_in(self):
    """Returns true if a user is currently logged in, false otherwise"""
    return self.auth.get_user_by_session() is not None

  def render(self, template_name, template_vars={}):
    # Preset values for the template
    values = {
      'url_for': self.uri_for,
      'logged_in': self.logged_in,
      'flashes': self.session.get_flashes()
    }

    # Add manually supplied template values
    values.update(template_vars)

    # read the template or 404.html
    try:
      self.response.write(self.jinja2.render_template(template_name, **values))
    except TemplateNotFound:
      self.abort(404)

  def head(self, *args):
    """Head is used by Twitter. If not there the tweet button shows 0"""
    pass

class RootHandler(BaseRequestHandler):
  def get(self):
    """Handles default landing page"""
    if self.logged_in:
        self.render('home.html', {
        'user': self.current_user,
        'session': self.auth.get_user_by_session()
      })
    else:
        self.render('home.html')

class ProfileHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /profile"""
    if self.logged_in:
      self.render('profile.html', {
        'user': self.current_user,
        'session': self.auth.get_user_by_session()
      })
    else:
      self.redirect('/')

class FollowersHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /followers"""
    if self.logged_in:
      user = self.current_user

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      followers = list()
      incomplete_list = False
      try:
          # the maximum per request is 200 according to
          # https://dev.twitter.com/rest/reference/get/followers/list
          for follower in tweepy.Cursor(api.followers,count=200).items():
              followers.append(follower)
      except TweepError as e:
          logging.error(e)
          limits = api.rate_limit_status('statuses')
          logging.info(limits)
          incomplete_list = True

      winner = False
      if len(followers) > 0:
        random_number = random.randint(0, len(followers)-1)
        winner = followers[random_number]

      self.render('followers.html', {
        'user': user,
        'session': self.auth.get_user_by_session(),
        'followers': followers,
        'winner': winner,
        'incomplete_list': incomplete_list,
      })
    else:
      self.redirect('/')

class RetweetsHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /retweets"""
    if self.logged_in:
      user = self.current_user
      tweet_id = self.request.get("id")

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      if not tweet_id:
          retweets_list = list()
          incomplete_list = False
          try:
            for tweet in api.retweets_of_me(count=100):
                retweets_list.append(tweet)
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('retweets.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'retweets': retweets_list,
            'incomplete_list': incomplete_list,
          })
      else:
          retweet = False
          winner = False
          incomplete_list = False
          try:
            retweet = api.get_status(tweet_id)
            retweeters = api.retweeters(tweet_id)
            if len(retweeters) > 0:
              random_number = random.randint(0, len(retweeters)-1)
              winner = api.get_user(retweeters[random_number])
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('retweets.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'winner': winner,
            'retweet': retweet,
            'incomplete_list': incomplete_list,
          })
    else:
      self.redirect('/')

class SearchesHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /searches"""
    if self.logged_in:
      user = self.current_user
      search_id = self.request.get("id")

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      if not search_id:
          searches_list = list()
          incomplete_list = False
          try:
            for search in api.saved_searches():
                searches_list.append(search)
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('searches.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'searches': searches_list,
            'incomplete_list': incomplete_list,
          })
      else:
          search = api.get_saved_search(search_id)
          print search.query
          statuses = list()
          tweeters = list()
          # max 100 https://dev.twitter.com/rest/reference/get/search/tweets
          for status in api.search(q=search.query, count=100):
            statuses.append(status)
            tweeters.append(status.user)
          random_number = random.randint(0, len(tweeters)-1)
          winner = tweeters[random_number]

          self.render('searches.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'winner': winner,
            'tweet': statuses[random_number],
          })
    else:
      self.redirect('/')

class AuthHandler(BaseRequestHandler, SimpleAuthHandler):
  """Authentication handler for OAuth 2.0, 1.0(a) and OpenID."""

  # Enable optional OAuth 2.0 CSRF guard
  OAUTH2_CSRF_STATE = True

  USER_ATTRS = {
    'facebook' : {
      'id'     : lambda id: ('avatar_url',
        'http://graph.facebook.com/{0}/picture?type=large'.format(id)),
      'name'   : 'name',
      'link'   : 'link'
    },
    'google'   : {
      'picture': 'avatar_url',
      'name'   : 'name',
      'profile': 'link'
    },
    'windows_live': {
      'avatar_url': 'avatar_url',
      'name'      : 'name',
      'link'      : 'link'
    },
    'twitter'  : {
      'profile_image_url': 'avatar_url',
      'screen_name'      : 'name',
      'link'             : 'link'
    },
    'linkedin' : {
      'picture-url'       : 'avatar_url',
      'first-name'        : 'name',
      'public-profile-url': 'link'
    },
    'linkedin2' : {
      'picture-url'       : 'avatar_url',
      'first-name'        : 'name',
      'public-profile-url': 'link'
    },
    'foursquare'   : {
      'photo'    : lambda photo: ('avatar_url', photo.get('prefix') + '100x100' + photo.get('suffix')),
      'firstName': 'firstName',
      'lastName' : 'lastName',
      'contact'  : lambda contact: ('email',contact.get('email')),
      'id'       : lambda id: ('link', 'http://foursquare.com/user/{0}'.format(id))
    },
    'openid'   : {
      'id'      : lambda id: ('avatar_url', '/img/missing-avatar.png'),
      'nickname': 'name',
      'email'   : 'link'
    }
  }

  def _on_signin(self, data, auth_info, provider):
    """Callback whenever a new or existing user is logging in.
     data is a user info dictionary.
     auth_info contains access token or oauth token and secret.
    """
    auth_id = '%s:%s' % (provider, data['id'])
    logging.info('Looking for a user with id %s', auth_id)

    user = self.auth.store.user_model.get_by_auth_id(auth_id)
    _attrs = self._to_user_model_attrs(data, self.USER_ATTRS[provider])

    if user:
      logging.info('Found existing user to log in')
      # Existing users might've changed their profile data so we update our
      # local model anyway. This might result in quite inefficient usage
      # of the Datastore, but we do this anyway for demo purposes.
      #
      # In a real app you could compare _attrs with user's properties fetched
      # from the datastore and update local user in case something's changed.
      user.populate(**_attrs)
      user.access_token = auth_info['oauth_token']
      user.access_token_secret = auth_info['oauth_token_secret']
      user.put()
      self.auth.set_session(
        self.auth.store.user_to_dict(user))

    else:
      # check whether there's a user currently logged in
      # then, create a new user if nobody's signed in,
      # otherwise add this auth_id to currently logged in user.

      if self.logged_in:
        logging.info('Updating currently logged in user')

        u = self.current_user
        u.populate(**_attrs)
        user.access_token = auth_info['oauth_token']
        user.access_token_secret = auth_info['oauth_token_secret']
        # The following will also do u.put(). Though, in a real app
        # you might want to check the result, which is
        # (boolean, info) tuple where boolean == True indicates success
        # See webapp2_extras.appengine.auth.models.User for details.
        u.add_auth_id(auth_id)

      else:
        logging.info('Creating a brand new user')
        ok, user = self.auth.store.user_model.create_user(auth_id, **_attrs)
        if ok:
          user.access_token = auth_info['oauth_token']
          user.access_token_secret = auth_info['oauth_token_secret']
          user.put()
          self.auth.set_session(self.auth.store.user_to_dict(user))

    # Remember auth data during redirect, just for this demo. You wouldn't
    # normally do this.
    #self.session.add_flash(data, 'data - from _on_signin(...)')
    #self.session.add_flash(auth_info, 'auth_info - from _on_signin(...)')

    self.redirect('/')

  def logout(self):
    self.auth.unset_session()
    self.redirect('/')

  def handle_exception(self, exception, debug):
    logging.error(exception)
    self.render('error.html', {'exception': exception})

  def _callback_uri_for(self, provider):
    return self.uri_for('auth_callback', provider=provider, _full=True)

  def _get_consumer_info_for(self, provider):
    """Returns a tuple (key, secret) for auth init requests."""
    return secrets.AUTH_CONFIG[provider]

  def _to_user_model_attrs(self, data, attrs_map):
    """Get the needed information from the provider dataset."""
    user_attrs = {}
    for k, v in attrs_map.iteritems():
      attr = (v, data.get(k)) if isinstance(v, str) else v(data.get(k))
      user_attrs.setdefault(*attr)

    return user_attrs
<commit_msg>searches: handle twitter rate limit<commit_after># -*- coding: utf-8 -*-
import logging
import secrets

import webapp2
from webapp2_extras import auth, sessions, jinja2
from jinja2.runtime import TemplateNotFound

from simpleauth import SimpleAuthHandler

import tweepy
import random
from tweepy.error import TweepError

# Consumer keys and access tokens, used for OAuth
consumer_key = secrets.consumer_key
consumer_secret = secrets.consumer_secret

class BaseRequestHandler(webapp2.RequestHandler):
  def dispatch(self):
    # Get a session store for this request.
    self.session_store = sessions.get_store(request=self.request)

    try:
      # Dispatch the request.
      webapp2.RequestHandler.dispatch(self)
    finally:
      # Save all sessions.
      self.session_store.save_sessions(self.response)

  @webapp2.cached_property
  def jinja2(self):
    """Returns a Jinja2 renderer cached in the app registry"""
    return jinja2.get_jinja2(app=self.app)

  @webapp2.cached_property
  def session(self):
    """Returns a session using the default cookie key"""
    return self.session_store.get_session()

  @webapp2.cached_property
  def auth(self):
      return auth.get_auth()

  @webapp2.cached_property
  def current_user(self):
    """Returns currently logged in user"""
    user_dict = self.auth.get_user_by_session()
    return self.auth.store.user_model.get_by_id(user_dict['user_id'])

  @webapp2.cached_property
  def logged_in(self):
    """Returns true if a user is currently logged in, false otherwise"""
    return self.auth.get_user_by_session() is not None

  def render(self, template_name, template_vars={}):
    # Preset values for the template
    values = {
      'url_for': self.uri_for,
      'logged_in': self.logged_in,
      'flashes': self.session.get_flashes()
    }

    # Add manually supplied template values
    values.update(template_vars)

    # read the template or 404.html
    try:
      self.response.write(self.jinja2.render_template(template_name, **values))
    except TemplateNotFound:
      self.abort(404)

  def head(self, *args):
    """Head is used by Twitter. If not there the tweet button shows 0"""
    pass

class RootHandler(BaseRequestHandler):
  def get(self):
    """Handles default landing page"""
    if self.logged_in:
        self.render('home.html', {
        'user': self.current_user,
        'session': self.auth.get_user_by_session()
      })
    else:
        self.render('home.html')

class ProfileHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /profile"""
    if self.logged_in:
      self.render('profile.html', {
        'user': self.current_user,
        'session': self.auth.get_user_by_session()
      })
    else:
      self.redirect('/')

class FollowersHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /followers"""
    if self.logged_in:
      user = self.current_user

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      followers = list()
      incomplete_list = False
      try:
          # the maximum per request is 200 according to
          # https://dev.twitter.com/rest/reference/get/followers/list
          for follower in tweepy.Cursor(api.followers,count=200).items():
              followers.append(follower)
      except TweepError as e:
          logging.error(e)
          limits = api.rate_limit_status('statuses')
          logging.info(limits)
          incomplete_list = True

      winner = False
      if len(followers) > 0:
        random_number = random.randint(0, len(followers)-1)
        winner = followers[random_number]

      self.render('followers.html', {
        'user': user,
        'session': self.auth.get_user_by_session(),
        'followers': followers,
        'winner': winner,
        'incomplete_list': incomplete_list,
      })
    else:
      self.redirect('/')

class RetweetsHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /retweets"""
    if self.logged_in:
      user = self.current_user
      tweet_id = self.request.get("id")

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      if not tweet_id:
          retweets_list = list()
          incomplete_list = False
          try:
            for tweet in api.retweets_of_me(count=100):
                retweets_list.append(tweet)
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('retweets.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'retweets': retweets_list,
            'incomplete_list': incomplete_list,
          })
      else:
          retweet = False
          winner = False
          incomplete_list = False
          try:
            retweet = api.get_status(tweet_id)
            retweeters = api.retweeters(tweet_id)
            if len(retweeters) > 0:
              random_number = random.randint(0, len(retweeters)-1)
              winner = api.get_user(retweeters[random_number])
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('retweets.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'winner': winner,
            'retweet': retweet,
            'incomplete_list': incomplete_list,
          })
    else:
      self.redirect('/')

class SearchesHandler(BaseRequestHandler):
  def get(self):
    """Handles GET /searches"""
    if self.logged_in:
      user = self.current_user
      search_id = self.request.get("id")

      # OAuth process, using the keys and tokens
      auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
      auth.set_access_token(user.access_token, user.access_token_secret)

      # Creation of the actual interface, using authentication
      api = tweepy.API(auth)

      if not search_id:
          searches_list = list()
          incomplete_list = False
          try:
            for search in api.saved_searches():
                searches_list.append(search)
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('searches.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'searches': searches_list,
            'incomplete_list': incomplete_list,
          })
      else:
          winner = False
          tweet = False
          incomplete_list = False
          try:
            search = api.get_saved_search(search_id)
            statuses = list()
            tweeters = list()
            # max 100 https://dev.twitter.com/rest/reference/get/search/tweets
            for status in api.search(q=search.query, count=100):
              statuses.append(status)
              tweeters.append(status.user)
            if len(tweeters) > 0:
              random_number = random.randint(0, len(tweeters)-1)
              winner = tweeters[random_number]
              tweet = statuses[random_number]
          except TweepError as e:
            logging.error(e)
            limits = api.rate_limit_status('statuses')
            logging.info(limits)
            incomplete_list = True

          self.render('searches.html', {
            'user': user,
            'session': self.auth.get_user_by_session(),
            'winner': winner,
            'tweet': tweet,
            'incomplete_list': incomplete_list,
          })
    else:
      self.redirect('/')

class AuthHandler(BaseRequestHandler, SimpleAuthHandler):
  """Authentication handler for OAuth 2.0, 1.0(a) and OpenID."""

  # Enable optional OAuth 2.0 CSRF guard
  OAUTH2_CSRF_STATE = True

  USER_ATTRS = {
    'facebook' : {
      'id'     : lambda id: ('avatar_url',
        'http://graph.facebook.com/{0}/picture?type=large'.format(id)),
      'name'   : 'name',
      'link'   : 'link'
    },
    'google'   : {
      'picture': 'avatar_url',
      'name'   : 'name',
      'profile': 'link'
    },
    'windows_live': {
      'avatar_url': 'avatar_url',
      'name'      : 'name',
      'link'      : 'link'
    },
    'twitter'  : {
      'profile_image_url': 'avatar_url',
      'screen_name'      : 'name',
      'link'             : 'link'
    },
    'linkedin' : {
      'picture-url'       : 'avatar_url',
      'first-name'        : 'name',
      'public-profile-url': 'link'
    },
    'linkedin2' : {
      'picture-url'       : 'avatar_url',
      'first-name'        : 'name',
      'public-profile-url': 'link'
    },
    'foursquare'   : {
      'photo'    : lambda photo: ('avatar_url', photo.get('prefix') + '100x100' + photo.get('suffix')),
      'firstName': 'firstName',
      'lastName' : 'lastName',
      'contact'  : lambda contact: ('email',contact.get('email')),
      'id'       : lambda id: ('link', 'http://foursquare.com/user/{0}'.format(id))
    },
    'openid'   : {
      'id'      : lambda id: ('avatar_url', '/img/missing-avatar.png'),
      'nickname': 'name',
      'email'   : 'link'
    }
  }

  def _on_signin(self, data, auth_info, provider):
    """Callback whenever a new or existing user is logging in.
     data is a user info dictionary.
     auth_info contains access token or oauth token and secret.
    """
    auth_id = '%s:%s' % (provider, data['id'])
    logging.info('Looking for a user with id %s', auth_id)

    user = self.auth.store.user_model.get_by_auth_id(auth_id)
    _attrs = self._to_user_model_attrs(data, self.USER_ATTRS[provider])

    if user:
      logging.info('Found existing user to log in')
      # Existing users might've changed their profile data so we update our
      # local model anyway. This might result in quite inefficient usage
      # of the Datastore, but we do this anyway for demo purposes.
      #
      # In a real app you could compare _attrs with user's properties fetched
      # from the datastore and update local user in case something's changed.
      user.populate(**_attrs)
      user.access_token = auth_info['oauth_token']
      user.access_token_secret = auth_info['oauth_token_secret']
      user.put()
      self.auth.set_session(
        self.auth.store.user_to_dict(user))

    else:
      # check whether there's a user currently logged in
      # then, create a new user if nobody's signed in,
      # otherwise add this auth_id to currently logged in user.

      if self.logged_in:
        logging.info('Updating currently logged in user')

        u = self.current_user
        u.populate(**_attrs)
        user.access_token = auth_info['oauth_token']
        user.access_token_secret = auth_info['oauth_token_secret']
        # The following will also do u.put(). Though, in a real app
        # you might want to check the result, which is
        # (boolean, info) tuple where boolean == True indicates success
        # See webapp2_extras.appengine.auth.models.User for details.
        u.add_auth_id(auth_id)

      else:
        logging.info('Creating a brand new user')
        ok, user = self.auth.store.user_model.create_user(auth_id, **_attrs)
        if ok:
          user.access_token = auth_info['oauth_token']
          user.access_token_secret = auth_info['oauth_token_secret']
          user.put()
          self.auth.set_session(self.auth.store.user_to_dict(user))

    # Remember auth data during redirect, just for this demo. You wouldn't
    # normally do this.
    #self.session.add_flash(data, 'data - from _on_signin(...)')
    #self.session.add_flash(auth_info, 'auth_info - from _on_signin(...)')

    self.redirect('/')

  def logout(self):
    self.auth.unset_session()
    self.redirect('/')

  def handle_exception(self, exception, debug):
    logging.error(exception)
    self.render('error.html', {'exception': exception})

  def _callback_uri_for(self, provider):
    return self.uri_for('auth_callback', provider=provider, _full=True)

  def _get_consumer_info_for(self, provider):
    """Returns a tuple (key, secret) for auth init requests."""
    return secrets.AUTH_CONFIG[provider]

  def _to_user_model_attrs(self, data, attrs_map):
    """Get the needed information from the provider dataset."""
    user_attrs = {}
    for k, v in attrs_map.iteritems():
      attr = (v, data.get(k)) if isinstance(v, str) else v(data.get(k))
      user_attrs.setdefault(*attr)

    return user_attrs
<|endoftext|>


<commit_before>import datetime
import logging
import re

from furl import furl
from lxml import etree
from bs4 import BeautifulSoup

from share import Harvester

logger = logging.getLogger(__name__)

class BiorxivHarvester(Harvester):
    url = 'http://biorxiv.org/search/'

    def do_harvest(self, start_date, end_date):

        end_date = end_date.date()
        start_date = start_date.date()

        # I wish I was wrong, the url is just "<key1>:<value> <key2>:<value>" and so on
        url = furl(self.url).add(
            path=' '.join([
                'limit_to:{}'.format(end_date),
                'limit_from:{}'.format(start_date),
                'numresults:100',
                'format_result:standard'
                'sort:publication-date'
                'direction:descending'
            ])
        ).url

        # Fetch records is a separate function for readability
        # Ends up returning a list of tuples with provider given id and the document itself
        return self.fetch_records(url, start_date, end_date)

    def fetch_records(self, url, start_date, end_date):
        page = 0
        resp = self.requests.get(url)

        while True:
            page += 1
            resp = self.requests.get(furl(url).set(query_params={'page': page}))
            links = re.findall(b'href="(/content/early/[^"]+?/[^"]+)"', resp.content)

            if not links:
                break

            for link in links:
                article = self.requests.get('http://biorxiv.org' + link.decode())

                yield (link.decode(), {
                    meta.attrs['name']: meta.attrs['content']
                    for meta in BeautifulSoup(article.content, 'html.parser').find_all('meta')
                    if 'name' in meta.attrs
                })
<commit_msg>Fix date ranges<commit_after>import logging
import re

import arrow
from furl import furl
from bs4 import BeautifulSoup

from share import Harvester

logger = logging.getLogger(__name__)

class BiorxivHarvester(Harvester):
    url = 'http://biorxiv.org/search/'

    def do_harvest(self, start_date, end_date):

        end_date = end_date.date()
        start_date = start_date.date()

        # I wish I was wrong, the url is just "<key1>:<value> <key2>:<value>" and so on
        url = furl(self.url).add(
            path=' '.join([
                'limit_from:{}'.format(start_date),
                'limit_to:{}'.format(end_date),
                'numresults:100',
                'sort:publication-date',
                'direction:descending',
                'format_result:standard',
            ])
        ).url

        # Fetch records is a separate function for readability
        # Ends up returning a list of tuples with provider given id and the document itself
        return self.fetch_records(url, start_date, end_date)

    def fetch_records(self, url, start_date, end_date):
        page = -1

        while True:
            page += 1
            resp = self.requests.get(furl(url).set(query_params={'page': page}))
            links = re.findall(b'href="(/content/early/[^"]+?/[^"]+)"', resp.content)

            if not links:
                return

            for link in links:
                date = arrow.get('-'.join(link.decode().split('/')[3:-1])).date()
                if date < start_date or date > end_date:  # Biorxiv's search sometimes "leaks"
                    return

                article = self.requests.get('http://biorxiv.org' + link.decode())

                yield (link.decode(), {
                    meta.attrs['name']: meta.attrs['content']
                    for meta in BeautifulSoup(article.content, 'html.parser').find_all('meta')
                    if 'name' in meta.attrs
                })
<|endoftext|>


<commit_before>from calendar import timegm
from datetime import datetime
from hashlib import md5
from json import dumps, JSONEncoder, loads

from django.core.urlresolvers import reverse
from django.db.models import Count
from django.db.models.query import QuerySet
from django.http import HttpResponse
from django.shortcuts import render
from django.utils.encoding import smart_bytes, smart_text

from twobuntu.accounts.models import Profile
from twobuntu.articles.models import Article
from twobuntu.categories.models import Category
from twobuntu.utils import timestamp_to_datetime

def index(request):
    """
    Display information about the API.
    """
    return render(request, 'api/v1_2/index.html', {
        'title': 'API',
    })

class APIException(Exception):
    """
    Base for all API exceptions.
    """

class ObjectEncoder(JSONEncoder):
    """
    JSON encoder for supported Django model instances.
    """

    def __init__(self, request, **kwargs):
        """
        Initialize the encoder.
        """
        self._request = request
        super(ObjectEncoder, self).__init__()

    def default(self, o):
        """
        Encode the provided object.
        """
        if type(o) is QuerySet:
            return list(o)
        elif type(o) is Article:
            return self._encode_article(o)
        elif type(o) is Category:
            return self._encode_category(o)
        elif type(o) is Profile:
            return self._encode_profile(o)
        else:
            return JSONEncoder.default(self, o)

    def _encode_article(self, article):
        """
        Encode an article.
        """
        return {
            'id': article.id,
            'title': smart_text(article),
            'author': {
                'id': article.author.id,
                'name': smart_text(article.author.profile),
                'email_hash': md5(smart_bytes(article.author.email)).hexdigest(),
            },
            'category': {
                'id': article.category.id,
                'name': article.category.name,
            },
            'body': article.render(),
            'cc_license': article.cc_license,
            'date': timegm(article.date.utctimetuple()),
            'url': self._request.build_absolute_uri(article.get_absolute_url()),
        }

    def _encode_category(self, category):
        """
        Encode a category.
        """
        return {
            'id': category.id,
            'name': category.name,
            'articles': category.num_articles,
            'url': self._request.build_absolute_uri(category.get_absolute_url()),
        }

    def _encode_profile(self, profile):
        """
        Encode a profile.
        """
        return {
            'id': profile.user.id,
            'name': smart_text(profile),
            'email_hash': md5(smart_bytes(profile.user.email)).hexdigest(),
            'age': profile.age(),
            'location': profile.location,
            'website': profile.website,
            'bio': profile.bio,
            'last_seen': timegm(profile.user.last_login.utctimetuple()),
            'url': self._request.build_absolute_uri(profile.get_absolute_url()),
        }

def endpoint(fn):
    """
    Wrap the API endpoint.
    """
    def wrap(request, **kwargs):
        try:
            json = dumps(
                fn(request, **kwargs),
                cls=ObjectEncoder,
                request=request,
            )
        except APIException as e:
            json = dumps({
                'error': smart_text(e),
            })
        if 'debug' in request.GET:
            return render(request, 'api/v1_2/debug.html', {
                'title': '2buntu API Debugger',
                'parent': {
                    'title': 'API',
                    'url': reverse('api:index'),
                },
                'json': dumps(loads(json), indent=4),
            })
        elif 'callback' in request.GET:
            return HttpResponse(
                '%s(%s)' % (request.GET['callback'], json),
                content_type='application/javascript; charset=utf-8',
            )
        else:
            return HttpResponse(json, content_type='application/json')
    return wrap

def paginate(fn):
    """
    Limit the number of items returned.
    """
    def wrap(request, **kwargs):
        try:
            page = max(int(request.GET['page']), 1) if 'page' in request.GET else 1
            size = min(max(int(request.GET['size']), 0), 20) if 'size' in request.GET else 20
        except ValueError:
            raise APIException("Invalid page and/or size parameter specified.")
        return fn(request, **kwargs)[(page - 1) * size:page * size]
    return wrap

def minmax(field):
    """
    Process minimum and maximum parameters.
    """
    def outer(fn):
        def inner(request, **kwargs):
            filters = {}
            try:
                if 'min' in request.GET:
                    filters['%s__gte' % field] = timestamp_to_datetime(int(request.GET['min']))
                if 'max' in request.GET:
                    filters['%s__lte' % field] = timestamp_to_datetime(int(request.GET['max']))
            except ValueError:
                raise APIException("Invalid min and/or max parameter specified.")
            return fn(request, **kwargs).filter(**filters)
        return inner
    return outer

@endpoint
@paginate
@minmax('date')
def articles(request):
    """
    Return all recent articles.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(status=Article.PUBLISHED)

@endpoint
def article_by_id(request, id):
    """
    Return the specified article.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(pk=id, status=Article.PUBLISHED)

@endpoint
@paginate
@minmax('user__last_login')
def authors(request):
    """
    Return most popular authors.
    """
    return Profile.objects.select_related('user').all()

@endpoint
def author_by_id(request, id):
    """
    Return the specified author.
    """
    return Profile.objects.select_related('user').filter(pk=id)

@endpoint
@paginate
@minmax('date')
def articles_by_author(request, id):
    """
    Return articles written by the specified author.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(author=id, status=Article.PUBLISHED)

@endpoint
@paginate
def categories(request):
    """
    Return most popular categories.
    """
    return Category.objects.all().annotate(num_articles=Count('article'))

@endpoint
@paginate
@minmax('date')
def articles_by_category(request, id):
    """
    Return recent articles in the specified category.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(category=id, status=Article.PUBLISHED)
<commit_msg>Removed unnecessary import.<commit_after>from calendar import timegm
from hashlib import md5
from json import dumps, JSONEncoder, loads

from django.core.urlresolvers import reverse
from django.db.models import Count
from django.db.models.query import QuerySet
from django.http import HttpResponse
from django.shortcuts import render
from django.utils.encoding import smart_bytes, smart_text

from twobuntu.accounts.models import Profile
from twobuntu.articles.models import Article
from twobuntu.categories.models import Category
from twobuntu.utils import timestamp_to_datetime

def index(request):
    """
    Display information about the API.
    """
    return render(request, 'api/v1_2/index.html', {
        'title': 'API',
    })

class APIException(Exception):
    """
    Base for all API exceptions.
    """

class ObjectEncoder(JSONEncoder):
    """
    JSON encoder for supported Django model instances.
    """

    def __init__(self, request, **kwargs):
        """
        Initialize the encoder.
        """
        self._request = request
        super(ObjectEncoder, self).__init__()

    def default(self, o):
        """
        Encode the provided object.
        """
        if type(o) is QuerySet:
            return list(o)
        elif type(o) is Article:
            return self._encode_article(o)
        elif type(o) is Category:
            return self._encode_category(o)
        elif type(o) is Profile:
            return self._encode_profile(o)
        else:
            return JSONEncoder.default(self, o)

    def _encode_article(self, article):
        """
        Encode an article.
        """
        return {
            'id': article.id,
            'title': smart_text(article),
            'author': {
                'id': article.author.id,
                'name': smart_text(article.author.profile),
                'email_hash': md5(smart_bytes(article.author.email)).hexdigest(),
            },
            'category': {
                'id': article.category.id,
                'name': article.category.name,
            },
            'body': article.render(),
            'cc_license': article.cc_license,
            'date': timegm(article.date.utctimetuple()),
            'url': self._request.build_absolute_uri(article.get_absolute_url()),
        }

    def _encode_category(self, category):
        """
        Encode a category.
        """
        return {
            'id': category.id,
            'name': category.name,
            'articles': category.num_articles,
            'url': self._request.build_absolute_uri(category.get_absolute_url()),
        }

    def _encode_profile(self, profile):
        """
        Encode a profile.
        """
        return {
            'id': profile.user.id,
            'name': smart_text(profile),
            'email_hash': md5(smart_bytes(profile.user.email)).hexdigest(),
            'age': profile.age(),
            'location': profile.location,
            'website': profile.website,
            'bio': profile.bio,
            'last_seen': timegm(profile.user.last_login.utctimetuple()),
            'url': self._request.build_absolute_uri(profile.get_absolute_url()),
        }

def endpoint(fn):
    """
    Wrap the API endpoint.
    """
    def wrap(request, **kwargs):
        try:
            json = dumps(
                fn(request, **kwargs),
                cls=ObjectEncoder,
                request=request,
            )
        except APIException as e:
            json = dumps({
                'error': smart_text(e),
            })
        if 'debug' in request.GET:
            return render(request, 'api/v1_2/debug.html', {
                'title': '2buntu API Debugger',
                'parent': {
                    'title': 'API',
                    'url': reverse('api:index'),
                },
                'json': dumps(loads(json), indent=4),
            })
        elif 'callback' in request.GET:
            return HttpResponse(
                '%s(%s)' % (request.GET['callback'], json),
                content_type='application/javascript; charset=utf-8',
            )
        else:
            return HttpResponse(json, content_type='application/json')
    return wrap

def paginate(fn):
    """
    Limit the number of items returned.
    """
    def wrap(request, **kwargs):
        try:
            page = max(int(request.GET['page']), 1) if 'page' in request.GET else 1
            size = min(max(int(request.GET['size']), 0), 20) if 'size' in request.GET else 20
        except ValueError:
            raise APIException("Invalid page and/or size parameter specified.")
        return fn(request, **kwargs)[(page - 1) * size:page * size]
    return wrap

def minmax(field):
    """
    Process minimum and maximum parameters.
    """
    def outer(fn):
        def inner(request, **kwargs):
            filters = {}
            try:
                if 'min' in request.GET:
                    filters['%s__gte' % field] = timestamp_to_datetime(int(request.GET['min']))
                if 'max' in request.GET:
                    filters['%s__lte' % field] = timestamp_to_datetime(int(request.GET['max']))
            except ValueError:
                raise APIException("Invalid min and/or max parameter specified.")
            return fn(request, **kwargs).filter(**filters)
        return inner
    return outer

@endpoint
@paginate
@minmax('date')
def articles(request):
    """
    Return all recent articles.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(status=Article.PUBLISHED)

@endpoint
def article_by_id(request, id):
    """
    Return the specified article.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(pk=id, status=Article.PUBLISHED)

@endpoint
@paginate
@minmax('user__last_login')
def authors(request):
    """
    Return most popular authors.
    """
    return Profile.objects.select_related('user').all()

@endpoint
def author_by_id(request, id):
    """
    Return the specified author.
    """
    return Profile.objects.select_related('user').filter(pk=id)

@endpoint
@paginate
@minmax('date')
def articles_by_author(request, id):
    """
    Return articles written by the specified author.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(author=id, status=Article.PUBLISHED)

@endpoint
@paginate
def categories(request):
    """
    Return most popular categories.
    """
    return Category.objects.all().annotate(num_articles=Count('article'))

@endpoint
@paginate
@minmax('date')
def articles_by_category(request, id):
    """
    Return recent articles in the specified category.
    """
    return Article.objects.select_related('author', 'author__profile', 'category').filter(category=id, status=Article.PUBLISHED)
<|endoftext|>


<commit_before>"""
SLD Profile Dialog for multifunctional models
"""
import wx
import os
import sys
from copy import deepcopy
from danse.common.plottools.plottables import Graph
from Plotter1D import ModelPanel1D as PlotPanel
from sans.guiframe.dataFitting import Data1D
from sans.guiframe.gui_style import GUIFRAME_ID
import pylab

DEFAULT_CMAP = None#pylab.cm.jet
_BOX_WIDTH = 76
_STATICBOX_WIDTH = 400
# X Y offset on plot
_X_OFF = 15
_Y_OFF = 0.5

#SLD panel size
if sys.platform.count("win32") > 0:
    _STATICBOX_WIDTH = 563
    PANEL_SIZE = 425
    FONT_VARIANT = 0
else:
    _STATICBOX_WIDTH = 605
    PANEL_SIZE = 500
    FONT_VARIANT = 1

class SLDPanel(wx.Dialog):
    """
    Provides the SLD profile plot panel.
    """
    ## Internal nickname for the window, used by the AUI manager
    window_name = "Scattering Length Density Profile"
    ## Name to appear on the window title bar
    window_caption = "Scattering Length Density Profile"
    ## Flag to tell the AUI manager to put this panel in the center pane
    CENTER_PANE = True
    def __init__(self, parent=None, base=None, data=None, axes =['Radius'],
                 id=-1, *args, **kwds):
        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        kwds["size"] = wx.Size(_STATICBOX_WIDTH, PANEL_SIZE)
        wx.Dialog.__init__(self, parent, id=id, *args, **kwds)

        if data != None:
            #Font size
            kwds = []
            self.SetWindowVariant(variant=FONT_VARIANT)

            self.SetTitle("Scattering Length Density Profile")
            self.parent = parent
            self._mgr = None
            self.data = data
            self.str = self.data.__str__()
            ## when 2 data have the same id override the 1 st plotted
            self.name = self.data.name
            # Panel for plot
            self.plotpanel = SLDplotpanel(self, axes, -1,
                                             style=wx.TRANSPARENT_WINDOW)
            self.cmap = DEFAULT_CMAP
            ## Create Artist and bind it
            self.subplot = self.plotpanel.subplot
            # layout
            self._setup_layout()
            # plot
            data_plot = deepcopy(self.data)
            data_plot.dy = self._set_dy_data()
            # unit increase to M times for users
            data_plot.y = self._set_y_data()

            self.newplot = Data1D(data_plot.x, data_plot.y, data_plot.dy)
            self.newplot.symbol = GUIFRAME_ID.CURVE_SYMBOL_NUM
            self.newplot.dy = None
            self.newplot.name = 'SLD'
            self.newplot.id = self.newplot.name
            self.plotpanel.add_image(self.newplot)
            self.plotpanel.subplot.set_ylim(min(data_plot.y) - _Y_OFF ,
                                                max(data_plot.y) + _Y_OFF)
            self.plotpanel.subplot.set_xlim(min(data_plot.x) - _X_OFF,
                                                max(data_plot.x) + _X_OFF)
            #self.Centre()
            self.plotpanel.render()
            self.plotpanel.resizing = False
            self.plotpanel.canvas.set_resizing(self.plotpanel.resizing)
            self.plotpanel.subplot.figure.canvas.draw_idle()

    def _set_dy_data(self):
        """
        make fake dy data

        :return dy:
        """
        # set dy as zero
        dy = [0 for y in self.data.y]
        return dy

    def _set_y_data(self):
        """
        make y data unit Mega times

        :return y_value:
        """
        # changes the unit
        y_value = [yvalue * 1e+006 for yvalue in self.data.y]

        return y_value

    def _setup_layout(self):
        """
        Set up the layout
        """
        # panel sizer
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.plotpanel, 0, wx.LEFT|wx.RIGHT, 5)
        sizer.Add(wx.StaticLine(self), 0, wx.ALL|wx.EXPAND, 5)
        sizer.Add((0, 5))
        #-----Button------------1
        id = wx.NewId()
        button_reset = wx.Button(self, id, "Close")
        button_reset.SetToolTipString("Close...")
        button_reset.Bind(wx.EVT_BUTTON, self._close,
                          id=button_reset.GetId())
        sizer.Add(button_reset, 0, wx.LEFT, _STATICBOX_WIDTH - 80)
        sizer.Add((0, 10))
        self.SetSizerAndFit(sizer)
        self.Centre()
        self.Show(True)
        button_reset.SetFocus()

    def _close(self, event):
        """
        Close the dialog
        """
        self.Close(True)

    def _draw_model(self, event):
        """
         on_close, update the model2d plot
        """
        pass

    def get_current_context_menu(self, graph=None):
        """
        When the context menu of a plot is rendered, the
        get_context_menu method will be called to give you a
        chance to add a menu item to the context menu.
        :param graph: the Graph object to which we attach the context menu

        :return: a list of menu items with call-back function
        """
        return []

    def set_schedule_full_draw(self, panel=None, func=None):
        """
        Set_schedule for full draw
        """
        # Not implemented
        pass

    def set_schedule(self, schedule=False):
        """
        Set schedule for redraw
        """
        # Not implemented
        pass

    def set_plot_unfocus(self):
        """
        Set_plot unfocus
        """
        # NOt implemented
        pass

    def on_change_caption(self, name, old_caption, new_caption):
        """
        """
        self.parent.parent.parent.on_change_caption(name, old_caption, new_caption)

    def disable_app_menu(self, panel):
        """
        Disable menu bar
        """
        # Not implemented!
        return

    def show_data1d(self, data, name):
        """
        Show data dialog
        """
        self.parent.parent.parent.show_data1d(data, name)

class SLDplotpanel(PlotPanel):
    """
    Panel
    """
    def __init__(self, parent, axes=[], id=-1, color=None, dpi=None,
                  **kwargs):
        """
        """
        PlotPanel.__init__(self, parent, id=id, xtransform='x', ytransform='y',
                           color=color, dpi=dpi,
                           size=(_STATICBOX_WIDTH, PANEL_SIZE-100), **kwargs)

        # Keep track of the parent Frame
        self.parent = parent
        self.window_name = "Scattering Length Density Profile"
        self.window_caption = self.window_name
        # Internal list of plottable names (because graph
        # doesn't have a dictionary of handles for the plottables)
        self.plots = {}
        self.graph = Graph()
        self.axes_label = []
        for idx in range(0, len(axes)):
            self.axes_label.append(axes[idx])
        self.xaxis_label = ''
        self.xaxis_unit = ''
        self.yaxis_label = ''
        self.yaxis_unit = ''
        self.resizing = True

    def add_image(self, plot):
        """
        Add image(Theory1D)
        """
        self.plots[plot.id] = plot
        self.plots[plot.id].is_data = True
        #add plot
        self.graph.add(plot)
        #add axes
        x1_label = self.axes_label[0]
        self.xaxis_label = '\\rm{%s} '% x1_label
        self.xaxis_unit = 'A'
        self.graph.xaxis(self.xaxis_label, self.xaxis_unit)
        self.yaxis_label = '\\rm{SLD} '
        self.yaxis_unit = '10^{-6}A^{-2}'
        self.graph.yaxis(self.yaxis_label, self.yaxis_unit)
        #self.subplot.figure.canvas.draw_idle()
        # For latter scale changes
        self.plots[plot.id].xaxis('\\rm{%s} '% x1_label, 'A')
        self.plots[plot.id].yaxis('\\rm{SLD} ', '10^{-6}A^{-2}')
        #draw
        self.graph.render(self)

    def on_set_focus(self, event):
        """
        send to the parenet the current panel on focus

        """
        #Not implemented
        pass

    def on_kill_focus(self, event):
        """
        reset the panel color

        """
        #Not implemented
        pass

    def _add_more_tool(self):
        """
        Not implemented
        """
        pass

    def onChangeCaption(self, event):
        """
        Not implemented
        """
        pass

    def _onSave(self, evt):
        """
        Save a data set to a text file

        :param evt: Menu event

        """
        menu = evt.GetEventObject()
        id = evt.GetId()
        self.set_selected_from_menu(menu, id)
        data = self.plots[self.graph.selected_plottable]
        default_name = data.label
        if default_name.count('.') > 0:
            default_name = default_name.split('.')[0]
        default_name += "_out"
        if self.parent != None:
            # What an ancestor!
            fit_panel = self.parent.parent.parent
            fit_panel.parent.save_data1d(data, default_name)

class ViewerFrame(wx.Frame):
    """
    Add comment
    """
    def __init__(self, parent, id, title):
        """
        comment
        :param parent: parent panel/container
        """
        # Initialize the Frame object
        wx.Frame.__init__(self, parent, id, title, wx.DefaultPosition,
                          wx.Size(_STATICBOX_WIDTH, PANEL_SIZE))
        # Panel for 1D plot
        self.plotpanel    = SLDplotpanel(self, -1, style=wx.RAISED_BORDER)

class ViewApp(wx.App):
    def OnInit(self):
        frame = ViewerFrame(None, -1, 'testView')
        frame.Show(True)
        self.SetTopWindow(frame)

        return True

if __name__ == "__main__":
    app = ViewApp(0)
    app.MainLoop()
<commit_msg>trying to fix sldprofile plot problem<commit_after>"""
SLD Profile Dialog for multifunctional models
"""
import wx
import os
import sys
from copy import deepcopy
from danse.common.plottools.plottables import Graph
from Plotter1D import ModelPanel1D as PlotPanel
from sans.guiframe.dataFitting import Data1D
from sans.guiframe.gui_style import GUIFRAME_ID
import pylab

DEFAULT_CMAP = None#pylab.cm.jet
_BOX_WIDTH = 76
_STATICBOX_WIDTH = 400
# X Y offset on plot
_X_OFF = 15
_Y_OFF = 0.5

#SLD panel size
if sys.platform.count("win32") > 0:
    _STATICBOX_WIDTH = 563
    PANEL_SIZE = 425
    FONT_VARIANT = 0
else:
    _STATICBOX_WIDTH = 605
    PANEL_SIZE = 500
    FONT_VARIANT = 1

class SLDPanel(wx.Dialog):
    """
    Provides the SLD profile plot panel.
    """
    ## Internal nickname for the window, used by the AUI manager
    window_name = "Scattering Length Density Profile"
    ## Name to appear on the window title bar
    window_caption = "Scattering Length Density Profile"
    ## Flag to tell the AUI manager to put this panel in the center pane
    CENTER_PANE = True
    def __init__(self, parent=None, base=None, data=None, axes =['Radius'],
                 id=-1, *args, **kwds):
        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        kwds["size"] = wx.Size(_STATICBOX_WIDTH, PANEL_SIZE)
        wx.Dialog.__init__(self, parent, id=id, *args, **kwds)

        if data != None:
            #Font size
            kwds = []
            self.SetWindowVariant(variant=FONT_VARIANT)

            self.SetTitle("Scattering Length Density Profile")
            self.parent = parent
            self._mgr = None
            self.data = data
            self.str = self.data.__str__()
            ## when 2 data have the same id override the 1 st plotted
            self.name = self.data.name
            # Panel for plot
            self.plotpanel = SLDplotpanel(self, axes, -1,
                                             style=wx.TRANSPARENT_WINDOW)
            self.cmap = DEFAULT_CMAP
            ## Create Artist and bind it
            self.subplot = self.plotpanel.subplot
            # layout
            self._setup_layout()
            # plot
            data_plot = deepcopy(self.data)
            data_plot.dy = self._set_dy_data()
            # unit increase to M times for users
            data_plot.y = self._set_y_data()

            self.newplot = Data1D(data_plot.x, data_plot.y, data_plot.dy)
            self.newplot.symbol = GUIFRAME_ID.CURVE_SYMBOL_NUM
            self.newplot.dy = None
            self.newplot.name = 'SLD'
            self.newplot.id = self.newplot.name
            self.plotpanel.add_image(self.newplot)
            self.plotpanel.subplot.set_ylim(min(data_plot.y) - _Y_OFF ,
                                                max(data_plot.y) + _Y_OFF)
            self.plotpanel.subplot.set_xlim(min(data_plot.x) - _X_OFF,
                                                max(data_plot.x) + _X_OFF)
            self.plotpanel.resizing = False
            self.plotpanel.canvas.set_resizing(self.plotpanel.resizing)
            self.plotpanel.graph.render(self.plotpanel)

    def _set_dy_data(self):
        """
        make fake dy data

        :return dy:
        """
        # set dy as zero
        dy = [0 for y in self.data.y]
        return dy

    def _set_y_data(self):
        """
        make y data unit Mega times

        :return y_value:
        """
        # changes the unit
        y_value = [yvalue * 1e+006 for yvalue in self.data.y]

        return y_value

    def _setup_layout(self):
        """
        Set up the layout
        """
        # panel sizer
        sizer = wx.BoxSizer(wx.VERTICAL)
        sizer.Add(self.plotpanel, 0, wx.LEFT|wx.RIGHT, 5)
        sizer.Add(wx.StaticLine(self), 0, wx.ALL|wx.EXPAND, 5)
        sizer.Add((0, 5))
        #-----Button------------1
        id = wx.NewId()
        button_reset = wx.Button(self, id, "Close")
        button_reset.SetToolTipString("Close...")
        button_reset.Bind(wx.EVT_BUTTON, self._close,
                          id=button_reset.GetId())
        sizer.Add(button_reset, 0, wx.LEFT, _STATICBOX_WIDTH - 80)
        sizer.Add((0, 10))
        self.SetSizerAndFit(sizer)
        self.Centre()
        self.Show(True)
        button_reset.SetFocus()

    def _close(self, event):
        """
        Close the dialog
        """
        self.Close(True)

    def _draw_model(self, event):
        """
         on_close, update the model2d plot
        """
        pass

    def get_current_context_menu(self, graph=None):
        """
        When the context menu of a plot is rendered, the
        get_context_menu method will be called to give you a
        chance to add a menu item to the context menu.
        :param graph: the Graph object to which we attach the context menu

        :return: a list of menu items with call-back function
        """
        return []

    def set_schedule_full_draw(self, panel=None, func=None):
        """
        Set_schedule for full draw
        """
        # Not implemented
        pass

    def set_schedule(self, schedule=False):
        """
        Set schedule for redraw
        """
        # Not implemented
        pass

    def set_plot_unfocus(self):
        """
        Set_plot unfocus
        """
        # NOt implemented
        pass

    def on_change_caption(self, name, old_caption, new_caption):
        """
        """
        self.parent.parent.parent.on_change_caption(name, old_caption, new_caption)

    def disable_app_menu(self, panel):
        """
        Disable menu bar
        """
        # Not implemented!
        return

    def show_data1d(self, data, name):
        """
        Show data dialog
        """
        self.parent.parent.parent.show_data1d(data, name)

class SLDplotpanel(PlotPanel):
    """
    Panel
    """
    def __init__(self, parent, axes=[], id=-1, color=None, dpi=None,
                  **kwargs):
        """
        """
        PlotPanel.__init__(self, parent, id=id, xtransform='x', ytransform='y',
                           color=color, dpi=dpi,
                           size=(_STATICBOX_WIDTH, PANEL_SIZE-100), **kwargs)

        # Keep track of the parent Frame
        self.parent = parent
        self.window_name = "Scattering Length Density Profile"
        self.window_caption = self.window_name
        # Internal list of plottable names (because graph
        # doesn't have a dictionary of handles for the plottables)
        self.plots = {}
        self.graph = Graph()
        self.axes_label = []
        for idx in range(0, len(axes)):
            self.axes_label.append(axes[idx])
        self.xaxis_label = ''
        self.xaxis_unit = ''
        self.yaxis_label = ''
        self.yaxis_unit = ''
        self.resizing = True

    def add_image(self, plot):
        """
        Add image(Theory1D)
        """
        self.plots[plot.id] = plot
        self.plots[plot.id].is_data = True
        #add plot
        self.graph.add(plot)
        #add axes
        x1_label = self.axes_label[0]
        self.xaxis_label = '\\rm{%s} '% x1_label
        self.xaxis_unit = 'A'
        self.graph.xaxis(self.xaxis_label, self.xaxis_unit)
        self.yaxis_label = '\\rm{SLD} '
        self.yaxis_unit = '10^{-6}A^{-2}'
        self.graph.yaxis(self.yaxis_label, self.yaxis_unit)
        #self.subplot.figure.canvas.draw_idle()
        # For latter scale changes
        self.plots[plot.id].xaxis('\\rm{%s} '% x1_label, 'A')
        self.plots[plot.id].yaxis('\\rm{SLD} ', '10^{-6}A^{-2}')
        #draw
        #self.graph.render(self)

    def on_set_focus(self, event):
        """
        send to the parenet the current panel on focus

        """
        #Not implemented
        pass

    def on_kill_focus(self, event):
        """
        reset the panel color

        """
        #Not implemented
        pass

    def _add_more_tool(self):
        """
        Not implemented
        """
        pass

    def onChangeCaption(self, event):
        """
        Not implemented
        """
        pass

    def _onSave(self, evt):
        """
        Save a data set to a text file

        :param evt: Menu event

        """
        menu = evt.GetEventObject()
        id = evt.GetId()
        self.set_selected_from_menu(menu, id)
        data = self.plots[self.graph.selected_plottable]
        default_name = data.label
        if default_name.count('.') > 0:
            default_name = default_name.split('.')[0]
        default_name += "_out"
        if self.parent != None:
            # What an ancestor!
            fit_panel = self.parent.parent.parent
            fit_panel.parent.save_data1d(data, default_name)

class ViewerFrame(wx.Frame):
    """
    Add comment
    """
    def __init__(self, parent, id, title):
        """
        comment
        :param parent: parent panel/container
        """
        # Initialize the Frame object
        wx.Frame.__init__(self, parent, id, title, wx.DefaultPosition,
                          wx.Size(_STATICBOX_WIDTH, PANEL_SIZE))
        # Panel for 1D plot
        self.plotpanel    = SLDplotpanel(self, -1, style=wx.RAISED_BORDER)

class ViewApp(wx.App):
    def OnInit(self):
        frame = ViewerFrame(None, -1, 'testView')
        frame.Show(True)
        self.SetTopWindow(frame)

        return True

if __name__ == "__main__":
    app = ViewApp(0)
    app.MainLoop()
<|endoftext|>


<commit_before>from selenium_tests.AdminDriverTest import AdminDriverTest
from selenium.webdriver.common.by import By

class TestUserAccounting(AdminDriverTest):
    def test_create_new_entry_button(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_first_button("Create New Entry")
        self.click_modal_footer_button("Cancel")

    def test_create_and_remove_user(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_first_button("Create New Entry")
        self.type_text_in_element_located(By.CSS_SELECTOR, ".modal-body > form > div > input", "mrenou")
        self.click_modal_footer_button("Submit")

        # Click remove button
        self.click_row_action_button("mrenou", "Remove")

        self.click_modal_footer_button("Ok")

    def test_admin_name_header_bug(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_row_action_button("test", "Policies")

        self.wait_until_text_inside_element_located(By.CSS_SELECTOR, "span.hidden-xs", "admin")
<commit_msg>Add tests<commit_after>from selenium_tests.AdminDriverTest import AdminDriverTest
from selenium.webdriver.common.by import By
import os

class TestUserAccounting(AdminDriverTest):
    def test_create_new_entry_button(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_first_button("Create New Entry")
        self.click_modal_footer_button("Cancel")

    def test_create_and_remove_user(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_first_button("Create New Entry")
        self.type_text_in_element_located(By.CSS_SELECTOR, ".modal-body > form > div > input", "mrenou")
        self.click_modal_footer_button("Submit")

        # Click remove button
        self.click_row_action_button("mrenou", "Remove")

        self.wait_until_invisibility_of_row("mrenou")

        self.click_modal_footer_button("Ok")

    def test_admin_name_header_bug(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_row_action_button("test", "Policies")

        self.wait_until_text_inside_element_located(By.CSS_SELECTOR, "span.hidden-xs", "admin")

    def test_user_id(self):
        self.click_first_element_located(By.LINK_TEXT, "Users")

        self.click_row_action_button("test", "Policies")

        self.wait_until_visibility_of_row("simphonyproject/simphonic-mayavi")

        self.driver.get(os.path.join(
            self.base_url,
            "user/admin/#/users/36/accounting"
        ))

        self.wait_until_presence_of_element_located(By.CSS_SELECTOR, "div.alert-danger")
        self.wait_until_invisibility_of_row("simphonyproject/simphonic-mayavi")
<|endoftext|>


<commit_before>from twisted.words.protocols import irc
from txircd.modbase import Command

class InfoCommand(Command):
    def onUse(self, user, data):
        user.sendMessage(irc.RPL_INFO, ":{}".format(self.ircd.version))
        user.sendMessage(irc.RPL_INFO, ":Developed by the Desert Bus for Hope Engineering Team")
        user.sendMessage(irc.RPL_INFO, ": -")
        user.sendMessage(irc.RPL_INFO, ":Original txircd by Fugiman, ElementalAlchemist, and ojii")
        user.sendMessage(irc.RPL_INFO, ":Modular txircd by ElementalAlchemist with some parts from the original version")
        user.sendMessage(irc.RPL_ENDOFINFO, ":End of /INFO list")

    def processParams(self, user, params):
        if user.registered > 0:
            user.sendMessage(irc.ERR_NOTYETREGISTERED, "INFO", ":You have not registered")
            return {}
        return {
            "user": user
        }

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd

    def spawn(self):
        return {
            "commands": {
                "INFO": InfoCommand()
            }
        }

    def cleanup(self):
        del self.ircd.commands["INFO"]<commit_msg>Now that Fugiman is getting back into it, may as well throw him into the new credits as well.<commit_after>from twisted.words.protocols import irc
from txircd.modbase import Command

class InfoCommand(Command):
    def onUse(self, user, data):
        user.sendMessage(irc.RPL_INFO, ":{}".format(self.ircd.version))
        user.sendMessage(irc.RPL_INFO, ":Developed by the Desert Bus for Hope Engineering Team")
        user.sendMessage(irc.RPL_INFO, ": -")
        user.sendMessage(irc.RPL_INFO, ":Original txircd by Fugiman, ElementalAlchemist, and ojii")
        user.sendMessage(irc.RPL_INFO, ":Modular txircd by ElementalAlchemist and Fugiman")
        user.sendMessage(irc.RPL_ENDOFINFO, ":End of /INFO list")

    def processParams(self, user, params):
        if user.registered > 0:
            user.sendMessage(irc.ERR_NOTYETREGISTERED, "INFO", ":You have not registered")
            return {}
        return {
            "user": user
        }

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd

    def spawn(self):
        return {
            "commands": {
                "INFO": InfoCommand()
            }
        }

    def cleanup(self):
        del self.ircd.commands["INFO"]<|endoftext|>


<commit_before>#!/usr/bin/env python3
import os
import time
from multiprocessing import Process
from tqdm import tqdm

os.environ['FILEREADER_CACHE'] = '1'

from common.realtime import config_realtime_process, Ratekeeper
from selfdrive.boardd.boardd import can_capnp_to_can_list
from tools.lib.logreader import LogReader

from panda import Panda
try:
  from panda_jungle import PandaJungle  # pylint: disable=import-error
except Exception:
  PandaJungle = None  # type: ignore

ROUTE = "77611a1fac303767/2020-03-24--09-50-38"
NUM_SEGS = 10 # route has 82 segments available

print("Loading log...")
CAN_MSGS = []
for i in tqdm(list(range(1, NUM_SEGS))):
  log_url = f"https://commadataci.blob.core.windows.net/openpilotci/{ROUTE}/{i}/rlog.bz2"
  lr = LogReader(log_url)
  CAN_MSGS += [can_capnp_to_can_list(m.can) for m in lr if m.which() == 'can']

def send_thread(sender, core):
  config_realtime_process(core, 55)

  if "Jungle" in str(type(sender)):
    sender.set_ignition(False)
    time.sleep(3)
    sender.set_ignition(True)
  else:
    sender.set_safety_mode(Panda.SAFETY_ALLOUTPUT)
  sender.set_can_loopback(False)

  log_idx = 0
  rk = Ratekeeper(100)
  while True:
    snd = CAN_MSGS[log_idx]
    log_idx = (log_idx + 1) % len(CAN_MSGS)
    snd = list(filter(lambda x: x[-1] <= 2, snd))
    sender.can_send_many(snd)

    # Drain panda message buffer
    sender.can_recv()
    rk.keep_time()

def connect():
  serials = {}
  while True:
    # look for new devices
    for p in [Panda, PandaJungle]:
      if p is None:
        continue

      for s in p.list():
        if s not in serials:
          print("starting send thread for", s)
          serials[s] = Process(target=send_thread, args=(p(s), 3))
          serials[s].start()

    # try to join all send procs
    cur_serials = serials.copy()
    for s, p in cur_serials.items():
      p.join(0.01)
      if p.exitcode is not None:
        del serials[s]

    time.sleep(1)

if __name__ == "__main__":
  while True:
    try:
      connect()
    except Exception:
      pass
<commit_msg>replay improvements for testing closet<commit_after>#!/usr/bin/env python3
import os
import time
from multiprocessing import Process
from tqdm import tqdm

os.environ['TESTING_CLOSET'] = '1'
os.environ['FILEREADER_CACHE'] = '1'

from common.realtime import config_realtime_process, Ratekeeper
from selfdrive.boardd.boardd import can_capnp_to_can_list
from selfdrive.pandad import set_panda_power
from tools.lib.logreader import LogReader

from panda import Panda
try:
  from panda_jungle import PandaJungle  # pylint: disable=import-error
except Exception:
  PandaJungle = None  # type: ignore

ROUTE = "77611a1fac303767/2020-03-24--09-50-38"
NUM_SEGS = 2 # route has 82 segments available

print("Loading log...")
CAN_MSGS = []
for i in tqdm(list(range(1, NUM_SEGS+1))):
  log_url = f"https://commadataci.blob.core.windows.net/openpilotci/{ROUTE}/{i}/rlog.bz2"
  lr = LogReader(log_url)
  CAN_MSGS += [can_capnp_to_can_list(m.can) for m in lr if m.which() == 'can']

def send_thread(sender, core):
  config_realtime_process(core, 55)

  if "Jungle" in str(type(sender)):
    for i in [0, 1, 2, 3, 0xFFFF]:
      sender.can_clear(i)
    sender.set_ignition(False)
    time.sleep(5)
    sender.set_ignition(True)
    sender.set_panda_power(True)
  else:
    sender.set_safety_mode(Panda.SAFETY_ALLOUTPUT)
  sender.set_can_loopback(False)

  log_idx = 0
  rk = Ratekeeper(100)
  while True:
    snd = CAN_MSGS[log_idx]
    log_idx = (log_idx + 1) % len(CAN_MSGS)
    snd = list(filter(lambda x: x[-1] <= 2, snd))
    sender.can_send_many(snd)

    # Drain panda message buffer
    sender.can_recv()
    rk.keep_time()

def connect():
  serials = {}
  while True:
    # look for new devices
    for p in [Panda, PandaJungle]:
      if p is None:
        continue

      for s in p.list():
        if s not in serials:
          print("starting send thread for", s)
          serials[s] = Process(target=send_thread, args=(p(s), 3))
          serials[s].start()

    # try to join all send procs
    cur_serials = serials.copy()
    for s, p in cur_serials.items():
      p.join(0.01)
      if p.exitcode is not None:
        del serials[s]

    time.sleep(1)

if __name__ == "__main__":
  set_panda_power(False)
  time.sleep(1)

  if "FLASH" in os.environ and PandaJungle is not None:
    for s in PandaJungle.list():
      PandaJungle(s).flash()

  while True:
    try:
      connect()
    except Exception:
      pass
<|endoftext|>


<commit_before>import codecs
import os
import sys
from optparse import make_option

from django.core.management.base import CommandError, NoArgsCommand
from leonardo.module.web.models import Widget, WidgetContentTheme, PageTheme, WidgetBaseTheme

from ._utils import get_or_create_template

"widget.verbose_name - template.name"
THEME_NAME_FORMAT = "{0} {1}"

class Command(NoArgsCommand):
    help = "Syncs file system templates and themes with the database bidirectionally.\
    based on dbtemplates.sync_templates"
    option_list = NoArgsCommand.option_list + (
        make_option("-e", "--ext",
                    dest="ext", action="store", default="html",
                    help="extension of the files you want to "
                         "sync with the database [default: %default]"),
        make_option("-f", "--force",
                    action="store_true", dest="force", default=False,
                    help="overwrite existing database templates"),
        make_option("-o", "--overwrite",
                    action="store", dest="overwrite", default='0',
                    help="'0' - ask always, '1' - overwrite database "
                         "templates from template files, '2' - overwrite "
                         "template files from database templates"),
        make_option("-a", "--app-first",
                    action="store_true", dest="app_first", default=False,
                    help="look for templates in applications "
                         "directories before project templates"),
        make_option("-d", "--delete",
                    action="store_true", dest="delete", default=False,
                    help="Delete templates after syncing"))

    def handle_noargs(self, **options):
        extension = options.get('ext')
        force = options.get('force')
        overwrite = options.get('overwrite')
        app_first = options.get('app_first')
        delete = options.get('delete')

        # base
        path = os.path.dirname(os.path.abspath(__file__))
        possible_topdir = os.path.normpath(os.path.join(path,
                                                        os.pardir,
                                                        os.pardir,
                                                        "templates"))

        # load widget base templates
        widget_base_dir = os.path.join(possible_topdir, "base", "widget")
        widget_base_template = None
        for dirpath, subdirs, filenames in os.walk(widget_base_dir):
            for f in filenames:
                # ignore private members
                if not f.startswith("_"):
                    w_base_template = get_or_create_template(
                        f, force=force, prefix="base/widget")

                    try:
                        widget_theme = WidgetBaseTheme.objects.get(
                            template__name__exact=f)
                    except WidgetBaseTheme.DoesNotExist:
                        widget_theme = WidgetBaseTheme()
                        widget_theme.name = f.split("/")[-1]
                        widget_theme.label = f.split("/")[-1].split(".")[0].title()
                        widget_theme.template = w_base_template
                        widget_theme.save()

                    if "default" in f:
                        widget_base_template = w_base_template

        created_themes = 0

        # load widget templates and create widget themes with default base
        for w in Widget.__subclasses__():
            templates = w.templates()
            for name in templates:
                # ignore private members
                if not name.startswith("_"):
                    widget_template = get_or_create_template(name, force=force)

                    if not widget_template:
                        self.stdout.write('Template for "%s" not found' % name)
                        continue

                    try:
                        widget_theme = WidgetContentTheme.objects.get(
                            template__name__exact=name)
                    except WidgetContentTheme.DoesNotExist:
                        widget_theme = WidgetContentTheme()
                        widget_theme.name = name.split("/")[-1]
                        widget_theme.label = THEME_NAME_FORMAT.format(
                            unicode(w._meta.verbose_name), name.split("/")[-1].split(".")[0])
                        widget_theme.template = widget_template
                        widget_theme.widget_class = w.__name__
                        widget_theme.save()
                        created_themes += 1

        self.stdout.write('Successfully created %s widget themes' % created_themes)
<commit_msg>fix duplicity<commit_after>import codecs
import os
import sys
from optparse import make_option

from django.core.management.base import CommandError, NoArgsCommand
from leonardo.module.web.models import Widget, WidgetContentTheme, PageTheme, WidgetBaseTheme

from ._utils import get_or_create_template

"widget.verbose_name - template.name"
THEME_NAME_FORMAT = "{0} {1}"

class Command(NoArgsCommand):
    help = "Syncs file system templates and themes with the database bidirectionally.\
    based on dbtemplates.sync_templates"
    option_list = NoArgsCommand.option_list + (
        make_option("-e", "--ext",
                    dest="ext", action="store", default="html",
                    help="extension of the files you want to "
                         "sync with the database [default: %default]"),
        make_option("-f", "--force",
                    action="store_true", dest="force", default=False,
                    help="overwrite existing database templates"),
        make_option("-o", "--overwrite",
                    action="store", dest="overwrite", default='0',
                    help="'0' - ask always, '1' - overwrite database "
                         "templates from template files, '2' - overwrite "
                         "template files from database templates"),
        make_option("-a", "--app-first",
                    action="store_true", dest="app_first", default=False,
                    help="look for templates in applications "
                         "directories before project templates"),
        make_option("-d", "--delete",
                    action="store_true", dest="delete", default=False,
                    help="Delete templates after syncing"))

    def handle_noargs(self, **options):
        extension = options.get('ext')
        force = options.get('force')
        overwrite = options.get('overwrite')
        app_first = options.get('app_first')
        delete = options.get('delete')

        # base
        path = os.path.dirname(os.path.abspath(__file__))
        possible_topdir = os.path.normpath(os.path.join(path,
                                                        os.pardir,
                                                        os.pardir,
                                                        "templates"))

        # load widget base templates
        widget_base_dir = os.path.join(possible_topdir, "base", "widget")
        widget_base_template = None
        for dirpath, subdirs, filenames in os.walk(widget_base_dir):
            for f in filenames:
                # ignore private members
                if not f.startswith("_"):
                    w_base_template = get_or_create_template(
                        f, force=force, prefix="base/widget")
                    name = f.split("/")[-1]
                    try:
                        widget_theme = WidgetBaseTheme.objects.get(
                            name__exact=name)
                    except WidgetBaseTheme.DoesNotExist:
                        widget_theme = WidgetBaseTheme()
                        widget_theme.name = name
                        widget_theme.label = name.split(".")[0].title()
                        widget_theme.template = w_base_template
                        widget_theme.save()

                    if "default" in f:
                        widget_base_template = widget_theme

        created_themes = 0

        # load widget templates and create widget themes with default base
        for w in Widget.__subclasses__():
            templates = w.templates()
            for name in templates:
                # ignore private members
                if not name.startswith("_"):
                    widget_template = get_or_create_template(name, force=force)

                    if not widget_template:
                        self.stdout.write('Template for "%s" not found' % name)
                        continue

                    try:
                        widget_theme = WidgetContentTheme.objects.get(
                            template__name__exact=name)
                    except WidgetContentTheme.DoesNotExist:
                        widget_theme = WidgetContentTheme()
                        widget_theme.name = name.split("/")[-1]
                        widget_theme.label = THEME_NAME_FORMAT.format(
                            unicode(w._meta.verbose_name), name.split("/")[-1].split(".")[0])
                        widget_theme.template = widget_template
                        widget_theme.widget_class = w.__name__
                        widget_theme.save()
                        created_themes += 1

        self.stdout.write('Successfully created %s widget themes' % created_themes)
<|endoftext|>


<commit_before>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower

class ModeCommand(Command):
	def onUse(self, user, data):
		if "targetchan" in data:
			cdata = self.ircd.channels[data["targetchan"]]
			self.chanUse(user, cdata, data["modes"])
		elif "targetuser" in data:
			if irc_lower(user) == irc_lower(data["targetuser"]):
				self.userUse(user, data["modes"])
			else:
				self.sendMessage(irc.ERR_USERSDONTMATCH, ":Can't operate on modes for other users")

	def chanUse(self, user, channel, modes):
		if not modes:
			self.sendMessage(irc.RPL_CHANNELMODEIS, channel.name, channel.modeString())
			self.sendMessage(irc.RPL_CREATIONTIME, channel.name, str(epoch(channel.created)))
			return
		modeDisplay = []
		for modedata in modes:
			modetype, adding, mode, param = modedata
			if adding and not self.ircd.channel_modes[modetype][mode].checkSet(user, channel, param):
				continue
			if not adding and not self.ircd.channel_modes[modetype][mode].checkUnset(user, channel, param):
				continue
			if modetype == -1:
				if param not in self.ircd.users:
					continue
				udata = self.ircd.users[param]
				if channel.name not in udata.channels:
					continue
				if adding:
					status = udata.channels[channel.name]["status"]
					statusList = list(status)
					for index, statusLevel in enumerate(status):
						if self.ircd.prefixes[statusLevel][1] < self.ircd.prefixes[mode][1]:
							statusList.insert(index, mode)
							break
					udata.channels[channel.name]["status"] = "".join(statusList)
					modeDisplay.append(modedata)
				else:
					if mode in udata.channels[channel.name]["status"]:
						udata.channels[channel.name]["status"].remove(mode)
						modeDisplay.append(modedata)
			elif modetype == 0:
				if not param:
					self.ircd.channel_modes[modetype][mode].showParam(user, channel)
				elif adding:
					if mode not in channel.mode:
						channel.mode[mode] = []
					if param not in channel.mode[mode]:
						channel.mode[mode].append(param)
						modeDisplay.append(modeData)
				else:
					if mode not in channel.mode:
						continue
					if param in channel.mode[mode]:
						channel.mode[mode].remove(param)
						modeDisplay.append(modeData)
						if not channel.mode[mode]:
							del channel.mode[mode]
			else:
				if adding:
					if mode in channel.mode and param == channel.mode[mode]:
						continue
					channel.mode[mode] = param
					modeDisplay.append(modeData)
				else:
					if mode not in channel.mode:
						continue
					if modetype == 1 and param != channel.mode[mode]:
						continue
					del channel.mode[mode]
					modeDisplay.append(modeData)
		if modeDisplay:
			adding = None
			modestring = [] # use an array of characters since we're repeatedly adding to this; join at the end
			params = []
			for mode in modeDisplay:
				if mode[1] and adding != "+":
					adding = "+"
					modestring.append("+")
				elif not mode[1] and adding != "-":
					adding = "-"
					modestring.append("-")
				modestring.append(mode[2])
				if mode[3]:
					params.append(mode[3])
			modeLine = "{} {}".format("".join(modestring), " ".join(params)) if params else "".join(modestring)
			for udata in channel.users:
				udata.sendMessage("MODE", modeLine, to=channel.name, prefix=user.prefix())

	def userUse(self, user, modes):
		if not modes:
			self.sendMessage(irc.RPL_UMODEIS, user.modeString())
			return
		modeDisplay = []
		for modedata in modes:
			modetype, adding, mode, param = modedata
			if adding and not self.ircd.user_modes[modetype][mode].checkSet(user, user, param):
				continue
			if not adding and not self.ircd.user_modes[modetype][mode].checkUnset(user, user, param):
				continue
			if modetype == 0:
				if not param:
					self.ircd.user_modes[modetype][mode].showParam(user, user)
				elif adding:
					if mode not in user.mode:
						user.mode[mode] = []
					if param not in user.mode[mode]:
						user.mode[mode].append(param)
						modeDisplay.append(modeData)
				else:
					if mode not in user.mode:
						continue
					if param in user.mode[mode]:
						user.mode[mode].remove(param)
						modeDisplay.append(modeData)
						if not user.mode[mode]:
							del user.mode[mode]
			else:
				if adding:
					if mode in user.mode and param == user.mode[mode]:
						continue
					user.mode[mode] = param
					modeDisplay.append(modeData)
				else:
					if mode not in user.mode:
						continue
					if modetype == 1 and param != user.mode[mode]:
						continue
					del user.mode[mode]
					modeDisplay.append(modeData)
		if modeDisplay:
			adding = None
			modestring = [] # use an array of characters since we're repeatedly adding to this; join at the end
			params = []
			for mode in modeDisplay:
				if mode[1] and adding != "+":
					adding = "+"
					modestring.append("+")
				elif not mode[1] and adding != "-":
					adding = "-"
					modestring.append("-")
				modestring.append(mode[2])
				if mode[3]:
					params.append(mode[3])
			modeLine = "{} {}".format("".join(modestring), " ".join(params)) if params else "".join(modestring)
			user.sendMessage("MODE", modeLine, prefix=user.prefix())

	def processParams(self, user, params):
		if user.registered > 0:
			user.sendMessage(irc.ERR_NOTREGISTERED, "MODE", ":You have not registered")
			return {}
		if not params:
			user.sendMessage(irc.ERR_NEEDMOREPARAMS, "MODE", ":Not enough parameters")
			return {}
		if params[0] in self.ircd.users:
			if len(params) > 1 and params[1]:
				modeChanges = []
				adding = True
				current_param = 2
				for mode in params[1]:
					if mode == "+":
						adding = True
					elif mode == "-":
						adding = False
					else:
						if mode not in self.ircd.user_mode_type:
							user.sendMessage(irc.ERR_UMODEUNKNOWNFLAG, mode, ":is unknown mode char to me")
							continue
						mode_type = self.ircd.user_mode_type[mode]
						if mode_type == 1 or (adding and mode_type == 2):
							if len(params) <= current_param:
								continue # Mode must have param that wasn't provided
							modeChanges.append([mode_type, adding, mode, params[current_param]])
							current_param += 1
						else:
							modeChanges.append([mode_type, adding, mode, None])
			return {
				"user": user,
				"targetuser": self.ircd.users[params[0]],
				"modes": []
			}
		if params[0] in self.ircd.channels:
			cdata = self.ircd.channels[params[0]]
			if len(params) > 1 and params[1]:
				if params[0] not in user.channels or not user.hasAccess(cdata.name, self.ircd.channel_mode_level):
					user.sendMessage(irc.ERR_CHANOPRIVSNEEDED, params[0], ":You must have channel operator access to set channel modes")
					return {}
				modeChanges = []
				adding = True
				current_param = 2
				for mode in params[1]:
					if mode == "+":
						adding = True
					elif mode == "-":
						adding = False
					else:
						if mode not in self.ircd.channel_mode_type:
							user.sendMessage(irc.ERR_UNKNOWNMODE, mode, ":is unknown mode char to me")
							continue
						mode_type = self.ircd.channel_mode_type[mode]
						if mode_type == 1 or (adding and mode_type == 2):
							if len(params) <= current_param:
								continue # Mode must have param that wasn't provided
							modeChanges.append([mode_type, adding, mode, params[current_param]])
							current_param += 1
						else:
							modeChanges.append([mode_type, adding, mode, None])
			return {
				"user": user,
				"targetchan": cdata,
				"modes": []
			}
		user.sendMessage(irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
		return {}

class Spawner(object):
	def __init__(self, ircd):
		self.ircd = ircd

	def spawn(self):
		return {
			"commands": {
				"MODE": ModeCommand()
			}
		}

	def cleanup(self):
		del self.ircd.commands["MODE"]<commit_msg>Handle mode parameters for list modes<commit_after>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower

class ModeCommand(Command):
	def onUse(self, user, data):
		if "targetchan" in data:
			cdata = self.ircd.channels[data["targetchan"]]
			self.chanUse(user, cdata, data["modes"])
		elif "targetuser" in data:
			if irc_lower(user) == irc_lower(data["targetuser"]):
				self.userUse(user, data["modes"])
			else:
				self.sendMessage(irc.ERR_USERSDONTMATCH, ":Can't operate on modes for other users")

	def chanUse(self, user, channel, modes):
		if not modes:
			self.sendMessage(irc.RPL_CHANNELMODEIS, channel.name, channel.modeString())
			self.sendMessage(irc.RPL_CREATIONTIME, channel.name, str(epoch(channel.created)))
			return
		modeDisplay = []
		for modedata in modes:
			modetype, adding, mode, param = modedata
			if adding and not self.ircd.channel_modes[modetype][mode].checkSet(user, channel, param):
				continue
			if not adding and not self.ircd.channel_modes[modetype][mode].checkUnset(user, channel, param):
				continue
			if modetype == -1:
				if param not in self.ircd.users:
					continue
				udata = self.ircd.users[param]
				if channel.name not in udata.channels:
					continue
				if adding:
					status = udata.channels[channel.name]["status"]
					statusList = list(status)
					for index, statusLevel in enumerate(status):
						if self.ircd.prefixes[statusLevel][1] < self.ircd.prefixes[mode][1]:
							statusList.insert(index, mode)
							break
					udata.channels[channel.name]["status"] = "".join(statusList)
					modeDisplay.append(modedata)
				else:
					if mode in udata.channels[channel.name]["status"]:
						udata.channels[channel.name]["status"].remove(mode)
						modeDisplay.append(modedata)
			elif modetype == 0:
				if not param:
					self.ircd.channel_modes[modetype][mode].showParam(user, channel)
				elif adding:
					if mode not in channel.mode:
						channel.mode[mode] = []
					if param not in channel.mode[mode]:
						channel.mode[mode].append(param)
						modeDisplay.append(modeData)
				else:
					if mode not in channel.mode:
						continue
					if param in channel.mode[mode]:
						channel.mode[mode].remove(param)
						modeDisplay.append(modeData)
						if not channel.mode[mode]:
							del channel.mode[mode]
			else:
				if adding:
					if mode in channel.mode and param == channel.mode[mode]:
						continue
					channel.mode[mode] = param
					modeDisplay.append(modeData)
				else:
					if mode not in channel.mode:
						continue
					if modetype == 1 and param != channel.mode[mode]:
						continue
					del channel.mode[mode]
					modeDisplay.append(modeData)
		if modeDisplay:
			adding = None
			modestring = [] # use an array of characters since we're repeatedly adding to this; join at the end
			params = []
			for mode in modeDisplay:
				if mode[1] and adding != "+":
					adding = "+"
					modestring.append("+")
				elif not mode[1] and adding != "-":
					adding = "-"
					modestring.append("-")
				modestring.append(mode[2])
				if mode[3]:
					params.append(mode[3])
			modeLine = "{} {}".format("".join(modestring), " ".join(params)) if params else "".join(modestring)
			for udata in channel.users:
				udata.sendMessage("MODE", modeLine, to=channel.name, prefix=user.prefix())

	def userUse(self, user, modes):
		if not modes:
			self.sendMessage(irc.RPL_UMODEIS, user.modeString())
			return
		modeDisplay = []
		for modedata in modes:
			modetype, adding, mode, param = modedata
			if adding and not self.ircd.user_modes[modetype][mode].checkSet(user, user, param):
				continue
			if not adding and not self.ircd.user_modes[modetype][mode].checkUnset(user, user, param):
				continue
			if modetype == 0:
				if not param:
					self.ircd.user_modes[modetype][mode].showParam(user, user)
				elif adding:
					if mode not in user.mode:
						user.mode[mode] = []
					if param not in user.mode[mode]:
						user.mode[mode].append(param)
						modeDisplay.append(modeData)
				else:
					if mode not in user.mode:
						continue
					if param in user.mode[mode]:
						user.mode[mode].remove(param)
						modeDisplay.append(modeData)
						if not user.mode[mode]:
							del user.mode[mode]
			else:
				if adding:
					if mode in user.mode and param == user.mode[mode]:
						continue
					user.mode[mode] = param
					modeDisplay.append(modeData)
				else:
					if mode not in user.mode:
						continue
					if modetype == 1 and param != user.mode[mode]:
						continue
					del user.mode[mode]
					modeDisplay.append(modeData)
		if modeDisplay:
			adding = None
			modestring = [] # use an array of characters since we're repeatedly adding to this; join at the end
			params = []
			for mode in modeDisplay:
				if mode[1] and adding != "+":
					adding = "+"
					modestring.append("+")
				elif not mode[1] and adding != "-":
					adding = "-"
					modestring.append("-")
				modestring.append(mode[2])
				if mode[3]:
					params.append(mode[3])
			modeLine = "{} {}".format("".join(modestring), " ".join(params)) if params else "".join(modestring)
			user.sendMessage("MODE", modeLine, prefix=user.prefix())

	def processParams(self, user, params):
		if user.registered > 0:
			user.sendMessage(irc.ERR_NOTREGISTERED, "MODE", ":You have not registered")
			return {}
		if not params:
			user.sendMessage(irc.ERR_NEEDMOREPARAMS, "MODE", ":Not enough parameters")
			return {}
		if params[0] in self.ircd.users:
			if len(params) > 1 and params[1]:
				modeChanges = []
				adding = True
				current_param = 2
				for mode in params[1]:
					if mode == "+":
						adding = True
					elif mode == "-":
						adding = False
					else:
						if mode not in self.ircd.user_mode_type:
							user.sendMessage(irc.ERR_UMODEUNKNOWNFLAG, mode, ":is unknown mode char to me")
							continue
						mode_type = self.ircd.user_mode_type[mode]
						if mode_type == 0 or mode_type == 1 or (adding and mode_type == 2):
							if len(params) <= current_param:
								continue # Mode must have param that wasn't provided
							modeChanges.append([mode_type, adding, mode, params[current_param]])
							current_param += 1
						else:
							modeChanges.append([mode_type, adding, mode, None])
			return {
				"user": user,
				"targetuser": self.ircd.users[params[0]],
				"modes": []
			}
		if params[0] in self.ircd.channels:
			cdata = self.ircd.channels[params[0]]
			if len(params) > 1 and params[1]:
				if params[0] not in user.channels or not user.hasAccess(cdata.name, self.ircd.channel_mode_level):
					user.sendMessage(irc.ERR_CHANOPRIVSNEEDED, params[0], ":You must have channel operator access to set channel modes")
					return {}
				modeChanges = []
				adding = True
				current_param = 2
				for mode in params[1]:
					if mode == "+":
						adding = True
					elif mode == "-":
						adding = False
					else:
						if mode not in self.ircd.channel_mode_type:
							user.sendMessage(irc.ERR_UNKNOWNMODE, mode, ":is unknown mode char to me")
							continue
						mode_type = self.ircd.channel_mode_type[mode]
						if mode_type == -1 or mode_type == 0 or mode_type == 1 or (adding and mode_type == 2):
							if len(params) <= current_param:
								continue # Mode must have param that wasn't provided
							modeChanges.append([mode_type, adding, mode, params[current_param]])
							current_param += 1
						else:
							modeChanges.append([mode_type, adding, mode, None])
			return {
				"user": user,
				"targetchan": cdata,
				"modes": []
			}
		user.sendMessage(irc.ERR_NOSUCHNICK, params[0], ":No such nick/channel")
		return {}

class Spawner(object):
	def __init__(self, ircd):
		self.ircd = ircd

	def spawn(self):
		return {
			"commands": {
				"MODE": ModeCommand()
			}
		}

	def cleanup(self):
		del self.ircd.commands["MODE"]<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import requests
import json
from ... import config

if "basistech" in config.engines:
	host = config.engines["basistech"]["host"]
	labels = config.engines["basistech"]["labels"]

def convert_label(label):
	if label in labels:
		return labels[label]
	else:
		print "basistech:", label
		return label

def extract_entities(text, lang):
	entities = {}
	headers = {'content-type': 'application/json', 'accept' : 'application/json' }

	# see RLP-AppDev-Guide.pdf
	data = {'text': text,
			'languageDetection': {"language": "UNKNOWN","strategy": "SINGLE"},
			'resultTypes': ["DEFAULT_NAMED_ENTITY"]}
	rp = requests.post(host + "/rws/services/doc/processText", headers=headers, data=json.dumps(data))
	try:
		result = rp.json()

		types = result["regions"][0]["resultAccess"]["NamedEntityTypeString"]
		lemmas = result["regions"][0]["resultAccess"]["NormalizedNamedEntity"]

		for lemma, entity_type in zip(lemmas, types):
			key = lemma
			entities[key] = convert_label(entity_type)
	except Exception as e:
		print "BasisTech API Error:", e
		print "BasisTech Response:", rp.content
	return entities<commit_msg>IP-1625: Activated co-reference resolution for basistech.<commit_after># -*- coding: utf-8 -*-
import requests
import json
from ... import config

if "basistech" in config.engines:
	host = config.engines["basistech"]["host"]
	labels = config.engines["basistech"]["labels"]

def convert_label(label):
	if label in labels:
		return labels[label]
	else:
		print "basistech:", label
		return label

def extract_entities(text, lang):
	entities = {}
	headers = {'content-type': 'application/json', 'accept' : 'application/json' }

	# To activate co-reference resolution one must add "NAMED_ENTITY_CHAIN_ID"
	# to the resultTypes (see chapter 5.10, RLP-AppDev-Guide.pdf)
	data = {'text': text,
			'languageDetection': {"language": "UNKNOWN","strategy": "SINGLE"},
			'resultTypes': ["DEFAULT_NAMED_ENTITY", "NAMED_ENTITY_CHAIN_ID"]}
	rp = requests.post(host + "/rws/services/doc/processText", headers=headers, data=json.dumps(data))
	try:
		result = rp.json()

		types = result["regions"][0]["resultAccess"]["NamedEntityTypeString"]
		lemmas = result["regions"][0]["resultAccess"]["NormalizedNamedEntity"]
		chain_ids = result["regions"][0]["resultAccess"]["NamedEntityChainId"]

		results = zip(lemmas, types, chain_ids)

		for lemma, entity_type, chain_id in results:
			# take the first element of the results triple where chain_id is pointing to.
			key = results[chain_id][0]
			entities[key] = convert_label(entity_type)
	except Exception as e:
		print "BasisTech API Error:", e
		print "BasisTech Response:", rp.content
	return entities<|endoftext|>