<commit_before>import sublime
import sys

class CrossPlaformCodecs():
    @classmethod
    def decode_line(self, line):
        line = line.rstrip()
        decoded_line = self.force_decode(line) if sys.version_info >= (3, 0) else line
        return str(decoded_line) + "\n"

    @classmethod
    def force_decode(self, text):
        try:
            text = text.decode('utf-8')
        except UnicodeDecodeError:
            if sublime.platform() == "windows":
                text = self.decode_windows_line(text)
        return text

    @classmethod
    def decode_windows_line(self, text):
        # Import only for Windows
        import locale, subprocess

        # STDERR gets the wrong encoding, use chcp to get the real one
        proccess = subprocess.Popen(["chcp"], shell=True, stdout=subprocess.PIPE)
        (chcp, _) = proccess.communicate()

        # Decode using the locale preferred encoding (for example 'cp1251') and remove newlines
        chcp = chcp.decode(locale.getpreferredencoding()).strip()

        # Get the actual number
        chcp = chcp.split(" ")[-1]

        # Actually decode
        return text.decode("cp" + chcp)

    @classmethod
    def encode_process_command(self, command):
        is_sublime_2_and_in_windows = sublime.platform() == "windows" and int(sublime.version()) < 3000
        return command.encode(sys.getfilesystemencoding()) if is_sublime_2_and_in_windows else command<commit_msg>Remove terminal Unicode Escape codes<commit_after>import sublime
import sys
import re

class CrossPlaformCodecs():
    @classmethod
    def decode_line(self, line):
        line = line.rstrip()
        decoded_line = self.force_decode(line) if sys.version_info >= (3, 0) else line
        decoded_line = re.sub(r'\033\[(\d{1,2}m|\d\w)', '', str(decoded_line))
        return decoded_line + "\n"

    @classmethod
    def force_decode(self, text):
        try:
            text = text.decode('utf-8')
        except UnicodeDecodeError:
            if sublime.platform() == "windows":
                text = self.decode_windows_line(text)
        return text

    @classmethod
    def decode_windows_line(self, text):
        # Import only for Windows
        import locale, subprocess

        # STDERR gets the wrong encoding, use chcp to get the real one
        proccess = subprocess.Popen(["chcp"], shell=True, stdout=subprocess.PIPE)
        (chcp, _) = proccess.communicate()

        # Decode using the locale preferred encoding (for example 'cp1251') and remove newlines
        chcp = chcp.decode(locale.getpreferredencoding()).strip()

        # Get the actual number
        chcp = chcp.split(" ")[-1]

        # Actually decode
        return text.decode("cp" + chcp)

    @classmethod
    def encode_process_command(self, command):
        is_sublime_2_and_in_windows = sublime.platform() == "windows" and int(sublime.version()) < 3000
        return command.encode(sys.getfilesystemencoding()) if is_sublime_2_and_in_windows else command<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#   Website info gathering
#

# TODO collect WHOIS information
# TODO extract meta information from homepage (head, meta)
# TODO add sitemap.xml support
# TODO add choice to select different report types about website
# TODO add choice to select different report format (text, json, html) about website
# TODO read robots.txt from string

__author__  = 'Andrey Usov <https://github.com/ownport/pywsinfo>'
__version__ = '0.1'

import re
import socket
import pprint
import requests
import urlparse
import datetime

# default setting for python-requests
REQUESTS_DEFAULTS = {}

def parse_url(url):
    ''' parse website url, remove path if exists '''
    url_parts = urlparse.urlparse(url)
    return {
            'source_url': urlparse.urlunsplit((url_parts.scheme,url_parts.netloc,'','','')),
            'host': url_parts.netloc
    }

def nslookup(host):
    ''' returns result of DNS lookup '''
    # return socket.gethostbyname(host)
    try:
        return socket.gethostbyname_ex(host)[2]
    except:
        return []

def parse_html_head(content):
    ''' parse HTML head, extract keywords & description '''

    result = dict()

    content = content.replace('\r', '')
    content = content.replace('\n', '')
    # select HEAD section
    head = ''.join(re.findall(r'<head(.+?)', content, re.I))
    # extract title information
    title = ''.join(re.findall(r'<title>(.+?)</title>', content, re.I))
    if title:
        result['title'] = title
    # select meta information: keywords and description
    metas = re.findall(r'<meta(.+?)[/]?>', head, re.I)
    for meta in metas:
        meta_dict = dict(re.findall(r'(\w+)\s*=\s*"(.+?)"', meta, re.I))
        if 'name' in meta_dict and 'content' in meta_dict:
            if meta_dict['name'].lower() == 'keywords':
                result['keywords'] = [c.strip() for c in meta_dict['content'].split(',')]
            if meta_dict['name'].lower() == 'description':
                result['description'] = meta_dict['content']
    return result

class WebsiteInfo(object):
    ''' website information '''

    def __init__(self, website_url):
        self._url = website_url
        self._homepage_content = None
        self._details = parse_url(self._url)

    def _make_request(self, method, url):
        ''' make request to website '''
        result = dict()
        resp = requests.request(method, url)

        result['final_url'] = resp.url
        result['status_code'] = resp.status_code
        if 'server' in resp.headers:
            result['server'] = resp.headers['server']
        if 'powered-by' in resp.headers:
            result['powered-by'] = resp.headers['x-powered-by']

        if resp.content:
            result['content'] = resp.content

        return result

    def run(self):
        ''' run website info retrieval '''

        # nslookup
        self._details['ip_addreses'] = nslookup(self._details['host'])

        # first request to web site. Try to detect
        if len(self._details['ip_addreses']) > 0:
            resp = self._make_request('GET', self._details['source_url'])
            for k in resp.keys():
                if k == 'content':
                    head_params = parse_html_head(resp[k])
                    for k in head_params:
                        self._details[k] = head_params[k]
                else:
                    self._details[k] = resp[k]

        # check robots.txt
        if self._details.get('status_code') == 200:
            robots_url = urlparse.urljoin(self._details['final_url'],'/robots.txt')
            resp = self._make_request('GET', robots_url)
            if resp['status_code'] == 200 and 'content' in resp:
                self._details['robots.txt'] = resp['content']
                # TODO extract link to sitemap.xml
                for line in self._details['robots.txt'].split('\n'):
                    if line.startswith('Sitemap'):
                        if 'sitemaps' not in self._details:
                            self._details['sitemaps'] = list()
                        self._details['sitemaps'].append(line.replace('Sitemap:', '').strip())

        # check default sitemap
        if 'sitemaps' not in self._details:
            sitemaps_url = urlparse.urljoin(self._details['final_url'],'/sitemap.xml')
            resp = self._make_request('HEAD', sitemaps_url)
            if resp['status_code'] == 200:
                self._details['sitemaps'] = sitemaps_url

        # latest update datetime
        self._details['last_update'] = str(datetime.datetime.now())

    def report(self):
        ''' website report '''

        pprint.pprint(self._details)

if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser(description='website info extraction')
    parser.add_argument('-u', '--url', help='website url')

    args = parser.parse_args()

    if args.url:
        wsinfo_process = WebsiteInfo(args.url)
        wsinfo_process.run()
        wsinfo_process.report()
    else:
        parser.print_help()

<commit_msg>starting to work with SitemapParser<commit_after>#!/usr/bin/env python
#
#   Website info gathering
#

# TODO add test server
# TODO change default user agent
# TODO collect WHOIS information
# TODO extract meta information from homepage (head, meta)
# TODO add sitemap.xml support
# TODO add choice to select different report types about website
# TODO add choice to select different report format (text, json, html) about website
# TODO read robots.txt from string

__author__  = 'Andrey Usov <https://github.com/ownport/pywsinfo>'
__version__ = '0.1'

import re
import socket
import pprint
import requests
import urlparse
import datetime

from gzip import GzipFile
from cStringIO import StringIO

# default setting for python-requests
REQUESTS_DEFAULTS = {}

def parse_url(url):
    ''' parse website url, remove path if exists '''
    url_parts = urlparse.urlparse(url)
    return {
            'source_url': urlparse.urlunsplit((url_parts.scheme,url_parts.netloc,'','','')),
            'host': url_parts.netloc
    }

def nslookup(host):
    ''' returns result of DNS lookup '''
    # return socket.gethostbyname(host)
    try:
        return socket.gethostbyname_ex(host)[2]
    except:
        return []

def parse_html_head(content):
    ''' parse HTML head, extract keywords & description '''

    result = dict()

    content = content.replace('\r', '')
    content = content.replace('\n', '')
    # select HEAD section
    head = ''.join(re.findall(r'<head(.+?)', content, re.I))
    # extract title information
    title = ''.join(re.findall(r'<title>(.+?)</title>', content, re.I))
    if title:
        result['title'] = title
    # select meta information: keywords and description
    metas = re.findall(r'<meta(.+?)[/]?>', head, re.I)
    for meta in metas:
        meta_dict = dict(re.findall(r'(\w+)\s*=\s*"(.+?)"', meta, re.I))
        if 'name' in meta_dict and 'content' in meta_dict:
            if meta_dict['name'].lower() == 'keywords':
                result['keywords'] = [c.strip() for c in meta_dict['content'].split(',')]
            if meta_dict['name'].lower() == 'description':
                result['description'] = meta_dict['content']
    return result

class SitemapParser(object):

    def __init__(self, sitemap):
        ''' init

        sitemap can be as one url or as list of urls to sitemaps
        '''
        self._sitemap_urls = list()
        if isinstance(sitemap, (str,unicode)):
            self._sitemap_urls.append(sitemap)
        elif isinstance(sitemap, (list,tuple)):
            self._sitemap_urls.extend(sitemap)
        else:
            raise RuntimeError('Unknow sitemap type: {}'.format(type(sitemap)))

    def _get(self, url):
        ''' get sitemap, if it compressed -> decompress'''
        SUPPORTED_PLAIN_CONTENT_TYPE = (
            'text/xml', 'application/xml',
        )
        SUPPORTED_COMPESSES_CONTENT_TYPE = (
            'application/octet-stream', 'application/x-gzip',
        )
        resp = requests.get(url)
        if resp.headers['content-type'].lower() in SUPPORTED_PLAIN_CONTENT_TYPE:
            return resp.content
        elif resp.headers['content-type'].lower() in SUPPORTED_COMPESSES_CONTENT_TYPE:
            return GzipFile(fileobj=StringIO(resp.content)).read()

    def _parse_sitemap(self, sitemap):
        ''' parse sitemap
        and return the list of (loc, lastmod, priority)'''
        print len(sitemap)

    def parse(self):
        ''' parse sitemap, if there's more than one sitemap url, the data will be merged '''
        for url in self._sitemap_urls:
            content = self._parse_sitemap(self._get(url))
            print content

class WebsiteInfo(object):
    ''' website information '''

    def __init__(self, website_url):
        self._url = website_url
        self._homepage_content = None
        self._details = parse_url(self._url)

    def _make_request(self, method, url):
        ''' make request to website '''
        result = dict()
        resp = requests.request(method, url)

        result['final_url'] = resp.url
        result['status_code'] = resp.status_code
        if 'server' in resp.headers:
            result['server'] = resp.headers['server']
        if 'powered-by' in resp.headers:
            result['powered-by'] = resp.headers['x-powered-by']

        if resp.content:
            result['content'] = resp.content

        return result

    def run(self):
        ''' run website info retrieval '''

        # nslookup
        self._details['ip_addreses'] = nslookup(self._details['host'])

        # first request to web site. Try to detect
        if len(self._details['ip_addreses']) > 0:
            resp = self._make_request('GET', self._details['source_url'])
            for k in resp.keys():
                if k == 'content':
                    head_params = parse_html_head(resp[k])
                    for k in head_params:
                        self._details[k] = head_params[k]
                else:
                    self._details[k] = resp[k]

        # check robots.txt
        if self._details.get('status_code') == 200:
            robots_url = urlparse.urljoin(self._details['final_url'],'/robots.txt')
            resp = self._make_request('GET', robots_url)
            if resp['status_code'] == 200 and 'content' in resp:
                self._details['robots.txt'] = resp['content']
                # TODO extract link to sitemap.xml
                for line in self._details['robots.txt'].split('\n'):
                    if line.startswith('Sitemap'):
                        if 'sitemaps' not in self._details:
                            self._details['sitemaps'] = list()
                        self._details['sitemaps'].append(line.replace('Sitemap:', '').strip())

        # check default sitemap
        if 'sitemaps' not in self._details:
            sitemaps_url = urlparse.urljoin(self._details['final_url'],'/sitemap.xml')
            resp = self._make_request('HEAD', sitemaps_url)
            if resp['status_code'] == 200:
                self._details['sitemaps'] = sitemaps_url

        # latest update datetime
        self._details['last_update'] = str(datetime.datetime.now())

    def report(self):
        ''' website report '''

        pprint.pprint(self._details)

if __name__ == '__main__':

    import argparse
    parser = argparse.ArgumentParser(description='website info extraction')
    parser.add_argument('-u', '--url', help='website url')
    parser.add_argument('-s', '--sitemap', help='get sitemap content')

    args = parser.parse_args()

    if args.url:
        wsinfo_process = WebsiteInfo(args.url)
        wsinfo_process.run()
        wsinfo_process.report()
    elif args.sitemap:
        sitemap_parser = SitemapParser(args.sitemap)
        sitemap_parser.parse()
    else:
        parser.print_help()

<|endoftext|>


<commit_before>#
#

# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""OS related QA tests.

"""

import os
import os.path

from ganeti import utils
from ganeti import constants

import qa_config
import qa_utils
import qa_error

from qa_utils import AssertCommand, AssertIn, AssertNotIn

_TEMP_OS_NAME = "TEMP-Ganeti-QA-OS"
_TEMP_OS_PATH = os.path.join(constants.OS_SEARCH_PATH[0], _TEMP_OS_NAME)

(_ALL_VALID,
 _ALL_INVALID,
 _PARTIALLY_VALID) = range(1, 4)

def TestOsList():
  """gnt-os list"""
  AssertCommand(["gnt-os", "list"])

def TestOsDiagnose():
  """gnt-os diagnose"""
  AssertCommand(["gnt-os", "diagnose"])

def _TestOsModify(hvp_dict, fail=False):
  """gnt-os modify"""
  cmd = ["gnt-os", "modify"]

  for hv_name, hv_params in hvp_dict.items():
    cmd.append("-H")
    options = []
    for key, value in hv_params.items():
      options.append("%s=%s" % (key, value))
    cmd.append("%s:%s" % (hv_name, ",".join(options)))

  cmd.append(_TEMP_OS_NAME)
  AssertCommand(cmd, fail=fail)

def _TestOsStates(os_name):
  """gnt-os modify, more stuff"""
  cmd = ["gnt-os", "modify"]

  for param in ["hidden", "blacklisted"]:
    for val in ["yes", "no"]:
      new_cmd = cmd + ["--%s" % param, val, os_name]
      AssertCommand(new_cmd)
      # check that double-running the command is OK
      AssertCommand(new_cmd)

def _SetupTempOs(node, dirname, valid):
  """Creates a temporary OS definition on the given node.

  """
  sq = utils.ShellQuoteArgs
  parts = [
    sq(["rm", "-rf", dirname]),
    sq(["mkdir", "-p", dirname]),
    sq(["cd", dirname]),
    sq(["ln", "-fs", "/bin/true", "export"]),
    sq(["ln", "-fs", "/bin/true", "import"]),
    sq(["ln", "-fs", "/bin/true", "rename"]),
    sq(["ln", "-fs", "/bin/true", "verify"]),
    ]

  if valid:
    parts.append(sq(["ln", "-fs", "/bin/true", "create"]))

  parts.append(sq(["echo", str(constants.OS_API_V20)]) +
               " >ganeti_api_version")

  parts.append(sq(["echo", "default"]) + " >variants.list")
  parts.append(sq(["echo", "funny this is funny"]) + " >parameters.list")

  cmd = " && ".join(parts)

  print qa_utils.FormatInfo("Setting up %s with %s OS definition" %
                            (node["primary"],
                             ["an invalid", "a valid"][int(valid)]))

  AssertCommand(cmd, node=node)

def _RemoveTempOs(node, dirname):
  """Removes a temporary OS definition.

  """
  AssertCommand(["rm", "-rf", dirname], node=node)

def _TestOs(mode, rapi_cb):
  """Generic function for OS definition testing

  """
  master = qa_config.GetMasterNode()

  name = _TEMP_OS_NAME
  dirname = _TEMP_OS_PATH

  # Ensure OS is usable
  cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name]
  AssertCommand(cmd)

  nodes = []
  try:
    for i, node in enumerate(qa_config.get("nodes")):
      nodes.append(node)
      if mode == _ALL_INVALID:
        valid = False
      elif mode == _ALL_VALID:
        valid = True
      elif mode == _PARTIALLY_VALID:
        valid = bool(i % 2)
      else:
        raise AssertionError("Unknown mode %s" % mode)
      _SetupTempOs(node, dirname, valid)

    # TODO: Use Python 2.6's itertools.permutations
    for (hidden, blacklisted) in [(False, False), (True, False),
                                  (False, True), (True, True)]:
      # Change OS' visibility
      cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)],
             "--blacklisted", ["no", "yes"][int(blacklisted)], name]
      AssertCommand(cmd)

      # Diagnose, checking exit status
      AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID))

      # Diagnose again, ignoring exit status
      output = qa_utils.GetCommandOutput(master["primary"],
                                         "gnt-os diagnose || :")
      for line in output.splitlines():
        if line.startswith("OS: %s [global status:" % name):
          break
      else:
        raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name)

      # Check info for all
      cmd = ["gnt-os", "info"]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd))
      AssertIn("%s:" % name, output.splitlines())

      # Check info for OS
      cmd = ["gnt-os", "info", name]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd)).splitlines()
      AssertIn("%s:" % name, output)
      for (field, value) in [("valid", mode == _ALL_VALID),
                             ("hidden", hidden),
                             ("blacklisted", blacklisted)]:
        AssertIn("  - %s: %s" % (field, value), output)

      # Only valid OSes should be listed
      cmd = ["gnt-os", "list", "--no-headers"]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd))
      if mode == _ALL_VALID and not (hidden or blacklisted):
        assert_fn = AssertIn
      else:
        assert_fn = AssertNotIn
      assert_fn(name, output.splitlines())

      # Check via RAPI
      if rapi_cb:
        assert_fn(name, rapi_cb())
  finally:
    for node in nodes:
      _RemoveTempOs(node, dirname)

def TestOsValid(rapi_cb):
  """Testing valid OS definition"""
  return _TestOs(_ALL_VALID, rapi_cb)

def TestOsInvalid(rapi_cb):
  """Testing invalid OS definition"""
  return _TestOs(_ALL_INVALID, rapi_cb)

def TestOsPartiallyValid(rapi_cb):
  """Testing partially valid OS definition"""
  return _TestOs(_PARTIALLY_VALID, rapi_cb)

def TestOsModifyValid():
  """Testing a valid os modify invocation"""
  hv_dict = {
    constants.HT_XEN_PVM: {
      constants.HV_ROOT_PATH: "/dev/sda5",
      },
    constants.HT_XEN_HVM: {
      constants.HV_ACPI: False,
      constants.HV_PAE: True,
      },
    }

  return _TestOsModify(hv_dict)

def TestOsModifyInvalid():
  """Testing an invalid os modify invocation"""
  hv_dict = {
    "blahblahblubb": {"bar": ""},
    }

  return _TestOsModify(hv_dict, fail=True)

def TestOsStatesNonExisting():
  """Testing OS states with non-existing OS"""
  AssertCommand(["test", "-e", _TEMP_OS_PATH], fail=True)
  return _TestOsStates(_TEMP_OS_NAME)
<commit_msg>Fix QA OS API failure<commit_after>#
#

# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""OS related QA tests.

"""

import os
import os.path

from ganeti import utils
from ganeti import constants

import qa_config
import qa_utils
import qa_error

from qa_utils import AssertCommand, AssertIn, AssertNotIn

_TEMP_OS_NAME = "TEMP-Ganeti-QA-OS"
_TEMP_OS_PATH = os.path.join(constants.OS_SEARCH_PATH[0], _TEMP_OS_NAME)

(_ALL_VALID,
 _ALL_INVALID,
 _PARTIALLY_VALID) = range(1, 4)

def TestOsList():
  """gnt-os list"""
  AssertCommand(["gnt-os", "list"])

def TestOsDiagnose():
  """gnt-os diagnose"""
  AssertCommand(["gnt-os", "diagnose"])

def _TestOsModify(hvp_dict, fail=False):
  """gnt-os modify"""
  cmd = ["gnt-os", "modify"]

  for hv_name, hv_params in hvp_dict.items():
    cmd.append("-H")
    options = []
    for key, value in hv_params.items():
      options.append("%s=%s" % (key, value))
    cmd.append("%s:%s" % (hv_name, ",".join(options)))

  cmd.append(_TEMP_OS_NAME)
  AssertCommand(cmd, fail=fail)

def _TestOsStates(os_name):
  """gnt-os modify, more stuff"""
  cmd = ["gnt-os", "modify"]

  for param in ["hidden", "blacklisted"]:
    for val in ["yes", "no"]:
      new_cmd = cmd + ["--%s" % param, val, os_name]
      AssertCommand(new_cmd)
      # check that double-running the command is OK
      AssertCommand(new_cmd)

def _SetupTempOs(node, dirname, variant, valid):
  """Creates a temporary OS definition on the given node.

  """
  sq = utils.ShellQuoteArgs
  parts = [
    sq(["rm", "-rf", dirname]),
    sq(["mkdir", "-p", dirname]),
    sq(["cd", dirname]),
    sq(["ln", "-fs", "/bin/true", "export"]),
    sq(["ln", "-fs", "/bin/true", "import"]),
    sq(["ln", "-fs", "/bin/true", "rename"]),
    sq(["ln", "-fs", "/bin/true", "verify"]),
    ]

  if valid:
    parts.append(sq(["ln", "-fs", "/bin/true", "create"]))

  parts.append(sq(["echo", str(constants.OS_API_V20)]) +
               " >ganeti_api_version")

  parts.append(sq(["echo", variant]) + " >variants.list")
  parts.append(sq(["echo", "funny this is funny"]) + " >parameters.list")

  cmd = " && ".join(parts)

  print qa_utils.FormatInfo("Setting up %s with %s OS definition" %
                            (node["primary"],
                             ["an invalid", "a valid"][int(valid)]))

  AssertCommand(cmd, node=node)

def _RemoveTempOs(node, dirname):
  """Removes a temporary OS definition.

  """
  AssertCommand(["rm", "-rf", dirname], node=node)

def _TestOs(mode, rapi_cb):
  """Generic function for OS definition testing

  """
  master = qa_config.GetMasterNode()

  name = _TEMP_OS_NAME
  variant = "default"
  fullname = "%s+%s" % (name, variant)
  dirname = _TEMP_OS_PATH

  # Ensure OS is usable
  cmd = ["gnt-os", "modify", "--hidden=no", "--blacklisted=no", name]
  AssertCommand(cmd)

  nodes = []
  try:
    for i, node in enumerate(qa_config.get("nodes")):
      nodes.append(node)
      if mode == _ALL_INVALID:
        valid = False
      elif mode == _ALL_VALID:
        valid = True
      elif mode == _PARTIALLY_VALID:
        valid = bool(i % 2)
      else:
        raise AssertionError("Unknown mode %s" % mode)
      _SetupTempOs(node, dirname, variant, valid)

    # TODO: Use Python 2.6's itertools.permutations
    for (hidden, blacklisted) in [(False, False), (True, False),
                                  (False, True), (True, True)]:
      # Change OS' visibility
      cmd = ["gnt-os", "modify", "--hidden", ["no", "yes"][int(hidden)],
             "--blacklisted", ["no", "yes"][int(blacklisted)], name]
      AssertCommand(cmd)

      # Diagnose, checking exit status
      AssertCommand(["gnt-os", "diagnose"], fail=(mode != _ALL_VALID))

      # Diagnose again, ignoring exit status
      output = qa_utils.GetCommandOutput(master["primary"],
                                         "gnt-os diagnose || :")
      for line in output.splitlines():
        if line.startswith("OS: %s [global status:" % name):
          break
      else:
        raise qa_error.Error("Didn't find OS '%s' in 'gnt-os diagnose'" % name)

      # Check info for all
      cmd = ["gnt-os", "info"]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd))
      AssertIn("%s:" % name, output.splitlines())

      # Check info for OS
      cmd = ["gnt-os", "info", name]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd)).splitlines()
      AssertIn("%s:" % name, output)
      for (field, value) in [("valid", mode == _ALL_VALID),
                             ("hidden", hidden),
                             ("blacklisted", blacklisted)]:
        AssertIn("  - %s: %s" % (field, value), output)

      # Only valid OSes should be listed
      cmd = ["gnt-os", "list", "--no-headers"]
      output = qa_utils.GetCommandOutput(master["primary"],
                                         utils.ShellQuoteArgs(cmd))
      if mode == _ALL_VALID and not (hidden or blacklisted):
        assert_fn = AssertIn
      else:
        assert_fn = AssertNotIn
      assert_fn(fullname, output.splitlines())

      # Check via RAPI
      if rapi_cb:
        assert_fn(fullname, rapi_cb())
  finally:
    for node in nodes:
      _RemoveTempOs(node, dirname)

def TestOsValid(rapi_cb):
  """Testing valid OS definition"""
  return _TestOs(_ALL_VALID, rapi_cb)

def TestOsInvalid(rapi_cb):
  """Testing invalid OS definition"""
  return _TestOs(_ALL_INVALID, rapi_cb)

def TestOsPartiallyValid(rapi_cb):
  """Testing partially valid OS definition"""
  return _TestOs(_PARTIALLY_VALID, rapi_cb)

def TestOsModifyValid():
  """Testing a valid os modify invocation"""
  hv_dict = {
    constants.HT_XEN_PVM: {
      constants.HV_ROOT_PATH: "/dev/sda5",
      },
    constants.HT_XEN_HVM: {
      constants.HV_ACPI: False,
      constants.HV_PAE: True,
      },
    }

  return _TestOsModify(hv_dict)

def TestOsModifyInvalid():
  """Testing an invalid os modify invocation"""
  hv_dict = {
    "blahblahblubb": {"bar": ""},
    }

  return _TestOsModify(hv_dict, fail=True)

def TestOsStatesNonExisting():
  """Testing OS states with non-existing OS"""
  AssertCommand(["test", "-e", _TEMP_OS_PATH], fail=True)
  return _TestOsStates(_TEMP_OS_NAME)
<|endoftext|>


<commit_before>#
# centroid.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.
from __future__ import division

class Centroid(object):
    '''Class to centroid data'''

    def __init__(self, pixels, mask=None, coords=None):
        '''Centroid the data.

        If coords is set then the centroid of the list of coordinates is done.
        Otherwise, if mask is set the the centroid of the masked image is done.
        Otherwise, a simple centroid of the image is done.

        Params:
            pixels The image (or list of pixel values) to centroid
            mask The mask of the image (optional)
            coords The coordianates for each pixel value

        Returns:
            The centroid data

        '''
        if coords:
            return self._centroid_coords(image, coords)
        else if mask:
            return self._centroid_mask(image, mask)
        else:
            return self._centroid_basic(image)

        def _centroid_coords(self, image, coords):
            from dials.image.centroid import CentroidList2d, CentroidList3d
            if isinstance(coords, flex.vec2_double):
                return CentroidList2d(image, coords)
            elif isinstance(coords, flex.vec3_double):
                return CentroidList3d(image, coords)
            else:
                raise RuntimeError("Bad coordinate type")

        def _centroid_mask(self, image, mask):
            from dials.image.centroid import MaskedCentroid2d, MaskedCentroid3d
            if len(image.all() == 2):
                return MaskedCentroid2d(image, mask)
            elif len(image.all() == 3):
                return MaskedCentroid3d(image, mask)
            else:
                raise RuntimeError("Bad dimensions")

        def _centroid_basic(self, image):
            from dials.image.centroid import Centroid2d, Centroid3d
            if len(image.all() == 2):
                return Centroid2d(image, mask)
            elif len(image.all() == 3):
                return Centroid3d(image, mask)
            else:
                raise RuntimeError("Bad dimensions")
<commit_msg>Added python code to get centroid in mm and pixels<commit_after>#
# centroid.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.
from __future__ import division

class Centroid(object):
    '''Class to centroid data'''

    def __init__(self, pixels, mask=None, coords=None):
        '''Centroid the data.

        If coords is set then the centroid of the list of coordinates is done.
        Otherwise, if mask is set the the centroid of the masked image is done.
        Otherwise, a simple centroid of the image is done.

        Params:
            pixels The image (or list of pixel values) to centroid
            mask The mask of the image (optional)
            coords The coordianates for each pixel value (optional)

        Returns:
            The centroid data

        '''
        # Choose the algorithm
        if coords:
            result self._centroid_coords(image, coords)
        elif mask:
            result self._centroid_mask(image, mask)
        else:
            result self._centroid_basic(image)

        # Extract the result quantities
        self.position = result.position
        self.variance = result.variance
        self.sq_width = result.sq_width

        def _centroid_coords(self, image, coords):
            '''Do the centroid on a list of coordinates.'''
            from dials.image.centroid import centroid_coord_list
            return centroid_coord_list(image, coords)

        def _centroid_mask(self, image, mask):
            '''Do the centroid on an image and mask.'''
            from dials.image.centroid import MaskedCentroid2d, MaskedCentroid3d
            if len(image.all() == 2):
                return MaskedCentroid2d(image, mask)
            elif len(image.all() == 3):
                return MaskedCentroid3d(image, mask)
            else:
                raise RuntimeError("Bad dimensions")

        def _centroid_basic(self, image):
            '''Do the centroid on an image.'''
            from dials.image.centroid import Centroid2d, Centroid3d
            if len(image.all() == 2):
                return Centroid2d(image, mask)
            elif len(image.all() == 3):
                return Centroid3d(image, mask)
            else:
                raise RuntimeError("Bad dimensions")

class CentroidPxAndMM(object):
    '''Class to calculate centroid in pixels and millimeters'''

    def __init__(self, detector, pixels, mask=None, coords=None):
        '''Calculate the centroid in pixels and millimeters.'''

        # Get the pixel to millimeter function
        pixel_to_mm = detector.pixel_to_millimeter

        # Calculate the centroid and extract the values
        centroid = Centroid(pixels, mask, coords)
        self.position_px = centroid.position
        self.variance_px = centroid.variance
        self.sq_width_px = centroid.sq_width

        # Get the centroid in millimeters
        self.position_mm = pixel_to_mm(self.position_px)
        self.variance_mm = pixel_to_mm(self.position_px)
        self.sq_width_mm = pixel_to_mm(self.position_px)
<|endoftext|>


<commit_before># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class AtSpi2Core(MesonPackage):
    """The At-Spi2 Core package provides a Service Provider Interface for the
       Assistive Technologies available on the GNOME platform and a library
       against which applications can be linked."""

    homepage = "http://www.linuxfromscratch.org/blfs/view/cvs/x/at-spi2-core.html"
    url      = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-core/2.28/at-spi2-core-2.28.0.tar.xz"
    list_url = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-core"
    list_depth = 1

    version('2.28.0', '9c42f79636ed1c0e908b7483d789b32e')

    depends_on('glib@2.56.1:')
    depends_on('dbus@1.12.8:')
    depends_on('libx11')
    depends_on('libxi')
    depends_on('libxtst', type='build')
    depends_on('recordproto', type='build')
    depends_on('inputproto', type='build')
    depends_on('fixesproto', type='build')
    depends_on('pkgconfig', type='build')
    depends_on('python', type='build')

    def url_for_version(self, version):
        """Handle gnome's version-based custom URLs."""
        url = 'http://ftp.gnome.org/pub/gnome/sources/at-spi2-core'
        return url + '/%s/at-spi2-core-%s.tar.xz' % (version.up_to(2), version)

    def setup_environment(self, spack_env, run_env):
        # this avoids an "import site" error in the build
        spack_env.unset('PYTHONHOME')
<commit_msg>Add missing gettext dependency to at-spi2-core (#10124)<commit_after># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class AtSpi2Core(MesonPackage):
    """The At-Spi2 Core package provides a Service Provider Interface for the
       Assistive Technologies available on the GNOME platform and a library
       against which applications can be linked."""

    homepage = "http://www.linuxfromscratch.org/blfs/view/cvs/x/at-spi2-core.html"
    url      = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-core/2.28/at-spi2-core-2.28.0.tar.xz"
    list_url = "http://ftp.gnome.org/pub/gnome/sources/at-spi2-core"
    list_depth = 1

    version('2.28.0', '9c42f79636ed1c0e908b7483d789b32e')

    depends_on('glib@2.56.1:')
    depends_on('dbus@1.12.8:')
    depends_on('gettext')
    depends_on('libx11')
    depends_on('libxi')
    depends_on('libxtst', type='build')
    depends_on('recordproto', type='build')
    depends_on('inputproto', type='build')
    depends_on('fixesproto', type='build')
    depends_on('pkgconfig', type='build')
    depends_on('python', type='build')

    def url_for_version(self, version):
        """Handle gnome's version-based custom URLs."""
        url = 'http://ftp.gnome.org/pub/gnome/sources/at-spi2-core'
        return url + '/%s/at-spi2-core-%s.tar.xz' % (version.up_to(2), version)

    def setup_environment(self, spack_env, run_env):
        # this avoids an "import site" error in the build
        spack_env.unset('PYTHONHOME')
<|endoftext|>


<commit_before>import operator
from functools import reduce
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.template import RequestContext, loader
from django.views.generic import CreateView, View, ListView, DetailView, UpdateView
from django.shortcuts import render, redirect, get_object_or_404
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.db.models import Q

from qa.models import (UserQAProfile, Question, Answer, AnswerVote,
                       QuestionVote, AnswerComment, QuestionComment)
from .mixins import LoginRequired

'''
Dear maintainer:

Once you are done trying to 'optimize' this routine, and have realized what a
terrible mistake that was, please increment the following counter as a warning
to the next guy:

total_hours_wasted_here = 2
'''

class QuestionIndexView(ListView):
    '''CBV to render the index view
    '''
    model = Question
    paginate_by = 10
    context_object_name = 'questions'
    template_name = 'qa/index.html'

    def get_context_data(self, *args, **kwargs):
        context = super(
            QuestionIndexView, self).get_context_data(*args, **kwargs)
        context['totalcount'] = Question.objects.count
        context['anscount'] = Answer.objects.count
        context['noans'] = Question.objects.order_by('-pub_date').filter(
            answer__isnull=True)[:10]
        context['reward'] = Question.objects.order_by('-reward').filter(
            answer__isnull=True, reward__gte=1)[:10]
        return context

class QuestionsSearchView(QuestionIndexView):
    """
    Display a ListView page inherithed from the QuestionIndexView filtered by
    the search query and sorted by the different elements aggregated.
    """

    def get_queryset(self):
        result = super(QuestionsSearchView, self).get_queryset()
        query = self.request.GET.get('word')
        if query:
            query_list = query.split()
            result = result.filter(
                reduce(operator.and_,
                       (Q(title__icontains=q) for q in query_list)) |
                reduce(operator.and_,
                       (Q(description__icontains=q) for q in query_list))
            )

        return result

        def get_context_data(self, *args, **kwargs):
            context = super(
                QuestionsSearchView, self).get_context_data(*args, **kwargs)
            context['totalcount'] = Question.objects.count
            context['anscount'] = Answer.objects.count
            context['noans'] = Question.objects.order_by('-pub_date').filter(
                answer__isnull=True)[:10]
            context['reward'] = Question.objects.order_by('-reward').filter(
                answer__isnull=True, reward__gte=1)[:10]
            return context

class QuestionsByTagView(ListView):
    '''View to call all the questions clasiffied under one specific tag.
    '''
    model = Question
    paginate_by = 10
    context_object_name = 'questions'
    template_name = 'qa/index.html'

    def get_queryset(self, **kwargs):
        return Question.objects.filter(tags__name__contains=self.kwargs['tag'])

    def get_context_data(self, *args, **kwargs):
        context = super(
            QuestionsByTagView, self).get_context_data(*args, **kwargs)
        context['totalcount'] = Question.objects.count
        context['anscount'] = Answer.objects.count
        context['noans'] = Question.objects.order_by('-pub_date').filter(
            tags__name__contains=self.kwargs['tag'], answer__isnull=True)[:10]
        context['reward'] = Question.objects.order_by('-reward').filter(
            tags__name__contains=self.kwargs['tag'], answer__isnull=True,
            reward__gte=1)[:10]
        return context

class CreateQuestionView(LoginRequired, CreateView):
    """
    View to handle the creation of a new question
    """
    template_name = 'qa/create_question.html'
    model = Question
    fields = ['title', 'description', 'tags']

    def form_valid(self, form):
        """
        Create the required relation
        """
        form.instance.user = self.request.user
        return super(CreateQuestionView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_index')

class UpdateQuestionView(LoginRequired, UpdateView):
    """
    Updates the question
    """
    template_name = 'qa/update_question.html'
    model = Question
    pk_url_kwarg = 'question_id'
    fields = ['title', 'description', 'tags']

    def get_success_url(self):
        question = self.get_object()
        return reverse('qa_detail', kwargs={'pk': question.pk})

class CreateAnswerView(LoginRequired, CreateView):
    """
    View to create new answers for a given question
    """
    template_name = 'qa/create_answer.html'
    model = Answer
    fields = ['answer_text']

    def form_valid(self, form):
        """
        Creates the required relationship between answer
        and user/question
        """
        form.instance.user = self.request.user
        form.instance.question_id = self.kwargs['question_id']
        return super(CreateAnswerView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_detail', kwargs={'pk': self.kwargs['question_id']})

class UpdateAnswerView(LoginRequired, UpdateView):
    """
    Updates the question answer
    """
    template_name = 'qa/update_answer.html'
    model = Answer
    pk_url_kwarg = 'answer_id'
    fields = ['answer_text']

    def get_success_url(self):
        answer = self.get_object()
        return reverse('qa_detail', kwargs={'pk': answer.question.pk})

class CreateAnswerCommentView(LoginRequired, CreateView):
    """
    View to create new comments for a given answer
    """
    template_name = 'qa/create_comment.html'
    model = AnswerComment
    fields = ['comment_text']

    def form_valid(self, form):
        """
        Creates the required relationship between answer
        and user/comment
        """
        form.instance.user = self.request.user
        form.instance.answer_id = self.kwargs['answer_id']
        return super(CreateAnswerCommentView, self).form_valid(form)

    def get_success_url(self):
        question_pk = Answer.objects.get(
            id=self.kwargs['answer_id']).question.pk
        return reverse('qa_detail', kwargs={'pk': question_pk})

class CreateQuestionCommentView(LoginRequired, CreateView):
    """
    View to create new comments for a given question
    """
    template_name = 'qa/create_comment.html'
    model = QuestionComment
    fields = ['comment_text']

    def form_valid(self, form):
        """
        Creates the required relationship between question
        and user/comment
        """
        form.instance.user = self.request.user
        form.instance.question_id = self.kwargs['question_id']
        return super(CreateQuestionCommentView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_detail', kwargs={'pk': self.kwargs['question_id']})

class UpdateQuestionCommentView(LoginRequired, UpdateView):
    """
    Updates the comment question
    """
    template_name = 'qa/create_comment.html'
    model = QuestionComment
    pk_url_kwarg = 'comment_id'
    fields = ['comment_text']

    def get_success_url(self):
        question_comment = self.get_object()
        return reverse(
            'qa_detail', kwargs={'pk': question_comment.question.pk})

class UpdateAnswerCommentView(UpdateQuestionCommentView):
    """
    Updates the comment answer
    """
    model = AnswerComment

    def get_success_url(self):
        answer_comment = self.get_object()
        return reverse(
            'qa_detail', kwargs={'pk': answer_comment.answer.question.pk})

class QuestionDetailView(DetailView):
    """
    View to call a question and to render all the details about that question.
    """
    model = Question
    template_name = 'qa/detail_question.html'
    context_object_name = 'question'

    def get_context_data(self, **kwargs):
        context = super(QuestionDetailView, self).get_context_data(**kwargs)
        context['last_comments'] = self.object.questioncomment_set.order_by(
            'pub_date')[:5]
        context['answers'] = self.object.answer_set.all().order_by('pub_date')
        return context

    def get_object(self):
        # Call the superclass
        question = super(QuestionDetailView, self).get_object()
        question.views += 1
        question.save()
        return question

class ParentVoteView(View):
    """
    Base class to create a vote for a given model (question/answer)
    """
    model = None
    vote_model = None

    def get_vote_kwargs(self, user, vote_target):
        """
        This takes the user and the vote and adjusts the kwargs
        depending on the used model.
        """
        object_kwargs = {'user': user}
        if self.model == Question:
            target_key = 'question'
        elif self.model == Answer:
            target_key = 'answer'
        else:
            raise ValidationError('Not a valid model for votes')
        object_kwargs[target_key] = vote_target
        return object_kwargs

    def post(self, request, object_id):
        vote_target = get_object_or_404(self.model, pk=object_id)
        if vote_target.user == request.user:
            raise ValidationError(
                'Sorry, voting for your own answer is not possible.')
        else:
            upvote = request.POST.get('upvote', None) is not None
            object_kwargs = self.get_vote_kwargs(request.user, vote_target)
            vote, created = self.vote_model.objects.get_or_create(
                defaults={'value': upvote},
                **object_kwargs)
            if created:
                vote_target.user.userqaprofile.points += 1 if upvote else -1
            elif not created:
                if vote.value == upvote:
                    vote.delete()
                    vote_target.user.userqaprofile.points += -1 if upvote else 1
                else:
                    vote_target.user.userqaprofile.points += 2 if upvote else -2
                    vote.value = upvote
                    vote.save()
            vote_target.user.userqaprofile.save()
            vote_target.save()

        next_url = request.POST.get('next', None)
        if next_url is not None:
            return redirect(next_url)
        else:
            return redirect(reverse('qa_index'))

class AnswerVoteView(ParentVoteView):
    """
    Class to upvote answers
    """
    model = Answer
    vote_model = AnswerVote

class QuestionVoteView(ParentVoteView):
    """
    Class to upvote questions
    """
    model = Question
    vote_model = QuestionVote

def profile(request, user_id):
    user_ob = get_user_model().objects.get(id=user_id)
    user = UserQAProfile.objects.get(user=user_ob)
    return render(request, 'qa/profile.html', {'user': user})
<commit_msg>added pagination to answers at question detail<commit_after>import operator
from functools import reduce
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.template import RequestContext, loader
from django.views.generic import CreateView, View, ListView, DetailView, UpdateView
from django.shortcuts import render, redirect, get_object_or_404
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.db.models import Q

from qa.models import (UserQAProfile, Question, Answer, AnswerVote,
                       QuestionVote, AnswerComment, QuestionComment)
from .mixins import LoginRequired

'''
Dear maintainer:

Once you are done trying to 'optimize' this routine, and have realized what a
terrible mistake that was, please increment the following counter as a warning
to the next guy:

total_hours_wasted_here = 2
'''

class QuestionIndexView(ListView):
    '''CBV to render the index view
    '''
    model = Question
    paginate_by = 10
    context_object_name = 'questions'
    template_name = 'qa/index.html'

    def get_context_data(self, *args, **kwargs):
        context = super(
            QuestionIndexView, self).get_context_data(*args, **kwargs)
        context['totalcount'] = Question.objects.count
        context['anscount'] = Answer.objects.count
        context['noans'] = Question.objects.order_by('-pub_date').filter(
            answer__isnull=True)[:10]
        context['reward'] = Question.objects.order_by('-reward').filter(
            answer__isnull=True, reward__gte=1)[:10]
        return context

class QuestionsSearchView(QuestionIndexView):
    """
    Display a ListView page inherithed from the QuestionIndexView filtered by
    the search query and sorted by the different elements aggregated.
    """

    def get_queryset(self):
        result = super(QuestionsSearchView, self).get_queryset()
        query = self.request.GET.get('word')
        if query:
            query_list = query.split()
            result = result.filter(
                reduce(operator.and_,
                       (Q(title__icontains=q) for q in query_list)) |
                reduce(operator.and_,
                       (Q(description__icontains=q) for q in query_list))
            )

        return result

        def get_context_data(self, *args, **kwargs):
            context = super(
                QuestionsSearchView, self).get_context_data(*args, **kwargs)
            context['totalcount'] = Question.objects.count
            context['anscount'] = Answer.objects.count
            context['noans'] = Question.objects.order_by('-pub_date').filter(
                answer__isnull=True)[:10]
            context['reward'] = Question.objects.order_by('-reward').filter(
                answer__isnull=True, reward__gte=1)[:10]
            return context

class QuestionsByTagView(ListView):
    '''View to call all the questions clasiffied under one specific tag.
    '''
    model = Question
    paginate_by = 10
    context_object_name = 'questions'
    template_name = 'qa/index.html'

    def get_queryset(self, **kwargs):
        return Question.objects.filter(tags__name__contains=self.kwargs['tag'])

    def get_context_data(self, *args, **kwargs):
        context = super(
            QuestionsByTagView, self).get_context_data(*args, **kwargs)
        context['totalcount'] = Question.objects.count
        context['anscount'] = Answer.objects.count
        context['noans'] = Question.objects.order_by('-pub_date').filter(
            tags__name__contains=self.kwargs['tag'], answer__isnull=True)[:10]
        context['reward'] = Question.objects.order_by('-reward').filter(
            tags__name__contains=self.kwargs['tag'], answer__isnull=True,
            reward__gte=1)[:10]
        return context

class CreateQuestionView(LoginRequired, CreateView):
    """
    View to handle the creation of a new question
    """
    template_name = 'qa/create_question.html'
    model = Question
    fields = ['title', 'description', 'tags']

    def form_valid(self, form):
        """
        Create the required relation
        """
        form.instance.user = self.request.user
        return super(CreateQuestionView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_index')

class UpdateQuestionView(LoginRequired, UpdateView):
    """
    Updates the question
    """
    template_name = 'qa/update_question.html'
    model = Question
    pk_url_kwarg = 'question_id'
    fields = ['title', 'description', 'tags']

    def get_success_url(self):
        question = self.get_object()
        return reverse('qa_detail', kwargs={'pk': question.pk})

class CreateAnswerView(LoginRequired, CreateView):
    """
    View to create new answers for a given question
    """
    template_name = 'qa/create_answer.html'
    model = Answer
    fields = ['answer_text']

    def form_valid(self, form):
        """
        Creates the required relationship between answer
        and user/question
        """
        form.instance.user = self.request.user
        form.instance.question_id = self.kwargs['question_id']
        return super(CreateAnswerView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_detail', kwargs={'pk': self.kwargs['question_id']})

class UpdateAnswerView(LoginRequired, UpdateView):
    """
    Updates the question answer
    """
    template_name = 'qa/update_answer.html'
    model = Answer
    pk_url_kwarg = 'answer_id'
    fields = ['answer_text']

    def get_success_url(self):
        answer = self.get_object()
        return reverse('qa_detail', kwargs={'pk': answer.question.pk})

class CreateAnswerCommentView(LoginRequired, CreateView):
    """
    View to create new comments for a given answer
    """
    template_name = 'qa/create_comment.html'
    model = AnswerComment
    fields = ['comment_text']

    def form_valid(self, form):
        """
        Creates the required relationship between answer
        and user/comment
        """
        form.instance.user = self.request.user
        form.instance.answer_id = self.kwargs['answer_id']
        return super(CreateAnswerCommentView, self).form_valid(form)

    def get_success_url(self):
        question_pk = Answer.objects.get(
            id=self.kwargs['answer_id']).question.pk
        return reverse('qa_detail', kwargs={'pk': question_pk})

class CreateQuestionCommentView(LoginRequired, CreateView):
    """
    View to create new comments for a given question
    """
    template_name = 'qa/create_comment.html'
    model = QuestionComment
    fields = ['comment_text']

    def form_valid(self, form):
        """
        Creates the required relationship between question
        and user/comment
        """
        form.instance.user = self.request.user
        form.instance.question_id = self.kwargs['question_id']
        return super(CreateQuestionCommentView, self).form_valid(form)

    def get_success_url(self):
        return reverse('qa_detail', kwargs={'pk': self.kwargs['question_id']})

class UpdateQuestionCommentView(LoginRequired, UpdateView):
    """
    Updates the comment question
    """
    template_name = 'qa/create_comment.html'
    model = QuestionComment
    pk_url_kwarg = 'comment_id'
    fields = ['comment_text']

    def get_success_url(self):
        question_comment = self.get_object()
        return reverse(
            'qa_detail', kwargs={'pk': question_comment.question.pk})

class UpdateAnswerCommentView(UpdateQuestionCommentView):
    """
    Updates the comment answer
    """
    model = AnswerComment

    def get_success_url(self):
        answer_comment = self.get_object()
        return reverse(
            'qa_detail', kwargs={'pk': answer_comment.answer.question.pk})

class QuestionDetailView(DetailView):
    """
    View to call a question and to render all the details about that question.
    """
    model = Question
    template_name = 'qa/detail_question.html'
    context_object_name = 'question'

    def get_context_data(self, **kwargs):
        answers_list = self.object.answer_set.all().order_by('pub_date')
        paginator = Paginator(answers_list, 10)
        page = self.request.GET.get('page')
        try:
            answers = paginator.page(page)
        except PageNotAnInteger:
            answers = paginator.page(1)
        except EmptyPage:
            answers = paginator.page(paginator.num_pages)
        context = super(QuestionDetailView, self).get_context_data(**kwargs)
        context['last_comments'] = self.object.questioncomment_set.order_by(
            'pub_date')[:5]
        context['answers'] = answers
        return context

    def get_object(self):
        # Call the superclass
        question = super(QuestionDetailView, self).get_object()
        question.views += 1
        question.save()
        return question

class ParentVoteView(View):
    """
    Base class to create a vote for a given model (question/answer)
    """
    model = None
    vote_model = None

    def get_vote_kwargs(self, user, vote_target):
        """
        This takes the user and the vote and adjusts the kwargs
        depending on the used model.
        """
        object_kwargs = {'user': user}
        if self.model == Question:
            target_key = 'question'
        elif self.model == Answer:
            target_key = 'answer'
        else:
            raise ValidationError('Not a valid model for votes')
        object_kwargs[target_key] = vote_target
        return object_kwargs

    def post(self, request, object_id):
        vote_target = get_object_or_404(self.model, pk=object_id)
        if vote_target.user == request.user:
            raise ValidationError(
                'Sorry, voting for your own answer is not possible.')
        else:
            upvote = request.POST.get('upvote', None) is not None
            object_kwargs = self.get_vote_kwargs(request.user, vote_target)
            vote, created = self.vote_model.objects.get_or_create(
                defaults={'value': upvote},
                **object_kwargs)
            if created:
                vote_target.user.userqaprofile.points += 1 if upvote else -1
            elif not created:
                if vote.value == upvote:
                    vote.delete()
                    vote_target.user.userqaprofile.points += -1 if upvote else 1
                else:
                    vote_target.user.userqaprofile.points += 2 if upvote else -2
                    vote.value = upvote
                    vote.save()
            vote_target.user.userqaprofile.save()
            vote_target.save()

        next_url = request.POST.get('next', None)
        if next_url is not None:
            return redirect(next_url)
        else:
            return redirect(reverse('qa_index'))

class AnswerVoteView(ParentVoteView):
    """
    Class to upvote answers
    """
    model = Answer
    vote_model = AnswerVote

class QuestionVoteView(ParentVoteView):
    """
    Class to upvote questions
    """
    model = Question
    vote_model = QuestionVote

def profile(request, user_id):
    user_ob = get_user_model().objects.get(id=user_id)
    user = UserQAProfile.objects.get(user=user_ob)
    return render(request, 'qa/profile.html', {'user': user})
<|endoftext|>


<commit_before>"""Test functions for python.shelxe"""

import os
import unittest
from ample import constants
from ample.util import ample_util, shelxe
from ample.testing import test_funcs

@unittest.skipUnless(test_funcs.found_exe("shelxe" + ample_util.EXE_EXT), "shelxe exec missing")
class Test(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.thisd = os.path.abspath(os.path.dirname(__file__))
        cls.ample_share = constants.SHARE_DIR
        cls.testfiles_dir = os.path.join(cls.ample_share, 'testfiles')

    def test_shelxe_1BYZ(self):
        shelxe_exe = ample_util.find_exe('shelxe' + ample_util.EXE_EXT)
        pdb = os.path.join(self.testfiles_dir, "1BYZ.pdb")
        mtz = os.path.join(self.testfiles_dir, "1BYZ-cad.mtz")
        mrPdb = os.path.join(self.testfiles_dir, "1BYZ_phaser_loc0_ALL_poly_ala_trunc_0.486615_rad_1_UNMOD.1.pdb")
        mrinfo = shelxe.MRinfo(shelxe_exe, pdb, mtz)
        mrinfo.analyse(mrPdb, cleanup=True)

        self.assertEqual(mrinfo.originShift, [0.326, 0.19, 0.275])
        # Need MRBUMP to be updated
        # self.assertEqual(mrinfo.MPE,78.5)
        self.assertEqual(mrinfo.wMPE, 74.5)

    def test_shelxe_1D7M(self):
        shelxe_exe = ample_util.find_exe('shelxe' + ample_util.EXE_EXT)
        pdb = os.path.join(self.testfiles_dir, "1D7M.pdb")
        mtz = os.path.join(self.testfiles_dir, "1D7M-cad.mtz")
        mrPdb = os.path.join(
            self.testfiles_dir, "1D7M_phaser_loc0_ALL_SCWRL_reliable_sidechains_trunc_5.241154_rad_1_UNMOD.1.pdb"
        )

        mrinfo = shelxe.MRinfo(shelxe_exe, pdb, mtz)
        mrinfo.analyse(mrPdb)

        self.assertEqual(mrinfo.originShift, [-0.0, -0.0, 0.5])
        # Need MRBUMP to be updated
        # self.assertEqual(mrinfo.MPE,65.5)
        self.assertEqual(mrinfo.wMPE, 57.4)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Changed test so that shelxe values are withing a 5% window of hard coded value<commit_after>"""Test functions for python.shelxe"""

import os
import unittest
from ample import constants
from ample.util import ample_util, shelxe
from ample.testing import test_funcs

@unittest.skipUnless(test_funcs.found_exe("shelxe" + ample_util.EXE_EXT), "shelxe exec missing")
class Test(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.thisd = os.path.abspath(os.path.dirname(__file__))
        cls.ample_share = constants.SHARE_DIR
        cls.testfiles_dir = os.path.join(cls.ample_share, 'testfiles')

    def test_shelxe_1BYZ(self):
        shelxe_exe = ample_util.find_exe('shelxe' + ample_util.EXE_EXT)
        pdb = os.path.join(self.testfiles_dir, "1BYZ.pdb")
        mtz = os.path.join(self.testfiles_dir, "1BYZ-cad.mtz")
        mrPdb = os.path.join(self.testfiles_dir, "1BYZ_phaser_loc0_ALL_poly_ala_trunc_0.486615_rad_1_UNMOD.1.pdb")
        mrinfo = shelxe.MRinfo(shelxe_exe, pdb, mtz)
        mrinfo.analyse(mrPdb, cleanup=True)

        self.assertEqual(mrinfo.originShift, [0.326, 0.19, 0.275])
        self.assertTrue(mrinfo.wMPE * 0.95 <= 74.5 <= mrinfo.wMPE * 1.05)

    def test_shelxe_1D7M(self):
        shelxe_exe = ample_util.find_exe('shelxe' + ample_util.EXE_EXT)
        pdb = os.path.join(self.testfiles_dir, "1D7M.pdb")
        mtz = os.path.join(self.testfiles_dir, "1D7M-cad.mtz")
        mrPdb = os.path.join(
            self.testfiles_dir, "1D7M_phaser_loc0_ALL_SCWRL_reliable_sidechains_trunc_5.241154_rad_1_UNMOD.1.pdb"
        )

        mrinfo = shelxe.MRinfo(shelxe_exe, pdb, mtz)
        mrinfo.analyse(mrPdb)

        self.assertEqual(mrinfo.originShift, [-0.0, -0.0, 0.5])
        self.assertTrue(mrinfo.wMPE * 0.95 <= 57.4 <= mrinfo.wMPE * 1.05)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import click
import ebisearch
from pprint import pprint

@click.group()
def main():
    pass

@click.command('get_results', short_help='Get list of results')
def get_results():
    """Return the list of domains in EBI"""
    ebisearch.get_results(verbose=True)

@click.command('get_filter_fields', short_help='Get filter fields')
@click.option(
    '--result',
    help='Id of a result (accessible with get_results)')
def get_filter_fields(result):
    """Get the filter fields of a result to build a query"""
    ebisearch.get_filter_fields(verbose=True)

@click.command('get_filter_types', short_help='Get filter types')
def get_filter_types():
    """Get the types of filters usable to build a query"""
    ebisearch.get_filter_types(verbose=True)

main.add_command(get_results)
main.add_command(get_filter_fields)
main.add_command(get_filter_types)

if __name__ == "__main__":
    main()
<commit_msg>Fix flake8 issue<commit_after>#!/usr/bin/env python

import click
import ebisearch

@click.group()
def main():
    pass

@click.command('get_results', short_help='Get list of results')
def get_results():
    """Return the list of domains in EBI"""
    ebisearch.get_results(verbose=True)

@click.command('get_filter_fields', short_help='Get filter fields')
@click.option(
    '--result',
    help='Id of a result (accessible with get_results)')
def get_filter_fields(result):
    """Get the filter fields of a result to build a query"""
    ebisearch.get_filter_fields(verbose=True)

@click.command('get_filter_types', short_help='Get filter types')
def get_filter_types():
    """Get the types of filters usable to build a query"""
    ebisearch.get_filter_types(verbose=True)

main.add_command(get_results)
main.add_command(get_filter_fields)
main.add_command(get_filter_types)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
A set of basic transformers for python asts
"""
import os
import ast

from ctypes import c_long, c_int, c_uint, c_byte, c_ulong, c_ushort, c_short, c_wchar_p, c_char_p, c_float

from ctree.nodes import Project, CtreeNode
from ctree.c.nodes import Op, Constant, String, SymbolRef, BinaryOp, TernaryOp, Return, While, MultiNode
from ctree.c.nodes import If, CFile, FunctionCall, FunctionDecl, For, Assign, AugAssign, ArrayRef, Literal
from ctree.c.nodes import Lt, Gt, AddAssign, SubAssign, MulAssign, DivAssign, BitAndAssign, BitShRAssign, BitShLAssign
from ctree.c.nodes import BitOrAssign, BitXorAssign, ModAssign, Break, Continue, Pass, Array

from ctree.c.nodes import Op

from ctree.types import get_ctype

from ctree.visitors import NodeTransformer
from ctree.util import flatten

class PyCtxScrubber(NodeTransformer):
    """
    Removes pesky ctx attributes from Python ast.Name nodes,
    yielding much cleaner python asts.
    """

    def visit_Name(self, node):
        node.ctx = None
        return node

class PyBasicConversions(NodeTransformer):
    """
    Convert constructs with obvious C analogues.
    """
    def __init__(self,names_dict={}, constants_dict={}):
        self.names_dict = names_dict
        self.constants_dict =constants_dict

    PY_OP_TO_CTREE_OP = {
        ast.Add: Op.Add,
        ast.Mod: Op.Mod,
        ast.Mult: Op.Mul,
        ast.Sub: Op.Sub,
        ast.Div: Op.Div,
        ast.Lt: Op.Lt,
        ast.Gt: Op.Gt,
        ast.LtE: Op.LtE,
        ast.GtE: Op.GtE,
        ast.BitAnd: Op.BitAnd,
        ast.BitOr: Op.BitOr,
        ast.Eq: Op.Eq,
        ast.NotEq: Op.NotEq,
        ast.Not: Op.Not,
        ast.And: Op.And,
        ast.Or: Op.Or,
        ast.BitXor: Op.BitXor,
        ast.LShift: Op.BitShL,
        ast.RShift: Op.BitShR,
        ast.Is: Op.Eq,
        ast.IsNot: Op. NotEq
        # TODO list the rest
    }

    def visit_Num(self, node):
        return Constant(node.n)

    def visit_Str(self, node):
        return String(node.s)

    def visit_Name(self, node):
        if node.id in self.constants_dict:
            return Constant(self.constants_dict[node.id])
        if node.id in self.names_dict:
            return SymbolRef(self.names_dict[node.id])
        return SymbolRef(node.id)

    def visit_BinOp(self, node):
        lhs = self.visit(node.left)
        rhs = self.visit(node.right)
        op = self.PY_OP_TO_CTREE_OP.get(type(node.op), type(node.op))()
        return BinaryOp(lhs, op, rhs)

    def visit_Return(self, node):
        if hasattr(node, 'value'):
            return Return(self.visit(node.value))
        else:
            return Return()

    def visit_For(self, node):
        """restricted, for now, to range as iterator with long-type args"""
        if isinstance(node, ast.For) and \
           isinstance(node.iter, ast.Call) and \
           isinstance(node.iter.func, ast.Name) and \
           node.iter.func.id in ('range', 'xrange'):
            Range = node.iter
            nArgs = len(Range.args)
            if nArgs == 1:
                stop = self.visit(Range.args[0])
                start, step = Constant(0), Constant(1)
            elif nArgs == 2:
                start, stop = map(self.visit, Range.args)
                step = Constant(1)
            elif nArgs == 3:
                start, stop, step = map(self.visit, Range.args)
            else:
                raise Exception("Cannot convert a for...range with %d args." % nArgs)

            #check no-op conditions.
            if all(isinstance(item, Constant) for item in (start, stop, step)):
                if step.value == 0:
                    raise ValueError("range() step argument must not be zero")
                if start.value == stop.value or \
                        (start.value < stop.value and step.value < 0) or \
                        (start.value > stop.value and step.value > 0):
                    return None

            # TODO allow any expressions castable to Long type
            target_type = c_long
            for el in (stop, start, step):
                if hasattr(el, 'get_type'): #typed item to try and guess type off of. Imperfect right now.
                    # TODO take the proper class instead of the last; if start, end are doubles, but step is long, target is double
                    t = el.get_type()
                    assert any(isinstance(t, klass) for klass in [
                        c_byte, c_int, c_long, c_short
                    ]), "Can only convert ranges with integer/long start/stop/step values"
                    target_type = t

            target = SymbolRef(node.target.id, target_type)
            op = Lt
            if hasattr(start,'value') and hasattr(stop,'value'):
                if start.value > stop.value:
                    op = Gt
            for_loop = For(
                Assign(target, start),
                op(target.copy(), stop),
                AddAssign(target.copy(), step),
                [self.visit(stmt) for stmt in node.body],
            )
            return for_loop
        node.body = list(map(self.visit, node.body))
        return node

    def visit_If(self, node):
        if isinstance(node, ast.If):
            cond = self.visit(node.test)
            then = [self.visit(t) for t in node.body]
            elze = [self.visit(t) for t in node.orelse] or None
            return If(cond, then, elze)
        else:
            return self.generic_visit(node)

    def visit_IfExp(self, node):
        cond = self.visit(node.test)
        then = self.visit(node.body)
        elze = self.visit(node.orelse)
        return TernaryOp(cond, then, elze)

    def visit_Compare(self, node):
        assert len(node.ops) == 1, \
            "PyBasicConversions doesn't support Compare nodes with more than one operator."
        lhs = self.visit(node.left)

        op = self.PY_OP_TO_CTREE_OP.get(type(node.ops[0]),type(node.ops[0]))()
        rhs = self.visit(node.comparators[0])
        return BinaryOp(lhs, op, rhs)

    def visit_Module(self, node):
        body = [self.visit(s) for s in node.body]
        return Project([CFile("module", body)])

    def visit_Call(self, node):
        args = [self.visit(a) for a in node.args]
        fn = self.visit(node.func)
        return FunctionCall(fn, args)

    def visit_FunctionDef(self, node):
        if ast.get_docstring(node):
            node.body.pop(0)
        params = [self.visit(p) for p in node.args.args]
        defn = [self.visit(s) for s in node.body]
        return FunctionDecl(None, node.name, params, defn)

    def visit_arg(self, node):
        return SymbolRef(node.arg, node.annotation)

    def visit_AugAssign(self, node):
        op = type(node.op)
        target = self.visit(node.target)
        value = self.visit(node.value)
        if op is ast.Add:
            return AddAssign(target, value)
        elif op is ast.Sub:
            return SubAssign(target, value)
        elif op is ast.Mult:
            return MulAssign(target, value)
        elif op is ast.Div:
            return DivAssign(target, value)
        elif op is ast.BitXor:
            return BitXorAssign(target, value)
        elif op is ast.BitAnd:
            return BitAndAssign(target, value)
        elif op is ast.BitOr:
            return BitOrAssign(target, value)
        elif op is ast.Mod:
            return ModAssign(target, value)
        elif op is ast.LShift:
            return BitShLAssign(target, value)
        elif op is ast.RShift:
            return BitShRAssign(target, value)
        # TODO: Error?
        return node

    def visit_Assign(self, node):
        target_value_list = []
        #a = b -> targets = [ast.Name], value = ast.Name
        #a = b = c... -> targets = [ast.Name, ast.Name....], value = ast.Name
        if all(isinstance(i, ast.Name) for i in node.targets):
            target_value_list.extend((target, node.value) for target in node.targets)

        #a, b = c,d -> targets = [ast.Tuple], value = ast.Tuple
        elif isinstance(node.targets[0], (ast.List, ast.Tuple)):
            target_value_list.extend((target, value) for target, value in zip(node.targets[0].elts, node.value.elts))

        else:
            return node

        target_value_list = [(self.visit(target), self.visit(value)) for target, value in target_value_list]

        #making a multinode no matter what. It's cleaner than branching a lot
        operation_body = []
        swap_body = []
        for target, value in target_value_list[:]:
            if isinstance(value, (Constant, String)):
                operation_body.append(Assign(target, value))
                target_value_list.remove((target,value))
                continue
            new_target = target.copy()
            new_target.name = "____temp__" + new_target.name
            operation_body.append(Assign(new_target, value))
            swap_body.append(Assign(target, new_target.copy()))
        return MultiNode(body = operation_body + swap_body)

    def visit_Subscript(self, node):
        if isinstance(node.slice,ast.Index):
            value = self.visit(node.value)
            index = self.visit(node.slice.value)
            return ArrayRef(value,index)
        else:
            return node

    def visit_While(self,node):
        cond = self.visit(node.test)
        body = [self.visit(i) for i in node.body]
        return While(cond, body)

    def visit_Lambda(self, node):

        if isinstance(node, ast.Lambda):
            def_node = ast.FunctionDef(name = "default", args = node.args, body = node.body, decorator_list = None)

            params = [self.visit(p) for p in def_node.args.args]
            defn = [Return(self.visit(def_node.body))]
            decl_node = FunctionDecl(None, def_node.name, params, defn)
            Lifter().visit_FunctionDecl(decl_node)

            return decl_node
        else:
            return node

    def visit_Break(self, node):
        return Break()

    def visit_Continue(self, node):
        return Continue()

    def visit_Pass(self, node):
        return Pass()

class ResolveGeneratedPathRefs(NodeTransformer):
    """
    Converts any instances of ctree.nodes.GeneratedPathRef into strings containing the absolute path
    of the target file.
    """

    def __init__(self, compilation_dir):
        self.compilation_dir = compilation_dir
        self.count = 0

    def visit_GeneratedPathRef(self, node):
        self.count += 1
        return String(os.path.join(self.compilation_dir, node.target.get_filename()))

class Lifter(NodeTransformer):
    """
    To aid in adding new includes or parameters during tree
    traversals, users can store them with arbitrary child nodes and call this
    transformation to move them to the correct position.
    """
    def __init__(self, lift_params=True, lift_includes=True):
        self.lift_params = lift_params
        self.lift_includes = lift_includes

    def visit_FunctionDecl(self, node):
        if self.lift_params:
            for child in ast.walk(node):
                for param in getattr(child, '_lift_params', []):
                    if param not in node.params:
                        node.params.append(param)
                    #del child._lift_params
        return self.generic_visit(node)

    def visit_CFile(self, node):
        if self.lift_includes:
            new_includes = []
            for child in ast.walk(node):
                for include in getattr(child, '_lift_includes', []):
                    if include not in new_includes:
                        new_includes.append(include)
            node.body = list(new_includes) + node.body
        return self.generic_visit(node)

class DeclarationFiller(NodeTransformer):
    def __init__(self):
        self.__environments = [{}]

    def __lookup(self, key):
        """
        :param key:
        :return: Looks up the last value corresponding to key in self.__environments
        """
        value = sentinel = object()
        for environment in self.__environments:
            if key in environment:
                value = environment[key]
        if value is sentinel:
            raise KeyError('Did not find {} in environments'.format(repr(key)))
        return value

    def __has_key(self, key):
        try:
            self.__lookup(key)
            return True
        except KeyError:
            return False

    def __add_entry(self, key, value):
        self.__environments[-1][key] = value

    def __add_environment(self):
        self.__environments.append({})

    def __pop_environment(self):
        return self.__environments.pop()

    def visit_FunctionDecl(self, node):
        #add current FunctionDecl's return type onto environments
        self.__add_entry(node.name, node.return_type)

        #new environment every time we enter a function
        self.__add_environment()

        for param in node.params:
            #binding types of parameters
            self.__add_entry(param.name, param.type)

        node.defn = [self.visit(i) for i in node.defn]
        self.__pop_environment()
        return node

    def visit_SymbolRef(self, node):

        if node.type:
            self.__add_entry(node.name, node.type)
        return node

    def visit_FunctionCall(self, node):
        if self.__has_key(node.func):
            node.type = self.__lookup(node.func)
        node.args = [self.visit(arg) for arg in node.args]
        return node

    def visit_BinaryOp(self, node):
        if isinstance(node.op, Op.Assign):
            node.left = self.visit(node.left)
            if isinstance(node.left, BinaryOp):
                return node
            node.right = self.visit(node.right)
            name = node.left
            value = node.right
            if hasattr(name, 'type') and name.type is not None:
                return node
            if not self.__has_key(name.name):
                if name.name.startswith('____temp__'): #temporary variable
                    stripped_name = name.name.lstrip('____temp__')
                    if self.__has_key(stripped_name):
                        node.left.type = self.__lookup(stripped_name)

                elif hasattr(value, 'get_type'):
                    node.left.type = value.get_type()
                elif isinstance(value, String):
                    node.left.type = c_char_p()
                elif isinstance(value, SymbolRef):
                    node.left.type = self.__lookup(value.name)
                elif isinstance(value, FunctionCall):
                    node.left.type = self.__lookup(value.func)

                self.__add_entry(node.left.name, node.left.type)
        return node

<commit_msg>fixed multiple assign for arbitrary nesting, i.e. a,(b,(c,d),e) = 1,(2,(3,4),5)<commit_after>"""
A set of basic transformers for python asts
"""
import os
import ast

from ctypes import c_long, c_int, c_uint, c_byte, c_ulong, c_ushort, c_short, c_wchar_p, c_char_p, c_float

from collections import deque
import itertools

from ctree.nodes import Project, CtreeNode
from ctree.c.nodes import Op, Constant, String, SymbolRef, BinaryOp, TernaryOp, Return, While, MultiNode
from ctree.c.nodes import If, CFile, FunctionCall, FunctionDecl, For, Assign, AugAssign, ArrayRef, Literal
from ctree.c.nodes import Lt, Gt, AddAssign, SubAssign, MulAssign, DivAssign, BitAndAssign, BitShRAssign, BitShLAssign
from ctree.c.nodes import BitOrAssign, BitXorAssign, ModAssign, Break, Continue, Pass, Array

from ctree.c.nodes import Op

from ctree.types import get_ctype

from ctree.visitors import NodeTransformer
from ctree.util import flatten

class PyCtxScrubber(NodeTransformer):
    """
    Removes pesky ctx attributes from Python ast.Name nodes,
    yielding much cleaner python asts.
    """

    def visit_Name(self, node):
        node.ctx = None
        return node

class PyBasicConversions(NodeTransformer):
    """
    Convert constructs with obvious C analogues.
    """
    def __init__(self,names_dict={}, constants_dict={}):
        self.names_dict = names_dict
        self.constants_dict =constants_dict

    PY_OP_TO_CTREE_OP = {
        ast.Add: Op.Add,
        ast.Mod: Op.Mod,
        ast.Mult: Op.Mul,
        ast.Sub: Op.Sub,
        ast.Div: Op.Div,
        ast.Lt: Op.Lt,
        ast.Gt: Op.Gt,
        ast.LtE: Op.LtE,
        ast.GtE: Op.GtE,
        ast.BitAnd: Op.BitAnd,
        ast.BitOr: Op.BitOr,
        ast.Eq: Op.Eq,
        ast.NotEq: Op.NotEq,
        ast.Not: Op.Not,
        ast.And: Op.And,
        ast.Or: Op.Or,
        ast.BitXor: Op.BitXor,
        ast.LShift: Op.BitShL,
        ast.RShift: Op.BitShR,
        ast.Is: Op.Eq,
        ast.IsNot: Op. NotEq
        # TODO list the rest
    }

    def visit_Num(self, node):
        return Constant(node.n)

    def visit_Str(self, node):
        return String(node.s)

    def visit_Name(self, node):
        if node.id in self.constants_dict:
            return Constant(self.constants_dict[node.id])
        if node.id in self.names_dict:
            return SymbolRef(self.names_dict[node.id])
        return SymbolRef(node.id)

    def visit_BinOp(self, node):
        lhs = self.visit(node.left)
        rhs = self.visit(node.right)
        op = self.PY_OP_TO_CTREE_OP.get(type(node.op), type(node.op))()
        return BinaryOp(lhs, op, rhs)

    def visit_Return(self, node):
        if hasattr(node, 'value'):
            return Return(self.visit(node.value))
        else:
            return Return()

    def visit_For(self, node):
        """restricted, for now, to range as iterator with long-type args"""
        if isinstance(node, ast.For) and \
           isinstance(node.iter, ast.Call) and \
           isinstance(node.iter.func, ast.Name) and \
           node.iter.func.id in ('range', 'xrange'):
            Range = node.iter
            nArgs = len(Range.args)
            if nArgs == 1:
                stop = self.visit(Range.args[0])
                start, step = Constant(0), Constant(1)
            elif nArgs == 2:
                start, stop = map(self.visit, Range.args)
                step = Constant(1)
            elif nArgs == 3:
                start, stop, step = map(self.visit, Range.args)
            else:
                raise Exception("Cannot convert a for...range with %d args." % nArgs)

            #check no-op conditions.
            if all(isinstance(item, Constant) for item in (start, stop, step)):
                if step.value == 0:
                    raise ValueError("range() step argument must not be zero")
                if start.value == stop.value or \
                        (start.value < stop.value and step.value < 0) or \
                        (start.value > stop.value and step.value > 0):
                    return None

            # TODO allow any expressions castable to Long type
            target_type = c_long
            for el in (stop, start, step):
                if hasattr(el, 'get_type'): #typed item to try and guess type off of. Imperfect right now.
                    # TODO take the proper class instead of the last; if start, end are doubles, but step is long, target is double
                    t = el.get_type()
                    assert any(isinstance(t, klass) for klass in [
                        c_byte, c_int, c_long, c_short
                    ]), "Can only convert ranges with integer/long start/stop/step values"
                    target_type = t

            target = SymbolRef(node.target.id, target_type)
            op = Lt
            if hasattr(start,'value') and hasattr(stop,'value'):
                if start.value > stop.value:
                    op = Gt
            for_loop = For(
                Assign(target, start),
                op(target.copy(), stop),
                AddAssign(target.copy(), step),
                [self.visit(stmt) for stmt in node.body],
            )
            return for_loop
        node.body = list(map(self.visit, node.body))
        return node

    def visit_If(self, node):
        if isinstance(node, ast.If):
            cond = self.visit(node.test)
            then = [self.visit(t) for t in node.body]
            elze = [self.visit(t) for t in node.orelse] or None
            return If(cond, then, elze)
        else:
            return self.generic_visit(node)

    def visit_IfExp(self, node):
        cond = self.visit(node.test)
        then = self.visit(node.body)
        elze = self.visit(node.orelse)
        return TernaryOp(cond, then, elze)

    def visit_Compare(self, node):
        assert len(node.ops) == 1, \
            "PyBasicConversions doesn't support Compare nodes with more than one operator."
        lhs = self.visit(node.left)

        op = self.PY_OP_TO_CTREE_OP.get(type(node.ops[0]),type(node.ops[0]))()
        rhs = self.visit(node.comparators[0])
        return BinaryOp(lhs, op, rhs)

    def visit_Module(self, node):
        body = [self.visit(s) for s in node.body]
        return Project([CFile("module", body)])

    def visit_Call(self, node):
        args = [self.visit(a) for a in node.args]
        fn = self.visit(node.func)
        return FunctionCall(fn, args)

    def visit_FunctionDef(self, node):
        if ast.get_docstring(node):
            node.body.pop(0)
        params = [self.visit(p) for p in node.args.args]
        defn = [self.visit(s) for s in node.body]
        return FunctionDecl(None, node.name, params, defn)

    def visit_arg(self, node):
        return SymbolRef(node.arg, node.annotation)

    def visit_AugAssign(self, node):
        op = type(node.op)
        target = self.visit(node.target)
        value = self.visit(node.value)
        if op is ast.Add:
            return AddAssign(target, value)
        elif op is ast.Sub:
            return SubAssign(target, value)
        elif op is ast.Mult:
            return MulAssign(target, value)
        elif op is ast.Div:
            return DivAssign(target, value)
        elif op is ast.BitXor:
            return BitXorAssign(target, value)
        elif op is ast.BitAnd:
            return BitAndAssign(target, value)
        elif op is ast.BitOr:
            return BitOrAssign(target, value)
        elif op is ast.Mod:
            return ModAssign(target, value)
        elif op is ast.LShift:
            return BitShLAssign(target, value)
        elif op is ast.RShift:
            return BitShRAssign(target, value)
        # TODO: Error?
        return node

    def visit_Assign(self, node):
        target_value_list = []
        #a = b -> targets = [ast.Name], value = ast.Name
        #a = b = c... -> targets = [ast.Name, ast.Name....], value = ast.Name

        def parse_pairs(node):
            def targets_to_list(targets): #parses target into nested lists
                res = []
                for elt in targets:
                    if not isinstance(elt, (ast.List, ast.Tuple)):
                        res.append(elt)
                    elif isinstance(elt, (ast.Tuple, ast.List)):
                        res.append(targets_to_list(elt.elts))
                return res

            def value_to_list(value): #parses value into nested lists for multiple assign
                res = []
                if not isinstance(value, (ast.List, ast.Tuple)):
                    return value
                for elt in value.elts:
                    if not isinstance(value, (ast.List, ast.Tuple)):
                        res.append(elt)
                    else:
                        res.append(value_to_list(elt))
                return res

            def pair_lists(targets, values):
                res = []
                queue = deque((target, values) for target in targets)
                sentinel = object()
                while queue:
                    target, value = queue.popleft()
                    if isinstance(target, list):
                        #target hasn't been completely unrolled yet
                        for sub_target, sub_value in itertools.izip_longest(target, value, fillvalue=sentinel):
                            if sub_target is sentinel or sub_value is sentinel:
                                raise ValueError('Incorrect number of values to unpack')
                            queue.append((sub_target, sub_value))
                    else:
                        res.append((target, value))
                return res

            targets = targets_to_list(node.targets)
            values = value_to_list(node.value)
            return pair_lists(targets, values)

        target_value_list = [(self.visit(target), self.visit(value)) for target, value in parse_pairs(node)]

        #making a multinode no matter what. It's cleaner than branching a lot
        operation_body = []
        swap_body = []
        for target, value in target_value_list[:]:
            if isinstance(value, (Constant, String)):
                operation_body.append(Assign(target, value))
                target_value_list.remove((target,value))
                continue
            new_target = target.copy()
            new_target.name = "____temp__" + new_target.name
            operation_body.append(Assign(new_target, value))
            swap_body.append(Assign(target, new_target.copy()))
        return MultiNode(body = operation_body + swap_body)

    def visit_Subscript(self, node):
        if isinstance(node.slice,ast.Index):
            value = self.visit(node.value)
            index = self.visit(node.slice.value)
            return ArrayRef(value,index)
        else:
            return node

    def visit_While(self,node):
        cond = self.visit(node.test)
        body = [self.visit(i) for i in node.body]
        return While(cond, body)

    def visit_Lambda(self, node):

        if isinstance(node, ast.Lambda):
            def_node = ast.FunctionDef(name = "default", args = node.args, body = node.body, decorator_list = None)

            params = [self.visit(p) for p in def_node.args.args]
            defn = [Return(self.visit(def_node.body))]
            decl_node = FunctionDecl(None, def_node.name, params, defn)
            Lifter().visit_FunctionDecl(decl_node)

            return decl_node
        else:
            return node

    def visit_Break(self, node):
        return Break()

    def visit_Continue(self, node):
        return Continue()

    def visit_Pass(self, node):
        return Pass()

class ResolveGeneratedPathRefs(NodeTransformer):
    """
    Converts any instances of ctree.nodes.GeneratedPathRef into strings containing the absolute path
    of the target file.
    """

    def __init__(self, compilation_dir):
        self.compilation_dir = compilation_dir
        self.count = 0

    def visit_GeneratedPathRef(self, node):
        self.count += 1
        return String(os.path.join(self.compilation_dir, node.target.get_filename()))

class Lifter(NodeTransformer):
    """
    To aid in adding new includes or parameters during tree
    traversals, users can store them with arbitrary child nodes and call this
    transformation to move them to the correct position.
    """
    def __init__(self, lift_params=True, lift_includes=True):
        self.lift_params = lift_params
        self.lift_includes = lift_includes

    def visit_FunctionDecl(self, node):
        if self.lift_params:
            for child in ast.walk(node):
                for param in getattr(child, '_lift_params', []):
                    if param not in node.params:
                        node.params.append(param)
                    #del child._lift_params
        return self.generic_visit(node)

    def visit_CFile(self, node):
        if self.lift_includes:
            new_includes = []
            for child in ast.walk(node):
                for include in getattr(child, '_lift_includes', []):
                    if include not in new_includes:
                        new_includes.append(include)
            node.body = list(new_includes) + node.body
        return self.generic_visit(node)

class DeclarationFiller(NodeTransformer):
    def __init__(self):
        self.__environments = [{}]

    def __lookup(self, key):
        """
        :param key:
        :return: Looks up the last value corresponding to key in self.__environments
        """
        value = sentinel = object()
        for environment in self.__environments:
            if key in environment:
                value = environment[key]
        if value is sentinel:
            raise KeyError('Did not find {} in environments'.format(repr(key)))
        return value

    def __has_key(self, key):
        try:
            self.__lookup(key)
            return True
        except KeyError:
            return False

    def __add_entry(self, key, value):
        self.__environments[-1][key] = value

    def __add_environment(self):
        self.__environments.append({})

    def __pop_environment(self):
        return self.__environments.pop()

    def visit_FunctionDecl(self, node):
        #add current FunctionDecl's return type onto environments
        self.__add_entry(node.name, node.return_type)

        #new environment every time we enter a function
        self.__add_environment()

        for param in node.params:
            #binding types of parameters
            self.__add_entry(param.name, param.type)

        node.defn = [self.visit(i) for i in node.defn]
        self.__pop_environment()
        return node

    def visit_SymbolRef(self, node):

        if node.type:
            self.__add_entry(node.name, node.type)
        return node

    def visit_FunctionCall(self, node):
        if self.__has_key(node.func):
            node.type = self.__lookup(node.func)
        node.args = [self.visit(arg) for arg in node.args]
        return node

    def visit_BinaryOp(self, node):
        if isinstance(node.op, Op.Assign):
            node.left = self.visit(node.left)
            if isinstance(node.left, BinaryOp):
                return node
            node.right = self.visit(node.right)
            name = node.left
            value = node.right
            if hasattr(name, 'type') and name.type is not None:
                return node
            if not self.__has_key(name.name):
                if name.name.startswith('____temp__'): #temporary variable
                    stripped_name = name.name.lstrip('____temp__')
                    if self.__has_key(stripped_name):
                        node.left.type = self.__lookup(stripped_name)

                elif hasattr(value, 'get_type'):
                    node.left.type = value.get_type()
                elif isinstance(value, String):
                    node.left.type = c_char_p()
                elif isinstance(value, SymbolRef):
                    node.left.type = self.__lookup(value.name)
                elif isinstance(value, FunctionCall):
                    node.left.type = self.__lookup(value.func)

                self.__add_entry(node.left.name, node.left.type)
        return node

<|endoftext|>