<commit_before>#! /usr/bin/env python

import logging
import pprint
from itertools import izip, chain
from collections import deque, defaultdict
from xml.sax import make_parser, handler, parseString, saxutils

taglist = ['country','doc-number','kind','date','application-reference','appl-type','classification-national','main-classification',
           'further-classification','abstract','invention-title','assignees','assignee','references-cited','citation','us-related-documents','parties','applicant',
           'addressbook','last-name','first-name','street','city','state','country','postcode','nationality','residence','agents','agent','orgname']

"""

By using the xml.sax.handler.ContentHandler, we can create a stack of the tags
we encounter when parsing the xml document. The information required by the
XMLPatentBase class can be described as tuples specifying the requisite nesting
of tags leading to the information that we want. E.g. (parties, applicant,
addressbook) would flexibly specify a nesting matching:

  <parties>
  <applicant>
  <addressbook> .... </addressbook>
  </applicant>
  </parties>

with an arbitrary number of tags between <parties>, <applicant> and
<addressbook>. We can map the contents of a tag to the tagstack at that point
in time. We can then search the keys of this dictionary for orderings of tags
matching the ordering in the tuple we are looking at. We have to be sure to
preserve the ordering of the tagstack keys as they are added to the dictionary
in case we need information from across multiple internal tags.

"""
class Patent(handler.ContentHandler):

  def __init__(self):
    self.tagstack = deque()
    self.contents = defaultdict(list)
    self.attributes = defaultdict(str)

  def _search(self, *terms):
    """
    Given a list [terms] of xml tags, we return the contents described
    by the nesting of tags in the order specified.
    """
    allkeys = self.contents.keys()
    searchdict = dict(zip(allkeys,allkeys))
    for term in terms:
      searchdict = dict((k,v[v.index(term):]) \
                    for k,v in searchdict.iteritems() if term in v)
    res = list(self.contents[k] for k in searchdict.iterkeys())
    return res if res else ''

  def startElement(self, name, attrs):
    self.tagstack.append(name)
    for k,v in attrs.items():
      self.attributes[k] = v

  def endElement(self, name):
    if self.tagstack:
      if self.tagstack[-1] == name:
        self.tagstack.pop()

  def characters(self, content):
    if content.strip():
      self.contents[tuple(self.tagstack)].append(content)

  def has_data(self, l):
    return filter(lambda x: x, l)

  def __classes__(self):
    res = []
    for tag in ['main-classification','further-classification']:
      tmp = self._search('classification-national', tag)
      if tmp:
        res.append(tmp[0])
    res = [[x[0][:3].replace(' ',''), x[0][3:].replace(' ','')] for x in res]
    return res

  def __asg_list__(self):
    toadd = []
    if self._search('assignees','first-name'):
      toadd = [1]
      data = self._search('assignees','last-name')
      toadd.extend(data[0] if data else [])
      data = self._search('assignees','first-name')
      toadd.extend(data[0] if data else [])
    else:
      toadd = [0]
      data = self._search('assignees','orgname')
      toadd.extend(data[0] if data else [])
      data = self._search('assignees','role')
      toadd.extend(data[0] if data else [])
    if len(toadd) == 1:
      return []
    tag = 'addressbook' if self._search('assignees','addressbook') else 'address'
    for inner in ['street','city','state','country','postcode']:
      data = self._search('assignees',tag,inner)
      toadd.extend(data[0] if data else [''])
    for tag in ['nationality','residence']:
      data = self._search('assignees',tag,'country')
      toadd.extend(data[0] if data else [''])
    return [toadd]

  def __cit_list__(self):
    basetag = 'references-cited' if self._search('references-cited') else 'citation'
    toadd = self._search(basetag,'category')
    if self._search(basetag,'patcit'):
      toadd.extend(self._search(basetag,'patcit','country'))
      toadd.extend(self._search(basetag,'patcit','doc-number'))
      toadd.extend(self._search(basetag,'patcit','date'))
      toadd.extend(self._search(basetag,'patcit','kind'))
      toadd.extend(self._search(basetag,'patcit','name'))
      toadd.append(['']*len(self._search(basetag,'patcit','name')[0]))
    elif self._search(basetag, 'othercit'):
      toadd.extend(['','','','',''])
      toadd.extend(self._search(basetag,'othercit'))
    return map(list,list(izip(*toadd)))

  def __rel_list__(self):
    d_list = []
    for tag in ['continuation-in-part','continuation','division','reissue']:
      if self._search('us-related-documents',tag):
        childtmp = []
        parenttmp = []
        parenttmpsecondary = []
        for nested in ['doc-number','country','kind']:
          data = self._search('us-related-documents',tag,'relation','child-doc',nested)
          childtmp.extend(data[0] if data else [''])
        for nested in ['doc-number','country','kind','date','parent-status']:
          data = self._search('us-related-documents',tag,'relation','parent-doc',nested)
          parenttmp.extend(data[0] if data else [''])
        for nested in ['doc-number','country','kind','date','parent-status']:
          data = self._search('us-related-documents',tag,'relation','parent-doc','parent-grant-document',nested)
          if data:
            parenttmpsecondary.extend(data[0])
          else:
            data = self._search('us-related-documents',tag,'relation','parent-doc','parent-pct-document',nested)
            parenttmpsecondary.extend(data[0] if data else [''])
        if self.has_data(childtmp):
          d_list.append([tag, -1] + childtmp)
        if self.has_data(parenttmp):
          d_list.append([tag, 1] + parenttmp)
        if self.has_data(parenttmpsecondary):
          d_list.append([tag, 1] + parenttmpsecondary)
    for tag in ['related-publication', 'us-provisional-application']:
      if self._search('us-related-documents', tag):
        for nested in ['doc-number','country','kind']:
          pass
    return d_list

  def __inv_list__(self):
    toadd = []
    for tag in ['parties', 'applicant']:
      for nested in ['last-name', 'first-name']:
        data = self._search(tag, 'addressbook', nested)
        toadd.append(data[0] if data else [''])
      for nested in ['street','city','state','country','postcode']:
        data = self._search(tag, 'addressbook', nested)
        toadd.append(data[0] if data else [''])
        data = self._search(tag, 'address', nested)
        toadd.append(data[0] if data else [''])
      for nested in ['nationality','residence']:
        data = self._search(tag,nested,'country')
        toadd.append(data[0] if data else [''])
    return map(list,list(izip(*toadd)))

  def __law_list__(self):
    pass

  def endDocument(self):
    self.country = self._search('publication-reference','country')[0][0]
    self.patent = self._search('publication-reference','doc-number')[0][0]
    self.kind = self._search('publication-reference','kind')[0][0]
    self.date_grant = self._search('publication-reference','date')[0][0]
    self.pat_type = self.attributes['appl-type']
    self.date_app = self._search('application-reference','date')[0][0]
    self.country_app = self._search('application-reference','country')[0][0]
    self.patent_app = self._search('application-reference','doc-number')[0][0]
    self.code_app = self._search('us-application-series-code')[0][0]
    self.clm_num = self._search('number-of-claims')[0][0]
    self.classes = self.__classes__()
    self.abstract = self._search('abstract')
    self.invention_title = self._search('invention-title')[0][0]
    self.asg_list = self.__asg_list__()
    self.cit_list = self.__cit_list__()
    self.rel_list = self.__rel_list__()
    self.inv_list = self.__inv_list__()
    self.law_list = self.__law_list__()
<commit_msg>further specification on inv_list<commit_after>#! /usr/bin/env python

import logging
import pprint
from itertools import izip, chain
from collections import deque, defaultdict
from xml.sax import make_parser, handler, parseString, saxutils

taglist = ['country','doc-number','kind','date','application-reference','appl-type','classification-national','main-classification',
           'further-classification','abstract','invention-title','assignees','assignee','references-cited','citation','us-related-documents','parties','applicant',
           'addressbook','last-name','first-name','street','city','state','country','postcode','nationality','residence','agents','agent','orgname']

"""

By using the xml.sax.handler.ContentHandler, we can create a stack of the tags
we encounter when parsing the xml document. The information required by the
XMLPatentBase class can be described as tuples specifying the requisite nesting
of tags leading to the information that we want. E.g. (parties, applicant,
addressbook) would flexibly specify a nesting matching:

  <parties>
  <applicant>
  <addressbook> .... </addressbook>
  </applicant>
  </parties>

with an arbitrary number of tags between <parties>, <applicant> and
<addressbook>. We can map the contents of a tag to the tagstack at that point
in time. We can then search the keys of this dictionary for orderings of tags
matching the ordering in the tuple we are looking at. We have to be sure to
preserve the ordering of the tagstack keys as they are added to the dictionary
in case we need information from across multiple internal tags.

"""
class Patent(handler.ContentHandler):

  def __init__(self):
    self.tagstack = deque()
    self.contents = defaultdict(list)
    self.attributes = defaultdict(str)

  def _search(self, *terms):
    """
    Given a list [terms] of xml tags, we return the contents described
    by the nesting of tags in the order specified.
    """
    allkeys = self.contents.keys()
    searchdict = dict(zip(allkeys,allkeys))
    for term in terms:
      searchdict = dict((k,v[v.index(term):]) \
                    for k,v in searchdict.iteritems() if term in v)
    res = list(self.contents[k] for k in searchdict.iterkeys())
    return res if res else ''

  def startElement(self, name, attrs):
    self.tagstack.append(name)
    for k,v in attrs.items():
      self.attributes[k] = v

  def endElement(self, name):
    if self.tagstack:
      if self.tagstack[-1] == name:
        self.tagstack.pop()

  def characters(self, content):
    if content.strip():
      self.contents[tuple(self.tagstack)].append(content)

  def has_data(self, l):
    return filter(lambda x: x, l)

  def __classes__(self):
    res = []
    for tag in ['main-classification','further-classification']:
      tmp = self._search('classification-national', tag)
      if tmp:
        res.append(tmp[0])
    res = [[x[0][:3].replace(' ',''), x[0][3:].replace(' ','')] for x in res]
    return res

  def __asg_list__(self):
    toadd = []
    if self._search('assignees','first-name'):
      toadd = [1]
      data = self._search('assignees','last-name')
      toadd.extend(data[0] if data else [])
      data = self._search('assignees','first-name')
      toadd.extend(data[0] if data else [])
    else:
      toadd = [0]
      data = self._search('assignees','orgname')
      toadd.extend(data[0] if data else [])
      data = self._search('assignees','role')
      toadd.extend(data[0] if data else [])
    if len(toadd) == 1:
      return []
    tag = 'addressbook' if self._search('assignees','addressbook') else 'address'
    for inner in ['street','city','state','country','postcode']:
      data = self._search('assignees',tag,inner)
      toadd.extend(data[0] if data else [''])
    for tag in ['nationality','residence']:
      data = self._search('assignees',tag,'country')
      toadd.extend(data[0] if data else [''])
    return [toadd]

  def __cit_list__(self):
    basetag = 'references-cited' if self._search('references-cited') else 'citation'
    toadd = self._search(basetag,'category')
    if self._search(basetag,'patcit'):
      toadd.extend(self._search(basetag,'patcit','country'))
      toadd.extend(self._search(basetag,'patcit','doc-number'))
      toadd.extend(self._search(basetag,'patcit','date'))
      toadd.extend(self._search(basetag,'patcit','kind'))
      toadd.extend(self._search(basetag,'patcit','name'))
      toadd.append(['']*len(self._search(basetag,'patcit','name')[0]))
    elif self._search(basetag, 'othercit'):
      toadd.extend(['','','','',''])
      toadd.extend(self._search(basetag,'othercit'))
    return map(list,list(izip(*toadd)))

  def __rel_list__(self):
    d_list = []
    for tag in ['continuation-in-part','continuation','division','reissue']:
      if self._search('us-related-documents',tag):
        childtmp = []
        parenttmp = []
        parenttmpsecondary = []
        for nested in ['doc-number','country','kind']:
          data = self._search('us-related-documents',tag,'relation','child-doc',nested)
          childtmp.extend(data[0] if data else [''])
        for nested in ['doc-number','country','kind','date','parent-status']:
          data = self._search('us-related-documents',tag,'relation','parent-doc',nested)
          parenttmp.extend(data[0] if data else [''])
        for nested in ['doc-number','country','kind','date','parent-status']:
          data = self._search('us-related-documents',tag,'relation','parent-doc','parent-grant-document',nested)
          if data:
            parenttmpsecondary.extend(data[0])
          else:
            data = self._search('us-related-documents',tag,'relation','parent-doc','parent-pct-document',nested)
            parenttmpsecondary.extend(data[0] if data else [''])
        if self.has_data(childtmp):
          d_list.append([tag, -1] + childtmp)
        if self.has_data(parenttmp):
          d_list.append([tag, 1] + parenttmp)
        if self.has_data(parenttmpsecondary):
          d_list.append([tag, 1] + parenttmpsecondary)
    for tag in ['related-publication', 'us-provisional-application']:
      if self._search('us-related-documents', tag):
        for nested in ['doc-number','country','kind']:
          pass
    return d_list

  def __inv_list__(self):
    toadd = []
    for nested in ['last-name', 'first-name']:
      data = self._search('parties','applicant', 'addressbook', nested)
      toadd.append(data[0] if data else [''])
    for nested in ['street','city','state','country','postcode']:
      data = self._search('parties','applicant', 'addressbook', nested)
      toadd.append(data[0] if data else [''])
      data = self._search('parties','applicant', 'address', nested)
      toadd.append(data[0] if data else [''])
    for nested in ['nationality','residence']:
      data = self._search('parties','applicant',nested,'country')
      toadd.append(data[0] if data else [''])
    return map(list,list(izip(*toadd)))

  def __law_list__(self):
    pass

  def endDocument(self):
    self.country = self._search('publication-reference','country')[0][0]
    self.patent = self._search('publication-reference','doc-number')[0][0]
    self.kind = self._search('publication-reference','kind')[0][0]
    self.date_grant = self._search('publication-reference','date')[0][0]
    self.pat_type = self.attributes['appl-type']
    self.date_app = self._search('application-reference','date')[0][0]
    self.country_app = self._search('application-reference','country')[0][0]
    self.patent_app = self._search('application-reference','doc-number')[0][0]
    self.code_app = self._search('us-application-series-code')[0][0]
    self.clm_num = self._search('number-of-claims')[0][0]
    self.classes = self.__classes__()
    self.abstract = self._search('abstract')
    self.invention_title = self._search('invention-title')[0][0]
    self.asg_list = self.__asg_list__()
    self.cit_list = self.__cit_list__()
    self.rel_list = self.__rel_list__()
    self.inv_list = self.__inv_list__()
    self.law_list = self.__law_list__()
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

from TTransport import *
import os
import errno
import socket
import sys

class TSocketBase(TTransportBase):
  def _resolveAddr(self):
    if self._unix_socket is not None:
      return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)]
    else:
      return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG)

  def close(self):
    if self.handle:
      self.handle.close()
      self.handle = None

class TSocket(TSocketBase):
  """Socket implementation of TTransport base."""

  def __init__(self, host='localhost', port=9090, unix_socket=None):
    """Initialize a TSocket

    @param host(str)  The host to connect to.
    @param port(int)  The (TCP) port to connect to.
    @param unix_socket(str)  The filename of a unix socket to connect to.
                             (host and port will be ignored.)
    """

    self.host = host
    self.port = port
    self.handle = None
    self._unix_socket = unix_socket
    self._timeout = None

  def setHandle(self, h):
    self.handle = h

  def isOpen(self):
    return self.handle is not None

  def setTimeout(self, ms):
    if ms is None:
      self._timeout = None
    else:
      self._timeout = ms/1000.0

    if self.handle is not None:
      self.handle.settimeout(self._timeout)

  def open(self):
    try:
      res0 = self._resolveAddr()
      for res in res0:
        self.handle = socket.socket(res[0], res[1])
        self.handle.settimeout(self._timeout)
        try:
          self.handle.connect(res[4])
        except socket.error, e:
          if res is not res0[-1]:
            continue
          else:
            raise e
        break
    except socket.error, e:
      if self._unix_socket:
        message = 'Could not connect to socket %s' % self._unix_socket
      else:
        message = 'Could not connect to %s:%d' % (self.host, self.port)
      raise TTransportException(type=TTransportException.NOT_OPEN, message=message)

  def read(self, sz):
    try:
      buff = self.handle.recv(sz)
    except socket.error, e:
      if (e.args[0] == errno.ECONNRESET and
          (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))):
        # freebsd and Mach don't follow POSIX semantic of recv
        # and fail with ECONNRESET if peer performed shutdown.
        # See corresponding comment and code in TSocket::read()
        # in lib/cpp/src/transport/TSocket.cpp.
        self.close()
        # Trigger the check to raise the END_OF_FILE exception below.
        buff = ''
      else:
        raise
    if len(buff) == 0:
      raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes')
    return buff

  def write(self, buff):
    if not self.handle:
      raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open')
    sent = 0
    have = len(buff)
    while sent < have:
      plus = self.handle.send(buff)
      if plus == 0:
        raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes')
      sent += plus
      buff = buff[plus:]

  def flush(self):
    pass

class TServerSocket(TSocketBase, TServerTransportBase):
  """Socket implementation of TServerTransport base."""

  def __init__(self, host=None, port=9090, unix_socket=None):
    self.host = host
    self.port = port
    self._unix_socket = unix_socket
    self.handle = None

  def listen(self):
    res0 = self._resolveAddr()
    for res in res0:
      if res[0] is socket.AF_INET6 or res is res0[-1]:
        break

    # We need remove the old unix socket if the file exists and
    # nobody is listening on it.
    if self._unix_socket:
      tmp = socket.socket(res[0], res[1])
      try:
        tmp.connect(res[4])
      except socket.error, err:
        eno, message = err.args
        if eno == errno.ECONNREFUSED:
          os.unlink(res[4])

    self.handle = socket.socket(res[0], res[1])
    self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    if hasattr(self.handle, 'set_timeout'):
      self.handle.set_timeout(None)
    self.handle.bind(res[4])
    self.handle.listen(128)

  def accept(self):
    client, addr = self.handle.accept()
    result = TSocket()
    result.setHandle(client)
    return result
<commit_msg>THRIFT-1407: Typo in Python socket server causes Thrift to fail when we enable a global socket timout Client: py Patch: adalc<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

from TTransport import *
import os
import errno
import socket
import sys

class TSocketBase(TTransportBase):
  def _resolveAddr(self):
    if self._unix_socket is not None:
      return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)]
    else:
      return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG)

  def close(self):
    if self.handle:
      self.handle.close()
      self.handle = None

class TSocket(TSocketBase):
  """Socket implementation of TTransport base."""

  def __init__(self, host='localhost', port=9090, unix_socket=None):
    """Initialize a TSocket

    @param host(str)  The host to connect to.
    @param port(int)  The (TCP) port to connect to.
    @param unix_socket(str)  The filename of a unix socket to connect to.
                             (host and port will be ignored.)
    """

    self.host = host
    self.port = port
    self.handle = None
    self._unix_socket = unix_socket
    self._timeout = None

  def setHandle(self, h):
    self.handle = h

  def isOpen(self):
    return self.handle is not None

  def setTimeout(self, ms):
    if ms is None:
      self._timeout = None
    else:
      self._timeout = ms/1000.0

    if self.handle is not None:
      self.handle.settimeout(self._timeout)

  def open(self):
    try:
      res0 = self._resolveAddr()
      for res in res0:
        self.handle = socket.socket(res[0], res[1])
        self.handle.settimeout(self._timeout)
        try:
          self.handle.connect(res[4])
        except socket.error, e:
          if res is not res0[-1]:
            continue
          else:
            raise e
        break
    except socket.error, e:
      if self._unix_socket:
        message = 'Could not connect to socket %s' % self._unix_socket
      else:
        message = 'Could not connect to %s:%d' % (self.host, self.port)
      raise TTransportException(type=TTransportException.NOT_OPEN, message=message)

  def read(self, sz):
    try:
      buff = self.handle.recv(sz)
    except socket.error, e:
      if (e.args[0] == errno.ECONNRESET and
          (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))):
        # freebsd and Mach don't follow POSIX semantic of recv
        # and fail with ECONNRESET if peer performed shutdown.
        # See corresponding comment and code in TSocket::read()
        # in lib/cpp/src/transport/TSocket.cpp.
        self.close()
        # Trigger the check to raise the END_OF_FILE exception below.
        buff = ''
      else:
        raise
    if len(buff) == 0:
      raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes')
    return buff

  def write(self, buff):
    if not self.handle:
      raise TTransportException(type=TTransportException.NOT_OPEN, message='Transport not open')
    sent = 0
    have = len(buff)
    while sent < have:
      plus = self.handle.send(buff)
      if plus == 0:
        raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket sent 0 bytes')
      sent += plus
      buff = buff[plus:]

  def flush(self):
    pass

class TServerSocket(TSocketBase, TServerTransportBase):
  """Socket implementation of TServerTransport base."""

  def __init__(self, host=None, port=9090, unix_socket=None):
    self.host = host
    self.port = port
    self._unix_socket = unix_socket
    self.handle = None

  def listen(self):
    res0 = self._resolveAddr()
    for res in res0:
      if res[0] is socket.AF_INET6 or res is res0[-1]:
        break

    # We need remove the old unix socket if the file exists and
    # nobody is listening on it.
    if self._unix_socket:
      tmp = socket.socket(res[0], res[1])
      try:
        tmp.connect(res[4])
      except socket.error, err:
        eno, message = err.args
        if eno == errno.ECONNREFUSED:
          os.unlink(res[4])

    self.handle = socket.socket(res[0], res[1])
    self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    if hasattr(self.handle, 'settimeout'):
      self.handle.settimeout(None)
    self.handle.bind(res[4])
    self.handle.listen(128)

  def accept(self):
    client, addr = self.handle.accept()
    result = TSocket()
    result.setHandle(client)
    return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

## This file is part of Invenio.
## Copyright (C) 2008, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Search Engine Summarizer, producing summary formats such as citesummary.
The main API is summarize_records().
"""

__lastupdated__ = """$Date$"""

__revision__ = "$Id$"

import ConfigParser
import sys

if sys.hexversion < 0x2040000:
    # pylint: disable=W0622
    from sets import Set as set
    # pylint: enable=W0622

from invenio.config import CFG_INSPIRE_SITE, CFG_ETCDIR
from invenio.bibrank_citation_searcher import get_cited_by_list
from invenio.bibrank_citation_indexer import tagify
from invenio.errorlib import register_exception
from invenio.bibformat_utils import parse_tag
from invenio.search_engine_utils import get_fieldvalues
from invenio.bibauthorid_searchinterface import get_personids_from_bibrec
from invenio.bibauthorid_searchinterface import get_person_bibrecs
import search_engine
import invenio.template
websearch_templates = invenio.template.load('websearch')

def load_config_file(key):
    filename = CFG_ETCDIR + "/bibrank/" + key + ".cfg"
    config = ConfigParser.ConfigParser()
    try:
        config.readfp(open(filename))
    except StandardError:
        raise Exception('Unable to load config file %s' % filename)
    return config

CITATION_CONFIG = load_config_file('citation')

## CFG_CITESUMMARY_COLLECTIONS -- how do we break down cite summary
## results according to collections?
if CFG_INSPIRE_SITE:
    CFG_CITESUMMARY_COLLECTIONS = [['All papers', 'collection:citeable'],
                                   ['Published only', 'collection:published']]
else:
    CFG_CITESUMMARY_COLLECTIONS = [['All papers', ''],
                                   ['Published only', 'collection:article']]

## CFG_CITESUMMARY_FAME_THRESHOLDS -- how do we break down cite
## summary results into famous and less famous paper groups?
CFG_CITESUMMARY_FAME_THRESHOLDS = [
                                   (500, 1000000, 'Renowned papers (500+)'),
                                   (250, 499, 'Famous papers (250-499)'),
                                   (100, 249, 'Very well-known papers (100-249)'),
                                   (50, 99, 'Well-known papers (50-99)'),
                                   (10, 49, 'Known papers (10-49)'),
                                   (1, 9, 'Less known papers (1-9)'),
                                   (0, 0, 'Unknown papers (0)')
                                   ]

## CFG_SELFCITATIONS_THRESHOLD -- only calculate self-citations stats if
## we are dealing with less than n records
CFG_CITESUMMARY_SELFCITES_THRESHOLD = 1000

def get_authors_tags(config=CITATION_CONFIG):
    """
    Get the tags for main author, coauthors, alternative authors from config
    """
    function = config.get("rank_method", "function")

    tags_names = [
        'first_author',
        'additional_author',
        'alternative_author_name',
        'collaboration_name',
    ]

    tags = {}
    for t in tags_names:
        r_tag = config.get(function, t)
        tags[t] = tagify(parse_tag(r_tag))

    return tags

def get_authors_from_record(recID, tags):
    """Get all authors for a record

    We need this function because there's 3 different types of authors
    and to fetch each one of them we need look through MARC tags
    """
    authors = get_personids_from_bibrec(recID)

    if not authors:
        mainauth_list = get_fieldvalues(recID, tags['first_author'])
        coauth_list   = get_fieldvalues(recID, tags['additional_author'])
        extauth_list  = get_fieldvalues(recID, tags['alternative_author_name'])

        authors = set(mainauth_list)
        authors.update(coauth_list)
        authors.update(extauth_list)

    return authors

def get_collaborations_from_record(recID, tags):
    """Get all collaborations for a record"""
    return get_fieldvalues(recID, tags['collaboration_name'])

def get_coauthors(author, tags, cache):
    """Get all coauthors for an author

    Given author A, returns all the authors having published
    a record with author A
    """
    if author in cache:
        return cache[author]

    friends = set()

    # sanity check: author may not exist
    if not author:
        return friends

    try:
        authorid = int(author)
        records = get_person_bibrecs(authorid)
    except ValueError:
        records = search_engine.search_pattern(p=author, f='author')
    for recid in records:
        friends.update(get_authors_from_record(recid, tags))

    cache[author] = friends
    return friends

def compute_self_citations(recid, lciters, authors_cache, tags):
    if not lciters:
        return 0

    total_citations = 0

    authors = frozenset(get_authors_from_record(recid, tags))

    collaborations = None
    if len(authors) > 20:
        collaborations = frozenset(
            get_collaborations_from_record(recid, tags))

    if collaborations:
        # Use collaborations names
        for cit in lciters:
            cit_collaborations = frozenset(
                get_collaborations_from_record(cit, tags))
            if not collaborations.intersection(cit_collaborations):
                total_citations += 1
    else:
        # Use authors names
        for cit in lciters:
            cit_authors = get_authors_from_record(cit, tags)
            if len(cit_authors) > 20 and \
                get_collaborations_from_record(cit, tags):
                # Record from a collaboration that cites
                # a record from an author, it's fine
                total_citations += 1
            else:
                cit_authors_set = set(cit_authors)
                # Extend with circle of friends
                for author in list(cit_authors)[:20]:
                    author_friends = get_coauthors(author, tags, authors_cache)
                    cit_authors_set.update(author_friends)

                if not authors.intersection(cit_authors_set):
                    total_citations += 1

    return total_citations

def render_self_citations(d_recids, d_total_recs, ln):
    try:
        tags = get_authors_tags()
    except IndexError, e:
        register_exception(prefix="attribute " + \
            str(e) + " missing in config", alert_admin=True)
        return ""

    d_recid_citers = {}
    d_total_cites = {}
    d_avg_cites = {}
    for coll, dummy_colldef in CFG_CITESUMMARY_COLLECTIONS:
        d_total_cites[coll] = 0
        d_avg_cites[coll] = 0

        d_recid_citers[coll] = get_cited_by_list(d_recids[coll])
        authors_cache = {}
        for recid, lciters in d_recid_citers[coll]:
            d_total_cites[coll] += \
                compute_self_citations(recid, lciters, authors_cache, tags)

        if d_total_recs[coll] != 0:
            d_avg_cites[coll] = d_total_cites[coll] * 1.0 / d_total_recs[coll]

    return websearch_templates.tmpl_citesummary_minus_self_cites(
        d_total_cites, d_avg_cites, CFG_CITESUMMARY_COLLECTIONS, ln)

def summarize_records(recids, of, ln, searchpattern="", searchfield="", req=None):
    """Write summary report for records RECIDS in the format OF in language LN.
       SEARCHPATTERN and SEARCHFIELD are search query that led to RECIDS,
       for instance p='Smith, Paul' and f='author'.  They are used for links.
       REQ is the Apache/mod_python request object.
    """
    if of == 'hcs':
        # this is HTML cite summary
        html = []
        compute_self_citations_p = True

        # 1) hcs prologue:
        d_recids = {}
        d_total_recs = {}
        for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
            if not colldef:
                d_recids[coll] = recids
            else:
                d_recids[coll] = recids & search_engine.search_pattern(p=colldef)
            d_total_recs[coll] = len(d_recids[coll])
            if d_total_recs[coll] > CFG_CITESUMMARY_SELFCITES_THRESHOLD:
                compute_self_citations_p = False

        prologue = websearch_templates.tmpl_citesummary_prologue(d_total_recs, CFG_CITESUMMARY_COLLECTIONS, searchpattern, searchfield, ln)

        if not req:
            html.append(prologue)
        elif hasattr(req, "write"):
            req.write(prologue)

        # 2) hcs overview:
        d_recid_citers = {}
        d_total_cites = {}
        d_avg_cites = {}
        d_recid_citecount_l = {}
        for coll, dummy_colldef in CFG_CITESUMMARY_COLLECTIONS:
            d_total_cites[coll] = 0
            d_avg_cites[coll] = 0
            d_recid_citecount_l[coll] = []
            d_recid_citers[coll] =  get_cited_by_list(d_recids[coll])
            for recid, lciters in d_recid_citers[coll]:
                if lciters:
                    d_total_cites[coll] += len(lciters)
                    d_recid_citecount_l[coll].append((recid, len(lciters)))
            if d_total_recs[coll] != 0:
                d_avg_cites[coll] = d_total_cites[coll] * 1.0 / d_total_recs[coll]
        overview = websearch_templates.tmpl_citesummary_overview(d_total_cites,
            d_avg_cites, CFG_CITESUMMARY_COLLECTIONS, ln)

        if not req:
            html.append(overview)
        elif hasattr(req, "write"):
            req.write(overview)

        # 3) compute self-citations
        if compute_self_citations_p:
            overview = render_self_citations(d_recids, d_total_recs, ln)

            if not req:
                html.append(overview)
            elif hasattr(req, "write"):
                req.write(overview)

        header = websearch_templates.tmpl_citesummary_breakdown_header(ln)
        if not req:
            html.append(header)
        elif hasattr(req, "write"):
            req.write(header)

        # 4) hcs break down by fame:
        for low, high, fame in CFG_CITESUMMARY_FAME_THRESHOLDS:
            d_cites = {}
            for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
                d_cites[coll] = 0
                for recid, lciters in d_recid_citers[coll]:
                    numcites = 0
                    if lciters:
                        numcites = len(lciters)
                    if numcites >= low and numcites <= high:
                        d_cites[coll] += 1
            fame_info = websearch_templates.tmpl_citesummary_breakdown_by_fame(d_cites, low, high, fame, CFG_CITESUMMARY_COLLECTIONS, searchpattern, searchfield, ln)

            if not req:
                html.append(fame_info)
            elif hasattr(req, "write"):
                req.write(fame_info)

        # 5) hcs calculate h index
        d_h_factors = {}
        def comparator(x, y):
            if x[1] > y[1]:
                return -1
            elif x[1] == y[1]:
                return 0
            else: return +1
        for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
            d_h_factors[coll] = 0
            d_recid_citecount_l[coll].sort(cmp=comparator)
            #req.write(repr(d_recid_citecount_l[coll])) # DEBUG
            for citecount in d_recid_citecount_l[coll]:
                d_h_factors[coll] += 1
                if d_h_factors[coll] > citecount[1]:
                    d_h_factors[coll] -= 1
                    break
        h_idx = websearch_templates.tmpl_citesummary_h_index(d_h_factors, CFG_CITESUMMARY_COLLECTIONS, ln)

        if not req:
            html.append(h_idx)
        elif hasattr(req, "write"):
            req.write(h_idx)

        # 6) hcs epilogue:
        eplilogue = websearch_templates.tmpl_citesummary_epilogue(ln)

        if not req:
            html.append(eplilogue)
        elif hasattr(req, "write"):
            req.write(eplilogue)

        if not req:
            return "\n".join(html)
        else:
            return ''

    elif of == 'xcs':
        # this is XML cite summary
        citedbylist = get_cited_by_list(recids)
        return print_citation_summary_xml(citedbylist)

#for citation summary, code xcs/hcs (unless changed)
def print_citation_summary_xml(citedbylist):
    """Prints citation summary in xml."""
    alldict = calculate_citations(citedbylist)
    avgstr = str(alldict['avgcites'])
    totalcites = str(alldict['totalcites'])
    #format avg so that it does not span 10 digits
    avgstr = avgstr[0:4]
    reciddict = alldict['reciddict']
    #output formatting
    outp = "<citationsummary records=\""+str(len(citedbylist))
    outp += "\" citations=\""+str(totalcites)+"\">"
    for low, high, name in CFG_CITESUMMARY_FAME_THRESHOLDS:
        #get the name, print the value
        if reciddict.has_key(name):
            recs = reciddict[name]
            outp += "<citationclass>"+name
            outp += "<records>"+str(recs)+"</records>"
            outp += "</citationclass>\n"
    outp = outp + "</citationsummary>"
    #req.write(outp)
    return outp #just to return something

def calculate_citations(citedbylist):
    """calculates records in classes of citations
       defined by thresholds. returns a dictionary that
       contains total, avg, records and a dictionary
       of threshold names and number corresponding to it"""
    totalcites = 0
    avgcites = 0
    reciddict = {}
    for recid, cites in citedbylist:
        numcites = 0
        if cites:
            numcites = len(cites)
        totalcites = totalcites + numcites
        #take the numbers in CFG_CITESUMMARY_FAME_THRESHOLDS
        for low, high, name in CFG_CITESUMMARY_FAME_THRESHOLDS:
            if (numcites >= low) and (numcites <= high):
                if reciddict.has_key(name):
                    tmp = reciddict[name]
                    tmp.append(recid)
                    reciddict[name] = tmp
                else:
                    reciddict[name] = [recid]
    if (len(citedbylist) == 0):
        avgcites = 0
    else:
        avgcites = totalcites*1.0/len(citedbylist)

    #create a dictionary that contains all the values
    alldict = {}
    alldict['records'] = len(citedbylist)
    alldict['totalcites'] = totalcites
    alldict['avgcites'] = avgcites
    alldict['reciddict'] = reciddict
    return alldict
<commit_msg>BibRank: two algorithms for self-citations<commit_after># -*- coding: utf-8 -*-

## This file is part of Invenio.
## Copyright (C) 2008, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Search Engine Summarizer, producing summary formats such as citesummary.
The main API is summarize_records().
"""

__lastupdated__ = """$Date$"""

__revision__ = "$Id$"

from invenio.config import CFG_INSPIRE_SITE, \
                           CFG_WEBSEARCH_CITESUMMARY_SELFCITES_THRESHOLD
from invenio.bibrank_citation_searcher import get_cited_by_list
from invenio.bibrank_selfcites_indexer import get_self_citations_count
import search_engine
import invenio.template
websearch_templates = invenio.template.load('websearch')

## CFG_CITESUMMARY_COLLECTIONS -- how do we break down cite summary
## results according to collections?
if CFG_INSPIRE_SITE:
    CFG_CITESUMMARY_COLLECTIONS = [['All papers', 'collection:citeable'],
                                   ['Published only', 'collection:published']]
else:
    CFG_CITESUMMARY_COLLECTIONS = [['All papers', ''],
                                   ['Published only', 'collection:article']]

## CFG_CITESUMMARY_FAME_THRESHOLDS -- how do we break down cite
## summary results into famous and less famous paper groups?
CFG_CITESUMMARY_FAME_THRESHOLDS = [
                                   (500, 1000000, 'Renowned papers (500+)'),
                                   (250, 499, 'Famous papers (250-499)'),
                                   (100, 249, 'Very well-known papers (100-249)'),
                                   (50, 99, 'Well-known papers (50-99)'),
                                   (10, 49, 'Known papers (10-49)'),
                                   (1, 9, 'Less known papers (1-9)'),
                                   (0, 0, 'Unknown papers (0)')
                                   ]

def render_self_citations(d_recids, d_total_recs, ln):
    """Render the html displayed for self-citations"""
    d_recid_citers = {}
    d_total_cites = {}
    d_avg_cites = {}
    for coll, dummy_colldef in CFG_CITESUMMARY_COLLECTIONS:
        d_total_cites[coll] = 0
        d_avg_cites[coll] = 0

        d_recid_citers[coll] = get_cited_by_list(d_recids[coll])
        d_total_cites[coll] = get_self_citations_count(d_recids[coll])

        if d_total_recs[coll] != 0:
            d_avg_cites[coll] = d_total_cites[coll] * 1.0 / d_total_recs[coll]

    return websearch_templates.tmpl_citesummary_minus_self_cites(
        d_total_cites, d_avg_cites, CFG_CITESUMMARY_COLLECTIONS, ln)

def summarize_records(recids, of, ln, searchpattern="", searchfield="", req=None):
    """Write summary report for records RECIDS in the format OF in language LN.
       SEARCHPATTERN and SEARCHFIELD are search query that led to RECIDS,
       for instance p='Smith, Paul' and f='author'.  They are used for links.
       REQ is the Apache/mod_python request object.
    """
    if of == 'hcs':
        # this is HTML cite summary
        html = []
        compute_self_citations_p = True

        # 1) hcs prologue:
        d_recids = {}
        d_total_recs = {}
        for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
            if not colldef:
                d_recids[coll] = recids
            else:
                d_recids[coll] = recids & search_engine.search_pattern(p=colldef)
            d_total_recs[coll] = len(d_recids[coll])
            if d_total_recs[coll] > CFG_WEBSEARCH_CITESUMMARY_SELFCITES_THRESHOLD:
                compute_self_citations_p = False

        prologue = websearch_templates.tmpl_citesummary_prologue(d_total_recs, CFG_CITESUMMARY_COLLECTIONS, searchpattern, searchfield, ln)

        if not req:
            html.append(prologue)
        elif hasattr(req, "write"):
            req.write(prologue)

        # 2) hcs overview:
        d_recid_citers = {}
        d_total_cites = {}
        d_avg_cites = {}
        d_recid_citecount_l = {}
        for coll, dummy_colldef in CFG_CITESUMMARY_COLLECTIONS:
            d_total_cites[coll] = 0
            d_avg_cites[coll] = 0
            d_recid_citecount_l[coll] = []
            d_recid_citers[coll] =  get_cited_by_list(d_recids[coll])
            for recid, lciters in d_recid_citers[coll]:
                if lciters:
                    d_total_cites[coll] += len(lciters)
                    d_recid_citecount_l[coll].append((recid, len(lciters)))
            if d_total_recs[coll] != 0:
                d_avg_cites[coll] = d_total_cites[coll] * 1.0 / d_total_recs[coll]
        overview = websearch_templates.tmpl_citesummary_overview(d_total_cites,
            d_avg_cites, CFG_CITESUMMARY_COLLECTIONS, ln)

        if not req:
            html.append(overview)
        elif hasattr(req, "write"):
            req.write(overview)

        # 3) compute self-citations
        if compute_self_citations_p:
            overview = render_self_citations(d_recids, d_total_recs, ln)

            if not req:
                html.append(overview)
            elif hasattr(req, "write"):
                req.write(overview)

        header = websearch_templates.tmpl_citesummary_breakdown_header(ln)
        if not req:
            html.append(header)
        elif hasattr(req, "write"):
            req.write(header)

        # 4) hcs break down by fame:
        for low, high, fame in CFG_CITESUMMARY_FAME_THRESHOLDS:
            d_cites = {}
            for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
                d_cites[coll] = 0
                for recid, lciters in d_recid_citers[coll]:
                    numcites = 0
                    if lciters:
                        numcites = len(lciters)
                    if numcites >= low and numcites <= high:
                        d_cites[coll] += 1
            fame_info = websearch_templates.tmpl_citesummary_breakdown_by_fame(d_cites, low, high, fame, CFG_CITESUMMARY_COLLECTIONS, searchpattern, searchfield, ln)

            if not req:
                html.append(fame_info)
            elif hasattr(req, "write"):
                req.write(fame_info)

        # 5) hcs calculate h index
        d_h_factors = {}
        def comparator(x, y):
            if x[1] > y[1]:
                return -1
            elif x[1] == y[1]:
                return 0
            else: return +1
        for coll, colldef in CFG_CITESUMMARY_COLLECTIONS:
            d_h_factors[coll] = 0
            d_recid_citecount_l[coll].sort(cmp=comparator)
            #req.write(repr(d_recid_citecount_l[coll])) # DEBUG
            for citecount in d_recid_citecount_l[coll]:
                d_h_factors[coll] += 1
                if d_h_factors[coll] > citecount[1]:
                    d_h_factors[coll] -= 1
                    break
        h_idx = websearch_templates.tmpl_citesummary_h_index(d_h_factors, CFG_CITESUMMARY_COLLECTIONS, ln)

        if not req:
            html.append(h_idx)
        elif hasattr(req, "write"):
            req.write(h_idx)

        # 6) hcs epilogue:
        eplilogue = websearch_templates.tmpl_citesummary_epilogue(ln)

        if not req:
            html.append(eplilogue)
        elif hasattr(req, "write"):
            req.write(eplilogue)

        if not req:
            return "\n".join(html)
        else:
            return ''

    elif of == 'xcs':
        # this is XML cite summary
        citedbylist = get_cited_by_list(recids)
        return print_citation_summary_xml(citedbylist)

#for citation summary, code xcs/hcs (unless changed)
def print_citation_summary_xml(citedbylist):
    """Prints citation summary in xml."""
    alldict = calculate_citations(citedbylist)
    avgstr = str(alldict['avgcites'])
    totalcites = str(alldict['totalcites'])
    #format avg so that it does not span 10 digits
    avgstr = avgstr[0:4]
    reciddict = alldict['reciddict']
    #output formatting
    outp = "<citationsummary records=\""+str(len(citedbylist))
    outp += "\" citations=\""+str(totalcites)+"\">"
    for low, high, name in CFG_CITESUMMARY_FAME_THRESHOLDS:
        #get the name, print the value
        if reciddict.has_key(name):
            recs = reciddict[name]
            outp += "<citationclass>"+name
            outp += "<records>"+str(recs)+"</records>"
            outp += "</citationclass>\n"
    outp = outp + "</citationsummary>"
    #req.write(outp)
    return outp #just to return something

def calculate_citations(citedbylist):
    """calculates records in classes of citations
       defined by thresholds. returns a dictionary that
       contains total, avg, records and a dictionary
       of threshold names and number corresponding to it"""
    totalcites = 0
    avgcites = 0
    reciddict = {}
    for recid, cites in citedbylist:
        numcites = 0
        if cites:
            numcites = len(cites)
        totalcites = totalcites + numcites
        #take the numbers in CFG_CITESUMMARY_FAME_THRESHOLDS
        for low, high, name in CFG_CITESUMMARY_FAME_THRESHOLDS:
            if (numcites >= low) and (numcites <= high):
                if reciddict.has_key(name):
                    tmp = reciddict[name]
                    tmp.append(recid)
                    reciddict[name] = tmp
                else:
                    reciddict[name] = [recid]
    if (len(citedbylist) == 0):
        avgcites = 0
    else:
        avgcites = totalcites*1.0/len(citedbylist)

    #create a dictionary that contains all the values
    alldict = {}
    alldict['records'] = len(citedbylist)
    alldict['totalcites'] = totalcites
    alldict['avgcites'] = avgcites
    alldict['reciddict'] = reciddict
    return alldict
<|endoftext|>


<commit_before>import sublime, sublime_plugin
import subprocess, os
import json, codecs, re

class Jfile:
    def __init__(self, fpath, encoding="utf-8"):
        self.encoding = encoding
        self.fpath = fpath

    def load(self, default=[]):
        self.fdir = os.path.dirname(self.fpath)
        if not os.path.isdir(self.fdir):
            os.makedirs(self.fdir)
        if os.path.exists(self.fpath):
            f = codecs.open(self.fpath, "r+", encoding=self.encoding)
            content = f.read()
            content = re.sub(r'^\s*//.*?$|^\s*/\*(?:.|\n)*?\*/', '', content, flags=re.MULTILINE)
            try:
                data = json.loads(content)
            except:
                data = default
            f.close()
        else:
            f = codecs.open(self.fpath, "w+", encoding=self.encoding)
            data = default
            f.close()
        return data

    def save(self, data, indent=4):
        self.fdir = os.path.dirname(self.fpath)
        if not os.path.isdir(self.fdir):
            os.makedirs(self.fdir)
        f = codecs.open(self.fpath, "w+", encoding=self.encoding)
        f.write(json.dumps(data, ensure_ascii=False, indent=indent))
        f.close()

    def remove(self):
        if os.path.exists(self.fpath): os.remove(self.fpath)

def subl(args=[]):
    # learnt from SideBarEnhancements
    executable_path = sublime.executable_path()
    if sublime.platform() == 'osx':
        app_path = executable_path[:executable_path.rfind(".app/")+5]
        executable_path = app_path+"Contents/SharedSupport/bin/subl"
    subprocess.Popen([executable_path]+args)

class Manager:
    def __init__(self, window):
        self.window = window
        self.projects_dir = os.path.join(sublime.packages_path(), "User", "Projects")

    def sublime_project(self, project):
        return os.path.join(self.projects_dir, "%s.sublime-project" % project)

    def sublime_workspace(self, project):
        return os.path.join(self.projects_dir, "%s.sublime-workspace" % project)

    def add_project(self):
        pd = self.window.project_data()
        if not pd:
            self.window.run_command("prompt_add_folder")
            delay = 300
        else:
            delay = 1

        def on_add(project):
            pd = self.window.project_data()
            Jfile(self.sublime_project(project)).save(pd)
            Jfile(self.sublime_workspace(project)).save({})
            self.window.run_command("close_workspace")
            self.window.run_command("close_project")
            self.switch_project(project)

        def show_input_panel():
            pd = self.window.project_data()
            if pd:
                project = os.path.basename(pd["folders"][0]["path"])
                self.window.show_input_panel("Project name:", project, on_add, None, None)

        sublime.set_timeout(show_input_panel, delay)

    def list_projects(self):
        if os.path.exists(self.projects_dir):
            ret = []
            for f in os.listdir(self.projects_dir):
                if f.endswith(".sublime-project"):
                    pd = Jfile(os.path.join(self.projects_dir,f)).load()
                    if pd and "folders" in pd and pd["folders"]:
                        ret.append([f.replace(".sublime-project",""), pd["folders"][0].get("path", "")])
                    else:
                        ret.append([f.replace(".sublime-project",""),""])
            return ret
        else:
            return []

    def get_project_data(self, project):
        return Jfile(self.sublime_project(project)).load()

    def check_project(self, project):
        if not os.path.exists(self.sublime_workspace(project)):
            Jfile(self.sublime_workspace).save({})

    def append_project(self, project):
        pd = self.get_project_data(project)
        paths = [f.get("path") for f in pd.get("folders")]
        subl(["-a"] + paths)

    def switch_project(self, project):
        self.window.run_command("close_workspace")
        self.check_project(project)
        # if project already opened, change focus
        for w in sublime.windows():
            print(w.project_file_name(), self.sublime_project(project))
            if w.project_file_name() == self.sublime_project(project):
                w.run_command("close_workspace")
                w.run_command("close_window")
                sublime.set_timeout_async(lambda: subl(["-n", self.sublime_project(project)]), 300)
                return

        if len(self.window.views())==0:
            sublime.set_timeout_async(lambda: subl([self.sublime_project(project)]), 300)
        else:
            sublime.set_timeout_async(lambda: subl(["-n", self.sublime_project(project)]), 300)

    def open_in_new_window(self, project):
        self.check_project(project)
        subl(["-n", self.sublime_project(project)])

    def remove_project(self, project):
        ok = sublime.ok_cancel_dialog("Remove Project %s?" % project)
        if ok:
            if self.window.project_file_name() == self.sublime_project(project):
                self.window.run_command("close_workspace")
            os.unlink(self.sublime_project(project))
            os.unlink(self.sublime_workspace(project))

    def edit_project(self, project):
        self.window.open_file(self.sublime_project(project))

    def rename_project(self, project):
        def on_rename(new_project):
            sublime_project = self.sublime_project(project)
            new_sublime_project = self.sublime_project(new_project)
            sublime_workspace = self.sublime_workspace(project)
            new_sublime_workspace = self.sublime_workspace(new_project)
            if self.window.project_file_name() == sublime_project:
                reopen = True
                self.window.run_command("close_workspace")
            else:
                reopen = False
            os.rename(sublime_project, new_sublime_project)
            os.rename(sublime_workspace, new_sublime_workspace)
            try:
                j = Jfile(new_sublime_workspace)
                data = j.load({})
                data["project"] = "%s.sublime-project" % new_project
                j.save(data)
            except:
                pass
            if reopen:
                self.switch_project(new_project)
        self.window.show_input_panel("New project name:", project, on_rename, None, None)

class ProjectManager(sublime_plugin.WindowCommand):

    def show_quick_panel(self, items, on_done):
        sublime.set_timeout(
            lambda: self.window.show_quick_panel(items, on_done),
            10)

    def run(self):
        self.manager = Manager(self.window)
        self.project_list = self.manager.list_projects()
        self.options = [
                ["[-] Project Manager", "More options"],
                ["[-] Add Project", "Add project to Project Manager"]
            ]

        self.show_quick_panel(self.options + self.project_list, self.on_open)

    def on_open(self, action):
        if action<0:
            return

        elif action==0:
            items = [
                ["Open Project", "Open project in current window"],
                ["Open Project in New Window", "Open project in a new window"],
                ["Append Project", "Append project to current window"],
                ["Edit Project", "Edit project settings"],
                ['Rename Project', "Rename project"],
                ["Remove Project", "Remove from Project Manager"]
            ]
            def callback(a):
                if a==0:
                    self.window.run_command("project_manager_list", args={"action": "switch"})
                elif a==1:
                    self.window.run_command("project_manager_list", args={"action": "new"})
                elif a==2:
                    self.window.run_command("project_manager_list", args={"action": "append"})
                elif a==3:
                    self.window.run_command("project_manager_list", args={"action": "edit"})
                elif a==4:
                    self.window.run_command("project_manager_list", args={"action": "rename"})
                elif a==5:
                    self.window.run_command("project_manager_list", args={"action": "remove"})
                else:
                    sublime.set_timeout(self.run, 10)

            self.show_quick_panel(items, callback)

        elif action==1:
            self.manager.add_project()

        elif action>=len(self.options):
            action = action-len(self.options)
            self.manager.switch_project(self.project_list[action][0])

class ProjectManagerAdd(sublime_plugin.WindowCommand):
    def run(self):
        self.manager = Manager(self.window)
        self.manager.add_project()

class ProjectManagerList(sublime_plugin.WindowCommand):

    def show_quick_panel(self, items, on_done):
        sublime.set_timeout(
            lambda: self.window.show_quick_panel(items, on_done),
            10)

    def run(self, action):
        self.manager = Manager(self.window)
        self.project_list = self.manager.list_projects()
        callback = eval("self.on_" + action)
        self.show_quick_panel(self.project_list, callback)

    def on_new(self, action):
        if action>=0:
            self.manager.open_in_new_window(self.project_list[action][0])

    def on_switch(self, action):
        if action>=0:
            self.manager.switch_project(self.project_list[action][0])

    def on_append(self, action):
        if action>=0:
            self.manager.append_project(self.project_list[action][0])

    def on_remove(self, action):
        if action>=0:
            self.manager.remove_project(self.project_list[action][0])

    def on_edit(self, action):
        if action>=0:
            self.manager.edit_project(self.project_list[action][0])

    def on_rename(self, action):
        if action>=0:
            self.manager.rename_project(self.project_list[action][0])
<commit_msg>gui update<commit_after>import sublime, sublime_plugin
import subprocess, os
import json, codecs, re
import copy

class Jfile:
    def __init__(self, fpath, encoding="utf-8"):
        self.encoding = encoding
        self.fpath = fpath

    def load(self, default=[]):
        self.fdir = os.path.dirname(self.fpath)
        if not os.path.isdir(self.fdir):
            os.makedirs(self.fdir)
        if os.path.exists(self.fpath):
            f = codecs.open(self.fpath, "r+", encoding=self.encoding)
            content = f.read()
            content = re.sub(r'^\s*//.*?$|^\s*/\*(?:.|\n)*?\*/', '', content, flags=re.MULTILINE)
            try:
                data = json.loads(content)
            except:
                data = default
            f.close()
        else:
            f = codecs.open(self.fpath, "w+", encoding=self.encoding)
            data = default
            f.close()
        return data

    def save(self, data, indent=4):
        self.fdir = os.path.dirname(self.fpath)
        if not os.path.isdir(self.fdir):
            os.makedirs(self.fdir)
        f = codecs.open(self.fpath, "w+", encoding=self.encoding)
        f.write(json.dumps(data, ensure_ascii=False, indent=indent))
        f.close()

    def remove(self):
        if os.path.exists(self.fpath): os.remove(self.fpath)

def subl(args=[]):
    # learnt from SideBarEnhancements
    executable_path = sublime.executable_path()
    if sublime.platform() == 'osx':
        app_path = executable_path[:executable_path.rfind(".app/")+5]
        executable_path = app_path+"Contents/SharedSupport/bin/subl"
    subprocess.Popen([executable_path]+args)

class Manager:
    def __init__(self, window):
        self.window = window
        self.projects_dir = os.path.join(sublime.packages_path(), "User", "Projects")

    def sublime_project(self, project):
        return os.path.join(self.projects_dir, "%s.sublime-project" % project)

    def sublime_workspace(self, project):
        return os.path.join(self.projects_dir, "%s.sublime-workspace" % project)

    def add_project(self):
        pd = self.window.project_data()
        if not pd:
            self.window.run_command("prompt_add_folder")
            delay = 300
        else:
            delay = 1

        def on_add(project):
            pd = self.window.project_data()
            Jfile(self.sublime_project(project)).save(pd)
            Jfile(self.sublime_workspace(project)).save({})
            self.window.run_command("close_workspace")
            self.window.run_command("close_project")
            self.switch_project(project)

        def show_input_panel():
            pd = self.window.project_data()
            if pd:
                project = os.path.basename(pd["folders"][0]["path"])
                self.window.show_input_panel("Project name:", project, on_add, None, None)

        sublime.set_timeout(show_input_panel, delay)

    def list_projects(self):
        if os.path.exists(self.projects_dir):
            ret = []
            for f in os.listdir(self.projects_dir):
                if f.endswith(".sublime-project"):
                    pd = Jfile(os.path.join(self.projects_dir,f)).load()
                    if pd and "folders" in pd and pd["folders"]:
                        ret.append([f.replace(".sublime-project",""), pd["folders"][0].get("path", "")])
                    else:
                        ret.append([f.replace(".sublime-project",""),""])
            return ret
        else:
            return []

    def get_project_data(self, project):
        return Jfile(self.sublime_project(project)).load()

    def check_project(self, project):
        if not os.path.exists(self.sublime_workspace(project)):
            Jfile(self.sublime_workspace).save({})

    def append_project(self, project):
        pd = self.get_project_data(project)
        paths = [f.get("path") for f in pd.get("folders")]
        subl(["-a"] + paths)

    def switch_project(self, project):
        self.window.run_command("close_workspace")
        self.check_project(project)
        for w in sublime.windows():
            if w.project_file_name() == self.sublime_project(project):
                w.run_command("close_workspace")
                w.run_command("close_window")
                sublime.set_timeout_async(lambda: subl(["-n", self.sublime_project(project)]), 300)
                return

        if len(self.window.views())==0:
            sublime.set_timeout_async(lambda: subl([self.sublime_project(project)]), 300)
        else:
            sublime.set_timeout_async(lambda: subl(["-n", self.sublime_project(project)]), 300)

    def open_in_new_window(self, project):
        self.check_project(project)
        subl(["-n", self.sublime_project(project)])

    def remove_project(self, project):
        ok = sublime.ok_cancel_dialog("Remove Project %s?" % project)
        if ok:
            if self.window.project_file_name() == self.sublime_project(project):
                self.window.run_command("close_workspace")
            os.unlink(self.sublime_project(project))
            os.unlink(self.sublime_workspace(project))

    def edit_project(self, project):
        self.window.open_file(self.sublime_project(project))

    def rename_project(self, project):
        def on_rename(new_project):
            sublime_project = self.sublime_project(project)
            new_sublime_project = self.sublime_project(new_project)
            sublime_workspace = self.sublime_workspace(project)
            new_sublime_workspace = self.sublime_workspace(new_project)
            if self.window.project_file_name() == sublime_project:
                reopen = True
                self.window.run_command("close_workspace")
            else:
                reopen = False
            os.rename(sublime_project, new_sublime_project)
            os.rename(sublime_workspace, new_sublime_workspace)
            try:
                j = Jfile(new_sublime_workspace)
                data = j.load({})
                data["project"] = "%s.sublime-project" % new_project
                j.save(data)
            except:
                pass
            if reopen:
                self.switch_project(new_project)
        self.window.show_input_panel("New project name:", project, on_rename, None, None)

class ProjectManager(sublime_plugin.WindowCommand):

    def show_quick_panel(self, items, on_done):
        sublime.set_timeout(
            lambda: self.window.show_quick_panel(items, on_done),
            10)

    def run(self, open_manager=False):
        self.manager = Manager(self.window)
        self.project_list = self.manager.list_projects()
        self.options = [
                ["[-] Project Manager", "More options"],
                ["[-] Add Project", "Add project to Project Manager"]
            ]
        if open_manager:
            self.on_open(0)
        else:
            self.show_quick_panel(self.options + self.project_list, self.on_open)

    def on_open(self, action):
        if action<0:
            return

        elif action==0:
            items = [
                ["Open Project", "Open project in current window"],
                ["Open Project in New Window", "Open project in a new window"],
                ["Append Project", "Append project to current window"],
                ["Edit Project", "Edit project settings"],
                ['Rename Project', "Rename project"],
                ["Remove Project", "Remove from Project Manager"]
            ]
            def callback(a):
                if a==0:
                    self.window.run_command("project_manager_list", args={"action": "switch", "caller" : "manager"})
                elif a==1:
                    self.window.run_command("project_manager_list", args={"action": "new", "caller" : "manager"})
                elif a==2:
                    self.window.run_command("project_manager_list", args={"action": "append", "caller" : "manager"})
                elif a==3:
                    self.window.run_command("project_manager_list", args={"action": "edit", "caller" : "manager"})
                elif a==4:
                    self.window.run_command("project_manager_list", args={"action": "rename", "caller" : "manager"})
                elif a==5:
                    self.window.run_command("project_manager_list", args={"action": "remove", "caller" : "manager"})
                else:
                    sublime.set_timeout(self.run, 10)

            self.show_quick_panel(items, callback)

        elif action==1:
            self.manager.add_project()

        elif action>=len(self.options):
            action = action-len(self.options)
            self.manager.switch_project(self.project_list[action][0])

class ProjectManagerAdd(sublime_plugin.WindowCommand):
    def run(self):
        self.manager = Manager(self.window)
        self.manager.add_project()

class ProjectManagerList(sublime_plugin.WindowCommand):

    def show_quick_panel(self, items, on_done):
        sublime.set_timeout(
            lambda: self.window.show_quick_panel(items, on_done),
            10)

    def run(self, action, caller=None):
        self.manager = Manager(self.window)
        self.project_list = self.manager.list_projects()
        display = copy.deepcopy(self.project_list)
        self.caller = caller
        callback = eval("self.on_" + action)
        for i, item in enumerate(display):
            if self.manager.sublime_project(item[0]) == self.window.project_file_name():
                display[i][0] = display[i][0] + "*"
                display.insert(0, display.pop(i))
                self.project_list.insert(0, self.project_list.pop(i))
                break
        self.show_quick_panel(display, callback)

    def on_new(self, action):
        if action>=0:
            self.manager.open_in_new_window(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_switch(self, action):
        if action>=0:
            self.manager.switch_project(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_append(self, action):
        if action>=0:
            self.manager.append_project(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_remove(self, action):
        if action>=0:
            self.manager.remove_project(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_edit(self, action):
        if action>=0:
            self.manager.edit_project(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_rename(self, action):
        if action>=0:
            self.manager.rename_project(self.project_list[action][0])
        elif action<0:
            self.on_cancel()

    def on_cancel(self):
        if self.caller == "manager":
            self.window.run_command("project_manager", args={"open_manager": True})
<|endoftext|>


<commit_before>import gate
test_f = "/home/nick/test/gate/pers_ment/11-JB_2015-07-12_PM_annotation.xml"
annotation_file = gate.AnnotationFile(test_f)
annotations = sorted(
    list( annotation_file.iter_annotations() ),
    key = lambda x : ( x._start_node, x._end_node )
)

for i, annotation in enumerate(annotations):
    if annotation._type.lower() == "token":
        if gate.is_overlapping(
            [
                annotation,
                annotations[i-1],
            ]
        ):
            print(
                annotation._type,
                annotations[i-1]._type
            )
            #TODO: try optimizing iter_overlapping_annotations()
            ## use something like a focal search
            #TODO: implement BIO tagging
        # j = i
        # while True:
            # annotation_type = annotations[j]._type
            # print(annotation_type)
            # if annotation_type.lower() == "person":
                # print()
                # break
            # j = j - 1
<commit_msg>Deleted pm.py<commit_after><|endoftext|>


<commit_before>import sqlite3
import sys

PREFIX = {}
OUTPUT_VAR = []
STATE = {'IN_PREFIX': False,
         'IN_SELECT': False,
         'IN_WHERE': False,
         'IN_FILTER': False}
SUB_QUERIES = []
SUB_VARS = []
FILTERS = []
FILTER_VAR = []

def main():
    if len(sys.argv) != 3:
        print("Usage: python q8.py <database file> <SPARQL query file>")
        sys.exit()

    infile = open(sys.argv[2], 'r')  # read the SPARQL query file
    whole_file = infile.read()
    infile.close()

    input_lines = reformat(whole_file)
    query = parseFile(input_lines)

    print(query)

    # conn = sqlite3.connect(sys.argv[1])  # connection to the database
    # c = conn.cursor()  # cursor
    # conn.create_function("convert",1,convert)
    # c.execute(query)
    # print(c.fetchall())

def convert(value):
    """
    this is a sql function that will try to convert value to int or float
    :param value: value of the object column that returned by a query
    :return: the converted value
    """
    try:
        if "." in value:
            a = float(value)
        else:
            a = int(value)

        return a
    except ValueError:
        return value

def reformat(string):
    """
    change the format of file so that:
      1. "SELECT" and "WHERE" will be in different lines
      2. (subj pred obj) and "WHERE {" will be in different lines
      3. there will be at least one space between SELECT and first ?var or *
      4. there will be at least one space between WHERE and {
      5. there will be at least one space between FILTER and (
    :param string: all the characters in the input file
    :return: a list of each line in the formatted file
    """
    # TODO: will the select and ?var in different lines?

    # changes #3
    select_index = string.upper().index("\nSELECT") + 6
    string = string[:select_index + 1] + " " + string[select_index + 1:]

    # changes #1,#2,#4
    where_index = 0
    while where_index < select_index:
        where_index = string.upper().find("WHERE", where_index + 1) + 4
    curly_open = string.find("{", where_index + 1)
    string = string[:where_index - 4] + "\nWHERE {" + string[curly_open + 1:]

    # changes #5
    filter_index = string.upper().find("FILTER", 0)
    while filter_index >= 0:
        string = string[:filter_index + 6] + " " + string[filter_index + 6:]
        filter_index = string.upper().find("FILTER", filter_index + 1)

    return string.split("\n")

def parseFile(sparql_lines):
    """
    parse the SPARQL query, translate and return it as an sqlite query
    :param sparql_lines: a list of each line in the SPARQL query file
    :return: a string containing sqlite query
    """
    for line in sparql_lines:
        line = line.strip()
        if line == "":  # if the line is blank, go to next line
            continue
        line = line.split()

        if line[0].upper() == "PREFIX":
            STATE['IN_PREFIX'] = True
            if len(line) == 3:  # if there are spaces between ':' and <url>
                PREFIX[line[1][:-1]] = line[2][1:-1]
            elif len(line) == 2:  # if there's no space:
                index_of_colon = line[1].index(":")
                PREFIX[line[1][:index_of_colon]] = line[1][index_of_colon + 2:-1]
            else:
                print("Unexpected error when reading prefix line with tokens:", line)
                sys.exit()

        elif line[0].upper() == "SELECT":
            STATE['IN_PREFIX'] = False
            STATE['IN_SELECT'] = True
            for token in line:
                if token.upper() == "SELECT":
                    continue
                elif "*" in token:
                    OUTPUT_VAR.append("*")
                    break
                elif "?" in token:
                    OUTPUT_VAR.append(token[1:])  # add the ?var into the outputVar list
                else:
                    print("Unexpected error when reading ?var in SELECT line with tokens:", line, "token:", token)
                    sys.exit()

        elif line[0].upper() == "WHERE":
            STATE['IN_SELECT'] = False
            STATE['IN_WHERE'] = True

        elif STATE['IN_WHERE']:
            if line[0][:6].upper() == "FILTER":  # if this is a filter line
                addFilter(line[1:])
            elif line[0] == "}":
                STATE['IN_WHERE'] = False
            else:
                readPattern(line)

        else:
            print("Unexpected error while parsing line with tokens:", line)
            sys.exit()

    result = buildQuery()
    return result

def addFilter(tokens):
    """
    add the filter line to the global variable
    :param tokens: all the token in the filter line
    """
    filter_line = "".join(tokens)
    variable_start = filter_line.find("?")
    var_name = ""
    variable_end = variable_start
    for char in filter_line[variable_start + 1:]:
        if char not in ",><=)":
            var_name += char
            variable_end += 1
        else:
            break
    FILTER_VAR.append(var_name)
    if "regex" in filter_line:

        first_quote = filter_line.find('''"''')
        if first_quote > 0:
            second_quote = filter_line.find('''"''', first_quote + 1)
        else:
            first_quote = filter_line.find("'")
            if first_quote < 0:
                print("error")
                sys.exit()
            second_quote = filter_line.find("'", first_quote + 1)
        target_string = filter_line[first_quote + 1:second_quote]

        FILTERS.append('''%s = "%s"''' % (var_name, target_string))
    else:
        # it has to be a number
        filter_line = filter_line[:variable_start + 1] + filter_line[variable_end + 1:]
        number = ""
        operation = ""
        for char in filter_line:
            if char.isnumeric() or char == ".":
                number += char

        operations = ["!=", ">=", "<=", "=", ">", "<"]
        for op in operations:
            if op in filter_line:
                operation = op
                break
        filter_line.find("!")
        FILTERS.append("%s %s %s" % (var_name, operation, number))

def readPattern(pattern):
    """
    read a (subj pred obej) pattern, store the variable and sub-sql-query in the dictionary
    :param pattern: a list as [subj, pred, obej] pattern
    """

    query_template = {(0,): ("subject as %s", '''predicate = "%s" and object = "%s"''', "predicate_object_index"),
                      (1,): ("predicate as %s", '''subject = "%s" and object = "%s"''', "subject_object_index"),
                      (2,): (
                      "convert(object) as %s", '''subject = "%s" and predicate = "%s"''', "subject_predicate_index"),
                      (0, 1): ("subject as %s, predicate as %s", '''object = "%s"''', "object_index"),
                      (0, 2): ("subject as %s, convert(object) as %s", '''predicate = "%s"''', "predicate_index"),
                      (1, 2): ("predicate as %s, convert(object) as %s", '''subject = "%s"''', "subject_index")}

    if pattern[2][-1] == ".":
        pattern[2] = pattern[2][:-1]

    variables = []
    conditions = []
    template_number = []

    for i in range(3):
        if pattern[i][0] == "?":  # if the term is a variable
            variables.append(pattern[i][1:])
            template_number.append(i)
        elif pattern[i][0] == "<":  # if the term is a url
            conditions.append(pattern[i][1:-1])
        elif ":" in pattern[i]:  # if it's a url with prefix
            index = pattern[i].index(":")
            if pattern[i][:index] not in PREFIX:
                print("Unknown prefix: ", pattern[i])
                sys.exit()
            url = PREFIX[pattern[i][:index]] + pattern[i][index + 1:]
            conditions.append(url)
        else:
            conditions.append(pattern[i])
    template_number = tuple(template_number)

    query = "SELECT DISTINCT %s \nFROM graph_data INDEXED BY %s \nWHERE %s " \
            % (query_template[template_number][0] % tuple(variables),
               query_template[template_number][2],
               query_template[template_number][1] % tuple(conditions))

    SUB_VARS.append(variables)
    SUB_QUERIES.append(query)

def buildQuery():
    """
    use the global variables to construct a final query
    :return: the final sql query
    """
    keys = SUB_VARS
    current_vars = set(keys.pop(0))
    # combine all the sub-queries:
    query = SUB_QUERIES.pop(0)
    used = []
    while len(used) != len(keys):
        for i in range(len(keys)):
            variables = keys[i]
            if not current_vars.isdisjoint(set(variables)):
                common_vars = tuple(current_vars.intersection(set(variables)))
                current_vars = current_vars.union(set(variables))
                query = "SELECT * \nFROM (\n(%s) \nJOIN \n(%s) \nUSING " % (query, SUB_QUERIES[i])
                query += "%s," * (len(common_vars) - 1)
                query += "%s)\n" % common_vars
                used.append(i)

    # output those needed:
    if OUTPUT_VAR != ["*"]:
        s = "%s, " * (len(OUTPUT_VAR) - 1)
        s += "%s "
        query = ("SELECT %s" % s) % tuple(OUTPUT_VAR) + "\nFROM ( " + query + ")"

        # check error
        if not current_vars.issuperset(set(OUTPUT_VAR)):
            print("At least one of the output var is not in the where clause, out:", OUTPUT_VAR, "where:", current_vars)
            sys.exit()

    if len(FILTERS) != 0:
        query += " \nWHERE "
        for i in range(len(FILTERS)):
            if i == 0:
                query += FILTERS[i]
            else:
                query += " \nand " + FILTERS[i]

    query += ";"
    return query

if __name__ == "__main__":
    main()
<commit_msg>q9 is working<commit_after>import sqlite3
import sys

PREFIX = {}
OUTPUT_VAR = []
STATE = {'IN_PREFIX': False,
         'IN_SELECT': False,
         'IN_WHERE': False,
         'IN_FILTER': False}
SUB_QUERIES = []
SUB_VARS = []
FILTERS = []
FILTER_VAR = []

def main():
    if len(sys.argv) != 3:
        print("Usage: python q8.py <database file> <SPARQL query file>")
        sys.exit()

    infile = open(sys.argv[2], 'r')  # read the SPARQL query file
    whole_file = infile.read()
    infile.close()

    input_lines = reformat(whole_file)
    query = parseFile(input_lines)

    print(query)

    conn = sqlite3.connect(sys.argv[1])  # connection to the database
    c = conn.cursor()  # cursor
    conn.create_function("convert",1,convert)
    c.execute(query)
    print(c.fetchall())
    conn.close()

def convert(value):
    """
    this is a sql function that will try to convert value to int or float
    :param value: value of the object column that returned by a query
    :return: the converted value
    """
    try:
        if "." in value:
            a = float(value)
        else:
            a = int(value)

        return a
    except ValueError:
        return value

def reformat(string):
    """
    change the format of file so that:
      1. "SELECT" and "WHERE" will be in different lines
      2. (subj pred obj) and "WHERE {" will be in different lines
      3. there will be at least one space between SELECT and first ?var or *
      4. there will be at least one space between WHERE and {
      5. there will be at least one space between FILTER and (
    :param string: all the characters in the input file
    :return: a list of each line in the formatted file
    """
    # TODO: will the select and ?var in different lines?

    # changes #3
    select_index = string.upper().index("\nSELECT") + 6
    string = string[:select_index + 1] + " " + string[select_index + 1:]

    # changes #1,#2,#4
    where_index = 0
    while where_index < select_index:
        where_index = string.upper().find("WHERE", where_index + 1) + 4
    curly_open = string.find("{", where_index + 1)
    string = string[:where_index - 4] + "\nWHERE {" + string[curly_open + 1:]

    # changes #5
    filter_index = string.upper().find("FILTER", 0)
    while filter_index >= 0:
        string = string[:filter_index + 6] + " " + string[filter_index + 6:]
        filter_index = string.upper().find("FILTER", filter_index + 1)

    return string.split("\n")

def parseFile(sparql_lines):
    """
    parse the SPARQL query, translate and return it as an sqlite query
    :param sparql_lines: a list of each line in the SPARQL query file
    :return: a string containing sqlite query
    """
    for line in sparql_lines:
        line = line.strip()
        if line == "":  # if the line is blank, go to next line
            continue
        line = line.split()

        if line[0].upper() == "PREFIX":
            STATE['IN_PREFIX'] = True
            if len(line) == 3:  # if there are spaces between ':' and <url>
                PREFIX[line[1][:-1]] = line[2][1:-1]
            elif len(line) == 2:  # if there's no space:
                index_of_colon = line[1].index(":")
                PREFIX[line[1][:index_of_colon]] = line[1][index_of_colon + 2:-1]
            else:
                print("Unexpected error when reading prefix line with tokens:", line)
                sys.exit()

        elif line[0].upper() == "SELECT":
            STATE['IN_PREFIX'] = False
            STATE['IN_SELECT'] = True
            for token in line:
                if token.upper() == "SELECT":
                    continue
                elif "*" in token:
                    OUTPUT_VAR.append("*")
                    break
                elif "?" in token:
                    OUTPUT_VAR.append(token[1:])  # add the ?var into the outputVar list
                else:
                    print("Unexpected error when reading ?var in SELECT line with tokens:", line, "token:", token)
                    sys.exit()

        elif line[0].upper() == "WHERE":
            STATE['IN_SELECT'] = False
            STATE['IN_WHERE'] = True

        elif STATE['IN_WHERE']:
            if line[0][:6].upper() == "FILTER":  # if this is a filter line
                addFilter(line[1:])
            elif line[0] == "}":
                STATE['IN_WHERE'] = False
            else:
                readPattern(line)

        else:
            print("Unexpected error while parsing line with tokens:", line)
            sys.exit()

    result = buildQuery()
    return result

def addFilter(tokens):
    """
    add the filter line to the global variable
    :param tokens: all the token in the filter line
    """
    filter_line = "".join(tokens)
    variable_start = filter_line.find("?")
    var_name = ""
    variable_end = variable_start
    for char in filter_line[variable_start + 1:]:
        if char not in ",><=)":
            var_name += char
            variable_end += 1
        else:
            break
    FILTER_VAR.append(var_name)
    if "regex" in filter_line:

        first_quote = filter_line.find('''"''')
        if first_quote > 0:
            second_quote = filter_line.find('''"''', first_quote + 1)
        else:
            first_quote = filter_line.find("'")
            if first_quote < 0:
                print("error")
                sys.exit()
            second_quote = filter_line.find("'", first_quote + 1)
        target_string = filter_line[first_quote + 1:second_quote]

        FILTERS.append('''%s = "%s"''' % (var_name, target_string))
    else:
        # it has to be a number
        filter_line = filter_line[:variable_start + 1] + filter_line[variable_end + 1:]
        number = ""
        operation = ""
        for char in filter_line:
            if char.isnumeric() or char == ".":
                number += char

        operations = ["!=", ">=", "<=", "=", ">", "<"]
        for op in operations:
            if op in filter_line:
                operation = op
                break
        filter_line.find("!")
        FILTERS.append("%s %s %s" % (var_name, operation, number))

def readPattern(pattern):
    """
    read a (subj pred obej) pattern, store the variable and sub-sql-query in the dictionary
    :param pattern: a list as [subj, pred, obej] pattern
    """

    query_template = {(0,): ("subject as %s ", '''predicate = "%s" and object = "%s"''', "predicate_object_index"),
                      (1,): ("predicate as %s ", '''subject = "%s" and object = "%s"''', "subject_object_index"),
                      (2,): (
                      "convert(object) as %s ", '''subject = "%s" and predicate = "%s"''', "subject_predicate_index"),
                      (0, 1): ("subject as %s , predicate as %s ", '''object = "%s"''', "object_index"),
                      (0, 2): ("subject as %s , convert(object) as %s ", '''predicate = "%s"''', "predicate_index"),
                      (1, 2): ("predicate as %s , convert(object) as %s ", '''subject = "%s"''', "subject_index")}

    if pattern[2][-1] == ".":
        pattern[2] = pattern[2][:-1]

    variables = []
    conditions = []
    template_number = []

    for i in range(3):
        if pattern[i][0] == "?":  # if the term is a variable
            variables.append(pattern[i][1:])
            template_number.append(i)
        elif pattern[i][0] == "<":  # if the term is a url
            conditions.append(pattern[i][1:-1])
        elif ":" in pattern[i]:  # if it's a url with prefix
            index = pattern[i].index(":")
            if pattern[i][:index] not in PREFIX:
                print("Unknown prefix: ", pattern[i])
                sys.exit()
            url = PREFIX[pattern[i][:index]] + pattern[i][index + 1:]
            conditions.append(url)
        else:
            conditions.append(pattern[i])
    template_number = tuple(template_number)

    query = "SELECT DISTINCT %s \nFROM graph_data INDEXED BY %s \nWHERE %s " \
            % (query_template[template_number][0] % tuple(variables),
               query_template[template_number][2],
               query_template[template_number][1] % tuple(conditions))

    SUB_VARS.append(variables)
    SUB_QUERIES.append(query)

def buildQuery():
    """
    use the global variables to construct a final query
    :return: the final sql query
    """
    keys = SUB_VARS
    current_vars = set(keys.pop(0))
    # combine all the sub-queries:
    query = SUB_QUERIES.pop(0)
    used = []
    while len(used) != len(keys):
        for i in range(len(keys)):
            variables = keys[i]
            if not current_vars.isdisjoint(set(variables)):
                common_vars = tuple(current_vars.intersection(set(variables)))
                current_vars = current_vars.union(set(variables))
                query = "SELECT * \nFROM (\n(%s) \nNATURAL JOIN \n(%s) )\n " % (query, SUB_QUERIES[i])
                #query += "%s," * (len(common_vars) - 1)
                #query += "%s)\n" % common_vars

                used.append(i)

    # output those needed:
    if OUTPUT_VAR != ["*"]:
        s = "%s, " * (len(OUTPUT_VAR) - 1)
        s += "%s "
        query = ("SELECT %s" % s) % tuple(OUTPUT_VAR) + "\nFROM ( " + query + ")"

        # check error
        if not current_vars.issuperset(set(OUTPUT_VAR)):
            print("At least one of the output var is not in the where clause, out:", OUTPUT_VAR, "where:", current_vars)
            sys.exit()

    if len(FILTERS) != 0:
        query += " \nWHERE "
        for i in range(len(FILTERS)):
            if i == 0:
                query += FILTERS[i]
            else:
                query += " \nand " + FILTERS[i]

    query += ";"
    return query

if __name__ == "__main__":
    main()
<|endoftext|>