<commit_before>from django.db import models
from django.http import HttpResponse
from urllib import request
from urllib.parse import urlencode
from django.utils import timezone
import json
from django.contrib.auth.models import User, Group
from datetime import timedelta
from django.conf import settings
import sys

# This gets the location for the image files for the Stock model.
def get_upload_location(instance, filename):
    return instance.symbol

class StockAPIError(Exception):
    pass

class TradeError(Exception):
    pass

class RemoteStockData:
    """
    An object that holds the data received when a stock is updated
    from the web API.
    @field symbol The symbol of the stock
    @field price The newest price available of the stock
    @field change The last available change of the stock
    @field name The name of the company
    """
    def __init__(self, symbol, name, price, change):
        self.symbol = symbol
        self.name = name
        self.price = price
        self.change = change
    def apply(self, stockObj = None):
        if not stockObj:
            stockObj = Stock.objects.get(symbol=self.symbol)
        stockObj.price = self.price
        stockObj.change = self.change
        if stockObj.company_name == "":
            stockObj.company_name = self.name
            stockObj.symbol = stockObj.symbol.upper()

class Stock(models.Model):
    company_name = models.CharField(max_length=50, default="", blank=True)
    symbol = models.CharField(max_length=4)
    last_updated = models.DateTimeField(default=timezone.now() - timedelta(minutes=20))
    # Set up a default image and maybe a way to get them automatically.
    image = models.ImageField(upload_to=get_upload_location, blank=True, default=settings.MEDIA_URL + "default")
    price = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    change = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    def __str__(self):
        return "{} ({})".format(self.company_name, self.symbol)
    def update(self):
        if timezone.now() - self.last_updated > timedelta(minutes=15):
            Stock.remote_load_price(self.symbol).apply(self)
            self.last_updated = timezone.now()
            self.save()
            # The database normalizes the input to two decimal places and makes
            # sure that the negative and positive work on the dashboard, so I
            # reload it here. With any luck, it's fast, but who knows.
            self.refresh_from_db()
    def force_update(self):
        self.last_updated -= timedelta(minutes=30)
        self.update()
    def get_price(self):
        self.update()
        return self.price
    def get_change(self):
        self.update()
        return self.change
    def format_for_json(self):
        return {"symbol": self.symbol, "name": self.company_name}
    @staticmethod
    def remote_load_price(symbol):
        """
        Given a symbol as a string, returns a RemoteStockData object with the given symbol's
        name, price, and last change.
        """
        URL = "http://finance.yahoo.com/webservice/v1/symbols/{}/quote?format=json&view=detail"
        try:
            jsonObj = json.loads(request.urlopen(URL.format(symbol)).read().decode("UTF-8"))['list']['resources'][0]['resource']['fields']
        except IndexError:
            if "." in symbol:
                return Stock.remote_load_price(symbol[:symbol.index(".")])
            else:
                raise RuntimeError("The stock with symbol {} can't be found!".format(symbol))
        return RemoteStockData(jsonObj["symbol"], jsonObj["name"], jsonObj["price"], jsonObj["change"])

class Floor(models.Model):
    OPEN = "open"
    CLOSED = "closed"
    PERMISSIVE = "permissive"
    PERMISSIVENESS_CHOICES = (
            (OPEN, "Open"),
            (CLOSED, "Closed"),
            (PERMISSIVE, "Permissive")
            )
    name = models.CharField(max_length=35)
    stocks = models.ManyToManyField(Stock)
    permissiveness = models.CharField(max_length=15, choices=PERMISSIVENESS_CHOICES, default=PERMISSIVE)
    owner = models.ForeignKey(User, null=True)
    # The model for this ForeignKey is a string because it doesn't know what it is yet because it's down there.
    floorPlayer = models.ForeignKey("Player", related_name="FloorPlayer", null=True)
    def __str__(self):
        return self.name
    def leaders(self):
        return Player.objects.filter(floor=self).exclude(user__groups__name__exact="Floor").order_by("-points")

# NB This model represents a specific player on a specific floor. The player account is represented by a Django `User`
# object, which this references. Setting these as ForeignKeys as opposed to something else will cause this object to be
# deleted if the it's `User` object or its floor is deleted.
class Player(models.Model):
    user = models.ForeignKey(User)
    floor = models.ForeignKey("Floor")
    stocks = models.ManyToManyField(Stock, blank=True)
    points = models.IntegerField(default=0)
    def __str__(self):
        return "{} on {}".format(str(self.user), str(self.floor))
    def get_name(self):
        """
        It's possible that somebody could not have a username, perhaps, so I have this to take
        care of that, and also prevent a bunch of ugly `player.user.username` calls.
        """
        return self.user.username if self.user.username else self.user.email
    def isFloor(self):
        return "Floor" in [i.name for i in self.user.groups.all()]
    def receivedTrades(self):
        return Trade.objects.filter(recipient=self)
    def sentTrades(self):
        return Trade.objects.filter(sender=self)
    def receivedRequests(self):
        return StockSuggestion.objects.filter(floor__owner=self.user)
    def numMessages(self):
        num = self.receivedTrades().count()
        if self.floor.owner == self.user and self.floor.permissiveness == 'permissive':
            num += self.receivedRequests().count()
        return num
    def seesSuggestions(self):
        """
        This returns a boolean telling whether this player needs a suggestions tab on his
        dashboard tab.
        """
        return self.floor.owner == self.user and self.floor.permissiveness == "permissive"

class Trade(models.Model):
    recipient = models.ForeignKey(Player)
    # recipientStocks and senderStocks are the stocks that those people have right now and will give away in the trade.
    recipientStocks = models.ManyToManyField(Stock, related_name="receivingPlayerStocks")
    floor = models.ForeignKey(Floor)
    sender = models.ForeignKey(Player, related_name="sendingPlayer")
    senderStocks = models.ManyToManyField(Stock)
    date = models.DateTimeField(auto_now_add=True)
    def __str__(self):
        return "Trade from {} to {} on {}".format(self.sender.user, self.recipient.user, self.floor)
    def accept(self):
        for s in self.recipientStocks.all():
            self.recipient.stocks.remove(s)
            self.sender.stocks.add(s)
        for s in self.senderStocks.all():
            self.sender.stocks.remove(s)
            self.recipient.stocks.add(s)
        self.sender.save()
        self.recipient.save()
        self.delete()
    def verify(self):
        if not (self.recipient.isFloor() and not self.floor.permissiveness == "closed"):
            for s in self.recipientStocks.all():
                if not s in self.recipient.stocks.all():
                    raise TradeError("One of the recipient stocks ({}) doesn't belong to the recipient ({})".format(s, self.recipient.user))
                if not s in self.floor.stocks.all():
                    raise TradeError("One of the recipient stocks ({}) doesn't belong to the floor ({})".format(s, self.floor))
        for s in self.senderStocks.all():
            if not s in self.sender.stocks.all():
                raise TradeError("One of the sender stocks ({}) doesn't belong to the sender ({})".format(s, self.recipient.user))
            if not s in self.floor.stocks.all():
                raise TradeError("One of the sender stocks ({}) doesn't belong to the floor ({})".format(s, self.floor))
        if self.recipient.isFloor():
            self.accept()
        elif self.sender.isFloor():
            raise RuntimeError("The floor sent a trade. This isn't good at all.")
    def toFormDict(self):
        d = {"other_user": self.sender.get_name(),
            "user_stocks": ",".join(i.symbol for i in self.recipientStocks.all()),
            "other_stocks": ",".join(i.symbol for i in self.senderStocks.all())}
        return d

class StockSuggestion(models.Model):
    stock = models.ForeignKey(Stock)
    requesting_player = models.ForeignKey(Player)
    floor = models.ForeignKey(Floor)
    date = models.DateTimeField(auto_now_add=True)
    def accept(self):
        if not self.stock in self.floor.stocks.all():
            self.floor.stocks.add(self.stock)
            self.floor.save()
            self.requesting_player.stocks.add(self.stock)
            self.requesting_player.save()
        self.delete()
<commit_msg>Possibly fix price updating bug<commit_after>from django.db import models
from django.http import HttpResponse
from urllib import request
from urllib.parse import urlencode
from django.utils import timezone
import json
from django.contrib.auth.models import User, Group
from datetime import timedelta
from django.conf import settings
import sys

# This gets the location for the image files for the Stock model.
def get_upload_location(instance, filename):
    return instance.symbol

class StockAPIError(Exception):
    pass

class TradeError(Exception):
    pass

class RemoteStockData:
    """
    An object that holds the data received when a stock is updated
    from the web API.
    @field symbol The symbol of the stock
    @field price The newest price available of the stock
    @field change The last available change of the stock
    @field name The name of the company
    """
    def __init__(self, symbol, name, price, change):
        self.symbol = symbol
        self.name = name
        self.price = price
        self.change = change
    def apply(self, stockObj = None):
        if not stockObj:
            stockObj = Stock.objects.get(symbol=self.symbol)
        stockObj.price = self.price
        stockObj.change = self.change
        if stockObj.company_name == "":
            stockObj.company_name = self.name
            stockObj.symbol = stockObj.symbol.upper()
    def __str__(self):
        return "{} at {}".format(self.symbol, self.price)

class Stock(models.Model):
    company_name = models.CharField(max_length=50, default="", blank=True)
    symbol = models.CharField(max_length=4)
    last_updated = models.DateTimeField(default=timezone.now() - timedelta(minutes=20))
    # Set up a default image and maybe a way to get them automatically.
    image = models.ImageField(upload_to=get_upload_location, blank=True, default=settings.MEDIA_URL + "default")
    price = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    change = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    def __str__(self):
        return "{} ({})".format(self.company_name, self.symbol)
    def update(self):
        if timezone.now() - self.last_updated > timedelta(minutes=15):
            Stock.remote_load_price(self.symbol).apply(self)
            self.last_updated = timezone.now()
            self.save()
            # The database normalizes the input to two decimal places and makes
            # sure that the negative and positive work on the dashboard, so I
            # reload it here. With any luck, it's fast, but who knows.
            self.refresh_from_db()
    def force_update(self):
        self.last_updated -= timedelta(minutes=30)
        self.update()
    def get_price(self):
        self.update()
        return self.price
    def get_change(self):
        self.update()
        return self.change
    def format_for_json(self):
        return {"symbol": self.symbol, "name": self.company_name}
    @staticmethod
    def remote_load_price(symbol):
        """
        Given a symbol as a string, returns a RemoteStockData object with the given symbol's
        name, price, and last change.
        """
        URL = "http://finance.yahoo.com/webservice/v1/symbols/{}/quote?format=json&view=detail"
        try:
            jsonObj = json.loads(request.urlopen(URL.format(symbol)).read().decode("UTF-8"))['list']['resources'][0]['resource']['fields']
        except IndexError:
            if "." in symbol:
                return Stock.remote_load_price(symbol.replace(".", "-"))
            else:
                raise RuntimeError("The stock with symbol {} can't be found!".format(symbol))
        return RemoteStockData(jsonObj["symbol"], jsonObj["name"], jsonObj["price"], jsonObj["change"])

class Floor(models.Model):
    OPEN = "open"
    CLOSED = "closed"
    PERMISSIVE = "permissive"
    PERMISSIVENESS_CHOICES = (
            (OPEN, "Open"),
            (CLOSED, "Closed"),
            (PERMISSIVE, "Permissive")
            )
    name = models.CharField(max_length=35)
    stocks = models.ManyToManyField(Stock)
    permissiveness = models.CharField(max_length=15, choices=PERMISSIVENESS_CHOICES, default=PERMISSIVE)
    owner = models.ForeignKey(User, null=True)
    # The model for this ForeignKey is a string because it doesn't know what it is yet because it's down there.
    floorPlayer = models.ForeignKey("Player", related_name="FloorPlayer", null=True)
    def __str__(self):
        return self.name
    def leaders(self):
        return Player.objects.filter(floor=self).exclude(user__groups__name__exact="Floor").order_by("-points")

# NB This model represents a specific player on a specific floor. The player account is represented by a Django `User`
# object, which this references. Setting these as ForeignKeys as opposed to something else will cause this object to be
# deleted if the it's `User` object or its floor is deleted.
class Player(models.Model):
    user = models.ForeignKey(User)
    floor = models.ForeignKey("Floor")
    stocks = models.ManyToManyField(Stock, blank=True)
    points = models.IntegerField(default=0)
    def __str__(self):
        return "{} on {}".format(str(self.user), str(self.floor))
    def get_name(self):
        """
        It's possible that somebody could not have a username, perhaps, so I have this to take
        care of that, and also prevent a bunch of ugly `player.user.username` calls.
        """
        return self.user.username if self.user.username else self.user.email
    def isFloor(self):
        return "Floor" in [i.name for i in self.user.groups.all()]
    def receivedTrades(self):
        return Trade.objects.filter(recipient=self)
    def sentTrades(self):
        return Trade.objects.filter(sender=self)
    def receivedRequests(self):
        return StockSuggestion.objects.filter(floor__owner=self.user)
    def numMessages(self):
        num = self.receivedTrades().count()
        if self.floor.owner == self.user and self.floor.permissiveness == 'permissive':
            num += self.receivedRequests().count()
        return num
    def seesSuggestions(self):
        """
        This returns a boolean telling whether this player needs a suggestions tab on his
        dashboard tab.
        """
        return self.floor.owner == self.user and self.floor.permissiveness == "permissive"

class Trade(models.Model):
    recipient = models.ForeignKey(Player)
    # recipientStocks and senderStocks are the stocks that those people have right now and will give away in the trade.
    recipientStocks = models.ManyToManyField(Stock, related_name="receivingPlayerStocks")
    floor = models.ForeignKey(Floor)
    sender = models.ForeignKey(Player, related_name="sendingPlayer")
    senderStocks = models.ManyToManyField(Stock)
    date = models.DateTimeField(auto_now_add=True)
    def __str__(self):
        return "Trade from {} to {} on {}".format(self.sender.user, self.recipient.user, self.floor)
    def accept(self):
        for s in self.recipientStocks.all():
            self.recipient.stocks.remove(s)
            self.sender.stocks.add(s)
        for s in self.senderStocks.all():
            self.sender.stocks.remove(s)
            self.recipient.stocks.add(s)
        self.sender.save()
        self.recipient.save()
        self.delete()
    def verify(self):
        if not (self.recipient.isFloor() and not self.floor.permissiveness == "closed"):
            for s in self.recipientStocks.all():
                if not s in self.recipient.stocks.all():
                    raise TradeError("One of the recipient stocks ({}) doesn't belong to the recipient ({})".format(s, self.recipient.user))
                if not s in self.floor.stocks.all():
                    raise TradeError("One of the recipient stocks ({}) doesn't belong to the floor ({})".format(s, self.floor))
        for s in self.senderStocks.all():
            if not s in self.sender.stocks.all():
                raise TradeError("One of the sender stocks ({}) doesn't belong to the sender ({})".format(s, self.recipient.user))
            if not s in self.floor.stocks.all():
                raise TradeError("One of the sender stocks ({}) doesn't belong to the floor ({})".format(s, self.floor))
        if self.recipient.isFloor():
            self.accept()
        elif self.sender.isFloor():
            raise RuntimeError("The floor sent a trade. This isn't good at all.")
    def toFormDict(self):
        d = {"other_user": self.sender.get_name(),
            "user_stocks": ",".join(i.symbol for i in self.recipientStocks.all()),
            "other_stocks": ",".join(i.symbol for i in self.senderStocks.all())}
        return d

class StockSuggestion(models.Model):
    stock = models.ForeignKey(Stock)
    requesting_player = models.ForeignKey(Player)
    floor = models.ForeignKey(Floor)
    date = models.DateTimeField(auto_now_add=True)
    def accept(self):
        if not self.stock in self.floor.stocks.all():
            self.floor.stocks.add(self.stock)
            self.floor.save()
            self.requesting_player.stocks.add(self.stock)
            self.requesting_player.save()
        self.delete()
<|endoftext|>


<commit_before>"""
========================================
Interpolation (:mod:`scipy.interpolate`)
========================================

.. currentmodule:: scipy.interpolate

Sub-package for objects used in interpolation.

As listed below, this sub-package contains spline functions and classes,
one-dimensional and multi-dimensional (univariate and multivariate)
interpolation classes, Lagrange and Taylor polynomial interpolators, and
wrappers for `FITPACK <http://www.netlib.org/dierckx/>`__
and DFITPACK functions.

Univariate interpolation
========================

.. autosummary::
   :toctree: generated/

   interp1d
   BarycentricInterpolator
   KroghInterpolator
   PiecewisePolynomial
   PchipInterpolator
   barycentric_interpolate
   krogh_interpolate
   piecewise_polynomial_interpolate
   pchip_interpolate
   Akima1DInterpolator
   PPoly
   BPoly

Multivariate interpolation
==========================

Unstructured data:

.. autosummary::
   :toctree: generated/

   griddata
   LinearNDInterpolator
   NearestNDInterpolator
   CloughTocher2DInterpolator
   Rbf
   interp2d

For data on a grid:

.. autosummary::
   :toctree: generated/

   interpn
   RegularGridInterpolator
   RectBivariateSpline

.. seealso:: `scipy.ndimage.interpolation.map_coordinates`

1-D Splines
===========

.. autosummary::
   :toctree: generated/

   UnivariateSpline
   InterpolatedUnivariateSpline
   LSQUnivariateSpline

Functional interface to FITPACK functions:

.. autosummary::
   :toctree: generated/

   splrep
   splprep
   splev
   splint
   sproot
   spalde
   splder
   splantider

2-D Splines
===========

For data on a grid:

.. autosummary::
   :toctree: generated/

   RectBivariateSpline
   RectSphereBivariateSpline

For unstructured data:

.. autosummary::
   :toctree: generated/

   BivariateSpline
   SmoothBivariateSpline
   SmoothSphereBivariateSpline
   LSQBivariateSpline
   LSQSphereBivariateSpline

Low-level interface to FITPACK functions:

.. autosummary::
   :toctree: generated/

   bisplrep
   bisplev

Additional tools
================

.. autosummary::
   :toctree: generated/

   lagrange
   approximate_taylor_polynomial

.. seealso::

   `scipy.ndimage.interpolation.map_coordinates`,
   `scipy.ndimage.interpolation.spline_filter`,
   `scipy.signal.resample`,
   `scipy.signal.bspline`,
   `scipy.signal.gauss_spline`,
   `scipy.signal.qspline1d`,
   `scipy.signal.cspline1d`,
   `scipy.signal.qspline1d_eval`,
   `scipy.signal.cspline1d_eval`,
   `scipy.signal.qspline2d`,
   `scipy.signal.cspline2d`.

"""
from __future__ import division, print_function, absolute_import

from .interpolate import *
from .fitpack import *

# New interface to fitpack library:
from .fitpack2 import *

from .rbf import Rbf

from .polyint import *

from ._monotone import *

from .ndgriddata import *

__all__ = [s for s in dir() if not s.startswith('_')]
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench
<commit_msg>DOC: interpolate: add missing items to refguide<commit_after>"""========================================
Interpolation (:mod:`scipy.interpolate`)
========================================

.. currentmodule:: scipy.interpolate

Sub-package for objects used in interpolation.

As listed below, this sub-package contains spline functions and classes,
one-dimensional and multi-dimensional (univariate and multivariate)
interpolation classes, Lagrange and Taylor polynomial interpolators, and
wrappers for `FITPACK <http://www.netlib.org/dierckx/>`__
and DFITPACK functions.

Univariate interpolation
========================

.. autosummary::
   :toctree: generated/

   interp1d
   BarycentricInterpolator
   KroghInterpolator
   PiecewisePolynomial
   PchipInterpolator
   barycentric_interpolate
   krogh_interpolate
   piecewise_polynomial_interpolate
   pchip_interpolate
   Akima1DInterpolator
   PPoly
   BPoly

Multivariate interpolation
==========================

Unstructured data:

.. autosummary::
   :toctree: generated/

   griddata
   LinearNDInterpolator
   NearestNDInterpolator
   CloughTocher2DInterpolator
   Rbf
   interp2d

For data on a grid:

.. autosummary::
   :toctree: generated/

   interpn
   RegularGridInterpolator
   RectBivariateSpline

.. seealso:: `scipy.ndimage.interpolation.map_coordinates`

1-D Splines
===========

.. autosummary::
   :toctree: generated/

   UnivariateSpline
   InterpolatedUnivariateSpline
   LSQUnivariateSpline

Functional interface to FITPACK functions:

.. autosummary::
   :toctree: generated/

   splrep
   splprep
   splev
   splint
   sproot
   spalde
   splder
   splantider
   insert

2-D Splines
===========

For data on a grid:

.. autosummary::
   :toctree: generated/

   RectBivariateSpline
   RectSphereBivariateSpline

For unstructured data:

.. autosummary::
   :toctree: generated/

   BivariateSpline
   SmoothBivariateSpline
   SmoothSphereBivariateSpline
   LSQBivariateSpline
   LSQSphereBivariateSpline

Low-level interface to FITPACK functions:

.. autosummary::
   :toctree: generated/

   bisplrep
   bisplev

Additional tools
================

.. autosummary::
   :toctree: generated/

   lagrange
   approximate_taylor_polynomial

.. seealso::

   `scipy.ndimage.interpolation.map_coordinates`,
   `scipy.ndimage.interpolation.spline_filter`,
   `scipy.signal.resample`,
   `scipy.signal.bspline`,
   `scipy.signal.gauss_spline`,
   `scipy.signal.qspline1d`,
   `scipy.signal.cspline1d`,
   `scipy.signal.qspline1d_eval`,
   `scipy.signal.cspline1d_eval`,
   `scipy.signal.qspline2d`,
   `scipy.signal.cspline2d`.

Functions existing for backward compatibility (should not be used in
new code):

.. autosummary::
   :toctree: generated/

   ppform
   spleval
   spline
   splmake
   spltopp
   pchip

"""
from __future__ import division, print_function, absolute_import

from .interpolate import *
from .fitpack import *

# New interface to fitpack library:
from .fitpack2 import *

from .rbf import Rbf

from .polyint import *

from ._monotone import *

from .ndgriddata import *

__all__ = [s for s in dir() if not s.startswith('_')]
from numpy.testing import Tester
test = Tester().test
bench = Tester().bench
<|endoftext|>


<commit_before>'''author@esilgard'''
#
# Copyright (c) 2013-2016 Fred Hutchinson Cancer Research Center
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from . import global_strings
__version__ = 'final_pathology_logic1.0'

def get(table_list):
    '''
    use all the extracted pathology elements from a given report to apply final logic
    add/delete values for the final output
    table_list = list of table dictionaries where
                table dictionary: tableName and a dictionary
    '''
    return_list = []
    ## iterate through the tables
    for table in table_list:
        field_list = []
        ## iterate through PathFinding values, find the specimen (recordKey) PathFindingHistology
        if table[global_strings.TABLE] == global_strings.FINDING_TABLE:
            specimen_histology_found = set([])
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.NAME] == "PathFindHistology" \
                   and fields[global_strings.VALUE]:
                    specimen_histology_found.add(fields[global_strings.KEY])
            ## iterate through fields again, only ones that have histologies to the final list
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.KEY] in specimen_histology_found:
                    field_list.append(fields)
        else:
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.VALUE]:
                    field_list.append(fields)
        if field_list:
            table[global_strings.FIELDS] = field_list
            return_list.append(table)
    for each in return_list:
        for k,v in each.items():
            print k, type(v)
            if type(v)==str: print v
        print type(each)
    return return_list
<commit_msg>removed printlns<commit_after>'''author@esilgard'''
#
# Copyright (c) 2013-2016 Fred Hutchinson Cancer Research Center
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from . import global_strings
__version__ = 'final_pathology_logic1.0'

def get(table_list):
    '''
    use all the extracted pathology elements from a given report to apply final logic
    add/delete values for the final output
    table_list = list of table dictionaries where
                table dictionary: tableName and a dictionary
    '''
    return_list = []
    ## iterate through the tables
    for table in table_list:
        field_list = []
        ## iterate through PathFinding values, find the specimen (recordKey) PathFindingHistology
        if table[global_strings.TABLE] == global_strings.FINDING_TABLE:
            specimen_histology_found = set([])
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.NAME] == "PathFindHistology" \
                   and fields[global_strings.VALUE]:
                    specimen_histology_found.add(fields[global_strings.KEY])
            ## iterate through fields again, only ones that have histologies to the final list
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.KEY] in specimen_histology_found:
                    field_list.append(fields)
        else:
            for fields in table[global_strings.FIELDS]:
                if fields[global_strings.VALUE]:
                    field_list.append(fields)
        if field_list:
            table[global_strings.FIELDS] = field_list
            return_list.append(table)
    return return_list
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

from unittest import TestCase

import numpy as np
import numpy.testing as npt
import astropy.units as u
from copy import deepcopy

from fil_finder import fil_finder_2D

from _testing_data import *

class Test_FilFinder(TestCase):

    def test_with_rht_branches(self):

        test1 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test1")

        test1.create_mask(border_masking=False)
        test1.medskel()
        test1.analyze_skeletons()
        test1.exec_rht(branches=True)
        test1.find_widths()
        test1.compute_filament_brightness()

        assert test1.number_of_filaments == len(table1["Lengths"])

        for i, param in enumerate(test1.width_fits["Names"]):
            npt.assert_allclose(test1.width_fits["Parameters"][:, i],
                                np.asarray(table1[param]), rtol=1e-4)
            npt.assert_allclose(test1.width_fits["Errors"][:, i],
                                np.asarray(table1[param + " Error"]),
                                rtol=1e-4)

        assert np.allclose(test1.lengths,
                           np.asarray(table1['Lengths']))

        assert (test1.width_fits['Type'] == table1['Fit Type']).all()

        assert np.allclose(test1.total_intensity,
                           np.asarray(table1['Total Intensity']))

        assert np.allclose(test1.filament_brightness,
                           np.asarray(table1['Median Brightness']))

        assert np.allclose(test1.branch_properties["number"],
                           np.asarray(table1['Branches']))

    def test_without_rht_branches(self):
        # Non-branches

        test2 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test2")

        test2.create_mask(border_masking=False)
        test2.medskel()
        test2.analyze_skeletons()
        test2.exec_rht(branches=False)
        test2.find_widths()
        test2.compute_filament_brightness()

        assert test2.number_of_filaments == len(table2["Lengths"])

        for i, param in enumerate(test2.width_fits["Names"]):
            npt.assert_allclose(test2.width_fits["Parameters"][:, i],
                                np.asarray(table2[param]), rtol=1e-4)
            npt.assert_allclose(test2.width_fits["Errors"][:, i],
                                np.asarray(table2[param + " Error"]), rtol=1e-4)

        assert np.allclose(test2.lengths,
                           np.asarray(table2['Lengths']))

        assert (test2.width_fits['Type'] == table2['Fit Type']).all()

        assert np.allclose(test2.total_intensity,
                           np.asarray(table2['Total Intensity']))

        assert np.allclose(test2.filament_brightness,
                           np.asarray(table2['Median Brightness']))

        assert np.allclose(test2.branch_properties["number"],
                           np.asarray(table2['Branches']))

        assert np.allclose(test2.rht_curvature['Orientation'],
                           np.asarray(table2['Orientation']))

        assert np.allclose(test2.rht_curvature['Curvature'],
                           np.asarray(table2['Curvature']))

    def test_equal_branches(self):
        '''
        Ensure the filament arrays are equal with and without computing the
        RHT branches.
        '''

        test1 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test1")

        test1.create_mask(border_masking=False)
        test1.medskel()
        test1.analyze_skeletons()
        test1.exec_rht(branches=True)

        test_copy = deepcopy(test1)

        test_copy.exec_rht(branches=False)

        for arr1, arr2 in zip(test1.filament_arrays['final'],
                              test_copy.filament_arrays['final']):
            assert np.allclose(arr1, arr2)

        for arr1, arr2 in zip(test1.filament_arrays['long path'],
                              test_copy.filament_arrays['long path']):
            assert np.allclose(arr1, arr2)
<commit_msg>Use numpy testing functions; add explicit testing of the mask and skeleton arrays<commit_after># Licensed under an MIT open source license - see LICENSE

from unittest import TestCase

import numpy as np
import numpy.testing as npt
import astropy.units as u
from copy import deepcopy

from .. import fil_finder_2D

from _testing_data import *

class Test_FilFinder(TestCase):

    def test_with_rht_branches(self):

        test1 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test1")

        test1.create_mask(border_masking=False)
        test1.medskel()
        test1.analyze_skeletons()
        test1.exec_rht(branches=True)
        test1.find_widths()
        test1.compute_filament_brightness()

        assert ((mask1 > 0) == test1.mask).all()
        assert ((skeletons1 > 0) == test1.skeleton).all()

        assert test1.number_of_filaments == len(table1["Lengths"])

        for i, param in enumerate(test1.width_fits["Names"]):
            npt.assert_allclose(test1.width_fits["Parameters"][:, i],
                                np.asarray(table1[param]), rtol=1e-4)
            npt.assert_allclose(test1.width_fits["Errors"][:, i],
                                np.asarray(table1[param + " Error"]),
                                rtol=1e-4)

        npt.assert_allclose(test1.lengths,
                            table1['Lengths'].quantity.value)

        assert (test1.width_fits['Type'] == table1['Fit Type']).all()

        npt.assert_allclose(test1.total_intensity,
                            table1['Total Intensity'].quantity.value)

        npt.assert_allclose(test1.filament_brightness,
                            table1['Median Brightness'].quantity.value)

        npt.assert_allclose(test1.branch_properties["number"],
                            table1['Branches'].quantity.value)

    def test_without_rht_branches(self):
        # Non-branches

        test2 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test2")

        test2.create_mask(border_masking=False)
        test2.medskel()
        test2.analyze_skeletons()
        test2.exec_rht(branches=False)
        test2.find_widths()
        test2.compute_filament_brightness()

        assert ((mask2 > 0) == test2.mask).all()
        assert ((skeletons2 > 0) == test2.skeleton).all()

        assert test2.number_of_filaments == len(table2["Lengths"])

        for i, param in enumerate(test2.width_fits["Names"]):
            npt.assert_allclose(test2.width_fits["Parameters"][:, i],
                                np.asarray(table2[param]), rtol=1e-4)
            npt.assert_allclose(test2.width_fits["Errors"][:, i],
                                np.asarray(table2[param + " Error"]),
                                rtol=1e-4)

        npt.assert_allclose(test2.lengths,
                            table2['Lengths'].quantity.value)

        assert (test2.width_fits['Type'] == table2['Fit Type']).all()

        npt.assert_allclose(test2.total_intensity,
                            table2['Total Intensity'].quantity.value)

        npt.assert_allclose(test2.filament_brightness,
                            table2['Median Brightness'].quantity.value)

        npt.assert_allclose(test2.branch_properties["number"],
                            table2['Branches'].quantity.value)

        npt.assert_allclose(test2.rht_curvature['Orientation'],
                            table2['Orientation'].quantity.value)

        npt.assert_allclose(test2.rht_curvature['Curvature'],
                            table2['Curvature'].quantity.value,
                            atol=5e-3)

    def test_equal_branches(self):
        '''
        Ensure the filament arrays are equal with and without computing the
        RHT branches.
        '''

        test1 = fil_finder_2D(img, header=hdr, beamwidth=10.0 * u.arcsec,
                              flatten_thresh=95,
                              distance=260 * u.pc, size_thresh=430,
                              glob_thresh=20, save_name="test1")

        test1.create_mask(border_masking=False)
        test1.medskel()
        test1.analyze_skeletons()
        test1.exec_rht(branches=True)

        test_copy = deepcopy(test1)

        test_copy.exec_rht(branches=False)

        for arr1, arr2 in zip(test1.filament_arrays['final'],
                              test_copy.filament_arrays['final']):
            assert np.allclose(arr1, arr2)

        for arr1, arr2 in zip(test1.filament_arrays['long path'],
                              test_copy.filament_arrays['long path']):
            assert np.allclose(arr1, arr2)
<|endoftext|>


<commit_before>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata

class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)

        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

        """
        Callback structure to handle the build_table specifics
        """
        self.tablecallbacks = {
            'assignee': self.build_assignee,
            'citation': self.build_citation,
            'class': self.build_class,
            'inventor': self.build_inventor,
            'patent': self.build_patent,
            'patdesc': self.build_patdesc,
            'lawyer': self.build_lawyer,
            'sciref': self.build_sciref,
            'usreldoc': self.build_usreldoc
            }

    def build_table(self, tablename):
        """
        Return list of requisite fields for inserting this patent into [tablename]
        """
        return self.tablecallbacks[tablename]()

    def build_assignee(self):
        ack = []
        for i,y in enumerate(self.asg_list):
            if not y[0]:
                ack.extend([[self, y[2], y[1], y[4], y[5], y[7], y[8], i]])
            else:
                ack.extend([[self, "00", "%s, %s" % (y[2], y[1]), y[4], y[5], y[6], y[7], y[8], i]])
        return ack

    def build_citation(self):
        ack = []
        for i,y in enumerate([x for x in self.cit_list if x[1] != ""]):
            ack.extend([[self, y[3], y[5], y[4], y[1], y[2], y[0], i]])
        return ack

    def build_class(self):
        ack = []
        for i,y in enumerate(self.classes):
            ack.extend([[self, (i==0)*1, y[0], y[1]]])
        return ack

    def build_inventor(self):
        ack = []
        for i,y in enumerate(self.inv_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], y[4], y[5], y[6], y[8], i]])
        return ack

    def build_self(self):
        return [[self, self.kind, self.clm_num, self.code_app, self_app, self.date_grant, self.date_grant[:4], self.date_app, self.date_app[:4], self.pat_type]]

    def build_patdesc(self):
        return [[self, self.abstract, self.invention_title]]

    def build_lawyer(self):
        ack = []
        for i,y in enumerate(self.law_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], i]])
        return ack

    def build_sciref(self):
        ack = []
        for i,y in enumerate([y for y in self.cit_list if y[1] == ""]):
            ack.extend([[self, y[-1], i]])
        return ack

    def build_usreldoc(self):
        ack = []
        for i,y in enumerate(self.rel_list):
            if y[1] == 1:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], y[5], y[6]]])
            else:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], "", ""]])
        return ack

    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain
        dom_element
        """
        #if blank return nothing as well!
        if dom_element == '':
            return ''
        else:
            rc = ""
            for node in dom_element.childNodes:
                if node.nodeType == node.TEXT_NODE:
                    rc = rc + node.data
            return rc

    def __tagSplit__(self, xmldoc, xmlList, tagList, baseList=[], idx=0, blank=False, iHTML=True, debug=False):
        """
        For each item in the list of xml tags returned by tagName, we treat each tag's contents
        as its own xml doc and search it for a list of xml tags. Everyything is appended together
        and returned
        """
        d_list = []
        for x in self.__tagNme__(xmldoc, tagList=xmlList, iHTML=False):
            record = copy.copy(baseList)
            for y in tagList:
                if debug:
                    print "-------------------"
                    print x.toxml()
                    print y
                record.extend(self.__tagNme__(x, tagList=y, blank=blank, iHTML=iHTML, debug=debug, idx=idx))
            d_list.append(record)
        return d_list

    def __tagNme__(self, xmldoc, tagList, idx=0, listType=False, blank=False, iHTML=True, debug=False):
        """
        takes in the xml.minidom parsed xml document [xmldoc] and a list of tags [tagList]
        [tagList] may sometimes be a list of lists?
        We iterate the taglist, searching another level deeper into the xml doc with each
        iteration. In tagList, index 0 is the first level of the xmldoc, index 1 is the list
        of all tags we look for at the level right under all the tags we found at index 0,
        and so on.
        For each item, we then iterate through all xmldocs we passed in (usually one?)
        and append the tag we wanted to an xmllist.
        We have empty strings if the tag was not found, and always return a flat list.
        **Returns:
        if the iHTML flag is set, we do the innerHTML method on the xmllist, otherwise
        we just return the xmllist.
        """
        xmldoc = [xmldoc]
        for i,x in enumerate(tagList):
            if type(x) is not ListType:
                x = [x]
            xmlnext = []
            for y in x:
                for z in xmldoc:
                    if z != '':
                        if i==0 and idx!=0:
                            if len(z.getElementsByTagName(y))>0:
                                xmlnext.append(z.getElementsByTagName(y)[idx-1])
                            else:
                                xmlnext.append('')
                        else:
                            blFlag = False
                            for za in z.getElementsByTagName(y):
                                blFlag = True
                                xmlnext.append(za)
                            if blFlag==False and blank==True:
                                xmlnext.append('')
                    else:
                        xmlnext.append('')
            xmldoc = xmlnext

        if debug:
            if len(xmldoc)==1 and iHTML:
                print self.__innerHTML__(xmldoc[0])
            elif iHTML:
                print [self.__innerHTML__(x) for x in xmldoc]
            else:
                print xmldoc

        if len(xmldoc)==1 and iHTML:
            if listType:
                return [self.__innerHTML__(xmldoc[0])]
            else:
                return self.__innerHTML__(xmldoc[0])
        elif iHTML:
            return [self.__innerHTML__(x) for x in xmldoc]
        else:
            return xmldoc

    def __asg_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            record = []
            if len(x.getElementsByTagName("first-name"))>0:
                record = [1]
                record.extend(self.__tagNme__(x, [["last-name", "first-name"]]))
            else:
                record = [0]
                record.extend(self.__tagNme__(x, [["orgname", "role"]]))
            record.extend(self.__tagNme__(x, ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], blank=True))
            record.extend(self.__tagNme__(x, [["nationality", "residence"], "country"], blank=True))
            d_list.append(record)
        return d_list

    def __cit_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            #this means patcit is part of the XML
            record = [self.__tagNme__(x, ["category"])]
            if len(x.getElementsByTagName("patcit"))>0:
                record.extend(self.__tagNme__(x, ["patcit", ["country", "doc-number", "date", "kind", "name"]], blank=True))
                record.extend([""])
            elif len(x.getElementsByTagName("othercit"))>0:
                record.extend(["", "", "", "", ""])
                record.extend([self.__allHTML__(x, ["othercit"])])
                #probably should grab date information
            else:
                print x.toxml()
            d_list.append(record)
        return d_list

    def __rel_detail__(self, xmldoc, debug=False):
        d_list = []
        for x in xmldoc:
            for y in ["continuation-in-part", "continuation", "division", "reissue"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["relation", "child-doc"], [[["doc-number", "country", "kind"]]], baseList=[y, -1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True, idx=1))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-grant-document"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-pct-document"],  [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
            for y in ["related-publication", "us-provisional-application"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["document-id"], [[["doc-number", "country", "kind"]]], baseList=[y, 0], blank=True))
            if debug:
                print "-------------------"
                for x in d_list:
                    print x
        return d_list

    def __repr__(self):
        return \
"""country = %s, patent = %s, pat_type = %s,
date_grant = %s, date_app = %s,
         abstract = %s
  invention_title = %s
len(classes, asg, cit, ret, inv, law) = %s""" % (self.country, self.patent, self.pat_type, self.date_grant, self.date_app,
       self.abstract[:50], self.invention_title[:50],
       str([len(self.classes),  len(self.asg_list), len(self.cit_list),
            len(self.rel_list), len(self.inv_list), len(self.law_list)]))
<commit_msg>add insert methods for db<commit_after>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata

class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)

        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

        """
        Callback structure to handle the build_table specifics
        """
        self.tablecallbacks = {
            'assignee': self.build_assignee,
            'citation': self.build_citation,
            'class': self.build_class,
            'inventor': self.build_inventor,
            'patent': self.build_patent,
            'patdesc': self.build_patdesc,
            'lawyer': self.build_lawyer,
            'sciref': self.build_sciref,
            'usreldoc': self.build_usreldoc
            }

    def build_table(self, tablename):
        """
        Return list of requisite fields for inserting this patent into [tablename]
        """
        return self.tablecallbacks[tablename]()

    def build_assignee(self):
        ack = []
        for i,y in enumerate(self.asg_list):
            if not y[0]:
                ack.extend([[self, y[2], y[1], y[4], y[5], y[7], y[8], i]])
            else:
                ack.extend([[self, "00", "%s, %s" % (y[2], y[1]), y[4], y[5], y[6], y[7], y[8], i]])
        return ack

    def build_citation(self):
        ack = []
        for i,y in enumerate([x for x in self.cit_list if x[1] != ""]):
            ack.extend([[self, y[3], y[5], y[4], y[1], y[2], y[0], i]])
        return ack

    def build_class(self):
        ack = []
        for i,y in enumerate(self.classes):
            ack.extend([[self, (i==0)*1, y[0], y[1]]])
        return ack

    def build_inventor(self):
        ack = []
        for i,y in enumerate(self.inv_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], y[4], y[5], y[6], y[8], i]])
        return ack

    def build_self(self):
        return [[self, self.kind, self.clm_num, self.code_app, self_app, self.date_grant, self.date_grant[:4], self.date_app, self.date_app[:4], self.pat_type]]

    def build_patdesc(self):
        return [[self, self.abstract, self.invention_title]]

    def build_lawyer(self):
        ack = []
        for i,y in enumerate(self.law_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], i]])
        return ack

    def build_sciref(self):
        ack = []
        for i,y in enumerate([y for y in self.cit_list if y[1] == ""]):
            ack.extend([[self, y[-1], i]])
        return ack

    def build_usreldoc(self):
        ack = []
        for i,y in enumerate(self.rel_list):
            if y[1] == 1:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], y[5], y[6]]])
            else:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], "", ""]])
        return ack

    def insert_assignee(self):
        conn = sqlite3.connect("assignee.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS assignee (
                Patent VARCHAR(8),      AsgType INTEGER,        Assignee VARCHAR(30),
                City VARCHAR(10),       State VARCHAR(2),       Country VARCHAR(2),
                Nationality VARCHAR(2), Residence VARCHAR(2),   AsgSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqAsg ON assignee (Patent, AsgSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["assignee"]())

    def insert_citation(self):
        conn = sqlite3.connect("citation.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS citation (
                Patent VARCHAR(8),      Cit_Date INTEGER,       Cit_Name VARCHAR(10),
                Cit_Kind VARCHAR(1),    Cit_Country VARCHAR(2), Citation VARCHAR(8),
                Category VARCHAR(15),   CitSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqCit ON citation (Patent, CitSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["citation"]())

    def insert_class(self):
        conn = sqlite3.connect("class.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS class (
                Patent VARCHAR(8),      Prim INTEGER,
                Class VARCHAR(3),       SubClass VARCHAR(3));
            CREATE UNIQUE INDEX IF NOT EXISTS uqClass ON class (Patent, Class, SubClass);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["class"]())

    def insert_inventor(self):
        conn = sqlite3.connect("inventor.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS inventor (
                Patent VARCHAR(8),      Firstname VARCHAR(15),  Lastname VARCHAR(15),
                Street VARCHAR(15),     City VARCHAR(10),
                State VARCHAR(2),       Country VARCHAR(12),
                Zipcode VARCHAR(5),     Nationality VARCHAR(2), InvSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqInv ON inventor (Patent, InvSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["inventor"]())

    def insert_patent(self):
        conn = sqlite3.connect("patent.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS patent (
                Patent VARCHAR(8),      Kind VARCHAR(3),        Claims INTEGER,
                AppType INTEGER,        AppNum VARCHAR(8),
                GDate INTEGER,          GYear INTEGER,
                AppDate INTEGER,        AppYear INTEGER, PatType VARCHAR(15) );
            CREATE UNIQUE INDEX IF NOT EXISTS uqPat on patent (Patent);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["patent"]())

    def insert_patdesc(self):
        conn = sqlite3.connect("patdesc.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS patdesc (
                Patent VARCHAR(8),
                Abstract VARCHAR(50),   Title VARCHAR(20));
            CREATE UNIQUE INDEX IF NOT EXISTS uqPatDesc ON patdesc (Patent);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?)""" % tbl, \
            self.tablecallbacks["patdesc"]())

    def insert_lawyer(self):
        conn = sqlite3.connect("lawyer.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS lawyer (
                Patent VARCHAR(8),      Firstname VARCHAR(15),  Lastname VARCHAR(15),
                LawCountry VARCHAR(2),  OrgName VARCHAR(20),    LawSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqLawyer ON lawyer (Patent, LawSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["lawyer"]())

    def insert_sciref(self):
        conn = sqlite3.connect("sciref.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS sciref (
                Patent VARCHAR(8),      Descrip VARCHAR(20),    CitSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqSciref ON sciref (Patent, CitSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?)""" % tbl, \
            self.tablecallbacks["sciref"]())

    def insert_usreldoc(self):
        conn = sqlite3.connect("usreldoc.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS usreldoc (
                Patent VARCHAR(8),      DocType VARCHAR(10),    OrderSeq INTEGER,
                Country VARCHAR(2),     RelPatent VARCHAR(8),   Kind VARCHAR(2),
                RelDate INTEGER,        Status VARCHAR(10));
            CREATE UNIQUE INDEX IF NOT EXISTS uqUSRelDoc ON usreldoc (Patent, OrderSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["usreldoc"]())

    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain
        dom_element
        """
        #if blank return nothing as well!
        if dom_element == '':
            return ''
        else:
            rc = ""
            for node in dom_element.childNodes:
                if node.nodeType == node.TEXT_NODE:
                    rc = rc + node.data
            return rc

    def __tagSplit__(self, xmldoc, xmlList, tagList, baseList=[], idx=0, blank=False, iHTML=True, debug=False):
        """
        For each item in the list of xml tags returned by tagName, we treat each tag's contents
        as its own xml doc and search it for a list of xml tags. Everyything is appended together
        and returned
        """
        d_list = []
        for x in self.__tagNme__(xmldoc, tagList=xmlList, iHTML=False):
            record = copy.copy(baseList)
            for y in tagList:
                if debug:
                    print "-------------------"
                    print x.toxml()
                    print y
                record.extend(self.__tagNme__(x, tagList=y, blank=blank, iHTML=iHTML, debug=debug, idx=idx))
            d_list.append(record)
        return d_list

    def __tagNme__(self, xmldoc, tagList, idx=0, listType=False, blank=False, iHTML=True, debug=False):
        """
        takes in the xml.minidom parsed xml document [xmldoc] and a list of tags [tagList]
        [tagList] may sometimes be a list of lists?
        We iterate the taglist, searching another level deeper into the xml doc with each
        iteration. In tagList, index 0 is the first level of the xmldoc, index 1 is the list
        of all tags we look for at the level right under all the tags we found at index 0,
        and so on.
        For each item, we then iterate through all xmldocs we passed in (usually one?)
        and append the tag we wanted to an xmllist.
        We have empty strings if the tag was not found, and always return a flat list.
        **Returns:
        if the iHTML flag is set, we do the innerHTML method on the xmllist, otherwise
        we just return the xmllist.
        """
        xmldoc = [xmldoc]
        for i,x in enumerate(tagList):
            if type(x) is not ListType:
                x = [x]
            xmlnext = []
            for y in x:
                for z in xmldoc:
                    if z != '':
                        if i==0 and idx!=0:
                            if len(z.getElementsByTagName(y))>0:
                                xmlnext.append(z.getElementsByTagName(y)[idx-1])
                            else:
                                xmlnext.append('')
                        else:
                            blFlag = False
                            for za in z.getElementsByTagName(y):
                                blFlag = True
                                xmlnext.append(za)
                            if blFlag==False and blank==True:
                                xmlnext.append('')
                    else:
                        xmlnext.append('')
            xmldoc = xmlnext

        if debug:
            if len(xmldoc)==1 and iHTML:
                print self.__innerHTML__(xmldoc[0])
            elif iHTML:
                print [self.__innerHTML__(x) for x in xmldoc]
            else:
                print xmldoc

        if len(xmldoc)==1 and iHTML:
            if listType:
                return [self.__innerHTML__(xmldoc[0])]
            else:
                return self.__innerHTML__(xmldoc[0])
        elif iHTML:
            return [self.__innerHTML__(x) for x in xmldoc]
        else:
            return xmldoc

    def __asg_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            record = []
            if len(x.getElementsByTagName("first-name"))>0:
                record = [1]
                record.extend(self.__tagNme__(x, [["last-name", "first-name"]]))
            else:
                record = [0]
                record.extend(self.__tagNme__(x, [["orgname", "role"]]))
            record.extend(self.__tagNme__(x, ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], blank=True))
            record.extend(self.__tagNme__(x, [["nationality", "residence"], "country"], blank=True))
            d_list.append(record)
        return d_list

    def __cit_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            #this means patcit is part of the XML
            record = [self.__tagNme__(x, ["category"])]
            if len(x.getElementsByTagName("patcit"))>0:
                record.extend(self.__tagNme__(x, ["patcit", ["country", "doc-number", "date", "kind", "name"]], blank=True))
                record.extend([""])
            elif len(x.getElementsByTagName("othercit"))>0:
                record.extend(["", "", "", "", ""])
                record.extend([self.__allHTML__(x, ["othercit"])])
                #probably should grab date information
            else:
                print x.toxml()
            d_list.append(record)
        return d_list

    def __rel_detail__(self, xmldoc, debug=False):
        d_list = []
        for x in xmldoc:
            for y in ["continuation-in-part", "continuation", "division", "reissue"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["relation", "child-doc"], [[["doc-number", "country", "kind"]]], baseList=[y, -1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True, idx=1))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-grant-document"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-pct-document"],  [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
            for y in ["related-publication", "us-provisional-application"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["document-id"], [[["doc-number", "country", "kind"]]], baseList=[y, 0], blank=True))
            if debug:
                print "-------------------"
                for x in d_list:
                    print x
        return d_list

    def __repr__(self):
        return \
"""country = %s, patent = %s, pat_type = %s,
date_grant = %s, date_app = %s,
         abstract = %s
  invention_title = %s
len(classes, asg, cit, ret, inv, law) = %s""" % (self.country, self.patent, self.pat_type, self.date_grant, self.date_app,
       self.abstract[:50], self.invention_title[:50],
       str([len(self.classes),  len(self.asg_list), len(self.cit_list),
            len(self.rel_list), len(self.inv_list), len(self.law_list)]))
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.api.settings")

import copy
import logging
from collections import namedtuple

import django

from api.webview.models import HarvesterResponse, Document

from scrapi import events
from scrapi.linter import RawDocument, NormalizedDocument
from scrapi.processing.base import BaseProcessor, BaseHarvesterResponse, BaseDatabaseManager

django.setup()
logger = logging.getLogger(__name__)

class DatabaseManager(BaseDatabaseManager):
    '''All database management is performed by django'''

    def setup(self):
        return True

    def tear_down(self):
        pass

    def clear(self, force=False):
        pass

    def celery_setup(self, *args, **kwargs):
        pass

class PostgresProcessor(BaseProcessor):
    NAME = 'postgres'

    manager = DatabaseManager()

    def documents(self, *sources):
        for source in sources:
            for doc in Document.objects.filter(source=source):
                yield RawDocument(doc.raw, clean=False, validate=False)

    def document_query(self, source, docID):
        try:
            document = Document.objects.get(source=source, docID=docID)
        except Document.DoesNotExist:
            return None

        DocumentTuple = namedtuple('Document', ['raw', 'normalized'])

        raw = RawDocument(document.raw, clean=False, validate=False)
        normalized = NormalizedDocument(document.normalized, validate=False)

        return DocumentTuple(raw, normalized)

    def document_delete(self, source, docID):
        doc = Document.objects.get(source=source, docID=docID)
        doc.delete()

    def document_create(self, attributes):
        Document.objects.create(
            source=attributes['source'],
            docID=attributes['docID'],
            providerUpdatedDateTime=None,
            raw=attributes,
            normalized=None
        ).save()

    @property
    def HarvesterResponseModel(self):
        return HarvesterResponseModel

    @events.logged(events.PROCESSING, 'raw.postgres')
    def process_raw(self, raw_doc):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        modified_doc = copy.deepcopy(raw_doc.attributes)
        if modified_doc.get('versions'):
            modified_doc['versions'] = list(map(str, modified_doc['versions']))

        document.raw = modified_doc

        document.save()

    @events.logged(events.PROCESSING, 'normalized.postgres')
    def process_normalized(self, raw_doc, normalized):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        document.normalized = normalized.attributes
        document.providerUpdatedDateTime = normalized['providerUpdatedDateTime']

        document.save()

    def _get_by_source_id(self, model, source, docID):
        try:
            return Document.objects.filter(source=source, docID=docID)[0]
        except IndexError:
            return None

class HarvesterResponseModel(BaseHarvesterResponse):

    response = None

    def __init__(self, *args, **kwargs):
        if kwargs:
            key = kwargs['method'].lower() + kwargs['url'].lower()
            self.response = HarvesterResponse(key=key, *args, **kwargs)
        else:
            self.response = args[0]

    @property
    def method(self):
        return str(self.response.method)

    @property
    def url(self):
        return str(self.response.url)

    @property
    def ok(self):
        return bool(self.response.ok)

    @property
    def content(self):
        if isinstance(self.response.content, memoryview):
            return self.response.content.tobytes()
        if isinstance(self.response.content, bytes):
            return self.response.content
        return str(self.response.content)

    @property
    def encoding(self):
        return str(self.response.encoding)

    @property
    def headers_str(self):
        return str(self.response.headers_str)

    @property
    def status_code(self):
        return int(self.response.status_code)

    @property
    def time_made(self):
        return str(self.response.time_made)

    def save(self, *args, **kwargs):
        self.response.save()
        return self

    def update(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self.response, k, v)
        return self.save()

    @classmethod
    def get(cls, url=None, method=None):
        key = method.lower() + url.lower()
        try:
            return cls(HarvesterResponse.objects.get(key=key))
        except HarvesterResponse.DoesNotExist:
            raise cls.DoesNotExist
<commit_msg>Doc tuple return, fix iterator without soruces, change names<commit_after>from __future__ import absolute_import

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.api.settings")

import copy
import logging
from collections import namedtuple

import django

from api.webview.models import HarvesterResponse, Document

from scrapi import events
from scrapi.linter import RawDocument, NormalizedDocument
from scrapi.processing import DocumentTuple
from scrapi.processing.base import BaseProcessor, BaseHarvesterResponse, BaseDatabaseManager

django.setup()
logger = logging.getLogger(__name__)

class DatabaseManager(BaseDatabaseManager):
    '''All database management is performed by django'''

    def setup(self):
        return True

    def tear_down(self):
        pass

    def clear(self, force=False):
        pass

    def celery_setup(self, *args, **kwargs):
        pass

class PostgresProcessor(BaseProcessor):
    NAME = 'postgres'

    manager = DatabaseManager()

    def documents(self, *sources):
        sources = sources
        q = Document.objects.all()
        querysets = (q.filter(source=source) for source in sources) if sources else [q]
        for query in querysets:
            for doc in query:
                try:
                    raw = RawDocument(doc.raw, clean=False, validate=False)
                except AttributeError as e:
                    logger.info('{}  -- Malformed rawdoc in database, skipping'.format(e))
                    raw = None
                    continue
                normalized = NormalizedDocument(doc.normalized, validate=False, clean=False) if doc.normalized else None
                yield DocumentTuple(raw, normalized)

    def get(self, source, docID):
        try:
            document = Document.objects.get(source=source, docID=docID)
        except Document.DoesNotExist:
            return None
        raw = RawDocument(document.raw, clean=False, validate=False)
        normalized = NormalizedDocument(document.normalized, validate=False, clean=False)

        return DocumentTuple(raw, normalized)

    def delete(self, source, docID):
        doc = Document.objects.get(source=source, docID=docID)
        doc.delete()

    def create(self, attributes):
        Document.objects.create(
            source=attributes['source'],
            docID=attributes['docID'],
            providerUpdatedDateTime=None,
            raw=attributes,
            normalized=None
        ).save()

    @property
    def HarvesterResponseModel(self):
        return HarvesterResponseModel

    @events.logged(events.PROCESSING, 'raw.postgres')
    def process_raw(self, raw_doc):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        modified_doc = copy.deepcopy(raw_doc.attributes)
        if modified_doc.get('versions'):
            modified_doc['versions'] = list(map(str, modified_doc['versions']))

        document.raw = modified_doc

        document.save()

    @events.logged(events.PROCESSING, 'normalized.postgres')
    def process_normalized(self, raw_doc, normalized):
        source, docID = raw_doc['source'], raw_doc['docID']
        document = self._get_by_source_id(Document, source, docID) or Document(source=source, docID=docID)

        document.normalized = normalized.attributes
        document.providerUpdatedDateTime = normalized['providerUpdatedDateTime']

        document.save()

    def _get_by_source_id(self, model, source, docID):
        try:
            return Document.objects.filter(source=source, docID=docID)[0]
        except IndexError:
            return None

class HarvesterResponseModel(BaseHarvesterResponse):

    response = None

    def __init__(self, *args, **kwargs):
        if kwargs:
            key = kwargs['method'].lower() + kwargs['url'].lower()
            self.response = HarvesterResponse(key=key, *args, **kwargs)
        else:
            self.response = args[0]

    @property
    def method(self):
        return str(self.response.method)

    @property
    def url(self):
        return str(self.response.url)

    @property
    def ok(self):
        return bool(self.response.ok)

    @property
    def content(self):
        if isinstance(self.response.content, memoryview):
            return self.response.content.tobytes()
        if isinstance(self.response.content, bytes):
            return self.response.content
        return str(self.response.content)

    @property
    def encoding(self):
        return str(self.response.encoding)

    @property
    def headers_str(self):
        return str(self.response.headers_str)

    @property
    def status_code(self):
        return int(self.response.status_code)

    @property
    def time_made(self):
        return str(self.response.time_made)

    def save(self, *args, **kwargs):
        self.response.save()
        return self

    def update(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self.response, k, v)
        return self.save()

    @classmethod
    def get(cls, url=None, method=None):
        key = method.lower() + url.lower()
        try:
            return cls(HarvesterResponse.objects.get(key=key))
        except HarvesterResponse.DoesNotExist:
            raise cls.DoesNotExist
<|endoftext|>