<commit_before># -*- coding: utf-8 -*-
# (c) 2015, 2016 Daniel Lobato <elobatocs@gmail.com>
# (c) 2016 Guido Günther <agx@sigxcpu.org>
# (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = '''
    callback: foreman
    type: notification
    short_description: Sends events to Foreman
    description:
      - This callback will report facts and task events to Foreman https://theforeman.org/
      - Before 2.4, if you wanted to use an ini configuration, the file must be placed in the same directory as this plugin and named foreman.ini
      - In 2.4 and above you can just put it in the main Ansible configuration file.
    version_added: "2.2"
    requirements:
      - whitelisting in configuration
      - requests (python library)
    options:
      url:
        description: URL to the Foreman server
        env:
          - name: FOREMAN_URL
        required: True
        default: http://localhost:3000
        ini:
          - section: callback_foreman
            key: url
      ssl_cert:
        description: X509 certificate to authenticate to Foreman if https is used
        env:
            - name: FOREMAN_SSL_CERT
        default: /etc/foreman/client_cert.pem
        ini:
          - section: callback_foreman
            key: ssl_cert
      ssl_key:
        description: the corresponding private key
        env:
          - name: FOREMAN_SSL_KEY
        default: /etc/foreman/client_key.pem
        ini:
          - section: callback_foreman
            key: ssl_key
      verify_certs:
        description:
          - Toggle to decidewhether to verify the Foreman certificate.
          - It can be set to '1' to verify SSL certificates using the installed CAs or to a path pointing to a CA bundle.
          - Set to '0' to disable certificate checking.
        env:
          - name: FOREMAN_SSL_VERIFY
        default: 1
        ini:
          - section: callback_foreman
            key: verify_certs
'''

import os
from datetime import datetime
from collections import defaultdict
import json
import time

try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False

from ansible.plugins.callback import CallbackBase

class CallbackModule(CallbackBase):
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'foreman'
    CALLBACK_NEEDS_WHITELIST = True

    FOREMAN_HEADERS = {
        "Content-Type": "application/json",
        "Accept": "application/json"
    }
    TIME_FORMAT = "%Y-%m-%d %H:%M:%S %f"

    def __init__(self):
        super(CallbackModule, self).__init__()
        self.items = defaultdict(list)
        self.start_time = int(time.time())

    def set_options(self, task_keys=None, var_options=None, direct=None):

        super(CallbackModule, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct)

        self.FOREMAN_URL = self.get_option('url')
        self.FOREMAN_SSL_CERT = (self.get_option['ssl_cert'], self.get_option['ssl_key'])
        self.FOREMAN_SSL_VERIFY = str(self.get_option['verify_certs'])

        self.ssl_verify = self._ssl_verify()

        if HAS_REQUESTS:
            requests_major = int(requests.__version__.split('.')[0])
            if requests_major < 2:
                self._disable_plugin('The `requests` python module is too old.')
        else:
            self._disable_plugin('The `requests` python module is not installed.')

        if self.FOREMAN_URL.startswith('https://'):
            if not os.path.exists(self.FOREMAN_SSL_CERT[0]):
                self._disable_plugin('FOREMAN_SSL_CERT %s not found.' % self.FOREMAN_SSL_CERT[0])

            if not os.path.exists(self.FOREMAN_SSL_CERT[1]):
                self._disable_plugin('FOREMAN_SSL_KEY %s not found.' % self.FOREMAN_SSL_CERT[1])

    def _disable_plugin(self, msg):
        self.disabled = True
        if msg:
            self._display.warning(msg + ' Disabling the Foreman callback plugin.')
        else:
            self._display.warning('Disabling the Foreman callback plugin.')

    def _ssl_verify(self):
        if self.FOREMAN_SSL_VERIFY.lower() in ["1", "true", "on"]:
            verify = True
        elif self.FOREMAN_SSL_VERIFY.lower() in ["0", "false", "off"]:
            requests.packages.urllib3.disable_warnings()
            self._display.warning("SSL verification of %s disabled" %
                                  self.FOREMAN_URL)
            verify = False
        else:  # Set to a CA bundle:
            verify = self.FOREMAN_SSL_VERIFY
        return verify

    def send_facts(self, host, data):
        """
        Sends facts to Foreman, to be parsed by foreman_ansible fact
        parser.  The default fact importer should import these facts
        properly.
        """
        data["_type"] = "ansible"
        data["_timestamp"] = datetime.now().strftime(self.TIME_FORMAT)
        facts = {"name": host,
                 "facts": data,
                 }
        try:
            r = requests.post(url=self.FOREMAN_URL + '/api/v2/hosts/facts',
                              data=json.dumps(facts),
                              headers=self.FOREMAN_HEADERS,
                              cert=self.FOREMAN_SSL_CERT,
                              verify=self.ssl_verify)
            r.raise_for_status()
        except requests.exceptions.RequestException as err:
            print(str(err))

    def _build_log(self, data):
        logs = []
        for entry in data:
            source, msg = entry
            if 'failed' in msg:
                level = 'err'
            elif 'changed' in msg and msg['changed']:
                level = 'notice'
            else:
                level = 'info'
            logs.append({
                "log": {
                    'sources': {
                        'source': source
                    },
                    'messages': {
                        'message': json.dumps(msg)
                    },
                    'level': level
                }
            })
        return logs

    def send_reports(self, stats):
        """
        Send reports to Foreman to be parsed by its config report
        importer. THe data is in a format that Foreman can handle
        without writing another report importer.
        """
        status = defaultdict(lambda: 0)
        metrics = {}

        for host in stats.processed.keys():
            sum = stats.summarize(host)
            status["applied"] = sum['changed']
            status["failed"] = sum['failures'] + sum['unreachable']
            status["skipped"] = sum['skipped']
            log = self._build_log(self.items[host])
            metrics["time"] = {"total": int(time.time()) - self.start_time}
            now = datetime.now().strftime(self.TIME_FORMAT)
            report = {
                "report": {
                    "host": host,
                    "reported_at": now,
                    "metrics": metrics,
                    "status": status,
                    "logs": log,
                }
            }
            # To be changed to /api/v2/config_reports in 1.11.  Maybe we
            # could make a GET request to get the Foreman version & do
            # this automatically.
            try:
                r = requests.post(url=self.FOREMAN_URL + '/api/v2/reports',
                                  data=json.dumps(report),
                                  headers=self.FOREMAN_HEADERS,
                                  cert=self.FOREMAN_SSL_CERT,
                                  verify=self.ssl_verify)
                r.raise_for_status()
            except requests.exceptions.RequestException as err:
                print(str(err))
            self.items[host] = []

    def append_result(self, result):
        name = result._task.get_name()
        host = result._host.get_name()
        self.items[host].append((name, result._result))

    # Ansible callback API
    def v2_runner_on_failed(self, result, ignore_errors=False):
        self.append_result(result)

    def v2_runner_on_unreachable(self, result):
        self.append_result(result)

    def v2_runner_on_async_ok(self, result, jid):
        self.append_result(result)

    def v2_runner_on_async_failed(self, result, jid):
        self.append_result(result)

    def v2_playbook_on_stats(self, stats):
        self.send_reports(stats)

    def v2_runner_on_ok(self, result):
        res = result._result
        module = result._task.action

        if module == 'setup' or 'ansible_facts' in res:
            host = result._host.get_name()
            self.send_facts(host, res)
        else:
            self.append_result(result)
<commit_msg>foreman: Fix get_option regression (#40534)<commit_after># -*- coding: utf-8 -*-
# (c) 2015, 2016 Daniel Lobato <elobatocs@gmail.com>
# (c) 2016 Guido Günther <agx@sigxcpu.org>
# (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = '''
    callback: foreman
    type: notification
    short_description: Sends events to Foreman
    description:
      - This callback will report facts and task events to Foreman https://theforeman.org/
      - Before 2.4, if you wanted to use an ini configuration, the file must be placed in the same directory as this plugin and named foreman.ini
      - In 2.4 and above you can just put it in the main Ansible configuration file.
    version_added: "2.2"
    requirements:
      - whitelisting in configuration
      - requests (python library)
    options:
      url:
        description: URL to the Foreman server
        env:
          - name: FOREMAN_URL
        required: True
        default: http://localhost:3000
        ini:
          - section: callback_foreman
            key: url
      ssl_cert:
        description: X509 certificate to authenticate to Foreman if https is used
        env:
            - name: FOREMAN_SSL_CERT
        default: /etc/foreman/client_cert.pem
        ini:
          - section: callback_foreman
            key: ssl_cert
      ssl_key:
        description: the corresponding private key
        env:
          - name: FOREMAN_SSL_KEY
        default: /etc/foreman/client_key.pem
        ini:
          - section: callback_foreman
            key: ssl_key
      verify_certs:
        description:
          - Toggle to decidewhether to verify the Foreman certificate.
          - It can be set to '1' to verify SSL certificates using the installed CAs or to a path pointing to a CA bundle.
          - Set to '0' to disable certificate checking.
        env:
          - name: FOREMAN_SSL_VERIFY
        default: 1
        ini:
          - section: callback_foreman
            key: verify_certs
'''

import os
from datetime import datetime
from collections import defaultdict
import json
import time

try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False

from ansible.plugins.callback import CallbackBase

class CallbackModule(CallbackBase):
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'foreman'
    CALLBACK_NEEDS_WHITELIST = True

    FOREMAN_HEADERS = {
        "Content-Type": "application/json",
        "Accept": "application/json"
    }
    TIME_FORMAT = "%Y-%m-%d %H:%M:%S %f"

    def __init__(self):
        super(CallbackModule, self).__init__()
        self.items = defaultdict(list)
        self.start_time = int(time.time())

    def set_options(self, task_keys=None, var_options=None, direct=None):

        super(CallbackModule, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct)

        self.FOREMAN_URL = self.get_option('url')
        self.FOREMAN_SSL_CERT = (self.get_option('ssl_cert'), self.get_option('ssl_key'))
        self.FOREMAN_SSL_VERIFY = str(self.get_option('verify_certs'))

        self.ssl_verify = self._ssl_verify()

        if HAS_REQUESTS:
            requests_major = int(requests.__version__.split('.')[0])
            if requests_major < 2:
                self._disable_plugin('The `requests` python module is too old.')
        else:
            self._disable_plugin('The `requests` python module is not installed.')

        if self.FOREMAN_URL.startswith('https://'):
            if not os.path.exists(self.FOREMAN_SSL_CERT[0]):
                self._disable_plugin('FOREMAN_SSL_CERT %s not found.' % self.FOREMAN_SSL_CERT[0])

            if not os.path.exists(self.FOREMAN_SSL_CERT[1]):
                self._disable_plugin('FOREMAN_SSL_KEY %s not found.' % self.FOREMAN_SSL_CERT[1])

    def _disable_plugin(self, msg):
        self.disabled = True
        if msg:
            self._display.warning(msg + ' Disabling the Foreman callback plugin.')
        else:
            self._display.warning('Disabling the Foreman callback plugin.')

    def _ssl_verify(self):
        if self.FOREMAN_SSL_VERIFY.lower() in ["1", "true", "on"]:
            verify = True
        elif self.FOREMAN_SSL_VERIFY.lower() in ["0", "false", "off"]:
            requests.packages.urllib3.disable_warnings()
            self._display.warning("SSL verification of %s disabled" %
                                  self.FOREMAN_URL)
            verify = False
        else:  # Set to a CA bundle:
            verify = self.FOREMAN_SSL_VERIFY
        return verify

    def send_facts(self, host, data):
        """
        Sends facts to Foreman, to be parsed by foreman_ansible fact
        parser.  The default fact importer should import these facts
        properly.
        """
        data["_type"] = "ansible"
        data["_timestamp"] = datetime.now().strftime(self.TIME_FORMAT)
        facts = {"name": host,
                 "facts": data,
                 }
        try:
            r = requests.post(url=self.FOREMAN_URL + '/api/v2/hosts/facts',
                              data=json.dumps(facts),
                              headers=self.FOREMAN_HEADERS,
                              cert=self.FOREMAN_SSL_CERT,
                              verify=self.ssl_verify)
            r.raise_for_status()
        except requests.exceptions.RequestException as err:
            print(str(err))

    def _build_log(self, data):
        logs = []
        for entry in data:
            source, msg = entry
            if 'failed' in msg:
                level = 'err'
            elif 'changed' in msg and msg['changed']:
                level = 'notice'
            else:
                level = 'info'
            logs.append({
                "log": {
                    'sources': {
                        'source': source
                    },
                    'messages': {
                        'message': json.dumps(msg)
                    },
                    'level': level
                }
            })
        return logs

    def send_reports(self, stats):
        """
        Send reports to Foreman to be parsed by its config report
        importer. THe data is in a format that Foreman can handle
        without writing another report importer.
        """
        status = defaultdict(lambda: 0)
        metrics = {}

        for host in stats.processed.keys():
            sum = stats.summarize(host)
            status["applied"] = sum['changed']
            status["failed"] = sum['failures'] + sum['unreachable']
            status["skipped"] = sum['skipped']
            log = self._build_log(self.items[host])
            metrics["time"] = {"total": int(time.time()) - self.start_time}
            now = datetime.now().strftime(self.TIME_FORMAT)
            report = {
                "report": {
                    "host": host,
                    "reported_at": now,
                    "metrics": metrics,
                    "status": status,
                    "logs": log,
                }
            }
            # To be changed to /api/v2/config_reports in 1.11.  Maybe we
            # could make a GET request to get the Foreman version & do
            # this automatically.
            try:
                r = requests.post(url=self.FOREMAN_URL + '/api/v2/reports',
                                  data=json.dumps(report),
                                  headers=self.FOREMAN_HEADERS,
                                  cert=self.FOREMAN_SSL_CERT,
                                  verify=self.ssl_verify)
                r.raise_for_status()
            except requests.exceptions.RequestException as err:
                print(str(err))
            self.items[host] = []

    def append_result(self, result):
        name = result._task.get_name()
        host = result._host.get_name()
        self.items[host].append((name, result._result))

    # Ansible callback API
    def v2_runner_on_failed(self, result, ignore_errors=False):
        self.append_result(result)

    def v2_runner_on_unreachable(self, result):
        self.append_result(result)

    def v2_runner_on_async_ok(self, result, jid):
        self.append_result(result)

    def v2_runner_on_async_failed(self, result, jid):
        self.append_result(result)

    def v2_playbook_on_stats(self, stats):
        self.send_reports(stats)

    def v2_runner_on_ok(self, result):
        res = result._result
        module = result._task.action

        if module == 'setup' or 'ansible_facts' in res:
            host = result._host.get_name()
            self.send_facts(host, res)
        else:
            self.append_result(result)
<|endoftext|>


<commit_before>from ..data import setup
from ..schema import schema

setup()

def test_str_schema():
    assert str(schema) == '''schema {
  query: Query
  mutation: Mutation
}

type Faction implements Node {
  id: ID!
  name: String
  ships(before: String, after: String, first: Int, last: Int): ShipConnection
}

input IntroduceShipInput {
  shipName: String!
  factionId: String!
  clientMutationId: String
}

type IntroduceShipPayload {
  ship: Ship
  faction: Faction
}

type Mutation {
  introduceShip(input: IntroduceShipInput): IntroduceShipPayload
}

interface Node {
  id: ID!
}

type PageInfo {
  hasNextPage: Boolean!
  hasPreviousPage: Boolean!
  startCursor: String
  endCursor: String
}

type Query {
  rebels: Faction
  empire: Faction
  node(id: ID!): Node
}

type Ship implements Node {
  id: ID!
  name: String
}

type ShipConnection {
  pageInfo: PageInfo!
  edges: [ShipEdge]
}

type ShipEdge {
  node: Ship
  cursor: String!
}
'''

def test_correctly_fetches_id_name_rebels():
    query = '''
      query RebelsQuery {
        rebels {
          id
          name
        }
      }
    '''
    expected = {
        'rebels': {
            'id': 'RmFjdGlvbjox',
            'name': 'Alliance to Restore the Republic'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_rebels():
    query = '''
      query RebelsRefetchQuery {
        node(id: "RmFjdGlvbjox") {
          id
          ... on Faction {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'RmFjdGlvbjox',
            'name': 'Alliance to Restore the Republic'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_fetches_id_name_empire():
    query = '''
      query EmpireQuery {
        empire {
          id
          name
        }
      }
    '''
    expected = {
        'empire': {
            'id': 'RmFjdGlvbjoy',
            'name': 'Galactic Empire'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_empire():
    query = '''
      query EmpireRefetchQuery {
        node(id: "RmFjdGlvbjoy") {
          id
          ... on Faction {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'RmFjdGlvbjoy',
            'name': 'Galactic Empire'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_xwing():
    query = '''
      query XWingRefetchQuery {
        node(id: "U2hpcDox") {
          id
          ... on Ship {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'U2hpcDox',
            'name': 'X-Wing'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected
<commit_msg>Fixed starwars relay tests schema representation<commit_after>from ..data import setup
from ..schema import schema

setup()

def test_str_schema():
    assert str(schema) == '''schema {
  query: Query
  mutation: Mutation
}

type Faction implements Node {
  id: ID!
  name: String
  ships(before: String, after: String, first: Int, last: Int): ShipConnection
}

input IntroduceShipInput {
  clientMutationId: String
  factionId: String!
  shipName: String!
}

type IntroduceShipPayload {
  ship: Ship
  faction: Faction
}

type Mutation {
  introduceShip(input: IntroduceShipInput!): IntroduceShipPayload
}

interface Node {
  id: ID!
}

type PageInfo {
  hasNextPage: Boolean!
  hasPreviousPage: Boolean!
  startCursor: String
  endCursor: String
}

type Query {
  rebels: Faction
  empire: Faction
  node(id: ID!): Node
}

type Ship implements Node {
  id: ID!
  name: String
}

type ShipConnection {
  pageInfo: PageInfo!
  edges: [ShipEdge]
}

type ShipEdge {
  node: Ship
  cursor: String!
}
'''

def test_correctly_fetches_id_name_rebels():
    query = '''
      query RebelsQuery {
        rebels {
          id
          name
        }
      }
    '''
    expected = {
        'rebels': {
            'id': 'RmFjdGlvbjox',
            'name': 'Alliance to Restore the Republic'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_rebels():
    query = '''
      query RebelsRefetchQuery {
        node(id: "RmFjdGlvbjox") {
          id
          ... on Faction {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'RmFjdGlvbjox',
            'name': 'Alliance to Restore the Republic'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_fetches_id_name_empire():
    query = '''
      query EmpireQuery {
        empire {
          id
          name
        }
      }
    '''
    expected = {
        'empire': {
            'id': 'RmFjdGlvbjoy',
            'name': 'Galactic Empire'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_empire():
    query = '''
      query EmpireRefetchQuery {
        node(id: "RmFjdGlvbjoy") {
          id
          ... on Faction {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'RmFjdGlvbjoy',
            'name': 'Galactic Empire'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected

def test_correctly_refetches_xwing():
    query = '''
      query XWingRefetchQuery {
        node(id: "U2hpcDox") {
          id
          ... on Ship {
            name
          }
        }
      }
    '''
    expected = {
        'node': {
            'id': 'U2hpcDox',
            'name': 'X-Wing'
        }
    }
    result = schema.execute(query)
    assert not result.errors
    assert result.data == expected
<|endoftext|>


<commit_before>

from apgl.util.Util import Util
from exp.sandbox.recommendation.IterativeSoftImpute import IterativeSoftImpute
from exp.sandbox.recommendation.SoftImpute import SoftImpute
import sys
import numpy
import unittest
import logging
import scipy.sparse
import numpy.linalg
import numpy.testing as nptst
import exp.util.SparseUtils as ExpSU

class IterativeSoftImputeTest(unittest.TestCase):
    def setUp(self):
        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
        numpy.set_printoptions(precision=3, suppress=True, linewidth=100)

        numpy.seterr(all="raise")
        numpy.random.seed(21)

        #Create a sequence of matrices
        n = 20
        m = 10
        r = 8
        U, s, V = ExpSU.SparseUtils.generateLowRank((n, m), r)

        numInds = 100
        inds = numpy.random.randint(0, n*m-1, numInds)
        inds = numpy.unique(inds)
        numpy.random.shuffle(inds)
        numInds = inds.shape[0]

        inds1 = inds[0:numInds/3]
        inds2 = inds[0:2*numInds/3]
        inds3 = inds

        X1 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds1)
        X2 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds2)
        X3 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds3)

        self.matrixList = [X1, X2, X3]
        self.indsList = [inds1, inds2, inds3]

    def testLearnModel(self):
        lmbda = 0.0
        eps = 0.1
        k = 10

        matrixIterator = iter(self.matrixList)
        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="propack")
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        #Check that ZList is the same as XList
        for i, Z in enumerate(ZList):
            U, s, V = Z
            Xhat = (U*s).dot(V.T)

            nptst.assert_array_almost_equal(Xhat, self.matrixList[i].todense())

        #Compare solution with that of SoftImpute class
        lmbdaList = [0.1, 0.2, 0.5, 1.0]

        for lmbda in lmbdaList:
            iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="propack", updateAlg="zero")

            matrixIterator = iter(self.matrixList)
            ZList = iterativeSoftImpute.learnModel(matrixIterator)

            lmbdas = numpy.array([lmbda])

            softImpute = SoftImpute(lmbdas, k=k, eps=eps)
            Z1 = softImpute.learnModel(self.matrixList[0])
            Z2 = softImpute.learnModel(self.matrixList[1])
            Z3 = softImpute.learnModel(self.matrixList[2])

            ZList2 = [Z1, Z2, Z3]

            for j, Zhat in enumerate(ZList):
                U, s, V = Zhat
                Z = (U*s).dot(V.T)
                nptst.assert_array_almost_equal(Z, ZList2[j].todense())

                #Also test with true solution Z = S_lambda(X + Z^\bot_\omega)
                Zomega = numpy.zeros(self.matrixList[j].shape)

                rowInds, colInds = self.matrixList[j].nonzero()
                for i in range(self.matrixList[j].nonzero()[0].shape[0]):
                    Zomega[rowInds[i], colInds[i]] = Z[rowInds[i], colInds[i]]

                U, s, V = ExpSU.SparseUtils.svdSoft(numpy.array(self.matrixList[j]-Zomega+Z), lmbda)

                tol = 0.1
                self.assertTrue(numpy.linalg.norm(Z -(U*s).dot(V.T))**2 < tol)

        #Test the SVD updating solution
        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=9, eps=eps, svdAlg="propack", updateAlg="zero")
        iterativeSoftImpute.svdAlg = "svdUpdate"
        matrixIterator = iter(self.matrixList)
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        #Test using Randomised SVD
        iterativeSoftImpute.svdAlg = "RSVD"
        matrixIterator = iter(self.matrixList)
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        #Test on an increasing then decreasing set of solutions

    @unittest.skip("")
    def testPredict(self):
        #Create a set of indices
        lmbda = 0.0

        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10)
        matrixIterator = iter(self.matrixList)
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        XhatList = iterativeSoftImpute.predict(ZList, self.indsList)

        #Check we get the exact matrices returned
        for i, Xhat in enumerate(XhatList):
            nptst.assert_array_almost_equal(numpy.array(Xhat.todense()), self.matrixList[i].todense())

            self.assertEquals(Xhat.nnz, self.indsList[i].shape[0])

    @unittest.skip("")
    def testModelSelect(self):
        lmbda = 0.1
        X = self.matrixList[0]

        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10)
        lmbdas = numpy.array([1.0, 0.8, 0.5, 0.2, 0.1])
        folds = 5
        iterativeSoftImpute.modelSelect(X, lmbdas, folds)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()
    <commit_msg>Better test for random SVD, model selection and predict <commit_after>

from apgl.util.Util import Util
from exp.sandbox.recommendation.IterativeSoftImpute import IterativeSoftImpute
from exp.sandbox.recommendation.SoftImpute import SoftImpute
import sys
import numpy
import unittest
import logging
import scipy.sparse
import numpy.linalg
import numpy.testing as nptst
import exp.util.SparseUtils as ExpSU

class IterativeSoftImputeTest(unittest.TestCase):
    def setUp(self):
        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
        numpy.set_printoptions(precision=4, suppress=True, linewidth=200)

        numpy.seterr(all="raise")
        numpy.random.seed(21)

        #Create a sequence of matrices
        n = 20
        m = 10
        r = 8
        U, s, V = ExpSU.SparseUtils.generateLowRank((n, m), r)

        numInds = 200
        inds = numpy.random.randint(0, n*m-1, numInds)
        inds = numpy.unique(inds)
        numpy.random.shuffle(inds)
        numInds = inds.shape[0]

        inds1 = inds[0:numInds/3]
        inds2 = inds[0:2*numInds/3]
        inds3 = inds

        X1 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds1)
        X2 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds2)
        X3 = ExpSU.SparseUtils.reconstructLowRank(U, s, V, inds3)

        self.matrixList = [X1, X2, X3]
        self.indsList = [inds1, inds2, inds3]

    def testLearnModel(self):
        lmbda = 0.0
        eps = 0.1
        k = 10

        matrixIterator = iter(self.matrixList)
        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="propack")
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        #Check that ZList is the same as XList
        for i, Z in enumerate(ZList):
            U, s, V = Z
            Xhat = (U*s).dot(V.T)

            nptst.assert_array_almost_equal(Xhat, self.matrixList[i].todense())

        #Compare solution with that of SoftImpute class
        lmbdaList = [0.1, 0.2, 0.5, 1.0]

        for lmbda in lmbdaList:
            iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="propack", updateAlg="zero")

            matrixIterator = iter(self.matrixList)
            ZList = iterativeSoftImpute.learnModel(matrixIterator)

            lmbdas = numpy.array([lmbda])

            softImpute = SoftImpute(lmbdas, k=k, eps=eps)
            Z1 = softImpute.learnModel(self.matrixList[0])
            Z2 = softImpute.learnModel(self.matrixList[1])
            Z3 = softImpute.learnModel(self.matrixList[2])

            ZList2 = [Z1, Z2, Z3]

            for j, Zhat in enumerate(ZList):
                U, s, V = Zhat
                Z = (U*s).dot(V.T)
                nptst.assert_array_almost_equal(Z, ZList2[j].todense())

                #Also test with true solution Z = S_lambda(X + Z^\bot_\omega)
                Zomega = numpy.zeros(self.matrixList[j].shape)

                rowInds, colInds = self.matrixList[j].nonzero()
                for i in range(self.matrixList[j].nonzero()[0].shape[0]):
                    Zomega[rowInds[i], colInds[i]] = Z[rowInds[i], colInds[i]]

                U, s, V = ExpSU.SparseUtils.svdSoft(numpy.array(self.matrixList[j]-Zomega+Z), lmbda)

                tol = 0.1
                self.assertTrue(numpy.linalg.norm(Z -(U*s).dot(V.T))**2 < tol)

    def testLearnModel2(self):
        #Test the SVD updating solution in the case where we get an exact solution
        lmbda = 0.0
        eps = 0.1
        k = 20

        matrixIterator = iter(self.matrixList)
        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="RSVD")
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        #Check that ZList is the same as XList
        for i, Z in enumerate(ZList):
            U, s, V = Z
            Xhat = (U*s).dot(V.T)

            nptst.assert_array_almost_equal(Xhat, self.matrixList[i].todense())

        #Compare solution with that of SoftImpute class
        lmbdaList = [0.1, 0.2, 0.5, 1.0]

        for lmbda in lmbdaList:
            iterativeSoftImpute = IterativeSoftImpute(lmbda, k=k, eps=eps, svdAlg="RSVD", updateAlg="zero")

            matrixIterator = iter(self.matrixList)
            ZList = iterativeSoftImpute.learnModel(matrixIterator)

            lmbdas = numpy.array([lmbda])

            softImpute = SoftImpute(lmbdas, k=k, eps=eps)
            Z1 = softImpute.learnModel(self.matrixList[0])
            Z2 = softImpute.learnModel(self.matrixList[1])
            Z3 = softImpute.learnModel(self.matrixList[2])

            ZList2 = [Z1, Z2, Z3]

            for j, Zhat in enumerate(ZList):
                U, s, V = Zhat
                Z = (U*s).dot(V.T)
                nptst.assert_array_almost_equal(Z, ZList2[j].todense())

                #Also test with true solution Z = S_lambda(X + Z^\bot_\omega)
                Zomega = numpy.zeros(self.matrixList[j].shape)

                rowInds, colInds = self.matrixList[j].nonzero()
                for i in range(self.matrixList[j].nonzero()[0].shape[0]):
                    Zomega[rowInds[i], colInds[i]] = Z[rowInds[i], colInds[i]]

                U, s, V = ExpSU.SparseUtils.svdSoft(numpy.array(self.matrixList[j]-Zomega+Z), lmbda)

                tol = 0.1
                self.assertTrue(numpy.linalg.norm(Z -(U*s).dot(V.T))**2 < tol)

    def testLearnModel3(self):
        #Test using Randomised SVD

        #Test on an increasing then decreasing set of solutions
        pass

    #@unittest.skip("")
    def testPredict(self):
        #Create a set of indices
        lmbda = 0.0

        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10)
        matrixIterator = iter(self.matrixList)
        ZList = iterativeSoftImpute.learnModel(matrixIterator)

        XhatList = iterativeSoftImpute.predict(ZList, self.indsList)

        #Check we get the exact matrices returned
        for i, Xhat in enumerate(XhatList):
            nptst.assert_array_almost_equal(numpy.array(Xhat.todense()), self.matrixList[i].todense())

            self.assertEquals(Xhat.nnz, self.indsList[i].shape[0])

        #Try moderate lambda
        lmbda = 0.1
        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10)
        matrixIterator = iter(self.matrixList)
        ZList = list(iterativeSoftImpute.learnModel(matrixIterator))

        XhatList = iterativeSoftImpute.predict(iter(ZList), self.indsList)

        for i, Xhat in enumerate(XhatList):
            for ind in self.indsList[i]:
                U, s, V = ZList[i]
                Z = (U*s).dot(V.T)
                self.assertEquals(Xhat[numpy.unravel_index(ind, Xhat.shape)], Z[numpy.unravel_index(ind, Xhat.shape)])

            self.assertEquals(Xhat.nnz, self.indsList[i].shape[0])

    #@unittest.skip("")
    def testModelSelect(self):
        lmbda = 0.1
        shape = (100, 100)
        r = 20
        k = 100
        noise = 0.2
        X = ExpSU.SparseUtils.generateSparseLowRank(shape, r, k, noise)

        print("--------------------")
        print(X.nnz)
        U, s, V = numpy.linalg.svd(X.todense())
        print(s)

        iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10, svdAlg="propack", updateAlg="zero")
        lmbdas = numpy.linspace(numpy.max(s), 0.01, 10)
        folds = 5
        iterativeSoftImpute.modelSelect(X, lmbdas, folds)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()
    <|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Uses the extended ContentHandler from xml_driver to extract the needed fields
from patent grant documents
"""

from cStringIO import StringIO
from datetime import datetime
from unidecode import unidecode
from handler import Patobj, PatentHandler
import re
import uuid
import xml.sax
import xml_util
import xml_driver

claim_num_regex = re.compile(r'^\d+\. *') # removes claim number from claim text

class Patent(PatentHandler):

    def __init__(self, xml_string, is_string=False):
        xh = xml_driver.XMLHandler()
        parser = xml_driver.make_parser()

        parser.setContentHandler(xh)
        parser.setFeature(xml_driver.handler.feature_external_ges, False)
        l = xml.sax.xmlreader.Locator()
        xh.setDocumentLocator(l)
        if is_string:
            parser.parse(StringIO(xml_string))
        else:
            parser.parse(xml_string)

        self.attributes = ['app','application','assignee_list','inventor_list',
                          'us_relation_list','us_classifications','ipcr_classifications',
                          'claims']

        self.xml = xh.root.patent_application_publication

        if self.xml.foreign_priority_data:
            self.country = self.xml.foreign_priority_data.contents_of('country_code')[0]
        else:
            self.country = self.xml.contents_of('country_code')[0]
        self.application = xml_util.normalize_document_identifier(self.xml.document_id.contents_of('doc_number')[0])
        self.kind = self.xml.document_id.contents_of('kind_code')[0]
        self.pat_type = None
        self.date_app = self.xml.document_id.contents_of('document_date')[0]
        self.clm_num = len(self.xml.subdoc_claims.claim)
        self.abstract = self.xml.subdoc_abstract.contents_of('paragraph', '', as_string=True, upper=False)
        self.invention_title = self._invention_title()

        self.app = {
            "id": self.application,
            "type": self.pat_type,
            "number": self.application,
            "country": self.country,
            "date": self._fix_date(self.date_app),
            "abstract": self.abstract,
            "title": self.invention_title,
            "kind": self.kind,
            "num_claims": self.clm_num
        }
        self.app["id"] = str(self.app["date"])[:4] + "/" + self.app["number"]

    def _invention_title(self):
        original = self.xml.contents_of('title_of_invention', upper=False)[0]
        if isinstance(original, list):
            original = ''.join(original)
        return original

    def _name_helper(self, tag_root):
        """
        Returns dictionary of firstname, lastname with prefix associated
        with lastname
        """
        firstname = tag_root.contents_of('given_name', as_string=True, upper=False)
        lastname = tag_root.contents_of('family_name', as_string=True, upper=False)
        return xml_util.associate_prefix(firstname, lastname)

    def _name_helper_dict(self, tag_root):
        """
        Returns dictionary of firstname, lastname with prefix associated
        with lastname
        """
        firstname = tag_root.contents_of('given_name', as_string=True, upper=False)
        lastname = tag_root.contents_of('family_name', as_string=True, upper=False)
        firstname, lastname = xml_util.associate_prefix(firstname, lastname)
        return {'name_first': firstname, 'name_last': lastname}

    def _fix_date(self, datestring):
        """
        Converts a number representing YY/MM to a Date
        """
        if not datestring:
            return None
        elif datestring[:4] < "1900":
            return None
        # default to first of month in absence of day
        if datestring[-4:-2] == '00':
            datestring = datestring[:-4] + '01' + datestring[-2:]
        if datestring[-2:] == '00':
            datestring = datestring[:6] + '01'
        try:
            datestring = datetime.strptime(datestring, '%Y%m%d')
            return datestring
        except Exception as inst:
            print inst, datestring
            return None

    @property
    def assignee_list(self):
        """
        Returns list of dictionaries:
        assignee:
          name_last
          name_first
          residence
          nationality
          organization
          sequence
        location:
          id
          city
          state
          country
        """
        assignees = self.xml.assignee
        if not assignees:
            return []
        res = []
        for i, assignee in enumerate(assignees):
            # add assignee data
            asg = {}
            asg.update(self._name_helper_dict(assignee))  # add firstname, lastname
            asg['organization'] = assignee.contents_of('organization_name', as_string=True, upper=False)
            asg['role'] = assignee.contents_of('role', as_string=True)
            if assignee.contents_of('country_code'):
                asg['nationality'] = assignee.contents_of('country_code')[0]
                asg['residence'] = assignee.contents_of('country_code')[0]
            # add location data for assignee
            loc = {}
            for tag in ['city', 'state']:
                loc[tag] = assignee.contents_of(tag, as_string=True, upper=False)
            if 'nationality' in asg:
                loc['country'] = asg['nationality']
            #this is created because of MySQL foreign key case sensitivities
                loc['id'] = unidecode(u"|".join([loc['city'], loc['state'], loc['country']]).lower())
            else:
                loc['id'] = u''
            if any(asg.values()) or any(loc.values()):
                asg['sequence'] = i
                asg['uuid'] = str(uuid.uuid1())
                res.append([asg, loc])
        return res

    @property
    def inventor_list(self):
        """
        Returns list of lists of applicant dictionary and location dictionary
        applicant:
          name_last
          name_first
          nationality
          sequence
        location:
          id
          city
          state
          country
        """
        applicants = self.xml.first_named_inventor + self.xml.inventors.inventor
        if not applicants:
            return []
        res = []
        for i, applicant in enumerate(applicants):
            # add applicant data
            app = {}
            app.update(self._name_helper_dict(applicant.name))
            app['nationality'] = applicant.contents_of('country_code', as_string=True)
            # add location data for applicant
            loc = {}
            for tag in ['city', 'state']:
                loc[tag] = applicant.residence.contents_of(tag, as_string=True, upper=False)
            loc['country'] = app['nationality']
            #this is created because of MySQL foreign key case sensitivities
            loc['id'] = unidecode("|".join([loc['city'], loc['state'], loc['country']]).lower())
            if any(app.values()) or any(loc.values()):
                app['sequence'] = i
                app['uuid'] = str(uuid.uuid1())
                res.append([app, loc])
        return res

    def _get_doc_info(self, root):
        """
        Accepts an XMLElement root as an argument. Returns list of
        [country, doc-number, kind, date] for the given root
        """
        res = {}
        country = root.contents_of('country_code')[0] if root.contents_of('country_code') else []
        kind = root.contents_of('kind_code')[0] if root.contents_of('kind_code') else []
        date = root.contents_of('document_date')[0] if root.contents_of('document_date') else []
        res['country'] = country if country else ''
        res['kind'] = kind if kind else ''
        res['date'] = date if date else ''
        res['number'] = xml_util.normalize_document_identifier(
            root.contents_of('doc_number')[0])
        return res

    @property
    def us_relation_list(self):
        """
        returns list of dictionaries for us reldoc:
        usreldoc:
          doctype
          status (parent status)
          date
          number
          kind
          country
          relationship
          sequence
        """
        root = self.xml.continuations
        if not root:
            return []
        root = root[0]
        res = []
        i = 0
        for reldoc in root.children:
            if reldoc._name == 'non_provisional_of_provisional':
                data = {'doctype': 'us_provisional_application'}
                data.update(self._get_doc_info(reldoc))
                data['date'] = self._fix_date(data['date'])
                if any(data.values()):
                    data['sequence'] = i
                    data['uuid'] = str(uuid.uuid1())
                    i = i + 1
                    res.append(data)
            for relation in reldoc.parent_child:
                for relationship in ['parent', 'child']:
                    data = {'doctype': u'relation'}
                    doc = getattr(relation, relationship)
                    if not doc:
                        continue
                    data.update(self._get_doc_info(doc[0]))
                    data['date'] = self._fix_date(data['date'])
                    data['status'] = relation.contents_of('parent_status', as_string=True)
                    data['relationship'] = relationship  # parent/child
                    if any(data.values()):
                        data['sequence'] = i
                        data['uuid'] = str(uuid.uuid1())
                        i = i + 1
                        res.append(data)
        return res

    @property
    def us_classifications(self):
        """
        Returns list of dictionaries representing us classification
        main:
          class
          subclass
        """
        classes = []
        i = 0
        main = self.xml.classification_us.classification_us_primary.uspc
        data = {'class': main.contents_of('class', as_string=True),
              'subclass': main.contents_of('subclass', as_string=True)}
        if any(data.values()):
            classes.append([
                {'uuid': str(uuid.uuid1()), 'sequence': i},
                {'id': data['class'].upper()},
                {'id': "{class}/{subclass}".format(**data).upper()}])
            i = i + 1
        if self.xml.classification_us.classification_us_secondary:
            further = self.xml.classification_us.classification_us_secondary
            for classification in further:
                data = {'class': classification.contents_of('class', as_string=True),
                        'subclass': classification.contents_of('class', as_string=True)}
                if any(data.values()):
                    classes.append([
                        {'uuid': str(uuid.uuid1()), 'sequence': i},
                        {'id': data['class'].upper()},
                        {'id': "{class}/{subclass}".format(**data).upper()}])
                    i = i + 1
        return classes

    @property
    def ipcr_classifications(self):
        """
        Returns list of dictionaries representing ipcr classifications
        ipcr:
          ipc_version_indicator
          classification_level
          section
          class
          subclass
          main_group
          subgroup
          symbol_position
          classification_value
          action_date
          classification_status
          classification_data_source
          sequence
        """
        ipcr_classifications = self.xml.classification_ipc
        if not ipcr_classifications:
            return []
        res = []
        # we can safely use [0] because there is only one ipcr_classifications tag
        for i, ipcr in enumerate(ipcr_classifications.classification_ipc_primary +
                                 ipcr_classifications.classification_ipc_secondary):
            data = {}
            data['classification_level'] = ipcr.contents_of('ipc', as_string=True)[0]
            data['class'] = ipcr.contents_of('ipc', as_string=True)[1:3]
            data['subclass'] = ipcr.contents_of('ipc', as_string=True)[3]
            data['main_group'] = ipcr.contents_of('ipc', as_string=True)[4:7]
            data['subgroup'] = ipcr.contents_of('ipc', as_string=True).split('/')[1]
            if any(data.values()):
                data['sequence'] = i
                data['uuid'] = str(uuid.uuid1())
                res.append(data)
        return res

    @property
    def claims(self):
        """
        Returns list of dictionaries representing claims
        claim:
          text
          dependent -- -1 if an independent claim, else this is the number
                       of the claim this one is dependent on
          sequence
        """
        claims = self.xml.claim
        res = []
        for i, claim in enumerate(claims):
            data = {}
            data['text'] = claim.contents_of('claim_text', as_string=True, upper=False)
            # remove leading claim num from text
            data['text'] = claim_num_regex.sub('', data['text'])
            data['sequence'] = i+1 # claims are 1-indexed
            if claim.dependent_claim_reference:
                # claim_refs are 'claim N', so we extract the N
                data['dependent'] = int(claim.dependent_claim_reference.contents_of('claim_text',\
                                        as_string=True).split(' ')[-1])
            data['uuid'] = str(uuid.uuid1())
            res.append(data)
        return res
<commit_msg>add handling for 2004 schema for dependent claims<commit_after>#!/usr/bin/env python

"""
Uses the extended ContentHandler from xml_driver to extract the needed fields
from patent grant documents
"""

from cStringIO import StringIO
from datetime import datetime
from unidecode import unidecode
from handler import Patobj, PatentHandler
import re
import uuid
import xml.sax
import xml_util
import xml_driver

claim_num_regex = re.compile(r'^\d+\. *') # removes claim number from claim text

class Patent(PatentHandler):

    def __init__(self, xml_string, is_string=False):
        xh = xml_driver.XMLHandler()
        parser = xml_driver.make_parser()

        parser.setContentHandler(xh)
        parser.setFeature(xml_driver.handler.feature_external_ges, False)
        l = xml.sax.xmlreader.Locator()
        xh.setDocumentLocator(l)
        if is_string:
            parser.parse(StringIO(xml_string))
        else:
            parser.parse(xml_string)

        self.attributes = ['app','application','assignee_list','inventor_list',
                          'us_relation_list','us_classifications','ipcr_classifications',
                          'claims']

        self.xml = xh.root.patent_application_publication

        if self.xml.foreign_priority_data:
            self.country = self.xml.foreign_priority_data.contents_of('country_code')[0]
        else:
            self.country = self.xml.contents_of('country_code')[0]
        self.application = xml_util.normalize_document_identifier(self.xml.document_id.contents_of('doc_number')[0])
        self.kind = self.xml.document_id.contents_of('kind_code')[0]
        self.pat_type = None
        self.date_app = self.xml.document_id.contents_of('document_date')[0]
        self.clm_num = len(self.xml.subdoc_claims.claim)
        self.abstract = self.xml.subdoc_abstract.contents_of('paragraph', '', as_string=True, upper=False)
        self.invention_title = self._invention_title()

        self.app = {
            "id": self.application,
            "type": self.pat_type,
            "number": self.application,
            "country": self.country,
            "date": self._fix_date(self.date_app),
            "abstract": self.abstract,
            "title": self.invention_title,
            "kind": self.kind,
            "num_claims": self.clm_num
        }
        self.app["id"] = str(self.app["date"])[:4] + "/" + self.app["number"]

    def _invention_title(self):
        original = self.xml.contents_of('title_of_invention', upper=False)[0]
        if isinstance(original, list):
            original = ''.join(original)
        return original

    def _name_helper(self, tag_root):
        """
        Returns dictionary of firstname, lastname with prefix associated
        with lastname
        """
        firstname = tag_root.contents_of('given_name', as_string=True, upper=False)
        lastname = tag_root.contents_of('family_name', as_string=True, upper=False)
        return xml_util.associate_prefix(firstname, lastname)

    def _name_helper_dict(self, tag_root):
        """
        Returns dictionary of firstname, lastname with prefix associated
        with lastname
        """
        firstname = tag_root.contents_of('given_name', as_string=True, upper=False)
        lastname = tag_root.contents_of('family_name', as_string=True, upper=False)
        firstname, lastname = xml_util.associate_prefix(firstname, lastname)
        return {'name_first': firstname, 'name_last': lastname}

    def _fix_date(self, datestring):
        """
        Converts a number representing YY/MM to a Date
        """
        if not datestring:
            return None
        elif datestring[:4] < "1900":
            return None
        # default to first of month in absence of day
        if datestring[-4:-2] == '00':
            datestring = datestring[:-4] + '01' + datestring[-2:]
        if datestring[-2:] == '00':
            datestring = datestring[:6] + '01'
        try:
            datestring = datetime.strptime(datestring, '%Y%m%d')
            return datestring
        except Exception as inst:
            print inst, datestring
            return None

    @property
    def assignee_list(self):
        """
        Returns list of dictionaries:
        assignee:
          name_last
          name_first
          residence
          nationality
          organization
          sequence
        location:
          id
          city
          state
          country
        """
        assignees = self.xml.assignee
        if not assignees:
            return []
        res = []
        for i, assignee in enumerate(assignees):
            # add assignee data
            asg = {}
            asg.update(self._name_helper_dict(assignee))  # add firstname, lastname
            asg['organization'] = assignee.contents_of('organization_name', as_string=True, upper=False)
            asg['role'] = assignee.contents_of('role', as_string=True)
            if assignee.contents_of('country_code'):
                asg['nationality'] = assignee.contents_of('country_code')[0]
                asg['residence'] = assignee.contents_of('country_code')[0]
            # add location data for assignee
            loc = {}
            for tag in ['city', 'state']:
                loc[tag] = assignee.contents_of(tag, as_string=True, upper=False)
            if 'nationality' in asg:
                loc['country'] = asg['nationality']
            #this is created because of MySQL foreign key case sensitivities
                loc['id'] = unidecode(u"|".join([loc['city'], loc['state'], loc['country']]).lower())
            else:
                loc['id'] = u''
            if any(asg.values()) or any(loc.values()):
                asg['sequence'] = i
                asg['uuid'] = str(uuid.uuid1())
                res.append([asg, loc])
        return res

    @property
    def inventor_list(self):
        """
        Returns list of lists of applicant dictionary and location dictionary
        applicant:
          name_last
          name_first
          nationality
          sequence
        location:
          id
          city
          state
          country
        """
        applicants = self.xml.first_named_inventor + self.xml.inventors.inventor
        if not applicants:
            return []
        res = []
        for i, applicant in enumerate(applicants):
            # add applicant data
            app = {}
            app.update(self._name_helper_dict(applicant.name))
            app['nationality'] = applicant.contents_of('country_code', as_string=True)
            # add location data for applicant
            loc = {}
            for tag in ['city', 'state']:
                loc[tag] = applicant.residence.contents_of(tag, as_string=True, upper=False)
            loc['country'] = app['nationality']
            #this is created because of MySQL foreign key case sensitivities
            loc['id'] = unidecode("|".join([loc['city'], loc['state'], loc['country']]).lower())
            if any(app.values()) or any(loc.values()):
                app['sequence'] = i
                app['uuid'] = str(uuid.uuid1())
                res.append([app, loc])
        return res

    def _get_doc_info(self, root):
        """
        Accepts an XMLElement root as an argument. Returns list of
        [country, doc-number, kind, date] for the given root
        """
        res = {}
        country = root.contents_of('country_code')[0] if root.contents_of('country_code') else []
        kind = root.contents_of('kind_code')[0] if root.contents_of('kind_code') else []
        date = root.contents_of('document_date')[0] if root.contents_of('document_date') else []
        res['country'] = country if country else ''
        res['kind'] = kind if kind else ''
        res['date'] = date if date else ''
        res['number'] = xml_util.normalize_document_identifier(
            root.contents_of('doc_number')[0])
        return res

    @property
    def us_relation_list(self):
        """
        returns list of dictionaries for us reldoc:
        usreldoc:
          doctype
          status (parent status)
          date
          number
          kind
          country
          relationship
          sequence
        """
        root = self.xml.continuations
        if not root:
            return []
        root = root[0]
        res = []
        i = 0
        for reldoc in root.children:
            if reldoc._name == 'non_provisional_of_provisional':
                data = {'doctype': 'us_provisional_application'}
                data.update(self._get_doc_info(reldoc))
                data['date'] = self._fix_date(data['date'])
                if any(data.values()):
                    data['sequence'] = i
                    data['uuid'] = str(uuid.uuid1())
                    i = i + 1
                    res.append(data)
            for relation in reldoc.parent_child:
                for relationship in ['parent', 'child']:
                    data = {'doctype': u'relation'}
                    doc = getattr(relation, relationship)
                    if not doc:
                        continue
                    data.update(self._get_doc_info(doc[0]))
                    data['date'] = self._fix_date(data['date'])
                    data['status'] = relation.contents_of('parent_status', as_string=True)
                    data['relationship'] = relationship  # parent/child
                    if any(data.values()):
                        data['sequence'] = i
                        data['uuid'] = str(uuid.uuid1())
                        i = i + 1
                        res.append(data)
        return res

    @property
    def us_classifications(self):
        """
        Returns list of dictionaries representing us classification
        main:
          class
          subclass
        """
        classes = []
        i = 0
        main = self.xml.classification_us.classification_us_primary.uspc
        data = {'class': main.contents_of('class', as_string=True),
              'subclass': main.contents_of('subclass', as_string=True)}
        if any(data.values()):
            classes.append([
                {'uuid': str(uuid.uuid1()), 'sequence': i},
                {'id': data['class'].upper()},
                {'id': "{class}/{subclass}".format(**data).upper()}])
            i = i + 1
        if self.xml.classification_us.classification_us_secondary:
            further = self.xml.classification_us.classification_us_secondary
            for classification in further:
                data = {'class': classification.contents_of('class', as_string=True),
                        'subclass': classification.contents_of('class', as_string=True)}
                if any(data.values()):
                    classes.append([
                        {'uuid': str(uuid.uuid1()), 'sequence': i},
                        {'id': data['class'].upper()},
                        {'id': "{class}/{subclass}".format(**data).upper()}])
                    i = i + 1
        return classes

    @property
    def ipcr_classifications(self):
        """
        Returns list of dictionaries representing ipcr classifications
        ipcr:
          ipc_version_indicator
          classification_level
          section
          class
          subclass
          main_group
          subgroup
          symbol_position
          classification_value
          action_date
          classification_status
          classification_data_source
          sequence
        """
        ipcr_classifications = self.xml.classification_ipc
        if not ipcr_classifications:
            return []
        res = []
        # we can safely use [0] because there is only one ipcr_classifications tag
        for i, ipcr in enumerate(ipcr_classifications.classification_ipc_primary +
                                 ipcr_classifications.classification_ipc_secondary):
            data = {}
            data['classification_level'] = ipcr.contents_of('ipc', as_string=True)[0]
            data['class'] = ipcr.contents_of('ipc', as_string=True)[1:3]
            data['subclass'] = ipcr.contents_of('ipc', as_string=True)[3]
            data['main_group'] = ipcr.contents_of('ipc', as_string=True)[4:7]
            data['subgroup'] = ipcr.contents_of('ipc', as_string=True).split('/')[1]
            if any(data.values()):
                data['sequence'] = i
                data['uuid'] = str(uuid.uuid1())
                res.append(data)
        return res

    @property
    def claims(self):
        """
        Returns list of dictionaries representing claims
        claim:
          text
          dependent -- -1 if an independent claim, else this is the number
                       of the claim this one is dependent on
          sequence
        """
        claims = self.xml.claim
        res = []
        for i, claim in enumerate(claims):
            data = {}
            data['text'] = claim.contents_of('claim_text', as_string=True, upper=False)
            # remove leading claim num from text
            data['text'] = claim_num_regex.sub('', data['text'])
            data['sequence'] = i+1 # claims are 1-indexed
            if claim.dependent_claim_reference and claim.dependent_claim_reference.claim_text:
                # claim_refs are 'claim N', so we extract the N
                data['dependent'] = int(claim.dependent_claim_reference.contents_of('claim_text',\
                                        as_string=True).split(' ')[-1])
            elif claim.dependent_claim_reference:
                data['dependent'] = int(claim.contents_of('dependent_claim_reference',\
                                        as_string=True).split(' ')[-1])
            data['uuid'] = str(uuid.uuid1())
            res.append(data)
        return res
<|endoftext|>


<commit_before>#
# JavaScript Tools - Optimizer for combining variable declarations
# Copyright 2010 Sebastian Werner
#

#
# Public API
#

def optimize(node):
    if not node.type in ("script", "block"):
        return

    for child in node:
        print child.type

    <commit_msg>Initial working version of declarations combiner<commit_after>#
# JavaScript Tools - Optimizer for combining variable declarations
# Copyright 2010 Sebastian Werner
#

#
# Public API
#

def optimize(node):
    if not node.type in ("script", "block"):
        return

    # Backwards processing and insertion into previous sibling
    length = len(node)
    pos = length-1
    while pos > 0:
        child = node[pos]
        prevChild = node[pos-1]

        if child.type == "var" and prevChild.type == "var":
            for variable in child:
                prevChild.append(variable)
            node.remove(child)

        pos -= 1

    <|endoftext|>


<commit_before># This file is Copyright (c) 2019 Tom Keddie <git@bronwenandtom.com>
# This file is Copyright (c) 2020 Antmicro <www.antmicro.com>
# This file is Copyright (c) 2020 Florent Kermarrec <florent@enjoy-digital.fr>
# License: BSD

# The Pano Logic Zero Client G2 is a thin commercial client from Pano Logic that can be repurposed
# as a generic development board thanks to reverse engineering efforts than can be found at:
# https://github.com/tomverbeure/panologic-g2

from litex.build.generic_platform import *
from litex.build.xilinx import XilinxPlatform
from litex.build.openocd import OpenOCD

# IOs ----------------------------------------------------------------------------------------------

_io = [
    # clock / reset
    ("clk125",    0, Pins("Y13"),  IOStandard("LVCMOS33")),
    ("rst_n",     0, Pins("AB14"), IOStandard("LVCMOS33")),

    # led
    ("user_led", 0, Pins("E12"),  IOStandard("LVCMOS33")),
    ("user_led", 1, Pins("H13"),  IOStandard("LVCMOS33")),
    ("user_led", 2, Pins("F13"),  IOStandard("LVCMOS33")),

    # btn
    ("user_btn_n", 0, Pins("H12"), IOStandard("LVCMOS33")),

    # serial
    ("serial", 0, # hdmi
        Subsignal("tx", Pins("AB19")),
        Subsignal("rx", Pins("AA21")),
        IOStandard("LVCMOS33")
    ),

    # serial
    ("serial", 1, # dvi
        Subsignal("tx", Pins("C14")),
        Subsignal("rx", Pins("C17")),
        IOStandard("LVCMOS33")
    ),

    # spi flash
    ("spiflash", 0,
        Subsignal("cs_n", Pins("T5"),   IOStandard("LVCMOS33")),
        Subsignal("clk",  Pins("Y21"),  IOStandard("LVCMOS33")),
        Subsignal("mosi", Pins("AB20"), IOStandard("LVCMOS33")),
        Subsignal("miso", Pins("AA20"), IOStandard("LVCMOS33"))
    ),

    # ddram
    ("ddram_clock_a", 0,
        Subsignal("p", Pins("H20")),
        Subsignal("n", Pins("J19")),
        IOStandard("DIFF_SSTL18_II"), Misc("IN_TERM=NONE")
    ),
    ("ddram_a", 0,
        Subsignal("a", Pins(
            "F21 F22 E22 G20 F20 K20 K19 E20",
            "C20 C22 G19 F19 D22"),
            IOStandard("SSTL18_II")),
        Subsignal("ba",    Pins("J17 K17 H18"), IOStandard("SSTL18_II")),
        Subsignal("ras_n", Pins("H21"), IOStandard("SSTL18_II")),
        Subsignal("cas_n", Pins("H22"), IOStandard("SSTL18_II")),
        Subsignal("we_n",  Pins("H19"), IOStandard("SSTL18_II")),
        Subsignal("dm",    Pins("M20 L19"), IOStandard("SSTL18_II")),
        Subsignal("dq",    Pins(
            "N20 N22 M21 M22 J20 J22 K21 K22",
            "P21 P22 R20 R22 U20 U22 V21 V22"),
            IOStandard("SSTL18_II")),
        Subsignal("dqs",   Pins("T21 L20"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("dqs_n", Pins("T22 L22"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("cke", Pins("D21"), IOStandard("SSTL18_II")),
        Subsignal("odt", Pins("G22"), IOStandard("SSTL18_II")),
    ),

    ("ddram_clock_b", 0,
        Subsignal("p", Pins("H4")),
        Subsignal("n", Pins("H3")),
        IOStandard("DIFF_SSTL18_II"), Misc("IN_TERM=NONE")
    ),
    ("ddram_b", 0,
        Subsignal("a", Pins(
            "H2 H1 H5 K6 F3 K3 J4 H6",
            "E3 E1 G4 C1 D1"),
            IOStandard("SSTL18_II")),
        Subsignal("ba",    Pins("G3 G1 F1"), IOStandard("SSTL18_II")),
        Subsignal("ras_n", Pins("K5"), IOStandard("SSTL18_II")),
        Subsignal("cas_n", Pins("K4"), IOStandard("SSTL18_II")),
        Subsignal("we_n",  Pins("F2"), IOStandard("SSTL18_II")),
        Subsignal("dm",    Pins("M3 L4"), IOStandard("SSTL18_II")),
        Subsignal("dq",    Pins(
            "N3 N1 M2 M1 J3 J1 K2 K1",
            "P2 P1 R3 R1 U3 U1 V2 V1"),
            IOStandard("SSTL18_II")),
        Subsignal("dqs",   Pins("T2 L3"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("dqs_n", Pins("T1 L1"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("cke", Pins("D2"), IOStandard("SSTL18_II")),
        Subsignal("odt", Pins("J6"), IOStandard("SSTL18_II")),
    ),

    # ethernet
    ("eth_rst_n",  0, Pins("R11"), IOStandard("LVCMOS33")),
    ("eth_clocks", 0,
        Subsignal("tx",  Pins("Y11")),
        Subsignal("gtx", Pins("AA12")),
        Subsignal("rx",  Pins("AB11")),
        IOStandard("LVCMOS33")
    ),
    ("eth", 0,
        Subsignal("rst_n",   Pins("R11")),
        Subsignal("int_n",   Pins("AA4")),
        Subsignal("mdio",    Pins("AA2")),
        Subsignal("mdc",     Pins("AB6")),
        Subsignal("rx_dv",   Pins("Y7")),
        Subsignal("rx_er",   Pins("Y8")),
        Subsignal("rx_data", Pins("Y3 Y4 R9 R7 V9 R8 U9 Y9")),
        Subsignal("tx_en",   Pins("AA8")),
        Subsignal("tx_er",   Pins("AB8")),
        Subsignal("tx_data", Pins("AB2 AB3 AB4 AB7 AB9 AB10 T7 Y10")),
        Subsignal("col",     Pins("V7")),
        Subsignal("crs",     Pins("W4")),
        IOStandard("LVCMOS33")
    ),

]

# Platform -----------------------------------------------------------------------------------------

class Platform(XilinxPlatform):
    default_clk_name   = "clk125"
    default_clk_period = 1e9/125e6

    def __init__(self, revision="c"):
        assert revision in ["b", "c"]
        device = {"b": "xc6slx150-2-fgg484", "c": "xc6slx100-2-fgg484"}[revision]
        XilinxPlatform.__init__(self, device, _io)
        self.add_platform_command("""CONFIG VCCAUX="2.5";""")
        self.add_period_constraint(self.lookup_request("clk125", loose=True), 1e9/125e6)

    def create_programmer(self):
        return OpenOCD("openocd_xc6_ft232.cfg")
<commit_msg>platforms/pano_logic_g2: -x.<commit_after># This file is Copyright (c) 2019 Tom Keddie <git@bronwenandtom.com>
# This file is Copyright (c) 2020 Antmicro <www.antmicro.com>
# This file is Copyright (c) 2020 Florent Kermarrec <florent@enjoy-digital.fr>
# License: BSD

# The Pano Logic Zero Client G2 is a thin commercial client from Pano Logic that can be repurposed
# as a generic development board thanks to reverse engineering efforts than can be found at:
# https://github.com/tomverbeure/panologic-g2

from litex.build.generic_platform import *
from litex.build.xilinx import XilinxPlatform
from litex.build.openocd import OpenOCD

# IOs ----------------------------------------------------------------------------------------------

_io = [
    # clock / reset
    ("clk125",    0, Pins("Y13"),  IOStandard("LVCMOS33")),
    ("rst_n",     0, Pins("AB14"), IOStandard("LVCMOS33")),

    # led
    ("user_led", 0, Pins("E12"),  IOStandard("LVCMOS33")),
    ("user_led", 1, Pins("H13"),  IOStandard("LVCMOS33")),
    ("user_led", 2, Pins("F13"),  IOStandard("LVCMOS33")),

    # btn
    ("user_btn_n", 0, Pins("H12"), IOStandard("LVCMOS33")),

    # serial
    ("serial", 0, # hdmi
        Subsignal("tx", Pins("AB19")),
        Subsignal("rx", Pins("AA21")),
        IOStandard("LVCMOS33")
    ),

    # serial
    ("serial", 1, # dvi
        Subsignal("tx", Pins("C14")),
        Subsignal("rx", Pins("C17")),
        IOStandard("LVCMOS33")
    ),

    # spi flash
    ("spiflash", 0,
        Subsignal("cs_n", Pins("T5"),   IOStandard("LVCMOS33")),
        Subsignal("clk",  Pins("Y21"),  IOStandard("LVCMOS33")),
        Subsignal("mosi", Pins("AB20"), IOStandard("LVCMOS33")),
        Subsignal("miso", Pins("AA20"), IOStandard("LVCMOS33"))
    ),

    # ddram
    ("ddram_clock_a", 0,
        Subsignal("p", Pins("H20")),
        Subsignal("n", Pins("J19")),
        IOStandard("DIFF_SSTL18_II"), Misc("IN_TERM=NONE")
    ),
    ("ddram_a", 0,
        Subsignal("a", Pins(
            "F21 F22 E22 G20 F20 K20 K19 E20",
            "C20 C22 G19 F19 D22"),
            IOStandard("SSTL18_II")),
        Subsignal("ba",    Pins("J17 K17 H18"), IOStandard("SSTL18_II")),
        Subsignal("ras_n", Pins("H21"), IOStandard("SSTL18_II")),
        Subsignal("cas_n", Pins("H22"), IOStandard("SSTL18_II")),
        Subsignal("we_n",  Pins("H19"), IOStandard("SSTL18_II")),
        Subsignal("dm",    Pins("M20 L19"), IOStandard("SSTL18_II")),
        Subsignal("dq",    Pins(
            "N20 N22 M21 M22 J20 J22 K21 K22",
            "P21 P22 R20 R22 U20 U22 V21 V22"),
            IOStandard("SSTL18_II")),
        Subsignal("dqs",   Pins("T21 L20"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("dqs_n", Pins("T22 L22"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("cke", Pins("D21"), IOStandard("SSTL18_II")),
        Subsignal("odt", Pins("G22"), IOStandard("SSTL18_II")),
    ),

    ("ddram_clock_b", 0,
        Subsignal("p", Pins("H4")),
        Subsignal("n", Pins("H3")),
        IOStandard("DIFF_SSTL18_II"), Misc("IN_TERM=NONE")
    ),
    ("ddram_b", 0,
        Subsignal("a", Pins(
            "H2 H1 H5 K6 F3 K3 J4 H6",
            "E3 E1 G4 C1 D1"),
            IOStandard("SSTL18_II")),
        Subsignal("ba",    Pins("G3 G1 F1"), IOStandard("SSTL18_II")),
        Subsignal("ras_n", Pins("K5"), IOStandard("SSTL18_II")),
        Subsignal("cas_n", Pins("K4"), IOStandard("SSTL18_II")),
        Subsignal("we_n",  Pins("F2"), IOStandard("SSTL18_II")),
        Subsignal("dm",    Pins("M3 L4"), IOStandard("SSTL18_II")),
        Subsignal("dq",    Pins(
            "N3 N1 M2 M1 J3 J1 K2 K1",
            "P2 P1 R3 R1 U3 U1 V2 V1"),
            IOStandard("SSTL18_II")),
        Subsignal("dqs",   Pins("T2 L3"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("dqs_n", Pins("T1 L1"), IOStandard("DIFF_SSTL18_II")),
        Subsignal("cke", Pins("D2"), IOStandard("SSTL18_II")),
        Subsignal("odt", Pins("J6"), IOStandard("SSTL18_II")),
    ),

    # ethernet
    ("eth_rst_n",  0, Pins("R11"), IOStandard("LVCMOS33")),
    ("eth_clocks", 0,
        Subsignal("tx",  Pins("Y11")),
        Subsignal("gtx", Pins("AA12")),
        Subsignal("rx",  Pins("AB11")),
        IOStandard("LVCMOS33")
    ),
    ("eth", 0,
        Subsignal("rst_n",   Pins("R11")),
        Subsignal("int_n",   Pins("AA4")),
        Subsignal("mdio",    Pins("AA2")),
        Subsignal("mdc",     Pins("AB6")),
        Subsignal("rx_dv",   Pins("Y7")),
        Subsignal("rx_er",   Pins("Y8")),
        Subsignal("rx_data", Pins("Y3 Y4 R9 R7 V9 R8 U9 Y9")),
        Subsignal("tx_en",   Pins("AA8")),
        Subsignal("tx_er",   Pins("AB8")),
        Subsignal("tx_data", Pins("AB2 AB3 AB4 AB7 AB9 AB10 T7 Y10")),
        Subsignal("col",     Pins("V7")),
        Subsignal("crs",     Pins("W4")),
        IOStandard("LVCMOS33")
    ),

]

# Platform -----------------------------------------------------------------------------------------

class Platform(XilinxPlatform):
    default_clk_name   = "clk125"
    default_clk_period = 1e9/125e6

    def __init__(self, revision="c"):
        assert revision in ["b", "c"]
        device = {"b": "xc6slx150-2-fgg484", "c": "xc6slx100-2-fgg484"}[revision]
        XilinxPlatform.__init__(self, device, _io)
        self.add_platform_command("""CONFIG VCCAUX="2.5";""")
        self.add_period_constraint(self.lookup_request("clk125", loose=True), 1e9/125e6)

    def create_programmer(self):
        return OpenOCD("openocd_xc6_ft232.cfg")
<|endoftext|>