<commit_before>from __future__ import absolute_import, division, print_function

import pytest

pymysql = pytest.importorskip('pymysql')

from datashape import var, DataShape, Record
import itertools
from odo.backends.csv import CSV
from odo import resource, odo
import sqlalchemy
import sqlalchemy as sa
import os
import sys
import csv as csv_module
import getpass
from odo import drop, discover
from odo.utils import tmpfile

pytestmark = pytest.mark.skipif(sys.platform == 'win32',
                                reason='not well tested on win32 mysql')

username = getpass.getuser()
url = 'mysql+pymysql://{0}@localhost:3306/test'.format(username)

def create_csv(data, file_name):
    with open(file_name, 'w') as f:
        csv_writer = csv_module.writer(f)
        for row in data:
            csv_writer.writerow(row)

data = [(1, 2), (10, 20), (100, 200)]
data_floats = [(1.02, 2.02), (102.02, 202.02), (1002.02, 2002.02)]

@pytest.yield_fixture
def csv():
    with tmpfile('.csv') as fn:
        create_csv(data, fn)
        yield CSV(fn)

@pytest.yield_fixture
def fcsv():
    with tmpfile('.csv') as fn:
        create_csv(data_floats, fn)
        yield CSV(fn, columns=list('ab'))

names = ('tbl%d' % i for i in itertools.count())

@pytest.fixture
def name():
    return next(names)

@pytest.fixture(scope='module')
def engine():
    return sqlalchemy.create_engine(url)

@pytest.yield_fixture
def sql(engine, csv, name):
    dshape = discover(csv)
    dshape = DataShape(var,
                       Record([(n, typ)
                               for n, typ in zip('ab', dshape.measure.types)]))
    try:
        t = resource('%s::%s' % (url, name), dshape=dshape)
    except sqlalchemy.exc.OperationalError as e:
        pytest.skip(str(e))
    else:
        yield t
        drop(t)

@pytest.yield_fixture
def fsql(engine, fcsv, name):
    dshape = discover(fcsv)
    dshape = DataShape(var,
                       Record([(n, typ)
                               for n, typ in zip('ab', dshape.measure.types)]))
    try:
        t = resource('%s::%s' % (url, name), dshape=dshape)
    except sqlalchemy.exc.OperationalError as e:
        pytest.skip(str(e))
    else:
        yield t
        drop(t)

@pytest.fixture
def dcsv():
    this_dir = os.path.dirname(__file__)
    file_name = os.path.join(this_dir, 'dummydata.csv')
    dshape = """var * {
        Name: string,
        RegistrationDate: date,
        ZipCode: int64,
        Consts: float64
    }"""

    return CSV(file_name, dshape=dshape)

@pytest.yield_fixture
def dsql(engine, dcsv, name):
    t = resource('%s::%s' % (url, name), dshape=discover(dcsv))
    yield t
    drop(t)

def test_csv_mysql_load(sql, csv):
    engine = sql.bind
    conn = engine.raw_connection()

    cursor = conn.cursor()
    full_path = os.path.abspath(csv.path)
    load = '''LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY ','
        lines terminated by '\n'
        '''.format(full_path, sql.name)
    cursor.execute(load)
    conn.commit()

def test_simple_into(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

def test_append(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

    odo(csv, sql, if_exists="append")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200),
                              (1, 2), (10, 20), (100, 200)]

def test_simple_float_into(fsql, fcsv):
    odo(fcsv, fsql, if_exists="replace")

    assert odo(fsql, list) == [(1.02, 2.02),
                               (102.02, 202.02),
                               (1002.02, 2002.02)]

def test_tryexcept_into(sql, csv):
    # uses multi-byte character and fails over to using sql.extend()
    odo(csv, sql, if_exists="replace", QUOTE="alpha", FORMAT="csv")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

@pytest.mark.xfail(raises=KeyError)
def test_failing_argument(sql, csv):
    odo(csv, sql, if_exists="replace", skipinitialspace="alpha")

def test_no_header_no_columns(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

@pytest.mark.xfail
def test_complex_into(dsql, dcsv):
    # data from: http://dummydata.me/generate
    odo(dcsv, dsql, if_exists="replace")
    assert odo(dsql, list) == odo(dcsv, list)

def test_sql_to_csv(sql, csv):
    sql = odo(csv, sql)
    with tmpfile('.csv') as fn:
        csv = odo(sql, fn)
        assert odo(csv, list) == data
        assert discover(csv).measure.names == discover(sql).measure.names

def test_sql_select_to_csv(sql, csv):
    sql = odo(csv, sql)
    query = sa.select([sql.c.a])
    with tmpfile('.csv') as fn:
        csv = odo(query, fn)
        assert odo(csv, list) == [(x,) for x, _ in data]

def test_csv_output_is_not_quoted_by_default(sql, csv):
    sql = odo(csv, sql)
    expected = "a,b\n1,2\n10,20\n100,200\n"
    with tmpfile('.csv') as fn:
        csv = odo(sql, fn)
        with open(fn, 'rt') as f:
            result = f.read()
        assert result == expected
<commit_msg>Test that we do NOT preserve the header for mysql -> CSV<commit_after>from __future__ import absolute_import, division, print_function

import pytest

pymysql = pytest.importorskip('pymysql')

from datashape import var, DataShape, Record
import itertools
from odo.backends.csv import CSV
from odo import resource, odo
import sqlalchemy
import sqlalchemy as sa
import os
import sys
import csv as csv_module
import getpass
from odo import drop, discover
from odo.utils import tmpfile

pytestmark = pytest.mark.skipif(sys.platform == 'win32',
                                reason='not well tested on win32 mysql')

username = getpass.getuser()
url = 'mysql+pymysql://{0}@localhost:3306/test'.format(username)

def create_csv(data, file_name):
    with open(file_name, 'w') as f:
        csv_writer = csv_module.writer(f)
        for row in data:
            csv_writer.writerow(row)

data = [(1, 2), (10, 20), (100, 200)]
data_floats = [(1.02, 2.02), (102.02, 202.02), (1002.02, 2002.02)]

@pytest.yield_fixture
def csv():
    with tmpfile('.csv') as fn:
        create_csv(data, fn)
        yield CSV(fn)

@pytest.yield_fixture
def fcsv():
    with tmpfile('.csv') as fn:
        create_csv(data_floats, fn)
        yield CSV(fn, columns=list('ab'))

names = ('tbl%d' % i for i in itertools.count())

@pytest.fixture
def name():
    return next(names)

@pytest.fixture(scope='module')
def engine():
    return sqlalchemy.create_engine(url)

@pytest.yield_fixture
def sql(engine, csv, name):
    dshape = discover(csv)
    dshape = DataShape(var,
                       Record([(n, typ)
                               for n, typ in zip('ab', dshape.measure.types)]))
    try:
        t = resource('%s::%s' % (url, name), dshape=dshape)
    except sqlalchemy.exc.OperationalError as e:
        pytest.skip(str(e))
    else:
        yield t
        drop(t)

@pytest.yield_fixture
def fsql(engine, fcsv, name):
    dshape = discover(fcsv)
    dshape = DataShape(var,
                       Record([(n, typ)
                               for n, typ in zip('ab', dshape.measure.types)]))
    try:
        t = resource('%s::%s' % (url, name), dshape=dshape)
    except sqlalchemy.exc.OperationalError as e:
        pytest.skip(str(e))
    else:
        yield t
        drop(t)

@pytest.fixture
def dcsv():
    this_dir = os.path.dirname(__file__)
    file_name = os.path.join(this_dir, 'dummydata.csv')
    dshape = """var * {
        Name: string,
        RegistrationDate: date,
        ZipCode: int64,
        Consts: float64
    }"""

    return CSV(file_name, dshape=dshape)

@pytest.yield_fixture
def dsql(engine, dcsv, name):
    t = resource('%s::%s' % (url, name), dshape=discover(dcsv))
    yield t
    drop(t)

def test_csv_mysql_load(sql, csv):
    engine = sql.bind
    conn = engine.raw_connection()

    cursor = conn.cursor()
    full_path = os.path.abspath(csv.path)
    load = '''LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY ','
        lines terminated by '\n'
        '''.format(full_path, sql.name)
    cursor.execute(load)
    conn.commit()

def test_simple_into(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

def test_append(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

    odo(csv, sql, if_exists="append")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200),
                              (1, 2), (10, 20), (100, 200)]

def test_simple_float_into(fsql, fcsv):
    odo(fcsv, fsql, if_exists="replace")

    assert odo(fsql, list) == [(1.02, 2.02),
                               (102.02, 202.02),
                               (1002.02, 2002.02)]

def test_tryexcept_into(sql, csv):
    # uses multi-byte character and fails over to using sql.extend()
    odo(csv, sql, if_exists="replace", QUOTE="alpha", FORMAT="csv")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

@pytest.mark.xfail(raises=KeyError)
def test_failing_argument(sql, csv):
    odo(csv, sql, if_exists="replace", skipinitialspace="alpha")

def test_no_header_no_columns(sql, csv):
    odo(csv, sql, if_exists="replace")
    assert odo(sql, list) == [(1, 2), (10, 20), (100, 200)]

@pytest.mark.xfail
def test_complex_into(dsql, dcsv):
    # data from: http://dummydata.me/generate
    odo(dcsv, dsql, if_exists="replace")
    assert odo(dsql, list) == odo(dcsv, list)

def test_sql_to_csv(sql, csv):
    sql = odo(csv, sql)
    with tmpfile('.csv') as fn:
        csv = odo(sql, fn)
        assert odo(csv, list) == data

        # explicitly test that we do NOT preserve the header here
        assert discover(csv).measure.names != discover(sql).measure.names

def test_sql_select_to_csv(sql, csv):
    sql = odo(csv, sql)
    query = sa.select([sql.c.a])
    with tmpfile('.csv') as fn:
        csv = odo(query, fn)
        assert odo(csv, list) == [(x,) for x, _ in data]

def test_csv_output_does_not_preserve_header(sql, csv):
    sql = odo(csv, sql)
    expected = "1,2\n10,20\n100,200\n"
    with tmpfile('.csv') as fn:
        csv = odo(sql, fn)
        with open(csv.path, 'rt') as f:
            result = f.read()
    assert result == expected
<|endoftext|>


<commit_before>import inspect

from java.awt import GridBagLayout
from java.awt import GridBagConstraints
from javax.swing import JPanel
from javax.swing import JLabel
from javax.swing import JTextField
from javax.swing import JButton
from javax.swing import JFileChooser
from javax.swing.filechooser import FileNameExtensionFilter

from java.util.logging import Level

from org.sleuthkit.autopsy.ingest import IngestModuleFactoryAdapter
from org.sleuthkit.autopsy.ingest import IngestModuleIngestJobSettingsPanel
from org.sleuthkit.autopsy.ingest import IngestModuleIngestJobSettings
from org.sleuthkit.autopsy.ingest import DataSourceIngestModule
from org.sleuthkit.autopsy.coreutils import Logger
from org.sleuthkit.autopsy.ingest import IngestMessage
from org.sleuthkit.autopsy.ingest import IngestServices

class VolatilityIngestModuleFactory(IngestModuleFactoryAdapter):
    def __init__(self):
        self.settings = None
        self.moduleName = "Volatility Processor"

    def getModuleDisplayName(self):
        return self.moduleName

    def getModuleDescription(self):
        return "Run Volatility against a Memory Image"

    def getModuleVersionNumber(self):
        return "1.0"

    def getDefaultIngestJobSettings(self):
        return VolatilityIngestModuleSettings()

    def hasIngestJobSettingsPanel(self):
        return True

    def getIngestJobSettingsPanel(self, settings):
        if not isinstance(settings, VolatilityIngestModuleSettings):
            raise IllegalArgumentException("Settings expected to be instnce of SampleIngestModuleSettings")
        self.settings = settings

        return VolatilityIngestModuleUISettingsPanel(self.settings)

    def isDataSourceIngestModule(self):
        return True

    def createDataSourceIngestModule(self, ingestOptions):
        return VolatilityIngestModule(self.settings)

class VolatilityIngestModuleUISettingsPanel(IngestModuleIngestJobSettingsPanel):
    def __init__(self, settings):
        self.localSettings = settings
        self.initLayout()

    def findDir(self, event):
        fileChooser = JFileChooser()
        fileExtentionFilter = FileNameExtensionFilter("Executable Files (*.exe)", ["exe"])
        fileChooser.addChoosableFileFilter(fileExtentionFilter)

        result = fileChooser.showDialog(self.mainPanel, "Select File")

        if result == JFileChooser.APPROVE_OPTION:
            file = fileChooser.getSelectedFile()
            canonicalPath = file.getCanonicalPath()

            self.localSettings.setVolatilityDir(canonicalPath)
            self.volatilityDirTextField.setText(canonicalPath)
            message = IngestMessage.createMessage(IngestMessage.MessageType.INFO, "Volatility Processor",
                                                  "Volatiity Executable Found",
                                                  "Volatity executable at " + canonicalPath)
        IngestServices.getInstance().postMessage(message)

    def getSettings(self):
        return self.localSettings

    def initLayout(self):
        self.mainPanel = JPanel()

        self.gridBagPanel = GridBagLayout()
        self.gridBagConstraints = GridBagConstraints()
        self.mainPanel.setLayout(self.gridBagPanel)

        # Volatility Executable Path
        self.dirLabel = JLabel("Volatility Executable Directory")
        self.dirLabel.setEnabled(True)
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 1
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.dirLabel, self.gridBagConstraints)
        self.mainPanel.add(self.dirLabel)

        self.volatilityDirTextField = JTextField(10)
        self.volatilityDirTextField.setEnabled(True)
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 3
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.volatilityDirTextField, self.gridBagConstraints)
        self.mainPanel.add(self.volatilityDirTextField)

        self.FindVolatilityPathButton = JButton("Find Dir", actionPerformed=self.findDir)
        self.FindVolatilityPathButton.setEnabled(True)
        # self.rbgmainPanel.add(self.FindVolatilityPathButton)
        self.gridBagConstraints.gridx = 6
        self.gridBagConstraints.gridy = 3
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.FindVolatilityPathButton, self.gridBagConstraints)
        self.mainPanel.add(self.FindVolatilityPathButton)

        self.add(self.mainPanel)

class VolatilityIngestModule(DataSourceIngestModule):
    def __init__(self, settings):
        self.context = None
        self.localSettings = settings
        self.databaseFile = ""
        self.isAutodetect = False
        self.AdditionalParams = ""
        self.PythonProgram = False
        # self.logger = Logger.getLogger(VolatilityIngestModuleFactory.moduleName)
        # self.setupLogger()

    def setupLogger(self):
        self.logger.setLogDirectory("ModuleLogs")

    def log(self, level, message):
        self.logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], message)

    def startUp(self, context):
        self.context = context

        self.log(Level.INFO, "Volatility Module Loaded")

    def process(self, dataSource, progressBar):
        progressBar.switchToIndeterminate()

class VolatilityIngestModuleSettings(IngestModuleIngestJobSettings):
    def __init__(self):
        self.versionUID = 1L
        self.VolatilityDir = ""
        self.Version = "2.5"
        self.Profile = "Autodetect"

    # Getters and setters
    def getVersionUID(self):
        return self.versionUID

    def getVolatilityDir(self):
        return self.VolatilityDir

    def getVersion(self):
        return self.Version

    def getProfile(self):
        return self.Profile

    def setVolatilityDir(self, dir):
        self.VolatilityDir = dir

    def setVersion(self, version):
        self.Version = version

    def setProfile(self, profile):
        self.Profile = profile
<commit_msg>Add rest of panel components<commit_after>import inspect

from java.awt import GridBagLayout
from java.awt import GridBagConstraints
from javax.swing import JPanel
from javax.swing import JLabel
from javax.swing import JTextField
from javax.swing import JButton
from javax.swing import JFileChooser
from javax.swing import JComboBox
from javax.swing.filechooser import FileNameExtensionFilter

from java.util.logging import Level

from org.sleuthkit.autopsy.ingest import IngestModuleFactoryAdapter
from org.sleuthkit.autopsy.ingest import IngestModuleIngestJobSettingsPanel
from org.sleuthkit.autopsy.ingest import IngestModuleIngestJobSettings
from org.sleuthkit.autopsy.ingest import DataSourceIngestModule
from org.sleuthkit.autopsy.coreutils import Logger
from org.sleuthkit.autopsy.ingest import IngestMessage
from org.sleuthkit.autopsy.ingest import IngestServices

class VolatilityIngestModuleFactory(IngestModuleFactoryAdapter):
    def __init__(self):
        self.settings = None
        self.moduleName = "Volatility Processor"

    def getModuleDisplayName(self):
        return self.moduleName

    def getModuleDescription(self):
        return "Run Volatility against a Memory Image"

    def getModuleVersionNumber(self):
        return "1.0"

    def getDefaultIngestJobSettings(self):
        return VolatilityIngestModuleSettings()

    def hasIngestJobSettingsPanel(self):
        return True

    def getIngestJobSettingsPanel(self, settings):
        if not isinstance(settings, VolatilityIngestModuleSettings):
            raise IllegalArgumentException("Settings expected to be instnce of SampleIngestModuleSettings")
        self.settings = settings

        return VolatilityIngestModuleUISettingsPanel(self.settings)

    def isDataSourceIngestModule(self):
        return True

    def createDataSourceIngestModule(self, ingestOptions):
        return VolatilityIngestModule(self.settings)

class VolatilityIngestModuleUISettingsPanel(IngestModuleIngestJobSettingsPanel):
    def __init__(self, settings):
        self.localSettings = settings
        self.initLayout()

    def findDir(self, event):
        fileChooser = JFileChooser()
        fileExtentionFilter = FileNameExtensionFilter("Executable Files (*.exe)", ["exe"])
        fileChooser.addChoosableFileFilter(fileExtentionFilter)

        result = fileChooser.showDialog(self.mainPanel, "Select File")

        if result == JFileChooser.APPROVE_OPTION:
            file = fileChooser.getSelectedFile()
            canonicalPath = file.getCanonicalPath()

            self.localSettings.setVolatilityDir(canonicalPath)
            self.volatilityDirTextField.setText(canonicalPath)
            message = IngestMessage.createMessage(IngestMessage.MessageType.INFO, "Volatility Processor",
                                                  "Volatiity Executable Found",
                                                  "Volatity executable at " + canonicalPath)
        IngestServices.getInstance().postMessage(message)

    def saveExecDir(self, event):
        rand = 0

    def getProfiles(self):
        return []

    def changeVersion(self):
        rand = 0

    def changeProfile(self):
        reand = 0

    def getSettings(self):
        return self.localSettings

    def initLayout(self):
        self.mainPanel = JPanel()

        self.gridBagPanel = GridBagLayout()
        self.gridBagConstraints = GridBagConstraints()
        self.mainPanel.setLayout(self.gridBagPanel)

        # Volatility Executable Path
        self.dirLabel = JLabel("Volatility Executable Directory")
        self.dirLabel.setEnabled(True)
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 1
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.dirLabel, self.gridBagConstraints)
        self.mainPanel.add(self.dirLabel)

        self.volatilityDirTextField = JTextField(10)
        self.volatilityDirTextField.setEnabled(True)
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 3
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.volatilityDirTextField, self.gridBagConstraints)
        self.mainPanel.add(self.volatilityDirTextField)

        self.findVolatilityPathButton = JButton("Find Dir", actionPerformed=self.findDir)
        self.findVolatilityPathButton.setEnabled(True)
        self.gridBagConstraints.gridx = 6
        self.gridBagConstraints.gridy = 3
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.findVolatilityPathButton, self.gridBagConstraints)
        self.mainPanel.add(self.findVolatilityPathButton)

        # Save dir button
        self.saveExecButton = JButton("Save Volatility Exec Dir", actionPerformed=self.saveExecDir)
        self.saveExecButton.setEnabled(True)
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 7
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.saveExecButton, self.gridBagConstraints)
        self.mainPanel.add(self.saveExecButton)

        # Version selector
        self.versionLabel = JLabel("Version:")
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 11
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.versionLabel, self.gridBagConstraints)
        self.mainPanel.add(self.versionLabel)

        self.versionList = ("2.5", "2.6")
        self.versionComboBox = JComboBox(self.versionList)
        self.versionComboBox.itemStateChanged = self.changeVersion
        self.gridBagConstraints.gridx = 6
        self.gridBagConstraints.gridy = 11
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.versionComboBox, self.gridBagConstraints)
        self.mainPanel.add(self.versionComboBox)

        # Profile selector
        self.profileLabel = JLabel("Profile:")
        self.gridBagConstraints.gridx = 2
        self.gridBagConstraints.gridy = 19
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 0
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.profileLabel, self.gridBagConstraints)
        self.mainPanel.add(self.profileLabel)

        self.profileList = self.getProfiles
        self.profileComboBox = JComboBox(self.profileList)
        self.profileComboBox.itemStateChanged = self.changeProfile
        self.gridBagConstraints.gridx = 6
        self.gridBagConstraints.gridy = 19
        self.gridBagConstraints.gridwidth = 1
        self.gridBagConstraints.gridheight = 1
        self.gridBagConstraints.fill = GridBagConstraints.BOTH
        self.gridBagConstraints.weightx = 1
        self.gridBagConstraints.weighty = 1
        self.gridBagConstraints.anchor = GridBagConstraints.NORTH
        self.gridBagPanel.setConstraints(self.profileComboBox, self.gridBagConstraints)
        self.mainPanel.add(self.profileComboBox)

        self.add(self.mainPanel)

class VolatilityIngestModule(DataSourceIngestModule):
    def __init__(self, settings):
        self.context = None
        self.localSettings = settings
        self.databaseFile = ""
        self.isAutodetect = False
        self.AdditionalParams = ""
        self.PythonProgram = False
        # self.logger = Logger.getLogger(VolatilityIngestModuleFactory.moduleName)
        # self.setupLogger()

    def setupLogger(self):
        self.logger.setLogDirectory("ModuleLogs")

    def log(self, level, message):
        self.logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], message)

    def startUp(self, context):
        self.context = context

        self.log(Level.INFO, "Volatility Module Loaded")

    def process(self, dataSource, progressBar):
        progressBar.switchToIndeterminate()

class VolatilityIngestModuleSettings(IngestModuleIngestJobSettings):
    def __init__(self):
        self.versionUID = 1L
        self.VolatilityDir = ""
        self.Version = "2.5"
        self.Profile = "Autodetect"

    # Getters and setters
    def getVersionUID(self):
        return self.versionUID

    def getVolatilityDir(self):
        return self.VolatilityDir

    def getVersion(self):
        return self.Version

    def getProfile(self):
        return self.Profile

    def setVolatilityDir(self, dir):
        self.VolatilityDir = dir

    def setVersion(self, version):
        self.Version = version

    def setProfile(self, profile):
        self.Profile = profile
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

from csv import DictReader
from urllib.parse import urlparse, urldefrag
from dspl2.jsonutil import (AsList, GetSchemaId, GetSchemaProp, GetUrl,
                            MakeIdKeyedDict)
from dspl2.rdfutil import (_DataFileFrame, FrameGraph, MakeSparqlSelectQuery,
                           SCHEMA)
import rdflib

class Dspl2RdfExpander(object):
  """Expand CSV files in an DSPL2 via the RDF graph"""
  def __init__(self, getter):
    self.getter = getter
    self.graph = getter.graph
    self.subjects = set(self.graph.subjects())

  def _ExpandDimensionValue(self, dim, equivalentTypes, row_id, row):
    self.graph.add((dim, SCHEMA.codeList, row_id))
    self.graph.add((rdflib.URIRef(row_id), rdflib.RDF.type,
                    SCHEMA.DimensionValue))
    self.graph.add(
        (rdflib.URIRef(row_id), SCHEMA.dimension, dim))
    for type_id in equivalentTypes:
      self.graph.add((rdflib.URIRef(row_id), rdflib.RDF.type,
                      rdflib.URIRef(type_id)))
    for key, val in row.items():
      fields = key.split('@')
      if len(fields) > 1:
        # A language code is specified
        self.graph.add((rdflib.URIRef(row_id), getattr(SCHEMA, fields[0]),
                        rdflib.Literal(val, language=fields[1])))
      else:
        self.graph.add((rdflib.URIRef(row_id), getattr(SCHEMA, key),
                        rdflib.Literal(val)))

  def _ExpandCodeList(self, dim):
    # Set up the DimensionValue's triples.
    # Start with types
    if urlparse(dim).fragment:
      id_prefix = str(dim)
    else:
      id_prefix = str(dim) + '#' + str(self.graph.triples(
          subject=dim,
          predicate=SCHEMA.name)[0])
    id_prefix += '='
    equivalentTypes = self.graph.objects(
        subject=dim,
        predicate=SCHEMA.equivalentType)
    for codeList in self.graph.objects(
        subject=dim,
        predicate=SCHEMA.codeList):
      if codeList not in self.subjects:
        self.graph.remove((dim, SCHEMA.codeList, codeList))
        with self.getter.Fetch(str(codeList)) as f:
          reader = DictReader(f)
          for row in reader:
            self._ExpandDimensionValue(
                dim, equivalentTypes,
                rdflib.URIRef(id_prefix + row['codeValue']), row)

  def _ExpandFootnotes(self):
    for result in self.graph.query(
        MakeSparqlSelectQuery(
            ('?ds', 'a', 'schema:StatisticalDataset'),
            ('?ds', 'schema:footnote', '?fn'),
            ns_manager=self.graph.namespace_manager)):
      if result['fn'] not in self.subjects:
        self.graph.remove((result['ds'], SCHEMA.footnote, result['fn']))
        id_prefix = urldefrag(str(result['ds'])).url
        with self.getter.Fetch(str(result['fn'])) as f:
          reader = DictReader(f)
          for row in reader:
            row_id = rdflib.URIRef(id_prefix + '#footnote=' + row['codeValue'])
            self.graph.add((result['ds'], SCHEMA.footnote, row_id))
            self.graph.add((row_id, rdflib.RDF.type,
                            SCHEMA.StatisticalAnnotation))
            for key, val in row.items():
              fields = key.split('@')
              if len(fields) > 1:
                # A language code is specified
                self.graph.add((row_id, getattr(SCHEMA, fields[0]),
                                rdflib.Literal(val, language=fields[1])))
              else:
                self.graph.add((row_id, getattr(SCHEMA, key),
                                rdflib.Literal(val)))

  def _GetDimensionDataForSlice(self, slice_id):
    ret = {}
    dims = sorted(
        self.graph.objects(
            subject=slice_id,
            predicate=SCHEMA.dimension))
    for dim_id in dims:
      dim_type = list(self.graph.objects(
          subject=dim_id,
          predicate=rdflib.RDF.type))
      dim_equiv_types = list(self.graph.objects(
          subject=dim_id,
          predicate=SCHEMA.equivalentType))
      csv_id = None
      for identifier in self.graph.objects(
          subject=dim_id,
          predicate=SCHEMA.identifier):
        csv_id = identifier
        break
      if not csv_id:
        csv_id = urldefrag(dim_id).fragment
      if not csv_id:
        print("Unable to determine CSV ID for dimension", dim_id)
        exit(1)
      ret[csv_id] = {
          'id': dim_id,
          'type': dim_type,
          'types': dim_equiv_types
      }
    return ret

  def _GetMeasureDataForSlice(self, slice_id):
    ret = {}
    measures = sorted(
        self.graph.objects(
            subject=slice_id,
            predicate=SCHEMA.measure))
    for measure_id in measures:
      unit_codes = list(self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.unitCode))
      unit_texts = list(self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.unitText))
      csv_id = None
      for identifier in self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.identifier):
        csv_id = identifier
        break
      if not csv_id:
        csv_id = urldefrag(measure_id).fragment
      if not csv_id:
        print("Unable to determine CSV ID for metric", measure_id)
        exit(1)
      ret[csv_id] = {
          'id': measure_id,
          'unit_code': unit_codes,
          'unit_text': unit_texts,
      }
    return ret

  def _MakeSliceDataRowId(self, slice_id, dims, measures, row):
    ret = str(slice_id)
    if not urldefrag(slice_id).fragment:
      ret += '#'
    else:
      ret += '/'
    for dim in dims:
      ret += dim + '=' + row[dim]
      ret += '/'
    for measure in measures:
      ret += measure
      ret += '/'
    return ret

  def _ExpandObservationDimensionValue(self, dim, data, row_id, row):
    node_id = rdflib.BNode()
    self.graph.add((row_id, SCHEMA.dimensionValues, node_id))
    self.graph.add((node_id, rdflib.RDF.type, SCHEMA.DimensionValue))
    self.graph.add((node_id, SCHEMA.dimension, data['id']))
    for dim_type in data['type']:
      if dim_type.endswith('CategoricalDimension'):
        for type_id in data['types']:
          self.graph.add((node_id, rdflib.RDF.type, type_id))
        self.graph.add((node_id, SCHEMA.codeValue, rdflib.Literal(row[dim])))
      else:
        if data['types']:
          self.graph.add(
              (node_id, SCHEMA.value,
               rdflib.Literal(row[dim],
                              datatype=rdflib.URIRef(data['types'][0]))))
        else:
          self.graph.add((node_id, SCHEMA.value, rdflib.Literal(row[dim])))

  def _ExpandObservationMeasureValue(self, measure, data, row_id, row):
    node_id = rdflib.BNode()
    self.graph.add((row_id, SCHEMA.measureValues, node_id))
    self.graph.add((node_id, rdflib.RDF.type, SCHEMA.MeasureValue))
    for unit_code in data['unit_code']:
      self.graph.add((node_id, SCHEMA.unitCode, rdflib.Literal(unit_code)))
    for unit_text in data['unit_text']:
      self.graph.add((node_id, SCHEMA.unitCode, rdflib.Literal(unit_text)))
    self.graph.add((node_id, SCHEMA.value, rdflib.Literal(row[measure])))
    for footnote in row.get(measure + '*', '').split(';'):
      footnote_id = rdflib.BNode()
      self.graph.add((node_id, SCHEMA.footnote, footnote_id))
      self.graph.add((footnote_id, rdflib.RDF.type,
                      SCHEMA.StatisticalAnnotation))
      self.graph.add((footnote_id, SCHEMA.codeValue, rdflib.Literal(footnote)))

  def _ExpandSliceData(self, slice_id):
    dim_data = self._GetDimensionDataForSlice(slice_id)
    measure_data = self._GetMeasureDataForSlice(slice_id)
    for data_id in self.graph.objects(
        subject=slice_id,
        predicate=SCHEMA.data):
      if data_id not in self.subjects:
        with self.getter.Fetch(data_id) as f:
          reader = DictReader(f)
          try:
            for row in reader:
              row_id = rdflib.URIRef(self._MakeSliceDataRowId(
                  slice_id, dim_data, measure_data, row))
              self.graph.add((slice_id, SCHEMA.data, row_id))
              self.graph.add((row_id, rdflib.RDF.type, SCHEMA.Observation))
              self.graph.add((row_id, SCHEMA.slice, slice_id))
              for dim, data in dim_data.items():
                self._ExpandObservationDimensionValue(dim, data, row_id, row)
              for measure, data in measure_data.items():
                self._ExpandObservationMeasureValue(measure, data, row_id, row)
          except Exception as e:
            raise RuntimeError(f"Error processing {data_id} at line {reader.line_num}") from e

  def Expand(self):
    for dim in set(self.graph.subjects(
        predicate=rdflib.RDF.type,
        object=SCHEMA.CategoricalDimension)):
      self._ExpandCodeList(dim)
    self._ExpandFootnotes()
    for slice_id in set(self.graph.subjects(
        predicate=rdflib.RDF.type,
        object=SCHEMA.DataSlice)):
      self._ExpandSliceData(slice_id)
    return self.graph

class Dspl2JsonLdExpander(object):
  """Expand CSV files in an DSPL2 directly as JSON-LD"""
  def __init__(self, getter):
    self.getter = getter

  def _ExpandCodeList(self, dim):
    """Load a code list from CSV and return a list of JSON-LD objects."""
    codeList = []
    with self.getter.Fetch(GetSchemaProp(dim, 'codeList')) as f:
      reader = DictReader(f)
      for row in reader:
        if GetSchemaProp(dim, 'equivalentType'):
          row['@type'] = ['DimensionValue',
                          GetSchemaProp(dim, 'equivalentType')]
        else:
          row['@type'] = 'DimensionValue'
        row['@id'] = GetSchemaId(dim) + '='
        row['@id'] += row['codeValue']
        row['dimension'] = GetSchemaId(dim)
        codeList.append(row)
    return codeList

  def _ExpandFootnotes(self, filename, json_val):
    """Load footnotes from CSV and return a list of JSON-LD objects."""
    footnotes = []
    with self.getter.Fetch(filename) as f:
      reader = DictReader(f)
      for row in reader:
        row['@type'] = 'StatisticalAnnotation'
        row['@id'] = GetSchemaId(json_val) + '#footnote='
        row['@id'] += row['codeValue']
        row['dataset'] = GetSchemaId(json_val)
        footnotes.append(row)
    return footnotes

  def _ExpandSliceData(self, slice, dim_defs_by_id):
    data = []
    with self.getter.Fetch(GetSchemaProp(slice, 'data')) as f:
      reader = DictReader(f)
      for row in reader:
        val = {}
        val['@type'] = 'Observation'
        val['slice'] = GetSchemaId(slice)
        val['dimensionValues'] = []
        val['measureValues'] = []
        for dim in AsList(GetSchemaProp(slice, 'dimension')):
          dim = GetUrl(dim)
          fragment = urlparse(dim).fragment
          dim_val = {
              '@type': 'DimensionValue',
              'dimension': dim,
          }
          dim_def = dim_defs_by_id.get(dim)
          if dim_def:
            if GetSchemaProp(dim_def, '@type') == 'CategoricalDimension':
              dim_val['codeValue'] = row[fragment]
            elif GetSchemaProp(dim_def, '@type') == 'TimeDimension':
              if GetSchemaProp(dim_def, 'equivalentType'):
                dim_val['value'] = {
                    '@type': GetSchemaProp(dim_def, 'equivalentType'),
                    '@value': row[fragment]
                }
              else:
                val['dimensionValues'][-1]['value'] = row[fragment]
          val['dimensionValues'].append(dim_val)

        for measure in AsList(GetSchemaProp(slice, 'measure')):
          measure = GetUrl(measure)
          fragment = urlparse(measure).fragment
          val['measureValues'].append({
              '@type': 'MeasureValue',
              'measure': measure,
              'value': row[fragment]
          })
          if row.get(fragment + '*'):
            val['measureValues'][-1]['footnote'] = [
                {
                    '@type': 'StatisticalAnnotation',
                    'codeValue': footnote
                }
                for footnote in row[fragment + '*'].split(';')
            ]
        data.append(val)
    return data

  def Expand(self):
    json_val = FrameGraph(self.getter.graph, frame=_DataFileFrame)
    for dim in AsList(GetSchemaProp(json_val, 'dimension')):
      if isinstance(dim.get('codeList'), str):
        dim['codeList'] = self._ExpandCodeList(dim)
    if isinstance(GetSchemaProp(json_val, 'footnote'), str):
      json_val['footnote'] = self._ExpandFootnotes(
          GetSchemaProp(json_val, 'footnote'), json_val)
    for slice in AsList(GetSchemaProp(json_val, 'slice')):
      dim_defs_by_id = MakeIdKeyedDict(
          AsList(GetSchemaProp(json_val, 'dimension')))
      if isinstance(GetSchemaProp(slice, 'data'), str):
        slice['data'] = self._ExpandSliceData(slice, dim_defs_by_id)
    return json_val
<commit_msg>fix TimeDimension value path with no equivalentType<commit_after># Copyright 2018 Google LLC
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

from csv import DictReader
from urllib.parse import urlparse, urldefrag
from dspl2.jsonutil import (AsList, GetSchemaId, GetSchemaProp, GetUrl,
                            MakeIdKeyedDict)
from dspl2.rdfutil import (_DataFileFrame, FrameGraph, MakeSparqlSelectQuery,
                           SCHEMA)
import rdflib

class Dspl2RdfExpander(object):
  """Expand CSV files in an DSPL2 via the RDF graph"""
  def __init__(self, getter):
    self.getter = getter
    self.graph = getter.graph
    self.subjects = set(self.graph.subjects())

  def _ExpandDimensionValue(self, dim, equivalentTypes, row_id, row):
    self.graph.add((dim, SCHEMA.codeList, row_id))
    self.graph.add((rdflib.URIRef(row_id), rdflib.RDF.type,
                    SCHEMA.DimensionValue))
    self.graph.add(
        (rdflib.URIRef(row_id), SCHEMA.dimension, dim))
    for type_id in equivalentTypes:
      self.graph.add((rdflib.URIRef(row_id), rdflib.RDF.type,
                      rdflib.URIRef(type_id)))
    for key, val in row.items():
      fields = key.split('@')
      if len(fields) > 1:
        # A language code is specified
        self.graph.add((rdflib.URIRef(row_id), getattr(SCHEMA, fields[0]),
                        rdflib.Literal(val, language=fields[1])))
      else:
        self.graph.add((rdflib.URIRef(row_id), getattr(SCHEMA, key),
                        rdflib.Literal(val)))

  def _ExpandCodeList(self, dim):
    # Set up the DimensionValue's triples.
    # Start with types
    if urlparse(dim).fragment:
      id_prefix = str(dim)
    else:
      id_prefix = str(dim) + '#' + str(self.graph.triples(
          subject=dim,
          predicate=SCHEMA.name)[0])
    id_prefix += '='
    equivalentTypes = self.graph.objects(
        subject=dim,
        predicate=SCHEMA.equivalentType)
    for codeList in self.graph.objects(
        subject=dim,
        predicate=SCHEMA.codeList):
      if codeList not in self.subjects:
        self.graph.remove((dim, SCHEMA.codeList, codeList))
        with self.getter.Fetch(str(codeList)) as f:
          reader = DictReader(f)
          for row in reader:
            self._ExpandDimensionValue(
                dim, equivalentTypes,
                rdflib.URIRef(id_prefix + row['codeValue']), row)

  def _ExpandFootnotes(self):
    for result in self.graph.query(
        MakeSparqlSelectQuery(
            ('?ds', 'a', 'schema:StatisticalDataset'),
            ('?ds', 'schema:footnote', '?fn'),
            ns_manager=self.graph.namespace_manager)):
      if result['fn'] not in self.subjects:
        self.graph.remove((result['ds'], SCHEMA.footnote, result['fn']))
        id_prefix = urldefrag(str(result['ds'])).url
        with self.getter.Fetch(str(result['fn'])) as f:
          reader = DictReader(f)
          for row in reader:
            row_id = rdflib.URIRef(id_prefix + '#footnote=' + row['codeValue'])
            self.graph.add((result['ds'], SCHEMA.footnote, row_id))
            self.graph.add((row_id, rdflib.RDF.type,
                            SCHEMA.StatisticalAnnotation))
            for key, val in row.items():
              fields = key.split('@')
              if len(fields) > 1:
                # A language code is specified
                self.graph.add((row_id, getattr(SCHEMA, fields[0]),
                                rdflib.Literal(val, language=fields[1])))
              else:
                self.graph.add((row_id, getattr(SCHEMA, key),
                                rdflib.Literal(val)))

  def _GetDimensionDataForSlice(self, slice_id):
    ret = {}
    dims = sorted(
        self.graph.objects(
            subject=slice_id,
            predicate=SCHEMA.dimension))
    for dim_id in dims:
      dim_type = list(self.graph.objects(
          subject=dim_id,
          predicate=rdflib.RDF.type))
      dim_equiv_types = list(self.graph.objects(
          subject=dim_id,
          predicate=SCHEMA.equivalentType))
      csv_id = None
      for identifier in self.graph.objects(
          subject=dim_id,
          predicate=SCHEMA.identifier):
        csv_id = identifier
        break
      if not csv_id:
        csv_id = urldefrag(dim_id).fragment
      if not csv_id:
        print("Unable to determine CSV ID for dimension", dim_id)
        exit(1)
      ret[csv_id] = {
          'id': dim_id,
          'type': dim_type,
          'types': dim_equiv_types
      }
    return ret

  def _GetMeasureDataForSlice(self, slice_id):
    ret = {}
    measures = sorted(
        self.graph.objects(
            subject=slice_id,
            predicate=SCHEMA.measure))
    for measure_id in measures:
      unit_codes = list(self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.unitCode))
      unit_texts = list(self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.unitText))
      csv_id = None
      for identifier in self.graph.objects(
          subject=measure_id,
          predicate=SCHEMA.identifier):
        csv_id = identifier
        break
      if not csv_id:
        csv_id = urldefrag(measure_id).fragment
      if not csv_id:
        print("Unable to determine CSV ID for metric", measure_id)
        exit(1)
      ret[csv_id] = {
          'id': measure_id,
          'unit_code': unit_codes,
          'unit_text': unit_texts,
      }
    return ret

  def _MakeSliceDataRowId(self, slice_id, dims, measures, row):
    ret = str(slice_id)
    if not urldefrag(slice_id).fragment:
      ret += '#'
    else:
      ret += '/'
    for dim in dims:
      ret += dim + '=' + row[dim]
      ret += '/'
    for measure in measures:
      ret += measure
      ret += '/'
    return ret

  def _ExpandObservationDimensionValue(self, dim, data, row_id, row):
    node_id = rdflib.BNode()
    self.graph.add((row_id, SCHEMA.dimensionValues, node_id))
    self.graph.add((node_id, rdflib.RDF.type, SCHEMA.DimensionValue))
    self.graph.add((node_id, SCHEMA.dimension, data['id']))
    for dim_type in data['type']:
      if dim_type.endswith('CategoricalDimension'):
        for type_id in data['types']:
          self.graph.add((node_id, rdflib.RDF.type, type_id))
        self.graph.add((node_id, SCHEMA.codeValue, rdflib.Literal(row[dim])))
      else:
        if data['types']:
          self.graph.add(
              (node_id, SCHEMA.value,
               rdflib.Literal(row[dim],
                              datatype=rdflib.URIRef(data['types'][0]))))
        else:
          self.graph.add((node_id, SCHEMA.value, rdflib.Literal(row[dim])))

  def _ExpandObservationMeasureValue(self, measure, data, row_id, row):
    node_id = rdflib.BNode()
    self.graph.add((row_id, SCHEMA.measureValues, node_id))
    self.graph.add((node_id, rdflib.RDF.type, SCHEMA.MeasureValue))
    for unit_code in data['unit_code']:
      self.graph.add((node_id, SCHEMA.unitCode, rdflib.Literal(unit_code)))
    for unit_text in data['unit_text']:
      self.graph.add((node_id, SCHEMA.unitCode, rdflib.Literal(unit_text)))
    self.graph.add((node_id, SCHEMA.value, rdflib.Literal(row[measure])))
    for footnote in row.get(measure + '*', '').split(';'):
      footnote_id = rdflib.BNode()
      self.graph.add((node_id, SCHEMA.footnote, footnote_id))
      self.graph.add((footnote_id, rdflib.RDF.type,
                      SCHEMA.StatisticalAnnotation))
      self.graph.add((footnote_id, SCHEMA.codeValue, rdflib.Literal(footnote)))

  def _ExpandSliceData(self, slice_id):
    dim_data = self._GetDimensionDataForSlice(slice_id)
    measure_data = self._GetMeasureDataForSlice(slice_id)
    for data_id in self.graph.objects(
        subject=slice_id,
        predicate=SCHEMA.data):
      if data_id not in self.subjects:
        with self.getter.Fetch(data_id) as f:
          reader = DictReader(f)
          try:
            for row in reader:
              row_id = rdflib.URIRef(self._MakeSliceDataRowId(
                  slice_id, dim_data, measure_data, row))
              self.graph.add((slice_id, SCHEMA.data, row_id))
              self.graph.add((row_id, rdflib.RDF.type, SCHEMA.Observation))
              self.graph.add((row_id, SCHEMA.slice, slice_id))
              for dim, data in dim_data.items():
                self._ExpandObservationDimensionValue(dim, data, row_id, row)
              for measure, data in measure_data.items():
                self._ExpandObservationMeasureValue(measure, data, row_id, row)
          except Exception as e:
            raise RuntimeError(f"Error processing {data_id} at line {reader.line_num}") from e

  def Expand(self):
    for dim in set(self.graph.subjects(
        predicate=rdflib.RDF.type,
        object=SCHEMA.CategoricalDimension)):
      self._ExpandCodeList(dim)
    self._ExpandFootnotes()
    for slice_id in set(self.graph.subjects(
        predicate=rdflib.RDF.type,
        object=SCHEMA.DataSlice)):
      self._ExpandSliceData(slice_id)
    return self.graph

class Dspl2JsonLdExpander(object):
  """Expand CSV files in an DSPL2 directly as JSON-LD"""
  def __init__(self, getter):
    self.getter = getter

  def _ExpandCodeList(self, dim):
    """Load a code list from CSV and return a list of JSON-LD objects."""
    codeList = []
    with self.getter.Fetch(GetSchemaProp(dim, 'codeList')) as f:
      reader = DictReader(f)
      for row in reader:
        if GetSchemaProp(dim, 'equivalentType'):
          row['@type'] = ['DimensionValue',
                          GetSchemaProp(dim, 'equivalentType')]
        else:
          row['@type'] = 'DimensionValue'
        row['@id'] = GetSchemaId(dim) + '='
        row['@id'] += row['codeValue']
        row['dimension'] = GetSchemaId(dim)
        codeList.append(row)
    return codeList

  def _ExpandFootnotes(self, filename, json_val):
    """Load footnotes from CSV and return a list of JSON-LD objects."""
    footnotes = []
    with self.getter.Fetch(filename) as f:
      reader = DictReader(f)
      for row in reader:
        row['@type'] = 'StatisticalAnnotation'
        row['@id'] = GetSchemaId(json_val) + '#footnote='
        row['@id'] += row['codeValue']
        row['dataset'] = GetSchemaId(json_val)
        footnotes.append(row)
    return footnotes

  def _ExpandSliceData(self, slice, dim_defs_by_id):
    data = []
    with self.getter.Fetch(GetSchemaProp(slice, 'data')) as f:
      reader = DictReader(f)
      for row in reader:
        val = {}
        val['@type'] = 'Observation'
        val['slice'] = GetSchemaId(slice)
        val['dimensionValues'] = []
        val['measureValues'] = []
        for dim in AsList(GetSchemaProp(slice, 'dimension')):
          dim = GetUrl(dim)
          fragment = urlparse(dim).fragment
          dim_val = {
              '@type': 'DimensionValue',
              'dimension': dim,
          }
          dim_def = dim_defs_by_id.get(dim)
          if dim_def:
            if GetSchemaProp(dim_def, '@type') == 'CategoricalDimension':
              dim_val['codeValue'] = row[fragment]
            elif GetSchemaProp(dim_def, '@type') == 'TimeDimension':
              if GetSchemaProp(dim_def, 'equivalentType'):
                dim_val['value'] = {
                    '@type': GetSchemaProp(dim_def, 'equivalentType'),
                    '@value': row[fragment]
                }
              else:
                dim_val['value'] = row[fragment]
          val['dimensionValues'].append(dim_val)

        for measure in AsList(GetSchemaProp(slice, 'measure')):
          measure = GetUrl(measure)
          fragment = urlparse(measure).fragment
          val['measureValues'].append({
              '@type': 'MeasureValue',
              'measure': measure,
              'value': row[fragment]
          })
          if row.get(fragment + '*'):
            val['measureValues'][-1]['footnote'] = [
                {
                    '@type': 'StatisticalAnnotation',
                    'codeValue': footnote
                }
                for footnote in row[fragment + '*'].split(';')
            ]
        data.append(val)
    return data

  def Expand(self):
    json_val = FrameGraph(self.getter.graph, frame=_DataFileFrame)
    for dim in AsList(GetSchemaProp(json_val, 'dimension')):
      if isinstance(dim.get('codeList'), str):
        dim['codeList'] = self._ExpandCodeList(dim)
    if isinstance(GetSchemaProp(json_val, 'footnote'), str):
      json_val['footnote'] = self._ExpandFootnotes(
          GetSchemaProp(json_val, 'footnote'), json_val)
    for slice in AsList(GetSchemaProp(json_val, 'slice')):
      dim_defs_by_id = MakeIdKeyedDict(
          AsList(GetSchemaProp(json_val, 'dimension')))
      if isinstance(GetSchemaProp(slice, 'data'), str):
        slice['data'] = self._ExpandSliceData(slice, dim_defs_by_id)
    return json_val
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# taken from
# https://github.com/pinax/django-forms-bootstrap

from django import template
from django.conf import settings
from django.template import Context
from django.template.loader import get_template

register = template.Library()

def _preprocess_fields(form):
    for field in form.fields:
        name = form.fields[field].widget.__class__.__name__.lower()
        if not name.startswith("radio") and not name.startswith("checkbox"):
            try:
                form.fields[field].widget.attrs["class"] += " form-control"
            except KeyError:
                form.fields[field].widget.attrs["class"] = " form-control"
    return form

@register.filter
def render_form(form):
    """same than  {{ form|crispy }} if crispy_forms is installed.
    render using a bootstrap3 templating otherwise"""

    if 'crispy_forms' in settings.INSTALLED_APPS:
        from crispy_forms.templatetags.crispy_forms_filters import as_crispy_form
        return as_crispy_form(form)

    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    c = Context({
        "form": form,
    })
    return template.render(c)

@register.filter
def as_bootstrap_inline(form):
    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    for field in form.fields:
        name = form.fields[field].widget.__class__.__name__.lower()
        if not name.startswith("radio") and not name.startswith("checkbox"):
            form.fields[field].widget.attrs["placeholder"] = form.fields[field].label

    css_classes = {
        "label": "sr-only",
        "single_container": "",
        "wrap": "",
    }

    c = Context({
        "form": form,
        "css_classes": css_classes,
    })
    return template.render(c)

@register.filter
def as_bootstrap_horizontal(form, label_classes=""):
    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    if label_classes == "":
        label_classes = "col-md-2"

    css_classes = {
        "label": label_classes,
        "single_container": "",
        "wrap": "",
    }

    for label_class in label_classes.split(" "):
        split_class, column_count = label_class.rsplit("-", 1)
        column_count = int(column_count)

        if column_count < 12:
            offset_class = "{split_class}-offset-{column_count}".format(
                split_class=split_class,
                column_count=column_count,
            )
            wrap_class = "{split_class}-{column_count}".format(
                split_class=split_class,
                column_count=12 - column_count,
            )
            css_classes["single_container"] += offset_class + " " + wrap_class + " "
            css_classes["wrap"] += wrap_class + " "

    c = Context({
        "form": form,
        "css_classes": css_classes,
    })
    return template.render(c)

@register.filter
def css_class(field):
    return field.field.widget.__class__.__name__.lower()
<commit_msg>remove deprectaed context class wrapper<commit_after># -*- coding: utf-8 -*-

# taken from
# https://github.com/pinax/django-forms-bootstrap

from django import template
from django.conf import settings
from django.template.loader import get_template

register = template.Library()

def _preprocess_fields(form):
    for field in form.fields:
        name = form.fields[field].widget.__class__.__name__.lower()
        if not name.startswith("radio") and not name.startswith("checkbox"):
            try:
                form.fields[field].widget.attrs["class"] += " form-control"
            except KeyError:
                form.fields[field].widget.attrs["class"] = " form-control"
    return form

@register.filter
def render_form(form):
    """same than  {{ form|crispy }} if crispy_forms is installed.
    render using a bootstrap3 templating otherwise"""

    if 'crispy_forms' in settings.INSTALLED_APPS:
        from crispy_forms.templatetags.crispy_forms_filters import as_crispy_form
        return as_crispy_form(form)

    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    return template.render({"form": form})

@register.filter
def as_bootstrap_inline(form):
    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    for field in form.fields:
        name = form.fields[field].widget.__class__.__name__.lower()
        if not name.startswith("radio") and not name.startswith("checkbox"):
            form.fields[field].widget.attrs["placeholder"] = form.fields[field].label
    css_classes = {
        "label": "sr-only",
        "single_container": "",
        "wrap": "",
    }
    context = {
        "form": form,
        "css_classes": css_classes,
    }
    return template.render(contex)

@register.filter
def as_bootstrap_horizontal(form, label_classes=""):
    template = get_template("bootstrap/form.html")
    form = _preprocess_fields(form)

    if label_classes == "":
        label_classes = "col-md-2"

    css_classes = {
        "label": label_classes,
        "single_container": "",
        "wrap": "",
    }

    for label_class in label_classes.split(" "):
        split_class, column_count = label_class.rsplit("-", 1)
        column_count = int(column_count)

        if column_count < 12:
            offset_class = "{split_class}-offset-{column_count}".format(
                split_class=split_class,
                column_count=column_count,
            )
            wrap_class = "{split_class}-{column_count}".format(
                split_class=split_class,
                column_count=12 - column_count,
            )
            css_classes["single_container"] += offset_class + " " + wrap_class + " "
            css_classes["wrap"] += wrap_class + " "
    context = {
        "form": form,
        "css_classes": css_classes,
    }
    return template.render(context)

@register.filter
def css_class(field):
    return field.field.widget.__class__.__name__.lower()
<|endoftext|>


<commit_before>collections_to_resource_type = {
    "stop_points": "stop_point", "routes": "route",
    "networks": "network", "commercial_modes": "commercial_mode",
    "physical_modes": "physical_mode", "companies": "company",
    "stop_areas": "stop_area", "lines": "line",
    "addresses": "address", "coords": "coord",
    "journey_pattern_points": "journey_pattern_point",
    "pois": "poi", "poi_types": "poi_type",
    "connections": "connection"}

resource_type_to_collection = dict((resource_type, collection)
                                   for (collection, resource_type)
                                   in collections_to_resource_type.iteritems())
<commit_msg>Jormungandr: add vehicle journeys to collections<commit_after>collections_to_resource_type = {
    "stop_points": "stop_point", "routes": "route",
    "networks": "network", "commercial_modes": "commercial_mode",
    "physical_modes": "physical_mode", "companies": "company",
    "stop_areas": "stop_area", "lines": "line",
    "addresses": "address", "coords": "coord",
    "journey_pattern_points": "journey_pattern_point",
    "pois": "poi", "poi_types": "poi_type",
    "connections": "connection", "vehicle_journeys":"vehicle_journey"}

resource_type_to_collection = dict((resource_type, collection)
                                   for (collection, resource_type)
                                   in collections_to_resource_type.iteritems())
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Macsio(CMakePackage):
    """A Multi-purpose, Application-Centric, Scalable I/O Proxy Application."""

    tags = ['proxy-app', 'ecp-proxy-app']

    homepage = "https://computation.llnl.gov/projects/co-design/macsio"
    url      = "https://github.com/LLNL/MACSio/archive/1.0.tar.gz"
    git      = "https://github.com/LLNL/MACSio.git"

    version('develop', branch='master')
    version('1.0', '90e8e00ea84af2a47bee387ad331dbde')

    variant('mpi', default=True, description="Build MPI plugin")
    variant('silo', default=True, description="Build with SILO plugin")
    # TODO: multi-level variants for hdf5
    variant('hdf5', default=False, description="Build HDF5 plugin")
    variant('zfp', default=False, description="Build HDF5 with ZFP compression")
    variant('szip', default=False, description="Build HDF5 with SZIP compression")
    variant('zlib', default=False, description="Build HDF5 with ZLIB compression")
    variant('pdb', default=False, description="Build PDB plugin")
    variant('exodus', default=False, description="Build EXODUS plugin")
    variant('scr', default=False, description="Build with SCR support")
    variant('typhonio', default=False, description="Build TYPHONIO plugin")

    depends_on('json-cwx')
    depends_on('mpi', when="+mpi")
    depends_on('silo', when="+silo")
    depends_on('hdf5+hl', when="+hdf5")
    # depends_on('hdf5+szip', when="+szip")
    depends_on('exodusii', when="+exodus")
    # pdb is packaged with silo
    depends_on('silo', when="+pdb")
    depends_on('typhonio', when="+typhonio")
    depends_on('scr', when="+scr")

    def cmake_args(self):
        spec = self.spec
        cmake_args = []

        if "~mpi" in spec:
            cmake_args.append("-DENABLE_MPI=OFF")

        if "~silo" in spec:
            cmake_args.append("-DENABLE_SILO_PLUGIN=OFF")

        if "+silo" in spec:
            cmake_args.append("-DWITH_SILO_PREFIX={0}"
                              .format(spec['silo'].prefix))

        if "+pdb" in spec:
            # pdb is a part of silo
            cmake_args.append("-DENABLE_PDB_PLUGIN=ON")
            cmake_args.append("-DWITH_SILO_PREFIX={0}"
                              .format(spec['silo'].prefix))
        if "+hdf5" in spec:
            cmake_args.append("-DENABLE_HDF5_PLUGIN=ON")
            cmake_args.append("-DWITH_HDF5_PREFIX={0}"
                              .format(spec['hdf5'].prefix))
            # TODO: Multi-level variants
            # ZFP not in hdf5 spack package??
            # if "+zfp" in spec:
            #     cmake_args.append("-DENABLE_HDF5_ZFP")
            #     cmake_args.append("-DWITH_ZFP_PREFIX={0}"
            #         .format(spec['silo'].prefix))
            # SZIP is an hdf5 spack variant
            # if "+szip" in spec:
            #     cmake_args.append("-DENABLE_HDF5_SZIP")
            #     cmake_args.append("-DWITH_SZIP_PREFIX={0}"
            #         .format(spec['SZIP'].prefix))
            # ZLIB is on by default, @1.1.2
            # if "+zlib" in spec:
            #     cmake_args.append("-DENABLE_HDF5_ZLIB")
            #     cmake_args.append("-DWITH_ZLIB_PREFIX={0}"
            #         .format(spec['silo'].prefix))

        if "+typhonio" in spec:
            cmake_args.append("-DENABLE_TYPHONIO_PLUGIN=ON")
            cmake_args.append("-DWITH_TYPHONIO_PREFIX={0}"
                              .format(spec['typhonio'].prefix))

        if "+exodus" in spec:
            cmake_args.append("-DENABLE_EXODUS_PLUGIN=ON")
            cmake_args.append("-DWITH_EXODUS_PREFIX={0}"
                              .format(spec['exodusii'].prefix))
            # exodus requires netcdf
            cmake_args.append("-DWITH_NETCDF_PREFIX={0}"
                              .format(spec['netcdf'].prefix))

        return cmake_args
<commit_msg>MACSio: update package (#9412)<commit_after>##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Macsio(CMakePackage):
    """A Multi-purpose, Application-Centric, Scalable I/O Proxy Application."""

    tags = ['proxy-app', 'ecp-proxy-app']

    homepage = "https://computation.llnl.gov/projects/co-design/macsio"
    url      = "https://github.com/LLNL/MACSio/archive/v1.1.tar.gz"
    git      = "https://github.com/LLNL/MACSio.git"

    version('develop', branch='master')

    version('1.1', sha256='a86249b0f10647c0b631773db69568388094605ec1a0af149d9e61e95e6961ec')
    version('1.0', '90e8e00ea84af2a47bee387ad331dbde')

    variant('mpi', default=True, description="Build MPI plugin")
    variant('silo', default=True, description="Build with SILO plugin")
    # TODO: multi-level variants for hdf5
    variant('hdf5', default=False, description="Build HDF5 plugin")
    variant('zfp', default=False, description="Build HDF5 with ZFP compression")
    variant('szip', default=False, description="Build HDF5 with SZIP compression")
    variant('zlib', default=False, description="Build HDF5 with ZLIB compression")
    variant('pdb', default=False, description="Build PDB plugin")
    variant('exodus', default=False, description="Build EXODUS plugin")
    variant('scr', default=False, description="Build with SCR support")
    variant('typhonio', default=False, description="Build TYPHONIO plugin")

    depends_on('json-cwx')
    depends_on('mpi', when="+mpi")
    depends_on('silo', when="+silo")
    depends_on('hdf5+hl', when="+hdf5")
    # depends_on('hdf5+szip', when="+szip")
    depends_on('exodusii', when="+exodus")
    # pdb is packaged with silo
    depends_on('silo', when="+pdb")
    depends_on('typhonio', when="+typhonio")
    depends_on('scr', when="+scr")

    def cmake_args(self):
        spec = self.spec
        cmake_args = []

        if "~mpi" in spec:
            cmake_args.append("-DENABLE_MPI=OFF")

        if "~silo" in spec:
            cmake_args.append("-DENABLE_SILO_PLUGIN=OFF")

        if "+silo" in spec:
            cmake_args.append("-DWITH_SILO_PREFIX={0}"
                              .format(spec['silo'].prefix))

        if "+pdb" in spec:
            # pdb is a part of silo
            cmake_args.append("-DENABLE_PDB_PLUGIN=ON")
            cmake_args.append("-DWITH_SILO_PREFIX={0}"
                              .format(spec['silo'].prefix))
        if "+hdf5" in spec:
            cmake_args.append("-DENABLE_HDF5_PLUGIN=ON")
            cmake_args.append("-DWITH_HDF5_PREFIX={0}"
                              .format(spec['hdf5'].prefix))
            # TODO: Multi-level variants
            # ZFP not in hdf5 spack package??
            # if "+zfp" in spec:
            #     cmake_args.append("-DENABLE_HDF5_ZFP")
            #     cmake_args.append("-DWITH_ZFP_PREFIX={0}"
            #         .format(spec['silo'].prefix))
            # SZIP is an hdf5 spack variant
            # if "+szip" in spec:
            #     cmake_args.append("-DENABLE_HDF5_SZIP")
            #     cmake_args.append("-DWITH_SZIP_PREFIX={0}"
            #         .format(spec['SZIP'].prefix))
            # ZLIB is on by default, @1.1.2
            # if "+zlib" in spec:
            #     cmake_args.append("-DENABLE_HDF5_ZLIB")
            #     cmake_args.append("-DWITH_ZLIB_PREFIX={0}"
            #         .format(spec['silo'].prefix))

        if "+typhonio" in spec:
            cmake_args.append("-DENABLE_TYPHONIO_PLUGIN=ON")
            cmake_args.append("-DWITH_TYPHONIO_PREFIX={0}"
                              .format(spec['typhonio'].prefix))

        if "+exodus" in spec:
            cmake_args.append("-DENABLE_EXODUS_PLUGIN=ON")
            cmake_args.append("-DWITH_EXODUS_PREFIX={0}"
                              .format(spec['exodusii'].prefix))
            # exodus requires netcdf
            cmake_args.append("-DWITH_NETCDF_PREFIX={0}"
                              .format(spec['netcdf'].prefix))

        return cmake_args
<|endoftext|>


<commit_before>import shutil
from conans import ConanFile, VisualStudioBuildEnvironment, CMake, tools

class CbindgenTestConan(ConanFile):
    name = "pact_mock_server_ffi"
    version = "0.0.1"
    description = "Pact/Rust FFI bindings"
    url = "https://github.com/pact-foundation/pact-reference"
    license = "MIT"
    settings = "os", "compiler", "build_type", "arch"
    no_copy_source = True
    requires = "openssl/1.1.1d"

    def build(self):
        if self.settings.os == "Windows":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-windows-x86_64.lib.gz"
                   % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.lib.gz")
            tools.unzip("libpact_mock_server_ffi.lib.gz")
        elif self.settings.os == "Linux":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-linux-x86_64.a.gz"
                % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.a.gz")
            tools.unzip("libpact_mock_server_ffi.a.gz")
        elif self.settings.os == "macOS":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-osx-x86_64.a.gz"
                   % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.a.gz")
            tools.unzip("libpact_mock_server_ffi.a.gz")
        else:
            raise Exception("Binary does not exist for these settings")
        tools.download(("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/pact_mock_server_ffi.h"
                % (str(self.version))), "include/pact_mock_server_ffi.h")

    def package(self):
        self.copy("libpact_mock_server_ffi*.a", "lib", "")
        self.copy("libpact_mock_server_ffi*.lib", "lib", "")
        self.copy("*.h", "", "")

    def package_info(self):  # still very useful for package consumers
        self.cpp_info.libs = ["pact_mock_server_ffi"]
<commit_msg>fix: Macos on conan package<commit_after>import shutil
from conans import ConanFile, VisualStudioBuildEnvironment, CMake, tools

class CbindgenTestConan(ConanFile):
    name = "pact_mock_server_ffi"
    version = "0.0.1"
    description = "Pact/Rust FFI bindings"
    url = "https://github.com/pact-foundation/pact-reference"
    license = "MIT"
    settings = "os", "compiler", "build_type", "arch"
    no_copy_source = True
    requires = "openssl/1.1.1d"

    def build(self):
        if self.settings.os == "Windows":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-windows-x86_64.lib.gz"
                   % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.lib.gz")
            tools.unzip("libpact_mock_server_ffi.lib.gz")
        elif self.settings.os == "Linux":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-linux-x86_64.a.gz"
                % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.a.gz")
            tools.unzip("libpact_mock_server_ffi.a.gz")
        elif self.settings.os == "Macos":
            url = ("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/libpact_mock_server_ffi-osx-x86_64.a.gz"
                   % (str(self.version)))
            tools.download(url, "libpact_mock_server_ffi.a.gz")
            tools.unzip("libpact_mock_server_ffi.a.gz")
        else:
            raise Exception("Binary does not exist for these settings")
        tools.download(("https://github.com/pact-foundation/pact-reference/releases/download/libpact_mock_server_ffi-v%s/pact_mock_server_ffi.h"
                % (str(self.version))), "include/pact_mock_server_ffi.h")

    def package(self):
        self.copy("libpact_mock_server_ffi*.a", "lib", "")
        self.copy("libpact_mock_server_ffi*.lib", "lib", "")
        self.copy("*.h", "", "")

    def package_info(self):  # still very useful for package consumers
        self.cpp_info.libs = ["pact_mock_server_ffi"]
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
import sys

class Pixman(AutotoolsPackage):
    """The Pixman package contains a library that provides low-level
    pixel manipulation features such as image compositing and
    trapezoid rasterization."""

    homepage = "http://www.pixman.org"
    url      = "http://cairographics.org/releases/pixman-0.32.6.tar.gz"

    version('0.38.0', sha256='a7592bef0156d7c27545487a52245669b00cf7e70054505381cff2136d890ca8')
    version('0.34.0', sha256='21b6b249b51c6800dc9553b65106e1e37d0e25df942c90531d4c3997aa20a88e')
    version('0.32.6', sha256='3dfed13b8060eadabf0a4945c7045b7793cc7e3e910e748a8bb0f0dc3e794904')

    depends_on('pkgconfig', type='build')
    depends_on('libpng')

    # As discussed here:
    # https://bugs.freedesktop.org/show_bug.cgi?id=104886
    # __builtin_shuffle was removed in clang 5.0.
    # From version 9.1 apple-clang is based on clang 5.0.
    # Patch is obtained from above link.
    patch('clang.patch', when='%clang@9.1.0-apple:')

    @run_before('build')
    def patch_config_h_for_intel(self):
        config_h = join_path(self.stage.source_path, 'config.h')

        # Intel disguises itself as GNU, but doesn't implement
        # the same builtin functions. This causes in this case
        # a positive detection of GCC vector extensions, which
        # is bound to fail at compile time because Intel has no
        # __builtin_shuffle. See also:
        #
        # https://software.intel.com/en-us/forums/intel-c-compiler/topic/758013
        #
        if '%intel' in self.spec:
            filter_file(
                r'#define HAVE_GCC_VECTOR_EXTENSIONS /\*\*/',
                '/* #undef HAVE_GCC_VECTOR_EXTENSIONS */',
                config_h
            )

    def configure_args(self):
        args = [
            '--enable-libpng',
            '--disable-gtk',
        ]

        if sys.platform == 'darwin':
            args.append('--disable-mmx')

        return args
<commit_msg>pixman: fix patch (#11173)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
import sys

class Pixman(AutotoolsPackage):
    """The Pixman package contains a library that provides low-level
    pixel manipulation features such as image compositing and
    trapezoid rasterization."""

    homepage = "http://www.pixman.org"
    url      = "http://cairographics.org/releases/pixman-0.32.6.tar.gz"

    version('0.38.0', sha256='a7592bef0156d7c27545487a52245669b00cf7e70054505381cff2136d890ca8')
    version('0.34.0', sha256='21b6b249b51c6800dc9553b65106e1e37d0e25df942c90531d4c3997aa20a88e')
    version('0.32.6', sha256='3dfed13b8060eadabf0a4945c7045b7793cc7e3e910e748a8bb0f0dc3e794904')

    depends_on('pkgconfig', type='build')
    depends_on('libpng')

    # As discussed here:
    # https://bugs.freedesktop.org/show_bug.cgi?id=104886
    # __builtin_shuffle was removed in clang 5.0.
    # From version 9.1 apple-clang is based on clang 5.0.
    # Patch is obtained from above link.
    patch('clang.patch', when='@0.34%clang@9.1.0-apple:')

    @run_before('build')
    def patch_config_h_for_intel(self):
        config_h = join_path(self.stage.source_path, 'config.h')

        # Intel disguises itself as GNU, but doesn't implement
        # the same builtin functions. This causes in this case
        # a positive detection of GCC vector extensions, which
        # is bound to fail at compile time because Intel has no
        # __builtin_shuffle. See also:
        #
        # https://software.intel.com/en-us/forums/intel-c-compiler/topic/758013
        #
        if '%intel' in self.spec:
            filter_file(
                r'#define HAVE_GCC_VECTOR_EXTENSIONS /\*\*/',
                '/* #undef HAVE_GCC_VECTOR_EXTENSIONS */',
                config_h
            )

    def configure_args(self):
        args = [
            '--enable-libpng',
            '--disable-gtk',
        ]

        if sys.platform == 'darwin':
            args.append('--disable-mmx')

        return args
<|endoftext|>