<commit_before>import os
import gviz_api
import pandas as pd
from igf_data.utils.seqrunutils import get_seqrun_date_from_igf_id

def _count_total_reads(data,seqrun_list):
  '''
  An internal function for counting total reads

  required params:
  data: A dictionary containing seqrun ids a key an read counts as values
  seqrun_list: A list of sequencing runs
  '''
  try:
    if len(seqrun_list) >1:
      if 'total_read' not in data:
        data['total_read']=0
        for run in seqrun_list:
          data['total_read'] += data[run]
    return data
  except:
    raise

def convert_project_data_gviz_data(input_data,output_file,
                                   sample_col='sample_igf_id',
                                   read_count_col='attribute_value',
                                   seqrun_col='flowcell_id'):
  '''
  A utility method for converting project's data availability information to
  gviz data table format
  https://developers.google.com/chart/interactive/docs/reference#DataTable

  required params:
  input_data: A pandas data frame, it should contain following columns
              sample_igf_id,
              flowcell_id,
              attribute_value (R1_READ_COUNT)
  sample_col: Column name for sample id, default sample_igf_id
  seqrun_col: Column name for sequencing run identifier, default flowcell_id
  read_count_col: Column name for sample read counts, default attribute_value
  output_file: A filepath for writing gviz json data
  '''
  try:
    if not isinstance(input_data, pd.DataFrame):
      raise AttributeError('Expecting a pandas dataframe and got {0}'.\
                           format(type(input_data)))
    input_data[read_count_col]=input_data[read_count_col].astype(int)           # convert reac counts to int
    processed_data=input_data.\
                   pivot_table(values=read_count_col,
                               index=[sample_col,
                                      seqrun_col],
                               aggfunc='sum')                                   # group data by sample id and seq runs
    print(processed_data.to_dict(orient='region'))
    processed_data.\
    reset_index([sample_col,
                 seqrun_col],
                inplace=True)                                                   # reset index for processed data
    intermediate_data=list()                                                    # define empty intermediate data structure
    seqrun_set=set()                                                            # define empty seqrun set
    print(processed_data.to_dict(orient='records'))
    for line in processed_data.to_dict(orient='records'):                       # reformat processed data to required structure
      tmp_data=dict()
      tmp_data.update({sample_col:line[sample_col],
                        line[seqrun_col]:line[read_count_col]})
      seqrun_set.add(line[seqrun_col])
      intermediate_data.append(tmp_data)

    intermediate_data=pd.DataFrame(intermediate_data)                           # convert intermediate data to dataframe
    intermediate_data.fillna(0,inplace=True)                                    # replace NAN values with zero
    intermediate_data=intermediate_data.\
                      pivot_table(index=sample_col,
                                  aggfunc='sum').\
                      reset_index(sample_col)                                   # group data by samples id
    intermediate_data=intermediate_data.\
                      apply(lambda line: \
                            _count_total_reads(data=line,
                                               seqrun_list=list(seqrun_set)),
                            axis=1)                                             # count total reads for multiple seq runs
    intermediate_data.fillna(0,inplace=True)                                    # fail safe for missing samples

    print(intermediate_data)
    description = {sample_col: ("string", "Sample ID")}                         # define description
    if len(list(seqrun_set)) >1:
        description.update({"total_read":("number", "Total Reads")})            # add total read column for samples with multiple runs
        intermediate_data['total_read']=intermediate_data['total_read'].\
                                        astype(float)                           # convert column to number

    for run in list(seqrun_set):
        description.update({run:("number",run)})                                # add seqrun columns
        intermediate_data[run]=intermediate_data[run].\
                               astype(float)                                    # convert column to number

    intermediate_data=intermediate_data.to_dict(orient='records')                # convert data frame to json
    print(intermediate_data)
    data_table = gviz_api.DataTable(description)                                # load description to gviz api
    data_table.LoadData(intermediate_data)                                      # load data to gviz_api
    column_list=[sample_col]                                                    # define column order
    column_list.extend(list(seqrun_set))
    if len(list(seqrun_set)) >1:
        column_list.append('total_read')                                        # total read is present only for multiple runs

    final_data=data_table.ToJSon(columns_order=tuple(column_list))              # create final data structure
    with open(output_file,'w') as jf:
        jf.write(final_data)                                                    # write final data to output file
  except:
    raise

def _modify_seqrun_data(data_series,seqrun_col,flowcell_col,path_col):
  '''
  An internal method for parsing seqrun dataframe and adding remote dir path
  required columns: seqrun_igf_id, flowcell_id
  seqrun_col: Column name for sequencing run id, default seqrun_igf_id
  flowcell_col: Column namae for flowcell id, default flowcell_id
  path_col: Column name for path, default path
  output column: flowcell_id, path
  '''
  try:
    if not isinstance(data_series,pd.Series):
      raise AttributeError('Expecting a pandas data series and got {0}'.\
                           format(type(data_series)))

    seqrun_igf_id=data_series[seqrun_col]
    flowcell_id=data_series[flowcell_col]
    seqrun_date=get_seqrun_date_from_igf_id(seqrun_igf_id)
    data_series[path_col]=os.path.join(seqrun_date,flowcell_id)                 # adding path to data series
    del data_series[seqrun_col]
    return data_series
  except:
    raise

def add_seqrun_path_info(input_data,output_file,seqrun_col='seqrun_igf_id',
                         flowcell_col='flowcell_id',path_col='path'):
  '''
  A utility method for adding remote path to a dataframe for each sequencing runs
  of a project

  required params:
  input_data: A input dataframe containing the following columns
              seqrun_igf_id
              flowcell_id
  seqrun_col: Column name for sequencing run id, default seqrun_igf_id
  flowcell_col: Column namae for flowcell id, default flowcell_id
  path_col: Column name for path, default path
  output_file: An output filepath for the json data
  '''
  try:
    if not isinstance(input_data,pd.DataFrame):
      raise AttributeError('Expecting a pandas dataframe and got {0}'.\
                           format(type(input_data)))

    input_data.drop_duplicates(inplace=True)                                    # remove duplicate entries
    input_data=input_data.\
               apply(lambda line: \
                     _modify_seqrun_data(data_series=line,
                                         seqrun_col=seqrun_col,
                                         flowcell_col=flowcell_col,
                                         path_col=path_col),
                     axis=1)                                                    # add remote seqrun path
    input_data=input_data.to_json(orient='records')                             # encode output json
    with open(output_file,'w') as j_data:
      j_data.write(input_data)                                                  # write output json file
  except:
    raise<commit_msg>removed extra print statements<commit_after>import os
import gviz_api
import pandas as pd
from igf_data.utils.seqrunutils import get_seqrun_date_from_igf_id

def _count_total_reads(data,seqrun_list):
  '''
  An internal function for counting total reads

  required params:
  data: A dictionary containing seqrun ids a key an read counts as values
  seqrun_list: A list of sequencing runs
  '''
  try:
    if len(seqrun_list) >1:
      if 'total_read' not in data:
        data['total_read']=0
        for run in seqrun_list:
          data['total_read'] += data[run]
    return data
  except:
    raise

def convert_project_data_gviz_data(input_data,output_file,
                                   sample_col='sample_igf_id',
                                   read_count_col='attribute_value',
                                   seqrun_col='flowcell_id'):
  '''
  A utility method for converting project's data availability information to
  gviz data table format
  https://developers.google.com/chart/interactive/docs/reference#DataTable

  required params:
  input_data: A pandas data frame, it should contain following columns
              sample_igf_id,
              flowcell_id,
              attribute_value (R1_READ_COUNT)
  sample_col: Column name for sample id, default sample_igf_id
  seqrun_col: Column name for sequencing run identifier, default flowcell_id
  read_count_col: Column name for sample read counts, default attribute_value
  output_file: A filepath for writing gviz json data
  '''
  try:
    if not isinstance(input_data, pd.DataFrame):
      raise AttributeError('Expecting a pandas dataframe and got {0}'.\
                           format(type(input_data)))
    input_data[read_count_col]=input_data[read_count_col].astype(int)           # convert reac counts to int
    processed_data=input_data.\
                   pivot_table(values=read_count_col,
                               index=[sample_col,
                                      seqrun_col],
                               aggfunc='sum')                                   # group data by sample id and seq runs
    processed_data.\
    reset_index([sample_col,
                 seqrun_col],
                inplace=True)                                                   # reset index for processed data
    intermediate_data=list()                                                    # define empty intermediate data structure
    seqrun_set=set()                                                            # define empty seqrun set
    for line in processed_data.to_dict(orient='records'):                       # reformat processed data to required structure
      tmp_data=dict()
      tmp_data.update({sample_col:line[sample_col],
                        line[seqrun_col]:line[read_count_col]})
      seqrun_set.add(line[seqrun_col])
      intermediate_data.append(tmp_data)

    intermediate_data=pd.DataFrame(intermediate_data)                           # convert intermediate data to dataframe
    intermediate_data.fillna(0,inplace=True)                                    # replace NAN values with zero
    intermediate_data=intermediate_data.\
                      pivot_table(index=sample_col,
                                  aggfunc='sum').\
                      reset_index(sample_col)                                   # group data by samples id
    intermediate_data=intermediate_data.\
                      apply(lambda line: \
                            _count_total_reads(data=line,
                                               seqrun_list=list(seqrun_set)),
                            axis=1)                                             # count total reads for multiple seq runs
    intermediate_data.fillna(0,inplace=True)                                    # fail safe for missing samples

    description = {sample_col: ("string", "Sample ID")}                         # define description
    if len(list(seqrun_set)) >1:
        description.update({"total_read":("number", "Total Reads")})            # add total read column for samples with multiple runs
        intermediate_data['total_read']=intermediate_data['total_read'].\
                                        astype(float)                           # convert column to number

    for run in list(seqrun_set):
        description.update({run:("number",run)})                                # add seqrun columns
        intermediate_data[run]=intermediate_data[run].\
                               astype(float)                                    # convert column to number

    intermediate_data=intermediate_data.to_dict(orient='records')                # convert data frame to json
    data_table = gviz_api.DataTable(description)                                # load description to gviz api
    data_table.LoadData(intermediate_data)                                      # load data to gviz_api
    column_list=[sample_col]                                                    # define column order
    column_list.extend(list(seqrun_set))
    if len(list(seqrun_set)) >1:
        column_list.append('total_read')                                        # total read is present only for multiple runs

    final_data=data_table.ToJSon(columns_order=tuple(column_list))              # create final data structure
    with open(output_file,'w') as jf:
        jf.write(final_data)                                                    # write final data to output file
  except:
    raise

def _modify_seqrun_data(data_series,seqrun_col,flowcell_col,path_col):
  '''
  An internal method for parsing seqrun dataframe and adding remote dir path
  required columns: seqrun_igf_id, flowcell_id
  seqrun_col: Column name for sequencing run id, default seqrun_igf_id
  flowcell_col: Column namae for flowcell id, default flowcell_id
  path_col: Column name for path, default path
  output column: flowcell_id, path
  '''
  try:
    if not isinstance(data_series,pd.Series):
      raise AttributeError('Expecting a pandas data series and got {0}'.\
                           format(type(data_series)))

    seqrun_igf_id=data_series[seqrun_col]
    flowcell_id=data_series[flowcell_col]
    seqrun_date=get_seqrun_date_from_igf_id(seqrun_igf_id)
    data_series[path_col]=os.path.join(seqrun_date,flowcell_id)                 # adding path to data series
    del data_series[seqrun_col]
    return data_series
  except:
    raise

def add_seqrun_path_info(input_data,output_file,seqrun_col='seqrun_igf_id',
                         flowcell_col='flowcell_id',path_col='path'):
  '''
  A utility method for adding remote path to a dataframe for each sequencing runs
  of a project

  required params:
  input_data: A input dataframe containing the following columns
              seqrun_igf_id
              flowcell_id
  seqrun_col: Column name for sequencing run id, default seqrun_igf_id
  flowcell_col: Column namae for flowcell id, default flowcell_id
  path_col: Column name for path, default path
  output_file: An output filepath for the json data
  '''
  try:
    if not isinstance(input_data,pd.DataFrame):
      raise AttributeError('Expecting a pandas dataframe and got {0}'.\
                           format(type(input_data)))

    input_data.drop_duplicates(inplace=True)                                    # remove duplicate entries
    input_data=input_data.\
               apply(lambda line: \
                     _modify_seqrun_data(data_series=line,
                                         seqrun_col=seqrun_col,
                                         flowcell_col=flowcell_col,
                                         path_col=path_col),
                     axis=1)                                                    # add remote seqrun path
    input_data=input_data.to_json(orient='records')                             # encode output json
    with open(output_file,'w') as j_data:
      j_data.write(input_data)                                                  # write output json file
  except:
    raise<|endoftext|>


<commit_before># coding=utf-8
from django.test import TestCase, Client
from django.core import mail
from imager_profile.tests.test_model import UserFactory
from .test_photos import PhotoFactory, AlbumFactory

class LibraryImageTestCase(TestCase):

    def setUp(self):
        self.client = Client()
        self.user = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user.profile
        )
        self.private_photo = PhotoFactory.create(
            owner=self.user.profile,
            published="CLOSED"
        )
        self.response = self.client.get('/images/library/')

    def test_view_exists(self):
        self.assertTrue(self.response.status_code == 200)

    def test_view_is_correct(self):
        self.assertTrue("
YOU ARE AT LIBRARY VIEW!
" in self.response.content.decode())

    def view_contains_images(self):
        self.assertTrue('
' in self.response.content.decode())

    def view_contains_images_except_private(self):
        self.assertFalse(self.private_photo.title in self.response.content.decode())

class AlbumViewTestCase(TestCase):

    def setUp(self):
        self.client = Client()

        self.user = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user.profile
        )
        self.album = AlbumFactory.create(
            owner=self.user.profile
        )
        self.album.add_photo(self.photo)
        self.response = self.client.get('/images/album/{}/'.format(self.album.id))

    def test_view_album_exists(self):
        self.assertEqual(self.response.status_code, 200)

    def test_view_album_model_is_correct(self):
        self.assertTrue(self.photo in self.album.photos.all())

    def test_view_album_view_is_correct(self):
        self.assertTrue(self.photo.title in self.response.content.decode())

class PhotoViewTestCase(TestCase):

    def setUp(self):
        self.client = Client()
        self.user1 = UserFactory.create()
        self.user2 = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user1.profile,
            published="CLOSED",
        )
        self.response = self.client.get('/images/photos/{0}/'.format(self.photo.id))

    def test_photo_exists(self):
        self.assertEqual(self.response.status_code, 200)

    def test_photo_appears(self):
        self.assertContains(self.response, self.photo.file.url)

    # TODO more photo tests
<commit_msg>Fixed attribute error<commit_after># coding=utf-8
from django.test import TestCase, Client
from django.core import mail
from imager_profile.tests.test_model import UserFactory
from .test_photos import PhotoFactory, AlbumFactory

class LibraryImageTestCase(TestCase):

    def setUp(self):
        self.client = Client()
        self.user = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user.profile
        )
        self.private_photo = PhotoFactory.create(
            owner=self.user.profile,
            published="CLOSED"
        )
        self.response = self.client.get('/images/library/')

    def test_view_exists(self):
        self.assertTrue(self.response.status_code == 200)

    def test_view_is_correct(self):
        self.assertTrue("
YOU ARE AT LIBRARY VIEW!
" in self.response.content.decode())

    def view_contains_images(self):
        self.assertTrue('
' in self.response.content.decode())

    def view_contains_images_except_private(self):
        self.assertFalse(self.private_photo.title in self.response.content.decode())

class AlbumViewTestCase(TestCase):

    def setUp(self):
        self.client = Client()

        self.user = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user.profile
        )
        self.album = AlbumFactory.create(
            owner=self.user.profile
        )
        self.album.add_photo(self.photo)
        self.response = self.client.get('/images/album/{}/'.format(self.album.id))

    def test_view_album_exists(self):
        self.assertEqual(self.response.status_code, 200)

    def test_view_album_model_is_correct(self):
        self.assertTrue(self.photo in self.album.photos.all())

    def test_view_album_view_is_correct(self):
        self.assertTrue(self.photo.title in self.response.content.decode())

class PhotoViewTestCase(TestCase):

    def setUp(self):
        self.client = Client()
        self.user1 = UserFactory.create()
        self.user2 = UserFactory.create()
        self.photo = PhotoFactory.create(
            owner=self.user1.profile,
            published="CLOSED",
        )
        self.response = self.client.get('/images/photos/{0}/'.format(self.photo.id))

    def test_photo_exists(self):
        self.assertEqual(self.response.status_code, 200)

    def test_photo_appears(self):
        self.assertContains(self.response, self.photo.photo.url)

    # TODO more photo tests
<|endoftext|>


<commit_before>##
##  See COPYING file distributed along with the redcap_rdf package for the
##  copyright and license terms
##
""" Data Dictionary Transformer to Semantic

@author: Victor Meyerson
"""

from rdflib import Graph, Literal, Namespace
from rdflib.namespace import DCTERMS, FOAF, RDF, RDFS, OWL, SKOS, VOID, XSD
import argparse
import csv
import os
import sys

# Header columns for data dictionary
FIELD_NAME = "Variable / Field Name"
FORM = "Form Name"
FIELD_TYPE = "Field Type"
FIELD_LABEL = "Field Label"
CHOICES = "Choices, Calculations, OR Slider Labels"
TEXT_TYPE = "Text Validation Type OR Show Slider Number"
TEXT_MIN = "Text Validation Min"
TEXT_MAX = "Text Validation Max"

# Header columns for config file
DIMENSION = "dimension"
CONCEPT = "concept"
CATEGORIES = "categories"
UNITS = "units"

class Transformer:
    def __init__(self):
        # clear internal data structures
        self._g = Graph()
        self._ns_dict = {}
        self._config_dict = {}
        self._add_prefixes()

    def build_graph(self, dd, config):
        self._build_config_lookup(config)
        if not os.path.isfile(dd):
            print("{} file not found".format(dd))
            return
        print("Processing: {}".format(dd))

        # constants
        rdf_property = self._get_ns("rdf")["Property"]
        dimension_property = self._get_ns("qb")["DimensionProperty"]
        measure_property = self._get_ns("qb")["MeasureProperty"]
        concept = self._get_ns("qb")["concept"]
        rdfs_label = self._get_ns("rdfs")["label"]
        rdfs_subPropertyOf = self._get_ns("rdfs")["subPropertyOf"]
        rdfs_range = self._get_ns("rdfs")["range"]

        with open(dd) as f:
            reader = csv.DictReader(f)
            for row in reader:
                field_name = row[FIELD_NAME]
                # TODO: Use Field Label if available else field_name.split('_')
                # and capitalize and join with a space.
                node = self._get_ns("ncanda")[field_name]
                self._g.add((node, rdfs_label, Literal(field_name)))
                prop = measure_property
                if (field_name in self._config_dict and
                        DIMENSION in self._config_dict[field_name]):
                    if self._config_dict[field_name][DIMENSION] == "y":
                        prop = dimension_property
                self._g.add((node, rdf_property, prop))
                if (field_name in self._config_dict and
                        CONCEPT in self._config_dict[field_name]):
                    obj = self._get_term(self._config_dict[field_name][CONCEPT])
                    self._g.add((node, concept, obj))
                if (field_name in self._config_dict and
                        UNITS in self._config_dict[field_name]):
                    obj = self._get_term(self._config_dict[field_name][UNITS])
                    self._g.add((node, rdfs_range, obj))

        # self._display_raw_triples(self._g)

    def display_graph(self):
        print(self._g.serialize(format='n3'))

    def _display_raw_triples(self, g):
        # Iterate over triples in store and print them out.
        print("--- printing raw triples ---")
        for s, p, o in g:
            print(s, p, o)

    def _add_prefix(self, prefix, namespace):
        ns = Namespace(namespace)
        self._g.bind(prefix, ns)
        self._ns_dict[prefix] = ns

    def _add_prefixes(self):
        self._add_prefix("owl", OWL)
        self._add_prefix("void", VOID)
        self._add_prefix("skos", SKOS)
        self._add_prefix("admingeo",
                         "http://data.ordnancesurvey.co.uk/ontology/admingeo/")
        self._add_prefix("interval",
                         "<http://reference.data.gov.uk/def/intervals/")

        self._add_prefix("qb", "http://purl.org/linked-data/cube#")

        self._add_prefix("sdmx-concept",
                         "http://purl.org/linked-data/sdmx/2009/concept#")
        self._add_prefix("sdmx-dimension",
                         "http://purl.org/linked-data/sdmx/2009/dimension#")
        self._add_prefix("sdmx-attribute",
                         "http://purl.org/linked-data/sdmx/2009/attribute#")
        self._add_prefix("sdmx-measure",
                         "http://purl.org/linked-data/sdmx/2009/measure#")
        self._add_prefix("sdmx-metadata",
                         "http://purl.org/linked-data/sdmx/2009/metadata#")
        self._add_prefix("sdmx-code",
                         "http://purl.org/linked-data/sdmx/2009/code#")
        self._add_prefix("sdmx-subject",
                         "http://purl.org/linked-data/sdmx/2009/subject#")

        self._add_prefix("ncanda", "http://ncanda.sri.com/terms.ttl#")
        self._add_prefix("fma", "http://purl.org/sig/fma#")
        self._add_prefix("prov", "http://w3c.org/ns/prov#")
        self._add_prefix("nidm", "http://purl.org/nidash/nidm#")
        self._add_prefix("fs", "http://www.incf.org/ns/nidash/fs#")

        # add in builtins
        self._add_prefix("rdf", RDF)
        self._add_prefix("rdfs", RDFS)
        self._add_prefix("xsd", XSD)
        self._add_prefix("dct", DCTERMS)
        self._add_prefix("foaf", FOAF)

        # placeholder
        self._add_prefix("ph", "file:///placeholder#")

    def _get_ns(self, prefix):
        if prefix in self._ns_dict:
            return self._ns_dict[prefix]
        return None

    def _get_term(self, term):
        parts = term.split(":")
        ns = parts[0]
        resource = parts[1]
        return self._get_ns(ns)[resource]

    def _build_config_lookup(self, config):
        if not os.path.isfile(config):
            print("{} file not found".format(config))
            return

        with open(config) as f:
            reader = csv.DictReader(f)
            for row in reader:
                # drop empty values
                res = dict((k, v) for k, v in row.iteritems() if v is not "")
                self._config_dict[row[FIELD_NAME]] = res

    # 'private' member data
    _g = Graph()
    _ns_dict = {}
    _config_dict = {}

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--datadict",
                        help="the data dictionary csv file",
                        type=str)
    parser.add_argument("-c", "--config",
                        help="a json config file",
                        type=str)
    parser.add_argument("-v", "--verbose",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()

    transformer = Transformer()
    transformer.build_graph(args.datadict, args.config)
    transformer.display_graph()

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>pep: import stdlib first<commit_after>##
##  See COPYING file distributed along with the redcap_rdf package for the
##  copyright and license terms
##
""" Data Dictionary Transformer to Semantic

@author: Victor Meyerson
"""
import os
import csv
import sys
import argparse

from rdflib import Graph, Literal, Namespace
from rdflib.namespace import DCTERMS, FOAF, RDF, RDFS, OWL, SKOS, VOID, XSD

# Header columns for data dictionary
FIELD_NAME = "Variable / Field Name"
FORM = "Form Name"
FIELD_TYPE = "Field Type"
FIELD_LABEL = "Field Label"
CHOICES = "Choices, Calculations, OR Slider Labels"
TEXT_TYPE = "Text Validation Type OR Show Slider Number"
TEXT_MIN = "Text Validation Min"
TEXT_MAX = "Text Validation Max"

# Header columns for config file
DIMENSION = "dimension"
CONCEPT = "concept"
CATEGORIES = "categories"
UNITS = "units"

class Transformer:
    def __init__(self):
        # clear internal data structures
        self._g = Graph()
        self._ns_dict = {}
        self._config_dict = {}
        self._add_prefixes()

    def build_graph(self, dd, config):
        self._build_config_lookup(config)
        if not os.path.isfile(dd):
            print("{} file not found".format(dd))
            return
        print("Processing: {}".format(dd))

        # constants
        rdf_property = self._get_ns("rdf")["Property"]
        dimension_property = self._get_ns("qb")["DimensionProperty"]
        measure_property = self._get_ns("qb")["MeasureProperty"]
        concept = self._get_ns("qb")["concept"]
        rdfs_label = self._get_ns("rdfs")["label"]
        rdfs_subPropertyOf = self._get_ns("rdfs")["subPropertyOf"]
        rdfs_range = self._get_ns("rdfs")["range"]

        with open(dd) as f:
            reader = csv.DictReader(f)
            for row in reader:
                field_name = row[FIELD_NAME]
                # TODO: Use Field Label if available else field_name.split('_')
                # and capitalize and join with a space.
                node = self._get_ns("ncanda")[field_name]
                self._g.add((node, rdfs_label, Literal(field_name)))
                prop = measure_property
                if (field_name in self._config_dict and
                        DIMENSION in self._config_dict[field_name]):
                    if self._config_dict[field_name][DIMENSION] == "y":
                        prop = dimension_property
                self._g.add((node, rdf_property, prop))
                if (field_name in self._config_dict and
                        CONCEPT in self._config_dict[field_name]):
                    obj = self._get_term(self._config_dict[field_name][CONCEPT])
                    self._g.add((node, concept, obj))
                if (field_name in self._config_dict and
                        UNITS in self._config_dict[field_name]):
                    obj = self._get_term(self._config_dict[field_name][UNITS])
                    self._g.add((node, rdfs_range, obj))

        # self._display_raw_triples(self._g)

    def display_graph(self):
        print(self._g.serialize(format='n3'))

    def _display_raw_triples(self, g):
        # Iterate over triples in store and print them out.
        print("--- printing raw triples ---")
        for s, p, o in g:
            print(s, p, o)

    def _add_prefix(self, prefix, namespace):
        ns = Namespace(namespace)
        self._g.bind(prefix, ns)
        self._ns_dict[prefix] = ns

    def _add_prefixes(self):
        self._add_prefix("owl", OWL)
        self._add_prefix("void", VOID)
        self._add_prefix("skos", SKOS)
        self._add_prefix("admingeo",
                         "http://data.ordnancesurvey.co.uk/ontology/admingeo/")
        self._add_prefix("interval",
                         "<http://reference.data.gov.uk/def/intervals/")

        self._add_prefix("qb", "http://purl.org/linked-data/cube#")

        self._add_prefix("sdmx-concept",
                         "http://purl.org/linked-data/sdmx/2009/concept#")
        self._add_prefix("sdmx-dimension",
                         "http://purl.org/linked-data/sdmx/2009/dimension#")
        self._add_prefix("sdmx-attribute",
                         "http://purl.org/linked-data/sdmx/2009/attribute#")
        self._add_prefix("sdmx-measure",
                         "http://purl.org/linked-data/sdmx/2009/measure#")
        self._add_prefix("sdmx-metadata",
                         "http://purl.org/linked-data/sdmx/2009/metadata#")
        self._add_prefix("sdmx-code",
                         "http://purl.org/linked-data/sdmx/2009/code#")
        self._add_prefix("sdmx-subject",
                         "http://purl.org/linked-data/sdmx/2009/subject#")

        self._add_prefix("ncanda", "http://ncanda.sri.com/terms.ttl#")
        self._add_prefix("fma", "http://purl.org/sig/fma#")
        self._add_prefix("prov", "http://w3c.org/ns/prov#")
        self._add_prefix("nidm", "http://purl.org/nidash/nidm#")
        self._add_prefix("fs", "http://www.incf.org/ns/nidash/fs#")

        # add in builtins
        self._add_prefix("rdf", RDF)
        self._add_prefix("rdfs", RDFS)
        self._add_prefix("xsd", XSD)
        self._add_prefix("dct", DCTERMS)
        self._add_prefix("foaf", FOAF)

        # placeholder
        self._add_prefix("ph", "file:///placeholder#")

    def _get_ns(self, prefix):
        if prefix in self._ns_dict:
            return self._ns_dict[prefix]
        return None

    def _get_term(self, term):
        parts = term.split(":")
        ns = parts[0]
        resource = parts[1]
        return self._get_ns(ns)[resource]

    def _build_config_lookup(self, config):
        if not os.path.isfile(config):
            print("{} file not found".format(config))
            return

        with open(config) as f:
            reader = csv.DictReader(f)
            for row in reader:
                # drop empty values
                res = dict((k, v) for k, v in row.iteritems() if v is not "")
                self._config_dict[row[FIELD_NAME]] = res

    # 'private' member data
    _g = Graph()
    _ns_dict = {}
    _config_dict = {}

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--datadict",
                        help="the data dictionary csv file",
                        type=str)
    parser.add_argument("-c", "--config",
                        help="a json config file",
                        type=str)
    parser.add_argument("-v", "--verbose",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()

    transformer = Transformer()
    transformer.build_graph(args.datadict, args.config)
    transformer.display_graph()

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>"""
Simple TensorFlow exercises
You should thoroughly test your code
"""

import tensorflow as tf

###############################################################################
# 1a: Create two random 0-d tensors x and y of any distribution.
# Create a TensorFlow object that returns x + y if x > y, and x - y otherwise.
# Hint: look up tf.cond()
# I do the first problem for you
###############################################################################

x = tf.random_uniform([])  # Empty array as shape creates a scalar.
y = tf.random_uniform([])
out = tf.cond(tf.less(x, y), lambda: tf.add(x, y), lambda: tf.sub(x, y))

###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from -1 and 1.
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################

# YOUR CODE

###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
###############################################################################

# YOUR CODE

###############################################################################
# 1d: Create the tensor x of value
# [29.05088806,  27.61298943,  31.19073486,  29.35532951,
#  30.97266006,  26.67541885,  38.08450317,  20.74983215,
#  34.94445419,  34.45999146,  29.06485367,  36.01657104,
#  27.88236427,  20.56035233,  30.20379066,  29.51215172,
#  33.71149445,  28.59134293,  36.05556488,  28.66994858].
# Get the indices of elements in x whose values are greater than 30.
# Hint: Use tf.where().
# Then extract elements whose values are greater than 30.
# Hint: Use tf.gather().
###############################################################################

# YOUR CODE

###############################################################################
# 1e: Create a diagnoal 2-d tensor of size 6 x 6 with the diagonal values of 1,
# 2, ..., 6
# Hint: Use tf.range() and tf.diag().
###############################################################################

# YOUR CODE

###############################################################################
# 1f: Create a random 2-d tensor of size 10 x 10 from any distribution.
# Calculate its determinant.
# Hint: Look at tf.matrix_determinant().
###############################################################################

# YOUR CODE

###############################################################################
# 1g: Create tensor x with value [5, 2, 3, 5, 10, 6, 2, 3, 4, 2, 1, 1, 0, 9].
# Return the unique elements in x
# Hint: use tf.unique(). Keep in mind that tf.unique() returns a tuple.
###############################################################################

# YOUR CODE

###############################################################################
# 1h: Create two tensors x and y of shape 300 from any normal distribution,
# as long as they are from the same distribution.
# Use tf.less() and tf.select() to return:
# - The mean squared error of (x - y) if the average of all elements in (x - y)
#   is negative, or
# - The sum of absolute value of all elements in the tensor (x - y) otherwise.
# Hint: see the Huber loss function in the lecture slides 3.
###############################################################################

# YOUR CODE<commit_msg>Warm up exercise<commit_after>"""
Simple TensorFlow exercises
You should thoroughly test your code
"""

import tensorflow as tf

###############################################################################
# 1a: Create two random 0-d tensors x and y of any distribution.
# Create a TensorFlow object that returns x + y if x > y, and x - y otherwise.
# Hint: look up tf.cond()
# I do the first problem for you
###############################################################################

x = tf.random_uniform([])  # Empty array as shape creates a scalar.
y = tf.random_uniform([])
out = tf.cond(tf.less(x, y), lambda: tf.add(x, y), lambda: tf.sub(x, y))

###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from -1 and 1.
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################

x = tf.random_uniform([],minval=-1,maxval=1)
y = tf.random_uniform([],minval=-1,maxval=1)
out = tf.case([(tf.less(x,y), lambda: x+y),
               (tf.greater(x,y), lambda: x-y)],
               default=lambda : tf.constant(0, dtype=tf.float32))

###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
###############################################################################

x = tf.Variable([[0, -2, -1], [0, 1, 2]])
y = tf.zeros_like(x)
out = tf.equal(x,y)

###############################################################################
# 1d: Create the tensor x of value
# [29.05088806,  27.61298943,  31.19073486,  29.35532951,
#  30.97266006,  26.67541885,  38.08450317,  20.74983215,
#  34.94445419,  34.45999146,  29.06485367,  36.01657104,
#  27.88236427,  20.56035233,  30.20379066,  29.51215172,
#  33.71149445,  28.59134293,  36.05556488,  28.66994858].
# Get the indices of elements in x whose values are greater than 30.
# Hint: Use tf.where().
# Then extract elements whose values are greater than 30.
# Hint: Use tf.gather().
###############################################################################

x = tf.Variable([29.05088806,  27.61298943,  31.19073486,  29.35532951,
                 30.97266006,  26.67541885,  38.08450317,  20.74983215,
                 34.94445419,  34.45999146,  29.06485367,  36.01657104,
                 27.88236427,  20.56035233,  30.20379066,  29.51215172,
                 33.71149445,  28.59134293,  36.05556488,  28.66994858])
y = tf.where(x > 30)
out = tf.gather(x,y)

###############################################################################
# 1e: Create a diagnoal 2-d tensor of size 6 x 6 with the diagonal values of 1,
# 2, ..., 6
# Hint: Use tf.range() and tf.diag().
###############################################################################

out = tf.diag(tf.range(1,7))

###############################################################################
# 1f: Create a random 2-d tensor of size 10 x 10 from any distribution.
# Calculate its determinant.
# Hint: Look at tf.matrix_determinant().
###############################################################################

x = tf.random_uniform(shape=(10,10))
out = tf.matrix_determinant(x)

###############################################################################
# 1g: Create tensor x with value [5, 2, 3, 5, 10, 6, 2, 3, 4, 2, 1, 1, 0, 9].
# Return the unique elements in x
# Hint: use tf.unique(). Keep in mind that tf.unique() returns a tuple.
###############################################################################

x = tf.Variable([5, 2, 3, 5, 10, 6, 2, 3, 4, 2, 1, 1, 0, 9])
y, idx = tf.unique(x)

###############################################################################
# 1h: Create two tensors x and y of shape 300 from any normal distribution,
# as long as they are from the same distribution.
# Use tf.less() and tf.select() to return:
# - The mean squared error of (x - y) if the average of all elements in (x - y)
#   is negative, or
# - The sum of absolute value of all elements in the tensor (x - y) otherwise.
# Hint: see the Huber loss function in the lecture slides 3.
###############################################################################

x = tf.random_uniform(shape=[300])
y = tf.random_uniform(shape=[300])
boolean = tf.less(tf.reduce_mean(x-y),tf.constant(0,dtype=tf.float32))
diff = x-y
mean_squared_error = tf.reduce_mean(tf.square(diff))
sum_abs_val = tf.reduce_sum(tf.abs(diff))
out = tf.cond(boolean,lambda : mean_squared_error,lambda : sum_abs_val)<|endoftext|>


<commit_before># Problem description:      Given two linked lists, return the node where the intersection begins, or if there's
#                           no intersection, return null.
# Solution time complexity: O(n)
# Comments:                 The solution with length offset. This solution makes use of the fact that if distance
#                           to the intersection node from headA is equal to headB, we can simply walk both lists
#                           incrementally until we encounter the intersection node.
#
#                           How do we do this? First, we walk both lists and find their lengths. Then we move the
#                           node of the bigger list as many times as required to make the count of nodes to the
#                           end equal in both lists. Now we can simply traverse both lists at the same time incre-
#                           mentally until we encounter the intersection node.

# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    @staticmethod
    def computeLinkedListLength(head):
        count = 0
        node  = head

        while node != None:
            count += 1
            node   = node.next

        return count

    def getIntersectionNode(self, headA, headB):
        """
        :type head1, head1: ListNode
        :rtype: ListNode
        """

        if headA == None or headB == None:
            return None

        lengthA = Solution.computeLinkedListLength(headA)
        lengthB = Solution.computeLinkedListLength(headB)

        nodeA   = headA
        nodeB   = headB

        countA  = lengthA
        countB  = lengthB

        while countA > countB:
            countA -= 1
            nodeA   = nodeA.next

        while countB > countA:
            countB -= 1
            nodeB   = nodeB.next

        while nodeA != None and nodeB != None:
            if nodeA == nodeB:
                return nodeA
            else:
                nodeA = nodeA.next
                nodeB = nodeB.next

        return None
<commit_msg>Update solution3.py<commit_after># Problem description:      Given two linked lists, return the node where the intersection begins, or if there's
#                           no intersection, return null.
# Solution time complexity: O(n)
# Comments:                 The solution with length offset. This solution makes use of the fact that if distance
#                           to the intersection node from headA is equal to headB, we can simply walk both lists
#                           incrementally until we encounter the intersection node.
#
#                           How do we do this? First, we walk both lists and find their lengths. Then we move the
#                           node of the bigger list as many times as required to make the count of nodes to the
#                           end equal in both lists. Now we can simply traverse both lists at the same time incre-
#                           mentally until we encounter the intersection node.
#
#                               a1->a2
#                                     \
#                                      c1->c2->c3
#                                     /
#                           b1->b2->b3
#
#
#                           i)
#                                  s
#                             x |------|
#                           |---|
#                               a1->a2
#                                     \
#                                      c1->c2->c3
#                                     /
#                           b1->b2->b3
#
#                           ii)
#                             x      s
#                           |---||------|
#                                      len1
#                               |----------------|
#                               a1->a2->c1->c2->c3
#                           b1->b2->b3->c1->c2->c3
#                            |-------------------|
#                                     len2
#
#                           x = abs(len1 - len2)
#
#                           iii)
#                               a1->a2                        o->a2
#                                     \                            \
#                                      c1->c2->c3                   c1->c2->c3
#                                     /                            /
#                               b2->b3                        o->b3
#
#                               o->o
#                                   \
#                                    c1 <-- intersection node
#                                   /
#                               o->o
#

# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    @staticmethod
    def computeLinkedListLength(head):
        count = 0
        node  = head

        while node != None:
            count += 1
            node   = node.next

        return count

    def getIntersectionNode(self, headA, headB):
        """
        :type head1, head1: ListNode
        :rtype: ListNode
        """

        if headA == None or headB == None:
            return None

        lengthA = Solution.computeLinkedListLength(headA)
        lengthB = Solution.computeLinkedListLength(headB)

        nodeA   = headA
        nodeB   = headB

        countA  = lengthA
        countB  = lengthB

        while countA > countB:
            countA -= 1
            nodeA   = nodeA.next

        while countB > countA:
            countB -= 1
            nodeB   = nodeB.next

        while nodeA != None and nodeB != None:
            if nodeA == nodeB:
                return nodeA
            else:
                nodeA = nodeA.next
                nodeB = nodeB.next

        return None
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

from datetime import date

from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.six.moves.urllib_parse import urljoin

import requests

from candidates.models import (
    AreaExtra, OrganizationExtra, PartySet, PostExtra, PostExtraElection
)
from elections.models import AreaType, Election
from elections.uk import mapit
from popolo.models import Area, Organization, Post

class Command(BaseCommand):

    help = 'Create posts and elections for the 2016 Scottish Parliament elections'

    def handle(self, **options):

        mapit_url = settings.MAPIT_BASE_URL

        gb_parties, _ = PartySet.objects.get_or_create(
            slug='gb', defaults={'name': 'Great Britain'}
        )
        ni_parties, _ = PartySet.objects.get_or_create(
            slug='ni', defaults={'name': 'Northern Ireland'}
        )

        elections = {
            'sp-2016-05-05-r': {
                'name': '2016 Scottish Parliament Election (Regions)',
                'for_post_role': 'Member of the Scottish Parliament',
                'label_format': 'Member of the Scottish Parliament for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': True,
                'mapit_code': 'SPE',
                'area_type_description': 'Scottish Parliament region',
                'organization_slug': 'scottish-parliament',
                'organization_name': 'Scottish Parliament',
            },
            'sp-2016-05-05-c': {
                'name': '2016 Scottish Parliament Election (Constituencies)',
                'for_post_role': 'Member of the Scottish Parliament',
                'label_format': 'Member of the Scottish Parliament for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'SPC',
                'area_type_description': 'Scottish Parliament constituency',
                'organization_slug': 'scottish-parliament',
                'organization_name': 'Scottish Parliament',
            },
            'naw-2016-05-05-r': {
                'name': '2016 Welsh Assembly Election (Regions)',
                'for_post_role': 'Member of the National Assembly for Wales',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': True,
                'mapit_code': 'WAE',
                'area_type_description': 'Welsh Assembly region',
                'organization_slug': 'welsh-assembly',
                'organization_name': 'National Assembly for Wales',
            },
            'naw-2016-05-05-c': {
                'name': '2016 Welsh Assembly Election (Constituencies)',
                'for_post_role': 'Member of the National Assembly for Wales',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'WAC',
                'area_type_description': 'Welsh Assembly constituency',
                'organization_slug': 'welsh-assembly',
                'organization_name': 'National Assembly for Wales',
            },
            'nia-2016-05-05': {
                'name': '2016 Northern Ireland Assembly Election',
                'for_post_role': 'Member of the Legislative Assembly',
                'label_format': 'Member of the Legislative Assembly for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'NIE',
                'area_type_description': 'Northern Ireland Assembly constituency',
                'organization_slug': 'northern-ireland-assembly',
                'organization_name': 'Northern Ireland Assembly',
            },
            'gla-2016-05-05-c': {
                'name': '2016 London Assembly Election (Constituencies)',
                'for_post_role': 'Member of the London Assembly',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'LAC',
                'area_type_description': 'London Assembly constituency',
                'organization_slug': 'london-assembly',
                'organization_name': 'London Assembly',
            },
            'gla-2016-05-05-a': {
                'name': '2016 London Assembly Election (Additional)',
                'for_post_role': 'Member of the London Assembly',
                'label_format': 'Assembly Member',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'GLA',
                'area_type_description': 'London Assembly constituency',
                'organization_slug': 'london-assembly',
                'organization_name': 'London Assembly',
            },
        }

        for election_slug, data in elections.items():
            # Make sure the parliament Organization and
            # OrganizationExtra objects exist:
            try:
                organization_extra = OrganizationExtra.objects.get(
                    slug=data['organization_slug']
                )
                organization = organization_extra.base
            except OrganizationExtra.DoesNotExist:
                organization = Organization.objects.create(
                    name=data['organization_name']
                )
                organization_extra = OrganizationExtra.objects.create(
                    base=organization,
                    slug=data['organization_slug']
                )
            # Make sure the Election object exists:
            election_defaults = {
                k: data[k] for k in
                [
                    'name', 'for_post_role', 'area_generation', 'election_date',
                    'party_lists_in_use',
                ]
            }
            election_defaults['current'] = True
            election_defaults['candidate_membership_role'] = 'Candidate'
            print('Creating:', election_defaults['name'], '...',)
            election, created = Election.objects.update_or_create(
                slug=election_slug,
                defaults=election_defaults
            )
            if created:
                print('[created]')
            else:
                print('[already existed]')

            area_type, _ = AreaType.objects.update_or_create(
                name=data['mapit_code'], defaults={'source': 'MapIt'}
            )
            if not election.area_types.filter(name=area_type.name).exists():
                election.area_types.add(area_type)
            url_path = '/areas/' + data['mapit_code']
            url = urljoin(mapit_url, url_path)
            r = requests.get(url)
            for mapit_area_id, mapit_area_data in r.json().items():

                area, _ = Area.objects.update_or_create(
                    identifier=str(mapit.format_code_from_area(mapit_area_data)),
                    defaults={'name': mapit_area_data['name']}
                )
                # Now make sure that the MapIt codes are present as identifiers:
                for scheme, identifier in mapit_area_data['codes'].items():
                    area.other_identifiers.update_or_create(
                        scheme=scheme,
                        defaults={'identifier': identifier},
                    )

                AreaExtra.objects.get_or_create(base=area, type=area_type)
                post, _ = Post.objects.update_or_create(
                    organization=organization,
                    area=area,
                    role='Member of the Scottish Parliament',
                    defaults={
                        'label': data['label_format'].format(
                            area_name=area.name
                        )
                    }
                )
                post_extra, _ = PostExtra.objects.update_or_create(
                    base=post,
                    defaults={
                        'slug': str(mapit.format_code_from_area(mapit_area_data)),
                        'party_set': gb_parties,
                    },
                )
                PostExtraElection.objects.update_or_create(
                    postextra=post_extra,
                    election=election
                )
<commit_msg>Fix regional election creation script to use the NI party set in NI<commit_after>from __future__ import print_function, unicode_literals

from datetime import date

from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils.six.moves.urllib_parse import urljoin

import requests

from candidates.models import (
    AreaExtra, OrganizationExtra, PartySet, PostExtra, PostExtraElection
)
from elections.models import AreaType, Election
from elections.uk import mapit
from popolo.models import Area, Organization, Post

class Command(BaseCommand):

    help = 'Create posts and elections for the 2016 Scottish Parliament elections'

    def handle(self, **options):

        mapit_url = settings.MAPIT_BASE_URL

        gb_parties, _ = PartySet.objects.get_or_create(
            slug='gb', defaults={'name': 'Great Britain'}
        )
        ni_parties, _ = PartySet.objects.get_or_create(
            slug='ni', defaults={'name': 'Northern Ireland'}
        )

        elections = {
            'sp-2016-05-05-r': {
                'name': '2016 Scottish Parliament Election (Regions)',
                'for_post_role': 'Member of the Scottish Parliament',
                'label_format': 'Member of the Scottish Parliament for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': True,
                'mapit_code': 'SPE',
                'area_type_description': 'Scottish Parliament region',
                'organization_slug': 'scottish-parliament',
                'organization_name': 'Scottish Parliament',
            },
            'sp-2016-05-05-c': {
                'name': '2016 Scottish Parliament Election (Constituencies)',
                'for_post_role': 'Member of the Scottish Parliament',
                'label_format': 'Member of the Scottish Parliament for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'SPC',
                'area_type_description': 'Scottish Parliament constituency',
                'organization_slug': 'scottish-parliament',
                'organization_name': 'Scottish Parliament',
            },
            'naw-2016-05-05-r': {
                'name': '2016 Welsh Assembly Election (Regions)',
                'for_post_role': 'Member of the National Assembly for Wales',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': True,
                'mapit_code': 'WAE',
                'area_type_description': 'Welsh Assembly region',
                'organization_slug': 'welsh-assembly',
                'organization_name': 'National Assembly for Wales',
            },
            'naw-2016-05-05-c': {
                'name': '2016 Welsh Assembly Election (Constituencies)',
                'for_post_role': 'Member of the National Assembly for Wales',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'WAC',
                'area_type_description': 'Welsh Assembly constituency',
                'organization_slug': 'welsh-assembly',
                'organization_name': 'National Assembly for Wales',
            },
            'nia-2016-05-05': {
                'name': '2016 Northern Ireland Assembly Election',
                'for_post_role': 'Member of the Legislative Assembly',
                'label_format': 'Member of the Legislative Assembly for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'NIE',
                'area_type_description': 'Northern Ireland Assembly constituency',
                'organization_slug': 'northern-ireland-assembly',
                'organization_name': 'Northern Ireland Assembly',
            },
            'gla-2016-05-05-c': {
                'name': '2016 London Assembly Election (Constituencies)',
                'for_post_role': 'Member of the London Assembly',
                'label_format': 'Assembly Member for {area_name}',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'LAC',
                'area_type_description': 'London Assembly constituency',
                'organization_slug': 'london-assembly',
                'organization_name': 'London Assembly',
            },
            'gla-2016-05-05-a': {
                'name': '2016 London Assembly Election (Additional)',
                'for_post_role': 'Member of the London Assembly',
                'label_format': 'Assembly Member',
                'area_generation': 22,
                'election_date': date(2016, 5, 5),
                'party_lists_in_use': False,
                'mapit_code': 'GLA',
                'area_type_description': 'London Assembly constituency',
                'organization_slug': 'london-assembly',
                'organization_name': 'London Assembly',
            },
        }

        for election_slug, data in elections.items():
            # Make sure the parliament Organization and
            # OrganizationExtra objects exist:
            try:
                organization_extra = OrganizationExtra.objects.get(
                    slug=data['organization_slug']
                )
                organization = organization_extra.base
            except OrganizationExtra.DoesNotExist:
                organization = Organization.objects.create(
                    name=data['organization_name']
                )
                organization_extra = OrganizationExtra.objects.create(
                    base=organization,
                    slug=data['organization_slug']
                )
            # Make sure the Election object exists:
            election_defaults = {
                k: data[k] for k in
                [
                    'name', 'for_post_role', 'area_generation', 'election_date',
                    'party_lists_in_use',
                ]
            }
            election_defaults['current'] = True
            election_defaults['candidate_membership_role'] = 'Candidate'
            print('Creating:', election_defaults['name'], '...',)
            election, created = Election.objects.update_or_create(
                slug=election_slug,
                defaults=election_defaults
            )
            if created:
                print('[created]')
            else:
                print('[already existed]')

            area_type, _ = AreaType.objects.update_or_create(
                name=data['mapit_code'], defaults={'source': 'MapIt'}
            )
            if not election.area_types.filter(name=area_type.name).exists():
                election.area_types.add(area_type)
            url_path = '/areas/' + data['mapit_code']
            url = urljoin(mapit_url, url_path)
            r = requests.get(url)
            for mapit_area_id, mapit_area_data in r.json().items():

                area, _ = Area.objects.update_or_create(
                    identifier=str(mapit.format_code_from_area(mapit_area_data)),
                    defaults={'name': mapit_area_data['name']}
                )
                # Now make sure that the MapIt codes are present as identifiers:
                for scheme, identifier in mapit_area_data['codes'].items():
                    area.other_identifiers.update_or_create(
                        scheme=scheme,
                        defaults={'identifier': identifier},
                    )

                AreaExtra.objects.get_or_create(base=area, type=area_type)
                post, _ = Post.objects.update_or_create(
                    organization=organization,
                    area=area,
                    role='Member of the Scottish Parliament',
                    defaults={
                        'label': data['label_format'].format(
                            area_name=area.name
                        )
                    }
                )
                if mapit_area_data['country_name'] == 'Northern Ireland':
                    party_set = ni_parties
                else:
                    party_set = gb_parties
                post_extra, _ = PostExtra.objects.update_or_create(
                    base=post,
                    defaults={
                        'slug': str(mapit.format_code_from_area(mapit_area_data)),
                        'party_set': party_set,
                    },
                )
                PostExtraElection.objects.update_or_create(
                    postextra=post_extra,
                    election=election
                )
<|endoftext|>


<commit_before><commit_msg>Convert examples in bysize to doctest.<commit_after><|endoftext|>


<commit_before><commit_msg>Bump version 0.17.0rc6 --> 0.17.0rc7<commit_after><|endoftext|>


<commit_before>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import StringIO

import sqlite3
from lib import openpgpfile, openpgpkey
import StringIO

class OpenPGPKeyTable:
    def __init__(self, db):
        self.db = db
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if "PGPKeys" not in tables:
            cu.execute("""CREATE TABLE PGPKeys(keyId INTEGER,
                                               userId INTEGER,
                                               fingerprint STRING(40) UNIQUE,
                                               pgpKey BINARY,
                                               PRIMARY KEY(keyId))""")
        if "PGPFingerprints" not in tables:
            cu.execute("""CREATE TABLE PGPFingerprints(
                                             keyId INTEGER,
                                             fingerprint STRING(40),
                                             PRIMARY KEY(fingerprint))""")
        db.commit()

        # set the global key cache to refer to the keys in the database
        keyCache = OpenPGPKeyDBCache(self)
        openpgpkey.setKeyCache(keyCache)

    def getFingerprint(self, keyId):
        cu = self.db.cursor()
        r = cu.execute('SELECT fingerprint FROM PGPFingerprints '
                       'WHERE fingerprint LIKE "%' + keyId + '%"')
        keyList = r.fetchall()
        if (len(keyList) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        return keyList[0][0]

    def addNewAsciiKey(self, userId, asciiData):
        keyData = openpgpfile.parseAsciiArmorKey(asciiData)
        if not keyData:
            raise IncompatibleKey('Unable to parse ASCII armored key')
        self.addNewKey(userId, keyData)

    def addNewKey(self, userId, pgpKeyData):
        cu = self.db.cursor()
        # start a transaction so that our SELECT is protected against
        # race conditions
        self.db._begin()
        r = cu.execute('SELECT IFNULL(MAX(keyId),0) + 1 FROM PGPKeys')
        keyId = r.fetchone()[0]
        keyRing = StringIO.StringIO(pgpKeyData)

        # make sure it's a public key
        keyType = openpgpfile.readBlockType(keyRing)
        keyRing.seek(-1,1)
        if (keyType >> 2) & 15 != openpgpfile.PKT_PUBLIC_KEY:
            raise IncompatibleKey('Key must be a public key')

        if openpgpfile.countKeys(keyRing) != 1:
            raise IncompatibleKey('Submit only one key at a time.')

        limit = len(pgpKeyData)
        while keyRing.tell() < limit:
            openpgpfile.verifySelfSignatures(openpgpfile.getKeyId(keyRing), keyRing)
            openpgpfile.seekNextKey(keyRing)
        keyRing.seek(0)

        mainFingerprint = openpgpfile.getKeyId(keyRing)

        # if this key already exists we need to ensure it's safe to overwrite
        # the old one, and then just do it.
        r = cu.execute('SELECT pgpKey FROM PGPKeys WHERE fingerprint=?',mainFingerprint)
        origKey = cu.fetchone()
        if origKey:
            # ensure new key is a superset of old key. we can't allow the repo
            # to let go of subkeys or revocations.
            openpgpfile.assertReplaceKeyAllowed(origKey[0], pgpKeyData)
            #reset the key cache so the changed key shows up
            keyCache = openpgpkey.getKeyCache()
            keyCache.reset()
        try:
            cu.execute('INSERT INTO PGPKeys VALUES(?, ?, ?, ?)',
                       (keyId, userId, mainFingerprint, pgpKeyData))
        except sqlite3.ProgrammingError:
            # controlled replacement of OpenPGP Keys is allowed. do NOT disable
            # assertReplaceKeyAllowed without disabling this
            cu.execute('UPDATE PGPKeys set pgpKey=? where fingerprint=?',
                       (pgpKeyData, mainFingerprint))
        keyFingerprints = openpgpfile.getFingerprints(keyRing)
        for fingerprint in keyFingerprints:
            try:
                cu.execute('INSERT INTO PGPFingerprints VALUES(?, ?)',
                       (keyId, fingerprint))
            except sqlite3.ProgrammingError:
                # ignore duplicate fingerprint errors.
                pass
        self.db.commit()
        keyRing.close()

    # to be used only with extreme caution. it can damage the repository.
    def deleteKey(self, keyId):
        fingerprint = self.getFingerprint(keyId)
        cu = self.db.cursor()
        cu.execute('SELECT keyId FROM PGPFingerprints '
                       'WHERE fingerprint=?', (fingerprint,))
        keyIds = cu.fetchall()
        if (len(keyIds) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        keyId = keyIds[0]
        cu.execute('DELETE FROM PGPFingerprints WHERE keyId=?', (keyId,))
        cu.execute('DELETE FROM PGPKeys WHERE keyId=?', (keyId,))
        self.db.commit()

    def getPGPKeyData(self, keyId):
        cu = self.db.cursor()
        cu.execute("""SELECT pgpKey FROM PGPKeys
                           LEFT JOIN PGPFingerprints ON
                             PGPKeys.keyId=PGPFingerprints.keyId
                        WHERE PGPFingerprints.fingerprint like "%%%s%%"
                        """ %keyId)
        keys = cu.fetchall()
        if (len(keys) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        return keys[0][0]

    def getUsersMainKeys(self, userId):
        cu = self.db.cursor()
        r = cu.execute('SELECT fingerprint FROM PGPKeys '
                       'WHERE userId=?', (userId,))
        return [ x[0] for x in r.fetchall() ]

    def getSubkeys(self, fingerprint):
        cu = self.db.cursor()
        r = cu.execute("""
                SELECT PGPFingerprints.fingerprint
                    FROM PGPFingerprints LEFT JOIN PGPKeys USING(keyid)
                WHERE PGPKeys.fingerprint=?
                     AND PGPFingerprints.fingerprint != PGPKeys.fingerprint""",
                       (fingerprint,))
        return [ x[0] for x in r.fetchall() ]

class OpenPGPKeyDBCache(openpgpkey.OpenPGPKeyCache):
    def __init__(self, keyTable = None):
        openpgpkey.OpenPGPKeyCache.__init__(self)
        self.keyTable = keyTable

    def setKeyTable(self, keyTable):
        self.keyTable = keyTable

    def getPublicKey(self, keyId):
        if keyId in self.publicDict:
            return self.publicDict[keyId]

        if self.keyTable is None:
            raise KeyNotFound(keyId, "Can't open database")

        # get the key data from the database
        fingerprint = self.keyTable.getFingerprint(keyId)
        keyData = self.keyTable.getPGPKeyData(keyId)

        # instantiate the crypto key object from the raw key data
        cryptoKey = openpgpfile.getPublicKeyFromString(keyId, keyData)

        # populate the cache
        self.publicDict[keyId] = openpgpkey.OpenPGPKey(fingerprint, cryptoKey)
        return self.publicDict[keyId]

<commit_msg>Fix a few improperly referenced exceptions, and qualify the update to only happen when the key is a duplicate<commit_after>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import StringIO

import sqlite3
from lib import openpgpfile, openpgpkey
import StringIO

class OpenPGPKeyTable:
    def __init__(self, db):
        self.db = db
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if "PGPKeys" not in tables:
            cu.execute("""CREATE TABLE PGPKeys(keyId INTEGER,
                                               userId INTEGER,
                                               fingerprint STRING(40) UNIQUE,
                                               pgpKey BINARY,
                                               PRIMARY KEY(keyId))""")
        if "PGPFingerprints" not in tables:
            cu.execute("""CREATE TABLE PGPFingerprints(
                                             keyId INTEGER,
                                             fingerprint STRING(40),
                                             PRIMARY KEY(fingerprint))""")
        db.commit()

        # set the global key cache to refer to the keys in the database
        keyCache = OpenPGPKeyDBCache(self)
        openpgpkey.setKeyCache(keyCache)

    def getFingerprint(self, keyId):
        cu = self.db.cursor()
        r = cu.execute('SELECT fingerprint FROM PGPFingerprints '
                       'WHERE fingerprint LIKE "%' + keyId + '%"')
        keyList = r.fetchall()
        if (len(keyList) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        return keyList[0][0]

    def addNewAsciiKey(self, userId, asciiData):
        keyData = openpgpfile.parseAsciiArmorKey(asciiData)
        if not keyData:
            raise openpgpfile.IncompatibleKey('Unable to parse ASCII armored key')
        self.addNewKey(userId, keyData)

    def addNewKey(self, userId, pgpKeyData):
        cu = self.db.cursor()
        # start a transaction so that our SELECT is protected against
        # race conditions
        self.db._begin()
        r = cu.execute('SELECT IFNULL(MAX(keyId),0) + 1 FROM PGPKeys')
        keyId = r.fetchone()[0]
        keyRing = StringIO.StringIO(pgpKeyData)

        # make sure it's a public key
        keyType = openpgpfile.readBlockType(keyRing)
        keyRing.seek(-1,1)
        if (keyType >> 2) & 15 != openpgpfile.PKT_PUBLIC_KEY:
            raise openpgpfile.IncompatibleKey('Key must be a public key')

        if openpgpfile.countKeys(keyRing) != 1:
            raise openpgpfile.IncompatibleKey('Submit only one key at a time.')

        limit = len(pgpKeyData)
        while keyRing.tell() < limit:
            openpgpfile.verifySelfSignatures(openpgpfile.getKeyId(keyRing), keyRing)
            openpgpfile.seekNextKey(keyRing)
        keyRing.seek(0)

        mainFingerprint = openpgpfile.getKeyId(keyRing)

        # if this key already exists we need to ensure it's safe to overwrite
        # the old one, and then just do it.
        r = cu.execute('SELECT pgpKey FROM PGPKeys WHERE fingerprint=?',mainFingerprint)
        origKey = cu.fetchone()
        if origKey:
            # ensure new key is a superset of old key. we can't allow the repo
            # to let go of subkeys or revocations.
            openpgpfile.assertReplaceKeyAllowed(origKey[0], pgpKeyData)
            #reset the key cache so the changed key shows up
            keyCache = openpgpkey.getKeyCache()
            keyCache.reset()
        try:
            cu.execute('INSERT INTO PGPKeys VALUES(?, ?, ?, ?)',
                       (keyId, userId, mainFingerprint, pgpKeyData))
        except sqlite3.ProgrammingError, e:
            # controlled replacement of OpenPGP Keys is allowed. do NOT disable
            # assertReplaceKeyAllowed without disabling this
            if e.args[0].startswith("column") and e.args[0].endswith("is not unique"):
                cu.execute('UPDATE PGPKeys set pgpKey=? where fingerprint=?',
                       (pgpKeyData, mainFingerprint))
            else:
                raise
        keyFingerprints = openpgpfile.getFingerprints(keyRing)
        for fingerprint in keyFingerprints:
            try:
                cu.execute('INSERT INTO PGPFingerprints VALUES(?, ?)',
                       (keyId, fingerprint))
            except sqlite3.ProgrammingError:
                # ignore duplicate fingerprint errors.
                pass
        self.db.commit()
        keyRing.close()

    # to be used only with extreme caution. it can damage the repository.
    def deleteKey(self, keyId):
        fingerprint = self.getFingerprint(keyId)
        cu = self.db.cursor()
        cu.execute('SELECT keyId FROM PGPFingerprints '
                       'WHERE fingerprint=?', (fingerprint,))
        keyIds = cu.fetchall()
        if (len(keyIds) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        keyId = keyIds[0]
        cu.execute('DELETE FROM PGPFingerprints WHERE keyId=?', (keyId,))
        cu.execute('DELETE FROM PGPKeys WHERE keyId=?', (keyId,))
        self.db.commit()

    def getPGPKeyData(self, keyId):
        cu = self.db.cursor()
        cu.execute("""SELECT pgpKey FROM PGPKeys
                           LEFT JOIN PGPFingerprints ON
                             PGPKeys.keyId=PGPFingerprints.keyId
                        WHERE PGPFingerprints.fingerprint like "%%%s%%"
                        """ %keyId)
        keys = cu.fetchall()
        if (len(keys) != 1):
            raise openpgpkey.KeyNotFound(keyId)
        return keys[0][0]

    def getUsersMainKeys(self, userId):
        cu = self.db.cursor()
        r = cu.execute('SELECT fingerprint FROM PGPKeys '
                       'WHERE userId=?', (userId,))
        return [ x[0] for x in r.fetchall() ]

    def getSubkeys(self, fingerprint):
        cu = self.db.cursor()
        r = cu.execute("""
                SELECT PGPFingerprints.fingerprint
                    FROM PGPFingerprints LEFT JOIN PGPKeys USING(keyid)
                WHERE PGPKeys.fingerprint=?
                     AND PGPFingerprints.fingerprint != PGPKeys.fingerprint""",
                       (fingerprint,))
        return [ x[0] for x in r.fetchall() ]

class OpenPGPKeyDBCache(openpgpkey.OpenPGPKeyCache):
    def __init__(self, keyTable = None):
        openpgpkey.OpenPGPKeyCache.__init__(self)
        self.keyTable = keyTable

    def setKeyTable(self, keyTable):
        self.keyTable = keyTable

    def getPublicKey(self, keyId):
        if keyId in self.publicDict:
            return self.publicDict[keyId]

        if self.keyTable is None:
            raise KeyNotFound(keyId, "Can't open database")

        # get the key data from the database
        fingerprint = self.keyTable.getFingerprint(keyId)
        keyData = self.keyTable.getPGPKeyData(keyId)

        # instantiate the crypto key object from the raw key data
        cryptoKey = openpgpfile.getPublicKeyFromString(keyId, keyData)

        # populate the cache
        self.publicDict[keyId] = openpgpkey.OpenPGPKey(fingerprint, cryptoKey)
        return self.publicDict[keyId]

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Helper functions for file operations"""
from __future__ import absolute_import, division, unicode_literals

import os

import xbmc
import xbmcvfs

from resources.lib.globals import g

def check_folder_path(path):
    """
    Check if folderpath ends with path delimator
    If not correct it (makes sure xbmcvfs.exists is working correct)
    """
    end = ''
    if '/' in path and not path.endswith('/'):
        end = '/'
    if '\\' in path and not path.endswith('\\'):
        end = '\\'
    return path + end

def file_exists(filename, data_path=g.DATA_PATH):
    """
    Checks if a given file exists
    :param filename: The filename
    :return: True if exists
    """
    return xbmcvfs.exists(xbmc.translatePath(os.path.join(data_path, filename)))

def copy_file(from_path, to_path):
    """
    Copy a file to destination
    :param from_path: File path to copy
    :param to_path: Destination file path
    :return: True if copied
    """
    try:
        return xbmcvfs.copy(xbmc.translatePath(from_path),
                            xbmc.translatePath(to_path))
    finally:
        pass

def save_file(filename, content, mode='wb'):
    """
    Saves the given content under given filename
    :param filename: The filename
    :param content: The content of the file
    """
    file_handle = xbmcvfs.File(
        xbmc.translatePath(os.path.join(g.DATA_PATH, filename)), mode)
    try:
        file_handle.write(bytearray(content))
    finally:
        file_handle.close()

def load_file(filename, mode='rb'):
    """
    Loads the content of a given filename
    :param filename: The file to load
    :return: The content of the file
    """
    file_handle = xbmcvfs.File(
        xbmc.translatePath(os.path.join(g.DATA_PATH, filename)), mode)
    try:
        return file_handle.readBytes().decode('utf-8')
    finally:
        file_handle.close()

def delete_file(filename):
    file_path = xbmc.translatePath(os.path.join(g.DATA_PATH, filename))
    try:
        xbmcvfs.delete(file_path)
    finally:
        pass

def list_dir(data_path=g.DATA_PATH):
    """
    List the contents of a folder
    :return: The contents of the folder
    """
    return xbmcvfs.listdir(xbmc.translatePath(data_path))

def delete_folder_contents(path):
    """Delete all files in a folder"""
    for filename in list_dir(path)[1]:
        xbmcvfs.delete(os.path.join(path, filename))

def delete_ndb_files(data_path=g.DATA_PATH):
    """Delete all .ndb files in a folder"""
    for filename in list_dir(data_path)[1]:
        if filename.endswith('.ndb'):
            xbmcvfs.delete(os.path.join(g.DATA_PATH, filename))
<commit_msg>Added delete subfolders to delete_folder_contents<commit_after># -*- coding: utf-8 -*-
"""Helper functions for file operations"""
from __future__ import absolute_import, division, unicode_literals

import os

import xbmc
import xbmcvfs

from resources.lib.globals import g

def check_folder_path(path):
    """
    Check if folderpath ends with path delimator
    If not correct it (makes sure xbmcvfs.exists is working correct)
    """
    end = ''
    if '/' in path and not path.endswith('/'):
        end = '/'
    if '\\' in path and not path.endswith('\\'):
        end = '\\'
    return path + end

def file_exists(filename, data_path=g.DATA_PATH):
    """
    Checks if a given file exists
    :param filename: The filename
    :return: True if exists
    """
    return xbmcvfs.exists(xbmc.translatePath(os.path.join(data_path, filename)))

def copy_file(from_path, to_path):
    """
    Copy a file to destination
    :param from_path: File path to copy
    :param to_path: Destination file path
    :return: True if copied
    """
    try:
        return xbmcvfs.copy(xbmc.translatePath(from_path),
                            xbmc.translatePath(to_path))
    finally:
        pass

def save_file(filename, content, mode='wb'):
    """
    Saves the given content under given filename
    :param filename: The filename
    :param content: The content of the file
    """
    file_handle = xbmcvfs.File(
        xbmc.translatePath(os.path.join(g.DATA_PATH, filename)), mode)
    try:
        file_handle.write(bytearray(content))
    finally:
        file_handle.close()

def load_file(filename, mode='rb'):
    """
    Loads the content of a given filename
    :param filename: The file to load
    :return: The content of the file
    """
    file_handle = xbmcvfs.File(
        xbmc.translatePath(os.path.join(g.DATA_PATH, filename)), mode)
    try:
        return file_handle.readBytes().decode('utf-8')
    finally:
        file_handle.close()

def delete_file(filename):
    file_path = xbmc.translatePath(os.path.join(g.DATA_PATH, filename))
    try:
        xbmcvfs.delete(file_path)
    finally:
        pass

def list_dir(data_path=g.DATA_PATH):
    """
    List the contents of a folder
    :return: The contents of the folder
    """
    return xbmcvfs.listdir(xbmc.translatePath(data_path))

def delete_folder_contents(path, delete_subfolders=False):
    """
    Delete all files in a folder
    :param path: Path to perform delete contents
    :param delete_subfolders: If True delete also all subfolders
    """
    directories, files = list_dir(path)
    for filename in files:
        xbmcvfs.delete(os.path.join(path, filename))
    if not delete_subfolders:
        return
    for directory in directories:
        delete_folder_contents(os.path.join(path, directory), True)
        # Give time because the system performs previous op. otherwise it can't delete the folder
        xbmc.sleep(80)
        xbmcvfs.rmdir(os.path.join(path, directory))

def delete_ndb_files(data_path=g.DATA_PATH):
    """Delete all .ndb files in a folder"""
    for filename in list_dir(data_path)[1]:
        if filename.endswith('.ndb'):
            xbmcvfs.delete(os.path.join(g.DATA_PATH, filename))
<|endoftext|>


<commit_before>import sys
import subprocess
import os

buildempath = sys.argv[1]
cmakepath = sys.argv[2]

goldenprefix = cmakepath + "/integration_tests/outputs/"
testoutprefix = cmakepath + "/integration_tests/temp_data/"

def compare_outputs(exe_string, stdoutfile, file_comps=None):
    exe_string = exe_string.replace("${BUILDEM_DIR}", buildempath)
    exe_string = exe_string.replace("${CMAKE_SOURCE_DIR}", cmakepath)

    if not os.path.exists(testoutprefix):
        os.makedirs(testoutprefix)

    golden_stdout_name = goldenprefix + stdoutfile
    testout_stdout_name = testoutprefix + stdoutfile

    p = subprocess.Popen(exe_string.split(), stdout=subprocess.PIPE)

    testout_stdout, err = p.communicate()

    # error returned from call
    if err is not None:
        exit(1)

    testout_stdout_arr = testout_stdout.split('\n')
    golden_stdout = open(golden_stdout_name).read()

    testout_stdout = ""
    for iter1 in range(0, len(testout_stdout_arr)):
        line = testout_stdout_arr[iter1]
        if "Time " in line:
            continue
        testout_stdout += line
        if iter1 != (len(testout_stdout_arr)-1):
            testout_stdout += "\n"

    fout = open(testout_stdout_name, 'w')
    fout.write(testout_stdout)
    fout.close()

    if testout_stdout != golden_stdout:
        exit(1)

    if file_comps is not None:
        for file_comp in file_comps:
            fing = open(goldenprefix + file_comp)
            fint = open(testoutprefix + file_comp)

            g_out = fing.read()
            t_out = fint.read()

            if g_out != t_out:
                exit(1)

            fing.close()
            fint.close()

    print "SUCCESS"

<commit_msg>tests: When comparing output, consider "nan" and "-nan" to be the same.<commit_after>import sys
import subprocess
import os

buildempath = sys.argv[1]
cmakepath = sys.argv[2]

goldenprefix = cmakepath + "/integration_tests/outputs/"
testoutprefix = cmakepath + "/integration_tests/temp_data/"

def compare_outputs(exe_string, stdoutfile, file_comps=None):
    exe_string = exe_string.replace("${BUILDEM_DIR}", buildempath)
    exe_string = exe_string.replace("${CMAKE_SOURCE_DIR}", cmakepath)

    if not os.path.exists(testoutprefix):
        os.makedirs(testoutprefix)

    golden_stdout_name = goldenprefix + stdoutfile
    testout_stdout_name = testoutprefix + stdoutfile

    p = subprocess.Popen(exe_string.split(), stdout=subprocess.PIPE)

    testout_stdout, err = p.communicate()

    # error returned from call
    if err is not None:
        sys.stderr.write("Error communicating with subprocess\n")
        exit(1)

    testout_stdout_arr = testout_stdout.split('\n')
    golden_stdout = open(golden_stdout_name).read()
    golden_stdout = golden_stdout.replace("-nan", "nan")

    testout_stdout = ""
    for line_index in range(0, len(testout_stdout_arr)):
        line = testout_stdout_arr[line_index]
        line = line.replace("-nan", "nan")
        if "Time " in line:
            continue
        testout_stdout += line
        if line_index != (len(testout_stdout_arr)-1):
            testout_stdout += "\n"

    fout = open(testout_stdout_name, 'w')
    fout.write(testout_stdout)
    fout.close()

    if testout_stdout != golden_stdout:
        sys.stderr.write("testout != golden_stdout\n")
        exit(1)

    if file_comps is not None:
        for file_comp in file_comps:
            fing = open(goldenprefix + file_comp)
            fint = open(testoutprefix + file_comp)

            g_out = fing.read()
            t_out = fint.read()

            g_out = g_out.replace("-nan", "nan")
            t_out = t_out.replace("-nan", "nan")

            if g_out != t_out:
                sys.stdout.write("g_out != t_out\n")
                exit(1)

            fing.close()
            fint.close()

    print "SUCCESS"

<|endoftext|>