<commit_before>#!/usr/bin/env python3

import unittest
import logging
import csv
import gzip
import json
from tests.test_source import SourceTestCase
from dipper.sources.IMPC import IMPC
from dipper.utils.CurieUtil import CurieUtil
from dipper.utils.TestUtils import TestUtils
from dipper import curie_map
from dipper.graph.RDFGraph import RDFGraph, URIRef

logging.basicConfig()
logging.getLogger().setLevel(logging.WARNING)
logger = logging.getLogger(__name__)

class IMPCTestCase(SourceTestCase):

    def setUp(self):
        self.source = IMPC('rdf_graph', True)  # Skolem Yes
        self.source.settestonly(True)
        self._setDirToSource()
        return

    def tearDown(self):
        self.source = None
        return

class EvidenceProvenanceTestCase(unittest.TestCase):

    def setUp(self):
        self.test_util = TestUtils()
        self.assoc_curie = 'MONARCH:test_association'
        self.eco_id = 'ECO:0000015'

        self.test_set_1 = (
            'MGI:1920145', 'Setd5', 'WTSI', 'MEFW', 'male', 'heterozygote',
            'MGI:4432631', 'Setd5<tm1a(EUCOMM)Wtsi>',
            'targeted mutation 1a, Wellcome Trust Sanger Institute',
            'MGI:2159965', 'C57BL/6N', 'MGP',
            'Wellcome Trust Sanger Institute Mouse Genetics Project',
            'MGP Select Pipeline', 'MGP_001', 'MGP_XRY_001', 'X-ray',
            'IMPC_XRY_008_001', 'Number of ribs right', 'MP:0005390',
            'skeleton phenotype', 'MP:0000480', 'increased rib number',
            '1.637023E-010', '', '8.885439E-007',
            'Wilcoxon rank sum test with continuity correction',
            'International Mouse Phenotyping Consortium')

        # Generate test curies, these are otherwise generated
        # within _add_evidence() and _add_study_provenance()
        # these blank nodes are hardcoded as NOT Skolemized  ...
        self.study_curie = "_:study"
        self.evidence_curie = "_:evidence"

        # IRIs for testing sparql output
        curie_dict = curie_map.get()
        curie_util = CurieUtil(curie_dict)
        self.assoc_iri = URIRef(curie_util.get_uri(self.assoc_curie))

        return

    def test_evidence_model(self):
        """
        Functional test for _add_evidence()
        """
        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)  # Reset graph
        # Test graph is empty
        self.assertTrue(len(list(impc.graph)) == 0)

        impc_map = impc.open_and_parse_yaml(impc.map_files['impc_map'])

        (p_value, percentage_change, effect_size) = self.test_set_1[23:26]

        impc._add_evidence(self.assoc_curie, self.eco_id, impc_map, p_value,
                           percentage_change, effect_size, self.study_curie)

        triples = """
    :MONARCH_test_association SEPIO:0000007 <https://monarchinitiative.org/.well-known/genid/b097a98087df7a99> .

    <https://monarchinitiative.org/.well-known/genid/b097a98087df7a99> a ECO:0000015 ;
        SEPIO:0000084 <https://monarchinitiative.org/.well-known/genid/b89ee584330837c9>,
            <https://monarchinitiative.org/.well-known/genid/bc0eeccdea27a1d8> ;
        SEPIO:0000085 <https://monarchinitiative.org/.well-known/genid/study> .

    <https://monarchinitiative.org/.well-known/genid/bc0eeccdea27a1d8> a OBI:0000175 ;
        RO:0002353 <https://monarchinitiative.org/.well-known/genid/study> ;
        STATO:0000129 1.637023e-10 .

    <https://monarchinitiative.org/.well-known/genid/b89ee584330837c9> a STATO:0000085 ;
        RO:0002353 <https://monarchinitiative.org/.well-known/genid/study> ;
        STATO:0000129 "8.885439E-007" .
        """

        self.assertTrue(self.test_util.test_graph_equality(
            triples, impc.graph))

    def test_provenance_model(self):
        """
        Functional test for _add_study_provenance()
        """
        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)
        self.assertTrue(len(list(impc.graph)) == 0)

        impc_map = impc.open_and_parse_yaml(impc.map_files['impc_map'])
        impress_map = json.loads(
            impc.fetch_from_url(
                impc.map_files['impress_map']).read().decode('utf-8'))

        (phenotyping_center, colony) = self.test_set_1[2:4]
        (project_fullname, pipeline_name, pipeline_stable_id,
         procedure_stable_id, procedure_name, parameter_stable_id,
         parameter_name) = self.test_set_1[12:19]
        (statistical_method, resource_name) = self.test_set_1[26:28]

        impc._add_study_provenance(
            impc_map, impress_map, phenotyping_center, colony,
            project_fullname, pipeline_name, pipeline_stable_id,
            procedure_stable_id, procedure_name,
            parameter_stable_id, parameter_name,
            statistical_method, resource_name)

        triples = """
    <https://monarchinitiative.org/.well-known/genid/bbdd05a8ca155dda> a OBI:0000471 ;
      BFO:0000051 OBO:STATO_0000076,
          <https://www.mousephenotype.org/impress/protocol/175/15> ;
      BFO:0000050  IMPRESS-procedure:15 ,
          <http://www.sanger.ac.uk/science/data/mouse-genomes-project> ;
      SEPIO:0000114 <https://www.mousephenotype.org/impress/parameterontologies/1867/91> ;
      SEPIO:0000017 <http://www.sanger.ac.uk/>  .

    <https://monarchinitiative.org/.well-known/genid/bc0b26361b8687b5> a owl:NamedIndividual ;
        rdfs:label "MEFW" .

    <http://www.sanger.ac.uk/> a foaf:organization ;
        rdfs:label "WTSI" .

    <http://www.sanger.ac.uk/science/data/mouse-genomes-project> a VIVO:Project ;
        rdfs:label "Wellcome Trust Sanger Institute Mouse Genetics Project" .

    <https://www.mousephenotype.org/impress/parameterontologies/1867/91> a owl:NamedIndividual ;
        rdfs:label "Number of ribs right (X-ray)" .

    IMPRESS-procedure:15 a owl:NamedIndividual ;
        rdfs:label "MGP Select Pipeline" .

    <https://www.mousephenotype.org/impress/protocol/175/15> a owl:NamedIndividual ;
        rdfs:label "X-ray" .
"""
        # dbg
        logger.debug(
            "Reference graph: %s", impc.graph.serialize(format="turtle").decode("utf-8")
        )
        # bitrot test
        #self.assertTrue(
        #    self.test_util.test_graph_equality(triples, impc.graph))

    def test_assertion_model(self):
        """
        Functional test for _add_study_provenance()
        """

        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)
        self.assertTrue(len(list(impc.graph)) == 0)

        impc_map = impc.open_and_parse_yaml(impc.map_files['impc_map'])

        impc._add_assertion_provenance(
            self.assoc_curie, self.evidence_curie, impc_map)

        triples = """
    MONARCH:test_association SEPIO:0000015 <https://monarchinitiative.org/.well-known/genid/bcb2c00a5c2f9c43> .
    <https://monarchinitiative.org/.well-known/genid/bcb2c00a5c2f9c43> a SEPIO:0000001 ;
        SEPIO:0000018 <http://www.mousephenotype.org/> ;
        SEPIO:0000111 <https://monarchinitiative.org/.well-known/genid/evidence>  .

    <http://www.mousephenotype.org/> a foaf:organization ;
        rdfs:label "International Mouse Phenotyping Consortium" .

        """
        # dbg
        logger.debug(
            "Reference graph: %s", impc.graph.serialize(format="turtle").decode("utf-8")
        )

        self.assertTrue(self.test_util.test_graph_equality(
            triples, impc.graph))

    def test_random_data_set(self):
        """
        Download dataset using fetch(), then take a row of data and
        run through evidence and provenance functions to test the output

        Line of data is hardcoded, but theoretically should work on any line
        """
        line_to_test = 1129
        count = 0
        impc = IMPC('rdf_graph', False)   # Not Skolem
        impress_map = json.loads(
            impc.fetch_from_url(
                impc.map_files['impress_map']).read().decode('utf-8'))
        impc_map = impc.open_and_parse_yaml(impc.map_files['impc_map'])

        # fetch file
        impc.fetch(True)
        file_path = '/'.join((impc.rawdir, impc.files['all']['file']))
        with gzip.open(file_path, 'rt') as csvfile:
            filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
            for row in filereader:
                count += 1
                if count == line_to_test:
                    self.test_set_1 = row
                    break

        # Some DRY violation with the above tests
        (phenotyping_center, colony) = row[2:4]
        (project_fullname, pipeline_name, pipeline_stable_id,
         procedure_stable_id, procedure_name, parameter_stable_id,
         parameter_name) = row[12:19]
        (statistical_method, resource_name) = row[26:28]

        (p_value, percentage_change, effect_size) = self.test_set_1[23:26]

        impc._add_evidence(self.assoc_curie, self.eco_id, impc_map, p_value,
                           percentage_change, effect_size, self.study_curie)

        impc._add_study_provenance(
            impc_map, impress_map, phenotyping_center, colony,
            project_fullname, pipeline_name, pipeline_stable_id,
            procedure_stable_id, procedure_name,
            parameter_stable_id, parameter_name,
            statistical_method, resource_name)

        # Note that this doesn't test much since we're dealing with
        # multiple part_of  and has_part links to individuals
        # which results in ambiguity = hard to test
        sparql_query = """
SELECT *
WHERE {
    ?assoc SEPIO:0000007 ?evidenceline .
    ?evidenceline a ECO:0000015 ;
        SEPIO:0000085 _:study .

    ?study a OBI:0000471 ;
        SEPIO:0000114 ?param ;
        SEPIO:0000017 ?agent .
}
"""
        sparql_output = impc.graph.query(sparql_query)
        # Test that query passes and returns one row
        self.assertEqual(len(list(sparql_output)), 1)

    def tearDown(self):
        return

if __name__ == '__main__':
    unittest.main()
<commit_msg>clean up (not fix) afterthought test<commit_after>#!/usr/bin/env python3

import unittest
import logging
import csv
import gzip
from tests.test_source import SourceTestCase
from dipper.sources.IMPC import IMPC
from dipper.utils.CurieUtil import CurieUtil
from dipper.utils.TestUtils import TestUtils
from dipper import curie_map
from dipper.graph.RDFGraph import RDFGraph, URIRef

logging.basicConfig()
logging.getLogger().setLevel(logging.WARNING)
logger = logging.getLogger(__name__)

class IMPCTestCase(SourceTestCase):

    def setUp(self):
        self.source = IMPC('rdf_graph', True)  # Skolem Yes
        self.source.settestonly(True)
        self._setDirToSource()
        return

    def tearDown(self):
        self.source = None
        return

class EvidenceProvenanceTestCase(unittest.TestCase):

    def setUp(self):
        self.test_util = TestUtils()
        self.assoc_curie = 'MONARCH:test_association'
        self.eco_id = 'ECO:0000015'

        self.test_set_1 = (
            'MGI:1920145', 'Setd5', 'WTSI', 'MEFW', 'male', 'heterozygote',
            'MGI:4432631', 'Setd5<tm1a(EUCOMM)Wtsi>',
            'targeted mutation 1a, Wellcome Trust Sanger Institute',
            'MGI:2159965', 'C57BL/6N', 'MGP',
            'Wellcome Trust Sanger Institute Mouse Genetics Project',
            'MGP Select Pipeline', 'MGP_001', 'MGP_XRY_001', 'X-ray',
            'IMPC_XRY_008_001', 'Number of ribs right', 'MP:0005390',
            'skeleton phenotype', 'MP:0000480', 'increased rib number',
            '1.637023E-010', '', '8.885439E-007',
            'Wilcoxon rank sum test with continuity correction',
            'International Mouse Phenotyping Consortium')

        # Generate test curies, these are otherwise generated
        # within _add_evidence() and _add_study_provenance()
        # these blank nodes are hardcoded as NOT Skolemized  ...
        self.study_curie = "_:study"
        self.evidence_curie = "_:evidence"

        # IRIs for testing sparql output
        curie_dict = curie_map.get()
        curie_util = CurieUtil(curie_dict)
        self.assoc_iri = URIRef(curie_util.get_uri(self.assoc_curie))

        return

    def test_evidence_model(self):
        """
        Functional test for _add_evidence()
        """
        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)  # Reset graph
        # Test graph is empty
        self.assertTrue(len(list(impc.graph)) == 0)

        (p_value, percentage_change, effect_size) = self.test_set_1[23:26]

        impc._add_evidence(
            self.assoc_curie, self.eco_id, p_value, percentage_change, effect_size,
            self.study_curie)

        triples = """
    :MONARCH_test_association SEPIO:0000007 <https://monarchinitiative.org/.well-known/genid/b097a98087df7a99> .

    <https://monarchinitiative.org/.well-known/genid/b097a98087df7a99> a ECO:0000015 ;
        SEPIO:0000084 <https://monarchinitiative.org/.well-known/genid/b89ee584330837c9>,
            <https://monarchinitiative.org/.well-known/genid/bc0eeccdea27a1d8> ;
        SEPIO:0000085 <https://monarchinitiative.org/.well-known/genid/study> .

    <https://monarchinitiative.org/.well-known/genid/bc0eeccdea27a1d8> a OBI:0000175 ;
        RO:0002353 <https://monarchinitiative.org/.well-known/genid/study> ;
        STATO:0000129 1.637023e-10 .

    <https://monarchinitiative.org/.well-known/genid/b89ee584330837c9> a STATO:0000085 ;
        RO:0002353 <https://monarchinitiative.org/.well-known/genid/study> ;
        STATO:0000129 "8.885439E-007" .
        """

        self.assertTrue(self.test_util.test_graph_equality(
            triples, impc.graph))

    def test_provenance_model(self):
        """
        Functional test for _add_study_provenance()
        """
        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)
        self.assertTrue(len(list(impc.graph)) == 0)

        (phenotyping_center, colony) = self.test_set_1[2:4]
        (project_fullname, pipeline_name, pipeline_stable_id,
         procedure_stable_id, procedure_name, parameter_stable_id,
         parameter_name) = self.test_set_1[12:19]
        (statistical_method, resource_name) = self.test_set_1[26:28]

        impc._add_study_provenance(
            phenotyping_center, colony,
            project_fullname, pipeline_name, pipeline_stable_id,
            procedure_stable_id, procedure_name,
            parameter_stable_id, parameter_name,
            statistical_method, resource_name)

        triples = """
    <https://monarchinitiative.org/.well-known/genid/bbdd05a8ca155dda> a OBI:0000471 ;
      BFO:0000051 OBO:STATO_0000076,
          <https://www.mousephenotype.org/impress/protocol/175/15> ;
      BFO:0000050  IMPRESS-procedure:15 ,
          <http://www.sanger.ac.uk/science/data/mouse-genomes-project> ;
      SEPIO:0000114 <https://www.mousephenotype.org/impress/parameterontologies/1867/91> ;
      SEPIO:0000017 <http://www.sanger.ac.uk/>  .

    <https://monarchinitiative.org/.well-known/genid/bc0b26361b8687b5> a owl:NamedIndividual ;
        rdfs:label "MEFW" .

    <http://www.sanger.ac.uk/> a foaf:organization ;
        rdfs:label "WTSI" .

    <http://www.sanger.ac.uk/science/data/mouse-genomes-project> a VIVO:Project ;
        rdfs:label "Wellcome Trust Sanger Institute Mouse Genetics Project" .

    <https://www.mousephenotype.org/impress/parameterontologies/1867/91> a owl:NamedIndividual ;
        rdfs:label "Number of ribs right (X-ray)" .

    IMPRESS-procedure:15 a owl:NamedIndividual ;
        rdfs:label "MGP Select Pipeline" .

    <https://www.mousephenotype.org/impress/protocol/175/15> a owl:NamedIndividual ;
        rdfs:label "X-ray" .
"""
        # dbg
        logger.debug(
            "Reference graph: %s", impc.graph.serialize(format="turtle").decode("utf-8")
        )
        # bitrot test
        # self.assertTrue(
        #    self.test_util.test_graph_equality(triples, impc.graph))

    def test_assertion_model(self):
        """
        Functional test for _add_study_provenance()
        """

        impc = IMPC('rdf_graph', True)
        impc.graph = RDFGraph(True)
        self.assertTrue(len(list(impc.graph)) == 0)

        impc._add_assertion_provenance(self.assoc_curie, self.evidence_curie)

        triples = """
    MONARCH:test_association SEPIO:0000015 <https://monarchinitiative.org/.well-known/genid/bcb2c00a5c2f9c43> .
    <https://monarchinitiative.org/.well-known/genid/bcb2c00a5c2f9c43> a SEPIO:0000001 ;
        SEPIO:0000018 <http://www.mousephenotype.org/> ;
        SEPIO:0000111 <https://monarchinitiative.org/.well-known/genid/evidence>  .

    <http://www.mousephenotype.org/> a foaf:organization ;
        rdfs:label "International Mouse Phenotyping Consortium" .

        """
        # dbg
        logger.debug(
            "Reference graph: %s", impc.graph.serialize(format="turtle").decode("utf-8")
        )

        self.assertTrue(self.test_util.test_graph_equality(
            triples, impc.graph))

    def test_random_data_set(self):
        """
        Download dataset using fetch(), then take a row of data and
        run through evidence and provenance functions to test the output

        Line of data is hardcoded, but theoretically should work on any line
        """
        line_to_test = 1129
        count = 0
        impc = IMPC('rdf_graph', False)   # Not Skolem

        # fetch file
        impc.fetch(True)
        file_path = '/'.join((impc.rawdir, impc.files['all']['file']))
        with gzip.open(file_path, 'rt') as csvfile:
            filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
            for row in filereader:
                count += 1
                if count == line_to_test:
                    self.test_set_1 = row
                    break

        # Some DRY violation with the above tests
        (phenotyping_center, colony) = row[2:4]
        (project_fullname, pipeline_name, pipeline_stable_id,
         procedure_stable_id, procedure_name, parameter_stable_id,
         parameter_name) = row[12:19]
        (statistical_method, resource_name) = row[26:28]

        (p_value, percentage_change, effect_size) = self.test_set_1[23:26]

        impc._add_evidence(
            self.assoc_curie, self.eco_id, p_value, percentage_change, effect_size,
            self.study_curie)

        impc._add_study_provenance(
            phenotyping_center, colony,
            project_fullname, pipeline_name, pipeline_stable_id,
            procedure_stable_id, procedure_name,
            parameter_stable_id, parameter_name,
            statistical_method, resource_name)

        # Note that this doesn't test much since we're dealing with
        # multiple part_of  and has_part links to individuals
        # which results in ambiguity = hard to test
        sparql_query = """
SELECT *
WHERE {
    ?assoc SEPIO:0000007 ?evidenceline .
    ?evidenceline a ECO:0000015 ;
        SEPIO:0000085 _:study .

    ?study a OBI:0000471 ;
        SEPIO:0000114 ?param ;
        SEPIO:0000017 ?agent .
}
"""
        sparql_output = impc.graph.query(sparql_query)
        # Test that query passes and returns one row
        self.assertEqual(len(list(sparql_output)), 1)

    def tearDown(self):
        return

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import re
import logging
from furl import furl
import requests
import grequests
logger = logging.getLogger("headliner")

class MostPopular(object):
    NUM_CONCURRENT = 1
    SECTIONS = ["arts", "automobiles", "business", "dining", "education", "fashion", "garden", "health", "movies", "music", "politics", "science", "sports", "style", "technology", "television", "travel", "your-money"]

    POPULARITY = ["mostviewed", "mostshared", "mostemailed"]

    WWW_PATH_PATTERN = re.compile(r"/[0-9]{4}/[0-9]{2}/[0-9]{2}/([a-z-]+)/([a-z-]+/)?[a-z0-9-]+\.html")
    BLOG_PATH_PATTERN = re.compile(r"/[0-9]{4}/")

    MAPPINGS = {
            "arts": {
                "__PATH": {
                    "design": ["Design"],
                    "music": ["Music"],
                    "television": ["Television"],
                    "video-games": ["Video-Games"],
                    "__NONE": ["Arts"],
                },
                "__FACET": {
                    "weddings and engagements": ["Weddings"],
                },
            },

            "automobiles": {
                "__ALL": ["Autos"],
            },

            "business": {
                "__ALL": ["Business"],
                "__PATH": {
                    "smallbusiness": ["Entrepreneur"],
                },
            },

            "dining": {
                "__ALL": ["Cooking"],
            },

            "education": {
                "__ALL": ["Ideas"],
            },

            "fashion": {
                "__ALL": ["Fashion-Men", "Fashion-Women"],
                "__PATH": {
                    "weddings": ["Weddings"],
                },
            },

            "garden": {
                "__ALL": ["Do-It-Yourself", "Home-Design"],
            },

            "health": {
                "__ALL": ["Health-Men", "Health-Women"],
            },

            "movies": {
                "__ALL": ["Movies"],
            },

            "politics": {
                "__ALL": ["Politics"],
            },

            "science": {
                "__ALL": ["Science"],
            },

            "sports": {
                "__ALL": ["Sports"],
                "__PATH": {
                    "baseball": ["Baseball"],
                    "basketball": ["Basketball"],
                    "football": ["Football"],
                    "golf": ["Golf"],
                    "hockey": ["Hockey"],
                    "soccer": ["Soccer"],
                    "tennis": ["Tennis"],
                },
                "__KEYWORD": {
                    "boxing": ["Boxing"],
                },
                "__COLUMN": {
                    "On Boxing": ["Boxing"],
                },
            },

            "style": {
                "__KEYWORD": {
                    "parenting": ["Parenting"]
                },
                "__FACET": {
                    "parenting": ["Parenting"]
                },
            },

            "technology": {
                "__ALL": ["Programming", "Technology"],
                "__PATH": {
                    "personaltech": ["Android", "Apple"],
                },
                "__FACET": {
                    "computer and video games": ["Video-Games"]
                },
            },

            "travel": {
                "__ALL": ["Travel"],
            },

            "your-money": {
                "__ALL": ["Business"],
            },
    }

    def __init__(self, config):
        self.config = config
        self._url_index = 0
        self.api_urls = self.gen_urls()

    def gen_urls(self):
        """
        Generates API URLS to be called by a class instance
        """
        urls = []
        for pop in MostPopular.POPULARITY:
            for section in MostPopular.SECTIONS:
                urls.append(self.config["api_url"].format(
                    popularity_type=pop,
                    section=section,
                    api_key=self.config["api_key"],
                ))
        return urls

    def next_url(self):
        output = self.api_urls[self._url_index]

        next_index = self._url_index + 1
        self._url_index = 0 if next_index >= len(self.api_urls) else next_index

        return output

    def next_urls(self, num=5):
        urls = []
        for i in range(num):
            urls.append(self.next_url())
        return urls

    def fetch_one(self):
        result_set = None
        url = self.next_url()
        req = requests.get(url)

        if req.status_code == 200:
            result_set = req.json()
        else:
            logger.warn("request_failed status_code:{0} reason:{1} url:{2}".format(req.status_code, req.reason, req.url))
        if result_set:
            return self.extract_data(result_set)
        return None

    def fetch_many(self, num):
        output = []
        urls = self.next_urls(num)
        rs = [grequests.get(u) for u in urls]
        responses = grequests.map(rs, size=self.NUM_CONCURRENT)

        result_groups = []
        for resp in responses:
            if resp.status_code == 200:
                result_groups.append(resp.json())
            else:
                logger.warn("request_failed status_code:{0} reason:{1} url:{2}".format(resp.status_code, resp.reason, resp.url))

        for result_set in result_groups:
            output.extend(self.extract_data(result_set))
        return output

    def extract_categorize(self, article):
        """
        Obtain categorization for a given article, capture data to be stored.
        """
        output = None
        section = article.get("section", "").lower()

        if MostPopular.MAPPINGS.has_key(section):
            uri = furl(article["url"])
            uri.query.add({"src": "moz-up"})

            data = {
                    "url": uri.url,
                    "title": article["title"],
                    "column": article.get("column", ""),
                    "media": article["media"],
            }

            labels = set()
            mapping = MostPopular.MAPPINGS[section]

            if mapping.has_key("__ALL"):
                labels.update(mapping["__ALL"])

            if mapping.has_key("__KEYWORD") and article.has_key("adx_keywords"):
                article_keywords = [k.lower() for k in article["adx_keywords"].split(";")]
                for k in article_keywords:
                    if mapping["__KEYWORD"].has_key(k):
                        labels.update(mapping["__KEYWORD"][k])

            if mapping.has_key("__FACET") and article.has_key("des_facet"):
                article_facets = [f.lower() for f in article["des_facet"]]
                for f in article_facets:
                    if mapping["__FACET"].has_key(f):
                        labels.update(mapping["__FACET"][f])

            if mapping.has_key("__COLUMN") and article.has_key("column"):
                col = article["column"].lower()
                if mapping["__COLUMN"].has_key(col):
                    labels.update(mapping["__COLUMN"][col])

            if mapping.has_key("__PATH"):
                path_match = self.WWW_PATH_PATTERN.match(uri.pathstr)
                if path_match:
                    section, sub_section = path_match.groups()
                    if sub_section:
                        # get rid of trailing '/'
                        sub_section = sub_section[:-1]

                    if sub_section is None and mapping["__PATH"].has_key("__NONE"):
                        labels.update(mapping["__PATH"]["__NONE"])
                    elif sub_section and mapping["__PATH"].has_key(sub_section):
                        labels.update(mapping["__PATH"][sub_section])

            output = {
                    "data": data,
                    "labels": list(labels),
                    "pub_date": article["published_date"]
            }

        return output

    def extract_data(self, input):
        articles = []
        if input['num_results'] > 0:
            for article in input['results']:
                article_data = self.extract_categorize(article)
                if article_data:
                    articles.append(article_data)
        return articles
<commit_msg>Closes #4 Use the section name instead of section path string Fix up sections: business, dining, fashion, garden, politics, your-money.<commit_after>import re
import logging
from furl import furl
import requests
import grequests
logger = logging.getLogger("headliner")

class MostPopular(object):
    NUM_CONCURRENT = 1
    SECTIONS = ["arts", "automobiles", "business", "dining", "education", "fashion", "garden", "health", "movies", "music", "politics", "science", "sports", "style", "technology", "television", "travel", "your-money"]

    POPULARITY = ["mostviewed", "mostshared", "mostemailed"]

    WWW_PATH_PATTERN = re.compile(r"/[0-9]{4}/[0-9]{2}/[0-9]{2}/([a-z-]+)/([a-z-]+/)?[a-z0-9-]+\.html")
    BLOG_PATH_PATTERN = re.compile(r"/[0-9]{4}/")

    MAPPINGS = {
            "arts": {
                "__PATH": {
                    "design": ["Design"],
                    "music": ["Music"],
                    "television": ["Television"],
                    "video-games": ["Video-Games"],
                    "__NONE": ["Arts"],
                },
                "__FACET": {
                    "weddings and engagements": ["Weddings"],
                },
            },

            "automobiles": {
                "__ALL": ["Autos"],
            },

            "business day": {
                "__ALL": ["Business"],
                "__PATH": {
                    "smallbusiness": ["Entrepreneur"],
                },
            },

            "dining & wine": {
                "__ALL": ["Cooking"],
            },

            "education": {
                "__ALL": ["Ideas"],
            },

            "fashion & style": {
                "__ALL": ["Fashion-Men", "Fashion-Women"],
                "__PATH": {
                    "weddings": ["Weddings"],
                },
            },

            "health": {
                "__ALL": ["Health-Men", "Health-Women"],
            },

            "home & garden": {
                "__ALL": ["Do-It-Yourself", "Home-Design"],
            },

            "movies": {
                "__ALL": ["Movies"],
            },

            "science": {
                "__ALL": ["Science"],
            },

            "sports": {
                "__ALL": ["Sports"],
                "__PATH": {
                    "baseball": ["Baseball"],
                    "basketball": ["Basketball"],
                    "football": ["Football"],
                    "golf": ["Golf"],
                    "hockey": ["Hockey"],
                    "soccer": ["Soccer"],
                    "tennis": ["Tennis"],
                },
                "__KEYWORD": {
                    "boxing": ["Boxing"],
                },
                "__COLUMN": {
                    "On Boxing": ["Boxing"],
                },
            },

            "style": {
                "__KEYWORD": {
                    "parenting": ["Parenting"]
                },
                "__FACET": {
                    "parenting": ["Parenting"]
                },
            },

            "technology": {
                "__ALL": ["Programming", "Technology"],
                "__PATH": {
                    "personaltech": ["Android", "Apple"],
                },
                "__FACET": {
                    "computer and video games": ["Video-Games"]
                },
            },

            "travel": {
                "__ALL": ["Travel"],
            },

            "u.s.": {
                "__ALL": ["Politics"],
            },

            "your money": {
                "__ALL": ["Business"],
            },
    }

    def __init__(self, config):
        self.config = config
        self._url_index = 0
        self.api_urls = self.gen_urls()

    def gen_urls(self):
        """
        Generates API URLS to be called by a class instance
        """
        urls = []
        for pop in MostPopular.POPULARITY:
            for section in MostPopular.SECTIONS:
                urls.append(self.config["api_url"].format(
                    popularity_type=pop,
                    section=section,
                    api_key=self.config["api_key"],
                ))
        return urls

    def next_url(self):
        output = self.api_urls[self._url_index]

        next_index = self._url_index + 1
        self._url_index = 0 if next_index >= len(self.api_urls) else next_index

        return output

    def next_urls(self, num=5):
        urls = []
        for i in range(num):
            urls.append(self.next_url())
        return urls

    def fetch_one(self):
        result_set = None
        url = self.next_url()
        req = requests.get(url)

        if req.status_code == 200:
            result_set = req.json()
        else:
            logger.warn("request_failed status_code:{0} reason:{1} url:{2}".format(req.status_code, req.reason, req.url))
        if result_set:
            return self.extract_data(result_set)
        return None

    def fetch_many(self, num):
        output = []
        urls = self.next_urls(num)
        rs = [grequests.get(u) for u in urls]
        responses = grequests.map(rs, size=self.NUM_CONCURRENT)

        result_groups = []
        for resp in responses:
            if resp.status_code == 200:
                result_groups.append(resp.json())
            else:
                logger.warn("request_failed status_code:{0} reason:{1} url:{2}".format(resp.status_code, resp.reason, resp.url))

        for result_set in result_groups:
            output.extend(self.extract_data(result_set))
        return output

    def extract_categorize(self, article):
        """
        Obtain categorization for a given article, capture data to be stored.
        """
        output = None
        section = article.get("section", "").lower()

        # NB: The article's section can be different from the section url path
        if MostPopular.MAPPINGS.has_key(section):
            uri = furl(article["url"])
            uri.query.add({"src": "moz-up"})

            data = {
                    "url": uri.url,
                    "title": article["title"],
                    "column": article.get("column", ""),
                    "media": article["media"],
            }

            labels = set()
            mapping = MostPopular.MAPPINGS[section]

            if mapping.has_key("__ALL"):
                labels.update(mapping["__ALL"])

            if mapping.has_key("__KEYWORD") and article.has_key("adx_keywords"):
                article_keywords = [k.lower() for k in article["adx_keywords"].split(";")]
                for k in article_keywords:
                    if mapping["__KEYWORD"].has_key(k):
                        labels.update(mapping["__KEYWORD"][k])

            if mapping.has_key("__FACET") and article.has_key("des_facet"):
                article_facets = [f.lower() for f in article["des_facet"]]
                for f in article_facets:
                    if mapping["__FACET"].has_key(f):
                        labels.update(mapping["__FACET"][f])

            if mapping.has_key("__COLUMN") and article.has_key("column"):
                col = article["column"].lower()
                if mapping["__COLUMN"].has_key(col):
                    labels.update(mapping["__COLUMN"][col])

            if mapping.has_key("__PATH"):
                path_match = self.WWW_PATH_PATTERN.match(uri.pathstr)
                if path_match:
                    section, sub_section = path_match.groups()
                    if sub_section:
                        # get rid of trailing '/'
                        sub_section = sub_section[:-1]

                    if sub_section is None and mapping["__PATH"].has_key("__NONE"):
                        labels.update(mapping["__PATH"]["__NONE"])
                    elif sub_section and mapping["__PATH"].has_key(sub_section):
                        labels.update(mapping["__PATH"][sub_section])

            output = {
                    "data": data,
                    "labels": list(labels),
                    "pub_date": article["published_date"]
            }

        return output

    def extract_data(self, input):
        articles = []
        if input['num_results'] > 0:
            for article in input['results']:
                article_data = self.extract_categorize(article)
                if article_data:
                    articles.append(article_data)
        return articles
<|endoftext|>


<commit_before>from django.template import Context, loader
from projects.models import Project
from django.http import HttpResponse, Http404, HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404, render
from projects.forms import ProjectForm
from django.utils import timezone

def index(request):
    latest_projects = Project.objects.all().order_by('-sub_date')[:5]
    t = loader.get_template('projects/index.html')
    c = Context({
        'latest_projects': latest_projects,
    })
    return HttpResponse(t.render(c))

def projectdisplay(request, project_id):
    p = get_object_or_404(Project, pk=project_id)
    return render_to_response('projects/projectdisplay.html', {'project': p})

def projectsubmit(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST)
        if form.is_valid():
            pd = form.save(commit=False)
            pd.sub_date = timezone.now()
            pd.approved = False

            # format the authors list
            al = pd.authors.split()
            al[-1] = 'and ' + al[-1]
            pd.authors = ', '.join(al)

            pd.save()
            return HttpResponseRedirect('/')
    else:
        form = ProjectForm()
    return render(request, 'projects/projectsubmit.html', {'form': form})
<commit_msg>Minor change to authors formatting<commit_after>from django.template import Context, loader
from projects.models import Project
from django.http import HttpResponse, Http404, HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404, render
from projects.forms import ProjectForm
from django.utils import timezone

def index(request):
    latest_projects = Project.objects.all().order_by('-sub_date')[:5]
    t = loader.get_template('projects/index.html')
    c = Context({
        'latest_projects': latest_projects,
    })
    return HttpResponse(t.render(c))

def projectdisplay(request, project_id):
    p = get_object_or_404(Project, pk=project_id)
    return render_to_response('projects/projectdisplay.html', {'project': p})

def projectsubmit(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST)
        if form.is_valid():
            pd = form.save(commit=False)
            pd.sub_date = timezone.now()
            pd.approved = False

            # format the authors list
            al = pd.authors.split()
            if(len(al) > 1):
                al[-1] = 'and ' + al[-1]
            if(len(al) > 2):
                pd.authors = ', '.join(al)
            else:
                pd.authors = ' '.join(al)

            pd.save()
            return HttpResponseRedirect('/')
    else:
        form = ProjectForm()
    return render(request, 'projects/projectsubmit.html', {'form': form})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ___        InstaBot V 1.2.0 by nickpettican           ___
# ___        Automate your Instagram activity           ___

# ___        Copyright 2017 Nicolas Pettican            ___

# ___   This software is licensed under the Apache 2    ___
# ___   license. You may not use this file except in    ___
# ___   compliance with the License.                    ___

# ___                   DISCLAIMER                      ___

# ___ InstaBot was created for educational purposes and ___
# ___ the end-user assumes sole responsibility of any   ___
# ___ consequences of it's misuse. Please be advised of ___
# ___ Instagram's monitoring, 1000 likes a day is the   ___
# ___ maximum you will get or you risk a temporary ban. ___
# ___ Choose your tags wisely or you may risk liking    ___
# ___ and commenting on undesirable media or spam.      ___

from lxml.etree import HTML
from json import loads as toJSON
from random import random, choice
from time import sleep
import itertools

# === INSTAGRAM FUNCTIONS ===

def refill(user_id, data, bucket, friends, tags_to_avoid, enabled, mode):
    # refill the bucket - returns dict with

    if mode == 'feed' and data:
        for i in data:
            bucket['codes'][i['media_id']] = i['url_code']
        # add feed posts to bucket
        if enabled['like_feed']:
            bucket['feed']['like'].extend([[i['media_id'], i['username']] for i in data
                                            if any(n.lower() == i['username'].lower() for n in friends)
                                            if not user_id == i['user_id']
                                            if not any(n == i['media_id'] for n in bucket['feed']['media_ids'])
                                            if not any(n[0] == i['media_id'] for n in bucket['feed']['done'])
                                            if not any(n in i['caption'] for n in tags_to_avoid)])
            bucket['feed']['media_ids'].extend([i['media_id'] for i in data])
    # add explored posts to bucket
    if mode == 'explore' and data['posts']:
        for i in data['posts']:
            bucket['codes'][i['media_id']] = i['url_code']
            bucket['user_ids'][i['user_id']] = i['username']
        # just to get the keys right
        tmp = [['like', 'media_id'], ['follow', 'user_id'], ['comment', 'media_id']]
        params = [param for param in tmp if enabled[param[0]]]
        # check if posts obbey the rules
        for param in params:
            if param:
                bucket[mode][param[0]].update([i[param[1]] for i in data['posts'] if not user_id == i['user_id']
                                            if not any(i[param[1]] in n for n in bucket[mode]['done'][param[0]])
                                            if not any(n in i['caption'] for n in tags_to_avoid)])
    elif mode == 'explore' and not data['posts']:
        raise Exception('No posts found')
    return bucket

def media_by_tag(browser, tag_url, media_url, tag, media_max_likes, media_min_likes):
    # returns list with the 14 'nodes' (posts) for the tag page

    result = {'posts': False, 'tag': tag}
    try:
        explore_site = browser.get(tag_url %(tag))
        tree = HTML(explore_site.text)
        data = return_sharedData(tree)
        if data:
            nodes = data['entry_data']['TagPage'][0]['graphql']['hashtag']['edge_hashtag_to_media']['edges']
            result['posts'] = [{'user_id': n['node']['owner']['id'],
                                'username': return_username(browser, media_url, n['node']['shortcode']),
                                'likes': n['node']['edge_liked_by']['count'],
                                'caption': n['node']['edge_media_to_caption']['edges'][0]['node']['text'],
                                'media_id': n['node']['id'],
                                'url_code': n['node']['shortcode']}
                               for n in nodes if media_min_likes <= n['node']['edge_liked_by']['count'] <= media_max_likes if not n['node']['comments_disabled']]
    except Exception as e:
        print '\nError in obtaining media by tag: %s' %(e)
    return result

def return_sharedData(tree):
    # returns the sharedData JSON object

    identifier = 'window._sharedData = '
    for a in tree.findall('.//script'):
        try:
            if a.text.startswith(identifier):
                try:
                    return toJSON(a.text.replace(identifier, '')[:-1])
                except Exception as e:
                    print '\nError returning sharedData JSON: %s' %(e)
        except Exception as e:
            continue
    return False

def return_username(browser, media_url, code):
    # returns the username from an image

    try:
        media_page = browser.get(media_url %(code))
        tree = HTML(media_page.text)
        data = return_sharedData(tree)
        return data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['owner']['username']
    except Exception as e:
        print '\nError obtaining username: %s' %(e)
    return False

def news_feed_media(browser, url, user_id):
    # returns the latest media on the news feed

    posts = False
    nodes = False
    try:
        news_feed = browser.get(url)
        tree = HTML(news_feed.text)
        data = return_sharedData(tree)
        if data:
            nodes = data['entry_data']['FeedPage'][0]['graphql']['user']['edge_web_feed_timeline']['edges']
        if nodes:
            posts = []
            for n in nodes:
                try:
                    if not n['node']['owner']['id'] == user_id and not n['node']['viewer_has_liked']:
                        post = {'user_id': n['node']['owner']['id'],
                                'username': n['node']['owner']['username'],
                                'likes': n['node']['edge_media_preview_like']['count'],
                                'caption': n['node']['edge_media_to_caption']['edges'][0]['node']['text'],
                                'media_id': n['node']['id'],
                                'url_code': n['node']['shortcode']}
                        posts.append(post)
                except:
                    continue
    except Exception as e:
        print '\nError getting new feed data: %s.' %(e)
    return posts

def check_user(browser, url, user):
    # checks the users profile to assess if it's fake

    result = {
    'fake': False, 'active': False, 'follower': False, 'data': {
        'username': '', 'user_id': '', 'media': '', 'follows': 0, 'followers': 0
    }}
    try:
        site = browser.get(url %(user))
        tree = HTML(site.text)
        data = return_sharedData(tree)
        user_data = data['entry_data']['ProfilePage'][0]['graphql']['user']
        if user_data:
            if user_data['follows_viewer'] or user_data['has_requested_viewer']:
                result['follower'] = True
            if user_data['edge_followed_by']['count'] > 0:
                try:
                    if user_data['edge_follow']['count'] / user_data['edge_followed_by']['count'] > 2 \
                        and user_data['edge_followed_by']['count'] < 10:
                        result['fake'] = True
                except ZeroDivisionError:
                    result['fake'] = True
                try:
                    if user_data['edge_follow']['count'] / user_data['edge_owner_to_timeline_media']['count'] < 10 \
                        and user_data['edge_followed_by']['count'] / user_data['edge_owner_to_timeline_media']['count'] < 10:
                        result['active'] = True
                except ZeroDivisionError:
                    pass
            else:
                result['fake'] = True
            result['data'] = {
                'username': user_data['username'],
                'user_id': user_data['id'],
                'media': user_data['edge_owner_to_timeline_media']['count'],
                'follows': user_data['edge_follow']['count'],
                'followers': user_data['edge_followed_by']['count']
            }

    except Exception as e:
        print '\nError checking user: %s.' %(e)

    sleep(5*random())
    return result

def generate_comment(comments_list):
    # returns a randomly generated generic comment

    # batch = list(itertools.product(
    #       ['Cool', 'Sweet', 'Awesome', 'Great'],
    #       ['😄', '🙌', '👍', '👌', '😊'],
    #       ['.', '!', '!!', '!!!']))
    batch = list(itertools.product(*comments_list))
    return ' '.join(choice(batch))

def post_data(browser, url, identifier, comment):
    # sends post request

    result = {'response': False, 'identifier': identifier}
    try:
        if comment:
            response = browser.post(url %(identifier), data= {'comment_text': comment})
        else:
            response = browser.post(url %(identifier))
        result['response'] = response
    except:
        pass
    return result
<commit_msg>Revert again<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ___        InstaBot V 1.2.0 by nickpettican           ___
# ___        Automate your Instagram activity           ___

# ___        Copyright 2017 Nicolas Pettican            ___

# ___   This software is licensed under the Apache 2    ___
# ___   license. You may not use this file except in    ___
# ___   compliance with the License.                    ___

# ___                   DISCLAIMER                      ___

# ___ InstaBot was created for educational purposes and ___
# ___ the end-user assumes sole responsibility of any   ___
# ___ consequences of it's misuse. Please be advised of ___
# ___ Instagram's monitoring, 1000 likes a day is the   ___
# ___ maximum you will get or you risk a temporary ban. ___
# ___ Choose your tags wisely or you may risk liking    ___
# ___ and commenting on undesirable media or spam.      ___

from lxml.etree import HTML
from json import loads as toJSON
from random import random, choice
from time import sleep
import itertools

# === INSTAGRAM FUNCTIONS ===

def refill(user_id, data, bucket, friends, tags_to_avoid, enabled, mode):
    # refill the bucket - returns dict with

    if mode == 'feed' and data:
        for i in data:
            bucket['codes'][i['media_id']] = i['url_code']
        # add feed posts to bucket
        if enabled['like_feed']:
            bucket['feed']['like'].extend([[i['media_id'], i['username']] for i in data
                                            if any(n.lower() == i['username'].lower() for n in friends)
                                            if not user_id == i['user_id']
                                            if not any(n == i['media_id'] for n in bucket['feed']['media_ids'])
                                            if not any(n[0] == i['media_id'] for n in bucket['feed']['done'])
                                            if not any(n in i['caption'] for n in tags_to_avoid)])
            bucket['feed']['media_ids'].extend([i['media_id'] for i in data])
    # add explored posts to bucket
    if mode == 'explore' and data['posts']:
        for i in data['posts']:
            bucket['codes'][i['media_id']] = i['url_code']
            bucket['user_ids'][i['user_id']] = i['username']
        # just to get the keys right
        tmp = [['like', 'media_id'], ['follow', 'user_id'], ['comment', 'media_id']]
        params = [param for param in tmp if enabled[param[0]]]
        # check if posts obbey the rules
        for param in params:
            if param:
                bucket[mode][param[0]].update([i[param[1]] for i in data['posts'] if not user_id == i['user_id']
                                            if not any(i[param[1]] in n for n in bucket[mode]['done'][param[0]])
                                            if not any(n in i['caption'] for n in tags_to_avoid)])
    elif mode == 'explore' and not data['posts']:
        raise Exception('No posts found')
    return bucket

def media_by_tag(browser, tag_url, media_url, tag, media_max_likes, media_min_likes):
    # returns list with the 14 'nodes' (posts) for the tag page

    result = {'posts': False, 'tag': tag}
    try:
        explore_site = browser.get(tag_url %(tag))
        tree = HTML(explore_site.text)
        data = return_sharedData(tree)
        if data:
            nodes = data['entry_data']['TagPage'][0]['graphql']['hashtag']['edge_hashtag_to_media']['edges']
            result['posts'] = [{'user_id': n['node']['owner']['id'],
                                'username': return_username(browser, media_url, n['node']['shortcode']),
                                'likes': n['node']['edge_liked_by']['count'],
                                'caption': n['node']['edge_media_to_caption']['edges'][0]['node']['text'],
                                'media_id': n['node']['id'],
                                'url_code': n['node']['shortcode']}
                               for n in nodes if media_min_likes <= n['node']['edge_liked_by']['count'] <= media_max_likes if not n['node']['comments_disabled']]
    except Exception as e:
        print '\nError in obtaining media by tag: %s' %(e)
    return result

def return_sharedData(tree):
    # returns the sharedData JSON object

    identifier = 'window._sharedData = '
    for a in tree.findall('.//script'):
        try:
            if a.text.startswith(identifier):
                try:
                    return toJSON(a.text.replace(identifier, '')[:-1])
                except Exception as e:
                    print '\nError returning sharedData JSON: %s' %(e)
        except Exception as e:
            continue
    return False

def return_username(browser, media_url, code):
    # returns the username from an image

    try:
        media_page = browser.get(media_url %(code))
        tree = HTML(media_page.text)
        data = return_sharedData(tree)
        return data['entry_data']['PostPage'][0]['graphql']['shortcode_media']['owner']['username']
    except Exception as e:
        print '\nError obtaining username: %s' %(e)
    return False

def news_feed_media(browser, url, user_id):
    # returns the latest media on the news feed

    posts = False
    nodes = False
    try:
        news_feed = browser.get(url)
        tree = HTML(news_feed.text)
        data = return_sharedData(tree)
        if data:
            nodes = data['entry_data']['FeedPage'][0]['graphql']['user']['edge_web_feed_timeline']['edges']
        if nodes:
            posts = []
            for n in nodes:
                try:
                    if not n['node']['owner']['id'] == user_id and not n['node']['viewer_has_liked']:
                        post = {'user_id': n['node']['owner']['id'],
                                'username': n['node']['owner']['username'],
                                'likes': n['node']['edge_media_preview_like']['count'],
                                'caption': n['node']['edge_media_to_caption']['edges'][0]['node']['text'],
                                'media_id': n['node']['id'],
                                'url_code': n['node']['shortcode']}
                        posts.append(post)
                except:
                    continue
    except Exception as e:
        print '\nError getting new feed data: %s.' %(e)
    return posts

def check_user(browser, url, user):
    # checks the users profile to assess if it's fake

    result = {
    'fake': False, 'active': False, 'follower': False, 'data': {
        'username': '', 'user_id': '', 'media': '', 'follows': 0, 'followers': 0
    }}
    try:
        site = browser.get(url %(user))
        tree = HTML(site.text)
        data = return_sharedData(tree)
        user_data = data['entry_data']['ProfilePage'][0]['graphql']['user']
        if user_data:
            if user_data['follows_viewer'] or user_data['has_requested_viewer']:
                result['follower'] = True
            if user_data['edge_followed_by']['count'] > 0:
                try:
                    if user_data['edge_follow']['count'] / user_data['edge_followed_by']['count'] > 2 \
                        and user_data['edge_followed_by']['count'] < 10:
                        result['fake'] = True
                except ZeroDivisionError:
                    result['fake'] = True
                try:
                    if user_data['edge_follow']['count'] / user_data['edge_owner_to_timeline_media']['count'] < 10 \
                        and user_data['edge_followed_by']['count'] / user_data['edge_owner_to_timeline_media']['count'] < 10:
                        result['active'] = True
                except ZeroDivisionError:
                    pass
            else:
                result['fake'] = True
            result['data'] = {
                'username': user_data['username'],
                'user_id': user_data['id'],
                'media': user_data['edge_owner_to_timeline_media']['count'],
                'follows': user_data['edge_follow']['count'],
                'followers': user_data['edge_followed_by']['count']
            }

    except Exception as e:
        print '\nError checking user: %s.' %(e)

    sleep(5*random())
    return result

def generate_comment(comments_list):
    # returns a randomly generated generic comment

    # batch = list(itertools.product(
    #       ['Cool', 'Sweet', 'Awesome', 'Great'],
    #       ['😄', '🙌', '👍', '👌', '😊'],
    #       ['.', '!', '!!', '!!!']))
    batch = list(itertools.product(*comments_list))
    return ' '.join(choice(batch))

def post_data(browser, url, identifier, comment):
    # sends post request

    result = {'response': False, 'identifier': identifier}
    try:
        if comment:
            response = browser.post(url %(identifier), data= {'comment_text': comment})
        else:
            response = browser.post(url %(identifier))
        result['response'] = response
    except:
        pass
    return result
<|endoftext|>


<commit_before># This should keep the .dev suffix in general, until a release is
# made.  The pre-release makefile target takes care of stripping the
# .dev out.  The post-release target increments the version and adds
# .dev back.  The first four numbers are the AG version. The fifth
# number is incremented with each release. Note that PEP-440 specifies
# that versions are automatically padded with zeros - that is, 6.2.1 =
# 6.2.1.0.0 (and also 6.2.0.dev = 6.2.0.0.dev).
__version__ = '6.2.1.0.1.dev'
<commit_msg>Release 6.2.1.0.1<commit_after># This should keep the .dev suffix in general, until a release is
# made.  The pre-release makefile target takes care of stripping the
# .dev out.  The post-release target increments the version and adds
# .dev back.  The first four numbers are the AG version. The fifth
# number is incremented with each release. Note that PEP-440 specifies
# that versions are automatically padded with zeros - that is, 6.2.1 =
# 6.2.1.0.0 (and also 6.2.0.dev = 6.2.0.0.dev).
__version__ = '6.2.1.0.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
"""JPEG XL conformance test runner.

Tool to perform a conformance test for a decoder.
"""

import argparse
import json
import numpy
import os
import subprocess
import sys
import tempfile

import lcms2

def Failure(message):
    print(message, flush=True)
    return False

def CompareNPY(ref, ref_icc, dec, dec_icc, frame_idx, rmse_limit, peak_error):
    """Compare a decoded numpy against the reference one."""
    if ref.shape != dec.shape:
        return Failure(f'Expected shape {ref.shape} but found {dec.shape}')
    ref_frame = ref[frame_idx]
    dec_frame = dec[frame_idx]
    num_channels = ref_frame.shape[2]

    if ref_icc != dec_icc:
        # Transform colors before comparison.
        if num_channels < 3:
            return Failure(f"Only RGB images are supported")
        dec_clr = dec_frame[:, :, 0:3]
        dec_frame[:, :, 0:3] = lcms2.convert_pixels(dec_icc, ref_icc, dec_clr)

    error = numpy.abs(ref_frame - dec_frame)
    actual_peak_error = error.max()
    error_by_channel = [error[:, :, ch] for ch in range(num_channels)]
    actual_rmses = [numpy.sqrt(numpy.mean(error_ch * error_ch)) for error_ch in error_by_channel]
    actual_rmse = max(actual_rmses)

    print(f"RMSE: {actual_rmses}, peak error: {actual_peak_error}", flush=True)

    if actual_rmse > rmse_limit:
        return Failure(f"RMSE too large: {actual_rmse} > {rmse_limit}")

    if actual_peak_error > peak_error:
        return Failure(
            f"Peak error too large: {actual_peak_error} > {peak_error}")
    return True

def CompareBinaries(ref_bin, dec_bin):
    """Compare a decoded binary file against the reference for exact contents."""
    with open(ref_bin, 'rb') as reff:
        ref_data = reff.read()

    with open(dec_bin, 'rb') as decf:
        dec_data = decf.read()

    if ref_data != dec_data:
        return Failure(
            f'Binary files mismatch: {ref_bin} {dec_bin}')
    return True

TEST_KEYS = set(
    ['reconstructed_jpeg', 'original_icc', 'rms_error', 'peak_error'])

def CheckMeta(dec, ref):
    if isinstance(ref, dict):
        if not isinstance(dec, dict):
            return Failure("Malformed metadata file")
        for k, v in ref.items():
            if k in TEST_KEYS:
                continue
            if k not in dec:
                return Failure(
                    f"Malformed metadata file: key {k} not found")
            vv = dec[k]
            return CheckMeta(vv, v)
    elif isinstance(ref, list):
        if not isinstance(dec, list) or len(dec) != len(ref):
            return Failure("Malformed metadata file")
        for vv, v in zip(dec, ref):
            return CheckMeta(vv, v)
    elif isinstance(ref, float):
        if not isinstance(dec, float):
            return Failure("Malformed metadata file")
        if abs(dec - ref) > 0.0001:
            return Failure(
                f"Metadata: Expected {ref}, found {dec}")
    elif dec != ref:
        return Failure(f"Metadata: Expected {ref}, found {dec}")
    return True

def ConformanceTestRunner(args):
    ok = True
    # We can pass either the .txt file or the directory which defaults to the
    # full corpus. This is useful to run a subset of the corpus in other .txt
    # files.
    if os.path.isdir(args.corpus):
        corpus_dir = args.corpus
        corpus_txt = os.path.join(args.corpus, 'corpus.txt')
    else:
        corpus_dir = os.path.dirname(args.corpus)
        corpus_txt = args.corpus

    with open(corpus_txt, 'r') as f:
        for test_id in f:
            test_id = test_id.rstrip('\n')
            print('Testing %s' % test_id, flush=True)
            test_dir = os.path.join(corpus_dir, test_id)

            with open(os.path.join(test_dir, 'test.json'), 'r') as f:
                descriptor = json.load(f)
                if 'sha256sums' in descriptor:
                    del descriptor['sha256sums']

            exact_tests = []

            with tempfile.TemporaryDirectory(prefix=test_id) as work_dir:
                input_filename = os.path.join(test_dir, 'input.jxl')
                pixel_prefix = os.path.join(work_dir, 'decoded')
                output_filename = pixel_prefix + '_image.npy'
                cmd = [args.decoder, input_filename, output_filename]
                cmd_jpeg = []
                if 'preview' in descriptor:
                    preview_filename = os.path.join(work_dir,
                                                    'decoded_preview.npy')
                    cmd.extend(['--preview_out', preview_filename])
                if 'reconstructed_jpeg' in descriptor:
                    jpeg_filename = os.path.join(work_dir, 'reconstructed.jpg')
                    cmd_jpeg = [args.decoder, input_filename, jpeg_filename]
                    exact_tests.append(('reconstructed.jpg', jpeg_filename))
                if 'original_icc' in descriptor:
                    decoded_original_icc = os.path.join(
                        work_dir, 'decoded_org.icc')
                    cmd.extend(['--orig_icc_out', decoded_original_icc])
                    exact_tests.append(('original.icc', decoded_original_icc))
                meta_filename = os.path.join(work_dir, 'meta.json')
                cmd.extend(['--metadata_out', meta_filename])
                cmd.extend(['--icc_out', pixel_prefix + '.icc'])
                cmd.extend(['--norender_spotcolors'])

                print(f"Running: {cmd}", flush=True)
                if subprocess.call(cmd) != 0:
                    ok = Failure('Running the decoder (%s) returned error' %
                                 ' '.join(cmd))
                    continue
                if cmd_jpeg:
                    print(f"Running: {cmd_jpeg}", flush=True)
                    if subprocess.call(cmd_jpeg) != 0:
                        ok = Failure(
                            'Running the decoder (%s) returned error' %
                            ' '.join(cmd_jpeg))
                        continue

                # Run validation of exact files.
                for reference_basename, decoded_filename in exact_tests:
                    reference_filename = os.path.join(test_dir,
                                                      reference_basename)
                    ok = ok & CompareBinaries(reference_filename, decoded_filename)

                # Validate metadata.
                with open(meta_filename, 'r') as f:
                    meta = json.load(f)

                ok = ok & CheckMeta(meta, descriptor)

                # Pixel data.
                decoded_icc = pixel_prefix + '.icc'
                with open(decoded_icc, 'rb') as f:
                    decoded_icc = f.read()
                reference_icc = os.path.join(test_dir, "reference.icc")
                with open(reference_icc, 'rb') as f:
                    reference_icc = f.read()

                reference_npy = os.path.join(test_dir, 'reference_image.npy')
                decoded_npy = os.path.join(work_dir, 'decoded_image.npy')

                if not os.path.exists(decoded_npy):
                    ok = Failure('File not decoded: decoded_image.npy')
                    continue

                reference_npy = numpy.load(reference_npy)
                decoded_npy = numpy.load(decoded_npy)

                for i, fd in enumerate(descriptor['frames']):
                    ok = ok & CompareNPY(reference_npy, reference_icc, decoded_npy,
                                         decoded_icc, i, fd['rms_error'],
                                         fd['peak_error'])

                if 'preview' in descriptor:
                    reference_npy = os.path.join(test_dir,
                                                 'reference_preview.npy')
                    decoded_npy = os.path.join(work_dir, 'decoded_preview.npy')

                    if not os.path.exists(decoded_npy):
                        ok = Failure(
                            'File not decoded: decoded_preview.npy')

                    reference_npy = numpy.load(reference_npy)
                    decoded_npy = numpy.load(decoded_npy)
                    ok = ok & CompareNPY(reference_npy, reference_icc, decoded_npy,
                                         decoded_icc, 0,
                                         descriptor['preview']['rms_error'],
                                         descriptor['preview']['peak_error'])

    return ok

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--decoder',
                        metavar='DECODER',
                        required=True,
                        help='path to the decoder binary under test.')
    parser.add_argument(
        '--corpus',
        metavar='CORPUS',
        required=True,
        help=('path to the corpus directory or corpus descriptor'
              ' text file.'))
    args = parser.parse_args()
    if not ConformanceTestRunner(args):
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>Colorize conformance output<commit_after>#!/usr/bin/env python3
# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
"""JPEG XL conformance test runner.

Tool to perform a conformance test for a decoder.
"""

import argparse
import json
import numpy
import os
import subprocess
import sys
import tempfile

import lcms2

def Failure(message):
    print(f"\033[91m{message}\033[0m", flush=True)
    return False

def CompareNPY(ref, ref_icc, dec, dec_icc, frame_idx, rmse_limit, peak_error):
    """Compare a decoded numpy against the reference one."""
    if ref.shape != dec.shape:
        return Failure(f'Expected shape {ref.shape} but found {dec.shape}')
    ref_frame = ref[frame_idx]
    dec_frame = dec[frame_idx]
    num_channels = ref_frame.shape[2]

    if ref_icc != dec_icc:
        # Transform colors before comparison.
        if num_channels < 3:
            return Failure(f"Only RGB images are supported")
        dec_clr = dec_frame[:, :, 0:3]
        dec_frame[:, :, 0:3] = lcms2.convert_pixels(dec_icc, ref_icc, dec_clr)

    error = numpy.abs(ref_frame - dec_frame)
    actual_peak_error = error.max()
    error_by_channel = [error[:, :, ch] for ch in range(num_channels)]
    actual_rmses = [numpy.sqrt(numpy.mean(error_ch * error_ch)) for error_ch in error_by_channel]
    actual_rmse = max(actual_rmses)

    print(f"RMSE: {actual_rmses}, peak error: {actual_peak_error}", flush=True)

    if actual_rmse > rmse_limit:
        return Failure(f"RMSE too large: {actual_rmse} > {rmse_limit}")

    if actual_peak_error > peak_error:
        return Failure(
            f"Peak error too large: {actual_peak_error} > {peak_error}")
    return True

def CompareBinaries(ref_bin, dec_bin):
    """Compare a decoded binary file against the reference for exact contents."""
    with open(ref_bin, 'rb') as reff:
        ref_data = reff.read()

    with open(dec_bin, 'rb') as decf:
        dec_data = decf.read()

    if ref_data != dec_data:
        return Failure(
            f'Binary files mismatch: {ref_bin} {dec_bin}')
    return True

TEST_KEYS = set(
    ['reconstructed_jpeg', 'original_icc', 'rms_error', 'peak_error'])

def CheckMeta(dec, ref):
    if isinstance(ref, dict):
        if not isinstance(dec, dict):
            return Failure("Malformed metadata file")
        for k, v in ref.items():
            if k in TEST_KEYS:
                continue
            if k not in dec:
                return Failure(
                    f"Malformed metadata file: key {k} not found")
            vv = dec[k]
            return CheckMeta(vv, v)
    elif isinstance(ref, list):
        if not isinstance(dec, list) or len(dec) != len(ref):
            return Failure("Malformed metadata file")
        for vv, v in zip(dec, ref):
            return CheckMeta(vv, v)
    elif isinstance(ref, float):
        if not isinstance(dec, float):
            return Failure("Malformed metadata file")
        if abs(dec - ref) > 0.0001:
            return Failure(
                f"Metadata: Expected {ref}, found {dec}")
    elif dec != ref:
        return Failure(f"Metadata: Expected {ref}, found {dec}")
    return True

def ConformanceTestRunner(args):
    ok = True
    # We can pass either the .txt file or the directory which defaults to the
    # full corpus. This is useful to run a subset of the corpus in other .txt
    # files.
    if os.path.isdir(args.corpus):
        corpus_dir = args.corpus
        corpus_txt = os.path.join(args.corpus, 'corpus.txt')
    else:
        corpus_dir = os.path.dirname(args.corpus)
        corpus_txt = args.corpus

    with open(corpus_txt, 'r') as f:
        for test_id in f:
            test_id = test_id.rstrip('\n')
            print(f"\033[94m\033[1mTesting {test_id}\033[0m", flush=True)
            test_dir = os.path.join(corpus_dir, test_id)

            with open(os.path.join(test_dir, 'test.json'), 'r') as f:
                descriptor = json.load(f)
                if 'sha256sums' in descriptor:
                    del descriptor['sha256sums']

            exact_tests = []

            with tempfile.TemporaryDirectory(prefix=test_id) as work_dir:
                input_filename = os.path.join(test_dir, 'input.jxl')
                pixel_prefix = os.path.join(work_dir, 'decoded')
                output_filename = pixel_prefix + '_image.npy'
                cmd = [args.decoder, input_filename, output_filename]
                cmd_jpeg = []
                if 'preview' in descriptor:
                    preview_filename = os.path.join(work_dir,
                                                    'decoded_preview.npy')
                    cmd.extend(['--preview_out', preview_filename])
                if 'reconstructed_jpeg' in descriptor:
                    jpeg_filename = os.path.join(work_dir, 'reconstructed.jpg')
                    cmd_jpeg = [args.decoder, input_filename, jpeg_filename]
                    exact_tests.append(('reconstructed.jpg', jpeg_filename))
                if 'original_icc' in descriptor:
                    decoded_original_icc = os.path.join(
                        work_dir, 'decoded_org.icc')
                    cmd.extend(['--orig_icc_out', decoded_original_icc])
                    exact_tests.append(('original.icc', decoded_original_icc))
                meta_filename = os.path.join(work_dir, 'meta.json')
                cmd.extend(['--metadata_out', meta_filename])
                cmd.extend(['--icc_out', pixel_prefix + '.icc'])
                cmd.extend(['--norender_spotcolors'])

                print(f"Running: {cmd}", flush=True)
                if subprocess.call(cmd) != 0:
                    ok = Failure('Running the decoder (%s) returned error' %
                                 ' '.join(cmd))
                    continue
                if cmd_jpeg:
                    print(f"Running: {cmd_jpeg}", flush=True)
                    if subprocess.call(cmd_jpeg) != 0:
                        ok = Failure(
                            'Running the decoder (%s) returned error' %
                            ' '.join(cmd_jpeg))
                        continue

                # Run validation of exact files.
                for reference_basename, decoded_filename in exact_tests:
                    reference_filename = os.path.join(test_dir,
                                                      reference_basename)
                    ok = ok & CompareBinaries(reference_filename, decoded_filename)

                # Validate metadata.
                with open(meta_filename, 'r') as f:
                    meta = json.load(f)

                ok = ok & CheckMeta(meta, descriptor)

                # Pixel data.
                decoded_icc = pixel_prefix + '.icc'
                with open(decoded_icc, 'rb') as f:
                    decoded_icc = f.read()
                reference_icc = os.path.join(test_dir, "reference.icc")
                with open(reference_icc, 'rb') as f:
                    reference_icc = f.read()

                reference_npy = os.path.join(test_dir, 'reference_image.npy')
                decoded_npy = os.path.join(work_dir, 'decoded_image.npy')

                if not os.path.exists(decoded_npy):
                    ok = Failure('File not decoded: decoded_image.npy')
                    continue

                reference_npy = numpy.load(reference_npy)
                decoded_npy = numpy.load(decoded_npy)

                for i, fd in enumerate(descriptor['frames']):
                    ok = ok & CompareNPY(reference_npy, reference_icc, decoded_npy,
                                         decoded_icc, i, fd['rms_error'],
                                         fd['peak_error'])

                if 'preview' in descriptor:
                    reference_npy = os.path.join(test_dir,
                                                 'reference_preview.npy')
                    decoded_npy = os.path.join(work_dir, 'decoded_preview.npy')

                    if not os.path.exists(decoded_npy):
                        ok = Failure(
                            'File not decoded: decoded_preview.npy')

                    reference_npy = numpy.load(reference_npy)
                    decoded_npy = numpy.load(decoded_npy)
                    ok = ok & CompareNPY(reference_npy, reference_icc, decoded_npy,
                                         decoded_icc, 0,
                                         descriptor['preview']['rms_error'],
                                         descriptor['preview']['peak_error'])

    return ok

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--decoder',
                        metavar='DECODER',
                        required=True,
                        help='path to the decoder binary under test.')
    parser.add_argument(
        '--corpus',
        metavar='CORPUS',
        required=True,
        help=('path to the corpus directory or corpus descriptor'
              ' text file.'))
    args = parser.parse_args()
    if not ConformanceTestRunner(args):
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
=============================================================================
Base classes for structure data (:mod:`sknano.io._base`)
=============================================================================

.. currentmodule:: sknano.io._base

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

from abc import ABCMeta, abstractmethod

from sknano.core.atoms import StructureAtom as Atom, StructureAtoms as Atoms
from sknano.version import short_version as version

default_comment_line = \
    'Structure data generated using scikit-nano version {}'.format(version)
default_structure_format = 'xyz'
supported_structure_formats = ('xyz', 'data', 'dump')

__all__ = ['Atom', 'Atoms',
           'StructureData',
           'StructureIO',
           'StructureReader',
           'StructureWriter',
           'StructureConverter',
           'StructureIOError',
           'StructureFormatSpec',
           'default_comment_line',
           'default_structure_format',
           'supported_structure_formats']

class StructureData(object):
    """Container class for structure data atoms."""
    def __init__(self):
        self.atoms = Atoms()

    @property
    def atoms(self):
        """Return :class:`~sknano.core.atoms.Atoms` instance object."""
        return self._atoms

    @atoms.setter
    def atoms(self, value):
        """Set :attr:`~StructureData.atoms` attribute."""
        if not isinstance(value, Atoms):
            raise TypeError('Expected an `Atoms` instance.')
        self._atoms = value

    @atoms.deleter
    def atoms(self):
        del self._atoms

    @property
    def bonds(self):
        return self.atoms.bonds

    def analyze(self):
        self.atoms.update_attrs()

    def clear(self):
        self.atoms.clear()

class StructureIO(object):
    """Base class for structure data file input and output.

    Parameters
    ----------
    fpath : {None, str}, optional

    """
    def __init__(self, fpath=None, fname=None, **kwargs):
        self.structure = StructureData()
        self.comment_line = default_comment_line
        if fpath is None and fname is not None:
            fpath = fname
        self.fpath = fpath
        self.kwargs = kwargs

    @property
    def comment_line(self):
        """Comment line."""
        return self._comment_line

    @comment_line.setter
    def comment_line(self, value):
        """Set the comment line string.

        Parameters
        ----------
        value : str

        """
        if not isinstance(value, str):
            raise TypeError('Expected a string.')
        self._comment_line = value

    @comment_line.deleter
    def comment_line(self):
        del self._comment_line

class StructureReader(object):
    """Structure data reader base class."""
    @classmethod
    def read(cls, fpath, structure_format=None, **kwargs):
        """Read structure data from file.

        Parameters
        ----------
        fpath : str
        structure_format : {None, str}
        """
        if fpath.endswith('.data') or structure_format == 'data':
            from ._lammps_data_format import DATAReader
            return DATAReader(fpath, **kwargs)
        elif fpath.endswith('.dump') or structure_format == 'dump':
            from ._lammps_dump_format import DUMPReader
            return DUMPReader(fpath, **kwargs)
        elif fpath.endswith('.xyz') or structure_format == 'xyz':
            from ._xyz_format import XYZReader
            return XYZReader.read(fpath)

class StructureWriter(object):
    """Structure data writer base class."""
    @classmethod
    def write(cls, fname=None, atoms=None, structure_format=None, **kwargs):
        """Write structure data to file.

        Parameters
        ----------
        fname : str, optional
        atoms : `~sknano.core.atoms.Atoms` instance
        structure_format : {None, str}, optional

        """
        if structure_format == 'data':
            from ._lammps_data_format import DATAWriter
            DATAWriter.write(fname=fname, atoms=atoms, **kwargs)
        elif structure_format == 'dump':
            from ._lammps_dump_format import DUMPWriter
            DUMPWriter.write(fname=fname, atoms=atoms, **kwargs)
        elif structure_format == 'xyz':
            from ._xyz_format import XYZWriter
            XYZWriter.write(fname=fname, atoms=atoms, **kwargs)

class StructureConverter(object):
    """Abstract base class for converting structure data.

    Parameters
    ----------
    infile : str
    outfile : str

    """
    __metaclass__ = ABCMeta

    def __init__(self, infile=None, outfile=None, **kwargs):
        self.infile = infile
        self.outfile = outfile
        self.kwargs = kwargs

    @abstractmethod
    def convert(self):
        """Convert structure data from one format to another format."""
        raise NotImplementedError('Subclasses of `StructureConverter` need '
                                  'to implement the `convert` method.')

class StructureFormatSpec(object):
    """Base class for defining a format specification.

    Parameters
    ----------

    """
    def __init__(self, **kwargs):
        pass

class StructureIOError(Exception):
    """Base class for `StructureIO` errors."""
    pass
<commit_msg>adding atoms attribute property to StructureIO class<commit_after># -*- coding: utf-8 -*-
"""
=============================================================================
Base classes for structure data (:mod:`sknano.io._base`)
=============================================================================

.. currentmodule:: sknano.io._base

"""
from __future__ import absolute_import, division, print_function
__docformat__ = 'restructuredtext en'

from abc import ABCMeta, abstractmethod

from sknano.core.atoms import StructureAtom as Atom, StructureAtoms as Atoms
from sknano.version import short_version as version

default_comment_line = \
    'Structure data generated using scikit-nano version {}'.format(version)
default_structure_format = 'xyz'
supported_structure_formats = ('xyz', 'data', 'dump')

__all__ = ['Atom', 'Atoms',
           'StructureData',
           'StructureIO',
           'StructureReader',
           'StructureWriter',
           'StructureConverter',
           'StructureIOError',
           'StructureFormatSpec',
           'default_comment_line',
           'default_structure_format',
           'supported_structure_formats']

class StructureData(object):
    """Container class for structure data atoms."""
    def __init__(self):
        self.atoms = Atoms()

    @property
    def atoms(self):
        """Return :class:`~sknano.core.atoms.Atoms` instance object."""
        return self._atoms

    @atoms.setter
    def atoms(self, value):
        """Set :attr:`~StructureData.atoms` attribute."""
        if not isinstance(value, Atoms):
            raise TypeError('Expected an `Atoms` instance.')
        self._atoms = value

    @atoms.deleter
    def atoms(self):
        del self._atoms

    @property
    def bonds(self):
        return self.atoms.bonds

    def analyze(self):
        self.atoms.update_attrs()

    def clear(self):
        self.atoms.clear()

class StructureIO(object):
    """Base class for structure data file input and output.

    Parameters
    ----------
    fpath : {None, str}, optional

    """
    def __init__(self, fpath=None, fname=None, **kwargs):
        self.structure_data = StructureData()
        self.comment_line = default_comment_line
        if fpath is None and fname is not None:
            fpath = fname
        self.fpath = fpath
        self.kwargs = kwargs

    @property
    def atoms(self):
        return self.structure_data.atoms

    @property
    def comment_line(self):
        """Comment line."""
        return self._comment_line

    @comment_line.setter
    def comment_line(self, value):
        """Set the comment line string.

        Parameters
        ----------
        value : str

        """
        if not isinstance(value, str):
            raise TypeError('Expected a string.')
        self._comment_line = value

    @comment_line.deleter
    def comment_line(self):
        del self._comment_line

class StructureReader(object):
    """Structure data reader base class."""
    @classmethod
    def read(cls, fpath, structure_format=None, **kwargs):
        """Read structure data from file.

        Parameters
        ----------
        fpath : str
        structure_format : {None, str}
        """
        if fpath.endswith('.data') or structure_format == 'data':
            from ._lammps_data_format import DATAReader
            return DATAReader(fpath, **kwargs)
        elif fpath.endswith('.dump') or structure_format == 'dump':
            from ._lammps_dump_format import DUMPReader
            return DUMPReader(fpath, **kwargs)
        elif fpath.endswith('.xyz') or structure_format == 'xyz':
            from ._xyz_format import XYZReader
            return XYZReader.read(fpath)

class StructureWriter(object):
    """Structure data writer base class."""
    @classmethod
    def write(cls, fname=None, atoms=None, structure_format=None, **kwargs):
        """Write structure data to file.

        Parameters
        ----------
        fname : str, optional
        atoms : `~sknano.core.atoms.Atoms` instance
        structure_format : {None, str}, optional

        """
        if structure_format == 'data':
            from ._lammps_data_format import DATAWriter
            DATAWriter.write(fname=fname, atoms=atoms, **kwargs)
        elif structure_format == 'dump':
            from ._lammps_dump_format import DUMPWriter
            DUMPWriter.write(fname=fname, atoms=atoms, **kwargs)
        elif structure_format == 'xyz':
            from ._xyz_format import XYZWriter
            XYZWriter.write(fname=fname, atoms=atoms, **kwargs)

class StructureConverter(object):
    """Abstract base class for converting structure data.

    Parameters
    ----------
    infile : str
    outfile : str

    """
    __metaclass__ = ABCMeta

    def __init__(self, infile=None, outfile=None, **kwargs):
        self.infile = infile
        self.outfile = outfile
        self.kwargs = kwargs

    @abstractmethod
    def convert(self):
        """Convert structure data from one format to another format."""
        raise NotImplementedError('Subclasses of `StructureConverter` need '
                                  'to implement the `convert` method.')

class StructureFormatSpec(object):
    """Base class for defining a format specification.

    Parameters
    ----------

    """
    def __init__(self, **kwargs):
        pass

class StructureIOError(Exception):
    """Base class for `StructureIO` errors."""
    pass
<|endoftext|>


<commit_before># coding: utf-8
from __future__ import unicode_literals

import unittest
from textwrap import dedent

from conllu import parse, parse_tree
from conllu.compat import text

class TestParse(unittest.TestCase):
    def test_multiple_sentences(self):
        data = dedent("""\
            1   The     the    DET    DT   Definite=Def|PronType=Art   4   det     _   _
            2   dog     dog    NOUN   NN   Number=Sing                 5   nmod    _   SpaceAfter=No
            3  .       .      PUNCT  .    _                           5   punct   _   _

            1   The     the    DET    DT   Definite=Def|PronType=Art   4   det     _   _
            2   dog     dog    NOUN   NN   Number=Sing                 5   nmod    _   SpaceAfter=No
            3  .       .      PUNCT  .    _                           5   punct   _   _

        """)
        self.assertEqual(
            text(parse(data)),
            "[TokenList<The, dog, .>, TokenList<The, dog, .>]"
        )
<commit_msg>Remove unused import.<commit_after># coding: utf-8
from __future__ import unicode_literals

import unittest
from textwrap import dedent

from conllu import parse
from conllu.compat import text

class TestParse(unittest.TestCase):
    def test_multiple_sentences(self):
        data = dedent("""\
            1   The     the    DET    DT   Definite=Def|PronType=Art   4   det     _   _
            2   dog     dog    NOUN   NN   Number=Sing                 5   nmod    _   SpaceAfter=No
            3  .       .      PUNCT  .    _                           5   punct   _   _

            1   The     the    DET    DT   Definite=Def|PronType=Art   4   det     _   _
            2   dog     dog    NOUN   NN   Number=Sing                 5   nmod    _   SpaceAfter=No
            3  .       .      PUNCT  .    _                           5   punct   _   _

        """)
        self.assertEqual(
            text(parse(data)),
            "[TokenList<The, dog, .>, TokenList<The, dog, .>]"
        )
<|endoftext|>


<commit_before>import os
import os.path
import sys
import string

# config vars, if you are using non-standard stuff change these
rom_dir         = '/home/pi/RetroPie/roms'
bak_dir         = '/home/pi/RetoPie/cleaned_up'
img_dir         = '/opt/retropie/configs/all/emulationstation/downloaded_images'
allowed_systems = ['amiga','amstradcpc','apple2','arcade','atari800','atari2600','atari5200','atari7800','atarilynx','atarist','c64','coco','dragon32','dreamcast','fba','fds','gamegear','gb','gba','gbc','intellivision','macintosh','mame-advmame','mame-libretro','mame-mame4all','mastersystem','megadrive','msx','n64','neogeo','nes','ngp','ngpc','pc','pcengine','psp','psx','sega32x','segacd','sg-1000','snes','vectrex','videopac','wonderswan','wonderswancolor','zmachine','zxspectrum']
delete_count    = 0
total_count     = 0

question = """
*****************************************************************
      WARNING! THIS SCRIPT WILL PERMANENTLY DELETE FILES!
         AND IS WRITTEN BY AN AMATEUR PYTHON DEVELOPER

        =====   THIS COULD GO VERY VERY WRONG!   =====
        ===== PLEASE BACKUP BEFORE RUNNING THIS! =====

        ARE YOU SURE YOU WANT TO PROCEED? (YES/TEST/NO)

      TYPE "TEST" TO DO A TEST RUN THAT (HOPEFULLY) WON'T
              ACTUALLY DELETE ANY FILES (MAYBE)

    TYPE "CLEAN" TO MOVE UNWANTED FILES TO A BACKUP DIRECTORY
                 (/home/pi/RetroPie/cleaned_up)

    TYPE "DELETE" IF YOU ARE SOUND MIND, UNDERSTAND THE RISKS
       AND WISH TO PROCEED WITH PERMANENTLY DELETING STUFF!
*****************************************************************
: """

user_input = raw_input(question)

# If your current working directory may change during script execution, it's recommended to
# immediately convert program arguments to an absolute path. Then the variable root below will
# be an absolute path as well. Example:
# walk_dir = os.path.abspath(walk_dir)

if user_input == 'DELETE' or user_input == 'delete' or user_input == 'test' or user_input == 'TEST' or user_input == 'clean' or user_input == 'CLEAN' :

    # check if bak_dir exists
    # if not, make it

    for root, subdirs, files in os.walk(rom_dir):

        list_file_path = os.path.join(root, 'foo.txt')

        with open(list_file_path, 'wb') as list_file:

            for filename in files:
                file_path = os.path.join(root, filename)

                rom_name = filename.split('.')[0]

                system_name = file_path.split('/')[-2]

                if system_name in allowed_systems:

                    total_count += 1

                    image_path = img_dir + '/' + system_name + '/' + rom_name + '-image.jpg'

                    #print image_path

                    if not os.path.isfile(image_path):
                        ## no image found, we should delete the rom!
                        print "DELETING: " + system_name + "/" + rom_name + " (" + filename + ")"
                        if user_input == 'delete' or user_input == 'DELETE':
                            ## os.remove(file_path)
                            delete_count += 1
                        elif user_input == 'clean' or user_input == 'CLEAN':
                            ## check if bak_dir/system_name does NOT exist
                            ## if so, create it
                            ## then move file to bak_dir/system_name
                            delete_count += 1
                        else:
                            ## do nothing
                            delete_count += 1

    if user_input == 'delete' or user_input == 'DELETE':
        print "\n--------------------------------------------------------------------"
        print " CLEANUP COMPLETE: " + str(delete_count) + " of " + str(total_count) + " files have been deleted! SHINY!"
        print "--------------------------------------------------------------------"
    elif user_input == 'clean' or user_input == 'CLEAN':
        print "\n--------------------------------------------------------------------"
        print " CLEAN COMPLETE: " + str(delete_count) + " of " + str(total_count) + " files have been moved! SHINY!"
        print "--------------------------------------------------------------------"
    else:
        print "\n--------------------------------------------------------------------"
        print " CLEANUP COMPLETE: " + str(delete_count) + " of " + str(total_count) + " are ripe to be tidied!"
        print "--------------------------------------------------------------------"

else :

    print "SCRIPT ABORTED (GOOD CHOICE!)"
<commit_msg>added cleanup support to remove-imageless-roms.py<commit_after>import os
import os.path
import sys
import string

# config vars, if you are using non-standard stuff change these
rom_dir         = '/home/pi/RetroPie/roms'
bak_dir         = '/home/pi/RetroPie/cleaned_up'
img_dir         = '/opt/retropie/configs/all/emulationstation/downloaded_images'
allowed_systems = ['amiga','amstradcpc','apple2','arcade','atari800','atari2600','atari5200','atari7800','atarilynx','atarist','c64','coco','dragon32','dreamcast','fba','fds','gamegear','gb','gba','gbc','intellivision','macintosh','mame-advmame','mame-libretro','mame-mame4all','mastersystem','megadrive','msx','n64','neogeo','nes','ngp','ngpc','pc','pcengine','psp','psx','sega32x','segacd','sg-1000','snes','vectrex','videopac','wonderswan','wonderswancolor','zmachine','zxspectrum']
delete_count    = 0
total_count     = 0

question = """
*****************************************************************
      WARNING! THIS SCRIPT WILL PERMANENTLY DELETE FILES!
         AND IS WRITTEN BY AN AMATEUR PYTHON DEVELOPER

        =====   THIS COULD GO VERY VERY WRONG!   =====
        ===== PLEASE BACKUP BEFORE RUNNING THIS! =====

        ARE YOU SURE YOU WANT TO PROCEED? (YES/TEST/NO)

      TYPE "TEST" TO DO A TEST RUN THAT (HOPEFULLY) WON'T
              ACTUALLY DELETE ANY FILES (MAYBE)

    TYPE "CLEAN" TO MOVE UNWANTED FILES TO A BACKUP DIRECTORY
                 (/home/pi/RetroPie/cleaned_up)

    TYPE "DELETE" IF YOU ARE SOUND MIND, UNDERSTAND THE RISKS
       AND WISH TO PROCEED WITH PERMANENTLY DELETING STUFF!
*****************************************************************
: """

user_input = raw_input(question)

if user_input == 'DELETE' or user_input == 'delete' or user_input == 'test' or user_input == 'TEST' or user_input == 'clean' or user_input == 'CLEAN' :

    # for the cleanup process, we need somewhere to put our backups
    if user_input == 'clean' or user_input == 'CLEAN':
        if not os.path.isdir(bak_dir):
            os.makedirs(bak_dir)

    for root, subdirs, files in os.walk(rom_dir):

        list_file_path = os.path.join(root, 'foo.txt')

        with open(list_file_path, 'wb') as list_file:

            os.remove(list_file_path)

            for filename in files:
                file_path = os.path.join(root, filename)

                rom_name = filename.split('.')[0]

                system_name = file_path.split('/')[-2]

                if system_name in allowed_systems:

                    total_count += 1

                    image_path = img_dir + '/' + system_name + '/' + rom_name + '-image.jpg'

                    if not os.path.isfile(image_path):
                        ## no image found, we should delete the rom!

                        if user_input == 'delete' or user_input == 'DELETE':

                            print "DELETING: " + system_name + "/" + rom_name + " (" + filename + ")"
                            os.remove(file_path)
                            delete_count += 1

                        elif user_input == 'clean' or user_input == 'CLEAN':

                            print "CLEANING: " + system_name + "/" + rom_name + " (" + filename + ")"
                            system_bak_dir = bak_dir + '/' + system_name
                            bak_file_path = system_bak_dir + '/' + filename

                            if not os.path.isdir(system_bak_dir):
                                os.makedirs(system_bak_dir)

                            os.rename(file_path, bak_file_path)
                            delete_count += 1

                        else:

                            print "TESTING: " + system_name + "/" + rom_name + " (" + filename + ")"
                            ## do nothing
                            delete_count += 1

    remaining_roms = total_count - delete_count

    if user_input == 'delete' or user_input == 'DELETE':
        print "\n--------------------------------------------------------------------"
        print "CLEANUP COMPLETE: " + str(delete_count) + " of " + str(total_count) + " files have been deleted! (" + str(remaining_roms) + " remain)"
        print "--------------------------------------------------------------------"
    elif user_input == 'clean' or user_input == 'CLEAN':
        print "\n--------------------------------------------------------------------"
        print "CLEAN COMPLETE: " + str(delete_count) + " of " + str(total_count) + " files have been moved!  (" + str(remaining_roms) + " remain)"
        print "--------------------------------------------------------------------"
    else:
        print "\n--------------------------------------------------------------------"
        print "TEST COMPLETE: " + str(delete_count) + " of " + str(total_count) + " are ripe to be tidied!  (" + str(remaining_roms) + " remain)"
        print "--------------------------------------------------------------------"

else :

    print "SCRIPT ABORTED (GOOD CHOICE!)"
<|endoftext|>